arg_utils.py 80.3 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import argparse
5
import copy
6
import dataclasses
7
import functools
8
import json
9
import sys
10
from collections.abc import Callable
11
from dataclasses import MISSING, dataclass, fields, is_dataclass
12
from itertools import permutations
13
from types import UnionType
14
15
16
17
18
from typing import (
    TYPE_CHECKING,
    Annotated,
    Any,
    Literal,
19
    TypeAlias,
20
21
22
23
24
25
    TypeVar,
    Union,
    cast,
    get_args,
    get_origin,
)
26

27
import huggingface_hub
28
import regex as re
29
import torch
30
from pydantic import TypeAdapter, ValidationError
31
from pydantic.fields import FieldInfo
32
from typing_extensions import TypeIs, deprecated
33

34
import vllm.envs as envs
35
36
37
38
39
40
41
42
43
44
45
from vllm.config import (
    CacheConfig,
    CompilationConfig,
    ConfigType,
    DeviceConfig,
    EPLBConfig,
    KVEventsConfig,
    KVTransferConfig,
    LoadConfig,
    LoRAConfig,
    ModelConfig,
46
    MultiModalConfig,
47
48
49
50
51
52
53
54
55
    ObservabilityConfig,
    ParallelConfig,
    PoolerConfig,
    SchedulerConfig,
    SpeculativeConfig,
    StructuredOutputsConfig,
    VllmConfig,
    get_attr_docs,
)
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
from vllm.config.cache import BlockSize, CacheDType, MambaDType, PrefixCachingHashAlgo
from vllm.config.device import Device
from vllm.config.model import (
    ConvertOption,
    HfOverrides,
    LogprobsMode,
    ModelDType,
    RunnerOption,
    TaskOption,
    TokenizerMode,
)
from vllm.config.multimodal import MMCacheType, MMEncoderTPMode
from vllm.config.observability import DetailedTraceModules
from vllm.config.parallel import DistributedExecutorBackend, ExpertPlacementStrategy
from vllm.config.scheduler import SchedulerPolicy
71
from vllm.config.utils import get_field
72
from vllm.logger import init_logger
73
from vllm.platforms import CpuArchEnum, current_platform
74
from vllm.plugins import load_general_plugins
75
from vllm.ray.lazy_utils import is_ray_initialized
76
from vllm.reasoning import ReasoningParserManager
77
from vllm.test_utils import MODEL_WEIGHTS_S3_BUCKET, MODELS_ON_S3
78
79
80
81
82
from vllm.transformers_utils.config import (
    get_model_path,
    is_interleaved,
    maybe_override_with_speculators,
)
83
from vllm.transformers_utils.utils import check_gguf_file
84
from vllm.utils import FlexibleArgumentParser, GiB_bytes, get_ip, is_in_ray_actor
85
from vllm.v1.sample.logits_processor import LogitsProcessor
86

87
88
89
if TYPE_CHECKING:
    from vllm.executor.executor_base import ExecutorBase
    from vllm.model_executor.layers.quantization import QuantizationMethods
90
    from vllm.model_executor.model_loader import LoadFormats
91
92
93
94
    from vllm.usage.usage_lib import UsageContext
else:
    ExecutorBase = Any
    QuantizationMethods = Any
95
    LoadFormats = Any
96
97
    UsageContext = Any

98
99
logger = init_logger(__name__)

100
101
# object is used to allow for special typing forms
T = TypeVar("T")
102
103
TypeHint: TypeAlias = type[Any] | object
TypeHintT: TypeAlias = type[T] | object
104

105

106
107
def parse_type(return_type: Callable[[str], T]) -> Callable[[str], T]:
    def _parse_type(val: str) -> T:
108
109
110
111
        try:
            return return_type(val)
        except ValueError as e:
            raise argparse.ArgumentTypeError(
112
113
                f"Value {val} cannot be converted to {return_type}."
            ) from e
114

115
116
117
    return _parse_type


118
119
def optional_type(return_type: Callable[[str], T]) -> Callable[[str], T | None]:
    def _optional_type(val: str) -> T | None:
120
121
122
123
        if val == "" or val == "None":
            return None
        return parse_type(return_type)(val)

124
    return _optional_type
125
126


127
def union_dict_and_str(val: str) -> str | dict[str, str] | None:
128
    if not re.match(r"(?s)^\s*{.*}\s*$", val):
129
        return str(val)
130
    return optional_type(json.loads)(val)
131
132


133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
def is_type(type_hint: TypeHint, type: TypeHintT) -> TypeIs[TypeHintT]:
    """Check if the type hint is a specific type."""
    return type_hint is type or get_origin(type_hint) is type


def contains_type(type_hints: set[TypeHint], type: TypeHintT) -> bool:
    """Check if the type hints contain a specific type."""
    return any(is_type(type_hint, type) for type_hint in type_hints)


def get_type(type_hints: set[TypeHint], type: TypeHintT) -> TypeHintT:
    """Get the specific type from the type hints."""
    return next((th for th in type_hints if is_type(th, type)), None)


148
def literal_to_kwargs(type_hints: set[TypeHint]) -> dict[str, Any]:
149
150
151
152
    """Get the `type` and `choices` from a `Literal` type hint in `type_hints`.

    If `type_hints` also contains `str`, we use `metavar` instead of `choices`.
    """
153
    type_hint = get_type(type_hints, Literal)
154
155
156
    options = get_args(type_hint)
    option_type = type(options[0])
    if not all(isinstance(option, option_type) for option in options):
157
        raise ValueError(
158
            "All options must be of the same type. "
159
160
            f"Got {options} with types {[type(c) for c in options]}"
        )
161
162
    kwarg = "metavar" if contains_type(type_hints, str) else "choices"
    return {"type": option_type, kwarg: sorted(options)}
163
164


165
166
167
168
169
def is_not_builtin(type_hint: TypeHint) -> bool:
    """Check if the class is not a built-in type."""
    return type_hint.__module__ != "builtins"


170
171
172
173
174
175
176
177
def get_type_hints(type_hint: TypeHint) -> set[TypeHint]:
    """Extract type hints from Annotated or Union type hints."""
    type_hints: set[TypeHint] = set()
    origin = get_origin(type_hint)
    args = get_args(type_hint)

    if origin is Annotated:
        type_hints.update(get_type_hints(args[0]))
178
179
    elif origin in {Union, UnionType}:
        # Union for Union[X, Y] and UnionType for X | Y
180
181
182
183
184
185
186
187
        for arg in args:
            type_hints.update(get_type_hints(arg))
    else:
        type_hints.add(type_hint)

    return type_hints


188
189
190
191
def is_online_quantization(quantization: Any) -> bool:
    return quantization in ["inc"]


192
NEEDS_HELP = (
193
194
    any("--help" in arg for arg in sys.argv)  # vllm SUBCOMMAND --help
    or (argv0 := sys.argv[0]).endswith("mkdocs")  # mkdocs SUBCOMMAND
195
196
197
198
    or argv0.endswith("mkdocs/__main__.py")  # python -m mkdocs SUBCOMMAND
)


199
@functools.lru_cache(maxsize=30)
200
def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]:
201
202
    # Save time only getting attr docs if we're generating help text
    cls_docs = get_attr_docs(cls) if NEEDS_HELP else {}
203
204
    kwargs = {}
    for field in fields(cls):
205
        # Get the set of possible types for the field
206
        type_hints: set[TypeHint] = get_type_hints(field.type)
207
208
209
210
211

        # If the field is a dataclass, we can use the model_validate_json
        generator = (th for th in type_hints if is_dataclass(th))
        dataclass_cls = next(generator, None)

212
        # Get the default value of the field
213
214
        if field.default is not MISSING:
            default = field.default
215
216
217
218
219
220
221
            # Handle pydantic.Field defaults
            if isinstance(default, FieldInfo):
                default = (
                    default.default
                    if default.default_factory is None
                    else default.default_factory()
                )
222
        elif field.default_factory is not MISSING:
223
            default = field.default_factory()
224
225
226

        # Get the help text for the field
        name = field.name
227
        help = cls_docs.get(name, "").strip()
228
229
230
231
232
233
234
        # Escape % for argparse
        help = help.replace("%", "%%")

        # Initialise the kwargs dictionary for the field
        kwargs[name] = {"default": default, "help": help}

        # Set other kwargs based on the type hints
235
236
237
        json_tip = (
            "Should either be a valid JSON string or JSON keys passed individually."
        )
238
        if dataclass_cls is not None:
239
240
241
242
243
244
245
246

            def parse_dataclass(val: str, cls=dataclass_cls) -> Any:
                try:
                    return TypeAdapter(cls).validate_json(val)
                except ValidationError as e:
                    raise argparse.ArgumentTypeError(repr(e)) from e

            kwargs[name]["type"] = parse_dataclass
247
            kwargs[name]["help"] += f"\n\n{json_tip}"
248
        elif contains_type(type_hints, bool):
249
250
251
            # Creates --no-<name> and --<name> flags
            kwargs[name]["action"] = argparse.BooleanOptionalAction
        elif contains_type(type_hints, Literal):
252
            kwargs[name].update(literal_to_kwargs(type_hints))
253
254
255
256
257
258
        elif contains_type(type_hints, tuple):
            type_hint = get_type(type_hints, tuple)
            types = get_args(type_hint)
            tuple_type = types[0]
            assert all(t is tuple_type for t in types if t is not Ellipsis), (
                "All non-Ellipsis tuple elements must be of the same "
259
260
                f"type. Got {types}."
            )
261
262
263
264
265
            kwargs[name]["type"] = tuple_type
            kwargs[name]["nargs"] = "+" if Ellipsis in types else len(types)
        elif contains_type(type_hints, list):
            type_hint = get_type(type_hints, list)
            types = get_args(type_hint)
266
            list_type = types[0]
267
268
            if get_origin(list_type) in {Union, UnionType}:
                # Union for Union[X, Y] and UnionType for X | Y
269
270
271
272
                msg = "List type must contain str if it is a Union."
                assert str in get_args(list_type), msg
                list_type = str
            kwargs[name]["type"] = list_type
273
274
275
            kwargs[name]["nargs"] = "+"
        elif contains_type(type_hints, int):
            kwargs[name]["type"] = int
276
            # Special case for large integers
277
278
279
280
281
282
            human_readable_ints = {
                "max_model_len",
                "max_num_batched_tokens",
                "kv_cache_memory_bytes",
            }
            if name in human_readable_ints:
283
                kwargs[name]["type"] = human_readable_int
284
                kwargs[name]["help"] += f"\n\n{human_readable_int.__doc__}"
285
286
        elif contains_type(type_hints, float):
            kwargs[name]["type"] = float
287
288
289
290
        elif contains_type(type_hints, dict) and (
            contains_type(type_hints, str)
            or any(is_not_builtin(th) for th in type_hints)
        ):
291
            kwargs[name]["type"] = union_dict_and_str
292
        elif contains_type(type_hints, dict):
293
            kwargs[name]["type"] = parse_type(json.loads)
294
            kwargs[name]["help"] += f"\n\n{json_tip}"
295
296
297
        elif contains_type(type_hints, str) or any(
            is_not_builtin(th) for th in type_hints
        ):
298
299
            kwargs[name]["type"] = str
        else:
300
            raise ValueError(f"Unsupported type {type_hints} for argument {name}.")
301

302
303
304
305
306
        # If the type hint was a sequence of literals, use the helper function
        # to update the type and choices
        if get_origin(kwargs[name].get("type")) is Literal:
            kwargs[name].update(literal_to_kwargs({kwargs[name]["type"]}))

307
308
309
310
311
312
313
        # If None is in type_hints, make the argument optional.
        # But not if it's a bool, argparse will handle this better.
        if type(None) in type_hints and not contains_type(type_hints, bool):
            kwargs[name]["type"] = optional_type(kwargs[name]["type"])
            if kwargs[name].get("choices"):
                kwargs[name]["choices"].append("None")
    return kwargs
314
315


316
def get_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]:
317
318
    """Return argparse kwargs for the given Config dataclass.

319
320
321
    If `--help` or `mkdocs` are not present in the command line command, the
    attribute documentation will not be included in the help output.

322
323
324
325
326
327
328
    The heavy computation is cached via functools.lru_cache, and a deep copy
    is returned so callers can mutate the dictionary without affecting the
    cached version.
    """
    return copy.deepcopy(_compute_kwargs(cls))


329
@dataclass
Zhuohan Li's avatar
Zhuohan Li committed
330
class EngineArgs:
Woosuk Kwon's avatar
Woosuk Kwon committed
331
    """Arguments for vLLM engine."""
332

333
    model: str = ModelConfig.model
334
335
336
    served_model_name: str | list[str] | None = ModelConfig.served_model_name
    tokenizer: str | None = ModelConfig.tokenizer
    hf_config_path: str | None = ModelConfig.hf_config_path
337
338
    runner: RunnerOption = ModelConfig.runner
    convert: ConvertOption = ModelConfig.convert
339
    task: TaskOption | None = ModelConfig.task
340
    skip_tokenizer_init: bool = ModelConfig.skip_tokenizer_init
341
    enable_prompt_embeds: bool = ModelConfig.enable_prompt_embeds
342
343
344
    tokenizer_mode: TokenizerMode = ModelConfig.tokenizer_mode
    trust_remote_code: bool = ModelConfig.trust_remote_code
    allowed_local_media_path: str = ModelConfig.allowed_local_media_path
345
346
    allowed_media_domains: list[str] | None = ModelConfig.allowed_media_domains
    download_dir: str | None = LoadConfig.download_dir
347
    safetensors_load_strategy: str = LoadConfig.safetensors_load_strategy
348
    load_format: str | LoadFormats = LoadConfig.load_format
349
350
    config_format: str = ModelConfig.config_format
    dtype: ModelDType = ModelConfig.dtype
351
    kv_cache_dtype: CacheDType = CacheConfig.cache_dtype
352
353
    seed: int | None = ModelConfig.seed
    max_model_len: int | None = ModelConfig.max_model_len
354
    cuda_graph_sizes: list[int] = get_field(SchedulerConfig, "cuda_graph_sizes")
355
356
357
    # Note: Specifying a custom executor backend by passing a class
    # is intended for expert use only. The API may change without
    # notice.
358
359
360
    distributed_executor_backend: (
        str | DistributedExecutorBackend | type[ExecutorBase] | None
    ) = ParallelConfig.distributed_executor_backend
361
    # number of P/D disaggregation (or other disaggregation) workers
362
363
    pipeline_parallel_size: int = ParallelConfig.pipeline_parallel_size
    tensor_parallel_size: int = ParallelConfig.tensor_parallel_size
364
    decode_context_parallel_size: int = ParallelConfig.decode_context_parallel_size
365
    data_parallel_size: int = ParallelConfig.data_parallel_size
366
367
368
369
370
    data_parallel_rank: int | None = None
    data_parallel_start_rank: int | None = None
    data_parallel_size_local: int | None = None
    data_parallel_address: str | None = None
    data_parallel_rpc_port: int | None = None
371
    data_parallel_hybrid_lb: bool = False
Rui Qiao's avatar
Rui Qiao committed
372
    data_parallel_backend: str = ParallelConfig.data_parallel_backend
373
    enable_expert_parallel: bool = ParallelConfig.enable_expert_parallel
374
    enable_dbo: bool = ParallelConfig.enable_dbo
375
376
    dbo_decode_token_threshold: int = ParallelConfig.dbo_decode_token_threshold
    dbo_prefill_token_threshold: int = ParallelConfig.dbo_prefill_token_threshold
377
378
379
    disable_nccl_for_dp_synchronization: bool = (
        ParallelConfig.disable_nccl_for_dp_synchronization
    )
380
    eplb_config: EPLBConfig = get_field(ParallelConfig, "eplb_config")
381
    enable_eplb: bool = ParallelConfig.enable_eplb
382
    expert_placement_strategy: ExpertPlacementStrategy = (
383
        ParallelConfig.expert_placement_strategy
384
    )
385
386
    _api_process_count: int = ParallelConfig._api_process_count
    _api_process_rank: int = ParallelConfig._api_process_rank
387
388
389
390
    num_redundant_experts: int = EPLBConfig.num_redundant_experts
    eplb_window_size: int = EPLBConfig.window_size
    eplb_step_interval: int = EPLBConfig.step_interval
    eplb_log_balancedness: bool = EPLBConfig.log_balancedness
391
    max_parallel_loading_workers: int | None = (
392
393
        ParallelConfig.max_parallel_loading_workers
    )
394
395
    block_size: BlockSize | None = CacheConfig.block_size
    enable_prefix_caching: bool | None = CacheConfig.enable_prefix_caching
396
    prefix_caching_hash_algo: PrefixCachingHashAlgo = (
397
        CacheConfig.prefix_caching_hash_algo
398
    )
399
400
    disable_sliding_window: bool = ModelConfig.disable_sliding_window
    disable_cascade_attn: bool = ModelConfig.disable_cascade_attn
401
402
403
    swap_space: float = CacheConfig.swap_space
    cpu_offload_gb: float = CacheConfig.cpu_offload_gb
    gpu_memory_utilization: float = CacheConfig.gpu_memory_utilization
404
405
    kv_cache_memory_bytes: int | None = CacheConfig.kv_cache_memory_bytes
    max_num_batched_tokens: int | None = SchedulerConfig.max_num_batched_tokens
406
407
    max_num_partial_prefills: int = SchedulerConfig.max_num_partial_prefills
    max_long_partial_prefills: int = SchedulerConfig.max_long_partial_prefills
408
    long_prefill_token_threshold: int = SchedulerConfig.long_prefill_token_threshold
409
    max_num_seqs: int | None = SchedulerConfig.max_num_seqs
410
    max_logprobs: int = ModelConfig.max_logprobs
411
    logprobs_mode: LogprobsMode = ModelConfig.logprobs_mode
412
    disable_log_stats: bool = False
413
    aggregate_engine_logging: bool = False
414
415
    revision: str | None = ModelConfig.revision
    code_revision: str | None = ModelConfig.code_revision
416
    rope_scaling: dict[str, Any] = get_field(ModelConfig, "rope_scaling")
417
418
    rope_theta: float | None = ModelConfig.rope_theta
    hf_token: bool | str | None = ModelConfig.hf_token
419
    hf_overrides: HfOverrides = get_field(ModelConfig, "hf_overrides")
420
421
    tokenizer_revision: str | None = ModelConfig.tokenizer_revision
    quantization: QuantizationMethods | None = ModelConfig.quantization
422
    enforce_eager: bool = ModelConfig.enforce_eager
423
    disable_custom_all_reduce: bool = ParallelConfig.disable_custom_all_reduce
424
    limit_mm_per_prompt: dict[str, int | dict[str, int]] = get_field(
425
426
        MultiModalConfig, "limit_per_prompt"
    )
427
    interleave_mm_strings: bool = MultiModalConfig.interleave_mm_strings
428
429
430
    media_io_kwargs: dict[str, dict[str, Any]] = get_field(
        MultiModalConfig, "media_io_kwargs"
    )
431
    mm_processor_kwargs: dict[str, Any] | None = MultiModalConfig.mm_processor_kwargs
432
    disable_mm_preprocessor_cache: bool = False  # DEPRECATED
433
    mm_processor_cache_gb: float = MultiModalConfig.mm_processor_cache_gb
434
    mm_processor_cache_type: MMCacheType | None = (
435
        MultiModalConfig.mm_processor_cache_type
436
437
    )
    mm_shm_cache_max_object_size_mb: int = (
438
        MultiModalConfig.mm_shm_cache_max_object_size_mb
439
    )
440
    mm_encoder_tp_mode: MMEncoderTPMode = MultiModalConfig.mm_encoder_tp_mode
441
    io_processor_plugin: str | None = None
442
    skip_mm_profiling: bool = MultiModalConfig.skip_mm_profiling
443
    video_pruning_rate: float = MultiModalConfig.video_pruning_rate
444
    # LoRA fields
445
    enable_lora: bool = False
446
447
    max_loras: int = LoRAConfig.max_loras
    max_lora_rank: int = LoRAConfig.max_lora_rank
448
    default_mm_loras: dict[str, str] | None = LoRAConfig.default_mm_loras
449
    fully_sharded_loras: bool = LoRAConfig.fully_sharded_loras
450
451
    max_cpu_loras: int | None = LoRAConfig.max_cpu_loras
    lora_dtype: str | torch.dtype | None = LoRAConfig.lora_dtype
452
453
    lora_extra_vocab_size: int = LoRAConfig.lora_extra_vocab_size

454
    ray_workers_use_nsight: bool = ParallelConfig.ray_workers_use_nsight
455
    num_gpu_blocks_override: int | None = CacheConfig.num_gpu_blocks_override
456
    num_lookahead_slots: int = SchedulerConfig.num_lookahead_slots
457
    model_loader_extra_config: dict = get_field(LoadConfig, "model_loader_extra_config")
458
    ignore_patterns: str | list[str] = get_field(LoadConfig, "ignore_patterns")
459

460
    enable_chunked_prefill: bool | None = SchedulerConfig.enable_chunked_prefill
461
    disable_chunked_mm_input: bool = SchedulerConfig.disable_chunked_mm_input
462

463
    disable_hybrid_kv_cache_manager: bool = (
464
465
        SchedulerConfig.disable_hybrid_kv_cache_manager
    )
466

467
    structured_outputs_config: StructuredOutputsConfig = get_field(
468
469
        VllmConfig, "structured_outputs_config"
    )
470
471
    reasoning_parser: str = StructuredOutputsConfig.reasoning_parser
    # Deprecated guided decoding fields
472
473
474
475
    guided_decoding_backend: str | None = None
    guided_decoding_disable_fallback: bool | None = None
    guided_decoding_disable_any_whitespace: bool | None = None
    guided_decoding_disable_additional_properties: bool | None = None
476

477
    logits_processor_pattern: str | None = ModelConfig.logits_processor_pattern
478

479
    speculative_config: dict[str, Any] | None = None
480

481
    show_hidden_metrics_for_version: str | None = (
482
        ObservabilityConfig.show_hidden_metrics_for_version
483
    )
484
485
    otlp_traces_endpoint: str | None = ObservabilityConfig.otlp_traces_endpoint
    collect_detailed_traces: list[DetailedTraceModules] | None = (
486
        ObservabilityConfig.collect_detailed_traces
487
    )
488
    scheduling_policy: SchedulerPolicy = SchedulerConfig.policy
489
    scheduler_cls: str | type[object] = SchedulerConfig.scheduler_cls
490

491
492
    pooler_config: PoolerConfig | None = ModelConfig.pooler_config
    override_pooler_config: dict | PoolerConfig | None = (
493
        ModelConfig.override_pooler_config
494
495
    )
    compilation_config: CompilationConfig = get_field(VllmConfig, "compilation_config")
496
497
    worker_cls: str = ParallelConfig.worker_cls
    worker_extension_cls: str = ParallelConfig.worker_extension_cls
498

499
500
    kv_transfer_config: KVTransferConfig | None = None
    kv_events_config: KVEventsConfig | None = None
501

502
503
    generation_config: str = ModelConfig.generation_config
    enable_sleep_mode: bool = ModelConfig.enable_sleep_mode
504
505
506
    override_generation_config: dict[str, Any] = get_field(
        ModelConfig, "override_generation_config"
    )
507
    model_impl: str = ModelConfig.model_impl
508
    override_attention_dtype: str = ModelConfig.override_attention_dtype
509

510
    calculate_kv_scales: bool = CacheConfig.calculate_kv_scales
511
512
    mamba_cache_dtype: MambaDType = CacheConfig.mamba_cache_dtype
    mamba_ssm_cache_dtype: MambaDType = CacheConfig.mamba_ssm_cache_dtype
513

514
    additional_config: dict[str, Any] = get_field(VllmConfig, "additional_config")
515

516
    use_tqdm_on_load: bool = LoadConfig.use_tqdm_on_load
517
    pt_load_map_location: str = LoadConfig.pt_load_map_location
518

519
520
    # DEPRECATED
    enable_multimodal_encoder_data_parallel: bool = False
521

522
    logits_processors: list[str | type[LogitsProcessor]] | None = (
523
524
        ModelConfig.logits_processors
    )
525
526
    """Custom logitproc types"""

527
528
    async_scheduling: bool = SchedulerConfig.async_scheduling

529
    kv_sharing_fast_prefill: bool = CacheConfig.kv_sharing_fast_prefill
530

531
    def __post_init__(self):
532
533
534
        # support `EngineArgs(compilation_config={...})`
        # without having to manually construct a
        # CompilationConfig object
535
        if isinstance(self.compilation_config, dict):
536
            self.compilation_config = CompilationConfig(**self.compilation_config)
537
        if isinstance(self.eplb_config, dict):
538
            self.eplb_config = EPLBConfig(**self.eplb_config)
539
        # Setup plugins
540
        from vllm.plugins import load_general_plugins
541

542
        load_general_plugins()
543
544
545
546
547
        # when use hf offline,replace model id to local model path
        if huggingface_hub.constants.HF_HUB_OFFLINE:
            model_id = self.model
            self.model = get_model_path(self.model, self.revision)
            logger.info(
548
549
550
551
                "HF_HUB_OFFLINE is True, replace model_id [%s] to model_path [%s]",
                model_id,
                self.model,
            )
552
553

    @staticmethod
554
    def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
Woosuk Kwon's avatar
Woosuk Kwon committed
555
        """Shared CLI arguments for vLLM engine."""
556

557
        # Model arguments
558
559
560
561
562
        model_kwargs = get_kwargs(ModelConfig)
        model_group = parser.add_argument_group(
            title="ModelConfig",
            description=ModelConfig.__doc__,
        )
563
        if not ("serve" in sys.argv[1:] and "--help" in sys.argv[1:]):
564
            model_group.add_argument("--model", **model_kwargs["model"])
565
566
        model_group.add_argument("--runner", **model_kwargs["runner"])
        model_group.add_argument("--convert", **model_kwargs["convert"])
567
        model_group.add_argument("--task", **model_kwargs["task"], deprecated=True)
568
        model_group.add_argument("--tokenizer", **model_kwargs["tokenizer"])
569
570
571
572
        model_group.add_argument("--tokenizer-mode", **model_kwargs["tokenizer_mode"])
        model_group.add_argument(
            "--trust-remote-code", **model_kwargs["trust_remote_code"]
        )
573
574
        model_group.add_argument("--dtype", **model_kwargs["dtype"])
        model_group.add_argument("--seed", **model_kwargs["seed"])
575
576
577
578
579
580
581
        model_group.add_argument("--hf-config-path", **model_kwargs["hf_config_path"])
        model_group.add_argument(
            "--allowed-local-media-path", **model_kwargs["allowed_local_media_path"]
        )
        model_group.add_argument(
            "--allowed-media-domains", **model_kwargs["allowed_media_domains"]
        )
582
        model_group.add_argument("--revision", **model_kwargs["revision"])
583
584
        model_group.add_argument("--code-revision", **model_kwargs["code_revision"])
        model_group.add_argument("--rope-scaling", **model_kwargs["rope_scaling"])
585
        model_group.add_argument("--rope-theta", **model_kwargs["rope_theta"])
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
        model_group.add_argument(
            "--tokenizer-revision", **model_kwargs["tokenizer_revision"]
        )
        model_group.add_argument("--max-model-len", **model_kwargs["max_model_len"])
        model_group.add_argument("--quantization", "-q", **model_kwargs["quantization"])
        model_group.add_argument("--enforce-eager", **model_kwargs["enforce_eager"])
        model_group.add_argument("--max-logprobs", **model_kwargs["max_logprobs"])
        model_group.add_argument("--logprobs-mode", **model_kwargs["logprobs_mode"])
        model_group.add_argument(
            "--disable-sliding-window", **model_kwargs["disable_sliding_window"]
        )
        model_group.add_argument(
            "--disable-cascade-attn", **model_kwargs["disable_cascade_attn"]
        )
        model_group.add_argument(
            "--skip-tokenizer-init", **model_kwargs["skip_tokenizer_init"]
        )
        model_group.add_argument(
            "--enable-prompt-embeds", **model_kwargs["enable_prompt_embeds"]
        )
        model_group.add_argument(
            "--served-model-name", **model_kwargs["served_model_name"]
        )
        model_group.add_argument("--config-format", **model_kwargs["config_format"])
610
611
        # This one is a special case because it can bool
        # or str. TODO: Handle this in get_kwargs
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
        model_group.add_argument(
            "--hf-token",
            type=str,
            nargs="?",
            const=True,
            default=model_kwargs["hf_token"]["default"],
            help=model_kwargs["hf_token"]["help"],
        )
        model_group.add_argument("--hf-overrides", **model_kwargs["hf_overrides"])
        model_group.add_argument("--pooler-config", **model_kwargs["pooler_config"])
        model_group.add_argument(
            "--override-pooler-config",
            **model_kwargs["override_pooler_config"],
            deprecated=True,
        )
        model_group.add_argument(
            "--logits-processor-pattern", **model_kwargs["logits_processor_pattern"]
        )
        model_group.add_argument(
            "--generation-config", **model_kwargs["generation_config"]
        )
        model_group.add_argument(
            "--override-generation-config", **model_kwargs["override_generation_config"]
        )
        model_group.add_argument(
            "--enable-sleep-mode", **model_kwargs["enable_sleep_mode"]
        )
639
        model_group.add_argument("--model-impl", **model_kwargs["model_impl"])
640
641
642
643
644
645
646
647
648
        model_group.add_argument(
            "--override-attention-dtype", **model_kwargs["override_attention_dtype"]
        )
        model_group.add_argument(
            "--logits-processors", **model_kwargs["logits_processors"]
        )
        model_group.add_argument(
            "--io-processor-plugin", **model_kwargs["io_processor_plugin"]
        )
649

650
651
652
653
654
655
        # Model loading arguments
        load_kwargs = get_kwargs(LoadConfig)
        load_group = parser.add_argument_group(
            title="LoadConfig",
            description=LoadConfig.__doc__,
        )
656
        load_group.add_argument("--load-format", **load_kwargs["load_format"])
657
658
659
660
661
662
663
664
665
666
667
668
        load_group.add_argument("--download-dir", **load_kwargs["download_dir"])
        load_group.add_argument(
            "--safetensors-load-strategy", **load_kwargs["safetensors_load_strategy"]
        )
        load_group.add_argument(
            "--model-loader-extra-config", **load_kwargs["model_loader_extra_config"]
        )
        load_group.add_argument("--ignore-patterns", **load_kwargs["ignore_patterns"])
        load_group.add_argument("--use-tqdm-on-load", **load_kwargs["use_tqdm_on_load"])
        load_group.add_argument(
            "--pt-load-map-location", **load_kwargs["pt_load_map_location"]
        )
669

670
671
672
673
674
        # Structured outputs arguments
        structured_outputs_kwargs = get_kwargs(StructuredOutputsConfig)
        structured_outputs_group = parser.add_argument_group(
            title="StructuredOutputsConfig",
            description=StructuredOutputsConfig.__doc__,
675
        )
676
        structured_outputs_group.add_argument(
677
            "--reasoning-parser",
678
            # This choice is a special case because it's not static
679
            choices=list(ReasoningParserManager.reasoning_parsers),
680
681
            **structured_outputs_kwargs["reasoning_parser"],
        )
682
683
684
685
686
687
688
689
690
691
692
        # Deprecated guided decoding arguments
        for arg, type in [
            ("--guided-decoding-backend", str),
            ("--guided-decoding-disable-fallback", bool),
            ("--guided-decoding-disable-any-whitespace", bool),
            ("--guided-decoding-disable-additional-properties", bool),
        ]:
            structured_outputs_group.add_argument(
                arg,
                type=type,
                help=(f"[DEPRECATED] {arg} will be removed in v0.12.0."),
693
694
                deprecated=True,
            )
695

696
        # Parallel arguments
697
698
699
700
701
702
        parallel_kwargs = get_kwargs(ParallelConfig)
        parallel_group = parser.add_argument_group(
            title="ParallelConfig",
            description=ParallelConfig.__doc__,
        )
        parallel_group.add_argument(
703
            "--distributed-executor-backend",
704
705
            **parallel_kwargs["distributed_executor_backend"],
        )
706
        parallel_group.add_argument(
707
708
709
710
            "--pipeline-parallel-size",
            "-pp",
            **parallel_kwargs["pipeline_parallel_size"],
        )
711
        parallel_group.add_argument(
712
713
            "--tensor-parallel-size", "-tp", **parallel_kwargs["tensor_parallel_size"]
        )
714
        parallel_group.add_argument(
715
716
717
718
719
720
721
722
723
724
            "--decode-context-parallel-size",
            "-dcp",
            **parallel_kwargs["decode_context_parallel_size"],
        )
        parallel_group.add_argument(
            "--data-parallel-size", "-dp", **parallel_kwargs["data_parallel_size"]
        )
        parallel_group.add_argument(
            "--data-parallel-rank",
            "-dpn",
725
            type=int,
726
727
728
            help="Data parallel rank of this instance. "
            "When set, enables external load balancer mode.",
        )
729
        parallel_group.add_argument(
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
            "--data-parallel-start-rank",
            "-dpr",
            type=int,
            help="Starting data parallel rank for secondary nodes.",
        )
        parallel_group.add_argument(
            "--data-parallel-size-local",
            "-dpl",
            type=int,
            help="Number of data parallel replicas to run on this node.",
        )
        parallel_group.add_argument(
            "--data-parallel-address",
            "-dpa",
            type=str,
            help="Address of data parallel cluster head-node.",
        )
        parallel_group.add_argument(
            "--data-parallel-rpc-port",
            "-dpp",
            type=int,
            help="Port for data parallel RPC communication.",
        )
        parallel_group.add_argument(
            "--data-parallel-backend",
            "-dpb",
            type=str,
            default="mp",
            help='Backend for data parallel, either "mp" or "ray".',
        )
760
        parallel_group.add_argument(
761
762
763
764
765
766
            "--data-parallel-hybrid-lb", **parallel_kwargs["data_parallel_hybrid_lb"]
        )
        parallel_group.add_argument(
            "--enable-expert-parallel", **parallel_kwargs["enable_expert_parallel"]
        )
        parallel_group.add_argument("--enable-dbo", **parallel_kwargs["enable_dbo"])
767
768
        parallel_group.add_argument(
            "--dbo-decode-token-threshold",
769
770
            **parallel_kwargs["dbo_decode_token_threshold"],
        )
771
772
        parallel_group.add_argument(
            "--dbo-prefill-token-threshold",
773
774
            **parallel_kwargs["dbo_prefill_token_threshold"],
        )
775
776
777
778
        parallel_group.add_argument(
            "--disable-nccl-for-dp-synchronization",
            **parallel_kwargs["disable_nccl_for_dp_synchronization"],
        )
779
780
        parallel_group.add_argument("--enable-eplb", **parallel_kwargs["enable_eplb"])
        parallel_group.add_argument("--eplb-config", **parallel_kwargs["eplb_config"])
781
782
        parallel_group.add_argument(
            "--expert-placement-strategy",
783
784
            **parallel_kwargs["expert_placement_strategy"],
        )
785
786
787
        parallel_group.add_argument(
            "--num-redundant-experts",
            type=int,
788
789
790
            help="[DEPRECATED] --num-redundant-experts will be removed in v0.12.0.",
            deprecated=True,
        )
791
792
793
794
        parallel_group.add_argument(
            "--eplb-window-size",
            type=int,
            help="[DEPRECATED] --eplb-window-size will be removed in v0.12.0.",
795
796
            deprecated=True,
        )
797
798
799
        parallel_group.add_argument(
            "--eplb-step-interval",
            type=int,
800
801
802
            help="[DEPRECATED] --eplb-step-interval will be removed in v0.12.0.",
            deprecated=True,
        )
803
804
805
        parallel_group.add_argument(
            "--eplb-log-balancedness",
            action=argparse.BooleanOptionalAction,
806
807
808
            help="[DEPRECATED] --eplb-log-balancedness will be removed in v0.12.0.",
            deprecated=True,
        )
809

810
        parallel_group.add_argument(
811
            "--max-parallel-loading-workers",
812
813
            **parallel_kwargs["max_parallel_loading_workers"],
        )
814
        parallel_group.add_argument(
815
816
            "--ray-workers-use-nsight", **parallel_kwargs["ray_workers_use_nsight"]
        )
817
        parallel_group.add_argument(
818
            "--disable-custom-all-reduce",
819
820
821
822
823
824
            **parallel_kwargs["disable_custom_all_reduce"],
        )
        parallel_group.add_argument("--worker-cls", **parallel_kwargs["worker_cls"])
        parallel_group.add_argument(
            "--worker-extension-cls", **parallel_kwargs["worker_extension_cls"]
        )
825
826
        parallel_group.add_argument(
            "--enable-multimodal-encoder-data-parallel",
827
            action="store_true",
828
829
            deprecated=True,
        )
830

831
832
833
834
835
        # KV cache arguments
        cache_kwargs = get_kwargs(CacheConfig)
        cache_group = parser.add_argument_group(
            title="CacheConfig",
            description=CacheConfig.__doc__,
836
        )
837
        cache_group.add_argument("--block-size", **cache_kwargs["block_size"])
838
839
840
841
842
843
        cache_group.add_argument(
            "--gpu-memory-utilization", **cache_kwargs["gpu_memory_utilization"]
        )
        cache_group.add_argument(
            "--kv-cache-memory-bytes", **cache_kwargs["kv_cache_memory_bytes"]
        )
844
        cache_group.add_argument("--swap-space", **cache_kwargs["swap_space"])
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
        cache_group.add_argument("--kv-cache-dtype", **cache_kwargs["cache_dtype"])
        cache_group.add_argument(
            "--num-gpu-blocks-override", **cache_kwargs["num_gpu_blocks_override"]
        )
        cache_group.add_argument(
            "--enable-prefix-caching", **cache_kwargs["enable_prefix_caching"]
        )
        cache_group.add_argument(
            "--prefix-caching-hash-algo", **cache_kwargs["prefix_caching_hash_algo"]
        )
        cache_group.add_argument("--cpu-offload-gb", **cache_kwargs["cpu_offload_gb"])
        cache_group.add_argument(
            "--calculate-kv-scales", **cache_kwargs["calculate_kv_scales"]
        )
        cache_group.add_argument(
            "--kv-sharing-fast-prefill", **cache_kwargs["kv_sharing_fast_prefill"]
        )
        cache_group.add_argument(
            "--mamba-cache-dtype", **cache_kwargs["mamba_cache_dtype"]
        )
        cache_group.add_argument(
            "--mamba-ssm-cache-dtype", **cache_kwargs["mamba_ssm_cache_dtype"]
        )
868

869
        # Multimodal related configs
870
871
872
873
874
        multimodal_kwargs = get_kwargs(MultiModalConfig)
        multimodal_group = parser.add_argument_group(
            title="MultiModalConfig",
            description=MultiModalConfig.__doc__,
        )
875
        multimodal_group.add_argument(
876
877
878
879
880
881
882
883
884
885
886
            "--limit-mm-per-prompt", **multimodal_kwargs["limit_per_prompt"]
        )
        multimodal_group.add_argument(
            "--media-io-kwargs", **multimodal_kwargs["media_io_kwargs"]
        )
        multimodal_group.add_argument(
            "--mm-processor-kwargs", **multimodal_kwargs["mm_processor_kwargs"]
        )
        multimodal_group.add_argument(
            "--mm-processor-cache-gb", **multimodal_kwargs["mm_processor_cache_gb"]
        )
887
        multimodal_group.add_argument(
888
889
            "--disable-mm-preprocessor-cache", action="store_true", deprecated=True
        )
890
        multimodal_group.add_argument(
891
892
            "--mm-processor-cache-type", **multimodal_kwargs["mm_processor_cache_type"]
        )
893
894
        multimodal_group.add_argument(
            "--mm-shm-cache-max-object-size-mb",
895
896
            **multimodal_kwargs["mm_shm_cache_max_object_size_mb"],
        )
897
        multimodal_group.add_argument(
898
899
900
901
902
            "--mm-encoder-tp-mode", **multimodal_kwargs["mm_encoder_tp_mode"]
        )
        multimodal_group.add_argument(
            "--interleave-mm-strings", **multimodal_kwargs["interleave_mm_strings"]
        )
903
        multimodal_group.add_argument(
904
905
            "--skip-mm-profiling", **multimodal_kwargs["skip_mm_profiling"]
        )
906

907
        multimodal_group.add_argument(
908
909
            "--video-pruning-rate", **multimodal_kwargs["video_pruning_rate"]
        )
910

911
        # LoRA related configs
912
913
914
915
916
917
        lora_kwargs = get_kwargs(LoRAConfig)
        lora_group = parser.add_argument_group(
            title="LoRAConfig",
            description=LoRAConfig.__doc__,
        )
        lora_group.add_argument(
918
            "--enable-lora",
919
            action=argparse.BooleanOptionalAction,
920
921
            help="If True, enable handling of LoRA adapters.",
        )
922
        lora_group.add_argument("--max-loras", **lora_kwargs["max_loras"])
923
924
925
926
        lora_group.add_argument("--max-lora-rank", **lora_kwargs["max_lora_rank"])
        lora_group.add_argument(
            "--lora-extra-vocab-size", **lora_kwargs["lora_extra_vocab_size"]
        )
927
        lora_group.add_argument(
928
            "--lora-dtype",
929
930
            **lora_kwargs["lora_dtype"],
        )
931
932
933
934
935
        lora_group.add_argument("--max-cpu-loras", **lora_kwargs["max_cpu_loras"])
        lora_group.add_argument(
            "--fully-sharded-loras", **lora_kwargs["fully_sharded_loras"]
        )
        lora_group.add_argument("--default-mm-loras", **lora_kwargs["default_mm_loras"])
936

937
938
939
940
941
942
943
944
        # Observability arguments
        observability_kwargs = get_kwargs(ObservabilityConfig)
        observability_group = parser.add_argument_group(
            title="ObservabilityConfig",
            description=ObservabilityConfig.__doc__,
        )
        observability_group.add_argument(
            "--show-hidden-metrics-for-version",
945
946
            **observability_kwargs["show_hidden_metrics_for_version"],
        )
947
        observability_group.add_argument(
948
949
            "--otlp-traces-endpoint", **observability_kwargs["otlp_traces_endpoint"]
        )
950
951
952
953
954
        # TODO: generalise this special case
        choices = observability_kwargs["collect_detailed_traces"]["choices"]
        metavar = f"{{{','.join(choices)}}}"
        observability_kwargs["collect_detailed_traces"]["metavar"] = metavar
        observability_kwargs["collect_detailed_traces"]["choices"] += [
955
            ",".join(p) for p in permutations(get_args(DetailedTraceModules), r=2)
956
957
958
        ]
        observability_group.add_argument(
            "--collect-detailed-traces",
959
960
            **observability_kwargs["collect_detailed_traces"],
        )
961

962
963
964
965
966
967
968
        # Scheduler arguments
        scheduler_kwargs = get_kwargs(SchedulerConfig)
        scheduler_group = parser.add_argument_group(
            title="SchedulerConfig",
            description=SchedulerConfig.__doc__,
        )
        scheduler_group.add_argument(
969
970
            "--max-num-batched-tokens", **scheduler_kwargs["max_num_batched_tokens"]
        )
971
        scheduler_group.add_argument(
972
973
974
975
976
            "--max-num-seqs", **scheduler_kwargs["max_num_seqs"]
        )
        scheduler_group.add_argument(
            "--max-num-partial-prefills", **scheduler_kwargs["max_num_partial_prefills"]
        )
977
978
        scheduler_group.add_argument(
            "--max-long-partial-prefills",
979
980
981
982
983
            **scheduler_kwargs["max_long_partial_prefills"],
        )
        scheduler_group.add_argument(
            "--cuda-graph-sizes", **scheduler_kwargs["cuda_graph_sizes"]
        )
984
985
        scheduler_group.add_argument(
            "--long-prefill-token-threshold",
986
987
988
989
990
            **scheduler_kwargs["long_prefill_token_threshold"],
        )
        scheduler_group.add_argument(
            "--num-lookahead-slots", **scheduler_kwargs["num_lookahead_slots"]
        )
991
992
        # multi-step scheduling has been removed; corresponding arguments
        # are no longer supported.
993
        scheduler_group.add_argument(
994
995
            "--scheduling-policy", **scheduler_kwargs["policy"]
        )
996
        scheduler_group.add_argument(
997
998
999
1000
1001
1002
1003
1004
            "--enable-chunked-prefill", **scheduler_kwargs["enable_chunked_prefill"]
        )
        scheduler_group.add_argument(
            "--disable-chunked-mm-input", **scheduler_kwargs["disable_chunked_mm_input"]
        )
        scheduler_group.add_argument(
            "--scheduler-cls", **scheduler_kwargs["scheduler_cls"]
        )
1005
1006
        scheduler_group.add_argument(
            "--disable-hybrid-kv-cache-manager",
1007
1008
1009
1010
1011
            **scheduler_kwargs["disable_hybrid_kv_cache_manager"],
        )
        scheduler_group.add_argument(
            "--async-scheduling", **scheduler_kwargs["async_scheduling"]
        )
1012
1013

        # vLLM arguments
1014
        vllm_kwargs = get_kwargs(VllmConfig)
1015
1016
1017
1018
        vllm_group = parser.add_argument_group(
            title="VllmConfig",
            description=VllmConfig.__doc__,
        )
1019
1020
1021
1022
        # We construct SpeculativeConfig using fields from other configs in
        # create_engine_config. So we set the type to a JSON string here to
        # delay the Pydantic validation that comes with SpeculativeConfig.
        vllm_kwargs["speculative_config"]["type"] = optional_type(json.loads)
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
        vllm_group.add_argument(
            "--speculative-config", **vllm_kwargs["speculative_config"]
        )
        vllm_group.add_argument(
            "--kv-transfer-config", **vllm_kwargs["kv_transfer_config"]
        )
        vllm_group.add_argument("--kv-events-config", **vllm_kwargs["kv_events_config"])
        vllm_group.add_argument(
            "--compilation-config", "-O", **vllm_kwargs["compilation_config"]
        )
        vllm_group.add_argument(
            "--additional-config", **vllm_kwargs["additional_config"]
        )
        vllm_group.add_argument(
            "--structured-outputs-config", **vllm_kwargs["structured_outputs_config"]
        )
1039

1040
        # Other arguments
1041
1042
1043
1044
1045
        parser.add_argument(
            "--disable-log-stats",
            action="store_true",
            help="Disable logging statistics.",
        )
1046

1047
1048
1049
1050
1051
1052
        parser.add_argument(
            "--aggregate-engine-logging",
            action="store_true",
            help="Log aggregate rather than per-engine statistics "
            "when using data parallelism.",
        )
1053
        return parser
1054
1055

    @classmethod
1056
    def from_cli_args(cls, args: argparse.Namespace):
1057
1058
1059
        # Get the list of attributes of this dataclass.
        attrs = [attr.name for attr in dataclasses.fields(cls)]
        # Set the attributes from the parsed arguments.
1060
1061
1062
        engine_args = cls(
            **{attr: getattr(args, attr) for attr in attrs if hasattr(args, attr)}
        )
Zhuohan Li's avatar
Zhuohan Li committed
1063
        return engine_args
1064

1065
    def create_model_config(self) -> ModelConfig:
1066
1067
1068
1069
1070
        # gguf file needs a specific model loader and doesn't use hf_repo
        if check_gguf_file(self.model):
            self.quantization = self.load_format = "gguf"

        # NOTE: This is to allow model loading from S3 in CI
1071
1072
1073
1074
1075
1076
        if (
            not isinstance(self, AsyncEngineArgs)
            and envs.VLLM_CI_USE_S3
            and self.model in MODELS_ON_S3
            and self.load_format == "auto"
        ):
1077
1078
            self.model = f"{MODEL_WEIGHTS_S3_BUCKET}/{self.model}"

1079
1080
1081
1082
        if self.disable_mm_preprocessor_cache:
            logger.warning(
                "`--disable-mm-preprocessor-cache` is deprecated "
                "and will be removed in v0.13. "
1083
1084
                "Please use `--mm-processor-cache-gb 0` instead.",
            )
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096

            self.mm_processor_cache_gb = 0
        elif envs.VLLM_MM_INPUT_CACHE_GIB != 4:
            logger.warning(
                "VLLM_MM_INPUT_CACHE_GIB` is deprecated "
                "and will be removed in v0.13. "
                "Please use `--mm-processor-cache-gb %d` instead.",
                envs.VLLM_MM_INPUT_CACHE_GIB,
            )

            self.mm_processor_cache_gb = envs.VLLM_MM_INPUT_CACHE_GIB

1097
1098
1099
1100
        if self.enable_multimodal_encoder_data_parallel:
            logger.warning(
                "--enable-multimodal-encoder-data-parallel` is deprecated "
                "and will be removed in v0.13. "
1101
1102
                "Please use `--mm-encoder-tp-mode data` instead."
            )
1103
1104
1105

            self.mm_encoder_tp_mode = "data"

1106
        return ModelConfig(
1107
            model=self.model,
1108
            hf_config_path=self.hf_config_path,
1109
1110
            runner=self.runner,
            convert=self.convert,
1111
            task=self.task,
1112
            tokenizer=self.tokenizer,
1113
1114
            tokenizer_mode=self.tokenizer_mode,
            trust_remote_code=self.trust_remote_code,
1115
            allowed_local_media_path=self.allowed_local_media_path,
1116
            allowed_media_domains=self.allowed_media_domains,
1117
1118
1119
1120
1121
            dtype=self.dtype,
            seed=self.seed,
            revision=self.revision,
            code_revision=self.code_revision,
            rope_scaling=self.rope_scaling,
1122
            rope_theta=self.rope_theta,
1123
            hf_token=self.hf_token,
1124
            hf_overrides=self.hf_overrides,
1125
1126
1127
1128
1129
            tokenizer_revision=self.tokenizer_revision,
            max_model_len=self.max_model_len,
            quantization=self.quantization,
            enforce_eager=self.enforce_eager,
            max_logprobs=self.max_logprobs,
1130
            logprobs_mode=self.logprobs_mode,
1131
            disable_sliding_window=self.disable_sliding_window,
1132
            disable_cascade_attn=self.disable_cascade_attn,
1133
            skip_tokenizer_init=self.skip_tokenizer_init,
1134
            enable_prompt_embeds=self.enable_prompt_embeds,
1135
            served_model_name=self.served_model_name,
1136
            limit_mm_per_prompt=self.limit_mm_per_prompt,
1137
            interleave_mm_strings=self.interleave_mm_strings,
1138
            media_io_kwargs=self.media_io_kwargs,
1139
            skip_mm_profiling=self.skip_mm_profiling,
1140
            config_format=self.config_format,
1141
            mm_processor_kwargs=self.mm_processor_kwargs,
1142
            mm_processor_cache_gb=self.mm_processor_cache_gb,
1143
            mm_processor_cache_type=self.mm_processor_cache_type,
1144
            mm_shm_cache_max_object_size_mb=self.mm_shm_cache_max_object_size_mb,
1145
            mm_encoder_tp_mode=self.mm_encoder_tp_mode,
1146
            pooler_config=self.pooler_config,
1147
            override_pooler_config=self.override_pooler_config,
1148
            logits_processor_pattern=self.logits_processor_pattern,
1149
            generation_config=self.generation_config,
1150
            override_generation_config=self.override_generation_config,
1151
            enable_sleep_mode=self.enable_sleep_mode,
1152
            model_impl=self.model_impl,
1153
            override_attention_dtype=self.override_attention_dtype,
1154
            logits_processors=self.logits_processors,
1155
            video_pruning_rate=self.video_pruning_rate,
1156
            io_processor_plugin=self.io_processor_plugin,
1157
        )
1158

1159
    def validate_tensorizer_args(self):
1160
1161
        from vllm.model_executor.model_loader.tensorizer import TensorizerConfig

1162
1163
        for key in self.model_loader_extra_config:
            if key in TensorizerConfig._fields:
1164
1165
1166
                self.model_loader_extra_config["tensorizer_config"][key] = (
                    self.model_loader_extra_config[key]
                )
1167

1168
    def create_load_config(self) -> LoadConfig:
1169
1170
        if self.quantization == "bitsandbytes":
            self.load_format = "bitsandbytes"
1171

1172
1173
1174
        if self.load_format == "tensorizer":
            if hasattr(self.model_loader_extra_config, "to_serializable"):
                self.model_loader_extra_config = (
1175
1176
                    self.model_loader_extra_config.to_serializable()
                )
1177
            self.model_loader_extra_config["tensorizer_config"] = {}
1178
1179
1180
            self.model_loader_extra_config["tensorizer_config"]["tensorizer_dir"] = (
                self.model
            )
1181
            self.validate_tensorizer_args()
1182

1183
1184
1185
        return LoadConfig(
            load_format=self.load_format,
            download_dir=self.download_dir,
1186
            safetensors_load_strategy=self.safetensors_load_strategy,
1187
            device="cpu" if is_online_quantization(self.quantization) else None,
1188
1189
            model_loader_extra_config=self.model_loader_extra_config,
            ignore_patterns=self.ignore_patterns,
1190
            use_tqdm_on_load=self.use_tqdm_on_load,
1191
            pt_load_map_location=self.pt_load_map_location,
1192
        )
1193

1194
1195
1196
1197
1198
1199
    def create_speculative_config(
        self,
        target_model_config: ModelConfig,
        target_parallel_config: ParallelConfig,
        enable_chunked_prefill: bool,
        disable_log_stats: bool,
1200
    ) -> SpeculativeConfig | None:
1201
1202
1203
1204
1205
1206
        """Initializes and returns a SpeculativeConfig object based on
        `speculative_config`.

        This function utilizes `speculative_config` to create a
        SpeculativeConfig object. The `speculative_config` can either be
        provided as a JSON string input via CLI arguments or directly as a
1207
        dictionary from the engine.
1208
1209
        """
        if self.speculative_config is None:
1210
            return None
1211

1212
1213
1214
        # Note(Shangming): These parameters are not obtained from the cli arg
        # '--speculative-config' and must be passed in when creating the engine
        # config.
1215
1216
1217
1218
1219
1220
1221
1222
        self.speculative_config.update(
            {
                "target_model_config": target_model_config,
                "target_parallel_config": target_parallel_config,
                "enable_chunked_prefill": enable_chunked_prefill,
                "disable_log_stats": disable_log_stats,
            }
        )
1223
        return SpeculativeConfig(**self.speculative_config)
1224

1225
1226
    def create_engine_config(
        self,
1227
        usage_context: UsageContext | None = None,
1228
        headless: bool = False,
1229
1230
1231
1232
1233
1234
1235
    ) -> VllmConfig:
        """
        Create the VllmConfig.

        NOTE: for autoselection of V0 vs V1 engine, we need to
        create the ModelConfig first, since ModelConfig's attrs
        (e.g. the model arch) are needed to make the decision.
Simon Mo's avatar
Simon Mo committed
1236

1237
1238
1239
1240
1241
1242
        This function set VLLM_USE_V1=X if VLLM_USE_V1 is
        unspecified by the user.

        If VLLM_USE_V1 is specified by the user but the VllmConfig
        is incompatible, we raise an error.
        """
1243
        current_platform.pre_register_and_update()
1244

1245
        device_config = DeviceConfig(device=cast(Device, current_platform.device_type))
1246

1247
1248
1249
1250
        model_config = self.create_model_config()
        self.model = model_config.model
        self.tokenizer = model_config.tokenizer

1251
1252
1253
1254
1255
1256
1257
1258
1259
        (self.model, self.tokenizer, self.speculative_config) = (
            maybe_override_with_speculators(
                model=self.model,
                tokenizer=self.tokenizer,
                revision=self.revision,
                trust_remote_code=self.trust_remote_code,
                vllm_speculative_config=self.speculative_config,
            )
        )
1260

1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
        # * If VLLM_USE_V1 is unset, we enable V1 for "supported features"
        #   and fall back to V0 for experimental or unsupported features.
        # * If VLLM_USE_V1=1, we enable V1 for supported + experimental
        #   features and raise error for unsupported features.
        # * If VLLM_USE_V1=0, we disable V1.
        use_v1 = False
        try_v1 = envs.VLLM_USE_V1 or not envs.is_set("VLLM_USE_V1")
        if try_v1 and self._is_v1_supported_oracle(model_config):
            use_v1 = True

        # If user explicitly set VLLM_USE_V1, sanity check we respect it.
        if envs.is_set("VLLM_USE_V1"):
            assert use_v1 == envs.VLLM_USE_V1
        # Otherwise, set the VLLM_USE_V1 variable globally.
        else:
            envs.set_vllm_use_v1(use_v1)

1278
1279
        # Set default arguments for V1 Engine.
        self._set_default_args(usage_context, model_config)
1280
        # Disable chunked prefill for POWER (ppc64le)/ARM/s390x/RISCV CPUs in V1
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
        if current_platform.is_cpu() and current_platform.get_cpu_architecture() in (
            CpuArchEnum.POWERPC,
            CpuArchEnum.S390X,
            CpuArchEnum.ARM,
            CpuArchEnum.RISCV,
        ):
            logger.info(
                "Chunked prefill is not supported for ARM and POWER, "
                "S390X and RISC-V CPUs; "
                "disabling it for V1 backend."
            )
1292
            self.enable_chunked_prefill = False
1293
1294
        assert self.enable_chunked_prefill is not None

1295
        sliding_window: int | None = None
1296
1297
1298
1299
1300
1301
        if not is_interleaved(model_config.hf_text_config):
            # Only set CacheConfig.sliding_window if the model is all sliding
            # window. Otherwise CacheConfig.sliding_window will override the
            # global layers in interleaved sliding window models.
            sliding_window = model_config.get_sliding_window()

1302
1303
1304
        # Note(hc): In the current implementation of decode context
        # parallel(DCP), tp_size needs to be divisible by dcp_size,
        # because the world size does not change by dcp, it simply
1305
        # reuses the GPUs of TP group, and split one TP group into
1306
        # tp_size//dcp_size DCP groups.
1307
        assert self.tensor_parallel_size % self.decode_context_parallel_size == 0, (
1308
1309
1310
1311
            f"tp_size={self.tensor_parallel_size} must be divisible by"
            f"dcp_size={self.decode_context_parallel_size}."
        )

1312
        cache_config = CacheConfig(
1313
            block_size=self.block_size,
1314
            gpu_memory_utilization=self.gpu_memory_utilization,
1315
            kv_cache_memory_bytes=self.kv_cache_memory_bytes,
1316
1317
            swap_space=self.swap_space,
            cache_dtype=self.kv_cache_dtype,
1318
            is_attention_free=model_config.is_attention_free,
1319
            num_gpu_blocks_override=self.num_gpu_blocks_override,
1320
            sliding_window=sliding_window,
1321
            enable_prefix_caching=self.enable_prefix_caching,
1322
            prefix_caching_hash_algo=self.prefix_caching_hash_algo,
1323
            cpu_offload_gb=self.cpu_offload_gb,
1324
            calculate_kv_scales=self.calculate_kv_scales,
1325
            kv_sharing_fast_prefill=self.kv_sharing_fast_prefill,
1326
1327
            mamba_cache_dtype=self.mamba_cache_dtype,
            mamba_ssm_cache_dtype=self.mamba_ssm_cache_dtype,
1328
        )
1329

1330
1331
1332
1333
1334
1335
        ray_runtime_env = None
        if is_ray_initialized():
            # Ray Serve LLM calls `create_engine_config` in the context
            # of a Ray task, therefore we check is_ray_initialized()
            # as opposed to is_in_ray_actor().
            import ray
1336

1337
            ray_runtime_env = ray.get_runtime_context().runtime_env
1338
1339
1340
1341
1342
1343
1344
            # Avoid logging sensitive environment variables
            sanitized_env = ray_runtime_env.to_dict() if ray_runtime_env else {}
            if "env_vars" in sanitized_env:
                sanitized_env["env_vars"] = {
                    k: "***" for k in sanitized_env["env_vars"]
                }
            logger.info("Using ray runtime env (env vars redacted): %s", sanitized_env)
1345

1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
        # Get the current placement group if Ray is initialized and
        # we are in a Ray actor. If so, then the placement group will be
        # passed to spawned processes.
        placement_group = None
        if is_in_ray_actor():
            import ray

            # This call initializes Ray automatically if it is not initialized,
            # but we should not do this here.
            placement_group = ray.util.get_current_placement_group()

1357
        assert not headless or not self.data_parallel_hybrid_lb, (
1358
1359
            "data_parallel_hybrid_lb is not applicable in headless mode"
        )
1360

1361
        data_parallel_external_lb = self.data_parallel_rank is not None
1362
        # Local DP rank = 1, use pure-external LB.
1363
1364
        if data_parallel_external_lb:
            assert self.data_parallel_size_local in (1, None), (
1365
1366
                "data_parallel_size_local must be 1 when data_parallel_rank is set"
            )
1367
            data_parallel_size_local = 1
1368
1369
            # Use full external lb if we have local_size of 1.
            self.data_parallel_hybrid_lb = False
1370
1371
        elif self.data_parallel_size_local is not None:
            data_parallel_size_local = self.data_parallel_size_local
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386

            if self.data_parallel_start_rank and not headless:
                # Infer hybrid LB mode.
                self.data_parallel_hybrid_lb = True

            if self.data_parallel_hybrid_lb and data_parallel_size_local == 1:
                # Use full external lb if we have local_size of 1.
                data_parallel_external_lb = True
                self.data_parallel_hybrid_lb = False

            if data_parallel_size_local == self.data_parallel_size:
                # Disable hybrid LB mode if set for a single node
                self.data_parallel_hybrid_lb = False

            self.data_parallel_rank = self.data_parallel_start_rank or 0
1387
        else:
1388
            assert not self.data_parallel_hybrid_lb, (
1389
1390
                "data_parallel_size_local must be set to use data_parallel_hybrid_lb."
            )
1391

1392
1393
            # Local DP size defaults to global DP size if not set.
            data_parallel_size_local = self.data_parallel_size
1394
1395
1396

        # DP address, used in multi-node case for torch distributed group
        # and ZMQ sockets.
Rui Qiao's avatar
Rui Qiao committed
1397
1398
1399
1400
        if self.data_parallel_address is None:
            if self.data_parallel_backend == "ray":
                host_ip = get_ip()
                logger.info(
1401
1402
                    "Using host IP %s as ray-based data parallel address", host_ip
                )
Rui Qiao's avatar
Rui Qiao committed
1403
1404
1405
1406
                data_parallel_address = host_ip
            else:
                assert self.data_parallel_backend == "mp", (
                    "data_parallel_backend can only be ray or mp, got %s",
1407
1408
                    self.data_parallel_backend,
                )
Rui Qiao's avatar
Rui Qiao committed
1409
1410
1411
                data_parallel_address = ParallelConfig.data_parallel_master_ip
        else:
            data_parallel_address = self.data_parallel_address
1412
1413
1414

        # This port is only used when there are remote data parallel engines,
        # otherwise the local IPC transport is used.
1415
        data_parallel_rpc_port = (
1416
            self.data_parallel_rpc_port
1417
1418
1419
            if (self.data_parallel_rpc_port is not None)
            else ParallelConfig.data_parallel_rpc_port
        )
1420

1421
1422
1423
1424
        if self.async_scheduling:
            # Async scheduling does not work with the uniprocess backend.
            if self.distributed_executor_backend is None:
                self.distributed_executor_backend = "mp"
1425
1426
1427
1428
                logger.info(
                    "Defaulting to mp-based distributed executor "
                    "backend for async scheduling."
                )
1429
            if self.pipeline_parallel_size > 1:
1430
1431
1432
                raise ValueError(
                    "Async scheduling is not supported with pipeline-parallel-size > 1."
                )
1433
1434
1435
1436
1437
1438

            # Currently, async scheduling does not support speculative decoding.
            # TODO(woosuk): Support it.
            if self.speculative_config is not None:
                raise ValueError(
                    "Currently, speculative decoding is not supported with "
1439
1440
                    "async scheduling."
                )
1441

1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
        # Forward the deprecated CLI args to the EPLB config.
        if self.num_redundant_experts is not None:
            self.eplb_config.num_redundant_experts = self.num_redundant_experts
        if self.eplb_window_size is not None:
            self.eplb_config.window_size = self.eplb_window_size
        if self.eplb_step_interval is not None:
            self.eplb_config.step_interval = self.eplb_step_interval
        if self.eplb_log_balancedness is not None:
            self.eplb_config.log_balancedness = self.eplb_log_balancedness

1452
        parallel_config = ParallelConfig(
1453
1454
            pipeline_parallel_size=self.pipeline_parallel_size,
            tensor_parallel_size=self.tensor_parallel_size,
1455
            data_parallel_size=self.data_parallel_size,
1456
1457
            data_parallel_rank=self.data_parallel_rank or 0,
            data_parallel_external_lb=data_parallel_external_lb,
1458
1459
1460
            data_parallel_size_local=data_parallel_size_local,
            data_parallel_master_ip=data_parallel_address,
            data_parallel_rpc_port=data_parallel_rpc_port,
1461
            data_parallel_backend=self.data_parallel_backend,
1462
            data_parallel_hybrid_lb=self.data_parallel_hybrid_lb,
1463
            enable_expert_parallel=self.enable_expert_parallel,
1464
1465
            enable_dbo=self.enable_dbo,
            dbo_decode_token_threshold=self.dbo_decode_token_threshold,
1466
            dbo_prefill_token_threshold=self.dbo_prefill_token_threshold,
1467
            disable_nccl_for_dp_synchronization=self.disable_nccl_for_dp_synchronization,
1468
            enable_eplb=self.enable_eplb,
1469
            eplb_config=self.eplb_config,
1470
            expert_placement_strategy=self.expert_placement_strategy,
1471
1472
1473
            max_parallel_loading_workers=self.max_parallel_loading_workers,
            disable_custom_all_reduce=self.disable_custom_all_reduce,
            ray_workers_use_nsight=self.ray_workers_use_nsight,
1474
            ray_runtime_env=ray_runtime_env,
1475
            placement_group=placement_group,
1476
1477
            distributed_executor_backend=self.distributed_executor_backend,
            worker_cls=self.worker_cls,
1478
            worker_extension_cls=self.worker_extension_cls,
1479
            decode_context_parallel_size=self.decode_context_parallel_size,
1480
1481
            _api_process_count=self._api_process_count,
            _api_process_rank=self._api_process_rank,
1482
        )
1483

1484
        speculative_config = self.create_speculative_config(
1485
1486
            target_model_config=model_config,
            target_parallel_config=parallel_config,
1487
            enable_chunked_prefill=self.enable_chunked_prefill,
1488
            disable_log_stats=self.disable_log_stats,
1489
1490
        )

1491
1492
1493
1494
1495
        # make sure num_lookahead_slots is set appropriately depending on
        # whether speculative decoding is enabled
        num_lookahead_slots = self.num_lookahead_slots
        if speculative_config is not None:
            num_lookahead_slots = speculative_config.num_lookahead_slots
1496

1497
        scheduler_config = SchedulerConfig(
1498
            runner_type=model_config.runner_type,
1499
1500
1501
            max_num_batched_tokens=self.max_num_batched_tokens,
            max_num_seqs=self.max_num_seqs,
            max_model_len=model_config.max_model_len,
1502
            cuda_graph_sizes=self.cuda_graph_sizes,
1503
            num_lookahead_slots=num_lookahead_slots,
1504
            enable_chunked_prefill=self.enable_chunked_prefill,
1505
            disable_chunked_mm_input=self.disable_chunked_mm_input,
1506
            is_multimodal_model=model_config.is_multimodal_model,
1507
            is_encoder_decoder=model_config.is_encoder_decoder,
1508
            send_delta_data=(envs.VLLM_USE_RAY_SPMD_WORKER and parallel_config.use_ray),
1509
            policy=self.scheduling_policy,
1510
            scheduler_cls=self.scheduler_cls,
1511
1512
1513
            max_num_partial_prefills=self.max_num_partial_prefills,
            max_long_partial_prefills=self.max_long_partial_prefills,
            long_prefill_token_threshold=self.long_prefill_token_threshold,
1514
            disable_hybrid_kv_cache_manager=self.disable_hybrid_kv_cache_manager,
1515
            async_scheduling=self.async_scheduling,
1516
        )
1517

1518
1519
1520
        if not model_config.is_multimodal_model and self.default_mm_loras:
            raise ValueError(
                "Default modality-specific LoRA(s) were provided for a "
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
                "non multimodal model"
            )

        lora_config = (
            LoRAConfig(
                max_lora_rank=self.max_lora_rank,
                max_loras=self.max_loras,
                default_mm_loras=self.default_mm_loras,
                fully_sharded_loras=self.fully_sharded_loras,
                lora_extra_vocab_size=self.lora_extra_vocab_size,
                lora_dtype=self.lora_dtype,
                max_cpu_loras=self.max_cpu_loras
                if self.max_cpu_loras and self.max_cpu_loras > 0
                else None,
            )
            if self.enable_lora
            else None
        )
1539

1540
1541
1542
1543
        # bitsandbytes pre-quantized model need a specific model loader
        if model_config.quantization == "bitsandbytes":
            self.quantization = self.load_format = "bitsandbytes"

1544
        load_config = self.create_load_config()
1545

1546
1547
        # Pass reasoning_parser into StructuredOutputsConfig
        if self.reasoning_parser:
1548
            self.structured_outputs_config.reasoning_parser = self.reasoning_parser
1549
1550
1551
1552

        # Forward the deprecated CLI args to the StructuredOutputsConfig
        so_config = self.structured_outputs_config
        if self.guided_decoding_backend is not None:
1553
            so_config.guided_decoding_backend = self.guided_decoding_backend
1554
        if self.guided_decoding_disable_fallback is not None:
1555
1556
1557
            so_config.guided_decoding_disable_fallback = (
                self.guided_decoding_disable_fallback
            )
1558
        if self.guided_decoding_disable_any_whitespace is not None:
1559
1560
1561
            so_config.guided_decoding_disable_any_whitespace = (
                self.guided_decoding_disable_any_whitespace
            )
1562
        if self.guided_decoding_disable_additional_properties is not None:
1563
1564
1565
            so_config.guided_decoding_disable_additional_properties = (
                self.guided_decoding_disable_additional_properties
            )
1566

1567
        observability_config = ObservabilityConfig(
1568
            show_hidden_metrics_for_version=(self.show_hidden_metrics_for_version),
1569
            otlp_traces_endpoint=self.otlp_traces_endpoint,
1570
            collect_detailed_traces=self.collect_detailed_traces,
1571
        )
1572

1573
        config = VllmConfig(
1574
1575
1576
1577
1578
1579
1580
1581
            model_config=model_config,
            cache_config=cache_config,
            parallel_config=parallel_config,
            scheduler_config=scheduler_config,
            device_config=device_config,
            lora_config=lora_config,
            speculative_config=speculative_config,
            load_config=load_config,
1582
            structured_outputs_config=self.structured_outputs_config,
1583
            observability_config=observability_config,
1584
            compilation_config=self.compilation_config,
1585
            kv_transfer_config=self.kv_transfer_config,
1586
            kv_events_config=self.kv_events_config,
1587
            additional_config=self.additional_config,
1588
        )
1589

1590
1591
        return config

1592
1593
1594
1595
1596
1597
    def _is_v1_supported_oracle(self, model_config: ModelConfig) -> bool:
        """Oracle for whether to use V0 or V1 Engine by default."""

        #############################################################
        # Unsupported Feature Flags on V1.

1598
1599
1600
1601
        if self.logits_processor_pattern != EngineArgs.logits_processor_pattern:
            _raise_or_fallback(
                feature_name="--logits-processor-pattern", recommend_to_remove=False
            )
1602
1603
            return False

1604
        # No Mamba or Encoder-Decoder so far.
1605
        if not model_config.is_v1_compatible:
1606
1607
1608
            _raise_or_fallback(
                feature_name=model_config.architectures, recommend_to_remove=False
            )
1609
1610
1611
            return False

        # No Concurrent Partial Prefills so far.
1612
1613
1614
1615
1616
1617
1618
1619
        if (
            self.max_num_partial_prefills != SchedulerConfig.max_num_partial_prefills
            or self.max_long_partial_prefills
            != SchedulerConfig.max_long_partial_prefills
        ):
            _raise_or_fallback(
                feature_name="Concurrent Partial Prefill", recommend_to_remove=False
            )
1620
1621
            return False

1622
        # V1 supports N-gram, Medusa, and Eagle speculative decoding.
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
        if self.speculative_config is not None:
            # speculative_config could still be a dict at this point
            if isinstance(self.speculative_config, dict):
                method = self.speculative_config.get("method", None)
            else:
                method = self.speculative_config.method

            if method == "draft_model":
                raise NotImplementedError(
                    "Draft model speculative decoding is not supported yet. "
                    "Please consider using other speculative decoding methods "
1634
1635
                    "such as ngram, medusa, eagle, or mtp."
                )
1636
1637

        V1_BACKENDS = [
1638
1639
            "FLASH_ATTN",
            "PALLAS",
1640
            "TRITON_ATTN",
1641
            "TRITON_MLA",
1642
            "CUTLASS_MLA",
1643
            "FLASHMLA",
1644
            "FLASH_ATTN_MLA",
1645
            "FLASHINFER",
1646
            "FLASHINFER_MLA",
1647
            "ROCM_AITER_MLA",
1648
            "TORCH_SDPA",
1649
            "FLEX_ATTENTION",
1650
            "TREE_ATTN",
1651
1652
            "XFORMERS",
            "ROCM_ATTN",
1653
            "ROCM_AITER_UNIFIED_ATTN",
1654
        ]
1655
1656
1657
1658
        if (
            envs.is_set("VLLM_ATTENTION_BACKEND")
            and envs.VLLM_ATTENTION_BACKEND not in V1_BACKENDS
        ):
1659
1660
1661
1662
1663
1664
1665
            name = f"VLLM_ATTENTION_BACKEND={envs.VLLM_ATTENTION_BACKEND}"
            _raise_or_fallback(feature_name=name, recommend_to_remove=True)
            return False

        #############################################################
        # Experimental Features - allow users to opt in.

1666
        if self.pipeline_parallel_size > 1:
1667
1668
1669
            supports_pp = getattr(
                self.distributed_executor_backend, "supports_pp", False
            )
1670
            if not supports_pp and self.distributed_executor_backend not in (
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
                ParallelConfig.distributed_executor_backend,
                "ray",
                "mp",
                "external_launcher",
            ):
                name = (
                    "Pipeline Parallelism without Ray distributed "
                    "executor or multiprocessing executor or external "
                    "launcher"
                )
                _raise_or_fallback(feature_name=name, recommend_to_remove=False)
1682
                return False
1683

1684
1685
1686
1687
        if current_platform.is_cpu() and model_config.get_sliding_window() is not None:
            _raise_or_fallback(
                feature_name="sliding window (CPU backend)", recommend_to_remove=False
            )
1688
1689
            return False

1690
1691
1692
1693
        #############################################################

        return True

1694
1695
1696
    def _set_default_args(
        self, usage_context: UsageContext, model_config: ModelConfig
    ) -> None:
1697
        """Set Default Arguments for V1 Engine."""
1698

1699
1700
1701
1702
1703
        # V1 always uses chunked prefills and prefix caching
        # for non-pooling tasks.
        # For pooling tasks the default is False
        if model_config.runner_type != "pooling":
            self.enable_chunked_prefill = True
1704
1705
1706

            # TODO: When prefix caching supports prompt embeds inputs, this
            # check can be removed.
1707
            if self.enable_prompt_embeds and self.enable_prefix_caching is not False:
1708
1709
1710
                logger.warning(
                    "--enable-prompt-embeds and --enable-prefix-caching "
                    "are not supported together in V1. Prefix caching has "
1711
1712
                    "been disabled."
                )
1713
1714
                self.enable_prefix_caching = False

1715
            if self.enable_prefix_caching is None:
1716
1717
1718
1719
1720
1721
                # Disable prefix caching default for hybrid models
                # since the feature is still experimental.
                if model_config.is_hybrid:
                    self.enable_prefix_caching = False
                else:
                    self.enable_prefix_caching = True
1722
1723
        else:
            pooling_type = model_config.pooler_config.pooling_type
1724
            is_causal = getattr(model_config.hf_config, "is_causal", True)
1725
1726
1727
1728
1729
            incremental_prefill_supported = (
                pooling_type is not None
                and pooling_type.lower() == "last"
                and is_causal
            )
1730

1731
            action = "Enabling" if incremental_prefill_supported else "Disabling"
1732
1733
1734
1735
1736
1737
1738
1739

            if self.enable_chunked_prefill is None:
                self.enable_chunked_prefill = incremental_prefill_supported
                logger.info("(%s) chunked prefill by default", action)
            if self.enable_prefix_caching is None:
                self.enable_prefix_caching = incremental_prefill_supported
                logger.info("(%s) prefix caching by default", action)

1740
1741
1742
        # V1 should use the new scheduler by default.
        # Swap it only if this arg is set to the original V0 default
        if self.scheduler_cls == EngineArgs.scheduler_cls:
1743
            self.scheduler_cls = "vllm.v1.core.sched.scheduler.Scheduler"
1744

1745
1746
        # When no user override, set the default values based on the usage
        # context.
1747
        # Use different default values for different hardware.
1748
1749
1750
1751
1752
1753
1754

        # Try to query the device name on the current platform. If it fails,
        # it may be because the platform that imports vLLM is not the same
        # as the platform that vLLM is running on (e.g. the case of scaling
        # vLLM with Ray) and has no GPUs. In this case we use the default
        # values for non-H100/H200 GPUs.
        try:
1755
            device_memory = current_platform.get_device_total_memory()
1756
            device_name = current_platform.get_device_name().lower()
1757
1758
        except Exception:
            # This is only used to set default_max_num_batched_tokens
1759
            device_memory = 0
1760

1761
1762
1763
        # NOTE(Kuntai): Setting large `max_num_batched_tokens` for A100 reduces
        # throughput, see PR #17885 for more details.
        # So here we do an extra device name check to prevent such regression.
1764
        from vllm.usage.usage_lib import UsageContext
1765

1766
        if device_memory >= 70 * GiB_bytes and "a100" not in device_name:
1767
            # For GPUs like H100 and MI300x, use larger default values.
1768
1769
1770
1771
            default_max_num_batched_tokens = {
                UsageContext.LLM_CLASS: 16384,
                UsageContext.OPENAI_API_SERVER: 8192,
            }
1772
1773
1774
1775
            default_max_num_seqs = {
                UsageContext.LLM_CLASS: 1024,
                UsageContext.OPENAI_API_SERVER: 1024,
            }
1776
1777
1778
1779
1780
1781
        else:
            # TODO(woosuk): Tune the default values for other hardware.
            default_max_num_batched_tokens = {
                UsageContext.LLM_CLASS: 8192,
                UsageContext.OPENAI_API_SERVER: 2048,
            }
1782
1783
1784
1785
            default_max_num_seqs = {
                UsageContext.LLM_CLASS: 256,
                UsageContext.OPENAI_API_SERVER: 256,
            }
1786

1787
1788
1789
1790
        # tpu specific default values.
        if current_platform.is_tpu():
            default_max_num_batched_tokens_tpu = {
                UsageContext.LLM_CLASS: {
1791
1792
1793
                    "V6E": 2048,
                    "V5E": 1024,
                    "V5P": 512,
1794
1795
                },
                UsageContext.OPENAI_API_SERVER: {
1796
1797
1798
1799
                    "V6E": 1024,
                    "V5E": 512,
                    "V5P": 256,
                },
1800
1801
            }

1802
1803
        # cpu specific default values.
        if current_platform.is_cpu():
1804
            world_size = self.pipeline_parallel_size * self.tensor_parallel_size
1805
            default_max_num_batched_tokens = {
1806
1807
                UsageContext.LLM_CLASS: 4096 * world_size,
                UsageContext.OPENAI_API_SERVER: 2048 * world_size,
1808
1809
            }
            default_max_num_seqs = {
1810
1811
                UsageContext.LLM_CLASS: 256 * world_size,
                UsageContext.OPENAI_API_SERVER: 128 * world_size,
1812
1813
            }

1814
        use_context_value = usage_context.value if usage_context else None
1815
1816
1817
1818
        if (
            self.max_num_batched_tokens is None
            and usage_context in default_max_num_batched_tokens
        ):
1819
1820
            if current_platform.is_tpu():
                chip_name = current_platform.get_device_name()
1821
1822
1823
1824
                if chip_name in default_max_num_batched_tokens_tpu[usage_context]:
                    self.max_num_batched_tokens = default_max_num_batched_tokens_tpu[
                        usage_context
                    ][chip_name]
1825
                else:
1826
1827
1828
                    self.max_num_batched_tokens = default_max_num_batched_tokens[
                        usage_context
                    ]
1829
            else:
1830
1831
1832
                if not self.enable_chunked_prefill:
                    self.max_num_batched_tokens = model_config.max_model_len
                else:
1833
1834
1835
                    self.max_num_batched_tokens = default_max_num_batched_tokens[
                        usage_context
                    ]
1836
            logger.debug(
1837
                "Setting max_num_batched_tokens to %d for %s usage context.",
1838
1839
1840
                self.max_num_batched_tokens,
                use_context_value,
            )
1841

1842
1843
1844
1845
1846
        if self.max_num_seqs is None and usage_context in default_max_num_seqs:
            self.max_num_seqs = min(
                default_max_num_seqs[usage_context],
                self.max_num_batched_tokens or sys.maxsize,
            )
1847

1848
1849
1850
1851
1852
            logger.debug(
                "Setting max_num_seqs to %d for %s usage context.",
                self.max_num_seqs,
                use_context_value,
            )
1853

1854

1855
@dataclass
Zhuohan Li's avatar
Zhuohan Li committed
1856
class AsyncEngineArgs(EngineArgs):
Woosuk Kwon's avatar
Woosuk Kwon committed
1857
    """Arguments for asynchronous vLLM engine."""
1858

1859
1860
1861
1862
1863
1864
    enable_log_requests: bool = False

    @property
    @deprecated(
        "`disable_log_requests` is deprecated and has been replaced with "
        "`enable_log_requests`. This will be removed in v0.12.0. Please use "
1865
1866
        "`enable_log_requests` instead."
    )
1867
1868
1869
1870
1871
1872
1873
    def disable_log_requests(self) -> bool:
        return not self.enable_log_requests

    @disable_log_requests.setter
    @deprecated(
        "`disable_log_requests` is deprecated and has been replaced with "
        "`enable_log_requests`. This will be removed in v0.12.0. Please use "
1874
1875
        "`enable_log_requests` instead."
    )
1876
1877
    def disable_log_requests(self, value: bool):
        self.enable_log_requests = not value
1878
1879

    @staticmethod
1880
1881
1882
    def add_cli_args(
        parser: FlexibleArgumentParser, async_args_only: bool = False
    ) -> FlexibleArgumentParser:
1883
        # Initialize plugin to update the parser, for example, The plugin may
1884
        # add a new kind of quantization method to --quantization argument or
1885
1886
        # a new device to --device argument.
        load_general_plugins()
1887
1888
        if not async_args_only:
            parser = EngineArgs.add_cli_args(parser)
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
        parser.add_argument(
            "--enable-log-requests",
            action=argparse.BooleanOptionalAction,
            default=AsyncEngineArgs.enable_log_requests,
            help="Enable logging requests.",
        )
        parser.add_argument(
            "--disable-log-requests",
            action=argparse.BooleanOptionalAction,
            default=not AsyncEngineArgs.enable_log_requests,
            help="[DEPRECATED] Disable logging requests.",
            deprecated=True,
        )
1902
        current_platform.pre_register_and_update(parser)
1903
        return parser
1904
1905


1906
1907
1908
def _raise_or_fallback(feature_name: str, recommend_to_remove: bool):
    if envs.is_set("VLLM_USE_V1") and envs.VLLM_USE_V1:
        raise NotImplementedError(
1909
1910
            f"VLLM_USE_V1=1 is not supported with {feature_name}."
        )
1911
1912
1913
1914
1915
1916
1917
1918
    msg = f"{feature_name} is not supported by the V1 Engine. "
    msg += "Falling back to V0. "
    if recommend_to_remove:
        msg += f"We recommend to remove {feature_name} from your config "
        msg += "in favor of the V1 Engine."
    logger.warning(msg)


1919
1920
1921
def human_readable_int(value):
    """Parse human-readable integers like '1k', '2M', etc.
    Including decimal values with decimal multipliers.
1922

1923
1924
1925
1926
1927
1928
    Examples:
    - '1k' -> 1,000
    - '1K' -> 1,024
    - '25.6k' -> 25,600
    """
    value = value.strip()
1929
    match = re.fullmatch(r"(\d+(?:\.\d+)?)([kKmMgGtT])", value)
1930
1931
    if match:
        decimal_multiplier = {
1932
1933
1934
            "k": 10**3,
            "m": 10**6,
            "g": 10**9,
1935
1936
        }
        binary_multiplier = {
1937
1938
1939
            "K": 2**10,
            "M": 2**20,
            "G": 2**30,
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
        }

        number, suffix = match.groups()
        if suffix in decimal_multiplier:
            mult = decimal_multiplier[suffix]
            return int(float(number) * mult)
        elif suffix in binary_multiplier:
            mult = binary_multiplier[suffix]
            # Do not allow decimals with binary multipliers
            try:
                return int(number) * mult
            except ValueError as e:
1952
1953
1954
1955
1956
                raise argparse.ArgumentTypeError(
                    "Decimals are not allowed "
                    f"with binary suffixes like {suffix}. Did you mean to use "
                    f"{number}{suffix.lower()} instead?"
                ) from e
1957
1958
1959

    # Regular plain number.
    return int(value)