arg_utils.py 87.8 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import os
5

6
import argparse
7
import copy
8
import dataclasses
9
import functools
10
import json
11
import sys
12
from collections.abc import Callable
13
from dataclasses import MISSING, dataclass, fields, is_dataclass
14
from itertools import permutations
15
from types import UnionType
16
17
18
19
20
from typing import (
    TYPE_CHECKING,
    Annotated,
    Any,
    Literal,
21
    TypeAlias,
22
23
24
25
26
27
    TypeVar,
    Union,
    cast,
    get_args,
    get_origin,
)
28

29
import huggingface_hub
30
import regex as re
31
import torch
32
from pydantic import TypeAdapter, ValidationError
33
from pydantic.fields import FieldInfo
34
from typing_extensions import TypeIs
35

36
import vllm.envs as envs
37
from vllm.config import (
38
    AttentionConfig,
39
40
41
42
    CacheConfig,
    CompilationConfig,
    ConfigType,
    DeviceConfig,
43
    ECTransferConfig,
44
45
46
47
48
49
    EPLBConfig,
    KVEventsConfig,
    KVTransferConfig,
    LoadConfig,
    LoRAConfig,
    ModelConfig,
50
    MultiModalConfig,
51
52
53
    ObservabilityConfig,
    ParallelConfig,
    PoolerConfig,
54
    ProfilerConfig,
55
56
57
58
59
60
    SchedulerConfig,
    SpeculativeConfig,
    StructuredOutputsConfig,
    VllmConfig,
    get_attr_docs,
)
61
62
63
64
65
66
67
from vllm.config.cache import (
    BlockSize,
    CacheDType,
    KVOffloadingBackend,
    MambaDType,
    PrefixCachingHashAlgo,
)
68
69
70
71
72
73
74
75
76
77
78
79
80
from vllm.config.device import Device
from vllm.config.model import (
    ConvertOption,
    HfOverrides,
    LogprobsMode,
    ModelDType,
    RunnerOption,
    TokenizerMode,
)
from vllm.config.multimodal import MMCacheType, MMEncoderTPMode
from vllm.config.observability import DetailedTraceModules
from vllm.config.parallel import DistributedExecutorBackend, ExpertPlacementStrategy
from vllm.config.scheduler import SchedulerPolicy
81
from vllm.config.utils import get_field
82
from vllm.config.vllm import OptimizationLevel
83
from vllm.logger import init_logger, suppress_logging
84
from vllm.platforms import CpuArchEnum, current_platform
85
from vllm.plugins import load_general_plugins
86
from vllm.ray.lazy_utils import is_in_ray_actor, is_ray_initialized
87
88
89
90
from vllm.transformers_utils.config import (
    is_interleaved,
    maybe_override_with_speculators,
)
91
from vllm.transformers_utils.gguf_utils import is_gguf
92
from vllm.transformers_utils.repo_utils import get_model_path
93
from vllm.transformers_utils.utils import is_cloud_storage
94
from vllm.utils.argparse_utils import FlexibleArgumentParser
95
from vllm.utils.mem_constants import GiB_bytes
96
from vllm.utils.network_utils import get_ip
97
from vllm.utils.torch_utils import resolve_kv_cache_dtype_string
98
from vllm.v1.attention.backends.registry import AttentionBackendEnum
99
from vllm.v1.sample.logits_processor import LogitsProcessor
100

101
102
if TYPE_CHECKING:
    from vllm.model_executor.layers.quantization import QuantizationMethods
103
    from vllm.model_executor.model_loader import LoadFormats
104
    from vllm.usage.usage_lib import UsageContext
105
    from vllm.v1.executor import Executor
106
else:
107
    Executor = Any
108
    QuantizationMethods = Any
109
    LoadFormats = Any
110
111
    UsageContext = Any

112

113
114
logger = init_logger(__name__)

115
116
# object is used to allow for special typing forms
T = TypeVar("T")
117
118
TypeHint: TypeAlias = type[Any] | object
TypeHintT: TypeAlias = type[T] | object
119

120

121
122
def parse_type(return_type: Callable[[str], T]) -> Callable[[str], T]:
    def _parse_type(val: str) -> T:
123
124
125
126
        try:
            return return_type(val)
        except ValueError as e:
            raise argparse.ArgumentTypeError(
127
128
                f"Value {val} cannot be converted to {return_type}."
            ) from e
129

130
131
132
    return _parse_type


133
134
def optional_type(return_type: Callable[[str], T]) -> Callable[[str], T | None]:
    def _optional_type(val: str) -> T | None:
135
136
137
138
        if val == "" or val == "None":
            return None
        return parse_type(return_type)(val)

139
    return _optional_type
140
141


142
def union_dict_and_str(val: str) -> str | dict[str, str] | None:
143
    if not re.match(r"(?s)^\s*{.*}\s*$", val):
144
        return str(val)
145
    return optional_type(json.loads)(val)
146
147


148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
def is_type(type_hint: TypeHint, type: TypeHintT) -> TypeIs[TypeHintT]:
    """Check if the type hint is a specific type."""
    return type_hint is type or get_origin(type_hint) is type


def contains_type(type_hints: set[TypeHint], type: TypeHintT) -> bool:
    """Check if the type hints contain a specific type."""
    return any(is_type(type_hint, type) for type_hint in type_hints)


def get_type(type_hints: set[TypeHint], type: TypeHintT) -> TypeHintT:
    """Get the specific type from the type hints."""
    return next((th for th in type_hints if is_type(th, type)), None)


163
def literal_to_kwargs(type_hints: set[TypeHint]) -> dict[str, Any]:
164
165
166
167
    """Get the `type` and `choices` from a `Literal` type hint in `type_hints`.

    If `type_hints` also contains `str`, we use `metavar` instead of `choices`.
    """
168
    type_hint = get_type(type_hints, Literal)
169
170
171
    options = get_args(type_hint)
    option_type = type(options[0])
    if not all(isinstance(option, option_type) for option in options):
172
        raise ValueError(
173
            "All options must be of the same type. "
174
175
            f"Got {options} with types {[type(c) for c in options]}"
        )
176
177
    kwarg = "metavar" if contains_type(type_hints, str) else "choices"
    return {"type": option_type, kwarg: sorted(options)}
178
179


180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
def collection_to_kwargs(type_hints: set[TypeHint], type: TypeHint) -> dict[str, Any]:
    type_hint = get_type(type_hints, type)
    types = get_args(type_hint)
    elem_type = types[0]

    # Handle Ellipsis
    assert all(t is elem_type for t in types if t is not Ellipsis), (
        f"All non-Ellipsis elements must be of the same type. Got {types}."
    )

    # Handle Union types
    if get_origin(elem_type) in {Union, UnionType}:
        # Union for Union[X, Y] and UnionType for X | Y
        assert str in get_args(elem_type), (
            "If element can have multiple types, one must be 'str' "
            f"(i.e. 'list[int | str]'). Got {elem_type}."
        )
        elem_type = str

    return {
        "type": elem_type,
        "nargs": "+" if type is not tuple or Ellipsis in types else len(types),
    }


205
206
207
208
209
def is_not_builtin(type_hint: TypeHint) -> bool:
    """Check if the class is not a built-in type."""
    return type_hint.__module__ != "builtins"


210
211
212
213
214
215
216
217
def get_type_hints(type_hint: TypeHint) -> set[TypeHint]:
    """Extract type hints from Annotated or Union type hints."""
    type_hints: set[TypeHint] = set()
    origin = get_origin(type_hint)
    args = get_args(type_hint)

    if origin is Annotated:
        type_hints.update(get_type_hints(args[0]))
218
219
    elif origin in {Union, UnionType}:
        # Union for Union[X, Y] and UnionType for X | Y
220
221
222
223
224
225
226
227
        for arg in args:
            type_hints.update(get_type_hints(arg))
    else:
        type_hints.add(type_hint)

    return type_hints


228
229
230
231
def is_online_quantization(quantization: Any) -> bool:
    return quantization in ["inc"]


232
NEEDS_HELP = (
233
234
    any("--help" in arg for arg in sys.argv)  # vllm SUBCOMMAND --help
    or (argv0 := sys.argv[0]).endswith("mkdocs")  # mkdocs SUBCOMMAND
235
236
237
238
    or argv0.endswith("mkdocs/__main__.py")  # python -m mkdocs SUBCOMMAND
)


239
@functools.lru_cache(maxsize=30)
240
def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]:
241
242
    # Save time only getting attr docs if we're generating help text
    cls_docs = get_attr_docs(cls) if NEEDS_HELP else {}
243
244
    kwargs = {}
    for field in fields(cls):
245
        # Get the set of possible types for the field
246
        type_hints: set[TypeHint] = get_type_hints(field.type)
247
248
249
250
251

        # If the field is a dataclass, we can use the model_validate_json
        generator = (th for th in type_hints if is_dataclass(th))
        dataclass_cls = next(generator, None)

252
        # Get the default value of the field
253
254
        if field.default is not MISSING:
            default = field.default
255
256
            # Handle pydantic.Field defaults
            if isinstance(default, FieldInfo):
257
258
259
260
261
262
263
                if default.default_factory is None:
                    default = default.default
                else:
                    # VllmConfig's Fields have default_factory set to config classes.
                    # These could emit logs on init, which would be confusing.
                    with suppress_logging():
                        default = default.default_factory()
264
        elif field.default_factory is not MISSING:
265
266
267
268
            default = field.default_factory()

        # Get the help text for the field
        name = field.name
269
        help = cls_docs.get(name, "").strip()
270
271
272
273
274
275
276
        # Escape % for argparse
        help = help.replace("%", "%%")

        # Initialise the kwargs dictionary for the field
        kwargs[name] = {"default": default, "help": help}

        # Set other kwargs based on the type hints
277
278
279
        json_tip = (
            "Should either be a valid JSON string or JSON keys passed individually."
        )
280
        if dataclass_cls is not None:
281
282
283
284
285
286
287
288

            def parse_dataclass(val: str, cls=dataclass_cls) -> Any:
                try:
                    return TypeAdapter(cls).validate_json(val)
                except ValidationError as e:
                    raise argparse.ArgumentTypeError(repr(e)) from e

            kwargs[name]["type"] = parse_dataclass
289
            kwargs[name]["help"] += f"\n\n{json_tip}"
290
        elif contains_type(type_hints, bool):
291
292
293
            # Creates --no-<name> and --<name> flags
            kwargs[name]["action"] = argparse.BooleanOptionalAction
        elif contains_type(type_hints, Literal):
294
            kwargs[name].update(literal_to_kwargs(type_hints))
295
        elif contains_type(type_hints, tuple):
296
            kwargs[name].update(collection_to_kwargs(type_hints, tuple))
297
        elif contains_type(type_hints, list):
298
299
300
            kwargs[name].update(collection_to_kwargs(type_hints, list))
        elif contains_type(type_hints, set):
            kwargs[name].update(collection_to_kwargs(type_hints, set))
301
        elif contains_type(type_hints, int):
302
303
304
305
            if name == "max_model_len":
                kwargs[name]["type"] = human_readable_int_or_auto
                kwargs[name]["help"] += f"\n\n{human_readable_int_or_auto.__doc__}"
            elif name in ("max_num_batched_tokens", "kv_cache_memory_bytes"):
306
                kwargs[name]["type"] = human_readable_int
307
                kwargs[name]["help"] += f"\n\n{human_readable_int.__doc__}"
308
309
            else:
                kwargs[name]["type"] = int
310
311
        elif contains_type(type_hints, float):
            kwargs[name]["type"] = float
312
313
314
315
        elif contains_type(type_hints, dict) and (
            contains_type(type_hints, str)
            or any(is_not_builtin(th) for th in type_hints)
        ):
316
            kwargs[name]["type"] = union_dict_and_str
317
        elif contains_type(type_hints, dict):
318
            kwargs[name]["type"] = parse_type(json.loads)
319
            kwargs[name]["help"] += f"\n\n{json_tip}"
320
321
322
        elif contains_type(type_hints, str) or any(
            is_not_builtin(th) for th in type_hints
        ):
323
324
            kwargs[name]["type"] = str
        else:
325
            raise ValueError(f"Unsupported type {type_hints} for argument {name}.")
326

327
328
329
330
331
        # If the type hint was a sequence of literals, use the helper function
        # to update the type and choices
        if get_origin(kwargs[name].get("type")) is Literal:
            kwargs[name].update(literal_to_kwargs({kwargs[name]["type"]}))

332
333
334
335
336
337
338
        # If None is in type_hints, make the argument optional.
        # But not if it's a bool, argparse will handle this better.
        if type(None) in type_hints and not contains_type(type_hints, bool):
            kwargs[name]["type"] = optional_type(kwargs[name]["type"])
            if kwargs[name].get("choices"):
                kwargs[name]["choices"].append("None")
    return kwargs
339
340


341
def get_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]:
342
343
    """Return argparse kwargs for the given Config dataclass.

344
345
346
    If `--help` or `mkdocs` are not present in the command line command, the
    attribute documentation will not be included in the help output.

347
348
349
350
351
352
353
    The heavy computation is cached via functools.lru_cache, and a deep copy
    is returned so callers can mutate the dictionary without affecting the
    cached version.
    """
    return copy.deepcopy(_compute_kwargs(cls))


354
355
356
357
358
359
360
361
362
class EnvironmentConfigError(Exception):
    pass

# def check_incompatible_config(env1: bool, env2: bool):
#     if env1 is True and env2 is True:
#         _s = "USE_FUSED_CUSTOM_ALL_REDUCE_RMS_QUANT and USE_FUSED_RMS_QUANT must not be enabled simultaneously!\n\n"
#         raise EnvironmentConfigError(_s)
    

363
@dataclass
Zhuohan Li's avatar
Zhuohan Li committed
364
class EngineArgs:
Woosuk Kwon's avatar
Woosuk Kwon committed
365
    """Arguments for vLLM engine."""
366

367
    model: str = ModelConfig.model
368
    enable_return_routed_experts: bool = ModelConfig.enable_return_routed_experts
369
    model_weights: str = ModelConfig.model_weights
370
371
372
    served_model_name: str | list[str] | None = ModelConfig.served_model_name
    tokenizer: str | None = ModelConfig.tokenizer
    hf_config_path: str | None = ModelConfig.hf_config_path
373
374
    runner: RunnerOption = ModelConfig.runner
    convert: ConvertOption = ModelConfig.convert
375
    skip_tokenizer_init: bool = ModelConfig.skip_tokenizer_init
376
    enable_prompt_embeds: bool = ModelConfig.enable_prompt_embeds
377
    tokenizer_mode: TokenizerMode | str = ModelConfig.tokenizer_mode
378
379
    trust_remote_code: bool = ModelConfig.trust_remote_code
    allowed_local_media_path: str = ModelConfig.allowed_local_media_path
380
381
    allowed_media_domains: list[str] | None = ModelConfig.allowed_media_domains
    download_dir: str | None = LoadConfig.download_dir
382
    safetensors_load_strategy: str = LoadConfig.safetensors_load_strategy
383
    load_format: str | LoadFormats = LoadConfig.load_format
384
385
    config_format: str = ModelConfig.config_format
    dtype: ModelDType = ModelConfig.dtype
386
    kv_cache_dtype: CacheDType = CacheConfig.cache_dtype
387
    seed: int = ModelConfig.seed
388
    max_model_len: int | None = ModelConfig.max_model_len
389
390
391
392
393
394
    cudagraph_capture_sizes: list[int] | None = (
        CompilationConfig.cudagraph_capture_sizes
    )
    max_cudagraph_capture_size: int | None = get_field(
        CompilationConfig, "max_cudagraph_capture_size"
    )
395
396
397
    # Note: Specifying a custom executor backend by passing a class
    # is intended for expert use only. The API may change without
    # notice.
398
    distributed_executor_backend: (
399
        str | DistributedExecutorBackend | type[Executor] | None
400
    ) = ParallelConfig.distributed_executor_backend
401
    # number of P/D disaggregation (or other disaggregation) workers
402
    pipeline_parallel_size: int = ParallelConfig.pipeline_parallel_size
403
404
405
406
    master_addr: str = ParallelConfig.master_addr
    master_port: int = ParallelConfig.master_port
    nnodes: int = ParallelConfig.nnodes
    node_rank: int = ParallelConfig.node_rank
407
    tensor_parallel_size: int = ParallelConfig.tensor_parallel_size
408
    prefill_context_parallel_size: int = ParallelConfig.prefill_context_parallel_size
409
    decode_context_parallel_size: int = ParallelConfig.decode_context_parallel_size
410
    dcp_kv_cache_interleave_size: int = ParallelConfig.dcp_kv_cache_interleave_size
411
    cp_kv_cache_interleave_size: int = ParallelConfig.cp_kv_cache_interleave_size
412
    data_parallel_size: int = ParallelConfig.data_parallel_size
413
414
415
416
417
    data_parallel_rank: int | None = None
    data_parallel_start_rank: int | None = None
    data_parallel_size_local: int | None = None
    data_parallel_address: str | None = None
    data_parallel_rpc_port: int | None = None
418
    data_parallel_hybrid_lb: bool = False
419
    data_parallel_external_lb: bool = False
Rui Qiao's avatar
Rui Qiao committed
420
    data_parallel_backend: str = ParallelConfig.data_parallel_backend
421
    enable_expert_parallel: bool = ParallelConfig.enable_expert_parallel
422
    all2all_backend: str = ParallelConfig.all2all_backend
423
    enable_dbo: bool = ParallelConfig.enable_dbo
424
    ubatch_size: int = ParallelConfig.ubatch_size
425
426
    dbo_decode_token_threshold: int = ParallelConfig.dbo_decode_token_threshold
    dbo_prefill_token_threshold: int = ParallelConfig.dbo_prefill_token_threshold
427
    disable_nccl_for_dp_synchronization: bool | None = (
428
429
        ParallelConfig.disable_nccl_for_dp_synchronization
    )
430
    eplb_config: EPLBConfig = get_field(ParallelConfig, "eplb_config")
431
    enable_eplb: bool = ParallelConfig.enable_eplb
432
    expert_placement_strategy: ExpertPlacementStrategy = (
433
        ParallelConfig.expert_placement_strategy
434
    )
435
436
    _api_process_count: int = ParallelConfig._api_process_count
    _api_process_rank: int = ParallelConfig._api_process_rank
437
    max_parallel_loading_workers: int | None = (
438
439
        ParallelConfig.max_parallel_loading_workers
    )
440
    block_size: BlockSize | None = CacheConfig.block_size
441
    enable_prefix_caching: bool | None = None
442
    prefix_caching_hash_algo: PrefixCachingHashAlgo = (
443
        CacheConfig.prefix_caching_hash_algo
444
    )
445
446
    disable_sliding_window: bool = ModelConfig.disable_sliding_window
    disable_cascade_attn: bool = ModelConfig.disable_cascade_attn
447
448
449
    swap_space: float = CacheConfig.swap_space
    cpu_offload_gb: float = CacheConfig.cpu_offload_gb
    gpu_memory_utilization: float = CacheConfig.gpu_memory_utilization
450
    kv_cache_memory_bytes: int | None = CacheConfig.kv_cache_memory_bytes
451
    max_num_batched_tokens: int | None = None
452
453
    max_num_partial_prefills: int = SchedulerConfig.max_num_partial_prefills
    max_long_partial_prefills: int = SchedulerConfig.max_long_partial_prefills
454
    long_prefill_token_threshold: int = SchedulerConfig.long_prefill_token_threshold
455
    max_num_seqs: int | None = None
456
    max_logprobs: int = ModelConfig.max_logprobs
457
    logprobs_mode: LogprobsMode = ModelConfig.logprobs_mode
458
    disable_log_stats: bool = False
459
    aggregate_engine_logging: bool = False
460
461
462
    revision: str | None = ModelConfig.revision
    code_revision: str | None = ModelConfig.code_revision
    hf_token: bool | str | None = ModelConfig.hf_token
463
    hf_overrides: HfOverrides = get_field(ModelConfig, "hf_overrides")
464
465
    tokenizer_revision: str | None = ModelConfig.tokenizer_revision
    quantization: QuantizationMethods | None = ModelConfig.quantization
466
    allow_deprecated_quantization: bool = ModelConfig.allow_deprecated_quantization
467
    enforce_eager: bool = ModelConfig.enforce_eager
468
    disable_custom_all_reduce: bool = ParallelConfig.disable_custom_all_reduce
469
    limit_mm_per_prompt: dict[str, int | dict[str, int]] = get_field(
470
471
        MultiModalConfig, "limit_per_prompt"
    )
472
    enable_mm_embeds: bool = MultiModalConfig.enable_mm_embeds
473
    interleave_mm_strings: bool = MultiModalConfig.interleave_mm_strings
474
475
476
    media_io_kwargs: dict[str, dict[str, Any]] = get_field(
        MultiModalConfig, "media_io_kwargs"
    )
477
    mm_processor_kwargs: dict[str, Any] | None = MultiModalConfig.mm_processor_kwargs
478
    mm_processor_cache_gb: float = MultiModalConfig.mm_processor_cache_gb
479
    mm_processor_cache_type: MMCacheType | None = (
480
        MultiModalConfig.mm_processor_cache_type
481
482
    )
    mm_shm_cache_max_object_size_mb: int = (
483
        MultiModalConfig.mm_shm_cache_max_object_size_mb
484
    )
485
    mm_encoder_tp_mode: MMEncoderTPMode = MultiModalConfig.mm_encoder_tp_mode
486
    mm_encoder_attn_backend: AttentionBackendEnum | str | None = (
487
488
        MultiModalConfig.mm_encoder_attn_backend
    )
489
    io_processor_plugin: str | None = None
490
    skip_mm_profiling: bool = MultiModalConfig.skip_mm_profiling
491
    video_pruning_rate: float = MultiModalConfig.video_pruning_rate
492
    # LoRA fields
493
    enable_lora: bool = False
494
495
    max_loras: int = LoRAConfig.max_loras
    max_lora_rank: int = LoRAConfig.max_lora_rank
496
    default_mm_loras: dict[str, str] | None = LoRAConfig.default_mm_loras
497
    fully_sharded_loras: bool = LoRAConfig.fully_sharded_loras
498
499
    max_cpu_loras: int | None = LoRAConfig.max_cpu_loras
    lora_dtype: str | torch.dtype | None = LoRAConfig.lora_dtype
500
    enable_tower_connector_lora: bool = LoRAConfig.enable_tower_connector_lora
501

502
    ray_workers_use_nsight: bool = ParallelConfig.ray_workers_use_nsight
503
    num_gpu_blocks_override: int | None = CacheConfig.num_gpu_blocks_override
504
    model_loader_extra_config: dict = get_field(LoadConfig, "model_loader_extra_config")
505
    ignore_patterns: str | list[str] = get_field(LoadConfig, "ignore_patterns")
506

507
    enable_chunked_prefill: bool | None = None
508
    disable_chunked_mm_input: bool = SchedulerConfig.disable_chunked_mm_input
509

510
    disable_hybrid_kv_cache_manager: bool | None = (
511
512
        SchedulerConfig.disable_hybrid_kv_cache_manager
    )
513

514
    structured_outputs_config: StructuredOutputsConfig = get_field(
515
516
        VllmConfig, "structured_outputs_config"
    )
517
    reasoning_parser: str = StructuredOutputsConfig.reasoning_parser
518
    reasoning_parser_plugin: str | None = None
519

520
    logits_processor_pattern: str | None = ModelConfig.logits_processor_pattern
521

522
    speculative_config: dict[str, Any] | None = None
523

524
    show_hidden_metrics_for_version: str | None = (
525
        ObservabilityConfig.show_hidden_metrics_for_version
526
    )
527
528
    otlp_traces_endpoint: str | None = ObservabilityConfig.otlp_traces_endpoint
    collect_detailed_traces: list[DetailedTraceModules] | None = (
529
        ObservabilityConfig.collect_detailed_traces
530
    )
531
532
533
534
    kv_cache_metrics: bool = ObservabilityConfig.kv_cache_metrics
    kv_cache_metrics_sample: float = get_field(
        ObservabilityConfig, "kv_cache_metrics_sample"
    )
535
    cudagraph_metrics: bool = ObservabilityConfig.cudagraph_metrics
536
537
538
    enable_layerwise_nvtx_tracing: bool = (
        ObservabilityConfig.enable_layerwise_nvtx_tracing
    )
539
    enable_mfu_metrics: bool = ObservabilityConfig.enable_mfu_metrics
540
541
542
    enable_logging_iteration_details: bool = (
        ObservabilityConfig.enable_logging_iteration_details
    )
543
    enable_mm_processor_stats: bool = ObservabilityConfig.enable_mm_processor_stats
544
    scheduling_policy: SchedulerPolicy = SchedulerConfig.policy
545
    scheduler_cls: str | type[object] | None = SchedulerConfig.scheduler_cls
546

547
    pooler_config: PoolerConfig | None = ModelConfig.pooler_config
548
    compilation_config: CompilationConfig = get_field(VllmConfig, "compilation_config")
549
    attention_config: AttentionConfig = get_field(VllmConfig, "attention_config")
550
551
    worker_cls: str = ParallelConfig.worker_cls
    worker_extension_cls: str = ParallelConfig.worker_extension_cls
552

553
554
    profiler_config: ProfilerConfig = get_field(VllmConfig, "profiler_config")

555
556
    kv_transfer_config: KVTransferConfig | None = None
    kv_events_config: KVEventsConfig | None = None
557

558
    ec_transfer_config: ECTransferConfig | None = None
559

560
561
    generation_config: str = ModelConfig.generation_config
    enable_sleep_mode: bool = ModelConfig.enable_sleep_mode
562
563
564
    override_generation_config: dict[str, Any] = get_field(
        ModelConfig, "override_generation_config"
    )
565
    model_impl: str = ModelConfig.model_impl
566
    override_attention_dtype: str = ModelConfig.override_attention_dtype
567
    attention_backend: AttentionBackendEnum | None = AttentionConfig.backend
568

569
    calculate_kv_scales: bool = CacheConfig.calculate_kv_scales
570
571
    mamba_cache_dtype: MambaDType = CacheConfig.mamba_cache_dtype
    mamba_ssm_cache_dtype: MambaDType = CacheConfig.mamba_ssm_cache_dtype
572
    mamba_block_size: int | None = get_field(CacheConfig, "mamba_block_size")
573

574
    additional_config: dict[str, Any] = get_field(VllmConfig, "additional_config")
575

576
    use_tqdm_on_load: bool = LoadConfig.use_tqdm_on_load
577
    pt_load_map_location: str = LoadConfig.pt_load_map_location
王敏's avatar
王敏 committed
578

579
    logits_processors: list[str | type[LogitsProcessor]] | None = (
580
581
        ModelConfig.logits_processors
    )
582
583
    """Custom logitproc types"""

584
    async_scheduling: bool | None = SchedulerConfig.async_scheduling
585

586
587
    stream_interval: int = SchedulerConfig.stream_interval

588
    kv_sharing_fast_prefill: bool = CacheConfig.kv_sharing_fast_prefill
589
    optimization_level: OptimizationLevel = VllmConfig.optimization_level
590

591
    kv_offloading_size: float | None = CacheConfig.kv_offloading_size
592
    kv_offloading_backend: KVOffloadingBackend = CacheConfig.kv_offloading_backend
593
    tokens_only: bool = False
594

595
    def __post_init__(self):
596
597
598
        # support `EngineArgs(compilation_config={...})`
        # without having to manually construct a
        # CompilationConfig object
599
        if isinstance(self.compilation_config, dict):
600
            self.compilation_config = CompilationConfig(**self.compilation_config)
601
602
        if isinstance(self.attention_config, dict):
            self.attention_config = AttentionConfig(**self.attention_config)
603
        if isinstance(self.eplb_config, dict):
604
            self.eplb_config = EPLBConfig(**self.eplb_config)
605
        # Setup plugins
606
        from vllm.plugins import load_general_plugins
607

608
        load_general_plugins()
609
        # when use hf offline,replace model and tokenizer id to local model path
610
611
612
        if huggingface_hub.constants.HF_HUB_OFFLINE:
            model_id = self.model
            self.model = get_model_path(self.model, self.revision)
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
            if model_id is not self.model:
                logger.info(
                    "HF_HUB_OFFLINE is True, replace model_id [%s] to model_path [%s]",
                    model_id,
                    self.model,
                )
            if self.tokenizer is not None:
                tokenizer_id = self.tokenizer
                self.tokenizer = get_model_path(self.tokenizer, self.tokenizer_revision)
                if tokenizer_id is not self.tokenizer:
                    logger.info(
                        "HF_HUB_OFFLINE is True, replace tokenizer_id [%s] "
                        "to tokenizer_path [%s]",
                        tokenizer_id,
                        self.tokenizer,
                    )
629
630

    @staticmethod
631
    def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
Woosuk Kwon's avatar
Woosuk Kwon committed
632
        """Shared CLI arguments for vLLM engine."""
633

634
        # Model arguments
635
636
637
638
639
        model_kwargs = get_kwargs(ModelConfig)
        model_group = parser.add_argument_group(
            title="ModelConfig",
            description=ModelConfig.__doc__,
        )
640
        if not ("serve" in sys.argv[1:] and "--help" in sys.argv[1:]):
641
            model_group.add_argument("--model", **model_kwargs["model"])
642
643
        model_group.add_argument("--runner", **model_kwargs["runner"])
        model_group.add_argument("--convert", **model_kwargs["convert"])
644
        model_group.add_argument("--tokenizer", **model_kwargs["tokenizer"])
645
646
647
648
        model_group.add_argument("--tokenizer-mode", **model_kwargs["tokenizer_mode"])
        model_group.add_argument(
            "--trust-remote-code", **model_kwargs["trust_remote_code"]
        )
649
650
        model_group.add_argument("--dtype", **model_kwargs["dtype"])
        model_group.add_argument("--seed", **model_kwargs["seed"])
651
652
653
654
655
656
657
        model_group.add_argument("--hf-config-path", **model_kwargs["hf_config_path"])
        model_group.add_argument(
            "--allowed-local-media-path", **model_kwargs["allowed_local_media_path"]
        )
        model_group.add_argument(
            "--allowed-media-domains", **model_kwargs["allowed_media_domains"]
        )
658
        model_group.add_argument("--revision", **model_kwargs["revision"])
659
660
661
662
663
664
        model_group.add_argument("--code-revision", **model_kwargs["code_revision"])
        model_group.add_argument(
            "--tokenizer-revision", **model_kwargs["tokenizer_revision"]
        )
        model_group.add_argument("--max-model-len", **model_kwargs["max_model_len"])
        model_group.add_argument("--quantization", "-q", **model_kwargs["quantization"])
665
666
667
668
        model_group.add_argument(
            "--allow-deprecated-quantization",
            **model_kwargs["allow_deprecated_quantization"],
        )
669
        model_group.add_argument("--enforce-eager", **model_kwargs["enforce_eager"])
670
671
672
673
        model_group.add_argument(
            "--enable-return-routed-experts",
            **model_kwargs["enable_return_routed_experts"],
        )
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
        model_group.add_argument("--max-logprobs", **model_kwargs["max_logprobs"])
        model_group.add_argument("--logprobs-mode", **model_kwargs["logprobs_mode"])
        model_group.add_argument(
            "--disable-sliding-window", **model_kwargs["disable_sliding_window"]
        )
        model_group.add_argument(
            "--disable-cascade-attn", **model_kwargs["disable_cascade_attn"]
        )
        model_group.add_argument(
            "--skip-tokenizer-init", **model_kwargs["skip_tokenizer_init"]
        )
        model_group.add_argument(
            "--enable-prompt-embeds", **model_kwargs["enable_prompt_embeds"]
        )
        model_group.add_argument(
            "--served-model-name", **model_kwargs["served_model_name"]
        )
        model_group.add_argument("--config-format", **model_kwargs["config_format"])
692
693
        # This one is a special case because it can bool
        # or str. TODO: Handle this in get_kwargs
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
        model_group.add_argument(
            "--hf-token",
            type=str,
            nargs="?",
            const=True,
            default=model_kwargs["hf_token"]["default"],
            help=model_kwargs["hf_token"]["help"],
        )
        model_group.add_argument("--hf-overrides", **model_kwargs["hf_overrides"])
        model_group.add_argument("--pooler-config", **model_kwargs["pooler_config"])
        model_group.add_argument(
            "--logits-processor-pattern", **model_kwargs["logits_processor_pattern"]
        )
        model_group.add_argument(
            "--generation-config", **model_kwargs["generation_config"]
        )
        model_group.add_argument(
            "--override-generation-config", **model_kwargs["override_generation_config"]
        )
        model_group.add_argument(
            "--enable-sleep-mode", **model_kwargs["enable_sleep_mode"]
        )
716
        model_group.add_argument("--model-impl", **model_kwargs["model_impl"])
717
718
719
720
721
722
723
724
725
        model_group.add_argument(
            "--override-attention-dtype", **model_kwargs["override_attention_dtype"]
        )
        model_group.add_argument(
            "--logits-processors", **model_kwargs["logits_processors"]
        )
        model_group.add_argument(
            "--io-processor-plugin", **model_kwargs["io_processor_plugin"]
        )
726

727
728
729
730
731
732
        # Model loading arguments
        load_kwargs = get_kwargs(LoadConfig)
        load_group = parser.add_argument_group(
            title="LoadConfig",
            description=LoadConfig.__doc__,
        )
733
        load_group.add_argument("--load-format", **load_kwargs["load_format"])
734
735
736
737
738
739
740
741
742
743
744
745
        load_group.add_argument("--download-dir", **load_kwargs["download_dir"])
        load_group.add_argument(
            "--safetensors-load-strategy", **load_kwargs["safetensors_load_strategy"]
        )
        load_group.add_argument(
            "--model-loader-extra-config", **load_kwargs["model_loader_extra_config"]
        )
        load_group.add_argument("--ignore-patterns", **load_kwargs["ignore_patterns"])
        load_group.add_argument("--use-tqdm-on-load", **load_kwargs["use_tqdm_on_load"])
        load_group.add_argument(
            "--pt-load-map-location", **load_kwargs["pt_load_map_location"]
        )
746

747
748
749
750
751
752
753
754
755
756
        # Attention arguments
        attention_kwargs = get_kwargs(AttentionConfig)
        attention_group = parser.add_argument_group(
            title="AttentionConfig",
            description=AttentionConfig.__doc__,
        )
        attention_group.add_argument(
            "--attention-backend", **attention_kwargs["backend"]
        )

757
758
759
760
761
        # Structured outputs arguments
        structured_outputs_kwargs = get_kwargs(StructuredOutputsConfig)
        structured_outputs_group = parser.add_argument_group(
            title="StructuredOutputsConfig",
            description=StructuredOutputsConfig.__doc__,
762
        )
763
        structured_outputs_group.add_argument(
764
            "--reasoning-parser",
765
            # Choices need to be validated after parsing to include plugins
766
767
            **structured_outputs_kwargs["reasoning_parser"],
        )
768
769
770
771
        structured_outputs_group.add_argument(
            "--reasoning-parser-plugin",
            **structured_outputs_kwargs["reasoning_parser_plugin"],
        )
772

773
        # Parallel arguments
774
775
776
777
778
779
        parallel_kwargs = get_kwargs(ParallelConfig)
        parallel_group = parser.add_argument_group(
            title="ParallelConfig",
            description=ParallelConfig.__doc__,
        )
        parallel_group.add_argument(
780
            "--distributed-executor-backend",
781
782
            **parallel_kwargs["distributed_executor_backend"],
        )
783
        parallel_group.add_argument(
784
785
786
787
            "--pipeline-parallel-size",
            "-pp",
            **parallel_kwargs["pipeline_parallel_size"],
        )
788
789
790
791
        parallel_group.add_argument("--master-addr", **parallel_kwargs["master_addr"])
        parallel_group.add_argument("--master-port", **parallel_kwargs["master_port"])
        parallel_group.add_argument("--nnodes", "-n", **parallel_kwargs["nnodes"])
        parallel_group.add_argument("--node-rank", "-r", **parallel_kwargs["node_rank"])
792
        parallel_group.add_argument(
793
794
            "--tensor-parallel-size", "-tp", **parallel_kwargs["tensor_parallel_size"]
        )
795
        parallel_group.add_argument(
796
797
798
799
            "--decode-context-parallel-size",
            "-dcp",
            **parallel_kwargs["decode_context_parallel_size"],
        )
800
        parallel_group.add_argument(
801
802
803
            "--dcp-kv-cache-interleave-size",
            **parallel_kwargs["dcp_kv_cache_interleave_size"],
        )
804
        parallel_group.add_argument(
805
806
807
            "--cp-kv-cache-interleave-size",
            **parallel_kwargs["cp_kv_cache_interleave_size"],
        )
808
        parallel_group.add_argument(
809
810
811
812
            "--prefill-context-parallel-size",
            "-pcp",
            **parallel_kwargs["prefill_context_parallel_size"],
        )
813
        parallel_group.add_argument(
814
815
            "--data-parallel-size", "-dp", **parallel_kwargs["data_parallel_size"]
        )
816
        parallel_group.add_argument(
817
818
            "--data-parallel-rank",
            "-dpn",
819
            type=int,
820
821
822
            help="Data parallel rank of this instance. "
            "When set, enables external load balancer mode.",
        )
823
        parallel_group.add_argument(
824
825
            "--data-parallel-start-rank",
            "-dpr",
826
            type=int,
827
828
            help="Starting data parallel rank for secondary nodes.",
        )
829
        parallel_group.add_argument(
830
831
            "--data-parallel-size-local",
            "-dpl",
832
            type=int,
833
834
            help="Number of data parallel replicas to run on this node.",
        )
835
        parallel_group.add_argument(
836
837
838
839
840
841
842
843
            "--data-parallel-address",
            "-dpa",
            type=str,
            help="Address of data parallel cluster head-node.",
        )
        parallel_group.add_argument(
            "--data-parallel-rpc-port",
            "-dpp",
844
            type=int,
845
846
            help="Port for data parallel RPC communication.",
        )
847
        parallel_group.add_argument(
848
849
850
851
852
853
            "--data-parallel-backend",
            "-dpb",
            type=str,
            default="mp",
            help='Backend for data parallel, either "mp" or "ray".',
        )
854
        parallel_group.add_argument(
855
856
857
858
859
860
861
862
            "--data-parallel-hybrid-lb",
            "-dph",
            **parallel_kwargs["data_parallel_hybrid_lb"],
        )
        parallel_group.add_argument(
            "--data-parallel-external-lb",
            "-dpe",
            **parallel_kwargs["data_parallel_external_lb"],
863
864
        )
        parallel_group.add_argument(
865
866
867
            "--enable-expert-parallel",
            "-ep",
            **parallel_kwargs["enable_expert_parallel"],
868
        )
869
870
871
        parallel_group.add_argument(
            "--all2all-backend", **parallel_kwargs["all2all_backend"]
        )
872
        parallel_group.add_argument("--enable-dbo", **parallel_kwargs["enable_dbo"])
873
874
875
876
        parallel_group.add_argument(
            "--ubatch-size",
            **parallel_kwargs["ubatch_size"],
        )
877
878
        parallel_group.add_argument(
            "--dbo-decode-token-threshold",
879
880
            **parallel_kwargs["dbo_decode_token_threshold"],
        )
881
882
        parallel_group.add_argument(
            "--dbo-prefill-token-threshold",
883
884
            **parallel_kwargs["dbo_prefill_token_threshold"],
        )
885
886
887
888
        parallel_group.add_argument(
            "--disable-nccl-for-dp-synchronization",
            **parallel_kwargs["disable_nccl_for_dp_synchronization"],
        )
889
890
        parallel_group.add_argument("--enable-eplb", **parallel_kwargs["enable_eplb"])
        parallel_group.add_argument("--eplb-config", **parallel_kwargs["eplb_config"])
891
892
        parallel_group.add_argument(
            "--expert-placement-strategy",
893
894
            **parallel_kwargs["expert_placement_strategy"],
        )
895

896
        parallel_group.add_argument(
897
            "--max-parallel-loading-workers",
898
899
            **parallel_kwargs["max_parallel_loading_workers"],
        )
900
        parallel_group.add_argument(
901
902
            "--ray-workers-use-nsight", **parallel_kwargs["ray_workers_use_nsight"]
        )
903
        parallel_group.add_argument(
904
            "--disable-custom-all-reduce",
905
906
907
908
909
910
            **parallel_kwargs["disable_custom_all_reduce"],
        )
        parallel_group.add_argument("--worker-cls", **parallel_kwargs["worker_cls"])
        parallel_group.add_argument(
            "--worker-extension-cls", **parallel_kwargs["worker_extension_cls"]
        )
911

912
913
914
915
916
        # KV cache arguments
        cache_kwargs = get_kwargs(CacheConfig)
        cache_group = parser.add_argument_group(
            title="CacheConfig",
            description=CacheConfig.__doc__,
917
        )
918
        cache_group.add_argument("--block-size", **cache_kwargs["block_size"])
919
920
921
922
923
924
        cache_group.add_argument(
            "--gpu-memory-utilization", **cache_kwargs["gpu_memory_utilization"]
        )
        cache_group.add_argument(
            "--kv-cache-memory-bytes", **cache_kwargs["kv_cache_memory_bytes"]
        )
925
        cache_group.add_argument("--swap-space", **cache_kwargs["swap_space"])
926
927
928
929
930
        cache_group.add_argument("--kv-cache-dtype", **cache_kwargs["cache_dtype"])
        cache_group.add_argument(
            "--num-gpu-blocks-override", **cache_kwargs["num_gpu_blocks_override"]
        )
        cache_group.add_argument(
931
932
933
934
935
            "--enable-prefix-caching",
            **{
                **cache_kwargs["enable_prefix_caching"],
                "default": None,
            },
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
        )
        cache_group.add_argument(
            "--prefix-caching-hash-algo", **cache_kwargs["prefix_caching_hash_algo"]
        )
        cache_group.add_argument("--cpu-offload-gb", **cache_kwargs["cpu_offload_gb"])
        cache_group.add_argument(
            "--calculate-kv-scales", **cache_kwargs["calculate_kv_scales"]
        )
        cache_group.add_argument(
            "--kv-sharing-fast-prefill", **cache_kwargs["kv_sharing_fast_prefill"]
        )
        cache_group.add_argument(
            "--mamba-cache-dtype", **cache_kwargs["mamba_cache_dtype"]
        )
        cache_group.add_argument(
            "--mamba-ssm-cache-dtype", **cache_kwargs["mamba_ssm_cache_dtype"]
        )
953
954
955
        cache_group.add_argument(
            "--mamba-block-size", **cache_kwargs["mamba_block_size"]
        )
956
957
958
959
960
961
        cache_group.add_argument(
            "--kv-offloading-size", **cache_kwargs["kv_offloading_size"]
        )
        cache_group.add_argument(
            "--kv-offloading-backend", **cache_kwargs["kv_offloading_backend"]
        )
962

963
        # Multimodal related configs
964
965
966
967
968
        multimodal_kwargs = get_kwargs(MultiModalConfig)
        multimodal_group = parser.add_argument_group(
            title="MultiModalConfig",
            description=MultiModalConfig.__doc__,
        )
969
        multimodal_group.add_argument(
970
971
            "--limit-mm-per-prompt", **multimodal_kwargs["limit_per_prompt"]
        )
972
973
974
        multimodal_group.add_argument(
            "--enable-mm-embeds", **multimodal_kwargs["enable_mm_embeds"]
        )
975
976
977
        multimodal_group.add_argument(
            "--media-io-kwargs", **multimodal_kwargs["media_io_kwargs"]
        )
978
        multimodal_group.add_argument(
979
980
            "--mm-processor-kwargs", **multimodal_kwargs["mm_processor_kwargs"]
        )
981
        multimodal_group.add_argument(
982
983
            "--mm-processor-cache-gb", **multimodal_kwargs["mm_processor_cache_gb"]
        )
984
        multimodal_group.add_argument(
985
986
            "--mm-processor-cache-type", **multimodal_kwargs["mm_processor_cache_type"]
        )
987
988
        multimodal_group.add_argument(
            "--mm-shm-cache-max-object-size-mb",
989
990
            **multimodal_kwargs["mm_shm_cache_max_object_size_mb"],
        )
991
        multimodal_group.add_argument(
992
993
            "--mm-encoder-tp-mode", **multimodal_kwargs["mm_encoder_tp_mode"]
        )
994
        multimodal_group.add_argument(
995
996
997
            "--mm-encoder-attn-backend",
            **multimodal_kwargs["mm_encoder_attn_backend"],
        )
998
999
1000
        multimodal_group.add_argument(
            "--interleave-mm-strings", **multimodal_kwargs["interleave_mm_strings"]
        )
1001
        multimodal_group.add_argument(
1002
1003
            "--skip-mm-profiling", **multimodal_kwargs["skip_mm_profiling"]
        )
1004

1005
        multimodal_group.add_argument(
1006
1007
            "--video-pruning-rate", **multimodal_kwargs["video_pruning_rate"]
        )
1008

1009
        # LoRA related configs
1010
1011
1012
1013
1014
1015
        lora_kwargs = get_kwargs(LoRAConfig)
        lora_group = parser.add_argument_group(
            title="LoRAConfig",
            description=LoRAConfig.__doc__,
        )
        lora_group.add_argument(
1016
            "--enable-lora",
1017
            action=argparse.BooleanOptionalAction,
1018
1019
            help="If True, enable handling of LoRA adapters.",
        )
1020
        lora_group.add_argument("--max-loras", **lora_kwargs["max_loras"])
1021
        lora_group.add_argument("--max-lora-rank", **lora_kwargs["max_lora_rank"])
1022
        lora_group.add_argument(
1023
            "--lora-dtype",
1024
1025
            **lora_kwargs["lora_dtype"],
        )
1026
1027
1028
1029
        lora_group.add_argument(
            "--enable-tower-connector-lora",
            **lora_kwargs["enable_tower_connector_lora"],
        )
1030
1031
1032
1033
1034
        lora_group.add_argument("--max-cpu-loras", **lora_kwargs["max_cpu_loras"])
        lora_group.add_argument(
            "--fully-sharded-loras", **lora_kwargs["fully_sharded_loras"]
        )
        lora_group.add_argument("--default-mm-loras", **lora_kwargs["default_mm_loras"])
1035

1036

1037
1038
1039
1040
1041
1042
1043
1044
        # Observability arguments
        observability_kwargs = get_kwargs(ObservabilityConfig)
        observability_group = parser.add_argument_group(
            title="ObservabilityConfig",
            description=ObservabilityConfig.__doc__,
        )
        observability_group.add_argument(
            "--show-hidden-metrics-for-version",
1045
1046
            **observability_kwargs["show_hidden_metrics_for_version"],
        )
1047
        observability_group.add_argument(
1048
1049
            "--otlp-traces-endpoint", **observability_kwargs["otlp_traces_endpoint"]
        )
1050
1051
1052
1053
1054
        # TODO: generalise this special case
        choices = observability_kwargs["collect_detailed_traces"]["choices"]
        metavar = f"{{{','.join(choices)}}}"
        observability_kwargs["collect_detailed_traces"]["metavar"] = metavar
        observability_kwargs["collect_detailed_traces"]["choices"] += [
1055
            ",".join(p) for p in permutations(get_args(DetailedTraceModules), r=2)
1056
1057
1058
        ]
        observability_group.add_argument(
            "--collect-detailed-traces",
1059
1060
            **observability_kwargs["collect_detailed_traces"],
        )
1061
1062
1063
1064
1065
1066
1067
        observability_group.add_argument(
            "--kv-cache-metrics", **observability_kwargs["kv_cache_metrics"]
        )
        observability_group.add_argument(
            "--kv-cache-metrics-sample",
            **observability_kwargs["kv_cache_metrics_sample"],
        )
1068
1069
1070
1071
        observability_group.add_argument(
            "--cudagraph-metrics",
            **observability_kwargs["cudagraph_metrics"],
        )
1072
1073
1074
1075
        observability_group.add_argument(
            "--enable-layerwise-nvtx-tracing",
            **observability_kwargs["enable_layerwise_nvtx_tracing"],
        )
1076
1077
1078
1079
        observability_group.add_argument(
            "--enable-mfu-metrics",
            **observability_kwargs["enable_mfu_metrics"],
        )
1080
1081
1082
1083
        observability_group.add_argument(
            "--enable-logging-iteration-details",
            **observability_kwargs["enable_logging_iteration_details"],
        )
1084

1085
1086
1087
1088
1089
1090
1091
        # Scheduler arguments
        scheduler_kwargs = get_kwargs(SchedulerConfig)
        scheduler_group = parser.add_argument_group(
            title="SchedulerConfig",
            description=SchedulerConfig.__doc__,
        )
        scheduler_group.add_argument(
1092
            "--max-num-batched-tokens",
1093
1094
1095
1096
            **{
                **scheduler_kwargs["max_num_batched_tokens"],
                "default": None,
            },
1097
        )
1098
        scheduler_group.add_argument(
1099
1100
1101
1102
1103
            "--max-num-seqs",
            **{
                **scheduler_kwargs["max_num_seqs"],
                "default": None,
            },
1104
1105
1106
1107
        )
        scheduler_group.add_argument(
            "--max-num-partial-prefills", **scheduler_kwargs["max_num_partial_prefills"]
        )
1108
1109
        scheduler_group.add_argument(
            "--max-long-partial-prefills",
1110
1111
            **scheduler_kwargs["max_long_partial_prefills"],
        )
1112
1113
        scheduler_group.add_argument(
            "--long-prefill-token-threshold",
1114
1115
            **scheduler_kwargs["long_prefill_token_threshold"],
        )
1116
1117
        # multi-step scheduling has been removed; corresponding arguments
        # are no longer supported.
1118
        scheduler_group.add_argument(
1119
1120
            "--scheduling-policy", **scheduler_kwargs["policy"]
        )
1121
        scheduler_group.add_argument(
1122
            "--enable-chunked-prefill",
1123
1124
1125
1126
            **{
                **scheduler_kwargs["enable_chunked_prefill"],
                "default": None,
            },
1127
        )
1128
        scheduler_group.add_argument(
1129
1130
1131
1132
1133
            "--disable-chunked-mm-input", **scheduler_kwargs["disable_chunked_mm_input"]
        )
        scheduler_group.add_argument(
            "--scheduler-cls", **scheduler_kwargs["scheduler_cls"]
        )
1134
1135
        scheduler_group.add_argument(
            "--disable-hybrid-kv-cache-manager",
1136
1137
1138
1139
1140
            **scheduler_kwargs["disable_hybrid_kv_cache_manager"],
        )
        scheduler_group.add_argument(
            "--async-scheduling", **scheduler_kwargs["async_scheduling"]
        )
1141
1142
1143
        scheduler_group.add_argument(
            "--stream-interval", **scheduler_kwargs["stream_interval"]
        )
1144

1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
        # Compilation arguments
        compilation_kwargs = get_kwargs(CompilationConfig)
        compilation_group = parser.add_argument_group(
            title="CompilationConfig",
            description=CompilationConfig.__doc__,
        )
        compilation_group.add_argument(
            "--cudagraph-capture-sizes", **compilation_kwargs["cudagraph_capture_sizes"]
        )
        compilation_group.add_argument(
            "--max-cudagraph-capture-size",
            **compilation_kwargs["max_cudagraph_capture_size"],
        )
1158
1159

        # vLLM arguments
1160
        vllm_kwargs = get_kwargs(VllmConfig)
1161
1162
1163
        vllm_group = parser.add_argument_group(
            title="VllmConfig",
            description=VllmConfig.__doc__,
1164
        )
1165
1166
1167
1168
        # We construct SpeculativeConfig using fields from other configs in
        # create_engine_config. So we set the type to a JSON string here to
        # delay the Pydantic validation that comes with SpeculativeConfig.
        vllm_kwargs["speculative_config"]["type"] = optional_type(json.loads)
1169
1170
1171
1172
1173
1174
1175
        vllm_group.add_argument(
            "--speculative-config", **vllm_kwargs["speculative_config"]
        )
        vllm_group.add_argument(
            "--kv-transfer-config", **vllm_kwargs["kv_transfer_config"]
        )
        vllm_group.add_argument("--kv-events-config", **vllm_kwargs["kv_events_config"])
1176
1177
1178
        vllm_group.add_argument(
            "--ec-transfer-config", **vllm_kwargs["ec_transfer_config"]
        )
1179
        vllm_group.add_argument(
1180
            "--compilation-config", "-cc", **vllm_kwargs["compilation_config"]
1181
        )
1182
1183
1184
        vllm_group.add_argument(
            "--attention-config", "-ac", **vllm_kwargs["attention_config"]
        )
1185
1186
1187
1188
1189
1190
        vllm_group.add_argument(
            "--additional-config", **vllm_kwargs["additional_config"]
        )
        vllm_group.add_argument(
            "--structured-outputs-config", **vllm_kwargs["structured_outputs_config"]
        )
1191
        vllm_group.add_argument("--profiler-config", **vllm_kwargs["profiler_config"])
1192
1193
1194
        vllm_group.add_argument(
            "--optimization-level", **vllm_kwargs["optimization_level"]
        )
1195

1196
        # Other arguments
1197
1198
1199
1200
1201
        parser.add_argument(
            "--disable-log-stats",
            action="store_true",
            help="Disable logging statistics.",
        )
1202

1203
1204
1205
1206
1207
1208
        parser.add_argument(
            "--aggregate-engine-logging",
            action="store_true",
            help="Log aggregate rather than per-engine statistics "
            "when using data parallelism.",
        )
1209
        return parser
1210
1211

    @classmethod
1212
    def from_cli_args(cls, args: argparse.Namespace):
1213
1214
1215
        # Get the list of attributes of this dataclass.
        attrs = [attr.name for attr in dataclasses.fields(cls)]
        # Set the attributes from the parsed arguments.
1216
1217
1218
        engine_args = cls(
            **{attr: getattr(args, attr) for attr in attrs if hasattr(args, attr)}
        )
Zhuohan Li's avatar
Zhuohan Li committed
1219
        return engine_args
1220

1221
    def create_model_config(self) -> ModelConfig:
1222
1223
        # gguf file needs a specific model loader
        if is_gguf(self.model):
1224
1225
            self.quantization = self.load_format = "gguf"

1226
1227
1228
1229
1230
1231
1232
        if not envs.VLLM_ENABLE_V1_MULTIPROCESSING:
            logger.warning(
                "The global random seed is set to %d. Since "
                "VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may "
                "affect the random state of the Python process that "
                "launched vLLM.",
                self.seed,
1233
            )
1234

1235
        return ModelConfig(
1236
            model=self.model,
1237
            model_weights=self.model_weights,
1238
            hf_config_path=self.hf_config_path,
1239
1240
            runner=self.runner,
            convert=self.convert,
1241
            tokenizer=self.tokenizer,
1242
1243
            tokenizer_mode=self.tokenizer_mode,
            trust_remote_code=self.trust_remote_code,
1244
            allowed_local_media_path=self.allowed_local_media_path,
1245
            allowed_media_domains=self.allowed_media_domains,
1246
1247
1248
1249
            dtype=self.dtype,
            seed=self.seed,
            revision=self.revision,
            code_revision=self.code_revision,
1250
            hf_token=self.hf_token,
1251
            hf_overrides=self.hf_overrides,
1252
1253
1254
            tokenizer_revision=self.tokenizer_revision,
            max_model_len=self.max_model_len,
            quantization=self.quantization,
1255
            allow_deprecated_quantization=self.allow_deprecated_quantization,
1256
            enforce_eager=self.enforce_eager,
1257
            enable_return_routed_experts=self.enable_return_routed_experts,
1258
            max_logprobs=self.max_logprobs,
1259
            logprobs_mode=self.logprobs_mode,
1260
            disable_sliding_window=self.disable_sliding_window,
1261
            disable_cascade_attn=self.disable_cascade_attn,
1262
            skip_tokenizer_init=self.skip_tokenizer_init,
1263
            enable_prompt_embeds=self.enable_prompt_embeds,
1264
            served_model_name=self.served_model_name,
1265
            limit_mm_per_prompt=self.limit_mm_per_prompt,
1266
            enable_mm_embeds=self.enable_mm_embeds,
1267
            interleave_mm_strings=self.interleave_mm_strings,
1268
            media_io_kwargs=self.media_io_kwargs,
1269
            skip_mm_profiling=self.skip_mm_profiling,
1270
            config_format=self.config_format,
1271
            mm_processor_kwargs=self.mm_processor_kwargs,
1272
            mm_processor_cache_gb=self.mm_processor_cache_gb,
1273
            mm_processor_cache_type=self.mm_processor_cache_type,
1274
            mm_shm_cache_max_object_size_mb=self.mm_shm_cache_max_object_size_mb,
1275
            mm_encoder_tp_mode=self.mm_encoder_tp_mode,
1276
            mm_encoder_attn_backend=self.mm_encoder_attn_backend,
1277
            pooler_config=self.pooler_config,
1278
            logits_processor_pattern=self.logits_processor_pattern,
1279
            generation_config=self.generation_config,
1280
            override_generation_config=self.override_generation_config,
1281
            enable_sleep_mode=self.enable_sleep_mode,
1282
            model_impl=self.model_impl,
1283
            override_attention_dtype=self.override_attention_dtype,
1284
            logits_processors=self.logits_processors,
1285
            video_pruning_rate=self.video_pruning_rate,
1286
            io_processor_plugin=self.io_processor_plugin,
1287
            enable_chunked_prefill=self.enable_chunked_prefill,
1288
        )
1289

1290
    def validate_tensorizer_args(self):
1291
1292
        from vllm.model_executor.model_loader.tensorizer import TensorizerConfig

1293
1294
        for key in self.model_loader_extra_config:
            if key in TensorizerConfig._fields:
1295
1296
1297
                self.model_loader_extra_config["tensorizer_config"][key] = (
                    self.model_loader_extra_config[key]
                )
1298

1299
    def create_load_config(self) -> LoadConfig:
1300
1301
        if self.quantization == "bitsandbytes":
            self.load_format = "bitsandbytes"
1302

1303
1304
1305
        if self.load_format == "tensorizer":
            if hasattr(self.model_loader_extra_config, "to_serializable"):
                self.model_loader_extra_config = (
1306
1307
                    self.model_loader_extra_config.to_serializable()
                )
1308
            self.model_loader_extra_config["tensorizer_config"] = {}
1309
1310
1311
            self.model_loader_extra_config["tensorizer_config"]["tensorizer_dir"] = (
                self.model
            )
1312
            self.validate_tensorizer_args()
1313

1314
1315
1316
        return LoadConfig(
            load_format=self.load_format,
            download_dir=self.download_dir,
1317
            safetensors_load_strategy=self.safetensors_load_strategy,
1318
            device="cpu" if is_online_quantization(self.quantization) else None,
1319
1320
            model_loader_extra_config=self.model_loader_extra_config,
            ignore_patterns=self.ignore_patterns,
1321
            use_tqdm_on_load=self.use_tqdm_on_load,
1322
            pt_load_map_location=self.pt_load_map_location,
1323
1324
        )

1325
1326
1327
1328
    def create_speculative_config(
        self,
        target_model_config: ModelConfig,
        target_parallel_config: ParallelConfig,
1329
    ) -> SpeculativeConfig | None:
1330
1331
1332
1333
1334
1335
        """Initializes and returns a SpeculativeConfig object based on
        `speculative_config`.

        This function utilizes `speculative_config` to create a
        SpeculativeConfig object. The `speculative_config` can either be
        provided as a JSON string input via CLI arguments or directly as a
1336
        dictionary from the engine.
1337
1338
        """
        if self.speculative_config is None:
1339
            return None
1340

1341
1342
1343
        # Note(Shangming): These parameters are not obtained from the cli arg
        # '--speculative-config' and must be passed in when creating the engine
        # config.
1344
1345
1346
1347
1348
1349
        self.speculative_config.update(
            {
                "target_model_config": target_model_config,
                "target_parallel_config": target_parallel_config,
            }
        )
1350
        return SpeculativeConfig(**self.speculative_config)
1351

1352
1353
    def create_engine_config(
        self,
1354
        usage_context: UsageContext | None = None,
1355
        headless: bool = False,
1356
1357
1358
    ) -> VllmConfig:
        """
        Create the VllmConfig.
1359

1360
        NOTE: If VllmConfig is incompatible, we raise an error.
1361
        """
1362
        current_platform.pre_register_and_update()
1363

1364
        device_config = DeviceConfig(device=cast(Device, current_platform.device_type))
1365

1366
1367
        # Check if the model is a speculator and override model/tokenizer/config
        # BEFORE creating ModelConfig, so the config is created with the target model
1368
1369
1370
1371
        # Skip speculator detection for cloud storage models (eg: S3, GCS) since
        # HuggingFace cannot load configs directly from S3 URLs. S3 models can still
        # use speculators with explicit --speculative-config.
        if not is_cloud_storage(self.model):
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
            (self.model, self.tokenizer, self.speculative_config) = (
                maybe_override_with_speculators(
                    model=self.model,
                    tokenizer=self.tokenizer,
                    revision=self.revision,
                    trust_remote_code=self.trust_remote_code,
                    vllm_speculative_config=self.speculative_config,
                )
            )

1382
        model_config = self.create_model_config()
1383
        self.model = model_config.model
1384
        self.model_weights = model_config.model_weights
1385
        self.tokenizer = model_config.tokenizer
1386

1387
        self._check_feature_supported(model_config)
1388
1389
1390
1391
        self._set_default_chunked_prefill_and_prefix_caching_args(model_config)
        self._set_default_max_num_seqs_and_batched_tokens_args(
            usage_context, model_config
        )
1392

1393
        sliding_window: int | None = None
1394
1395
1396
1397
1398
1399
        if not is_interleaved(model_config.hf_text_config):
            # Only set CacheConfig.sliding_window if the model is all sliding
            # window. Otherwise CacheConfig.sliding_window will override the
            # global layers in interleaved sliding window models.
            sliding_window = model_config.get_sliding_window()

1400
1401
1402
        # Note(hc): In the current implementation of decode context
        # parallel(DCP), tp_size needs to be divisible by dcp_size,
        # because the world size does not change by dcp, it simply
1403
        # reuses the GPUs of TP group, and split one TP group into
1404
        # tp_size//dcp_size DCP groups.
1405
        assert self.tensor_parallel_size % self.decode_context_parallel_size == 0, (
1406
1407
1408
1409
            f"tp_size={self.tensor_parallel_size} must be divisible by"
            f"dcp_size={self.decode_context_parallel_size}."
        )

1410
1411
1412
1413
1414
        # Resolve "auto" kv_cache_dtype to actual value from model config
        resolved_cache_dtype = resolve_kv_cache_dtype_string(
            self.kv_cache_dtype, model_config
        )

1415
        cache_config = CacheConfig(
1416
            block_size=self.block_size,
1417
            gpu_memory_utilization=self.gpu_memory_utilization,
1418
            kv_cache_memory_bytes=self.kv_cache_memory_bytes,
1419
            swap_space=self.swap_space,
1420
            cache_dtype=resolved_cache_dtype,
1421
            is_attention_free=model_config.is_attention_free,
1422
            num_gpu_blocks_override=self.num_gpu_blocks_override,
1423
            sliding_window=sliding_window,
1424
            enable_prefix_caching=self.enable_prefix_caching,
1425
            prefix_caching_hash_algo=self.prefix_caching_hash_algo,
1426
            cpu_offload_gb=self.cpu_offload_gb,
1427
            calculate_kv_scales=self.calculate_kv_scales,
1428
            kv_sharing_fast_prefill=self.kv_sharing_fast_prefill,
1429
1430
            mamba_cache_dtype=self.mamba_cache_dtype,
            mamba_ssm_cache_dtype=self.mamba_ssm_cache_dtype,
1431
            mamba_block_size=self.mamba_block_size,
1432
1433
            kv_offloading_size=self.kv_offloading_size,
            kv_offloading_backend=self.kv_offloading_backend,
1434
        )
1435

1436
1437
1438
1439
1440
1441
        ray_runtime_env = None
        if is_ray_initialized():
            # Ray Serve LLM calls `create_engine_config` in the context
            # of a Ray task, therefore we check is_ray_initialized()
            # as opposed to is_in_ray_actor().
            import ray
1442

1443
            ray_runtime_env = ray.get_runtime_context().runtime_env
1444
1445
1446
1447
1448
1449
1450
            # Avoid logging sensitive environment variables
            sanitized_env = ray_runtime_env.to_dict() if ray_runtime_env else {}
            if "env_vars" in sanitized_env:
                sanitized_env["env_vars"] = {
                    k: "***" for k in sanitized_env["env_vars"]
                }
            logger.info("Using ray runtime env (env vars redacted): %s", sanitized_env)
1451

1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
        # Get the current placement group if Ray is initialized and
        # we are in a Ray actor. If so, then the placement group will be
        # passed to spawned processes.
        placement_group = None
        if is_in_ray_actor():
            import ray

            # This call initializes Ray automatically if it is not initialized,
            # but we should not do this here.
            placement_group = ray.util.get_current_placement_group()

1463
        assert not headless or not self.data_parallel_hybrid_lb, (
1464
1465
            "data_parallel_hybrid_lb is not applicable in headless mode"
        )
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
        assert not (self.data_parallel_hybrid_lb and self.data_parallel_external_lb), (
            "data_parallel_hybrid_lb and data_parallel_external_lb cannot both be True."
        )
        assert self.data_parallel_backend == "mp" or self.nnodes == 1, (
            "nnodes > 1 is only supported with data_parallel_backend=mp"
        )
        inferred_data_parallel_rank = 0
        if self.nnodes > 1:
            world_size = (
                self.data_parallel_size
                * self.pipeline_parallel_size
                * self.tensor_parallel_size
            )
            world_size_within_dp = (
                self.pipeline_parallel_size * self.tensor_parallel_size
            )
            local_world_size = world_size // self.nnodes
            assert world_size % self.nnodes == 0, (
                f"world_size={world_size} must be divisible by nnodes={self.nnodes}."
            )
            assert self.node_rank < self.nnodes, (
                f"node_rank={self.node_rank} must be less than nnodes={self.nnodes}."
            )
            inferred_data_parallel_rank = (
                self.node_rank * local_world_size
            ) // world_size_within_dp
            if self.data_parallel_size > 1 and self.data_parallel_external_lb:
                self.data_parallel_rank = inferred_data_parallel_rank
                logger.info(
                    "Inferred data_parallel_rank %d from node_rank %d for external lb",
                    self.data_parallel_rank,
                    self.node_rank,
                )
            elif self.data_parallel_size_local is None:
                # Infer data parallel size local for internal dplb:
                self.data_parallel_size_local = max(
                    local_world_size // world_size_within_dp, 1
                )
        data_parallel_external_lb = (
            self.data_parallel_external_lb or self.data_parallel_rank is not None
        )
1507
        # Local DP rank = 1, use pure-external LB.
1508
        if data_parallel_external_lb:
1509
            assert self.data_parallel_rank is not None, (
1510
                "data_parallel_rank or node_rank must be specified if "
1511
1512
                "data_parallel_external_lb is enable."
            )
1513
            assert self.data_parallel_size_local in (1, None), (
1514
1515
                "data_parallel_size_local must be 1 or None when data_parallel_rank "
                "is set"
1516
            )
1517
            data_parallel_size_local = 1
1518
1519
            # Use full external lb if we have local_size of 1.
            self.data_parallel_hybrid_lb = False
1520
1521
        elif self.data_parallel_size_local is not None:
            data_parallel_size_local = self.data_parallel_size_local
1522
1523
1524
1525
1526
1527
1528

            if self.data_parallel_start_rank and not headless:
                # Infer hybrid LB mode.
                self.data_parallel_hybrid_lb = True

            if self.data_parallel_hybrid_lb and data_parallel_size_local == 1:
                # Use full external lb if we have local_size of 1.
1529
1530
1531
1532
1533
                logger.warning(
                    "data_parallel_hybrid_lb is not eligible when "
                    "data_parallel_size_local = 1, autoswitch to "
                    "data_parallel_external_lb."
                )
1534
1535
1536
1537
1538
1539
1540
                data_parallel_external_lb = True
                self.data_parallel_hybrid_lb = False

            if data_parallel_size_local == self.data_parallel_size:
                # Disable hybrid LB mode if set for a single node
                self.data_parallel_hybrid_lb = False

1541
1542
1543
1544
1545
1546
1547
1548
1549
            self.data_parallel_rank = (
                self.data_parallel_start_rank or inferred_data_parallel_rank
            )
            if self.nnodes > 1:
                logger.info(
                    "Inferred data_parallel_rank %d from node_rank %d",
                    self.data_parallel_rank,
                    self.node_rank,
                )
1550
        else:
1551
            assert not self.data_parallel_hybrid_lb, (
1552
1553
                "data_parallel_size_local must be set to use data_parallel_hybrid_lb."
            )
1554

1555
1556
1557
1558
1559
1560
1561
1562
1563
            if self.data_parallel_backend == "ray" and (
                envs.VLLM_RAY_DP_PACK_STRATEGY == "span"
            ):
                # Data parallel size defaults to 1 if DP ranks are spanning
                # multiple nodes
                data_parallel_size_local = 1
            else:
                # Otherwise local DP size defaults to global DP size if not set
                data_parallel_size_local = self.data_parallel_size
1564
1565
1566

        # DP address, used in multi-node case for torch distributed group
        # and ZMQ sockets.
Rui Qiao's avatar
Rui Qiao committed
1567
1568
1569
1570
        if self.data_parallel_address is None:
            if self.data_parallel_backend == "ray":
                host_ip = get_ip()
                logger.info(
1571
1572
                    "Using host IP %s as ray-based data parallel address", host_ip
                )
Rui Qiao's avatar
Rui Qiao committed
1573
1574
1575
1576
                data_parallel_address = host_ip
            else:
                assert self.data_parallel_backend == "mp", (
                    "data_parallel_backend can only be ray or mp, got %s",
1577
1578
                    self.data_parallel_backend,
                )
1579
1580
1581
                data_parallel_address = (
                    self.master_addr or ParallelConfig.data_parallel_master_ip
                )
Rui Qiao's avatar
Rui Qiao committed
1582
1583
        else:
            data_parallel_address = self.data_parallel_address
1584
1585
1586

        # This port is only used when there are remote data parallel engines,
        # otherwise the local IPC transport is used.
1587
        data_parallel_rpc_port = (
1588
            self.data_parallel_rpc_port
1589
1590
1591
            if (self.data_parallel_rpc_port is not None)
            else ParallelConfig.data_parallel_rpc_port
        )
1592

1593
1594
1595
1596
        if self.tokens_only and not model_config.skip_tokenizer_init:
            model_config.skip_tokenizer_init = True
            logger.info("Skipping tokenizer initialization for tokens-only mode.")

1597
        parallel_config = ParallelConfig(
1598
1599
            pipeline_parallel_size=self.pipeline_parallel_size,
            tensor_parallel_size=self.tensor_parallel_size,
1600
            prefill_context_parallel_size=self.prefill_context_parallel_size,
1601
            data_parallel_size=self.data_parallel_size,
1602
1603
            data_parallel_rank=self.data_parallel_rank or 0,
            data_parallel_external_lb=data_parallel_external_lb,
1604
            data_parallel_size_local=data_parallel_size_local,
1605
1606
1607
1608
            master_addr=self.master_addr,
            master_port=self.master_port,
            nnodes=self.nnodes,
            node_rank=self.node_rank,
1609
1610
            data_parallel_master_ip=data_parallel_address,
            data_parallel_rpc_port=data_parallel_rpc_port,
1611
            data_parallel_backend=self.data_parallel_backend,
1612
            data_parallel_hybrid_lb=self.data_parallel_hybrid_lb,
1613
            is_moe_model=model_config.is_moe,
1614
            enable_expert_parallel=self.enable_expert_parallel,
1615
            all2all_backend=self.all2all_backend,
1616
            enable_dbo=self.enable_dbo,
1617
            ubatch_size=self.ubatch_size,
1618
            dbo_decode_token_threshold=self.dbo_decode_token_threshold,
1619
            dbo_prefill_token_threshold=self.dbo_prefill_token_threshold,
1620
            disable_nccl_for_dp_synchronization=self.disable_nccl_for_dp_synchronization,
1621
            enable_eplb=self.enable_eplb,
1622
            eplb_config=self.eplb_config,
1623
            expert_placement_strategy=self.expert_placement_strategy,
1624
1625
1626
            max_parallel_loading_workers=self.max_parallel_loading_workers,
            disable_custom_all_reduce=self.disable_custom_all_reduce,
            ray_workers_use_nsight=self.ray_workers_use_nsight,
1627
            ray_runtime_env=ray_runtime_env,
1628
            placement_group=placement_group,
1629
1630
            distributed_executor_backend=self.distributed_executor_backend,
            worker_cls=self.worker_cls,
1631
            worker_extension_cls=self.worker_extension_cls,
1632
            decode_context_parallel_size=self.decode_context_parallel_size,
1633
            dcp_kv_cache_interleave_size=self.dcp_kv_cache_interleave_size,
1634
            cp_kv_cache_interleave_size=self.cp_kv_cache_interleave_size,
1635
1636
            _api_process_count=self._api_process_count,
            _api_process_rank=self._api_process_rank,
1637
        )
1638

1639
        speculative_config = self.create_speculative_config(
1640
1641
1642
1643
            target_model_config=model_config,
            target_parallel_config=parallel_config,
        )

1644
1645
        # check_incompatible_config(envs.USE_FUSED_CUSTOM_ALL_REDUCE_RMS_QUANT, envs.USE_FUSED_RMS_QUANT)
        
1646
        scheduler_config = SchedulerConfig(
1647
            runner_type=model_config.runner_type,
1648
1649
1650
            max_num_batched_tokens=self.max_num_batched_tokens,
            max_num_seqs=self.max_num_seqs,
            max_model_len=model_config.max_model_len,
1651
            enable_chunked_prefill=self.enable_chunked_prefill,
1652
            disable_chunked_mm_input=self.disable_chunked_mm_input,
1653
            is_multimodal_model=model_config.is_multimodal_model,
1654
            is_encoder_decoder=model_config.is_encoder_decoder,
1655
            policy=self.scheduling_policy,
1656
            scheduler_cls=self.scheduler_cls,
1657
1658
1659
            max_num_partial_prefills=self.max_num_partial_prefills,
            max_long_partial_prefills=self.max_long_partial_prefills,
            long_prefill_token_threshold=self.long_prefill_token_threshold,
1660
            disable_hybrid_kv_cache_manager=self.disable_hybrid_kv_cache_manager,
1661
            async_scheduling=self.async_scheduling,
1662
            stream_interval=self.stream_interval,
1663
        )
1664

1665
1666
1667
        if not model_config.is_multimodal_model and self.default_mm_loras:
            raise ValueError(
                "Default modality-specific LoRA(s) were provided for a "
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
                "non multimodal model"
            )

        lora_config = (
            LoRAConfig(
                max_lora_rank=self.max_lora_rank,
                max_loras=self.max_loras,
                default_mm_loras=self.default_mm_loras,
                fully_sharded_loras=self.fully_sharded_loras,
                lora_dtype=self.lora_dtype,
1678
                enable_tower_connector_lora=self.enable_tower_connector_lora,
1679
1680
1681
1682
1683
1684
1685
                max_cpu_loras=self.max_cpu_loras
                if self.max_cpu_loras and self.max_cpu_loras > 0
                else None,
            )
            if self.enable_lora
            else None
        )
1686

1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
        if (
            lora_config is not None
            and speculative_config is not None
            and scheduler_config.max_num_batched_tokens
            < (
                scheduler_config.max_num_seqs
                * (speculative_config.num_speculative_tokens + 1)
            )
        ):
            raise ValueError(
                "Consider increasing max_num_batched_tokens or "
                "decreasing num_speculative_tokens"
            )
1700

1701
1702
1703
1704
        # bitsandbytes pre-quantized model need a specific model loader
        if model_config.quantization == "bitsandbytes":
            self.quantization = self.load_format = "bitsandbytes"

1705
1706
1707
1708
1709
1710
1711
1712
        # Attention config overrides
        attention_config = copy.deepcopy(self.attention_config)
        if self.attention_backend is not None:
            if attention_config.backend is not None:
                raise ValueError(
                    "attention_backend and attention_config.backend "
                    "are mutually exclusive"
                )
1713
1714
1715
1716
1717
1718
1719
            # Convert string to enum if needed (CLI parsing returns a string)
            if isinstance(self.attention_backend, str):
                attention_config.backend = AttentionBackendEnum[
                    self.attention_backend.upper()
                ]
            else:
                attention_config.backend = self.attention_backend
1720

1721
        load_config = self.create_load_config()
1722

1723
1724
        # Pass reasoning_parser into StructuredOutputsConfig
        if self.reasoning_parser:
1725
            self.structured_outputs_config.reasoning_parser = self.reasoning_parser
1726

1727
1728
1729
1730
        if self.reasoning_parser_plugin:
            self.structured_outputs_config.reasoning_parser_plugin = (
                self.reasoning_parser_plugin
            )
1731

1732
        observability_config = ObservabilityConfig(
1733
            show_hidden_metrics_for_version=self.show_hidden_metrics_for_version,
1734
            otlp_traces_endpoint=self.otlp_traces_endpoint,
1735
            collect_detailed_traces=self.collect_detailed_traces,
1736
1737
            kv_cache_metrics=self.kv_cache_metrics,
            kv_cache_metrics_sample=self.kv_cache_metrics_sample,
1738
            cudagraph_metrics=self.cudagraph_metrics,
1739
            enable_layerwise_nvtx_tracing=self.enable_layerwise_nvtx_tracing,
1740
            enable_mfu_metrics=self.enable_mfu_metrics,
1741
            enable_mm_processor_stats=self.enable_mm_processor_stats,
1742
            enable_logging_iteration_details=self.enable_logging_iteration_details,
1743
        )
1744

1745
        # Compilation config overrides
1746
        compilation_config = copy.deepcopy(self.compilation_config)
1747
        if self.cudagraph_capture_sizes is not None:
1748
            if compilation_config.cudagraph_capture_sizes is not None:
1749
1750
1751
1752
                raise ValueError(
                    "cudagraph_capture_sizes and compilation_config."
                    "cudagraph_capture_sizes are mutually exclusive"
                )
1753
            compilation_config.cudagraph_capture_sizes = self.cudagraph_capture_sizes
1754
        if self.max_cudagraph_capture_size is not None:
1755
            if compilation_config.max_cudagraph_capture_size is not None:
1756
1757
1758
1759
                raise ValueError(
                    "max_cudagraph_capture_size and compilation_config."
                    "max_cudagraph_capture_size are mutually exclusive"
                )
1760
            compilation_config.max_cudagraph_capture_size = (
1761
1762
                self.max_cudagraph_capture_size
            )
1763
        config = VllmConfig(
1764
1765
1766
1767
1768
            model_config=model_config,
            cache_config=cache_config,
            parallel_config=parallel_config,
            scheduler_config=scheduler_config,
            device_config=device_config,
1769
1770
            load_config=load_config,
            attention_config=attention_config,
1771
1772
            lora_config=lora_config,
            speculative_config=speculative_config,
1773
            structured_outputs_config=self.structured_outputs_config,
1774
            observability_config=observability_config,
1775
            compilation_config=compilation_config,
1776
            kv_transfer_config=self.kv_transfer_config,
1777
            kv_events_config=self.kv_events_config,
1778
            ec_transfer_config=self.ec_transfer_config,
1779
            profiler_config=self.profiler_config,
1780
            additional_config=self.additional_config,
1781
            optimization_level=self.optimization_level,
1782
        )
1783

1784
1785
        return config

1786

1787
1788
    def _check_feature_supported(self, model_config: ModelConfig):
        """Raise an error if the feature is not supported."""
1789
        if self.logits_processor_pattern != EngineArgs.logits_processor_pattern:
1790
            _raise_unsupported_error(feature_name="--logits-processor-pattern")
1791
1792

        # No Concurrent Partial Prefills so far.
1793
1794
1795
1796
1797
        if (
            self.max_num_partial_prefills != SchedulerConfig.max_num_partial_prefills
            or self.max_long_partial_prefills
            != SchedulerConfig.max_long_partial_prefills
        ):
1798
            _raise_unsupported_error(feature_name="Concurrent Partial Prefill")
1799

1800
        # N-gram, Medusa, and Eagle are supported for speculative decoding.
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
        if self.speculative_config is not None:
            # speculative_config could still be a dict at this point
            if isinstance(self.speculative_config, dict):
                method = self.speculative_config.get("method", None)
            else:
                method = self.speculative_config.method

            if method == "draft_model":
                raise NotImplementedError(
                    "Draft model speculative decoding is not supported yet. "
                    "Please consider using other speculative decoding methods "
1812
1813
                    "such as ngram, medusa, eagle, or mtp."
                )
1814

1815
        if self.pipeline_parallel_size > 1:
1816
1817
1818
            supports_pp = getattr(
                self.distributed_executor_backend, "supports_pp", False
            )
1819
            if not supports_pp and self.distributed_executor_backend not in (
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
                ParallelConfig.distributed_executor_backend,
                "ray",
                "mp",
                "external_launcher",
            ):
                name = (
                    "Pipeline Parallelism without Ray distributed "
                    "executor or multiprocessing executor or external "
                    "launcher"
                )
1830
                _raise_unsupported_error(feature_name=name)
1831

1832
1833
1834
1835
1836
1837
    @classmethod
    def get_batch_defaults(
        cls,
        world_size: int,
    ) -> tuple[dict[UsageContext | None, int], dict[UsageContext | None, int]]:
        from vllm.usage.usage_lib import UsageContext
1838

1839
1840
        default_max_num_batched_tokens: dict[UsageContext | None, int]
        default_max_num_seqs: dict[UsageContext | None, int]
1841

1842
1843
        # When no user override, set the default values based on the usage
        # context.
1844
        # Use different default values for different hardware.
1845
1846
1847
1848
1849
1850
1851

        # Try to query the device name on the current platform. If it fails,
        # it may be because the platform that imports vLLM is not the same
        # as the platform that vLLM is running on (e.g. the case of scaling
        # vLLM with Ray) and has no GPUs. In this case we use the default
        # values for non-H100/H200 GPUs.
        try:
1852
            device_memory = current_platform.get_device_total_memory()
1853
            device_name = current_platform.get_device_name().lower()
1854
1855
        except Exception:
            # This is only used to set default_max_num_batched_tokens
1856
            device_memory = 0
1857
            device_name = ""
1858

1859
1860
1861
1862
        # NOTE(Kuntai): Setting large `max_num_batched_tokens` for A100 reduces
        # throughput, see PR #17885 for more details.
        # So here we do an extra device name check to prevent such regression.
        if device_memory >= 70 * GiB_bytes and "a100" not in device_name:
1863
            # For GPUs like H100 and MI300x, use larger default values.
1864
1865
1866
1867
            default_max_num_batched_tokens = {
                UsageContext.LLM_CLASS: 16384,
                UsageContext.OPENAI_API_SERVER: 8192,
            }
1868
1869
1870
1871
            default_max_num_seqs = {
                UsageContext.LLM_CLASS: 1024,
                UsageContext.OPENAI_API_SERVER: 1024,
            }
1872
1873
1874
1875
        else:
            # TODO(woosuk): Tune the default values for other hardware.
            default_max_num_batched_tokens = {
                UsageContext.LLM_CLASS: 8192,
1876
                UsageContext.OPENAI_API_SERVER: 10240,
1877
            }
1878
1879
1880
1881
            default_max_num_seqs = {
                UsageContext.LLM_CLASS: 256,
                UsageContext.OPENAI_API_SERVER: 256,
            }
1882

1883
1884
        # tpu specific default values.
        if current_platform.is_tpu():
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
            chip_name = current_platform.get_device_name()

            if chip_name == "V6E":
                default_max_num_batched_tokens = {
                    UsageContext.LLM_CLASS: 2048,
                    UsageContext.OPENAI_API_SERVER: 1024,
                }
            elif chip_name == "V5E":
                default_max_num_batched_tokens = {
                    UsageContext.LLM_CLASS: 1024,
                    UsageContext.OPENAI_API_SERVER: 512,
                }
            elif chip_name == "V5P":
                default_max_num_batched_tokens = {
                    UsageContext.LLM_CLASS: 512,
                    UsageContext.OPENAI_API_SERVER: 256,
1901
1902
                }

1903
1904
1905
        # cpu specific default values.
        if current_platform.is_cpu():
            default_max_num_batched_tokens = {
1906
1907
                UsageContext.LLM_CLASS: 4096 * world_size,
                UsageContext.OPENAI_API_SERVER: 2048 * world_size,
1908
1909
            }
            default_max_num_seqs = {
1910
1911
                UsageContext.LLM_CLASS: 256 * world_size,
                UsageContext.OPENAI_API_SERVER: 128 * world_size,
1912
1913
            }

1914
1915
        return default_max_num_batched_tokens, default_max_num_seqs

1916
1917
    def _set_default_chunked_prefill_and_prefix_caching_args(
        self, model_config: ModelConfig
1918
    ) -> None:
1919
1920
        default_chunked_prefill = model_config.is_chunked_prefill_supported
        default_prefix_caching = model_config.is_prefix_caching_supported
1921
1922
1923
1924
1925
1926
1927
1928

        if self.enable_chunked_prefill is None:
            self.enable_chunked_prefill = default_chunked_prefill

            logger.debug(
                "%s chunked prefill by default",
                "Enabling" if default_chunked_prefill else "Disabling",
            )
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
        elif (
            model_config.runner_type == "generate"
            and not self.enable_chunked_prefill
            and default_chunked_prefill
        ):
            logger.warning_once(
                "This model does not officially support disabling chunked prefill. "
                "Disabling this manually may cause the engine to crash "
                "or produce incorrect outputs.",
                scope="local",
            )
1940
1941
1942
1943
        elif (
            model_config.runner_type == "pooling"
            and self.enable_chunked_prefill
            and not default_chunked_prefill
1944
        ):
1945
            logger.warning_once(
1946
1947
1948
                "This model does not officially support chunked prefill. "
                "Enabling this manually may cause the engine to crash "
                "or produce incorrect outputs.",
1949
                scope="local",
1950
1951
1952
1953
1954
            )

        if self.enable_prefix_caching is None:
            self.enable_prefix_caching = default_prefix_caching

1955
            logger.debug(
1956
1957
1958
1959
1960
1961
1962
1963
                "%s prefix caching by default",
                "Enabling" if default_prefix_caching else "Disabling",
            )
        elif (
            model_config.runner_type == "pooling"
            and self.enable_prefix_caching
            and not default_prefix_caching
        ):
1964
            logger.warning_once(
1965
1966
1967
                "This model does not officially support prefix caching. "
                "Enabling this manually may cause the engine to crash "
                "or produce incorrect outputs.",
1968
                scope="local",
1969
1970
            )

1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
        # Disable chunked prefill and prefix caching for:
        # POWER (ppc64le)/s390x/RISCV CPUs in V1
        if current_platform.is_cpu() and current_platform.get_cpu_architecture() in (
            CpuArchEnum.POWERPC,
            CpuArchEnum.S390X,
            CpuArchEnum.RISCV,
        ):
            logger.info(
                "Chunked prefill is not supported for ARM and POWER, "
                "S390X and RISC-V CPUs; "
                "disabling it for V1 backend."
            )
            self.enable_chunked_prefill = False
            logger.info(
                "Prefix caching is not supported for ARM and POWER, "
                "S390X and RISC-V CPUs; "
                "disabling it for V1 backend."
            )
            self.enable_prefix_caching = False

    def _set_default_max_num_seqs_and_batched_tokens_args(
1992
1993
1994
        self,
        usage_context: UsageContext | None,
        model_config: ModelConfig,
1995
    ):
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
        world_size = self.pipeline_parallel_size * self.tensor_parallel_size
        (
            default_max_num_batched_tokens,
            default_max_num_seqs,
        ) = self.get_batch_defaults(world_size)

        orig_max_num_batched_tokens = self.max_num_batched_tokens
        orig_max_num_seqs = self.max_num_seqs

        if self.max_num_batched_tokens is None:
            self.max_num_batched_tokens = default_max_num_batched_tokens.get(
                usage_context,
                SchedulerConfig.DEFAULT_MAX_NUM_BATCHED_TOKENS,
            )

        if self.max_num_seqs is None:
            self.max_num_seqs = default_max_num_seqs.get(
                usage_context,
                SchedulerConfig.DEFAULT_MAX_NUM_SEQS,
            )

        if orig_max_num_batched_tokens is None:
            if not self.enable_chunked_prefill:
                # If max_model_len is too short, use the default for higher throughput.
                self.max_num_batched_tokens = max(
                    model_config.max_model_len,
                    self.max_num_batched_tokens,
                )

            # When using default settings,
            # Ensure max_num_batched_tokens does not exceed model limit.
            # Some models (e.g., Whisper) have embeddings tied to max length.
            self.max_num_batched_tokens = min(
                self.max_num_seqs * model_config.max_model_len,
2030
2031
                self.max_num_batched_tokens,
            )
2032

2033
            logger.debug(
2034
2035
2036
                "Defaulting max_num_batched_tokens to %d for %s usage context.",
                self.max_num_batched_tokens,
                usage_context.value if usage_context else None,
2037
            )
2038

2039
2040
2041
        if orig_max_num_seqs is None:
            assert self.max_num_batched_tokens is not None  # For type checking
            self.max_num_seqs = min(self.max_num_seqs, self.max_num_batched_tokens)
2042

2043
            logger.debug(
2044
                "Defaulting max_num_seqs to %d for %s usage context.",
2045
                self.max_num_seqs,
2046
                usage_context.value if usage_context else None,
2047
            )
2048

2049

2050
@dataclass
Zhuohan Li's avatar
Zhuohan Li committed
2051
class AsyncEngineArgs(EngineArgs):
Woosuk Kwon's avatar
Woosuk Kwon committed
2052
    """Arguments for asynchronous vLLM engine."""
2053
2054

    enable_log_requests: bool = False
2055
2056

    @staticmethod
2057
2058
2059
    def add_cli_args(
        parser: FlexibleArgumentParser, async_args_only: bool = False
    ) -> FlexibleArgumentParser:
2060
        # Initialize plugin to update the parser, for example, The plugin may
2061
        # add a new kind of quantization method to --quantization argument or
2062
2063
        # a new device to --device argument.
        load_general_plugins()
2064
2065
        if not async_args_only:
            parser = EngineArgs.add_cli_args(parser)
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
        parser.add_argument(
            "--enable-log-requests",
            action=argparse.BooleanOptionalAction,
            default=AsyncEngineArgs.enable_log_requests,
            help="Enable logging requests.",
        )
        parser.add_argument(
            "--disable-log-requests",
            action=argparse.BooleanOptionalAction,
            default=not AsyncEngineArgs.enable_log_requests,
            help="[DEPRECATED] Disable logging requests.",
            deprecated=True,
        )
2079
        current_platform.pre_register_and_update(parser)
2080
        return parser
2081
2082


2083
2084
2085
2086
2087
2088
def _raise_unsupported_error(feature_name: str):
    msg = (
        f"{feature_name} is not supported. We recommend to "
        f"remove {feature_name} from your config."
    )
    raise NotImplementedError(msg)
2089
2090


2091
def human_readable_int(value: str) -> int:
2092
2093
    """Parse human-readable integers like '1k', '2M', etc.
    Including decimal values with decimal multipliers.
2094

2095
2096
2097
2098
2099
2100
    Examples:
    - '1k' -> 1,000
    - '1K' -> 1,024
    - '25.6k' -> 25,600
    """
    value = value.strip()
2101

2102
    match = re.fullmatch(r"(\d+(?:\.\d+)?)([kKmMgGtT])", value)
2103
2104
    if match:
        decimal_multiplier = {
2105
2106
2107
            "k": 10**3,
            "m": 10**6,
            "g": 10**9,
2108
            "t": 10**12,
2109
2110
        }
        binary_multiplier = {
2111
2112
2113
            "K": 2**10,
            "M": 2**20,
            "G": 2**30,
2114
            "T": 2**40,
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
        }

        number, suffix = match.groups()
        if suffix in decimal_multiplier:
            mult = decimal_multiplier[suffix]
            return int(float(number) * mult)
        elif suffix in binary_multiplier:
            mult = binary_multiplier[suffix]
            # Do not allow decimals with binary multipliers
            try:
                return int(number) * mult
            except ValueError as e:
2127
2128
2129
2130
2131
                raise argparse.ArgumentTypeError(
                    "Decimals are not allowed "
                    f"with binary suffixes like {suffix}. Did you mean to use "
                    f"{number}{suffix.lower()} instead?"
                ) from e
2132
2133
2134

    # Regular plain number.
    return int(value)
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153


def human_readable_int_or_auto(value: str) -> int:
    """Parse human-readable integers like '1k', '2M', etc.
    Including decimal values with decimal multipliers.
    Also accepts -1 or 'auto' as a special value for auto-detection.

    Examples:
    - '1k' -> 1,000
    - '1K' -> 1,024
    - '25.6k' -> 25,600
    - '-1' or 'auto' -> -1 (special value for auto-detection)
    """
    value = value.strip()

    if value == "-1" or value.lower() == "auto":
        return -1

    return human_readable_int(value)