arg_utils.py 32.1 KB
Newer Older
1
import argparse
2
import dataclasses
3
import json
4
from dataclasses import dataclass
5
from typing import List, Optional, Tuple, Union
6

7
from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
8
9
                         EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
                         ParallelConfig, SchedulerConfig, SpeculativeConfig,
10
                         TokenizerPoolConfig, VisionLanguageConfig)
11
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
12
from vllm.utils import str_to_int_tuple
13
14


15
16
17
18
19
20
def nullable_str(val: str):
    if not val or val == "None":
        return None
    return val


21
@dataclass
Zhuohan Li's avatar
Zhuohan Li committed
22
class EngineArgs:
Woosuk Kwon's avatar
Woosuk Kwon committed
23
    """Arguments for vLLM engine."""
24
    model: str
25
    served_model_name: Optional[Union[List[str]]] = None
26
    tokenizer: Optional[str] = None
27
    skip_tokenizer_init: bool = False
28
    tokenizer_mode: str = 'auto'
29
    trust_remote_code: bool = False
30
    download_dir: Optional[str] = None
31
    load_format: str = 'auto'
32
    dtype: str = 'auto'
33
    kv_cache_dtype: str = 'auto'
34
    quantization_param_path: Optional[str] = None
35
    seed: int = 0
36
    max_model_len: Optional[int] = None
37
    worker_use_ray: bool = False
38
    distributed_executor_backend: Optional[str] = None
39
40
    pipeline_parallel_size: int = 1
    tensor_parallel_size: int = 1
41
    max_parallel_loading_workers: Optional[int] = None
42
    block_size: int = 16
43
    enable_prefix_caching: bool = False
44
    disable_sliding_window: bool = False
45
    use_v2_block_manager: bool = False
46
    swap_space: int = 4  # GiB
47
    gpu_memory_utilization: float = 0.90
48
    max_num_batched_tokens: Optional[int] = None
49
    max_num_seqs: int = 256
50
    max_logprobs: int = 5  # OpenAI default value
51
    disable_log_stats: bool = False
Jasmond L's avatar
Jasmond L committed
52
    revision: Optional[str] = None
53
    code_revision: Optional[str] = None
54
    rope_scaling: Optional[dict] = None
55
    tokenizer_revision: Optional[str] = None
56
    quantization: Optional[str] = None
57
    enforce_eager: bool = False
58
59
    max_context_len_to_capture: Optional[int] = None
    max_seq_len_to_capture: int = 8192
60
    disable_custom_all_reduce: bool = False
61
62
63
    tokenizer_pool_size: int = 0
    tokenizer_pool_type: str = "ray"
    tokenizer_pool_extra_config: Optional[dict] = None
64
65
66
    enable_lora: bool = False
    max_loras: int = 1
    max_lora_rank: int = 16
67
    fully_sharded_loras: bool = False
68
    lora_extra_vocab_size: int = 256
69
    long_lora_scaling_factors: Optional[Tuple[float]] = None
70
71
    lora_dtype = 'auto'
    max_cpu_loras: Optional[int] = None
72
    device: str = 'auto'
73
    ray_workers_use_nsight: bool = False
74
    num_gpu_blocks_override: Optional[int] = None
75
    num_lookahead_slots: int = 0
76
    model_loader_extra_config: Optional[dict] = None
77

78
79
80
81
82
    # Related to Vision-language models such as llava
    image_input_type: Optional[str] = None
    image_token_id: Optional[int] = None
    image_input_shape: Optional[str] = None
    image_feature_size: Optional[int] = None
83
    scheduler_delay_factor: float = 0.0
84
    enable_chunked_prefill: bool = False
85

86
    guided_decoding_backend: str = 'outlines'
87
88
89
    # Speculative decoding configuration.
    speculative_model: Optional[str] = None
    num_speculative_tokens: Optional[int] = None
90
    speculative_max_model_len: Optional[int] = None
91
    speculative_disable_by_batch_size: Optional[int] = None
92
93
    ngram_prompt_lookup_max: Optional[int] = None
    ngram_prompt_lookup_min: Optional[int] = None
94

95
    def __post_init__(self):
96
97
        if self.tokenizer is None:
            self.tokenizer = self.model
98
99
100

    @staticmethod
    def add_cli_args(
101
            parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
Woosuk Kwon's avatar
Woosuk Kwon committed
102
        """Shared CLI arguments for vLLM engine."""
103

104
        # Model arguments
105
106
107
108
        parser.add_argument(
            '--model',
            type=str,
            default='facebook/opt-125m',
109
            help='Name or path of the huggingface model to use.')
110
111
        parser.add_argument(
            '--tokenizer',
112
            type=nullable_str,
113
            default=EngineArgs.tokenizer,
114
            help='Name or path of the huggingface tokenizer to use.')
115
116
117
118
        parser.add_argument(
            '--skip-tokenizer-init',
            action='store_true',
            help='Skip initialization of tokenizer and detokenizer')
Jasmond L's avatar
Jasmond L committed
119
120
        parser.add_argument(
            '--revision',
121
            type=nullable_str,
Jasmond L's avatar
Jasmond L committed
122
            default=None,
123
            help='The specific model version to use. It can be a branch '
Jasmond L's avatar
Jasmond L committed
124
125
            'name, a tag name, or a commit id. If unspecified, will use '
            'the default version.')
126
127
        parser.add_argument(
            '--code-revision',
128
            type=nullable_str,
129
            default=None,
130
            help='The specific revision to use for the model code on '
131
132
            'Hugging Face Hub. It can be a branch name, a tag name, or a '
            'commit id. If unspecified, will use the default version.')
133
134
        parser.add_argument(
            '--tokenizer-revision',
135
            type=nullable_str,
136
            default=None,
137
            help='The specific tokenizer version to use. It can be a branch '
138
139
            'name, a tag name, or a commit id. If unspecified, will use '
            'the default version.')
140
141
142
143
144
145
146
147
        parser.add_argument(
            '--tokenizer-mode',
            type=str,
            default=EngineArgs.tokenizer_mode,
            choices=['auto', 'slow'],
            help='The tokenizer mode.\n\n* "auto" will use the '
            'fast tokenizer if available.\n* "slow" will '
            'always use the slow tokenizer.')
148
149
        parser.add_argument('--trust-remote-code',
                            action='store_true',
150
                            help='Trust remote code from huggingface.')
151
        parser.add_argument('--download-dir',
152
                            type=nullable_str,
Zhuohan Li's avatar
Zhuohan Li committed
153
                            default=EngineArgs.download_dir,
154
                            help='Directory to download and load the weights, '
155
                            'default to the default cache dir of '
156
                            'huggingface.')
157
158
159
160
        parser.add_argument(
            '--load-format',
            type=str,
            default=EngineArgs.load_format,
161
162
163
            choices=[
                'auto', 'pt', 'safetensors', 'npcache', 'dummy', 'tensorizer'
            ],
164
165
            help='The format of the model weights to load.\n\n'
            '* "auto" will try to load the weights in the safetensors format '
166
            'and fall back to the pytorch bin format if safetensors format '
167
168
169
170
171
172
173
174
            'is not available.\n'
            '* "pt" will load the weights in the pytorch bin format.\n'
            '* "safetensors" will load the weights in the safetensors format.\n'
            '* "npcache" will load the weights in pytorch format and store '
            'a numpy cache to speed up the loading.\n'
            '* "dummy" will initialize the weights with random values, '
            'which is mainly for profiling.\n'
            '* "tensorizer" will load the weights using tensorizer from '
175
176
            'CoreWeave. See the Tensorize vLLM Model script in the Examples'
            'section for more information.\n')
177
178
179
180
        parser.add_argument(
            '--dtype',
            type=str,
            default=EngineArgs.dtype,
Woosuk Kwon's avatar
Woosuk Kwon committed
181
182
183
            choices=[
                'auto', 'half', 'float16', 'bfloat16', 'float', 'float32'
            ],
184
185
186
187
188
189
190
191
            help='Data type for model weights and activations.\n\n'
            '* "auto" will use FP16 precision for FP32 and FP16 models, and '
            'BF16 precision for BF16 models.\n'
            '* "half" for FP16. Recommended for AWQ quantization.\n'
            '* "float16" is the same as "half".\n'
            '* "bfloat16" for a balance between precision and range.\n'
            '* "float" is shorthand for FP32 precision.\n'
            '* "float32" for FP32 precision.')
192
193
194
        parser.add_argument(
            '--kv-cache-dtype',
            type=str,
195
            choices=['auto', 'fp8', 'fp8_e5m2', 'fp8_e4m3'],
196
            default=EngineArgs.kv_cache_dtype,
197
            help='Data type for kv cache storage. If "auto", will use model '
198
199
            'data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. '
            'ROCm (AMD GPU) supports fp8 (=fp8_e4m3)')
200
201
        parser.add_argument(
            '--quantization-param-path',
202
            type=nullable_str,
203
204
205
206
207
208
209
            default=None,
            help='Path to the JSON file containing the KV cache '
            'scaling factors. This should generally be supplied, when '
            'KV cache dtype is FP8. Otherwise, KV cache scaling factors '
            'default to 1.0, which may cause accuracy issues. '
            'FP8_E5M2 (without scaling) is only supported on cuda version'
            'greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead '
210
            'supported for common inference criteria.')
211
212
        parser.add_argument('--max-model-len',
                            type=int,
213
                            default=EngineArgs.max_model_len,
214
215
                            help='Model context length. If unspecified, will '
                            'be automatically derived from the model config.')
216
217
218
219
220
221
        parser.add_argument(
            '--guided-decoding-backend',
            type=str,
            default='outlines',
            choices=['outlines', 'lm-format-enforcer'],
            help='Which engine will be used for guided decoding'
222
223
224
225
226
            ' (JSON schema / regex etc) by default. Currently support '
            'https://github.com/outlines-dev/outlines and '
            'https://github.com/noamgat/lm-format-enforcer.'
            ' Can be overridden per request via guided_decoding_backend'
            ' parameter.')
227
        # Parallel arguments
228
229
230
231
232
233
234
235
236
237
238
        parser.add_argument(
            '--distributed-executor-backend',
            choices=['ray', 'mp'],
            default=EngineArgs.distributed_executor_backend,
            help='Backend to use for distributed serving. When more than 1 GPU '
            'is used, will be automatically set to "ray" if installed '
            'or "mp" (multiprocessing) otherwise.')
        parser.add_argument(
            '--worker-use-ray',
            action='store_true',
            help='Deprecated, use --distributed-executor-backend=ray.')
239
240
241
        parser.add_argument('--pipeline-parallel-size',
                            '-pp',
                            type=int,
Zhuohan Li's avatar
Zhuohan Li committed
242
                            default=EngineArgs.pipeline_parallel_size,
243
                            help='Number of pipeline stages.')
244
245
246
        parser.add_argument('--tensor-parallel-size',
                            '-tp',
                            type=int,
Zhuohan Li's avatar
Zhuohan Li committed
247
                            default=EngineArgs.tensor_parallel_size,
248
                            help='Number of tensor parallel replicas.')
249
250
251
        parser.add_argument(
            '--max-parallel-loading-workers',
            type=int,
252
            default=EngineArgs.max_parallel_loading_workers,
253
            help='Load model sequentially in multiple batches, '
254
            'to avoid RAM OOM when using tensor '
255
            'parallel and large models.')
256
257
258
        parser.add_argument(
            '--ray-workers-use-nsight',
            action='store_true',
259
            help='If specified, use nsight to profile Ray workers.')
260
        # KV cache arguments
261
262
        parser.add_argument('--block-size',
                            type=int,
Zhuohan Li's avatar
Zhuohan Li committed
263
                            default=EngineArgs.block_size,
264
                            choices=[8, 16, 32],
265
266
                            help='Token block size for contiguous chunks of '
                            'tokens.')
267
268
269

        parser.add_argument('--enable-prefix-caching',
                            action='store_true',
270
                            help='Enables automatic prefix caching.')
271
272
273
274
        parser.add_argument('--disable-sliding-window',
                            action='store_true',
                            help='Disables sliding window, '
                            'capping to sliding window size')
275
276
        parser.add_argument('--use-v2-block-manager',
                            action='store_true',
277
                            help='Use BlockSpaceMangerV2.')
278
279
280
281
282
283
284
285
        parser.add_argument(
            '--num-lookahead-slots',
            type=int,
            default=EngineArgs.num_lookahead_slots,
            help='Experimental scheduling config necessary for '
            'speculative decoding. This will be replaced by '
            'speculative config in the future; it is present '
            'to enable correctness tests until then.')
286

287
288
289
        parser.add_argument('--seed',
                            type=int,
                            default=EngineArgs.seed,
290
                            help='Random seed for operations.')
291
292
        parser.add_argument('--swap-space',
                            type=int,
Zhuohan Li's avatar
Zhuohan Li committed
293
                            default=EngineArgs.swap_space,
294
                            help='CPU swap space size (GiB) per GPU.')
295
296
297
298
        parser.add_argument(
            '--gpu-memory-utilization',
            type=float,
            default=EngineArgs.gpu_memory_utilization,
299
300
301
302
            help='The fraction of GPU memory to be used for the model '
            'executor, which can range from 0 to 1. For example, a value of '
            '0.5 would imply 50%% GPU memory utilization. If unspecified, '
            'will use the default value of 0.9.')
303
        parser.add_argument(
304
            '--num-gpu-blocks-override',
305
306
307
308
            type=int,
            default=None,
            help='If specified, ignore GPU profiling result and use this number'
            'of GPU blocks. Used for testing preemption.')
309
310
        parser.add_argument('--max-num-batched-tokens',
                            type=int,
Zhuohan Li's avatar
Zhuohan Li committed
311
                            default=EngineArgs.max_num_batched_tokens,
312
313
                            help='Maximum number of batched tokens per '
                            'iteration.')
314
315
        parser.add_argument('--max-num-seqs',
                            type=int,
Zhuohan Li's avatar
Zhuohan Li committed
316
                            default=EngineArgs.max_num_seqs,
317
                            help='Maximum number of sequences per iteration.')
318
319
320
321
        parser.add_argument(
            '--max-logprobs',
            type=int,
            default=EngineArgs.max_logprobs,
322
323
            help=('Max number of log probs to return logprobs is specified in'
                  ' SamplingParams.'))
324
325
        parser.add_argument('--disable-log-stats',
                            action='store_true',
326
                            help='Disable logging statistics.')
327
328
329
        # Quantization settings.
        parser.add_argument('--quantization',
                            '-q',
330
                            type=nullable_str,
331
                            choices=[*QUANTIZATION_METHODS, None],
332
                            default=EngineArgs.quantization,
333
334
335
336
337
338
                            help='Method used to quantize the weights. If '
                            'None, we first check the `quantization_config` '
                            'attribute in the model config file. If that is '
                            'None, we assume the model weights are not '
                            'quantized and use `dtype` to determine the data '
                            'type of the weights.')
339
340
341
342
343
        parser.add_argument('--rope-scaling',
                            default=None,
                            type=json.loads,
                            help='RoPE scaling configuration in JSON format. '
                            'For example, {"type":"dynamic","factor":2.0}')
344
345
346
347
348
349
350
351
        parser.add_argument('--enforce-eager',
                            action='store_true',
                            help='Always use eager-mode PyTorch. If False, '
                            'will use eager mode and CUDA graph in hybrid '
                            'for maximal performance and flexibility.')
        parser.add_argument('--max-context-len-to-capture',
                            type=int,
                            default=EngineArgs.max_context_len_to_capture,
352
                            help='Maximum context length covered by CUDA '
353
                            'graphs. When a sequence has context length '
354
                            'larger than this, we fall back to eager mode. '
355
                            '(DEPRECATED. Use --max-seq-len-to-capture instead'
356
                            ')')
357
        parser.add_argument('--max-seq-len-to-capture',
358
359
360
361
                            type=int,
                            default=EngineArgs.max_seq_len_to_capture,
                            help='Maximum sequence length covered by CUDA '
                            'graphs. When a sequence has context length '
362
                            'larger than this, we fall back to eager mode.')
363
364
365
        parser.add_argument('--disable-custom-all-reduce',
                            action='store_true',
                            default=EngineArgs.disable_custom_all_reduce,
366
                            help='See ParallelConfig.')
367
368
369
370
371
372
373
374
375
376
377
378
379
        parser.add_argument('--tokenizer-pool-size',
                            type=int,
                            default=EngineArgs.tokenizer_pool_size,
                            help='Size of tokenizer pool to use for '
                            'asynchronous tokenization. If 0, will '
                            'use synchronous tokenization.')
        parser.add_argument('--tokenizer-pool-type',
                            type=str,
                            default=EngineArgs.tokenizer_pool_type,
                            help='Type of tokenizer pool to use for '
                            'asynchronous tokenization. Ignored '
                            'if tokenizer_pool_size is 0.')
        parser.add_argument('--tokenizer-pool-extra-config',
380
                            type=nullable_str,
381
382
383
384
385
                            default=EngineArgs.tokenizer_pool_extra_config,
                            help='Extra config for tokenizer pool. '
                            'This should be a JSON string that will be '
                            'parsed into a dictionary. Ignored if '
                            'tokenizer_pool_size is 0.')
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
        # LoRA related configs
        parser.add_argument('--enable-lora',
                            action='store_true',
                            help='If True, enable handling of LoRA adapters.')
        parser.add_argument('--max-loras',
                            type=int,
                            default=EngineArgs.max_loras,
                            help='Max number of LoRAs in a single batch.')
        parser.add_argument('--max-lora-rank',
                            type=int,
                            default=EngineArgs.max_lora_rank,
                            help='Max LoRA rank.')
        parser.add_argument(
            '--lora-extra-vocab-size',
            type=int,
            default=EngineArgs.lora_extra_vocab_size,
            help=('Maximum size of extra vocabulary that can be '
                  'present in a LoRA adapter (added to the base '
                  'model vocabulary).'))
        parser.add_argument(
            '--lora-dtype',
            type=str,
            default=EngineArgs.lora_dtype,
            choices=['auto', 'float16', 'bfloat16', 'float32'],
            help=('Data type for LoRA. If auto, will default to '
                  'base model dtype.'))
412
413
414
415
416
417
418
419
420
421
422
        parser.add_argument(
            '--long-lora-scaling-factors',
            type=nullable_str,
            default=EngineArgs.long_lora_scaling_factors,
            help=('Specify multiple scaling factors (which can '
                  'be different from base model scaling factor '
                  '- see eg. Long LoRA) to allow for multiple '
                  'LoRA adapters trained with those scaling '
                  'factors to be used at the same time. If not '
                  'specified, only adapters trained with the '
                  'base model scaling factor are allowed.'))
423
424
425
426
427
428
429
        parser.add_argument(
            '--max-cpu-loras',
            type=int,
            default=EngineArgs.max_cpu_loras,
            help=('Maximum number of LoRAs to store in CPU memory. '
                  'Must be >= than max_num_seqs. '
                  'Defaults to max_num_seqs.'))
430
431
432
433
434
435
436
437
        parser.add_argument(
            '--fully-sharded-loras',
            action='store_true',
            help=('By default, only half of the LoRA computation is '
                  'sharded with tensor parallelism. '
                  'Enabling this will use the fully sharded layers. '
                  'At high sequence length, max rank or '
                  'tensor parallel size, this is likely faster.'))
438
439
440
        parser.add_argument("--device",
                            type=str,
                            default=EngineArgs.device,
441
                            choices=["auto", "cuda", "neuron", "cpu"],
442
                            help='Device type for vLLM execution.')
443
444
445
        # Related to Vision-language models such as llava
        parser.add_argument(
            '--image-input-type',
446
            type=nullable_str,
447
448
449
450
451
452
453
454
455
456
457
458
            default=None,
            choices=[
                t.name.lower() for t in VisionLanguageConfig.ImageInputType
            ],
            help=('The image input type passed into vLLM. '
                  'Should be one of "pixel_values" or "image_features".'))
        parser.add_argument('--image-token-id',
                            type=int,
                            default=None,
                            help=('Input id for image token.'))
        parser.add_argument(
            '--image-input-shape',
459
            type=nullable_str,
460
461
462
463
464
465
466
467
            default=None,
            help=('The biggest image input shape (worst for memory footprint) '
                  'given an input type. Only used for vLLM\'s profile_run.'))
        parser.add_argument(
            '--image-feature-size',
            type=int,
            default=None,
            help=('The image feature size along the context dimension.'))
468
469
470
471
472
473
        parser.add_argument(
            '--scheduler-delay-factor',
            type=float,
            default=EngineArgs.scheduler_delay_factor,
            help='Apply a delay (of delay factor multiplied by previous'
            'prompt latency) before scheduling next prompt.')
474
475
        parser.add_argument(
            '--enable-chunked-prefill',
476
477
            action='store_true',
            help='If set, the prefill requests can be chunked based on the '
478
            'max_num_batched_tokens.')
479
480
481

        parser.add_argument(
            '--speculative-model',
482
            type=nullable_str,
483
            default=EngineArgs.speculative_model,
484
485
486
487
488
489
            help=
            'The name of the draft model to be used in speculative decoding.')

        parser.add_argument(
            '--num-speculative-tokens',
            type=int,
490
            default=EngineArgs.num_speculative_tokens,
491
            help='The number of speculative tokens to sample from '
492
            'the draft model in speculative decoding.')
493

494
495
        parser.add_argument(
            '--speculative-max-model-len',
496
            type=int,
497
498
499
500
501
            default=EngineArgs.speculative_max_model_len,
            help='The maximum sequence length supported by the '
            'draft model. Sequences over this length will skip '
            'speculation.')

502
503
504
505
506
507
508
        parser.add_argument(
            '--speculative-disable-by-batch-size',
            type=int,
            default=EngineArgs.speculative_disable_by_batch_size,
            help='Disable speculative decoding for new incoming requests '
            'if the number of enqueue requests is larger than this value.')

509
510
511
512
513
514
515
516
517
518
519
520
521
522
        parser.add_argument(
            '--ngram-prompt-lookup-max',
            type=int,
            default=EngineArgs.ngram_prompt_lookup_max,
            help='Max size of window for ngram prompt lookup in speculative '
            'decoding.')

        parser.add_argument(
            '--ngram-prompt-lookup-min',
            type=int,
            default=EngineArgs.ngram_prompt_lookup_min,
            help='Min size of window for ngram prompt lookup in speculative '
            'decoding.')

523
        parser.add_argument('--model-loader-extra-config',
524
                            type=nullable_str,
525
526
527
528
529
530
531
                            default=EngineArgs.model_loader_extra_config,
                            help='Extra config for model loader. '
                            'This will be passed to the model loader '
                            'corresponding to the chosen load_format. '
                            'This should be a JSON string that will be '
                            'parsed into a dictionary.')

532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
        parser.add_argument(
            "--served-model-name",
            nargs="+",
            type=str,
            default=None,
            help="The model name(s) used in the API. If multiple "
            "names are provided, the server will respond to any "
            "of the provided names. The model name in the model "
            "field of a response will be the first name in this "
            "list. If not specified, the model name will be the "
            "same as the `--model` argument. Noted that this name(s)"
            "will also be used in `model_name` tag content of "
            "prometheus metrics, if multiple names provided, metrics"
            "tag will take the first one.")

547
        return parser
548
549

    @classmethod
550
    def from_cli_args(cls, args: argparse.Namespace):
551
552
553
        # Get the list of attributes of this dataclass.
        attrs = [attr.name for attr in dataclasses.fields(cls)]
        # Set the attributes from the parsed arguments.
Zhuohan Li's avatar
Zhuohan Li committed
554
555
        engine_args = cls(**{attr: getattr(args, attr) for attr in attrs})
        return engine_args
556

557
    def create_engine_config(self, ) -> EngineConfig:
558
        device_config = DeviceConfig(self.device)
559
560
        model_config = ModelConfig(
            self.model, self.tokenizer, self.tokenizer_mode,
561
            self.trust_remote_code, self.dtype, self.seed, self.revision,
562
563
564
565
            self.code_revision, self.rope_scaling, self.tokenizer_revision,
            self.max_model_len, self.quantization,
            self.quantization_param_path, self.enforce_eager,
            self.max_context_len_to_capture, self.max_seq_len_to_capture,
566
567
            self.max_logprobs, self.disable_sliding_window,
            self.skip_tokenizer_init, self.served_model_name)
568
569
        cache_config = CacheConfig(self.block_size,
                                   self.gpu_memory_utilization,
570
                                   self.swap_space, self.kv_cache_dtype,
571
                                   self.num_gpu_blocks_override,
572
573
                                   model_config.get_sliding_window(),
                                   self.enable_prefix_caching)
574
        parallel_config = ParallelConfig(
575
576
577
578
            self.pipeline_parallel_size,
            self.tensor_parallel_size,
            self.worker_use_ray,
            self.max_parallel_loading_workers,
579
580
581
582
583
            self.disable_custom_all_reduce,
            TokenizerPoolConfig.create_config(
                self.tokenizer_pool_size,
                self.tokenizer_pool_type,
                self.tokenizer_pool_extra_config,
584
585
586
            ),
            self.ray_workers_use_nsight,
            distributed_executor_backend=self.distributed_executor_backend)
587
588
589
590
591
592
593

        speculative_config = SpeculativeConfig.maybe_create_spec_config(
            target_model_config=model_config,
            target_parallel_config=parallel_config,
            target_dtype=self.dtype,
            speculative_model=self.speculative_model,
            num_speculative_tokens=self.num_speculative_tokens,
594
595
            speculative_disable_by_batch_size=self.
            speculative_disable_by_batch_size,
596
597
598
            speculative_max_model_len=self.speculative_max_model_len,
            enable_chunked_prefill=self.enable_chunked_prefill,
            use_v2_block_manager=self.use_v2_block_manager,
599
600
            ngram_prompt_lookup_max=self.ngram_prompt_lookup_max,
            ngram_prompt_lookup_min=self.ngram_prompt_lookup_min,
601
602
        )

603
604
605
606
607
        scheduler_config = SchedulerConfig(
            self.max_num_batched_tokens,
            self.max_num_seqs,
            model_config.max_model_len,
            self.use_v2_block_manager,
608
609
610
            num_lookahead_slots=(self.num_lookahead_slots
                                 if speculative_config is None else
                                 speculative_config.num_lookahead_slots),
611
612
            delay_factor=self.scheduler_delay_factor,
            enable_chunked_prefill=self.enable_chunked_prefill,
613
            embedding_mode=model_config.embedding_mode,
614
        )
615
616
617
        lora_config = LoRAConfig(
            max_lora_rank=self.max_lora_rank,
            max_loras=self.max_loras,
618
            fully_sharded_loras=self.fully_sharded_loras,
619
            lora_extra_vocab_size=self.lora_extra_vocab_size,
620
            long_lora_scaling_factors=self.long_lora_scaling_factors,
621
622
623
            lora_dtype=self.lora_dtype,
            max_cpu_loras=self.max_cpu_loras if self.max_cpu_loras
            and self.max_cpu_loras > 0 else None) if self.enable_lora else None
624

625
626
627
628
        load_config = LoadConfig(
            load_format=self.load_format,
            download_dir=self.download_dir,
            model_loader_extra_config=self.model_loader_extra_config,
629
630
        )

631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
        if self.image_input_type:
            if (not self.image_token_id or not self.image_input_shape
                    or not self.image_feature_size):
                raise ValueError(
                    'Specify `image_token_id`, `image_input_shape` and '
                    '`image_feature_size` together with `image_input_type`.')
            vision_language_config = VisionLanguageConfig(
                image_input_type=VisionLanguageConfig.
                get_image_input_enum_type(self.image_input_type),
                image_token_id=self.image_token_id,
                image_input_shape=str_to_int_tuple(self.image_input_shape),
                image_feature_size=self.image_feature_size,
            )
        else:
            vision_language_config = None

647
648
649
        decoding_config = DecodingConfig(
            guided_decoding_backend=self.guided_decoding_backend)

650
        if (model_config.get_sliding_window() is not None
651
652
                and scheduler_config.chunked_prefill_enabled
                and not scheduler_config.use_v2_block_manager):
653
            raise ValueError(
654
655
                "Chunked prefill is not supported with sliding window. "
                "Set --disable-sliding-window to disable sliding window.")
656

657
658
659
660
661
662
663
        return EngineConfig(model_config=model_config,
                            cache_config=cache_config,
                            parallel_config=parallel_config,
                            scheduler_config=scheduler_config,
                            device_config=device_config,
                            lora_config=lora_config,
                            vision_language_config=vision_language_config,
664
                            speculative_config=speculative_config,
665
666
                            load_config=load_config,
                            decoding_config=decoding_config)
667
668


669
@dataclass
Zhuohan Li's avatar
Zhuohan Li committed
670
class AsyncEngineArgs(EngineArgs):
Woosuk Kwon's avatar
Woosuk Kwon committed
671
    """Arguments for asynchronous vLLM engine."""
Zhuohan Li's avatar
Zhuohan Li committed
672
    engine_use_ray: bool = False
673
    disable_log_requests: bool = False
674
    max_log_len: Optional[int] = None
675
676

    @staticmethod
677
678
679
680
    def add_cli_args(parser: argparse.ArgumentParser,
                     async_args_only: bool = False) -> argparse.ArgumentParser:
        if not async_args_only:
            parser = EngineArgs.add_cli_args(parser)
681
682
        parser.add_argument('--engine-use-ray',
                            action='store_true',
683
                            help='Use Ray to start the LLM engine in a '
684
685
686
                            'separate process as the server process.')
        parser.add_argument('--disable-log-requests',
                            action='store_true',
687
                            help='Disable logging requests.')
688
689
690
        parser.add_argument('--max-log-len',
                            type=int,
                            default=None,
691
692
693
                            help='Max number of prompt characters or prompt '
                            'ID numbers being printed in log.'
                            '\n\nDefault: Unlimited')
694
        return parser
695
696
697
698
699
700
701
702
703
704


# These functions are used by sphinx to build the documentation
def _engine_args_parser():
    return EngineArgs.add_cli_args(argparse.ArgumentParser())


def _async_engine_args_parser():
    return AsyncEngineArgs.add_cli_args(argparse.ArgumentParser(),
                                        async_args_only=True)