arg_utils.py 23.4 KB
Newer Older
1
import argparse
2
3
import dataclasses
from dataclasses import dataclass
4
from typing import Optional
5

6
7
8
from vllm.config import (CacheConfig, DeviceConfig, EngineConfig, LoRAConfig,
                         ModelConfig, ParallelConfig, SchedulerConfig,
                         SpeculativeConfig, TokenizerPoolConfig,
9
10
                         VisionLanguageConfig)
from vllm.utils import str_to_int_tuple
11
12


13
@dataclass
Zhuohan Li's avatar
Zhuohan Li committed
14
class EngineArgs:
Woosuk Kwon's avatar
Woosuk Kwon committed
15
    """Arguments for vLLM engine."""
16
    model: str
17
    tokenizer: Optional[str] = None
18
    tokenizer_mode: str = 'auto'
19
    trust_remote_code: bool = False
20
    download_dir: Optional[str] = None
21
    load_format: str = 'auto'
22
    dtype: str = 'auto'
23
    kv_cache_dtype: str = 'auto'
24
    quantization_param_path: Optional[str] = None
25
    seed: int = 0
26
    max_model_len: Optional[int] = None
27
    worker_use_ray: bool = False
28
29
    pipeline_parallel_size: int = 1
    tensor_parallel_size: int = 1
30
    max_parallel_loading_workers: Optional[int] = None
31
    block_size: int = 16
32
    enable_prefix_caching: bool = False
33
    use_v2_block_manager: bool = False
34
    swap_space: int = 4  # GiB
35
    gpu_memory_utilization: float = 0.90
36
    max_num_batched_tokens: Optional[int] = None
37
    max_num_seqs: int = 256
38
    max_logprobs: int = 5  # OpenAI default value
39
    disable_log_stats: bool = False
Jasmond L's avatar
Jasmond L committed
40
    revision: Optional[str] = None
41
    code_revision: Optional[str] = None
42
    tokenizer_revision: Optional[str] = None
43
    quantization: Optional[str] = None
44
45
    enforce_eager: bool = False
    max_context_len_to_capture: int = 8192
46
    disable_custom_all_reduce: bool = False
47
48
49
    tokenizer_pool_size: int = 0
    tokenizer_pool_type: str = "ray"
    tokenizer_pool_extra_config: Optional[dict] = None
50
51
52
53
54
55
    enable_lora: bool = False
    max_loras: int = 1
    max_lora_rank: int = 16
    lora_extra_vocab_size: int = 256
    lora_dtype = 'auto'
    max_cpu_loras: Optional[int] = None
56
    device: str = 'auto'
57
    ray_workers_use_nsight: bool = False
58
    num_gpu_blocks_override: Optional[int] = None
59
    num_lookahead_slots: int = 0
60

61
62
63
64
65
    # Related to Vision-language models such as llava
    image_input_type: Optional[str] = None
    image_token_id: Optional[int] = None
    image_input_shape: Optional[str] = None
    image_feature_size: Optional[int] = None
66

67
    scheduler_delay_factor: float = 0.0
68
    enable_chunked_prefill: bool = False
69

70
71
72
73
    # Speculative decoding configuration.
    speculative_model: Optional[str] = None
    num_speculative_tokens: Optional[int] = None

74
    def __post_init__(self):
75
76
        if self.tokenizer is None:
            self.tokenizer = self.model
77
78
79

    @staticmethod
    def add_cli_args(
80
            parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
Woosuk Kwon's avatar
Woosuk Kwon committed
81
        """Shared CLI arguments for vLLM engine."""
82
83
84
85

        # NOTE: If you update any of the arguments below, please also
        # make sure to update docs/source/models/engine_args.rst

86
        # Model arguments
87
88
89
90
91
92
93
94
95
96
        parser.add_argument(
            '--model',
            type=str,
            default='facebook/opt-125m',
            help='name or path of the huggingface model to use')
        parser.add_argument(
            '--tokenizer',
            type=str,
            default=EngineArgs.tokenizer,
            help='name or path of the huggingface tokenizer to use')
Jasmond L's avatar
Jasmond L committed
97
98
99
100
101
102
103
        parser.add_argument(
            '--revision',
            type=str,
            default=None,
            help='the specific model version to use. It can be a branch '
            'name, a tag name, or a commit id. If unspecified, will use '
            'the default version.')
104
105
106
107
108
109
110
        parser.add_argument(
            '--code-revision',
            type=str,
            default=None,
            help='the specific revision to use for the model code on '
            'Hugging Face Hub. It can be a branch name, a tag name, or a '
            'commit id. If unspecified, will use the default version.')
111
112
113
114
115
116
117
        parser.add_argument(
            '--tokenizer-revision',
            type=str,
            default=None,
            help='the specific tokenizer version to use. It can be a branch '
            'name, a tag name, or a commit id. If unspecified, will use '
            'the default version.')
118
119
        parser.add_argument('--tokenizer-mode',
                            type=str,
120
121
122
                            default=EngineArgs.tokenizer_mode,
                            choices=['auto', 'slow'],
                            help='tokenizer mode. "auto" will use the fast '
123
124
                            'tokenizer if available, and "slow" will '
                            'always use the slow tokenizer.')
125
126
127
        parser.add_argument('--trust-remote-code',
                            action='store_true',
                            help='trust remote code from huggingface')
128
129
        parser.add_argument('--download-dir',
                            type=str,
Zhuohan Li's avatar
Zhuohan Li committed
130
                            default=EngineArgs.download_dir,
131
                            help='directory to download and load the weights, '
132
133
                            'default to the default cache dir of '
                            'huggingface')
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
        parser.add_argument(
            '--load-format',
            type=str,
            default=EngineArgs.load_format,
            choices=['auto', 'pt', 'safetensors', 'npcache', 'dummy'],
            help='The format of the model weights to load. '
            '"auto" will try to load the weights in the safetensors format '
            'and fall back to the pytorch bin format if safetensors format '
            'is not available. '
            '"pt" will load the weights in the pytorch bin format. '
            '"safetensors" will load the weights in the safetensors format. '
            '"npcache" will load the weights in pytorch format and store '
            'a numpy cache to speed up the loading. '
            '"dummy" will initialize the weights with random values, '
            'which is mainly for profiling.')
149
150
151
152
        parser.add_argument(
            '--dtype',
            type=str,
            default=EngineArgs.dtype,
Woosuk Kwon's avatar
Woosuk Kwon committed
153
154
155
            choices=[
                'auto', 'half', 'float16', 'bfloat16', 'float', 'float32'
            ],
156
157
158
159
            help='data type for model weights and activations. '
            'The "auto" option will use FP16 precision '
            'for FP32 and FP16 models, and BF16 precision '
            'for BF16 models.')
160
161
162
        parser.add_argument(
            '--kv-cache-dtype',
            type=str,
163
            choices=['auto', 'fp8'],
164
            default=EngineArgs.kv_cache_dtype,
165
            help='Data type for kv cache storage. If "auto", will use model '
166
167
168
169
170
171
172
173
174
175
176
177
178
179
            'data type. FP8_E5M2 (without scaling) is only supported on cuda '
            'version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead '
            'supported for common inference criteria. ')
        parser.add_argument(
            '--quantization-param-path',
            type=str,
            default=None,
            help='Path to the JSON file containing the KV cache '
            'scaling factors. This should generally be supplied, when '
            'KV cache dtype is FP8. Otherwise, KV cache scaling factors '
            'default to 1.0, which may cause accuracy issues. '
            'FP8_E5M2 (without scaling) is only supported on cuda version'
            'greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead '
            'supported for common inference criteria. ')
180
181
        parser.add_argument('--max-model-len',
                            type=int,
182
                            default=EngineArgs.max_model_len,
183
184
                            help='model context length. If unspecified, '
                            'will be automatically derived from the model.')
185
        # Parallel arguments
186
187
        parser.add_argument('--worker-use-ray',
                            action='store_true',
188
                            help='use Ray for distributed serving, will be '
189
190
191
192
                            'automatically set when using more than 1 GPU')
        parser.add_argument('--pipeline-parallel-size',
                            '-pp',
                            type=int,
Zhuohan Li's avatar
Zhuohan Li committed
193
                            default=EngineArgs.pipeline_parallel_size,
194
                            help='number of pipeline stages')
195
196
197
        parser.add_argument('--tensor-parallel-size',
                            '-tp',
                            type=int,
Zhuohan Li's avatar
Zhuohan Li committed
198
                            default=EngineArgs.tensor_parallel_size,
199
                            help='number of tensor parallel replicas')
200
201
202
        parser.add_argument(
            '--max-parallel-loading-workers',
            type=int,
203
            default=EngineArgs.max_parallel_loading_workers,
204
205
206
            help='load model sequentially in multiple batches, '
            'to avoid RAM OOM when using tensor '
            'parallel and large models')
207
208
209
210
        parser.add_argument(
            '--ray-workers-use-nsight',
            action='store_true',
            help='If specified, use nsight to profile ray workers')
211
        # KV cache arguments
212
213
        parser.add_argument('--block-size',
                            type=int,
Zhuohan Li's avatar
Zhuohan Li committed
214
                            default=EngineArgs.block_size,
215
                            choices=[8, 16, 32, 128],
216
                            help='token block size')
217
218
219
220

        parser.add_argument('--enable-prefix-caching',
                            action='store_true',
                            help='Enables automatic prefix caching')
221
222
223
        parser.add_argument('--use-v2-block-manager',
                            action='store_true',
                            help='Use BlockSpaceMangerV2')
224
225
226
227
228
229
230
231
        parser.add_argument(
            '--num-lookahead-slots',
            type=int,
            default=EngineArgs.num_lookahead_slots,
            help='Experimental scheduling config necessary for '
            'speculative decoding. This will be replaced by '
            'speculative config in the future; it is present '
            'to enable correctness tests until then.')
232

233
234
235
        parser.add_argument('--seed',
                            type=int,
                            default=EngineArgs.seed,
236
                            help='random seed')
237
238
        parser.add_argument('--swap-space',
                            type=int,
Zhuohan Li's avatar
Zhuohan Li committed
239
                            default=EngineArgs.swap_space,
240
                            help='CPU swap space size (GiB) per GPU')
241
242
243
244
245
246
247
        parser.add_argument(
            '--gpu-memory-utilization',
            type=float,
            default=EngineArgs.gpu_memory_utilization,
            help='the fraction of GPU memory to be used for '
            'the model executor, which can range from 0 to 1.'
            'If unspecified, will use the default value of 0.9.')
248
        parser.add_argument(
249
            '--num-gpu-blocks-override',
250
251
252
253
            type=int,
            default=None,
            help='If specified, ignore GPU profiling result and use this number'
            'of GPU blocks. Used for testing preemption.')
254
255
        parser.add_argument('--max-num-batched-tokens',
                            type=int,
Zhuohan Li's avatar
Zhuohan Li committed
256
                            default=EngineArgs.max_num_batched_tokens,
257
                            help='maximum number of batched tokens per '
258
259
260
                            'iteration')
        parser.add_argument('--max-num-seqs',
                            type=int,
Zhuohan Li's avatar
Zhuohan Li committed
261
                            default=EngineArgs.max_num_seqs,
262
                            help='maximum number of sequences per iteration')
263
264
265
266
267
268
        parser.add_argument(
            '--max-logprobs',
            type=int,
            default=EngineArgs.max_logprobs,
            help=('max number of log probs to return logprobs is specified in'
                  ' SamplingParams'))
269
270
        parser.add_argument('--disable-log-stats',
                            action='store_true',
271
                            help='disable logging statistics')
272
273
274
275
        # Quantization settings.
        parser.add_argument('--quantization',
                            '-q',
                            type=str,
CHU Tianxiang's avatar
CHU Tianxiang committed
276
                            choices=['awq', 'gptq', 'squeezellm', None],
277
                            default=EngineArgs.quantization,
278
279
280
281
282
283
                            help='Method used to quantize the weights. If '
                            'None, we first check the `quantization_config` '
                            'attribute in the model config file. If that is '
                            'None, we assume the model weights are not '
                            'quantized and use `dtype` to determine the data '
                            'type of the weights.')
284
285
286
287
288
289
290
291
292
293
294
        parser.add_argument('--enforce-eager',
                            action='store_true',
                            help='Always use eager-mode PyTorch. If False, '
                            'will use eager mode and CUDA graph in hybrid '
                            'for maximal performance and flexibility.')
        parser.add_argument('--max-context-len-to-capture',
                            type=int,
                            default=EngineArgs.max_context_len_to_capture,
                            help='maximum context length covered by CUDA '
                            'graphs. When a sequence has context length '
                            'larger than this, we fall back to eager mode.')
295
296
297
298
        parser.add_argument('--disable-custom-all-reduce',
                            action='store_true',
                            default=EngineArgs.disable_custom_all_reduce,
                            help='See ParallelConfig')
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
        parser.add_argument('--tokenizer-pool-size',
                            type=int,
                            default=EngineArgs.tokenizer_pool_size,
                            help='Size of tokenizer pool to use for '
                            'asynchronous tokenization. If 0, will '
                            'use synchronous tokenization.')
        parser.add_argument('--tokenizer-pool-type',
                            type=str,
                            default=EngineArgs.tokenizer_pool_type,
                            help='Type of tokenizer pool to use for '
                            'asynchronous tokenization. Ignored '
                            'if tokenizer_pool_size is 0.')
        parser.add_argument('--tokenizer-pool-extra-config',
                            type=str,
                            default=EngineArgs.tokenizer_pool_extra_config,
                            help='Extra config for tokenizer pool. '
                            'This should be a JSON string that will be '
                            'parsed into a dictionary. Ignored if '
                            'tokenizer_pool_size is 0.')
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
        # LoRA related configs
        parser.add_argument('--enable-lora',
                            action='store_true',
                            help='If True, enable handling of LoRA adapters.')
        parser.add_argument('--max-loras',
                            type=int,
                            default=EngineArgs.max_loras,
                            help='Max number of LoRAs in a single batch.')
        parser.add_argument('--max-lora-rank',
                            type=int,
                            default=EngineArgs.max_lora_rank,
                            help='Max LoRA rank.')
        parser.add_argument(
            '--lora-extra-vocab-size',
            type=int,
            default=EngineArgs.lora_extra_vocab_size,
            help=('Maximum size of extra vocabulary that can be '
                  'present in a LoRA adapter (added to the base '
                  'model vocabulary).'))
        parser.add_argument(
            '--lora-dtype',
            type=str,
            default=EngineArgs.lora_dtype,
            choices=['auto', 'float16', 'bfloat16', 'float32'],
            help=('Data type for LoRA. If auto, will default to '
                  'base model dtype.'))
        parser.add_argument(
            '--max-cpu-loras',
            type=int,
            default=EngineArgs.max_cpu_loras,
            help=('Maximum number of LoRAs to store in CPU memory. '
                  'Must be >= than max_num_seqs. '
                  'Defaults to max_num_seqs.'))
351
352
353
        parser.add_argument("--device",
                            type=str,
                            default=EngineArgs.device,
354
                            choices=["auto", "cuda", "neuron", "cpu"],
355
                            help='Device type for vLLM execution.')
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
        # Related to Vision-language models such as llava
        parser.add_argument(
            '--image-input-type',
            type=str,
            default=None,
            choices=[
                t.name.lower() for t in VisionLanguageConfig.ImageInputType
            ],
            help=('The image input type passed into vLLM. '
                  'Should be one of "pixel_values" or "image_features".'))
        parser.add_argument('--image-token-id',
                            type=int,
                            default=None,
                            help=('Input id for image token.'))
        parser.add_argument(
            '--image-input-shape',
            type=str,
            default=None,
            help=('The biggest image input shape (worst for memory footprint) '
                  'given an input type. Only used for vLLM\'s profile_run.'))
        parser.add_argument(
            '--image-feature-size',
            type=int,
            default=None,
            help=('The image feature size along the context dimension.'))
381
382
383
384
385
386
        parser.add_argument(
            '--scheduler-delay-factor',
            type=float,
            default=EngineArgs.scheduler_delay_factor,
            help='Apply a delay (of delay factor multiplied by previous'
            'prompt latency) before scheduling next prompt.')
387
388
        parser.add_argument(
            '--enable-chunked-prefill',
389
390
            action='store_true',
            help='If set, the prefill requests can be chunked based on the '
391
            'max_num_batched_tokens')
392
393
394
395
396
397
398
399
400
401
402
403
404
405

        parser.add_argument(
            '--speculative-model',
            type=str,
            default=None,
            help=
            'The name of the draft model to be used in speculative decoding.')

        parser.add_argument(
            '--num-speculative-tokens',
            type=int,
            default=None,
            help='The number of speculative tokens to sample from '
            'the draft model in speculative decoding')
406
        return parser
407
408

    @classmethod
409
    def from_cli_args(cls, args: argparse.Namespace) -> 'EngineArgs':
410
411
412
        # Get the list of attributes of this dataclass.
        attrs = [attr.name for attr in dataclasses.fields(cls)]
        # Set the attributes from the parsed arguments.
Zhuohan Li's avatar
Zhuohan Li committed
413
414
        engine_args = cls(**{attr: getattr(args, attr) for attr in attrs})
        return engine_args
415

416
    def create_engine_config(self, ) -> EngineConfig:
417
        device_config = DeviceConfig(self.device)
418
419
420
421
422
        model_config = ModelConfig(
            self.model, self.tokenizer, self.tokenizer_mode,
            self.trust_remote_code, self.download_dir, self.load_format,
            self.dtype, self.seed, self.revision, self.code_revision,
            self.tokenizer_revision, self.max_model_len, self.quantization,
423
424
            self.quantization_param_path, self.enforce_eager,
            self.max_context_len_to_capture, self.max_logprobs)
425
426
        cache_config = CacheConfig(self.block_size,
                                   self.gpu_memory_utilization,
427
                                   self.swap_space, self.kv_cache_dtype,
428
                                   self.num_gpu_blocks_override,
429
430
                                   model_config.get_sliding_window(),
                                   self.enable_prefix_caching)
431
432
433
434
435
436
437
438
439
        parallel_config = ParallelConfig(
            self.pipeline_parallel_size, self.tensor_parallel_size,
            self.worker_use_ray, self.max_parallel_loading_workers,
            self.disable_custom_all_reduce,
            TokenizerPoolConfig.create_config(
                self.tokenizer_pool_size,
                self.tokenizer_pool_type,
                self.tokenizer_pool_extra_config,
            ), self.ray_workers_use_nsight)
440
441
442
443
444
445
446
447
448

        speculative_config = SpeculativeConfig.maybe_create_spec_config(
            target_model_config=model_config,
            target_parallel_config=parallel_config,
            target_dtype=self.dtype,
            speculative_model=self.speculative_model,
            num_speculative_tokens=self.num_speculative_tokens,
        )

449
450
451
452
453
        scheduler_config = SchedulerConfig(
            self.max_num_batched_tokens,
            self.max_num_seqs,
            model_config.max_model_len,
            self.use_v2_block_manager,
454
455
456
            num_lookahead_slots=(self.num_lookahead_slots
                                 if speculative_config is None else
                                 speculative_config.num_lookahead_slots),
457
458
459
            delay_factor=self.scheduler_delay_factor,
            enable_chunked_prefill=self.enable_chunked_prefill,
        )
460
461
462
463
464
465
466
        lora_config = LoRAConfig(
            max_lora_rank=self.max_lora_rank,
            max_loras=self.max_loras,
            lora_extra_vocab_size=self.lora_extra_vocab_size,
            lora_dtype=self.lora_dtype,
            max_cpu_loras=self.max_cpu_loras if self.max_cpu_loras
            and self.max_cpu_loras > 0 else None) if self.enable_lora else None
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483

        if self.image_input_type:
            if (not self.image_token_id or not self.image_input_shape
                    or not self.image_feature_size):
                raise ValueError(
                    'Specify `image_token_id`, `image_input_shape` and '
                    '`image_feature_size` together with `image_input_type`.')
            vision_language_config = VisionLanguageConfig(
                image_input_type=VisionLanguageConfig.
                get_image_input_enum_type(self.image_input_type),
                image_token_id=self.image_token_id,
                image_input_shape=str_to_int_tuple(self.image_input_shape),
                image_feature_size=self.image_feature_size,
            )
        else:
            vision_language_config = None

484
485
486
487
488
489
490
491
        return EngineConfig(model_config=model_config,
                            cache_config=cache_config,
                            parallel_config=parallel_config,
                            scheduler_config=scheduler_config,
                            device_config=device_config,
                            lora_config=lora_config,
                            vision_language_config=vision_language_config,
                            speculative_config=speculative_config)
492
493


494
@dataclass
Zhuohan Li's avatar
Zhuohan Li committed
495
class AsyncEngineArgs(EngineArgs):
Woosuk Kwon's avatar
Woosuk Kwon committed
496
    """Arguments for asynchronous vLLM engine."""
Zhuohan Li's avatar
Zhuohan Li committed
497
    engine_use_ray: bool = False
498
    disable_log_requests: bool = False
499
    max_log_len: Optional[int] = None
500
501
502

    @staticmethod
    def add_cli_args(
503
            parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
Zhuohan Li's avatar
Zhuohan Li committed
504
        parser = EngineArgs.add_cli_args(parser)
505
506
        parser.add_argument('--engine-use-ray',
                            action='store_true',
Zhuohan Li's avatar
Zhuohan Li committed
507
                            help='use Ray to start the LLM engine in a '
508
509
510
                            'separate process as the server process.')
        parser.add_argument('--disable-log-requests',
                            action='store_true',
511
                            help='disable logging requests')
512
513
514
515
516
517
        parser.add_argument('--max-log-len',
                            type=int,
                            default=None,
                            help='max number of prompt characters or prompt '
                            'ID numbers being printed in log. '
                            'Default: unlimited.')
518
        return parser