server_args.py 56.3 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
Lianmin Zheng's avatar
Lianmin Zheng committed
14
15
"""The arguments of the server."""

Lianmin Zheng's avatar
Lianmin Zheng committed
16
17
import argparse
import dataclasses
18
import json
19
import logging
20
import os
21
import random
22
import tempfile
23
from typing import List, Literal, Optional
Lianmin Zheng's avatar
Lianmin Zheng committed
24

25
from sglang.srt.hf_transformers_utils import check_gguf_file, get_config
Xihuai Wang's avatar
Xihuai Wang committed
26
from sglang.srt.reasoning_parser import ReasoningParser
27
from sglang.srt.utils import (
Vincent's avatar
Vincent committed
28
    configure_ipv6,
29
    get_device,
Lianmin Zheng's avatar
Lianmin Zheng committed
30
    get_device_memory_capacity,
31
    is_flashinfer_available,
HAI's avatar
HAI committed
32
    is_hip,
33
    is_port_available,
34
    is_remote_url,
35
    is_valid_ipv6_address,
bjmsong's avatar
bjmsong committed
36
    nullable_str,
37
)
38

39
40
logger = logging.getLogger(__name__)

Lianmin Zheng's avatar
Lianmin Zheng committed
41
42
43

@dataclasses.dataclass
class ServerArgs:
Lianmin Zheng's avatar
Lianmin Zheng committed
44
    # Model and tokenizer
Lianmin Zheng's avatar
Lianmin Zheng committed
45
46
47
    model_path: str
    tokenizer_path: Optional[str] = None
    tokenizer_mode: str = "auto"
48
    skip_tokenizer_init: bool = False
49
    enable_tokenizer_batch_encode: bool = False
Lianmin Zheng's avatar
Lianmin Zheng committed
50
    load_format: str = "auto"
51
    trust_remote_code: bool = False
Lianmin Zheng's avatar
Lianmin Zheng committed
52
    dtype: str = "auto"
53
    kv_cache_dtype: str = "auto"
Lianmin Zheng's avatar
Lianmin Zheng committed
54
    quantization: Optional[str] = None
Vincent's avatar
Vincent committed
55
    quantization_param_path: Optional[str] = None
56
    context_length: Optional[int] = None
57
    device: Optional[str] = None
58
    served_model_name: Optional[str] = None
Lianmin Zheng's avatar
Lianmin Zheng committed
59
    chat_template: Optional[str] = None
60
    completion_template: Optional[str] = None
61
    is_embedding: bool = False
62
    revision: Optional[str] = None
Lianmin Zheng's avatar
Lianmin Zheng committed
63

64
    # Port for the HTTP server
Lianmin Zheng's avatar
Lianmin Zheng committed
65
66
67
68
    host: str = "127.0.0.1"
    port: int = 30000

    # Memory and scheduling
Lianmin Zheng's avatar
Lianmin Zheng committed
69
    mem_fraction_static: Optional[float] = None
70
    max_running_requests: Optional[int] = None
71
    max_total_tokens: Optional[int] = None
72
    chunked_prefill_size: Optional[int] = None
73
    max_prefill_tokens: int = 16384
74
    schedule_policy: str = "fcfs"
75
    schedule_conservativeness: float = 1.0
76
    cpu_offload_gb: int = 0
77
    page_size: int = 1
Lianmin Zheng's avatar
Lianmin Zheng committed
78
79
80

    # Other runtime options
    tp_size: int = 1
81
82
    pp_size: int = 1
    max_micro_batch_size: Optional[int] = None
83
    stream_interval: int = 1
84
    stream_output: bool = False
85
    random_seed: Optional[int] = None
86
    constrained_json_whitespace_pattern: Optional[str] = None
87
    watchdog_timeout: float = 300
88
    dist_timeout: Optional[int] = None  # timeout for torch.distributed
89
    download_dir: Optional[str] = None
90
    base_gpu_id: int = 0
91
    gpu_id_step: int = 1
Lianmin Zheng's avatar
Lianmin Zheng committed
92
93
94

    # Logging
    log_level: str = "info"
95
    log_level_http: Optional[str] = None
96
    log_requests: bool = False
97
    log_requests_level: int = 0
Liangsheng Yin's avatar
Liangsheng Yin committed
98
    show_time_cost: bool = False
99
    enable_metrics: bool = False
100
    decode_log_interval: int = 40
Liangsheng Yin's avatar
Liangsheng Yin committed
101

102
    # API related
103
    api_key: Optional[str] = None
104
    file_storage_path: str = "sglang_storage"
105
    enable_cache_report: bool = False
Xihuai Wang's avatar
Xihuai Wang committed
106
    reasoning_parser: Optional[str] = None
Lianmin Zheng's avatar
Lianmin Zheng committed
107

108
109
110
    # Data parallelism
    dp_size: int = 1
    load_balance_method: str = "round_robin"
111

xiaobochen's avatar
xiaobochen committed
112
113
    # Expert parallelism
    ep_size: int = 1
114

115
    # Multi-node distributed serving
116
    dist_init_addr: Optional[str] = None
Lianmin Zheng's avatar
Lianmin Zheng committed
117
    nnodes: int = 1
118
    node_rank: int = 0
Lianmin Zheng's avatar
Lianmin Zheng committed
119
120
121
122

    # Model override args in JSON
    json_model_override_args: str = "{}"

123
124
125
    # LoRA
    lora_paths: Optional[List[str]] = None
    max_loras_per_batch: int = 8
126
    lora_backend: str = "triton"
127
128

    # Kernel backend
129
130
    attention_backend: Optional[str] = None
    sampling_backend: Optional[str] = None
131
    grammar_backend: Optional[str] = None
132

133
134
    # Speculative decoding
    speculative_algorithm: Optional[str] = None
135
    speculative_draft_model_path: Optional[str] = None
136
137
138
    speculative_num_steps: Optional[int] = None
    speculative_eagle_topk: Optional[int] = None
    speculative_num_draft_tokens: Optional[int] = None
139
140
    speculative_accept_threshold_single: float = 1.0
    speculative_accept_threshold_acc: float = 1.0
141
    speculative_token_map: Optional[str] = None
142
143
144

    # Double Sparsity
    enable_double_sparsity: bool = False
Vincent's avatar
Vincent committed
145
    ds_channel_config_path: Optional[str] = None
146
147
148
149
150
    ds_heavy_channel_num: int = 32
    ds_heavy_token_num: int = 256
    ds_heavy_channel_type: str = "qk"
    ds_sparse_decode_threshold: int = 4096

151
    # Optimization/debug options
Lianmin Zheng's avatar
Lianmin Zheng committed
152
    disable_radix_cache: bool = False
153
    disable_cuda_graph: bool = False
154
    disable_cuda_graph_padding: bool = False
155
    enable_nccl_nvls: bool = False
156
    disable_outlines_disk_cache: bool = False
157
    disable_custom_all_reduce: bool = False
158
    enable_multimodal: Optional[bool] = None
159
    disable_overlap_schedule: bool = False
160
    enable_mixed_chunk: bool = False
Ke Bao's avatar
Ke Bao committed
161
    enable_dp_attention: bool = False
xiaobochen's avatar
xiaobochen committed
162
    enable_ep_moe: bool = False
163
    enable_deepep_moe: bool = False
164
    deepep_mode: Optional[Literal["auto", "normal", "low_latency"]] = "auto"
165
    enable_torch_compile: bool = False
166
    torch_compile_max_bs: int = 32
167
    cuda_graph_max_bs: Optional[int] = None
168
    cuda_graph_bs: Optional[List[int]] = None
169
    torchao_config: str = ""
170
    enable_nan_detection: bool = False
171
    enable_p2p_check: bool = False
172
    triton_attention_reduce_in_fp32: bool = False
173
    triton_attention_num_kv_splits: int = 8
174
    num_continuous_decode_steps: int = 1
175
    delete_ckpt_after_loading: bool = False
176
    enable_memory_saver: bool = False
177
    allow_auto_truncate: bool = False
178
    enable_custom_logit_processor: bool = False
Vincent's avatar
Vincent committed
179
    tool_call_parser: Optional[str] = None
180
    enable_hierarchical_cache: bool = False
181
    hicache_ratio: float = 2.0
Zhiqiang Xie's avatar
Zhiqiang Xie committed
182
183
    hicache_size: int = 0
    hicache_write_policy: str = "write_through_selective"
184
    flashinfer_mla_disable_ragged: bool = False
185
    warmups: Optional[str] = None
186
    moe_dense_tp_size: Optional[int] = None
187
    n_share_experts_fusion: int = 0
188
    disable_chunked_prefix_cache: bool = False
Lianmin Zheng's avatar
Lianmin Zheng committed
189
    disable_fast_image_processor: bool = False
190
191
192
193
194

    # Debug tensor dumps
    debug_tensor_dump_output_folder: Optional[str] = None
    debug_tensor_dump_input_file: Optional[str] = None
    debug_tensor_dump_inject: bool = False
195

Byron Hsu's avatar
Byron Hsu committed
196
197
198
    # For PD disaggregation: can be "null" (not disaggregated), "prefill" (prefill-only), or "decode" (decode-only)
    disaggregation_mode: str = "null"
    disaggregation_bootstrap_port: int = 8998
199
    disaggregation_transfer_backend: str = "mooncake"
200
    disaggregation_ib_device: Optional[str] = None
201
    pdlb_url: Optional[str] = None
Byron Hsu's avatar
Byron Hsu committed
202

Lianmin Zheng's avatar
Lianmin Zheng committed
203
    def __post_init__(self):
204
205
206
        # Expert parallelism
        if self.enable_ep_moe:
            self.ep_size = self.tp_size
Lianmin Zheng's avatar
Lianmin Zheng committed
207
            logger.warning(
208
209
210
                f"EP MoE is enabled. The expert parallel size is adjusted to be the same as the tensor parallel size[{self.tp_size}]."
            )

211
        # Set missing default values
Lianmin Zheng's avatar
Lianmin Zheng committed
212
213
        if self.tokenizer_path is None:
            self.tokenizer_path = self.model_path
214

215
216
217
        if self.device is None:
            self.device = get_device()

218
219
220
        if self.served_model_name is None:
            self.served_model_name = self.model_path

221
222
223
        if self.random_seed is None:
            self.random_seed = random.randint(0, 1 << 30)

Lianmin Zheng's avatar
Lianmin Zheng committed
224
        gpu_mem = get_device_memory_capacity(self.device)
225
226

        # Set mem fraction static, which depends on the tensor parallelism size
Lianmin Zheng's avatar
Lianmin Zheng committed
227
        if self.mem_fraction_static is None:
228
229
230
231
232
233
234
235
236
237
238
239
            parallel_size = self.tp_size * self.pp_size
            if gpu_mem <= 81920:
                if parallel_size >= 16:
                    self.mem_fraction_static = 0.79
                elif parallel_size >= 8:
                    self.mem_fraction_static = 0.81
                elif parallel_size >= 4:
                    self.mem_fraction_static = 0.85
                elif parallel_size >= 2:
                    self.mem_fraction_static = 0.87
                else:
                    self.mem_fraction_static = 0.88
Ying Sheng's avatar
Ying Sheng committed
240
            else:
241
242
243
244
245
246
247
248
                self.mem_fraction_static = 0.88
            if gpu_mem > 96 * 1024:
                mem_fraction = self.mem_fraction_static
                self.mem_fraction_static = min(
                    mem_fraction + 48 * 1024 * (1 - mem_fraction) / gpu_mem,
                    (gpu_mem - 1024 * 18)
                    / gpu_mem,  # 15 GB + additional 3GB for cuda graph
                )
249

250
251
        # Set chunked prefill size, which depends on the gpu memory capacity
        if self.chunked_prefill_size is None:
252
            if gpu_mem is not None and gpu_mem < 25_000:
253
                self.chunked_prefill_size = 2048
254
255
            elif self.disaggregation_mode != "null":
                self.chunked_prefill_size = 16384
256
257
            else:
                self.chunked_prefill_size = 8192
Lianmin Zheng's avatar
Lianmin Zheng committed
258
259
        assert self.chunked_prefill_size % self.page_size == 0

260
261
262
        assert self.moe_dense_tp_size in {
            1,
            None,
Lianmin Zheng's avatar
Lianmin Zheng committed
263
        }, "moe_dense_tp_size only support 1 and None currently"
264

265
        if self.attention_backend == "flashmla":
266
267
268
269
            logger.warning(
                "FlashMLA only supports a page_size of 64, change page_size to 64."
            )
            self.page_size = 64
Lianmin Zheng's avatar
Lianmin Zheng committed
270

271
272
273
274
275
276
        if self.attention_backend == "cutlass_mla":
            logger.warning(
                "Cutlass MLA only supports a page_size of 128, change page_size to 128."
            )
            self.page_size = 128

277
278
        # Set cuda graph max batch size
        if self.cuda_graph_max_bs is None:
279
            # Based on detailed statistics, when serving TP1/TP2 models on lower-end GPUs with HBM<25G, you can either disable cuda graph or set `cuda_graph_max_bs` to a very small value to reduce the memory overhead of creating cuda graphs, with almost no impact on performance. However, when serving models with TP4 or TP8, we need to enable cuda graph to maintain high performance. In this case, we can set `cuda_graph_max_bs` to 80 (half of the default value 160) to reduce the memory overhead of creating cuda graphs. Looking at the logs from TP4 serving of qwen2-72b, a value of 80 is sufficient and can reduce the memory overhead of creating cuda graphs on lower-end GPUs compared to the original 160, avoiding OOM issues.
280
            if gpu_mem is not None and gpu_mem < 25_000:
281
282
283
284
                if self.tp_size < 4:
                    self.cuda_graph_max_bs = 8
                else:
                    self.cuda_graph_max_bs = 80
285

286
        # Set kernel backends for hpu device
287
288
289
290
        if self.device == "hpu":
            self.attention_backend = "torch_native"
            self.sampling_backend = "pytorch"

Lianmin Zheng's avatar
Lianmin Zheng committed
291
        # Set kernel backends
292
        if self.sampling_backend is None:
293
294
295
296
297
            self.sampling_backend = (
                "flashinfer" if is_flashinfer_available() else "pytorch"
            )

        if self.attention_backend == "torch_native":
298
            logger.warning(
299
300
301
                "Cuda graph is disabled because of using torch native attention backend"
            )
            self.disable_cuda_graph = True
302

303
304
305
        # Choose grammar backend
        if self.grammar_backend is None:
            self.grammar_backend = "xgrammar"
306

307
        # Data parallelism attention
Ke Bao's avatar
Ke Bao committed
308
        if self.enable_dp_attention:
309
            self.schedule_conservativeness = self.schedule_conservativeness * 0.3
Lianmin Zheng's avatar
Lianmin Zheng committed
310
311
312
313
314
            assert (
                self.dp_size > 1
            ), "Please set a dp-size > 1. You can use 1 < dp-size <= tp-size "
            assert self.tp_size % self.dp_size == 0
            self.chunked_prefill_size = self.chunked_prefill_size // self.dp_size
315
            logger.warning(
316
                f"DP attention is enabled. The chunked prefill size is adjusted to {self.chunked_prefill_size} to avoid MoE kernel issues. "
317
            )
318
319

        # DeepEP MoE
Lianmin Zheng's avatar
Lianmin Zheng committed
320
        self.enable_sp_layernorm = False
321
        if self.enable_deepep_moe:
322
323
324
325
            if self.deepep_mode == "auto":
                assert (
                    not self.enable_dp_attention
                ), "DeepEP MoE `auto` mode is not supported with DP Attention."
326
327
328
329
            self.ep_size = self.tp_size
            self.enable_sp_layernorm = (
                self.dp_size < self.tp_size if self.enable_dp_attention else True
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
330
            logger.warning(
331
332
                f"DeepEP MoE is enabled. The expert parallel size is adjusted to be the same as the tensor parallel size[{self.tp_size}]."
            )
333

334
        # Speculative Decoding
335
336
337
338
        if self.speculative_algorithm == "NEXTN":
            # NEXTN shares the same implementation of EAGLE
            self.speculative_algorithm = "EAGLE"

Lianmin Zheng's avatar
Lianmin Zheng committed
339
        if self.speculative_algorithm in ("EAGLE", "EAGLE3"):
340
            if self.max_running_requests is None:
341
                self.max_running_requests = 48
342
            self.disable_overlap_schedule = True
Lianmin Zheng's avatar
Lianmin Zheng committed
343
            logger.warning(
344
                "Overlap scheduler is disabled because of using "
345
                "eagle speculative decoding."
346
            )
347

348
349
350
            model_arch = get_model_arch(self)

            # Auto set draft_model_path DeepSeek-V3/R1
351
352
353
354
355
356
357
            if model_arch == "DeepseekV3ForCausalLM":
                if self.speculative_draft_model_path is None:
                    self.speculative_draft_model_path = self.model_path
                else:
                    logger.warning(
                        "DeepSeek MTP does not require setting speculative_draft_model_path."
                    )
358

359
360
361
362
363
364
365
366
367
368
            # Auto choose parameters
            if self.speculative_num_steps is None:
                assert (
                    self.speculative_eagle_topk is None
                    and self.speculative_num_draft_tokens is None
                )
                (
                    self.speculative_num_steps,
                    self.speculative_eagle_topk,
                    self.speculative_num_draft_tokens,
369
                ) = auto_choose_speculative_params(model_arch)
370
371
372

            if self.page_size > 1 and self.speculative_eagle_topk > 1:
                self.speculative_eagle_topk = 1
Lianmin Zheng's avatar
Lianmin Zheng committed
373
                logger.warning(
374
375
376
377
378
379
380
                    "speculative_eagle_topk is adjusted to 1 when page_size > 1"
                )

            if (
                self.speculative_eagle_topk == 1
                and self.speculative_num_draft_tokens != self.speculative_num_steps + 1
            ):
Lianmin Zheng's avatar
Lianmin Zheng committed
381
                logger.warning(
382
383
384
                    "speculative_num_draft_tokens is adjusted to speculative_num_steps + 1 when speculative_eagle_topk == 1"
                )
                self.speculative_num_draft_tokens = self.speculative_num_steps + 1
385

386
            # The token generated from the verify step is counted.
387
            # If sepculative_num_steps >= speculative_num_draft_tokens, the additional tokens will definitely be discarded.
388
            # assert self.speculative_num_steps < self.speculative_num_draft_tokens
389

390
391
392
393
394
395
        # GGUF
        if (
            self.load_format == "auto" or self.load_format == "gguf"
        ) and check_gguf_file(self.model_path):
            self.quantization = self.load_format = "gguf"

396
397
398
        if is_remote_url(self.model_path):
            self.load_format = "remote"

399
400
401
402
        # AMD-specific Triton attention KV splits default number
        if is_hip():
            self.triton_attention_num_kv_splits = 16

Byron Hsu's avatar
Byron Hsu committed
403
404
405
        # PD disaggregation
        if self.disaggregation_mode == "prefill":
            self.disable_cuda_graph = True
406
            logger.warning("Cuda graph is disabled for prefill server")
Byron Hsu's avatar
Byron Hsu committed
407
408
        elif self.disaggregation_mode == "decode":
            self.disable_radix_cache = True
409
            logger.warning("KV cache is forced as chunk cache for decode server")
Byron Hsu's avatar
Byron Hsu committed
410

411
412
413
        os.environ["SGLANG_ENABLE_TORCH_COMPILE"] = (
            "1" if self.enable_torch_compile else "0"
        )
414
415
416
417
        # Set env var before grammar backends init
        os.environ["SGLANG_DISABLE_OUTLINES_DISK_CACHE"] = (
            "1" if self.disable_outlines_disk_cache else "0"
        )
418

Lianmin Zheng's avatar
Lianmin Zheng committed
419
420
    @staticmethod
    def add_cli_args(parser: argparse.ArgumentParser):
421
        # Model and port args
Lianmin Zheng's avatar
Lianmin Zheng committed
422
423
424
425
426
427
428
429
430
431
432
433
        parser.add_argument(
            "--model-path",
            type=str,
            help="The path of the model weights. This can be a local folder or a Hugging Face repo ID.",
            required=True,
        )
        parser.add_argument(
            "--tokenizer-path",
            type=str,
            default=ServerArgs.tokenizer_path,
            help="The path of the tokenizer.",
        )
Yuanhan Zhang's avatar
Yuanhan Zhang committed
434
435
436
437
438
439
        parser.add_argument(
            "--host", type=str, default=ServerArgs.host, help="The host of the server."
        )
        parser.add_argument(
            "--port", type=int, default=ServerArgs.port, help="The port of the server."
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
440
441
442
443
444
445
446
447
448
        parser.add_argument(
            "--tokenizer-mode",
            type=str,
            default=ServerArgs.tokenizer_mode,
            choices=["auto", "slow"],
            help="Tokenizer mode. 'auto' will use the fast "
            "tokenizer if available, and 'slow' will "
            "always use the slow tokenizer.",
        )
449
450
451
        parser.add_argument(
            "--skip-tokenizer-init",
            action="store_true",
452
            help="If set, skip init tokenizer and pass input_ids in generate request.",
453
        )
454
455
456
457
458
        parser.add_argument(
            "--enable-tokenizer-batch-encode",
            action="store_true",
            help="Enable batch tokenization for improved performance when processing multiple text inputs. Do not use with image inputs, pre-tokenized input_ids, or input_embeds.",
        )
459
        parser.add_argument(
Lianmin Zheng's avatar
Lianmin Zheng committed
460
461
462
            "--load-format",
            type=str,
            default=ServerArgs.load_format,
463
464
465
466
467
468
            choices=[
                "auto",
                "pt",
                "safetensors",
                "npcache",
                "dummy",
469
                "sharded_state",
470
471
                "gguf",
                "bitsandbytes",
472
                "layered",
473
                "remote",
474
            ],
Lianmin Zheng's avatar
Lianmin Zheng committed
475
476
477
478
479
480
481
482
483
            help="The format of the model weights to load. "
            '"auto" will try to load the weights in the safetensors format '
            "and fall back to the pytorch bin format if safetensors format "
            "is not available. "
            '"pt" will load the weights in the pytorch bin format. '
            '"safetensors" will load the weights in the safetensors format. '
            '"npcache" will load the weights in pytorch format and store '
            "a numpy cache to speed up the loading. "
            '"dummy" will initialize the weights with random values, '
484
            "which is mainly for profiling."
485
486
            '"gguf" will load the weights in the gguf format. '
            '"bitsandbytes" will load the weights using bitsandbytes '
487
488
489
490
            "quantization."
            '"layered" loads weights layer by layer so that one can quantize a '
            "layer before loading another to make the peak memory envelope "
            "smaller.",
Lianmin Zheng's avatar
Lianmin Zheng committed
491
        )
492
493
494
495
496
        parser.add_argument(
            "--trust-remote-code",
            action="store_true",
            help="Whether or not to allow for custom models defined on the Hub in their own modeling files.",
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
497
        parser.add_argument(
Lianmin Zheng's avatar
Lianmin Zheng committed
498
            "--dtype",
Cody Yu's avatar
Cody Yu committed
499
            type=str,
Lianmin Zheng's avatar
Lianmin Zheng committed
500
            default=ServerArgs.dtype,
Ying Sheng's avatar
Ying Sheng committed
501
502
            choices=["auto", "half", "float16", "bfloat16", "float", "float32"],
            help="Data type for model weights and activations.\n\n"
Lianmin Zheng's avatar
Lianmin Zheng committed
503
            '* "auto" will use FP16 precision for FP32 and FP16 models, and '
Ying Sheng's avatar
Ying Sheng committed
504
            "BF16 precision for BF16 models.\n"
Lianmin Zheng's avatar
Lianmin Zheng committed
505
506
507
508
            '* "half" for FP16. Recommended for AWQ quantization.\n'
            '* "float16" is the same as "half".\n'
            '* "bfloat16" for a balance between precision and range.\n'
            '* "float" is shorthand for FP32 precision.\n'
Ying Sheng's avatar
Ying Sheng committed
509
510
            '* "float32" for FP32 precision.',
        )
511
512
513
514
        parser.add_argument(
            "--kv-cache-dtype",
            type=str,
            default=ServerArgs.kv_cache_dtype,
bjmsong's avatar
bjmsong committed
515
516
517
            choices=["auto", "fp8_e5m2", "fp8_e4m3"],
            help='Data type for kv cache storage. "auto" will use model data type. "fp8_e5m2" and "fp8_e4m3" is supported for CUDA 11.8+.',
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
518
519
520
521
        parser.add_argument(
            "--quantization",
            type=str,
            default=ServerArgs.quantization,
Ying Sheng's avatar
Ying Sheng committed
522
523
524
525
526
527
            choices=[
                "awq",
                "fp8",
                "gptq",
                "marlin",
                "gptq_marlin",
Ying Sheng's avatar
Ying Sheng committed
528
                "awq_marlin",
Ying Sheng's avatar
Ying Sheng committed
529
                "bitsandbytes",
530
                "gguf",
531
                "modelopt",
532
                "modelopt_fp4",
533
                "w8a8_int8",
HandH1998's avatar
HandH1998 committed
534
                "w8a8_fp8",
AniZpZ's avatar
AniZpZ committed
535
                "moe_wna16",
Ying Sheng's avatar
Ying Sheng committed
536
            ],
Lianmin Zheng's avatar
Lianmin Zheng committed
537
538
            help="The quantization method.",
        )
539
540
541
542
543
544
545
546
547
        parser.add_argument(
            "--quantization-param-path",
            type=nullable_str,
            default=None,
            help="Path to the JSON file containing the KV cache "
            "scaling factors. This should generally be supplied, when "
            "KV cache dtype is FP8. Otherwise, KV cache scaling factors "
            "default to 1.0, which may cause accuracy issues. ",
        )
548
549
550
551
552
553
554
555
556
        parser.add_argument(
            "--context-length",
            type=int,
            default=ServerArgs.context_length,
            help="The model's maximum context length. Defaults to None (will use the value from the model's config.json instead).",
        )
        parser.add_argument(
            "--device",
            type=str,
557
            default=ServerArgs.device,
558
            help="The device to use ('cuda', 'xpu', 'hpu', 'npu', 'cpu'). Defaults to auto-detection if not specified.",
559
        )
560
561
562
563
564
565
        parser.add_argument(
            "--served-model-name",
            type=str,
            default=ServerArgs.served_model_name,
            help="Override the model name returned by the v1/models endpoint in OpenAI API server.",
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
566
567
568
569
570
571
        parser.add_argument(
            "--chat-template",
            type=str,
            default=ServerArgs.chat_template,
            help="The buliltin chat template name or the path of the chat template file. This is only used for OpenAI-compatible API server.",
        )
572
573
574
575
576
577
        parser.add_argument(
            "--completion-template",
            type=str,
            default=ServerArgs.completion_template,
            help="The buliltin completion template name or the path of the completion template file. This is only used for OpenAI-compatible API server. only for code completion currently.",
        )
578
579
580
581
582
        parser.add_argument(
            "--is-embedding",
            action="store_true",
            help="Whether to use a CausalLM as an embedding model.",
        )
583
584
585
586
587
588
589
590
        parser.add_argument(
            "--revision",
            type=str,
            default=None,
            help="The specific model version to use. It can be a branch "
            "name, a tag name, or a commit id. If unspecified, will use "
            "the default version.",
        )
591

592
        # Memory and scheduling
Lianmin Zheng's avatar
Lianmin Zheng committed
593
594
595
596
        parser.add_argument(
            "--mem-fraction-static",
            type=float,
            default=ServerArgs.mem_fraction_static,
597
            help="The fraction of the memory used for static allocation (model weights and KV cache memory pool). Use a smaller value if you see out-of-memory errors.",
Lianmin Zheng's avatar
Lianmin Zheng committed
598
        )
599
600
601
602
603
604
        parser.add_argument(
            "--max-running-requests",
            type=int,
            default=ServerArgs.max_running_requests,
            help="The maximum number of running requests.",
        )
605
606
607
608
        parser.add_argument(
            "--max-total-tokens",
            type=int,
            default=ServerArgs.max_total_tokens,
609
610
            help="The maximum number of tokens in the memory pool. If not specified, it will be automatically calculated based on the memory usage fraction. "
            "This option is typically used for development and debugging purposes.",
611
        )
612
613
614
615
        parser.add_argument(
            "--chunked-prefill-size",
            type=int,
            default=ServerArgs.chunked_prefill_size,
616
            help="The maximum number of tokens in a chunk for the chunked prefill. Setting this to -1 means disabling chunked prefill.",
617
618
619
620
621
622
623
        )
        parser.add_argument(
            "--max-prefill-tokens",
            type=int,
            default=ServerArgs.max_prefill_tokens,
            help="The maximum number of tokens in a prefill batch. The real bound will be the maximum of this value and the model's maximum context length.",
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
624
        parser.add_argument(
625
            "--schedule-policy",
Lianmin Zheng's avatar
Lianmin Zheng committed
626
            type=str,
627
            default=ServerArgs.schedule_policy,
Liangsheng Yin's avatar
Liangsheng Yin committed
628
            choices=["lpm", "random", "fcfs", "dfs-weight"],
629
            help="The scheduling policy of the requests.",
Lianmin Zheng's avatar
Lianmin Zheng committed
630
        )
631
632
633
634
        parser.add_argument(
            "--schedule-conservativeness",
            type=float,
            default=ServerArgs.schedule_conservativeness,
635
            help="How conservative the schedule policy is. A larger value means more conservative scheduling. Use a larger value if you see requests being retracted frequently.",
636
        )
637
638
639
640
        parser.add_argument(
            "--cpu-offload-gb",
            type=int,
            default=ServerArgs.cpu_offload_gb,
641
            help="How many GBs of RAM to reserve for CPU offloading.",
642
        )
643
644
645
646
647
648
        parser.add_argument(
            "--page-size",
            type=int,
            default=ServerArgs.page_size,
            help="The number of tokens in a page.",
        )
649

650
        # Other runtime options
Lianmin Zheng's avatar
Lianmin Zheng committed
651
        parser.add_argument(
652
            "--tensor-parallel-size",
Lianmin Zheng's avatar
Lianmin Zheng committed
653
            "--tp-size",
Lianmin Zheng's avatar
Lianmin Zheng committed
654
            type=int,
Lianmin Zheng's avatar
Lianmin Zheng committed
655
            default=ServerArgs.tp_size,
656
            help="The tensor parallelism size.",
657
        )
658
659
660
661
662
663
664
665
666
667
668
669
670
        parser.add_argument(
            "--pipeline-parallel-size",
            "--pp-size",
            type=int,
            default=ServerArgs.pp_size,
            help="The pipeline parallelism size.",
        )
        parser.add_argument(
            "--max-micro-batch-size",
            type=int,
            default=ServerArgs.max_micro_batch_size,
            help="The maximum micro batch size in pipeline parallelism.",
        )
671
672
673
        parser.add_argument(
            "--stream-interval",
            type=int,
Lianmin Zheng's avatar
Lianmin Zheng committed
674
            default=ServerArgs.stream_interval,
675
            help="The interval (or buffer size) for streaming in terms of the token length. A smaller value makes streaming smoother, while a larger value makes the throughput higher",
676
        )
677
678
679
680
681
        parser.add_argument(
            "--stream-output",
            action="store_true",
            help="Whether to output as a sequence of disjoint segments.",
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
682
683
684
685
        parser.add_argument(
            "--random-seed",
            type=int,
            default=ServerArgs.random_seed,
686
            help="The random seed.",
Lianmin Zheng's avatar
Lianmin Zheng committed
687
        )
688
689
690
691
692
693
        parser.add_argument(
            "--constrained-json-whitespace-pattern",
            type=str,
            default=ServerArgs.constrained_json_whitespace_pattern,
            help=r"Regex pattern for syntactic whitespaces allowed in JSON constrained output. For example, to allow the model generate consecutive whitespaces, set the pattern to [\n\t ]*",
        )
694
695
696
697
698
699
        parser.add_argument(
            "--watchdog-timeout",
            type=float,
            default=ServerArgs.watchdog_timeout,
            help="Set watchdog timeout in seconds. If a forward batch takes longer than this, the server will crash to prevent hanging.",
        )
700
701
702
703
704
705
        parser.add_argument(
            "--dist-timeout",
            type=int,
            default=ServerArgs.dist_timeout,
            help="Set timeout for torch.distributed initialization.",
        )
706
707
708
709
        parser.add_argument(
            "--download-dir",
            type=str,
            default=ServerArgs.download_dir,
Lianmin Zheng's avatar
Lianmin Zheng committed
710
            help="Model download directory for huggingface.",
711
        )
712
713
714
715
716
717
        parser.add_argument(
            "--base-gpu-id",
            type=int,
            default=ServerArgs.base_gpu_id,
            help="The base GPU ID to start allocating GPUs from. Useful when running multiple instances on the same machine.",
        )
718
719
720
721
722
723
        parser.add_argument(
            "--gpu-id-step",
            type=int,
            default=ServerArgs.gpu_id_step,
            help="The delta between consecutive GPU IDs that are used. For example, setting it to 2 will use GPU 0,2,4,...",
        )
724
725

        # Logging
Lianmin Zheng's avatar
Lianmin Zheng committed
726
727
728
729
        parser.add_argument(
            "--log-level",
            type=str,
            default=ServerArgs.log_level,
730
            help="The logging level of all loggers.",
Lianmin Zheng's avatar
Lianmin Zheng committed
731
        )
732
        parser.add_argument(
733
734
735
736
            "--log-level-http",
            type=str,
            default=ServerArgs.log_level_http,
            help="The logging level of HTTP server. If not set, reuse --log-level by default.",
737
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
738
        parser.add_argument(
739
            "--log-requests",
Lianmin Zheng's avatar
Lianmin Zheng committed
740
            action="store_true",
741
742
743
744
745
746
747
748
            help="Log metadata, inputs, outputs of all requests. The verbosity is decided by --log-requests-level",
        )
        parser.add_argument(
            "--log-requests-level",
            type=int,
            default=0,
            help="0: Log metadata. 1. Log metadata and partial input/output. 2. Log every input/output.",
            choices=[0, 1, 2],
Lianmin Zheng's avatar
Lianmin Zheng committed
749
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
750
751
752
        parser.add_argument(
            "--show-time-cost",
            action="store_true",
Ying Sheng's avatar
Ying Sheng committed
753
            help="Show time cost of custom marks.",
Lianmin Zheng's avatar
Lianmin Zheng committed
754
        )
755
756
757
758
759
        parser.add_argument(
            "--enable-metrics",
            action="store_true",
            help="Enable log prometheus metrics.",
        )
760
761
762
763
        parser.add_argument(
            "--decode-log-interval",
            type=int,
            default=ServerArgs.decode_log_interval,
764
            help="The log interval of decode batch.",
765
        )
766

767
        # API related
Liangsheng Yin's avatar
Liangsheng Yin committed
768
769
770
771
        parser.add_argument(
            "--api-key",
            type=str,
            default=ServerArgs.api_key,
772
            help="Set API key of the server. It is also used in the OpenAI API compatible server.",
Liangsheng Yin's avatar
Liangsheng Yin committed
773
        )
774
        parser.add_argument(
775
            "--file-storage-path",
776
            type=str,
777
            default=ServerArgs.file_storage_path,
778
779
            help="The path of the file storage in backend.",
        )
780
781
782
783
784
        parser.add_argument(
            "--enable-cache-report",
            action="store_true",
            help="Return number of cached tokens in usage.prompt_tokens_details for each openai request.",
        )
Xihuai Wang's avatar
Xihuai Wang committed
785
786
787
788
789
790
791
        parser.add_argument(
            "--reasoning-parser",
            type=str,
            choices=list(ReasoningParser.DetectorMap.keys()),
            default=ServerArgs.reasoning_parser,
            help=f"Specify the parser for reasoning models, supported parsers are: {list(ReasoningParser.DetectorMap.keys())}.",
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
792

793
794
        # Data parallelism
        parser.add_argument(
795
            "--data-parallel-size",
796
797
798
            "--dp-size",
            type=int,
            default=ServerArgs.dp_size,
799
            help="The data parallelism size.",
800
801
802
803
804
        )
        parser.add_argument(
            "--load-balance-method",
            type=str,
            default=ServerArgs.load_balance_method,
805
            help="The load balancing strategy for data parallelism.",
806
807
808
809
810
            choices=[
                "round_robin",
                "shortest_queue",
            ],
        )
811

xiaobochen's avatar
xiaobochen committed
812
813
814
815
816
817
818
819
        # Expert parallelism
        parser.add_argument(
            "--expert-parallel-size",
            "--ep-size",
            type=int,
            default=ServerArgs.ep_size,
            help="The expert parallelism size.",
        )
820

821
        # Multi-node distributed serving
822
        parser.add_argument(
823
824
            "--dist-init-addr",
            "--nccl-init-addr",  # For backward compatbility. This will be removed in the future.
825
            type=str,
826
            help="The host address for initializing distributed backend (e.g., `192.168.0.2:25000`).",
827
828
        )
        parser.add_argument(
Liangsheng Yin's avatar
Liangsheng Yin committed
829
            "--nnodes", type=int, default=ServerArgs.nnodes, help="The number of nodes."
830
        )
831
832
833
        parser.add_argument(
            "--node-rank", type=int, default=ServerArgs.node_rank, help="The node rank."
        )
834

Lianmin Zheng's avatar
Lianmin Zheng committed
835
836
837
838
839
840
841
842
        # Model override args
        parser.add_argument(
            "--json-model-override-args",
            type=str,
            help="A dictionary in JSON string format used to override default model configurations.",
            default=ServerArgs.json_model_override_args,
        )

843
844
845
846
847
848
849
        # LoRA
        parser.add_argument(
            "--lora-paths",
            type=str,
            nargs="*",
            default=None,
            action=LoRAPathAction,
850
            help="The list of LoRA adapters. You can provide a list of either path in str or renamed path in the format {name}={path}.",
851
852
853
854
855
        )
        parser.add_argument(
            "--max-loras-per-batch",
            type=int,
            default=8,
856
857
858
859
860
861
862
            help="Maximum number of adapters for a running batch, include base-only request.",
        )
        parser.add_argument(
            "--lora-backend",
            type=str,
            default="triton",
            help="Choose the kernel backend for multi-LoRA serving.",
863
864
865
        )

        # Kernel backend
866
867
868
        parser.add_argument(
            "--attention-backend",
            type=str,
869
870
871
872
873
874
875
876
            choices=[
                "flashinfer",
                "triton",
                "torch_native",
                "fa3",
                "flashmla",
                "cutlass_mla",
            ],
877
878
879
            default=ServerArgs.attention_backend,
            help="Choose the kernels for attention layers.",
        )
880
881
882
883
884
885
886
        parser.add_argument(
            "--sampling-backend",
            type=str,
            choices=["flashinfer", "pytorch"],
            default=ServerArgs.sampling_backend,
            help="Choose the kernels for sampling layers.",
        )
887
888
889
        parser.add_argument(
            "--grammar-backend",
            type=str,
890
            choices=["xgrammar", "outlines", "llguidance", "none"],
891
            default=ServerArgs.grammar_backend,
Lianmin Zheng's avatar
Lianmin Zheng committed
892
            help="Choose the backend for grammar-guided decoding.",
893
        )
894
895
        parser.add_argument(
            "--enable-flashinfer-mla",
896
897
            action=DeprecatedAction,
            help="--enable-flashinfer-mla is deprecated. Please use '--attention-backend flashinfer' instead.",
898
        )
lukec's avatar
lukec committed
899
900
        parser.add_argument(
            "--enable-flashmla",
901
902
            action=DeprecatedAction,
            help="--enable-flashmla is deprecated. Please use '--attention-backend flashmla' instead.",
lukec's avatar
lukec committed
903
        )
904
905
906
907
908
        parser.add_argument(
            "--flashinfer-mla-disable-ragged",
            action="store_true",
            help="Not using ragged prefill wrapper when running flashinfer mla",
        )
909

910
911
912
913
        # Speculative decoding
        parser.add_argument(
            "--speculative-algorithm",
            type=str,
James Liu's avatar
James Liu committed
914
            choices=["EAGLE", "EAGLE3", "NEXTN"],
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
            help="Speculative algorithm.",
        )
        parser.add_argument(
            "--speculative-draft-model-path",
            type=str,
            help="The path of the draft model weights. This can be a local folder or a Hugging Face repo ID.",
        )
        parser.add_argument(
            "--speculative-num-steps",
            type=int,
            help="The number of steps sampled from draft model in Speculative Decoding.",
            default=ServerArgs.speculative_num_steps,
        )
        parser.add_argument(
            "--speculative-eagle-topk",
            type=int,
931
            help="The number of tokens sampled from the draft model in eagle2 each step.",
932
933
            default=ServerArgs.speculative_eagle_topk,
        )
934
935
936
        parser.add_argument(
            "--speculative-num-draft-tokens",
            type=int,
937
            help="The number of tokens sampled from the draft model in Speculative Decoding.",
938
939
            default=ServerArgs.speculative_num_draft_tokens,
        )
940
941
942
943
944
945
946
947
948
949
950
951
        parser.add_argument(
            "--speculative-accept-threshold-single",
            type=float,
            help="Accept a draft token if its probability in the target model is greater than this threshold.",
            default=ServerArgs.speculative_accept_threshold_single,
        )
        parser.add_argument(
            "--speculative-accept-threshold-acc",
            type=float,
            help="The accept probability of a draft token is raised from its target probability p to min(1, p / threshold_acc).",
            default=ServerArgs.speculative_accept_threshold_acc,
        )
952
953
954
955
956
957
        parser.add_argument(
            "--speculative-token-map",
            type=str,
            help="The path of the draft model's small vocab table.",
            default=ServerArgs.speculative_token_map,
        )
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995

        # Double Sparsity
        parser.add_argument(
            "--enable-double-sparsity",
            action="store_true",
            help="Enable double sparsity attention",
        )
        parser.add_argument(
            "--ds-channel-config-path",
            type=str,
            default=ServerArgs.ds_channel_config_path,
            help="The path of the double sparsity channel config",
        )
        parser.add_argument(
            "--ds-heavy-channel-num",
            type=int,
            default=ServerArgs.ds_heavy_channel_num,
            help="The number of heavy channels in double sparsity attention",
        )
        parser.add_argument(
            "--ds-heavy-token-num",
            type=int,
            default=ServerArgs.ds_heavy_token_num,
            help="The number of heavy tokens in double sparsity attention",
        )
        parser.add_argument(
            "--ds-heavy-channel-type",
            type=str,
            default=ServerArgs.ds_heavy_channel_type,
            help="The type of heavy channels in double sparsity attention",
        )
        parser.add_argument(
            "--ds-sparse-decode-threshold",
            type=int,
            default=ServerArgs.ds_sparse_decode_threshold,
            help="The type of heavy channels in double sparsity attention",
        )

996
        # Optimization/debug options
Liangsheng Yin's avatar
Liangsheng Yin committed
997
        parser.add_argument(
Lianmin Zheng's avatar
Lianmin Zheng committed
998
            "--disable-radix-cache",
Liangsheng Yin's avatar
Liangsheng Yin committed
999
            action="store_true",
Ying Sheng's avatar
Ying Sheng committed
1000
            help="Disable RadixAttention for prefix caching.",
Liangsheng Yin's avatar
Liangsheng Yin committed
1001
        )
1002
1003
1004
1005
1006
        parser.add_argument(
            "--disable-cuda-graph",
            action="store_true",
            help="Disable cuda graph.",
        )
1007
        parser.add_argument(
1008
1009
1010
1011
            "--disable-cuda-graph-padding",
            action="store_true",
            help="Disable cuda graph when padding is needed. Still uses cuda graph when padding is not needed.",
        )
1012
1013
1014
1015
1016
        parser.add_argument(
            "--enable-nccl-nvls",
            action="store_true",
            help="Enable NCCL NVLS for prefill heavy requests when available.",
        )
1017
        parser.add_argument(
1018
            "--disable-outlines-disk-cache",
1019
            action="store_true",
1020
            help="Disable disk cache of outlines to avoid possible crashes related to file system or high concurrency.",
1021
        )
1022
1023
1024
1025
1026
        parser.add_argument(
            "--disable-custom-all-reduce",
            action="store_true",
            help="Disable the custom all-reduce kernel and fall back to NCCL.",
        )
1027
        parser.add_argument(
1028
1029
            "--enable-multimodal",
            default=ServerArgs.enable_multimodal,
1030
            action="store_true",
1031
            help="Enable the multimodal functionality for the served model. If the model being served is not multimodal, nothing will happen",
1032
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
1033
        parser.add_argument(
1034
            "--disable-overlap-schedule",
Lianmin Zheng's avatar
Lianmin Zheng committed
1035
            action="store_true",
1036
            help="Disable the overlap scheduler, which overlaps the CPU scheduler with GPU model worker.",
Lianmin Zheng's avatar
Lianmin Zheng committed
1037
        )
1038
1039
1040
        parser.add_argument(
            "--enable-mixed-chunk",
            action="store_true",
1041
            help="Enabling mixing prefill and decode in a batch when using chunked prefill.",
1042
        )
Ke Bao's avatar
Ke Bao committed
1043
1044
1045
1046
1047
        parser.add_argument(
            "--enable-dp-attention",
            action="store_true",
            help="Enabling data parallelism for attention and tensor parallelism for FFN. The dp size should be equal to the tp size. Currently only DeepSeek-V2 is supported.",
        )
xiaobochen's avatar
xiaobochen committed
1048
1049
1050
1051
1052
        parser.add_argument(
            "--enable-ep-moe",
            action="store_true",
            help="Enabling expert parallelism for moe. The ep size is equal to the tp size.",
        )
1053
1054
1055
        parser.add_argument(
            "--enable-torch-compile",
            action="store_true",
1056
1057
            help="Optimize the model with torch.compile. Experimental feature.",
        )
1058
        parser.add_argument(
1059
            "--torch-compile-max-bs",
1060
            type=int,
1061
            default=ServerArgs.torch_compile_max_bs,
1062
1063
            help="Set the maximum batch size when using torch compile.",
        )
1064
        parser.add_argument(
1065
            "--cuda-graph-max-bs",
1066
            type=int,
1067
            default=ServerArgs.cuda_graph_max_bs,
1068
1069
            help="Set the maximum batch size for cuda graph.",
        )
1070
1071
1072
1073
1074
1075
        parser.add_argument(
            "--cuda-graph-bs",
            type=int,
            nargs="+",
            help="Set the list of batch sizes for cuda graph.",
        )
1076
1077
1078
1079
        parser.add_argument(
            "--torchao-config",
            type=str,
            default=ServerArgs.torchao_config,
1080
            help="Optimize the model with torchao. Experimental feature. Current choices are: int8dq, int8wo, int4wo-<group_size>, fp8wo, fp8dq-per_tensor, fp8dq-per_row",
1081
        )
1082
1083
1084
1085
1086
        parser.add_argument(
            "--enable-nan-detection",
            action="store_true",
            help="Enable the NaN detection for debugging purposes.",
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
1087
        parser.add_argument(
1088
            "--enable-p2p-check",
Lianmin Zheng's avatar
Lianmin Zheng committed
1089
            action="store_true",
1090
            help="Enable P2P check for GPU access, otherwise the p2p access is allowed by default.",
Lianmin Zheng's avatar
Lianmin Zheng committed
1091
        )
1092
        parser.add_argument(
1093
            "--triton-attention-reduce-in-fp32",
1094
            action="store_true",
1095
            help="Cast the intermidiate attention results to fp32 to avoid possible crashes related to fp16."
1096
            "This only affects Triton attention kernels.",
1097
        )
1098
1099
1100
1101
1102
1103
        parser.add_argument(
            "--triton-attention-num-kv-splits",
            type=int,
            default=ServerArgs.triton_attention_num_kv_splits,
            help="The number of KV splits in flash decoding Triton kernel. Larger value is better in longer context scenarios. The default value is 8.",
        )
1104
1105
1106
1107
1108
1109
1110
1111
        parser.add_argument(
            "--num-continuous-decode-steps",
            type=int,
            default=ServerArgs.num_continuous_decode_steps,
            help="Run multiple continuous decoding steps to reduce scheduling overhead. "
            "This can potentially increase throughput but may also increase time-to-first-token latency. "
            "The default value is 1, meaning only run one decoding step at a time.",
        )
1112
1113
1114
1115
1116
        parser.add_argument(
            "--delete-ckpt-after-loading",
            action="store_true",
            help="Delete the model checkpoint after loading the model.",
        )
1117
1118
1119
1120
1121
        parser.add_argument(
            "--enable-memory-saver",
            action="store_true",
            help="Allow saving memory using release_memory_occupation and resume_memory_occupation",
        )
1122
1123
1124
1125
1126
        parser.add_argument(
            "--allow-auto-truncate",
            action="store_true",
            help="Allow automatically truncating requests that exceed the maximum input length instead of returning an error.",
        )
1127
1128
1129
1130
1131
        parser.add_argument(
            "--enable-custom-logit-processor",
            action="store_true",
            help="Enable users to pass custom logit processors to the server (disabled by default for security)",
        )
YAMY's avatar
YAMY committed
1132
1133
1134
        parser.add_argument(
            "--tool-call-parser",
            type=str,
1135
            choices=["qwen25", "mistral", "llama3", "deepseekv3", "pythonic"],
YAMY's avatar
YAMY committed
1136
            default=ServerArgs.tool_call_parser,
1137
            help="Specify the parser for handling tool-call interactions. Options include: 'qwen25', 'mistral', 'llama3', 'deepseekv3', and 'pythonic'.",
YAMY's avatar
YAMY committed
1138
        )
1139
1140
1141
1142
1143
        parser.add_argument(
            "--enable-hierarchical-cache",
            action="store_true",
            help="Enable hierarchical cache",
        )
1144
1145
1146
1147
1148
1149
        parser.add_argument(
            "--hicache-ratio",
            type=float,
            default=ServerArgs.hicache_ratio,
            help="The ratio of the size of host KV cache memory pool to the size of device pool.",
        )
Zhiqiang Xie's avatar
Zhiqiang Xie committed
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
        parser.add_argument(
            "--hicache-size",
            type=int,
            default=ServerArgs.hicache_size,
            help="The size of host KV cache memory pool in gigabytes, which will override the hicache_ratio if set.",
        )
        parser.add_argument(
            "--hicache-write-policy",
            type=str,
            choices=["write_back", "write_through", "write_through_selective"],
            default=ServerArgs.hicache_write_policy,
            help="The write policy of hierarchical cache.",
        )
1163
1164
1165
1166
1167
        parser.add_argument(
            "--enable-deepep-moe",
            action="store_true",
            help="Enabling DeepEP MoE implementation for EP MoE.",
        )
1168
1169
1170
1171
1172
1173
        parser.add_argument(
            "--moe-dense-tp-size",
            type=int,
            default=ServerArgs.moe_dense_tp_size,
            help="TP size for MoE dense MLP layers. This flag is useful when, with large TP size, there are errors caused by weights in MLP layers having dimension smaller than the min dimension GEMM supports.",
        )
1174
1175
1176
1177
        parser.add_argument(
            "--deepep-mode",
            type=str,
            choices=["normal", "low_latency", "auto"],
1178
            default="auto",
1179
1180
            help="Select the mode when enable DeepEP MoE, could be `normal`, `low_latency` or `auto`. Default is `auto`, which means `low_latency` for decode batch and `normal` for prefill batch.",
        )
1181

1182
1183
1184
        parser.add_argument(
            "--n-share-experts-fusion",
            type=int,
1185
            default=0,
1186
1187
            help="The number of shared_experts need to be replicated to fuse with normal experts in deepseek v3/r1, "
            "set it to tp_size can get best optimized performace.",
1188
        )
1189
1190
1191
1192
1193
        parser.add_argument(
            "--disable-chunked-prefix-cache",
            action="store_true",
            help="Disable chunked prefix cache feature for deepseek, which should save overhead for short sequences.",
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
1194
1195
1196
1197
1198
        parser.add_argument(
            "--disable-fast-image-processor",
            action="store_true",
            help="Adopt base image processor instead of fast image processor.",
        )
1199

1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
        # Server warmups
        parser.add_argument(
            "--warmups",
            type=str,
            required=False,
            help="Specify custom warmup functions (csv) to run before server starts eg. --warmups=warmup_name1,warmup_name2 "
            "will run the functions `warmup_name1` and `warmup_name2` specified in warmup.py before the server starts listening for requests",
        )

        # Debug tensor dumps
        parser.add_argument(
            "--debug-tensor-dump-output-folder",
            type=str,
            default=ServerArgs.debug_tensor_dump_output_folder,
            help="The output folder for dumping tensors.",
        )
        parser.add_argument(
            "--debug-tensor-dump-input-file",
            type=str,
            default=ServerArgs.debug_tensor_dump_input_file,
            help="The input filename for dumping tensors",
        )
        parser.add_argument(
            "--debug-tensor-dump-inject",
            type=str,
            default=ServerArgs.debug_tensor_dump_inject,
            help="Inject the outputs from jax as the input of every layer.",
        )

Byron Hsu's avatar
Byron Hsu committed
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
        # Disaggregation
        parser.add_argument(
            "--disaggregation-mode",
            type=str,
            default="null",
            choices=["null", "prefill", "decode"],
            help='Only used for PD disaggregation. "prefill" for prefill-only server, and "decode" for decode-only server. If not specified, it is not PD disaggregated',
        )
        parser.add_argument(
            "--disaggregation-bootstrap-port",
            type=int,
            default=ServerArgs.disaggregation_bootstrap_port,
            help="Bootstrap server port on the prefill server. Default is 8998.",
        )
1243
1244
1245
1246
        parser.add_argument(
            "--disaggregation-transfer-backend",
            type=str,
            default=ServerArgs.disaggregation_transfer_backend,
1247
            choices=["mooncake", "nixl"],
1248
1249
            help="The backend for disaggregation transfer. Default is mooncake.",
        )
1250
1251
1252
1253
        parser.add_argument(
            "--disaggregation-ib-device",
            type=str,
            default=ServerArgs.disaggregation_ib_device,
1254
1255
1256
            help="The InfiniBand devices for disaggregation transfer, accepts single device (e.g., --disaggregation-ib-device mlx5_0) "
            "or multiple comma-separated devices (e.g., --disaggregation-ib-device mlx5_0,mlx5_1). "
            "Default is None, which triggers automatic device detection when mooncake backend is enabled.",
1257
        )
1258
1259
1260
1261
1262
1263
        parser.add_argument(
            "--pdlb-url",
            type=str,
            default=None,
            help="The URL of the PD disaggregation load balancer. If set, the prefill/decode server will register with the load balancer.",
        )
Byron Hsu's avatar
Byron Hsu committed
1264

Lianmin Zheng's avatar
Lianmin Zheng committed
1265
1266
    @classmethod
    def from_cli_args(cls, args: argparse.Namespace):
1267
        args.tp_size = args.tensor_parallel_size
1268
        args.pp_size = args.pipeline_parallel_size
1269
        args.dp_size = args.data_parallel_size
xiaobochen's avatar
xiaobochen committed
1270
        args.ep_size = args.expert_parallel_size
Lianmin Zheng's avatar
Lianmin Zheng committed
1271
1272
1273
1274
        attrs = [attr.name for attr in dataclasses.fields(cls)]
        return cls(**{attr: getattr(args, attr) for attr in attrs})

    def url(self):
1275
        if is_valid_ipv6_address(self.host):
1276
1277
1278
            return f"http://[{self.host}]:{self.port}"
        else:
            return f"http://{self.host}:{self.port}"
Lianmin Zheng's avatar
Lianmin Zheng committed
1279

1280
1281
    def check_server_args(self):
        assert (
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
            self.tp_size * self.pp_size
        ) % self.nnodes == 0, "tp_size must be divisible by number of nodes"

        # FIXME pp constraints
        if self.pp_size > 1:
            logger.warning(f"Turn off overlap scheule for pipeline parallelism.")
            self.disable_overlap_schedule = True
            assert (
                self.disable_overlap_schedule
                and self.speculative_algorithm is None
                and not self.enable_mixed_chunk
            ), "Pipeline parallelism is not compatible with overlap schedule, speculative decoding, mixed chunked prefill."

1295
        assert not (
1296
1297
            self.dp_size > 1 and self.nnodes != 1 and not self.enable_dp_attention
        ), "multi-node data parallel is not supported unless dp attention!"
1298
1299
1300
1301
1302
        assert (
            self.max_loras_per_batch > 0
            # FIXME
            and (self.lora_paths is None or self.disable_radix_cache)
        ), "compatibility of lora and cuda graph and radix attention is in progress"
1303
        assert self.base_gpu_id >= 0, "base_gpu_id must be non-negative"
1304
        assert self.gpu_id_step >= 1, "gpu_id_step must be positive"
1305

1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
        if isinstance(self.lora_paths, list):
            lora_paths = self.lora_paths
            self.lora_paths = {}
            for lora_path in lora_paths:
                if "=" in lora_path:
                    name, path = lora_path.split("=", 1)
                    self.lora_paths[name] = path
                else:
                    self.lora_paths[lora_path] = lora_path

Lianmin Zheng's avatar
Lianmin Zheng committed
1316

Lianmin Zheng's avatar
Lianmin Zheng committed
1317
def prepare_server_args(argv: List[str]) -> ServerArgs:
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
    """
    Prepare the server arguments from the command line arguments.

    Args:
        args: The command line arguments. Typically, it should be `sys.argv[1:]`
            to ensure compatibility with `parse_args` when no arguments are passed.

    Returns:
        The server arguments.
    """
    parser = argparse.ArgumentParser()
    ServerArgs.add_cli_args(parser)
Lianmin Zheng's avatar
Lianmin Zheng committed
1330
    raw_args = parser.parse_args(argv)
1331
1332
1333
1334
    server_args = ServerArgs.from_cli_args(raw_args)
    return server_args


1335
1336
1337
ZMQ_TCP_PORT_DELTA = 233


Lianmin Zheng's avatar
Lianmin Zheng committed
1338
1339
@dataclasses.dataclass
class PortArgs:
1340
1341
1342
1343
1344
1345
    # The ipc filename for tokenizer to receive inputs from detokenizer (zmq)
    tokenizer_ipc_name: str
    # The ipc filename for scheduler (rank 0) to receive inputs from tokenizer (zmq)
    scheduler_input_ipc_name: str
    # The ipc filename for detokenizer to receive inputs from scheduler (zmq)
    detokenizer_ipc_name: str
1346

1347
1348
    # The port for nccl initialization (torch.dist)
    nccl_port: int
1349

1350
1351
1352
    # The ipc filename for rpc call between Engine and Scheduler
    rpc_ipc_name: str

1353
    @staticmethod
1354
    def init_new(server_args, dp_rank: Optional[int] = None) -> "PortArgs":
1355
        port = server_args.port + random.randint(100, 1000)
1356
1357
1358
        while True:
            if is_port_available(port):
                break
TianYu GUO's avatar
TianYu GUO committed
1359
1360
1361
1362
            if port < 60000:
                port += 42
            else:
                port -= 43
1363

1364
1365
1366
1367
1368
1369
1370
        if not server_args.enable_dp_attention:
            # Normal case, use IPC within a single node
            return PortArgs(
                tokenizer_ipc_name=f"ipc://{tempfile.NamedTemporaryFile(delete=False).name}",
                scheduler_input_ipc_name=f"ipc://{tempfile.NamedTemporaryFile(delete=False).name}",
                detokenizer_ipc_name=f"ipc://{tempfile.NamedTemporaryFile(delete=False).name}",
                nccl_port=port,
1371
                rpc_ipc_name=f"ipc://{tempfile.NamedTemporaryFile(delete=False).name}",
1372
1373
1374
1375
1376
            )
        else:
            # DP attention. Use TCP + port to handle both single-node and multi-node.
            if server_args.nnodes == 1 and server_args.dist_init_addr is None:
                dist_init_addr = ("127.0.0.1", server_args.port + ZMQ_TCP_PORT_DELTA)
Vincent's avatar
Vincent committed
1377
1378
1379
            elif server_args.dist_init_addr.startswith("["):  # ipv6 address
                port_num, host = configure_ipv6(server_args.dist_init_addr)
                dist_init_addr = (host, str(port_num))
1380
1381
            else:
                dist_init_addr = server_args.dist_init_addr.split(":")
Vincent's avatar
Vincent committed
1382

1383
1384
1385
1386
1387
1388
1389
1390
            assert (
                len(dist_init_addr) == 2
            ), "please provide --dist-init-addr as host:port of head node"

            dist_init_host, dist_init_port = dist_init_addr
            port_base = int(dist_init_port) + 1
            if dp_rank is None:
                scheduler_input_port = (
1391
                    port_base + 3
1392
                )  # TokenizerManager to DataParallelController
1393
            else:
1394
                scheduler_input_port = port_base + 3 + 1 + dp_rank
1395
1396
1397
1398
1399
1400

            return PortArgs(
                tokenizer_ipc_name=f"tcp://{dist_init_host}:{port_base}",
                scheduler_input_ipc_name=f"tcp://{dist_init_host}:{scheduler_input_port}",
                detokenizer_ipc_name=f"tcp://{dist_init_host}:{port_base + 1}",
                nccl_port=port,
1401
                rpc_ipc_name=f"tcp://{dist_init_host}:{port_base + 2}",
1402
            )
1403

1404
1405
1406
1407
1408
1409
1410
1411
1412
1413

class LoRAPathAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
        setattr(namespace, self.dest, {})
        for lora_path in values:
            if "=" in lora_path:
                name, path = lora_path.split("=", 1)
                getattr(namespace, self.dest)[name] = path
            else:
                getattr(namespace, self.dest)[lora_path] = lora_path
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423


class DeprecatedAction(argparse.Action):
    def __init__(self, option_strings, dest, nargs=0, **kwargs):
        super(DeprecatedAction, self).__init__(
            option_strings, dest, nargs=nargs, **kwargs
        )

    def __call__(self, parser, namespace, values, option_string=None):
        raise ValueError(self.help)
1424
1425


1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
def get_model_arch(args: ServerArgs):
    hf_config = get_config(
        args.model_path,
        trust_remote_code=args.trust_remote_code,
        revision=args.revision,
        model_override_args=json.loads(args.json_model_override_args),
    )
    return hf_config.architectures[0]


def auto_choose_speculative_params(arch: str):
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
    """
    Automatically choose the parameters for speculative decoding.

    You can tune them on your own models and prompts with scripts/playground/bench_speculative.py
    """
    if arch in ["LlamaForCausalLM"]:
        # The default value for llama
        return (5, 4, 8)
    elif arch in ["DeepseekV3ForCausalLM", "DeepseekV2ForCausalLM"]:
        # The default value for deepseek
        return (5, 4, 8)
    elif arch in ["Grok1ForCausalLM", "Grok1VForCausalLM"]:
        return (5, 4, 8)
    else:
        # The default value for all other models
        return (5, 4, 8)