"test/vscode:/vscode.git/clone" did not exist on "548a57b1f3ca52a7268f5e96e54c80ff2f4df4cd"
server_args.py 64.3 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
Lianmin Zheng's avatar
Lianmin Zheng committed
14
15
"""The arguments of the server."""

Lianmin Zheng's avatar
Lianmin Zheng committed
16
17
import argparse
import dataclasses
18
import json
19
import logging
20
import os
21
import random
22
import tempfile
23
from typing import List, Literal, Optional
Lianmin Zheng's avatar
Lianmin Zheng committed
24

25
from sglang.srt.hf_transformers_utils import check_gguf_file, get_config
Xihuai Wang's avatar
Xihuai Wang committed
26
from sglang.srt.reasoning_parser import ReasoningParser
27
from sglang.srt.utils import (
Vincent's avatar
Vincent committed
28
    configure_ipv6,
29
    get_device,
Lianmin Zheng's avatar
Lianmin Zheng committed
30
    get_device_memory_capacity,
31
    is_flashinfer_available,
HAI's avatar
HAI committed
32
    is_hip,
33
    is_port_available,
34
    is_remote_url,
35
    is_valid_ipv6_address,
bjmsong's avatar
bjmsong committed
36
    nullable_str,
37
)
38

39
40
logger = logging.getLogger(__name__)

Lianmin Zheng's avatar
Lianmin Zheng committed
41
42
43

@dataclasses.dataclass
class ServerArgs:
Lianmin Zheng's avatar
Lianmin Zheng committed
44
    # Model and tokenizer
Lianmin Zheng's avatar
Lianmin Zheng committed
45
46
47
    model_path: str
    tokenizer_path: Optional[str] = None
    tokenizer_mode: str = "auto"
48
    skip_tokenizer_init: bool = False
Lianmin Zheng's avatar
Lianmin Zheng committed
49
    load_format: str = "auto"
50
    trust_remote_code: bool = False
Lianmin Zheng's avatar
Lianmin Zheng committed
51
    dtype: str = "auto"
52
    kv_cache_dtype: str = "auto"
Lianmin Zheng's avatar
Lianmin Zheng committed
53
    quantization: Optional[str] = None
Vincent's avatar
Vincent committed
54
    quantization_param_path: Optional[str] = None
55
    context_length: Optional[int] = None
56
    device: Optional[str] = None
57
    served_model_name: Optional[str] = None
Lianmin Zheng's avatar
Lianmin Zheng committed
58
    chat_template: Optional[str] = None
59
    completion_template: Optional[str] = None
60
    is_embedding: bool = False
61
    enable_multimodal: Optional[bool] = None
62
    revision: Optional[str] = None
Lianmin Zheng's avatar
Lianmin Zheng committed
63

64
    # Port for the HTTP server
Lianmin Zheng's avatar
Lianmin Zheng committed
65
66
67
68
    host: str = "127.0.0.1"
    port: int = 30000

    # Memory and scheduling
Lianmin Zheng's avatar
Lianmin Zheng committed
69
    mem_fraction_static: Optional[float] = None
70
    max_running_requests: Optional[int] = None
71
    max_total_tokens: Optional[int] = None
72
    chunked_prefill_size: Optional[int] = None
73
    max_prefill_tokens: int = 16384
74
    schedule_policy: str = "fcfs"
75
    schedule_conservativeness: float = 1.0
76
    cpu_offload_gb: int = 0
77
    page_size: int = 1
Lianmin Zheng's avatar
Lianmin Zheng committed
78
79
80

    # Other runtime options
    tp_size: int = 1
81
82
    pp_size: int = 1
    max_micro_batch_size: Optional[int] = None
83
    stream_interval: int = 1
84
    stream_output: bool = False
85
    random_seed: Optional[int] = None
86
    constrained_json_whitespace_pattern: Optional[str] = None
87
    watchdog_timeout: float = 300
88
    dist_timeout: Optional[int] = None  # timeout for torch.distributed
89
    download_dir: Optional[str] = None
90
    base_gpu_id: int = 0
91
    gpu_id_step: int = 1
Lianmin Zheng's avatar
Lianmin Zheng committed
92
93
94

    # Logging
    log_level: str = "info"
95
    log_level_http: Optional[str] = None
96
    log_requests: bool = False
97
    log_requests_level: int = 0
Liangsheng Yin's avatar
Liangsheng Yin committed
98
    show_time_cost: bool = False
99
    enable_metrics: bool = False
100
101
102
103
    bucket_time_to_first_token: Optional[List[float]] = None
    bucket_e2e_request_latency: Optional[List[float]] = None
    bucket_inter_token_latency: Optional[List[float]] = None
    collect_tokens_histogram: bool = False
104
    decode_log_interval: int = 40
105
    enable_request_time_stats_logging: bool = False
106
    kv_events_config: Optional[str] = None
Liangsheng Yin's avatar
Liangsheng Yin committed
107

108
    # API related
109
    api_key: Optional[str] = None
110
    file_storage_path: str = "sglang_storage"
111
    enable_cache_report: bool = False
Xihuai Wang's avatar
Xihuai Wang committed
112
    reasoning_parser: Optional[str] = None
Lianmin Zheng's avatar
Lianmin Zheng committed
113

114
115
116
    # Data parallelism
    dp_size: int = 1
    load_balance_method: str = "round_robin"
117

xiaobochen's avatar
xiaobochen committed
118
119
    # Expert parallelism
    ep_size: int = 1
120

121
    # Multi-node distributed serving
122
    dist_init_addr: Optional[str] = None
Lianmin Zheng's avatar
Lianmin Zheng committed
123
    nnodes: int = 1
124
    node_rank: int = 0
Lianmin Zheng's avatar
Lianmin Zheng committed
125
126
127

    # Model override args in JSON
    json_model_override_args: str = "{}"
128
    preferred_sampling_params: Optional[str] = None
Lianmin Zheng's avatar
Lianmin Zheng committed
129

130
131
132
    # LoRA
    lora_paths: Optional[List[str]] = None
    max_loras_per_batch: int = 8
133
    lora_backend: str = "triton"
134
135

    # Kernel backend
136
137
    attention_backend: Optional[str] = None
    sampling_backend: Optional[str] = None
138
    grammar_backend: Optional[str] = None
139

140
141
    # Speculative decoding
    speculative_algorithm: Optional[str] = None
142
    speculative_draft_model_path: Optional[str] = None
143
144
145
    speculative_num_steps: Optional[int] = None
    speculative_eagle_topk: Optional[int] = None
    speculative_num_draft_tokens: Optional[int] = None
146
147
    speculative_accept_threshold_single: float = 1.0
    speculative_accept_threshold_acc: float = 1.0
148
    speculative_token_map: Optional[str] = None
149
150
151

    # Double Sparsity
    enable_double_sparsity: bool = False
Vincent's avatar
Vincent committed
152
    ds_channel_config_path: Optional[str] = None
153
154
155
156
157
    ds_heavy_channel_num: int = 32
    ds_heavy_token_num: int = 256
    ds_heavy_channel_type: str = "qk"
    ds_sparse_decode_threshold: int = 4096

158
    # Optimization/debug options
Lianmin Zheng's avatar
Lianmin Zheng committed
159
    disable_radix_cache: bool = False
160
    disable_cuda_graph: bool = False
161
    disable_cuda_graph_padding: bool = False
162
    enable_nccl_nvls: bool = False
163
    enable_tokenizer_batch_encode: bool = False
164
    disable_outlines_disk_cache: bool = False
165
    disable_custom_all_reduce: bool = False
166
    disable_overlap_schedule: bool = False
167
    enable_mixed_chunk: bool = False
Ke Bao's avatar
Ke Bao committed
168
    enable_dp_attention: bool = False
169
    enable_dp_lm_head: bool = False
170
    enable_two_batch_overlap: bool = False
xiaobochen's avatar
xiaobochen committed
171
    enable_ep_moe: bool = False
172
    enable_deepep_moe: bool = False
173
    deepep_mode: Optional[Literal["auto", "normal", "low_latency"]] = "auto"
174
    ep_num_redundant_experts: int = 0
175
    ep_dispatch_algorithm: Optional[Literal["static", "dynamic", "fake"]] = None
176
    init_expert_location: str = "trivial"
177
    enable_eplb: bool = False
178
    eplb_algorithm: str = "auto"
179
    eplb_rebalance_num_iterations: int = 1000
180
181
182
183
    expert_distribution_recorder_mode: Optional[
        Literal["stat", "per_pass", "per_token"]
    ] = None
    expert_distribution_recorder_buffer_size: Optional[int] = None
184
    enable_expert_distribution_metrics: bool = False
185
    deepep_config: Optional[str] = None
186
    enable_torch_compile: bool = False
187
    torch_compile_max_bs: int = 32
188
    cuda_graph_max_bs: Optional[int] = None
189
    cuda_graph_bs: Optional[List[int]] = None
190
    torchao_config: str = ""
191
    enable_nan_detection: bool = False
192
    enable_p2p_check: bool = False
193
    triton_attention_reduce_in_fp32: bool = False
194
    triton_attention_num_kv_splits: int = 8
195
    num_continuous_decode_steps: int = 1
196
    delete_ckpt_after_loading: bool = False
197
    enable_memory_saver: bool = False
198
    allow_auto_truncate: bool = False
199
    enable_custom_logit_processor: bool = False
Vincent's avatar
Vincent committed
200
    tool_call_parser: Optional[str] = None
201
    enable_hierarchical_cache: bool = False
202
    hicache_ratio: float = 2.0
Zhiqiang Xie's avatar
Zhiqiang Xie committed
203
204
    hicache_size: int = 0
    hicache_write_policy: str = "write_through_selective"
205
    flashinfer_mla_disable_ragged: bool = False
206
    warmups: Optional[str] = None
207
    moe_dense_tp_size: Optional[int] = None
208
    n_share_experts_fusion: int = 0
209
    disable_chunked_prefix_cache: bool = False
Lianmin Zheng's avatar
Lianmin Zheng committed
210
    disable_fast_image_processor: bool = False
211
    mm_attention_backend: Optional[str] = None
212
213
214
215
216

    # Debug tensor dumps
    debug_tensor_dump_output_folder: Optional[str] = None
    debug_tensor_dump_input_file: Optional[str] = None
    debug_tensor_dump_inject: bool = False
217

Byron Hsu's avatar
Byron Hsu committed
218
219
220
    # For PD disaggregation: can be "null" (not disaggregated), "prefill" (prefill-only), or "decode" (decode-only)
    disaggregation_mode: str = "null"
    disaggregation_bootstrap_port: int = 8998
221
    disaggregation_transfer_backend: str = "mooncake"
222
    disaggregation_ib_device: Optional[str] = None
223
    pdlb_url: Optional[str] = None
Byron Hsu's avatar
Byron Hsu committed
224

Lianmin Zheng's avatar
Lianmin Zheng committed
225
    def __post_init__(self):
226
227
228
        # Expert parallelism
        if self.enable_ep_moe:
            self.ep_size = self.tp_size
Lianmin Zheng's avatar
Lianmin Zheng committed
229
            logger.warning(
230
231
232
                f"EP MoE is enabled. The expert parallel size is adjusted to be the same as the tensor parallel size[{self.tp_size}]."
            )

233
        # Set missing default values
Lianmin Zheng's avatar
Lianmin Zheng committed
234
235
        if self.tokenizer_path is None:
            self.tokenizer_path = self.model_path
236

237
238
239
        if self.device is None:
            self.device = get_device()

240
241
242
        if self.served_model_name is None:
            self.served_model_name = self.model_path

243
244
245
        if self.random_seed is None:
            self.random_seed = random.randint(0, 1 << 30)

Lianmin Zheng's avatar
Lianmin Zheng committed
246
        gpu_mem = get_device_memory_capacity(self.device)
247
248

        # Set mem fraction static, which depends on the tensor parallelism size
Lianmin Zheng's avatar
Lianmin Zheng committed
249
        if self.mem_fraction_static is None:
250
            parallel_size = self.tp_size * self.pp_size
Yi Liu's avatar
Yi Liu committed
251
            if gpu_mem is not None and gpu_mem <= 81920:
252
253
254
255
256
257
258
259
260
261
                if parallel_size >= 16:
                    self.mem_fraction_static = 0.79
                elif parallel_size >= 8:
                    self.mem_fraction_static = 0.81
                elif parallel_size >= 4:
                    self.mem_fraction_static = 0.85
                elif parallel_size >= 2:
                    self.mem_fraction_static = 0.87
                else:
                    self.mem_fraction_static = 0.88
Ying Sheng's avatar
Ying Sheng committed
262
            else:
263
                self.mem_fraction_static = 0.88
264
265
266
            if gpu_mem is not None and gpu_mem > 180 * 1000:
                self.mem_fraction_static = 0.79
            elif gpu_mem is not None and gpu_mem > 96 * 1024:
267
                mem_fraction = self.mem_fraction_static
268
269
270
271
272
                # 15 GB + additional 3GB for cuda graph
                reserve_mem = 1024 * 18
                # need reserve more memory for spec cuda graph
                if self.speculative_algorithm is not None:
                    reserve_mem = 1024 * 20
273
274
                self.mem_fraction_static = min(
                    mem_fraction + 48 * 1024 * (1 - mem_fraction) / gpu_mem,
275
                    (gpu_mem - reserve_mem) / gpu_mem,
276
                )
277
278
279
            else:
                if self.speculative_algorithm is not None:
                    self.mem_fraction_static *= 0.95
280

281
282
        # Set chunked prefill size, which depends on the gpu memory capacity
        if self.chunked_prefill_size is None:
283
284
285
            if gpu_mem is not None and gpu_mem > 180_000:
                self.chunked_prefill_size = 16384
            elif gpu_mem is not None and gpu_mem < 25_000:
286
                self.chunked_prefill_size = 2048
287
288
            elif self.disaggregation_mode != "null":
                self.chunked_prefill_size = 16384
289
290
            else:
                self.chunked_prefill_size = 8192
Lianmin Zheng's avatar
Lianmin Zheng committed
291
292
        assert self.chunked_prefill_size % self.page_size == 0

293
294
295
        assert self.moe_dense_tp_size in {
            1,
            None,
Lianmin Zheng's avatar
Lianmin Zheng committed
296
        }, "moe_dense_tp_size only support 1 and None currently"
297

298
        if self.attention_backend == "flashmla":
299
300
301
302
            logger.warning(
                "FlashMLA only supports a page_size of 64, change page_size to 64."
            )
            self.page_size = 64
Lianmin Zheng's avatar
Lianmin Zheng committed
303

304
305
306
307
308
309
        if self.attention_backend == "cutlass_mla":
            logger.warning(
                "Cutlass MLA only supports a page_size of 128, change page_size to 128."
            )
            self.page_size = 128

310
        # Set cuda graph max batch size
311
        if self.cuda_graph_max_bs is None:
312
            # Based on detailed statistics, when serving TP1/TP2 models on lower-end GPUs with HBM<25G, you can either disable cuda graph or set `cuda_graph_max_bs` to a very small value to reduce the memory overhead of creating cuda graphs, with almost no impact on performance. However, when serving models with TP4 or TP8, we need to enable cuda graph to maintain high performance. In this case, we can set `cuda_graph_max_bs` to 80 (half of the default value 160) to reduce the memory overhead of creating cuda graphs. Looking at the logs from TP4 serving of qwen2-72b, a value of 80 is sufficient and can reduce the memory overhead of creating cuda graphs on lower-end GPUs compared to the original 160, avoiding OOM issues.
313
            if gpu_mem is not None and gpu_mem < 25_000:
314
315
316
317
                if self.tp_size < 4:
                    self.cuda_graph_max_bs = 8
                else:
                    self.cuda_graph_max_bs = 80
318

319
        # Set kernel backends for hpu device
320
321
322
323
        if self.device == "hpu":
            self.attention_backend = "torch_native"
            self.sampling_backend = "pytorch"

Lianmin Zheng's avatar
Lianmin Zheng committed
324
        # Set kernel backends
325
        if self.sampling_backend is None:
326
327
328
329
330
            self.sampling_backend = (
                "flashinfer" if is_flashinfer_available() else "pytorch"
            )

        if self.attention_backend == "torch_native":
331
            logger.warning(
332
333
334
                "Cuda graph is disabled because of using torch native attention backend"
            )
            self.disable_cuda_graph = True
335

336
337
338
        # Choose grammar backend
        if self.grammar_backend is None:
            self.grammar_backend = "xgrammar"
339

340
        # Data parallelism attention
Ke Bao's avatar
Ke Bao committed
341
        if self.enable_dp_attention:
342
            self.schedule_conservativeness = self.schedule_conservativeness * 0.3
Lianmin Zheng's avatar
Lianmin Zheng committed
343
344
345
346
347
            assert (
                self.dp_size > 1
            ), "Please set a dp-size > 1. You can use 1 < dp-size <= tp-size "
            assert self.tp_size % self.dp_size == 0
            self.chunked_prefill_size = self.chunked_prefill_size // self.dp_size
348
            logger.warning(
349
                f"DP attention is enabled. The chunked prefill size is adjusted to {self.chunked_prefill_size} to avoid MoE kernel issues. "
350
            )
351

352
353
354
355
356
        if self.enable_dp_lm_head:
            assert (
                self.enable_dp_attention
            ), "Please enable dp attention when setting enable_dp_attention. "

357
        # DeepEP MoE
Lianmin Zheng's avatar
Lianmin Zheng committed
358
        self.enable_sp_layernorm = False
359
        if self.enable_deepep_moe:
360
361
362
363
            if self.deepep_mode == "auto":
                assert (
                    not self.enable_dp_attention
                ), "DeepEP MoE `auto` mode is not supported with DP Attention."
364
365
366
            if self.deepep_mode == "normal":
                logger.warning("Cuda graph is disabled because deepep_mode=`normal`")
                self.disable_cuda_graph = True
367
368
369
370
            self.ep_size = self.tp_size
            self.enable_sp_layernorm = (
                self.dp_size < self.tp_size if self.enable_dp_attention else True
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
371
            logger.warning(
372
373
                f"DeepEP MoE is enabled. The expert parallel size is adjusted to be the same as the tensor parallel size[{self.tp_size}]."
            )
374

375
376
377
378
379
380
        if self.pp_size > 1:
            self.disable_overlap_schedule = True
            logger.warning(
                "Pipeline parallelism is incompatible with overlap schedule."
            )

381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
        if self.enable_eplb and (self.expert_distribution_recorder_mode is None):
            self.expert_distribution_recorder_mode = "stat"
            logger.info(
                f"EPLB is enabled. The expert_distribution_recorder_mode is automatically set."
            )

        if (self.enable_eplb or (self.init_expert_location is not None)) and (
            self.ep_dispatch_algorithm is None
        ):
            self.ep_dispatch_algorithm = "static"
            logger.info(
                f"EPLB is enabled or init_expert_location is provided. ep_dispatch_algorithm is configured."
            )

        if self.enable_expert_distribution_metrics and (
            self.expert_distribution_recorder_mode is None
        ):
            self.expert_distribution_recorder_mode = "stat"

400
        if self.expert_distribution_recorder_buffer_size is None:
401
402
            if (x := self.eplb_rebalance_num_iterations) is not None:
                self.expert_distribution_recorder_buffer_size = x
403
404
405
            elif self.expert_distribution_recorder_mode is not None:
                self.expert_distribution_recorder_buffer_size = 1000

406
        # Speculative Decoding
407
408
409
410
        if self.speculative_algorithm == "NEXTN":
            # NEXTN shares the same implementation of EAGLE
            self.speculative_algorithm = "EAGLE"

Lianmin Zheng's avatar
Lianmin Zheng committed
411
        if self.speculative_algorithm in ("EAGLE", "EAGLE3"):
412
            if self.max_running_requests is None:
413
                self.max_running_requests = 48
414
            self.disable_overlap_schedule = True
Lianmin Zheng's avatar
Lianmin Zheng committed
415
            logger.warning(
416
                "Overlap scheduler is disabled because of using "
417
                "eagle speculative decoding."
418
            )
419

420
421
422
            model_arch = get_model_arch(self)

            # Auto set draft_model_path DeepSeek-V3/R1
423
424
425
426
427
428
429
            if model_arch == "DeepseekV3ForCausalLM":
                if self.speculative_draft_model_path is None:
                    self.speculative_draft_model_path = self.model_path
                else:
                    logger.warning(
                        "DeepSeek MTP does not require setting speculative_draft_model_path."
                    )
430

431
432
433
434
435
436
437
438
439
440
            # Auto choose parameters
            if self.speculative_num_steps is None:
                assert (
                    self.speculative_eagle_topk is None
                    and self.speculative_num_draft_tokens is None
                )
                (
                    self.speculative_num_steps,
                    self.speculative_eagle_topk,
                    self.speculative_num_draft_tokens,
441
                ) = auto_choose_speculative_params(model_arch)
442
443
444

            if self.page_size > 1 and self.speculative_eagle_topk > 1:
                self.speculative_eagle_topk = 1
Lianmin Zheng's avatar
Lianmin Zheng committed
445
                logger.warning(
446
447
448
449
450
451
452
                    "speculative_eagle_topk is adjusted to 1 when page_size > 1"
                )

            if (
                self.speculative_eagle_topk == 1
                and self.speculative_num_draft_tokens != self.speculative_num_steps + 1
            ):
Lianmin Zheng's avatar
Lianmin Zheng committed
453
                logger.warning(
454
455
456
                    "speculative_num_draft_tokens is adjusted to speculative_num_steps + 1 when speculative_eagle_topk == 1"
                )
                self.speculative_num_draft_tokens = self.speculative_num_steps + 1
457

458
            # The token generated from the verify step is counted.
459
            # If sepculative_num_steps >= speculative_num_draft_tokens, the additional tokens will definitely be discarded.
460
            # assert self.speculative_num_steps < self.speculative_num_draft_tokens
461

462
463
464
465
466
467
        # GGUF
        if (
            self.load_format == "auto" or self.load_format == "gguf"
        ) and check_gguf_file(self.model_path):
            self.quantization = self.load_format = "gguf"

468
469
470
        if is_remote_url(self.model_path):
            self.load_format = "remote"

471
472
473
474
        # AMD-specific Triton attention KV splits default number
        if is_hip():
            self.triton_attention_num_kv_splits = 16

Byron Hsu's avatar
Byron Hsu committed
475
476
477
        # PD disaggregation
        if self.disaggregation_mode == "prefill":
            self.disable_cuda_graph = True
478
            logger.warning("Cuda graph is disabled for prefill server")
Byron Hsu's avatar
Byron Hsu committed
479
480
        elif self.disaggregation_mode == "decode":
            self.disable_radix_cache = True
481
            logger.warning("KV cache is forced as chunk cache for decode server")
Byron Hsu's avatar
Byron Hsu committed
482

483
484
485
        os.environ["SGLANG_ENABLE_TORCH_COMPILE"] = (
            "1" if self.enable_torch_compile else "0"
        )
486
487
488
489
        # Set env var before grammar backends init
        os.environ["SGLANG_DISABLE_OUTLINES_DISK_CACHE"] = (
            "1" if self.disable_outlines_disk_cache else "0"
        )
490

Lianmin Zheng's avatar
Lianmin Zheng committed
491
492
    @staticmethod
    def add_cli_args(parser: argparse.ArgumentParser):
493
        # Model and port args
Lianmin Zheng's avatar
Lianmin Zheng committed
494
495
496
497
498
499
500
501
502
503
504
505
        parser.add_argument(
            "--model-path",
            type=str,
            help="The path of the model weights. This can be a local folder or a Hugging Face repo ID.",
            required=True,
        )
        parser.add_argument(
            "--tokenizer-path",
            type=str,
            default=ServerArgs.tokenizer_path,
            help="The path of the tokenizer.",
        )
Yuanhan Zhang's avatar
Yuanhan Zhang committed
506
507
508
509
510
511
        parser.add_argument(
            "--host", type=str, default=ServerArgs.host, help="The host of the server."
        )
        parser.add_argument(
            "--port", type=int, default=ServerArgs.port, help="The port of the server."
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
512
513
514
515
516
517
518
519
520
        parser.add_argument(
            "--tokenizer-mode",
            type=str,
            default=ServerArgs.tokenizer_mode,
            choices=["auto", "slow"],
            help="Tokenizer mode. 'auto' will use the fast "
            "tokenizer if available, and 'slow' will "
            "always use the slow tokenizer.",
        )
521
522
523
        parser.add_argument(
            "--skip-tokenizer-init",
            action="store_true",
524
            help="If set, skip init tokenizer and pass input_ids in generate request.",
525
        )
526
        parser.add_argument(
Lianmin Zheng's avatar
Lianmin Zheng committed
527
528
529
            "--load-format",
            type=str,
            default=ServerArgs.load_format,
530
531
532
533
534
535
            choices=[
                "auto",
                "pt",
                "safetensors",
                "npcache",
                "dummy",
536
                "sharded_state",
537
538
                "gguf",
                "bitsandbytes",
539
                "layered",
540
                "remote",
541
            ],
Lianmin Zheng's avatar
Lianmin Zheng committed
542
543
544
545
546
547
548
549
550
            help="The format of the model weights to load. "
            '"auto" will try to load the weights in the safetensors format '
            "and fall back to the pytorch bin format if safetensors format "
            "is not available. "
            '"pt" will load the weights in the pytorch bin format. '
            '"safetensors" will load the weights in the safetensors format. '
            '"npcache" will load the weights in pytorch format and store '
            "a numpy cache to speed up the loading. "
            '"dummy" will initialize the weights with random values, '
551
            "which is mainly for profiling."
552
553
            '"gguf" will load the weights in the gguf format. '
            '"bitsandbytes" will load the weights using bitsandbytes '
554
555
556
557
            "quantization."
            '"layered" loads weights layer by layer so that one can quantize a '
            "layer before loading another to make the peak memory envelope "
            "smaller.",
Lianmin Zheng's avatar
Lianmin Zheng committed
558
        )
559
560
561
562
563
        parser.add_argument(
            "--trust-remote-code",
            action="store_true",
            help="Whether or not to allow for custom models defined on the Hub in their own modeling files.",
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
564
        parser.add_argument(
Lianmin Zheng's avatar
Lianmin Zheng committed
565
            "--dtype",
Cody Yu's avatar
Cody Yu committed
566
            type=str,
Lianmin Zheng's avatar
Lianmin Zheng committed
567
            default=ServerArgs.dtype,
Ying Sheng's avatar
Ying Sheng committed
568
569
            choices=["auto", "half", "float16", "bfloat16", "float", "float32"],
            help="Data type for model weights and activations.\n\n"
Lianmin Zheng's avatar
Lianmin Zheng committed
570
            '* "auto" will use FP16 precision for FP32 and FP16 models, and '
Ying Sheng's avatar
Ying Sheng committed
571
            "BF16 precision for BF16 models.\n"
Lianmin Zheng's avatar
Lianmin Zheng committed
572
573
574
575
            '* "half" for FP16. Recommended for AWQ quantization.\n'
            '* "float16" is the same as "half".\n'
            '* "bfloat16" for a balance between precision and range.\n'
            '* "float" is shorthand for FP32 precision.\n'
Ying Sheng's avatar
Ying Sheng committed
576
577
            '* "float32" for FP32 precision.',
        )
578
579
580
581
        parser.add_argument(
            "--kv-cache-dtype",
            type=str,
            default=ServerArgs.kv_cache_dtype,
bjmsong's avatar
bjmsong committed
582
583
584
            choices=["auto", "fp8_e5m2", "fp8_e4m3"],
            help='Data type for kv cache storage. "auto" will use model data type. "fp8_e5m2" and "fp8_e4m3" is supported for CUDA 11.8+.',
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
585
586
587
588
        parser.add_argument(
            "--quantization",
            type=str,
            default=ServerArgs.quantization,
Ying Sheng's avatar
Ying Sheng committed
589
590
591
592
593
594
            choices=[
                "awq",
                "fp8",
                "gptq",
                "marlin",
                "gptq_marlin",
Ying Sheng's avatar
Ying Sheng committed
595
                "awq_marlin",
Ying Sheng's avatar
Ying Sheng committed
596
                "bitsandbytes",
597
                "gguf",
598
                "modelopt",
599
                "modelopt_fp4",
600
                "w8a8_int8",
HandH1998's avatar
HandH1998 committed
601
                "w8a8_fp8",
AniZpZ's avatar
AniZpZ committed
602
                "moe_wna16",
HandH1998's avatar
HandH1998 committed
603
                "qoq",
Ying Sheng's avatar
Ying Sheng committed
604
            ],
Lianmin Zheng's avatar
Lianmin Zheng committed
605
606
            help="The quantization method.",
        )
607
608
609
610
611
612
613
614
615
        parser.add_argument(
            "--quantization-param-path",
            type=nullable_str,
            default=None,
            help="Path to the JSON file containing the KV cache "
            "scaling factors. This should generally be supplied, when "
            "KV cache dtype is FP8. Otherwise, KV cache scaling factors "
            "default to 1.0, which may cause accuracy issues. ",
        )
616
617
618
619
620
621
622
623
624
        parser.add_argument(
            "--context-length",
            type=int,
            default=ServerArgs.context_length,
            help="The model's maximum context length. Defaults to None (will use the value from the model's config.json instead).",
        )
        parser.add_argument(
            "--device",
            type=str,
625
            default=ServerArgs.device,
626
            help="The device to use ('cuda', 'xpu', 'hpu', 'npu', 'cpu'). Defaults to auto-detection if not specified.",
627
        )
628
629
630
631
632
633
        parser.add_argument(
            "--served-model-name",
            type=str,
            default=ServerArgs.served_model_name,
            help="Override the model name returned by the v1/models endpoint in OpenAI API server.",
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
634
635
636
637
638
639
        parser.add_argument(
            "--chat-template",
            type=str,
            default=ServerArgs.chat_template,
            help="The buliltin chat template name or the path of the chat template file. This is only used for OpenAI-compatible API server.",
        )
640
641
642
643
644
645
        parser.add_argument(
            "--completion-template",
            type=str,
            default=ServerArgs.completion_template,
            help="The buliltin completion template name or the path of the completion template file. This is only used for OpenAI-compatible API server. only for code completion currently.",
        )
646
647
648
649
650
        parser.add_argument(
            "--is-embedding",
            action="store_true",
            help="Whether to use a CausalLM as an embedding model.",
        )
651
652
653
654
655
656
        parser.add_argument(
            "--enable-multimodal",
            default=ServerArgs.enable_multimodal,
            action="store_true",
            help="Enable the multimodal functionality for the served model. If the model being served is not multimodal, nothing will happen",
        )
657
658
659
660
661
662
663
664
        parser.add_argument(
            "--revision",
            type=str,
            default=None,
            help="The specific model version to use. It can be a branch "
            "name, a tag name, or a commit id. If unspecified, will use "
            "the default version.",
        )
665

666
        # Memory and scheduling
Lianmin Zheng's avatar
Lianmin Zheng committed
667
668
669
670
        parser.add_argument(
            "--mem-fraction-static",
            type=float,
            default=ServerArgs.mem_fraction_static,
671
            help="The fraction of the memory used for static allocation (model weights and KV cache memory pool). Use a smaller value if you see out-of-memory errors.",
Lianmin Zheng's avatar
Lianmin Zheng committed
672
        )
673
674
675
676
677
678
        parser.add_argument(
            "--max-running-requests",
            type=int,
            default=ServerArgs.max_running_requests,
            help="The maximum number of running requests.",
        )
679
680
681
682
        parser.add_argument(
            "--max-total-tokens",
            type=int,
            default=ServerArgs.max_total_tokens,
683
684
            help="The maximum number of tokens in the memory pool. If not specified, it will be automatically calculated based on the memory usage fraction. "
            "This option is typically used for development and debugging purposes.",
685
        )
686
687
688
689
        parser.add_argument(
            "--chunked-prefill-size",
            type=int,
            default=ServerArgs.chunked_prefill_size,
690
            help="The maximum number of tokens in a chunk for the chunked prefill. Setting this to -1 means disabling chunked prefill.",
691
692
693
694
695
696
697
        )
        parser.add_argument(
            "--max-prefill-tokens",
            type=int,
            default=ServerArgs.max_prefill_tokens,
            help="The maximum number of tokens in a prefill batch. The real bound will be the maximum of this value and the model's maximum context length.",
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
698
        parser.add_argument(
699
            "--schedule-policy",
Lianmin Zheng's avatar
Lianmin Zheng committed
700
            type=str,
701
            default=ServerArgs.schedule_policy,
Liangsheng Yin's avatar
Liangsheng Yin committed
702
            choices=["lpm", "random", "fcfs", "dfs-weight"],
703
            help="The scheduling policy of the requests.",
Lianmin Zheng's avatar
Lianmin Zheng committed
704
        )
705
706
707
708
        parser.add_argument(
            "--schedule-conservativeness",
            type=float,
            default=ServerArgs.schedule_conservativeness,
709
            help="How conservative the schedule policy is. A larger value means more conservative scheduling. Use a larger value if you see requests being retracted frequently.",
710
        )
711
712
713
714
        parser.add_argument(
            "--cpu-offload-gb",
            type=int,
            default=ServerArgs.cpu_offload_gb,
715
            help="How many GBs of RAM to reserve for CPU offloading.",
716
        )
717
718
719
720
721
722
        parser.add_argument(
            "--page-size",
            type=int,
            default=ServerArgs.page_size,
            help="The number of tokens in a page.",
        )
723

724
        # Other runtime options
Lianmin Zheng's avatar
Lianmin Zheng committed
725
        parser.add_argument(
726
            "--tensor-parallel-size",
Lianmin Zheng's avatar
Lianmin Zheng committed
727
            "--tp-size",
Lianmin Zheng's avatar
Lianmin Zheng committed
728
            type=int,
Lianmin Zheng's avatar
Lianmin Zheng committed
729
            default=ServerArgs.tp_size,
730
            help="The tensor parallelism size.",
731
        )
732
733
734
735
736
737
738
739
740
741
742
743
744
        parser.add_argument(
            "--pipeline-parallel-size",
            "--pp-size",
            type=int,
            default=ServerArgs.pp_size,
            help="The pipeline parallelism size.",
        )
        parser.add_argument(
            "--max-micro-batch-size",
            type=int,
            default=ServerArgs.max_micro_batch_size,
            help="The maximum micro batch size in pipeline parallelism.",
        )
745
746
747
        parser.add_argument(
            "--stream-interval",
            type=int,
Lianmin Zheng's avatar
Lianmin Zheng committed
748
            default=ServerArgs.stream_interval,
749
            help="The interval (or buffer size) for streaming in terms of the token length. A smaller value makes streaming smoother, while a larger value makes the throughput higher",
750
        )
751
752
753
754
755
        parser.add_argument(
            "--stream-output",
            action="store_true",
            help="Whether to output as a sequence of disjoint segments.",
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
756
757
758
759
        parser.add_argument(
            "--random-seed",
            type=int,
            default=ServerArgs.random_seed,
760
            help="The random seed.",
Lianmin Zheng's avatar
Lianmin Zheng committed
761
        )
762
763
764
765
766
767
        parser.add_argument(
            "--constrained-json-whitespace-pattern",
            type=str,
            default=ServerArgs.constrained_json_whitespace_pattern,
            help=r"Regex pattern for syntactic whitespaces allowed in JSON constrained output. For example, to allow the model generate consecutive whitespaces, set the pattern to [\n\t ]*",
        )
768
769
770
771
772
773
        parser.add_argument(
            "--watchdog-timeout",
            type=float,
            default=ServerArgs.watchdog_timeout,
            help="Set watchdog timeout in seconds. If a forward batch takes longer than this, the server will crash to prevent hanging.",
        )
774
775
776
777
778
779
        parser.add_argument(
            "--dist-timeout",
            type=int,
            default=ServerArgs.dist_timeout,
            help="Set timeout for torch.distributed initialization.",
        )
780
781
782
783
        parser.add_argument(
            "--download-dir",
            type=str,
            default=ServerArgs.download_dir,
784
            help="Model download directory for huggingface.",
785
        )
786
787
788
789
790
791
        parser.add_argument(
            "--base-gpu-id",
            type=int,
            default=ServerArgs.base_gpu_id,
            help="The base GPU ID to start allocating GPUs from. Useful when running multiple instances on the same machine.",
        )
792
793
794
795
796
797
        parser.add_argument(
            "--gpu-id-step",
            type=int,
            default=ServerArgs.gpu_id_step,
            help="The delta between consecutive GPU IDs that are used. For example, setting it to 2 will use GPU 0,2,4,...",
        )
798
799

        # Logging
Lianmin Zheng's avatar
Lianmin Zheng committed
800
801
802
803
        parser.add_argument(
            "--log-level",
            type=str,
            default=ServerArgs.log_level,
804
            help="The logging level of all loggers.",
Lianmin Zheng's avatar
Lianmin Zheng committed
805
        )
806
        parser.add_argument(
807
808
809
810
            "--log-level-http",
            type=str,
            default=ServerArgs.log_level_http,
            help="The logging level of HTTP server. If not set, reuse --log-level by default.",
811
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
812
        parser.add_argument(
813
            "--log-requests",
Lianmin Zheng's avatar
Lianmin Zheng committed
814
            action="store_true",
815
816
817
818
819
820
821
822
            help="Log metadata, inputs, outputs of all requests. The verbosity is decided by --log-requests-level",
        )
        parser.add_argument(
            "--log-requests-level",
            type=int,
            default=0,
            help="0: Log metadata. 1. Log metadata and partial input/output. 2. Log every input/output.",
            choices=[0, 1, 2],
Lianmin Zheng's avatar
Lianmin Zheng committed
823
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
824
825
826
        parser.add_argument(
            "--show-time-cost",
            action="store_true",
Ying Sheng's avatar
Ying Sheng committed
827
            help="Show time cost of custom marks.",
Lianmin Zheng's avatar
Lianmin Zheng committed
828
        )
829
830
831
832
833
        parser.add_argument(
            "--enable-metrics",
            action="store_true",
            help="Enable log prometheus metrics.",
        )
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
        parser.add_argument(
            "--bucket-time-to-first-token",
            type=float,
            nargs="+",
            default=ServerArgs.bucket_time_to_first_token,
            help="The buckets of time to first token, specified as a list of floats.",
        )
        parser.add_argument(
            "--bucket-inter-token-latency",
            type=float,
            nargs="+",
            default=ServerArgs.bucket_inter_token_latency,
            help="The buckets of inter-token latency, specified as a list of floats.",
        )
        parser.add_argument(
            "--bucket-e2e-request-latency",
            type=float,
            nargs="+",
            default=ServerArgs.bucket_e2e_request_latency,
            help="The buckets of end-to-end request latency, specified as a list of floats.",
        )
        parser.add_argument(
            "--collect-tokens-histogram",
            action="store_true",
            default=ServerArgs.collect_tokens_histogram,
            help="Collect prompt/generation tokens histogram.",
        )
861
862
863
864
865
866
        parser.add_argument(
            "--kv-events-config",
            type=str,
            default=None,
            help="Config in json format for NVIDIA dynamo KV event publishing. Publishing will be enabled if this flag is used.",
        )
867
868
869
870
        parser.add_argument(
            "--decode-log-interval",
            type=int,
            default=ServerArgs.decode_log_interval,
871
            help="The log interval of decode batch.",
872
        )
873
874
875
876
877
878
        parser.add_argument(
            "--enable-request-time-stats-logging",
            action="store_true",
            default=ServerArgs.enable_request_time_stats_logging,
            help="Enable per request time stats logging",
        )
879

880
        # API related
Liangsheng Yin's avatar
Liangsheng Yin committed
881
882
883
884
        parser.add_argument(
            "--api-key",
            type=str,
            default=ServerArgs.api_key,
885
            help="Set API key of the server. It is also used in the OpenAI API compatible server.",
Liangsheng Yin's avatar
Liangsheng Yin committed
886
        )
887
        parser.add_argument(
888
            "--file-storage-path",
889
            type=str,
890
            default=ServerArgs.file_storage_path,
891
892
            help="The path of the file storage in backend.",
        )
893
894
895
896
897
        parser.add_argument(
            "--enable-cache-report",
            action="store_true",
            help="Return number of cached tokens in usage.prompt_tokens_details for each openai request.",
        )
Xihuai Wang's avatar
Xihuai Wang committed
898
899
900
901
902
903
904
        parser.add_argument(
            "--reasoning-parser",
            type=str,
            choices=list(ReasoningParser.DetectorMap.keys()),
            default=ServerArgs.reasoning_parser,
            help=f"Specify the parser for reasoning models, supported parsers are: {list(ReasoningParser.DetectorMap.keys())}.",
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
905

906
907
        # Data parallelism
        parser.add_argument(
908
            "--data-parallel-size",
909
910
911
            "--dp-size",
            type=int,
            default=ServerArgs.dp_size,
912
            help="The data parallelism size.",
913
914
915
916
917
        )
        parser.add_argument(
            "--load-balance-method",
            type=str,
            default=ServerArgs.load_balance_method,
918
            help="The load balancing strategy for data parallelism.",
919
920
921
922
923
            choices=[
                "round_robin",
                "shortest_queue",
            ],
        )
924

xiaobochen's avatar
xiaobochen committed
925
926
927
928
929
930
931
932
        # Expert parallelism
        parser.add_argument(
            "--expert-parallel-size",
            "--ep-size",
            type=int,
            default=ServerArgs.ep_size,
            help="The expert parallelism size.",
        )
933

934
        # Multi-node distributed serving
935
        parser.add_argument(
936
            "--dist-init-addr",
937
            "--nccl-init-addr",  # For backward compatibility. This will be removed in the future.
938
            type=str,
939
            help="The host address for initializing distributed backend (e.g., `192.168.0.2:25000`).",
940
941
        )
        parser.add_argument(
Liangsheng Yin's avatar
Liangsheng Yin committed
942
            "--nnodes", type=int, default=ServerArgs.nnodes, help="The number of nodes."
943
        )
944
945
946
        parser.add_argument(
            "--node-rank", type=int, default=ServerArgs.node_rank, help="The node rank."
        )
947

Lianmin Zheng's avatar
Lianmin Zheng committed
948
949
950
951
952
953
954
        # Model override args
        parser.add_argument(
            "--json-model-override-args",
            type=str,
            help="A dictionary in JSON string format used to override default model configurations.",
            default=ServerArgs.json_model_override_args,
        )
955
956
957
958
959
        parser.add_argument(
            "--preferred-sampling-params",
            type=str,
            help="json-formatted sampling settings that will be returned in /get_model_info",
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
960

961
962
963
964
965
966
967
        # LoRA
        parser.add_argument(
            "--lora-paths",
            type=str,
            nargs="*",
            default=None,
            action=LoRAPathAction,
968
            help="The list of LoRA adapters. You can provide a list of either path in str or renamed path in the format {name}={path}.",
969
970
971
972
973
        )
        parser.add_argument(
            "--max-loras-per-batch",
            type=int,
            default=8,
974
975
976
977
978
979
980
            help="Maximum number of adapters for a running batch, include base-only request.",
        )
        parser.add_argument(
            "--lora-backend",
            type=str,
            default="triton",
            help="Choose the kernel backend for multi-LoRA serving.",
981
982
983
        )

        # Kernel backend
984
985
986
        parser.add_argument(
            "--attention-backend",
            type=str,
987
            choices=[
988
                "aiter",
989
990
991
992
993
994
995
                "flashinfer",
                "triton",
                "torch_native",
                "fa3",
                "flashmla",
                "cutlass_mla",
            ],
996
997
998
            default=ServerArgs.attention_backend,
            help="Choose the kernels for attention layers.",
        )
999
1000
1001
1002
1003
1004
1005
        parser.add_argument(
            "--sampling-backend",
            type=str,
            choices=["flashinfer", "pytorch"],
            default=ServerArgs.sampling_backend,
            help="Choose the kernels for sampling layers.",
        )
1006
1007
1008
        parser.add_argument(
            "--grammar-backend",
            type=str,
1009
            choices=["xgrammar", "outlines", "llguidance", "none"],
1010
            default=ServerArgs.grammar_backend,
Lianmin Zheng's avatar
Lianmin Zheng committed
1011
            help="Choose the backend for grammar-guided decoding.",
1012
        )
1013
1014
        parser.add_argument(
            "--enable-flashinfer-mla",
1015
1016
            action=DeprecatedAction,
            help="--enable-flashinfer-mla is deprecated. Please use '--attention-backend flashinfer' instead.",
1017
        )
lukec's avatar
lukec committed
1018
1019
        parser.add_argument(
            "--enable-flashmla",
1020
1021
            action=DeprecatedAction,
            help="--enable-flashmla is deprecated. Please use '--attention-backend flashmla' instead.",
lukec's avatar
lukec committed
1022
        )
1023
1024
1025
1026
1027
        parser.add_argument(
            "--flashinfer-mla-disable-ragged",
            action="store_true",
            help="Not using ragged prefill wrapper when running flashinfer mla",
        )
1028

1029
1030
1031
1032
        # Speculative decoding
        parser.add_argument(
            "--speculative-algorithm",
            type=str,
James Liu's avatar
James Liu committed
1033
            choices=["EAGLE", "EAGLE3", "NEXTN"],
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
            help="Speculative algorithm.",
        )
        parser.add_argument(
            "--speculative-draft-model-path",
            type=str,
            help="The path of the draft model weights. This can be a local folder or a Hugging Face repo ID.",
        )
        parser.add_argument(
            "--speculative-num-steps",
            type=int,
            help="The number of steps sampled from draft model in Speculative Decoding.",
            default=ServerArgs.speculative_num_steps,
        )
        parser.add_argument(
            "--speculative-eagle-topk",
            type=int,
1050
            help="The number of tokens sampled from the draft model in eagle2 each step.",
1051
1052
            default=ServerArgs.speculative_eagle_topk,
        )
1053
1054
1055
        parser.add_argument(
            "--speculative-num-draft-tokens",
            type=int,
1056
            help="The number of tokens sampled from the draft model in Speculative Decoding.",
1057
1058
            default=ServerArgs.speculative_num_draft_tokens,
        )
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
        parser.add_argument(
            "--speculative-accept-threshold-single",
            type=float,
            help="Accept a draft token if its probability in the target model is greater than this threshold.",
            default=ServerArgs.speculative_accept_threshold_single,
        )
        parser.add_argument(
            "--speculative-accept-threshold-acc",
            type=float,
            help="The accept probability of a draft token is raised from its target probability p to min(1, p / threshold_acc).",
            default=ServerArgs.speculative_accept_threshold_acc,
        )
1071
1072
1073
1074
1075
1076
        parser.add_argument(
            "--speculative-token-map",
            type=str,
            help="The path of the draft model's small vocab table.",
            default=ServerArgs.speculative_token_map,
        )
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114

        # Double Sparsity
        parser.add_argument(
            "--enable-double-sparsity",
            action="store_true",
            help="Enable double sparsity attention",
        )
        parser.add_argument(
            "--ds-channel-config-path",
            type=str,
            default=ServerArgs.ds_channel_config_path,
            help="The path of the double sparsity channel config",
        )
        parser.add_argument(
            "--ds-heavy-channel-num",
            type=int,
            default=ServerArgs.ds_heavy_channel_num,
            help="The number of heavy channels in double sparsity attention",
        )
        parser.add_argument(
            "--ds-heavy-token-num",
            type=int,
            default=ServerArgs.ds_heavy_token_num,
            help="The number of heavy tokens in double sparsity attention",
        )
        parser.add_argument(
            "--ds-heavy-channel-type",
            type=str,
            default=ServerArgs.ds_heavy_channel_type,
            help="The type of heavy channels in double sparsity attention",
        )
        parser.add_argument(
            "--ds-sparse-decode-threshold",
            type=int,
            default=ServerArgs.ds_sparse_decode_threshold,
            help="The type of heavy channels in double sparsity attention",
        )

1115
        # Optimization/debug options
Liangsheng Yin's avatar
Liangsheng Yin committed
1116
        parser.add_argument(
Lianmin Zheng's avatar
Lianmin Zheng committed
1117
            "--disable-radix-cache",
Liangsheng Yin's avatar
Liangsheng Yin committed
1118
            action="store_true",
Ying Sheng's avatar
Ying Sheng committed
1119
            help="Disable RadixAttention for prefix caching.",
Liangsheng Yin's avatar
Liangsheng Yin committed
1120
        )
1121
1122
1123
        parser.add_argument(
            "--disable-cuda-graph",
            action="store_true",
1124
            help="Disable cuda graph.",
1125
        )
1126
        parser.add_argument(
1127
1128
            "--disable-cuda-graph-padding",
            action="store_true",
1129
            help="Disable cuda graph when padding is needed. Still uses cuda graph when padding is not needed.",
1130
        )
1131
1132
1133
1134
1135
        parser.add_argument(
            "--enable-nccl-nvls",
            action="store_true",
            help="Enable NCCL NVLS for prefill heavy requests when available.",
        )
1136
1137
1138
1139
1140
        parser.add_argument(
            "--enable-tokenizer-batch-encode",
            action="store_true",
            help="Enable batch tokenization for improved performance when processing multiple text inputs. Do not use with image inputs, pre-tokenized input_ids, or input_embeds.",
        )
1141
        parser.add_argument(
1142
            "--disable-outlines-disk-cache",
1143
            action="store_true",
1144
            help="Disable disk cache of outlines to avoid possible crashes related to file system or high concurrency.",
1145
        )
1146
1147
1148
1149
1150
        parser.add_argument(
            "--disable-custom-all-reduce",
            action="store_true",
            help="Disable the custom all-reduce kernel and fall back to NCCL.",
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
1151
        parser.add_argument(
1152
            "--disable-overlap-schedule",
Lianmin Zheng's avatar
Lianmin Zheng committed
1153
            action="store_true",
1154
            help="Disable the overlap scheduler, which overlaps the CPU scheduler with GPU model worker.",
Lianmin Zheng's avatar
Lianmin Zheng committed
1155
        )
1156
1157
1158
        parser.add_argument(
            "--enable-mixed-chunk",
            action="store_true",
1159
            help="Enabling mixing prefill and decode in a batch when using chunked prefill.",
1160
        )
Ke Bao's avatar
Ke Bao committed
1161
1162
1163
        parser.add_argument(
            "--enable-dp-attention",
            action="store_true",
1164
            help="Enabling data parallelism for attention and tensor parallelism for FFN. The dp size should be equal to the tp size. Currently DeepSeek-V2 and Qwen 2/3 MoE models are supported.",
Ke Bao's avatar
Ke Bao committed
1165
        )
1166
1167
1168
1169
1170
        parser.add_argument(
            "--enable-dp-lm-head",
            action="store_true",
            help="Enable vocabulary parallel across the attention TP group to avoid all-gather across DP groups, optimizing performance under DP attention.",
        )
xiaobochen's avatar
xiaobochen committed
1171
1172
1173
        parser.add_argument(
            "--enable-ep-moe",
            action="store_true",
1174
            help="Enabling expert parallelism for moe. The ep size is equal to the tp size.",
xiaobochen's avatar
xiaobochen committed
1175
        )
1176
1177
1178
1179
1180
        parser.add_argument(
            "--enable-two-batch-overlap",
            action="store_true",
            help="Enabling two micro batches to overlap.",
        )
1181
1182
1183
        parser.add_argument(
            "--enable-torch-compile",
            action="store_true",
1184
1185
            help="Optimize the model with torch.compile. Experimental feature.",
        )
1186
        parser.add_argument(
1187
            "--torch-compile-max-bs",
1188
            type=int,
1189
            default=ServerArgs.torch_compile_max_bs,
1190
1191
            help="Set the maximum batch size when using torch compile.",
        )
1192
        parser.add_argument(
1193
            "--cuda-graph-max-bs",
1194
            type=int,
1195
            default=ServerArgs.cuda_graph_max_bs,
1196
            help="Set the maximum batch size for cuda graph. It will extend the cuda graph capture batch size to this value.",
1197
        )
1198
1199
1200
1201
        parser.add_argument(
            "--cuda-graph-bs",
            type=int,
            nargs="+",
1202
            help="Set the list of batch sizes for cuda graph.",
1203
        )
1204
1205
1206
1207
        parser.add_argument(
            "--torchao-config",
            type=str,
            default=ServerArgs.torchao_config,
1208
            help="Optimize the model with torchao. Experimental feature. Current choices are: int8dq, int8wo, int4wo-<group_size>, fp8wo, fp8dq-per_tensor, fp8dq-per_row",
1209
        )
1210
1211
1212
1213
1214
        parser.add_argument(
            "--enable-nan-detection",
            action="store_true",
            help="Enable the NaN detection for debugging purposes.",
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
1215
        parser.add_argument(
1216
            "--enable-p2p-check",
Lianmin Zheng's avatar
Lianmin Zheng committed
1217
            action="store_true",
1218
            help="Enable P2P check for GPU access, otherwise the p2p access is allowed by default.",
Lianmin Zheng's avatar
Lianmin Zheng committed
1219
        )
1220
        parser.add_argument(
1221
            "--triton-attention-reduce-in-fp32",
1222
            action="store_true",
1223
            help="Cast the intermediate attention results to fp32 to avoid possible crashes related to fp16."
1224
            "This only affects Triton attention kernels.",
1225
        )
1226
1227
1228
1229
1230
1231
        parser.add_argument(
            "--triton-attention-num-kv-splits",
            type=int,
            default=ServerArgs.triton_attention_num_kv_splits,
            help="The number of KV splits in flash decoding Triton kernel. Larger value is better in longer context scenarios. The default value is 8.",
        )
1232
1233
1234
1235
1236
1237
1238
1239
        parser.add_argument(
            "--num-continuous-decode-steps",
            type=int,
            default=ServerArgs.num_continuous_decode_steps,
            help="Run multiple continuous decoding steps to reduce scheduling overhead. "
            "This can potentially increase throughput but may also increase time-to-first-token latency. "
            "The default value is 1, meaning only run one decoding step at a time.",
        )
1240
1241
1242
1243
1244
        parser.add_argument(
            "--delete-ckpt-after-loading",
            action="store_true",
            help="Delete the model checkpoint after loading the model.",
        )
1245
1246
1247
1248
1249
        parser.add_argument(
            "--enable-memory-saver",
            action="store_true",
            help="Allow saving memory using release_memory_occupation and resume_memory_occupation",
        )
1250
1251
1252
1253
1254
        parser.add_argument(
            "--allow-auto-truncate",
            action="store_true",
            help="Allow automatically truncating requests that exceed the maximum input length instead of returning an error.",
        )
1255
1256
1257
1258
1259
        parser.add_argument(
            "--enable-custom-logit-processor",
            action="store_true",
            help="Enable users to pass custom logit processors to the server (disabled by default for security)",
        )
YAMY's avatar
YAMY committed
1260
1261
1262
        parser.add_argument(
            "--tool-call-parser",
            type=str,
1263
            choices=["qwen25", "mistral", "llama3", "deepseekv3", "pythonic"],
YAMY's avatar
YAMY committed
1264
            default=ServerArgs.tool_call_parser,
1265
            help="Specify the parser for handling tool-call interactions. Options include: 'qwen25', 'mistral', 'llama3', 'deepseekv3', and 'pythonic'.",
YAMY's avatar
YAMY committed
1266
        )
1267
1268
1269
1270
1271
        parser.add_argument(
            "--enable-hierarchical-cache",
            action="store_true",
            help="Enable hierarchical cache",
        )
1272
1273
1274
1275
1276
1277
        parser.add_argument(
            "--hicache-ratio",
            type=float,
            default=ServerArgs.hicache_ratio,
            help="The ratio of the size of host KV cache memory pool to the size of device pool.",
        )
Zhiqiang Xie's avatar
Zhiqiang Xie committed
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
        parser.add_argument(
            "--hicache-size",
            type=int,
            default=ServerArgs.hicache_size,
            help="The size of host KV cache memory pool in gigabytes, which will override the hicache_ratio if set.",
        )
        parser.add_argument(
            "--hicache-write-policy",
            type=str,
            choices=["write_back", "write_through", "write_through_selective"],
            default=ServerArgs.hicache_write_policy,
            help="The write policy of hierarchical cache.",
        )
1291
1292
1293
1294
1295
        parser.add_argument(
            "--enable-deepep-moe",
            action="store_true",
            help="Enabling DeepEP MoE implementation for EP MoE.",
        )
1296
1297
1298
1299
1300
1301
        parser.add_argument(
            "--moe-dense-tp-size",
            type=int,
            default=ServerArgs.moe_dense_tp_size,
            help="TP size for MoE dense MLP layers. This flag is useful when, with large TP size, there are errors caused by weights in MLP layers having dimension smaller than the min dimension GEMM supports.",
        )
1302
1303
1304
1305
        parser.add_argument(
            "--deepep-mode",
            type=str,
            choices=["normal", "low_latency", "auto"],
1306
            default="auto",
1307
1308
            help="Select the mode when enable DeepEP MoE, could be `normal`, `low_latency` or `auto`. Default is `auto`, which means `low_latency` for decode batch and `normal` for prefill batch.",
        )
1309
1310
1311
1312
1313
1314
        parser.add_argument(
            "--ep-num-redundant-experts",
            type=int,
            default=ServerArgs.ep_num_redundant_experts,
            help="Allocate this number of redundant experts in expert parallel.",
        )
1315
1316
1317
1318
1319
1320
        parser.add_argument(
            "--ep-dispatch-algorithm",
            type=str,
            default=ServerArgs.ep_dispatch_algorithm,
            help="The algorithm to choose ranks for redundant experts in expert parallel.",
        )
1321
1322
1323
1324
1325
1326
        parser.add_argument(
            "--init-expert-location",
            type=str,
            default=ServerArgs.init_expert_location,
            help="Initial location of EP experts.",
        )
1327
1328
1329
1330
1331
        parser.add_argument(
            "--enable-eplb",
            action="store_true",
            help="Enable EPLB algorithm",
        )
1332
1333
1334
1335
1336
1337
        parser.add_argument(
            "--eplb-algorithm",
            type=str,
            default=ServerArgs.eplb_algorithm,
            help="Chosen EPLB algorithm",
        )
1338
1339
1340
1341
1342
1343
        parser.add_argument(
            "--eplb-rebalance-num-iterations",
            type=int,
            default=ServerArgs.eplb_rebalance_num_iterations,
            help="Number of iterations to automatically trigger a EPLB re-balance.",
        )
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
        parser.add_argument(
            "--expert-distribution-recorder-mode",
            type=str,
            default=ServerArgs.expert_distribution_recorder_mode,
            help="Mode of expert distribution recorder.",
        )
        parser.add_argument(
            "--expert-distribution-recorder-buffer-size",
            type=int,
            default=ServerArgs.expert_distribution_recorder_buffer_size,
            help="Circular buffer size of expert distribution recorder. Set to -1 to denote infinite buffer.",
        )
1356
1357
1358
1359
1360
        parser.add_argument(
            "--enable-expert-distribution-metrics",
            action="store_true",
            help="Enable logging metrics for expert balancedness",
        )
1361
1362
1363
1364
        parser.add_argument(
            "--deepep-config",
            type=str,
            default=ServerArgs.deepep_config,
1365
            help="Tuned DeepEP config suitable for your own cluster. It can be either a string with JSON content or a file path.",
1366
        )
1367

1368
1369
1370
        parser.add_argument(
            "--n-share-experts-fusion",
            type=int,
1371
            default=0,
1372
            help="The number of shared_experts need to be replicated to fuse with normal experts in deepseek v3/r1, "
1373
            "set it to tp_size can get best optimized performance. Note that for architectures with SM==90, we have enabled the shared experts fusion optimization by default for DeepSeek V3/R1, with n_share_experts_fusion automatically set to the TP size.",
1374
        )
1375
1376
1377
1378
1379
        parser.add_argument(
            "--disable-chunked-prefix-cache",
            action="store_true",
            help="Disable chunked prefix cache feature for deepseek, which should save overhead for short sequences.",
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
1380
1381
1382
1383
1384
        parser.add_argument(
            "--disable-fast-image-processor",
            action="store_true",
            help="Adopt base image processor instead of fast image processor.",
        )
1385

1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
        # Server warmups
        parser.add_argument(
            "--warmups",
            type=str,
            required=False,
            help="Specify custom warmup functions (csv) to run before server starts eg. --warmups=warmup_name1,warmup_name2 "
            "will run the functions `warmup_name1` and `warmup_name2` specified in warmup.py before the server starts listening for requests",
        )

        # Debug tensor dumps
        parser.add_argument(
            "--debug-tensor-dump-output-folder",
            type=str,
            default=ServerArgs.debug_tensor_dump_output_folder,
            help="The output folder for dumping tensors.",
        )
        parser.add_argument(
            "--debug-tensor-dump-input-file",
            type=str,
            default=ServerArgs.debug_tensor_dump_input_file,
            help="The input filename for dumping tensors",
        )
        parser.add_argument(
            "--debug-tensor-dump-inject",
            type=str,
            default=ServerArgs.debug_tensor_dump_inject,
            help="Inject the outputs from jax as the input of every layer.",
        )

Byron Hsu's avatar
Byron Hsu committed
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
        # Disaggregation
        parser.add_argument(
            "--disaggregation-mode",
            type=str,
            default="null",
            choices=["null", "prefill", "decode"],
            help='Only used for PD disaggregation. "prefill" for prefill-only server, and "decode" for decode-only server. If not specified, it is not PD disaggregated',
        )
        parser.add_argument(
            "--disaggregation-bootstrap-port",
            type=int,
            default=ServerArgs.disaggregation_bootstrap_port,
            help="Bootstrap server port on the prefill server. Default is 8998.",
        )
1429
1430
1431
1432
        parser.add_argument(
            "--disaggregation-transfer-backend",
            type=str,
            default=ServerArgs.disaggregation_transfer_backend,
1433
            choices=["mooncake", "nixl"],
1434
1435
            help="The backend for disaggregation transfer. Default is mooncake.",
        )
1436
1437
1438
1439
        parser.add_argument(
            "--disaggregation-ib-device",
            type=str,
            default=ServerArgs.disaggregation_ib_device,
1440
1441
1442
            help="The InfiniBand devices for disaggregation transfer, accepts single device (e.g., --disaggregation-ib-device mlx5_0) "
            "or multiple comma-separated devices (e.g., --disaggregation-ib-device mlx5_0,mlx5_1). "
            "Default is None, which triggers automatic device detection when mooncake backend is enabled.",
1443
        )
1444
1445
1446
1447
1448
1449
        parser.add_argument(
            "--pdlb-url",
            type=str,
            default=None,
            help="The URL of the PD disaggregation load balancer. If set, the prefill/decode server will register with the load balancer.",
        )
Byron Hsu's avatar
Byron Hsu committed
1450

1451
1452
1453
1454
1455
1456
1457
1458
        parser.add_argument(
            "--mm-attention-backend",
            type=str,
            choices=["sdpa", "fa3", "triton_attn"],
            default=ServerArgs.mm_attention_backend,
            help="Set multimodal attention backend.",
        )

Lianmin Zheng's avatar
Lianmin Zheng committed
1459
1460
    @classmethod
    def from_cli_args(cls, args: argparse.Namespace):
1461
        args.tp_size = args.tensor_parallel_size
1462
        args.pp_size = args.pipeline_parallel_size
1463
        args.dp_size = args.data_parallel_size
xiaobochen's avatar
xiaobochen committed
1464
        args.ep_size = args.expert_parallel_size
Lianmin Zheng's avatar
Lianmin Zheng committed
1465
1466
1467
1468
        attrs = [attr.name for attr in dataclasses.fields(cls)]
        return cls(**{attr: getattr(args, attr) for attr in attrs})

    def url(self):
1469
        if is_valid_ipv6_address(self.host):
1470
1471
1472
            return f"http://[{self.host}]:{self.port}"
        else:
            return f"http://{self.host}:{self.port}"
Lianmin Zheng's avatar
Lianmin Zheng committed
1473

1474
1475
    def check_server_args(self):
        assert (
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
            self.tp_size * self.pp_size
        ) % self.nnodes == 0, "tp_size must be divisible by number of nodes"

        # FIXME pp constraints
        if self.pp_size > 1:
            assert (
                self.disable_overlap_schedule
                and self.speculative_algorithm is None
                and not self.enable_mixed_chunk
            ), "Pipeline parallelism is not compatible with overlap schedule, speculative decoding, mixed chunked prefill."

1487
        assert not (
1488
1489
            self.dp_size > 1 and self.nnodes != 1 and not self.enable_dp_attention
        ), "multi-node data parallel is not supported unless dp attention!"
1490
1491
1492
1493
        assert (
            self.max_loras_per_batch > 0
            # FIXME
            and (self.lora_paths is None or self.disable_radix_cache)
1494
        ), "compatibility of lora and radix attention is in progress"
1495
        assert self.base_gpu_id >= 0, "base_gpu_id must be non-negative"
1496
        assert self.gpu_id_step >= 1, "gpu_id_step must be positive"
1497

1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
        if isinstance(self.lora_paths, list):
            lora_paths = self.lora_paths
            self.lora_paths = {}
            for lora_path in lora_paths:
                if "=" in lora_path:
                    name, path = lora_path.split("=", 1)
                    self.lora_paths[name] = path
                else:
                    self.lora_paths[lora_path] = lora_path

Lianmin Zheng's avatar
Lianmin Zheng committed
1508

Lianmin Zheng's avatar
Lianmin Zheng committed
1509
def prepare_server_args(argv: List[str]) -> ServerArgs:
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
    """
    Prepare the server arguments from the command line arguments.

    Args:
        args: The command line arguments. Typically, it should be `sys.argv[1:]`
            to ensure compatibility with `parse_args` when no arguments are passed.

    Returns:
        The server arguments.
    """
    parser = argparse.ArgumentParser()
    ServerArgs.add_cli_args(parser)
Lianmin Zheng's avatar
Lianmin Zheng committed
1522
    raw_args = parser.parse_args(argv)
1523
1524
1525
1526
    server_args = ServerArgs.from_cli_args(raw_args)
    return server_args


1527
1528
1529
ZMQ_TCP_PORT_DELTA = 233


Lianmin Zheng's avatar
Lianmin Zheng committed
1530
1531
@dataclasses.dataclass
class PortArgs:
1532
1533
1534
1535
1536
1537
    # The ipc filename for tokenizer to receive inputs from detokenizer (zmq)
    tokenizer_ipc_name: str
    # The ipc filename for scheduler (rank 0) to receive inputs from tokenizer (zmq)
    scheduler_input_ipc_name: str
    # The ipc filename for detokenizer to receive inputs from scheduler (zmq)
    detokenizer_ipc_name: str
1538

1539
1540
    # The port for nccl initialization (torch.dist)
    nccl_port: int
1541

1542
1543
1544
    # The ipc filename for rpc call between Engine and Scheduler
    rpc_ipc_name: str

1545
    @staticmethod
1546
    def init_new(server_args, dp_rank: Optional[int] = None) -> "PortArgs":
1547
        port = server_args.port + random.randint(100, 1000)
1548
1549
1550
        while True:
            if is_port_available(port):
                break
TianYu GUO's avatar
TianYu GUO committed
1551
1552
1553
1554
            if port < 60000:
                port += 42
            else:
                port -= 43
1555

1556
1557
1558
1559
1560
1561
1562
        if not server_args.enable_dp_attention:
            # Normal case, use IPC within a single node
            return PortArgs(
                tokenizer_ipc_name=f"ipc://{tempfile.NamedTemporaryFile(delete=False).name}",
                scheduler_input_ipc_name=f"ipc://{tempfile.NamedTemporaryFile(delete=False).name}",
                detokenizer_ipc_name=f"ipc://{tempfile.NamedTemporaryFile(delete=False).name}",
                nccl_port=port,
1563
                rpc_ipc_name=f"ipc://{tempfile.NamedTemporaryFile(delete=False).name}",
1564
1565
1566
1567
1568
            )
        else:
            # DP attention. Use TCP + port to handle both single-node and multi-node.
            if server_args.nnodes == 1 and server_args.dist_init_addr is None:
                dist_init_addr = ("127.0.0.1", server_args.port + ZMQ_TCP_PORT_DELTA)
Vincent's avatar
Vincent committed
1569
1570
1571
            elif server_args.dist_init_addr.startswith("["):  # ipv6 address
                port_num, host = configure_ipv6(server_args.dist_init_addr)
                dist_init_addr = (host, str(port_num))
1572
1573
            else:
                dist_init_addr = server_args.dist_init_addr.split(":")
Vincent's avatar
Vincent committed
1574

1575
1576
1577
1578
1579
1580
1581
1582
            assert (
                len(dist_init_addr) == 2
            ), "please provide --dist-init-addr as host:port of head node"

            dist_init_host, dist_init_port = dist_init_addr
            port_base = int(dist_init_port) + 1
            if dp_rank is None:
                scheduler_input_port = (
1583
                    port_base + 3
1584
                )  # TokenizerManager to DataParallelController
1585
            else:
1586
                scheduler_input_port = port_base + 3 + 1 + dp_rank
1587
1588
1589
1590
1591
1592

            return PortArgs(
                tokenizer_ipc_name=f"tcp://{dist_init_host}:{port_base}",
                scheduler_input_ipc_name=f"tcp://{dist_init_host}:{scheduler_input_port}",
                detokenizer_ipc_name=f"tcp://{dist_init_host}:{port_base + 1}",
                nccl_port=port,
1593
                rpc_ipc_name=f"tcp://{dist_init_host}:{port_base + 2}",
1594
            )
1595

1596
1597
1598
1599
1600
1601
1602
1603
1604
1605

class LoRAPathAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
        setattr(namespace, self.dest, {})
        for lora_path in values:
            if "=" in lora_path:
                name, path = lora_path.split("=", 1)
                getattr(namespace, self.dest)[name] = path
            else:
                getattr(namespace, self.dest)[lora_path] = lora_path
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615


class DeprecatedAction(argparse.Action):
    def __init__(self, option_strings, dest, nargs=0, **kwargs):
        super(DeprecatedAction, self).__init__(
            option_strings, dest, nargs=nargs, **kwargs
        )

    def __call__(self, parser, namespace, values, option_string=None):
        raise ValueError(self.help)
1616
1617


1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
def get_model_arch(args: ServerArgs):
    hf_config = get_config(
        args.model_path,
        trust_remote_code=args.trust_remote_code,
        revision=args.revision,
        model_override_args=json.loads(args.json_model_override_args),
    )
    return hf_config.architectures[0]


def auto_choose_speculative_params(arch: str):
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
    """
    Automatically choose the parameters for speculative decoding.

    You can tune them on your own models and prompts with scripts/playground/bench_speculative.py
    """
    if arch in ["LlamaForCausalLM"]:
        # The default value for llama
        return (5, 4, 8)
    elif arch in ["DeepseekV3ForCausalLM", "DeepseekV2ForCausalLM"]:
        # The default value for deepseek
        return (5, 4, 8)
    elif arch in ["Grok1ForCausalLM", "Grok1VForCausalLM"]:
        return (5, 4, 8)
    else:
        # The default value for all other models
        return (5, 4, 8)