cli_args.py 12 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
3
4
5
6
7
8
9
10
"""
This file contains the command line arguments for the vLLM's
OpenAI-compatible server. It is kept in a separate file for documentation
purposes.
"""

import argparse
import json
import ssl
11
12
from collections.abc import Sequence
from typing import Optional, Union, get_args
13

14
import vllm.envs as envs
15
from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
16
17
from vllm.entrypoints.chat_utils import (ChatTemplateContentFormatOption,
                                         validate_chat_template)
18
from vllm.entrypoints.openai.serving_models import (LoRAModulePath,
19
                                                    PromptAdapterPath)
20
from vllm.entrypoints.openai.tool_parsers import ToolParserManager
21
from vllm.logger import init_logger
22
from vllm.utils import FlexibleArgumentParser
23

24
25
logger = init_logger(__name__)

26
27
28

class LoRAParserAction(argparse.Action):

29
30
31
32
33
34
35
36
37
38
39
40
    def __call__(
        self,
        parser: argparse.ArgumentParser,
        namespace: argparse.Namespace,
        values: Optional[Union[str, Sequence[str]]],
        option_string: Optional[str] = None,
    ):
        if values is None:
            values = []
        if isinstance(values, str):
            raise TypeError("Expected values to be a list")

41
        lora_list: list[LoRAModulePath] = []
42
        for item in values:
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
            if item in [None, '']:  # Skip if item is None or empty string
                continue
            if '=' in item and ',' not in item:  # Old format: name=path
                name, path = item.split('=')
                lora_list.append(LoRAModulePath(name, path))
            else:  # Assume JSON format
                try:
                    lora_dict = json.loads(item)
                    lora = LoRAModulePath(**lora_dict)
                    lora_list.append(lora)
                except json.JSONDecodeError:
                    parser.error(
                        f"Invalid JSON format for --lora-modules: {item}")
                except TypeError as e:
                    parser.error(
                        f"Invalid fields for --lora-modules: {item} - {str(e)}"
                    )
60
61
62
        setattr(namespace, self.dest, lora_list)


63
64
class PromptAdapterParserAction(argparse.Action):

65
66
67
68
69
70
71
72
73
74
75
76
    def __call__(
        self,
        parser: argparse.ArgumentParser,
        namespace: argparse.Namespace,
        values: Optional[Union[str, Sequence[str]]],
        option_string: Optional[str] = None,
    ):
        if values is None:
            values = []
        if isinstance(values, str):
            raise TypeError("Expected values to be a list")

77
        adapter_list: list[PromptAdapterPath] = []
78
79
80
81
82
83
        for item in values:
            name, path = item.split('=')
            adapter_list.append(PromptAdapterPath(name, path))
        setattr(namespace, self.dest, adapter_list)


Ethan Xu's avatar
Ethan Xu committed
84
def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
85
    parser.add_argument("--host",
86
                        type=optional_type(str),
87
                        default=None,
88
89
                        help="Host name.")
    parser.add_argument("--port", type=int, default=8000, help="Port number.")
90
91
92
93
94
    parser.add_argument(
        "--uvicorn-log-level",
        type=str,
        default="info",
        choices=['debug', 'info', 'warning', 'error', 'critical', 'trace'],
95
        help="Log level for uvicorn.")
96
97
98
    parser.add_argument("--disable-uvicorn-access-log",
                        action="store_true",
                        help="Disable uvicorn access log.")
99
100
    parser.add_argument("--allow-credentials",
                        action="store_true",
101
                        help="Allow credentials.")
102
103
104
    parser.add_argument("--allowed-origins",
                        type=json.loads,
                        default=["*"],
105
                        help="Allowed origins.")
106
107
108
    parser.add_argument("--allowed-methods",
                        type=json.loads,
                        default=["*"],
109
                        help="Allowed methods.")
110
111
112
    parser.add_argument("--allowed-headers",
                        type=json.loads,
                        default=["*"],
113
                        help="Allowed headers.")
114
    parser.add_argument("--api-key",
115
                        type=optional_type(str),
116
117
118
119
120
                        default=None,
                        help="If provided, the server will require this key "
                        "to be presented in the header.")
    parser.add_argument(
        "--lora-modules",
121
        type=optional_type(str),
122
123
124
        default=None,
        nargs='+',
        action=LoRAParserAction,
125
126
        help="LoRA module configurations in either 'name=path' format"
        "or JSON format. "
127
        "Example (old format): ``'name=path'`` "
128
        "Example (new format): "
129
        "``{\"name\": \"name\", \"path\": \"lora_path\", "
130
        "\"base_model_name\": \"id\"}``")
131
132
    parser.add_argument(
        "--prompt-adapters",
133
        type=optional_type(str),
134
135
136
137
138
        default=None,
        nargs='+',
        action=PromptAdapterParserAction,
        help="Prompt adapter configurations in the format name=path. "
        "Multiple adapters can be specified.")
139
    parser.add_argument("--chat-template",
140
                        type=optional_type(str),
141
142
143
                        default=None,
                        help="The file path to the chat template, "
                        "or the template in single-line form "
144
                        "for the specified model.")
145
146
147
148
149
150
151
152
    parser.add_argument(
        '--chat-template-content-format',
        type=str,
        default="auto",
        choices=get_args(ChatTemplateContentFormatOption),
        help='The format to render message content within a chat template.'
        '\n\n'
        '* "string" will render the content as a string. '
153
        'Example: ``"Hello World"``\n'
154
155
        '* "openai" will render the content as a list of dictionaries, '
        'similar to OpenAI schema. '
156
        'Example: ``[{"type": "text", "text": "Hello world!"}]``')
157
    parser.add_argument("--response-role",
158
                        type=optional_type(str),
159
160
                        default="assistant",
                        help="The role name to return if "
161
                        "``request.add_generation_prompt=true``.")
162
    parser.add_argument("--ssl-keyfile",
163
                        type=optional_type(str),
164
                        default=None,
165
                        help="The file path to the SSL key file.")
166
    parser.add_argument("--ssl-certfile",
167
                        type=optional_type(str),
168
                        default=None,
169
                        help="The file path to the SSL cert file.")
170
    parser.add_argument("--ssl-ca-certs",
171
                        type=optional_type(str),
172
                        default=None,
173
                        help="The CA certificates file.")
174
175
176
177
178
    parser.add_argument(
        "--enable-ssl-refresh",
        action="store_true",
        default=False,
        help="Refresh SSL Context when SSL certificate files change")
179
180
181
182
    parser.add_argument(
        "--ssl-cert-reqs",
        type=int,
        default=int(ssl.CERT_NONE),
183
        help="Whether client certificate is required (see stdlib ssl module's)."
184
185
186
    )
    parser.add_argument(
        "--root-path",
187
        type=optional_type(str),
188
        default=None,
189
190
        help="FastAPI root_path when app is behind a path based routing proxy."
    )
191
192
    parser.add_argument(
        "--middleware",
193
        type=optional_type(str),
194
195
196
197
198
199
        action="append",
        default=[],
        help="Additional ASGI middleware to apply to the app. "
        "We accept multiple --middleware arguments. "
        "The value should be an import path. "
        "If a function is provided, vLLM will add it to the server "
200
        "using ``@app.middleware('http')``. "
201
        "If a class is provided, vLLM will add it to the server "
202
        "using ``app.add_middleware()``. ")
203
204
205
    parser.add_argument(
        "--return-tokens-as-token-ids",
        action="store_true",
206
207
208
        help="When ``--max-logprobs`` is specified, represents single tokens "
        " as strings of the form 'token_id:{token_id}' so that tokens "
        "that are not JSON-encodable can be identified.")
209
210
211
212
213
    parser.add_argument(
        "--disable-frontend-multiprocessing",
        action="store_true",
        help="If specified, will run the OpenAI frontend server in the same "
        "process as the model serving engine.")
214
215
216
217
218
    parser.add_argument(
        "--enable-request-id-headers",
        action="store_true",
        help="If specified, API server will add X-Request-Id header to "
        "responses. Caution: this hurts performance at high QPS.")
219
220
221
222
    parser.add_argument(
        "--enable-auto-tool-choice",
        action="store_true",
        default=False,
223
224
        help="Enable auto tool choice for supported models. Use "
        "``--tool-call-parser`` to specify which parser to use.")
225

226
    valid_tool_parsers = ToolParserManager.tool_parsers.keys()
227
228
229
    parser.add_argument(
        "--tool-call-parser",
        type=str,
230
231
        metavar="{" + ",".join(valid_tool_parsers) + "} or name registered in "
        "--tool-parser-plugin",
232
233
234
235
        default=None,
        help=
        "Select the tool call parser depending on the model that you're using."
        " This is used to parse the model-generated tool call into OpenAI API "
236
        "format. Required for ``--enable-auto-tool-choice``.")
237

238
239
240
241
242
243
244
    parser.add_argument(
        "--tool-parser-plugin",
        type=str,
        default="",
        help=
        "Special the tool parser plugin write to parse the model-generated tool"
        " into OpenAI API format, the name register in this plugin can be used "
245
        "in ``--tool-call-parser``.")
246

247
248
249
250
251
252
253
    parser.add_argument(
        "--log-config-file",
        type=str,
        default=envs.VLLM_LOGGING_CONFIG_PATH,
        help="Path to logging config JSON file for both vllm and uvicorn",
    )

254
    parser = AsyncEngineArgs.add_cli_args(parser)
255
256
257
258
259
260

    parser.add_argument('--max-log-len',
                        type=int,
                        default=None,
                        help='Max number of prompt characters or prompt '
                        'ID numbers being printed in log.'
261
                        ' The default of None means unlimited.')
262

263
264
265
266
    parser.add_argument(
        "--disable-fastapi-docs",
        action='store_true',
        default=False,
267
        help="Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint."
268
    )
269
270
271
272
273
    parser.add_argument(
        "--enable-prompt-tokens-details",
        action='store_true',
        default=False,
        help="If set to True, enable prompt_tokens_details in usage.")
274
275
276
277
278
279
280
    parser.add_argument(
        "--enable-server-load-tracking",
        action='store_true',
        default=False,
        help=
        "If set to True, enable tracking server_load_metrics in the app state."
    )
281

282
    return parser
Ethan Xu's avatar
Ethan Xu committed
283
284


285
286
287
288
289
290
291
292
293
294
295
296
def validate_parsed_serve_args(args: argparse.Namespace):
    """Quick checks for model serve args that raise prior to loading."""
    if hasattr(args, "subparser") and args.subparser != "serve":
        return

    # Ensure that the chat template is valid; raises if it likely isn't
    validate_chat_template(args.chat_template)

    # Enable auto tool needs a tool call parser to be valid
    if args.enable_auto_tool_choice and not args.tool_call_parser:
        raise TypeError("Error: --enable-auto-tool-choice requires "
                        "--tool-call-parser")
297
298
299
    if args.enable_prompt_embeds and args.enable_prompt_adapter:
        raise ValueError(
            "Cannot use prompt embeds and prompt adapter at the same time.")
300
301


302
303
304
305
306
307
308
309
310
def log_non_default_args(args: argparse.Namespace):
    non_default_args = {}
    parser = make_arg_parser(FlexibleArgumentParser())
    for arg, default in vars(parser.parse_args([])).items():
        if default != getattr(args, arg):
            non_default_args[arg] = getattr(args, arg)
    logger.info("non-default args: %s", non_default_args)


Ethan Xu's avatar
Ethan Xu committed
311
312
313
314
def create_parser_for_docs() -> FlexibleArgumentParser:
    parser_for_docs = FlexibleArgumentParser(
        prog="-m vllm.entrypoints.openai.api_server")
    return make_arg_parser(parser_for_docs)