api_server.py 24.1 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import asyncio
4
5
import importlib
import inspect
6
import multiprocessing
7
import multiprocessing.forkserver as forkserver
8
import os
9
import signal
10
import socket
11
import tempfile
12
import warnings
13
from argparse import Namespace
14
from collections.abc import AsyncIterator
15
from contextlib import asynccontextmanager
16
from typing import Any
17

18
import uvloop
19
from fastapi import FastAPI, HTTPException
Zhuohan Li's avatar
Zhuohan Li committed
20
21
from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware
22
from starlette.datastructures import State
Zhuohan Li's avatar
Zhuohan Li committed
23

24
import vllm.envs as envs
25
from vllm.config import VllmConfig
Woosuk Kwon's avatar
Woosuk Kwon committed
26
from vllm.engine.arg_utils import AsyncEngineArgs
27
from vllm.engine.protocol import EngineClient
28
from vllm.entrypoints.chat_utils import load_chat_template
29
from vllm.entrypoints.launcher import serve_http
30
from vllm.entrypoints.logger import RequestLogger
31
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
32
from vllm.entrypoints.openai.engine.protocol import GenerationError
33
from vllm.entrypoints.openai.models.protocol import BaseModelPath
34
35
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.openai.server_utils import (
36
37
    engine_error_handler,
    exception_handler,
38
    generation_error_handler,
39
40
41
42
43
    get_uvicorn_log_config,
    http_exception_handler,
    lifespan,
    log_response,
    validation_exception_handler,
44
)
45
from vllm.entrypoints.sagemaker.api_router import sagemaker_standards_bootstrap
46
47
48
49
from vllm.entrypoints.serve.elastic_ep.middleware import (
    ScalingMiddleware,
)
from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
50
51
52
from vllm.entrypoints.utils import (
    cli_env_setup,
    log_non_default_args,
53
    log_version_and_model,
54
    process_lora_modules,
55
)
56
from vllm.logger import init_logger
57
from vllm.reasoning import ReasoningParserManager
58
from vllm.tasks import POOLING_TASKS, SupportedTask
59
from vllm.tool_parsers import ToolParserManager
60
from vllm.tracing import instrument
yhu422's avatar
yhu422 committed
61
from vllm.usage.usage_lib import UsageContext
Cyrus Leung's avatar
Cyrus Leung committed
62
from vllm.utils.argparse_utils import FlexibleArgumentParser
63
from vllm.utils.network_utils import is_valid_ipv6_address
64
from vllm.utils.system_utils import decorate_logs, set_ulimit
65
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
66
from vllm.version import __version__ as VLLM_VERSION
Zhuohan Li's avatar
Zhuohan Li committed
67

68
prometheus_multiproc_dir: tempfile.TemporaryDirectory
69

70
# Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765)
71
logger = init_logger("vllm.entrypoints.openai.api_server")
72

73
74
_FALLBACK_SUPPORTED_TASKS: tuple[SupportedTask, ...] = ("generate",)

75

76
@asynccontextmanager
77
async def build_async_engine_client(
78
    args: Namespace,
79
80
    *,
    usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
81
82
    disable_frontend_multiprocessing: bool | None = None,
    client_config: dict[str, Any] | None = None,
83
) -> AsyncIterator[EngineClient]:
84
85
86
87
    if os.getenv("VLLM_WORKER_MULTIPROC_METHOD") == "forkserver":
        # The executor is expected to be mp.
        # Pre-import heavy modules in the forkserver process
        logger.debug("Setup forkserver with pre-imports")
88
        multiprocessing.set_start_method("forkserver")
89
90
91
92
        multiprocessing.set_forkserver_preload(["vllm.v1.engine.async_llm"])
        forkserver.ensure_running()
        logger.debug("Forkserver setup complete!")

93
    # Context manager to handle engine_client lifecycle
94
95
    # Ensures everything is shutdown and cleaned up on error/exit
    engine_args = AsyncEngineArgs.from_cli_args(args)
96
97
98
    if client_config:
        engine_args._api_process_count = client_config.get("client_count", 1)
        engine_args._api_process_rank = client_config.get("client_index", 0)
99

100
    if disable_frontend_multiprocessing is None:
101
        disable_frontend_multiprocessing = bool(args.disable_frontend_multiprocessing)
102

103
    async with build_async_engine_client_from_engine_args(
104
105
106
107
        engine_args,
        usage_context=usage_context,
        disable_frontend_multiprocessing=disable_frontend_multiprocessing,
        client_config=client_config,
108
    ) as engine:
109
110
111
112
113
114
        yield engine


@asynccontextmanager
async def build_async_engine_client_from_engine_args(
    engine_args: AsyncEngineArgs,
115
116
    *,
    usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
117
    disable_frontend_multiprocessing: bool = False,
118
    client_config: dict[str, Any] | None = None,
119
) -> AsyncIterator[EngineClient]:
120
    """
121
    Create EngineClient, either:
122
123
124
125
126
127
        - in-process using the AsyncLLMEngine Directly
        - multiprocess using AsyncLLMEngine RPC

    Returns the Client or None if the creation failed.
    """

128
129
130
    # Create the EngineConfig (determines if we can use V1).
    vllm_config = engine_args.create_engine_config(usage_context=usage_context)

131
    if disable_frontend_multiprocessing:
132
        logger.warning("V1 is enabled, but got --disable-frontend-multiprocessing.")
133

134
    from vllm.v1.engine.async_llm import AsyncLLM
135

136
    async_llm: AsyncLLM | None = None
137
138
139
140
141
142

    # Don't mutate the input client_config
    client_config = dict(client_config) if client_config else {}
    client_count = client_config.pop("client_count", 1)
    client_index = client_config.pop("client_index", 0)

143
144
145
146
147
    try:
        async_llm = AsyncLLM.from_vllm_config(
            vllm_config=vllm_config,
            usage_context=usage_context,
            enable_log_requests=engine_args.enable_log_requests,
148
            aggregate_engine_logging=engine_args.aggregate_engine_logging,
149
150
151
            disable_log_stats=engine_args.disable_log_stats,
            client_addresses=client_config,
            client_count=client_count,
152
153
            client_index=client_index,
        )
154
155

        # Don't keep the dummy data in memory
156
        assert async_llm is not None
157
158
159
160
161
162
        await async_llm.reset_mm_cache()

        yield async_llm
    finally:
        if async_llm:
            async_llm.shutdown()
163
164


165
166
167
168
169
170
171
172
173
174
175
176
177
def build_app(
    args: Namespace, supported_tasks: tuple["SupportedTask", ...] | None = None
) -> FastAPI:
    if supported_tasks is None:
        warnings.warn(
            "The 'supported_tasks' parameter was not provided to "
            "build_app and will be required in a future version. "
            "Defaulting to ('generate',).",
            DeprecationWarning,
            stacklevel=2,
        )
        supported_tasks = _FALLBACK_SUPPORTED_TASKS

178
    if args.disable_fastapi_docs:
179
180
181
        app = FastAPI(
            openapi_url=None, docs_url=None, redoc_url=None, lifespan=lifespan
        )
182
183
    elif args.enable_offline_docs:
        app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)
184
185
    else:
        app = FastAPI(lifespan=lifespan)
186
    app.state.args = args
187

188
    from vllm.entrypoints.serve import register_vllm_serve_api_routers
189

190
    register_vllm_serve_api_routers(app)
191

192
193
    from vllm.entrypoints.openai.models.api_router import (
        attach_router as register_models_api_router,
194
195
    )

196
    register_models_api_router(app)
197

198
199
    from vllm.entrypoints.sagemaker.api_router import (
        attach_router as register_sagemaker_api_router,
200
201
    )

202
    register_sagemaker_api_router(app, supported_tasks)
203

204
    if "generate" in supported_tasks:
205
206
207
        from vllm.entrypoints.openai.generate.api_router import (
            register_generate_api_routers,
        )
208

209
        register_generate_api_routers(app)
210

211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
        from vllm.entrypoints.serve.disagg.api_router import (
            attach_router as attach_disagg_router,
        )

        attach_disagg_router(app)

        from vllm.entrypoints.serve.rlhf.api_router import (
            attach_router as attach_rlhf_router,
        )

        attach_rlhf_router(app)

        from vllm.entrypoints.serve.elastic_ep.api_router import (
            attach_router as elastic_ep_attach_router,
        )

        elastic_ep_attach_router(app)

229
230
231
232
233
234
235
    if "generate" in supported_tasks or "render" in supported_tasks:
        from vllm.entrypoints.serve.render.api_router import (
            attach_router as attach_render_router,
        )

        attach_render_router(app)

236
    if "transcription" in supported_tasks:
237
238
        from vllm.entrypoints.openai.speech_to_text.api_router import (
            attach_router as register_speech_to_text_api_router,
239
        )
240

241
        register_speech_to_text_api_router(app)
Zhuohan Li's avatar
Zhuohan Li committed
242

243
244
245
246
247
248
249
    if "realtime" in supported_tasks:
        from vllm.entrypoints.openai.realtime.api_router import (
            attach_router as register_realtime_api_router,
        )

        register_realtime_api_router(app)

250
251
    if any(task in POOLING_TASKS for task in supported_tasks):
        from vllm.entrypoints.pooling import register_pooling_api_routers
252

253
        register_pooling_api_routers(app, supported_tasks)
254

255
    app.root_path = args.root_path
Zhuohan Li's avatar
Zhuohan Li committed
256
257
258
259
260
261
262
263
    app.add_middleware(
        CORSMiddleware,
        allow_origins=args.allowed_origins,
        allow_credentials=args.allow_credentials,
        allow_methods=args.allowed_methods,
        allow_headers=args.allowed_headers,
    )

264
265
    app.exception_handler(HTTPException)(http_exception_handler)
    app.exception_handler(RequestValidationError)(validation_exception_handler)
266
267
    app.exception_handler(EngineGenerateError)(engine_error_handler)
    app.exception_handler(EngineDeadError)(engine_error_handler)
268
    app.exception_handler(GenerationError)(generation_error_handler)
269
    app.exception_handler(Exception)(exception_handler)
Ethan Xu's avatar
Ethan Xu committed
270

271
    # Ensure --api-key option from CLI takes precedence over VLLM_API_KEY
272
    if tokens := [key for key in (args.api_key or [envs.VLLM_API_KEY]) if key]:
273
274
        from vllm.entrypoints.openai.server_utils import AuthenticationMiddleware

275
        app.add_middleware(AuthenticationMiddleware, tokens=tokens)
276

277
    if args.enable_request_id_headers:
278
279
        from vllm.entrypoints.openai.server_utils import XRequestIdMiddleware

280
        app.add_middleware(XRequestIdMiddleware)
281

282
283
284
    # Add scaling middleware to check for scaling state
    app.add_middleware(ScalingMiddleware)

285
286
287
288
289
290
291
292
    if "realtime" in supported_tasks:
        # Add WebSocket metrics middleware
        from vllm.entrypoints.openai.realtime.metrics import (
            WebSocketMetricsMiddleware,
        )

        app.add_middleware(WebSocketMetricsMiddleware)

293
    if envs.VLLM_DEBUG_LOG_API_SERVER_RESPONSE:
294
295
296
297
298
        logger.warning(
            "CAUTION: Enabling log response in the API Server. "
            "This can include sensitive information and should be "
            "avoided in production."
        )
299
        app.middleware("http")(log_response)
300

301
302
303
304
    for middleware in args.middleware:
        module_path, object_name = middleware.rsplit(".", 1)
        imported = getattr(importlib.import_module(module_path), object_name)
        if inspect.isclass(imported):
305
            app.add_middleware(imported)  # type: ignore[arg-type]
306
307
308
        elif inspect.iscoroutinefunction(imported):
            app.middleware("http")(imported)
        else:
309
310
311
            raise ValueError(
                f"Invalid middleware {middleware}. Must be a function or a class."
            )
312

313
    app = sagemaker_standards_bootstrap(app)
Ethan Xu's avatar
Ethan Xu committed
314
315
316
    return app


317
async def init_app_state(
318
    engine_client: EngineClient,
319
    state: State,
320
    args: Namespace,
321
    supported_tasks: tuple["SupportedTask", ...] | None = None,
322
) -> None:
323
    vllm_config = engine_client.vllm_config
324
325
326
327
328
329
330
331
332
    if supported_tasks is None:
        warnings.warn(
            "The 'supported_tasks' parameter was not provided to "
            "init_app_state and will be required in a future version. "
            "Please pass 'supported_tasks' explicitly.",
            DeprecationWarning,
            stacklevel=2,
        )
        supported_tasks = _FALLBACK_SUPPORTED_TASKS
333

334
    if args.served_model_name is not None:
335
        served_model_names = args.served_model_name
336
    else:
337
        served_model_names = [args.model]
338

339
    if args.enable_log_requests:
340
        request_logger = RequestLogger(max_log_len=args.max_log_len)
341
342
    else:
        request_logger = None
343

344
    base_model_paths = [
345
        BaseModelPath(name=name, model_path=args.model) for name in served_model_names
346
347
    ]

348
    state.engine_client = engine_client
349
    state.log_stats = not args.disable_log_stats
350
    state.vllm_config = vllm_config
351
    state.args = args
352
    resolved_chat_template = load_chat_template(args.chat_template)
353

354
    # Merge default_mm_loras into the static lora_modules
355
356
357
358
359
360
    default_mm_loras = (
        vllm_config.lora_config.default_mm_loras
        if vllm_config.lora_config is not None
        else {}
    )
    lora_modules = process_lora_modules(args.lora_modules, default_mm_loras)
361

362
    state.openai_serving_models = OpenAIServingModels(
363
        engine_client=engine_client,
364
        base_model_paths=base_model_paths,
365
        lora_modules=lora_modules,
366
    )
367
    await state.openai_serving_models.init_static_loras()
368
    state.openai_serving_tokenization = OpenAIServingTokenization(
369
        engine_client,
370
        state.openai_serving_models,
371
        request_logger=request_logger,
372
373
        chat_template=resolved_chat_template,
        chat_template_content_format=args.chat_template_content_format,
374
        default_chat_template_kwargs=args.default_chat_template_kwargs,
375
        trust_request_chat_template=args.trust_request_chat_template,
376
    )
377

378
    if "generate" in supported_tasks:
379
380
381
382
        from vllm.entrypoints.openai.generate.api_router import init_generate_state

        await init_generate_state(
            engine_client, state, args, request_logger, supported_tasks
383
        )
384
385

    if "transcription" in supported_tasks:
386
        from vllm.entrypoints.openai.speech_to_text.api_router import (
387
            init_transcription_state,
388
        )
389
390
391

        init_transcription_state(
            engine_client, state, args, request_logger, supported_tasks
392
        )
393

394
395
396
397
398
    if "realtime" in supported_tasks:
        from vllm.entrypoints.openai.realtime.api_router import init_realtime_state

        init_realtime_state(engine_client, state, args, request_logger, supported_tasks)

399
400
    if any(task in POOLING_TASKS for task in supported_tasks):
        from vllm.entrypoints.pooling import init_pooling_state
401

402
        init_pooling_state(engine_client, state, args, request_logger, supported_tasks)
403

404
405
406
    state.enable_server_load_tracking = args.enable_server_load_tracking
    state.server_load_metrics = 0

407

408
409
410
411
412
413
414
415
416
417
418
419
420
async def init_render_app_state(
    vllm_config: VllmConfig,
    state: State,
    args: Namespace,
) -> None:
    """Initialise FastAPI app state for a CPU-only render server.

    Unlike :func:`init_app_state` this function does not require an
    :class:`~vllm.engine.protocol.EngineClient`; it bootstraps the
    preprocessing pipeline (renderer, io_processor, input_processor)
    directly from the :class:`~vllm.config.VllmConfig`.
    """
    from vllm.entrypoints.chat_utils import load_chat_template
421
    from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
422
423
424
425
426
    from vllm.entrypoints.serve.render.serving import OpenAIServingRender
    from vllm.plugins.io_processors import get_io_processor
    from vllm.renderers import renderer_from_config

    served_model_names = args.served_model_name or [args.model]
427
428
429
430
431
432
433
    model_registry = OpenAIModelRegistry(
        model_config=vllm_config.model_config,
        base_model_paths=[
            BaseModelPath(name=name, model_path=args.model)
            for name in served_model_names
        ],
    )
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449

    if args.enable_log_requests:
        request_logger = RequestLogger(max_log_len=args.max_log_len)
    else:
        request_logger = None

    renderer = renderer_from_config(vllm_config)
    io_processor = get_io_processor(
        vllm_config, renderer, vllm_config.model_config.io_processor_plugin
    )
    resolved_chat_template = load_chat_template(args.chat_template)

    state.openai_serving_render = OpenAIServingRender(
        model_config=vllm_config.model_config,
        renderer=renderer,
        io_processor=io_processor,
450
        model_registry=model_registry,
451
452
453
454
455
456
457
458
459
460
461
        request_logger=request_logger,
        chat_template=resolved_chat_template,
        chat_template_content_format=args.chat_template_content_format,
        trust_request_chat_template=args.trust_request_chat_template,
        enable_auto_tools=args.enable_auto_tool_choice,
        exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
        tool_parser=args.tool_call_parser,
        default_chat_template_kwargs=args.default_chat_template_kwargs,
        log_error_stack=args.log_error_stack,
    )

462
    state.openai_serving_models = model_registry
463

464
465
466
    # Expose tokenization via the render handler (no engine required).
    state.openai_serving_tokenization = state.openai_serving_render

467
468
469
470
471
472
473
474
475
    state.vllm_config = vllm_config
    # Disable stats logging — there is no engine to poll.
    state.log_stats = False
    state.engine_client = None
    state.args = args
    state.enable_server_load_tracking = False
    state.server_load_metrics = 0


476
def create_server_socket(addr: tuple[str, int]) -> socket.socket:
477
478
479
480
481
482
    family = socket.AF_INET
    if is_valid_ipv6_address(addr[0]):
        family = socket.AF_INET6

    sock = socket.socket(family=family, type=socket.SOCK_STREAM)
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
483
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
484
485
486
487
488
    sock.bind(addr)

    return sock


489
490
491
492
493
494
def create_server_unix_socket(path: str) -> socket.socket:
    sock = socket.socket(family=socket.AF_UNIX, type=socket.SOCK_STREAM)
    sock.bind(path)
    return sock


495
def validate_api_server_args(args):
496
    valid_tool_parses = ToolParserManager.list_registered()
497
498
499
500
501
    if args.enable_auto_tool_choice and args.tool_call_parser not in valid_tool_parses:
        raise KeyError(
            f"invalid tool call parser: {args.tool_call_parser} "
            f"(chose from {{ {','.join(valid_tool_parses)} }})"
        )
502

503
    valid_reasoning_parsers = ReasoningParserManager.list_registered()
504
505
    if (
        reasoning_parser := args.structured_outputs_config.reasoning_parser
506
    ) and reasoning_parser not in valid_reasoning_parsers:
507
        raise KeyError(
508
            f"invalid reasoning parser: {reasoning_parser} "
509
            f"(chose from {{ {','.join(valid_reasoning_parsers)} }})"
510
        )
511

512

513
@instrument(span_name="API server setup")
514
515
516
517
def setup_server(args):
    """Validate API server args, set up signal handler, create socket
    ready to serve."""

518
    log_version_and_model(logger, VLLM_VERSION, args.model)
519
520
521
522
523
    log_non_default_args(args)

    if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
        ToolParserManager.import_tool_parser(args.tool_parser_plugin)

524
525
526
    if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:
        ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)

527
528
    validate_api_server_args(args)

529
530
531
    # workaround to make sure that we bind the port before the engine is set up.
    # This avoids race conditions with ray.
    # see https://github.com/vllm-project/vllm/issues/8204
532
533
534
535
536
    if args.uds:
        sock = create_server_unix_socket(args.uds)
    else:
        sock_addr = (args.host or "", args.port)
        sock = create_server_socket(sock_addr)
537

538
539
540
541
    # workaround to avoid footguns where uvicorn drops requests with too
    # many concurrent requests active
    set_ulimit()

542
543
544
545
546
547
    def signal_handler(*_) -> None:
        # Interrupt server on sigterm while initializing
        raise KeyboardInterrupt("terminated")

    signal.signal(signal.SIGTERM, signal_handler)

548
549
550
551
552
    if args.uds:
        listen_address = f"unix:{args.uds}"
    else:
        addr, port = sock_addr
        is_ssl = args.ssl_keyfile and args.ssl_certfile
553
        host_part = f"[{addr}]" if is_valid_ipv6_address(addr) else addr or "0.0.0.0"
554
        listen_address = f"http{'s' if is_ssl else ''}://{host_part}:{port}"
555
556
557
    return listen_address, sock


558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
async def build_and_serve(
    engine_client: EngineClient,
    listen_address: str,
    sock: socket.socket,
    args: Namespace,
    **uvicorn_kwargs,
) -> asyncio.Task:
    """Build FastAPI app, initialize state, and start serving.

    Returns the shutdown task for the caller to await.
    """

    # Get uvicorn log config (from file or with endpoint filter)
    log_config = get_uvicorn_log_config(args)
    if log_config is not None:
        uvicorn_kwargs["log_config"] = log_config

    supported_tasks = await engine_client.get_supported_tasks()
    logger.info("Supported tasks: %s", supported_tasks)
    app = build_app(args, supported_tasks)
    await init_app_state(engine_client, app.state, args, supported_tasks)

    logger.info("Starting vLLM server on %s", listen_address)

    return await serve_http(
        app,
        sock=sock,
        enable_ssl_refresh=args.enable_ssl_refresh,
        host=args.host,
        port=args.port,
        log_level=args.uvicorn_log_level,
        # NOTE: When the 'disable_uvicorn_access_log' value is True,
        # no access log will be output.
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
        access_log=not args.disable_uvicorn_access_log,
        timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
        ssl_keyfile=args.ssl_keyfile,
        ssl_certfile=args.ssl_certfile,
        ssl_ca_certs=args.ssl_ca_certs,
        ssl_cert_reqs=args.ssl_cert_reqs,
        ssl_ciphers=args.ssl_ciphers,
        h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
        h11_max_header_count=args.h11_max_header_count,
        **uvicorn_kwargs,
    )


async def build_and_serve_renderer(
    vllm_config: VllmConfig,
    listen_address: str,
    sock: socket.socket,
    args: Namespace,
    **uvicorn_kwargs,
) -> asyncio.Task:
    """Build FastAPI app for a CPU-only render server, initialize state, and
    start serving.

    Returns the shutdown task for the caller to await.
    """

    # Get uvicorn log config (from file or with endpoint filter)
    log_config = get_uvicorn_log_config(args)
    if log_config is not None:
        uvicorn_kwargs["log_config"] = log_config

    app = build_app(args, ("render",))
    await init_render_app_state(vllm_config, app.state, args)

    logger.info("Starting vLLM server on %s", listen_address)

    return await serve_http(
        app,
        sock=sock,
        enable_ssl_refresh=args.enable_ssl_refresh,
        host=args.host,
        port=args.port,
        log_level=args.uvicorn_log_level,
        # NOTE: When the 'disable_uvicorn_access_log' value is True,
        # no access log will be output.
636
637
638
639
640
641
642
643
644
645
646
647
648
        access_log=not args.disable_uvicorn_access_log,
        timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
        ssl_keyfile=args.ssl_keyfile,
        ssl_certfile=args.ssl_certfile,
        ssl_ca_certs=args.ssl_ca_certs,
        ssl_cert_reqs=args.ssl_cert_reqs,
        ssl_ciphers=args.ssl_ciphers,
        h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
        h11_max_header_count=args.h11_max_header_count,
        **uvicorn_kwargs,
    )


649
650
async def run_server(args, **uvicorn_kwargs) -> None:
    """Run a single-worker API server."""
651
652

    # Add process-specific prefix to stdout and stderr.
653
    decorate_logs("APIServer")
654

655
656
657
658
    listen_address, sock = setup_server(args)
    await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)


659
660
661
async def run_server_worker(
    listen_address, sock, args, client_config=None, **uvicorn_kwargs
) -> None:
662
663
664
665
666
    """Run a single API server worker."""

    if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
        ToolParserManager.import_tool_parser(args.tool_parser_plugin)

667
668
669
    if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:
        ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)

670
    async with build_async_engine_client(
671
672
        args,
        client_config=client_config,
673
    ) as engine_client:
674
675
        shutdown_task = await build_and_serve(
            engine_client, listen_address, sock, args, **uvicorn_kwargs
676
        )
677
    # NB: Await server shutdown only after the backend context is exited
678
679
680
681
    try:
        await shutdown_task
    finally:
        sock.close()
682

Ethan Xu's avatar
Ethan Xu committed
683
684
685

if __name__ == "__main__":
    # NOTE(simon):
686
687
    # This section should be in sync with vllm/entrypoints/cli/main.py for CLI
    # entrypoints.
688
    cli_env_setup()
Ethan Xu's avatar
Ethan Xu committed
689
    parser = FlexibleArgumentParser(
690
691
        description="vLLM OpenAI-Compatible RESTful API server."
    )
Ethan Xu's avatar
Ethan Xu committed
692
693
    parser = make_arg_parser(parser)
    args = parser.parse_args()
694
    validate_parsed_serve_args(args)
695

696
    uvloop.run(run_server(args))