api_server.py 24.9 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import asyncio
4
5
import importlib
import inspect
6
import multiprocessing
7
import multiprocessing.forkserver as forkserver
8
import os
9
import signal
10
import socket
11
import tempfile
12
import warnings
13
from argparse import Namespace
14
from collections.abc import AsyncIterator
15
from contextlib import asynccontextmanager
16
from typing import Any
17

18
import uvloop
19
from fastapi import FastAPI, HTTPException
Zhuohan Li's avatar
Zhuohan Li committed
20
21
from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware
22
from starlette.datastructures import State
Zhuohan Li's avatar
Zhuohan Li committed
23

24
import vllm.envs as envs
25
from vllm.config import VllmConfig
Woosuk Kwon's avatar
Woosuk Kwon committed
26
from vllm.engine.arg_utils import AsyncEngineArgs
27
from vllm.engine.protocol import EngineClient
28
from vllm.entrypoints.chat_utils import load_chat_template
29
from vllm.entrypoints.launcher import serve_http
30
from vllm.entrypoints.logger import RequestLogger
31
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
32
from vllm.entrypoints.openai.engine.protocol import GenerationError
33
from vllm.entrypoints.openai.models.protocol import BaseModelPath
34
35
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.openai.server_utils import (
36
37
    engine_error_handler,
    exception_handler,
38
    generation_error_handler,
39
40
41
42
43
    get_uvicorn_log_config,
    http_exception_handler,
    lifespan,
    log_response,
    validation_exception_handler,
44
)
45
from vllm.entrypoints.sagemaker.api_router import sagemaker_standards_bootstrap
46
47
48
from vllm.entrypoints.serve.elastic_ep.middleware import (
    ScalingMiddleware,
)
49
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
50
from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
51
52
53
from vllm.entrypoints.utils import (
    cli_env_setup,
    log_non_default_args,
54
    log_version_and_model,
55
    process_lora_modules,
56
)
57
from vllm.logger import init_logger
58
from vllm.reasoning import ReasoningParserManager
59
from vllm.tasks import POOLING_TASKS, SupportedTask
60
from vllm.tool_parsers import ToolParserManager
61
from vllm.tracing import instrument
yhu422's avatar
yhu422 committed
62
from vllm.usage.usage_lib import UsageContext
Cyrus Leung's avatar
Cyrus Leung committed
63
from vllm.utils.argparse_utils import FlexibleArgumentParser
64
from vllm.utils.network_utils import is_valid_ipv6_address
65
from vllm.utils.system_utils import decorate_logs, set_ulimit
66
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
67
from vllm.version import __version__ as VLLM_VERSION
Zhuohan Li's avatar
Zhuohan Li committed
68

69
prometheus_multiproc_dir: tempfile.TemporaryDirectory
70

71
# Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765)
72
logger = init_logger("vllm.entrypoints.openai.api_server")
73

74
75
_FALLBACK_SUPPORTED_TASKS: tuple[SupportedTask, ...] = ("generate",)

76

77
@asynccontextmanager
78
async def build_async_engine_client(
79
    args: Namespace,
80
81
    *,
    usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
82
83
    disable_frontend_multiprocessing: bool | None = None,
    client_config: dict[str, Any] | None = None,
84
) -> AsyncIterator[EngineClient]:
85
86
87
88
    if os.getenv("VLLM_WORKER_MULTIPROC_METHOD") == "forkserver":
        # The executor is expected to be mp.
        # Pre-import heavy modules in the forkserver process
        logger.debug("Setup forkserver with pre-imports")
89
        multiprocessing.set_start_method("forkserver")
90
91
92
93
        multiprocessing.set_forkserver_preload(["vllm.v1.engine.async_llm"])
        forkserver.ensure_running()
        logger.debug("Forkserver setup complete!")

94
    # Context manager to handle engine_client lifecycle
95
96
    # Ensures everything is shutdown and cleaned up on error/exit
    engine_args = AsyncEngineArgs.from_cli_args(args)
97
98
99
    if client_config:
        engine_args._api_process_count = client_config.get("client_count", 1)
        engine_args._api_process_rank = client_config.get("client_index", 0)
100

101
    if disable_frontend_multiprocessing is None:
102
        disable_frontend_multiprocessing = bool(args.disable_frontend_multiprocessing)
103

104
    async with build_async_engine_client_from_engine_args(
105
106
107
108
        engine_args,
        usage_context=usage_context,
        disable_frontend_multiprocessing=disable_frontend_multiprocessing,
        client_config=client_config,
109
    ) as engine:
110
111
112
113
114
115
        yield engine


@asynccontextmanager
async def build_async_engine_client_from_engine_args(
    engine_args: AsyncEngineArgs,
116
117
    *,
    usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
118
    disable_frontend_multiprocessing: bool = False,
119
    client_config: dict[str, Any] | None = None,
120
) -> AsyncIterator[EngineClient]:
121
    """
122
    Create EngineClient, either:
123
124
125
126
127
128
        - in-process using the AsyncLLMEngine Directly
        - multiprocess using AsyncLLMEngine RPC

    Returns the Client or None if the creation failed.
    """

129
130
131
    # Create the EngineConfig (determines if we can use V1).
    vllm_config = engine_args.create_engine_config(usage_context=usage_context)

132
    if disable_frontend_multiprocessing:
133
        logger.warning("V1 is enabled, but got --disable-frontend-multiprocessing.")
134

135
    from vllm.v1.engine.async_llm import AsyncLLM
136

137
    async_llm: AsyncLLM | None = None
138
139
140
141
142
143

    # Don't mutate the input client_config
    client_config = dict(client_config) if client_config else {}
    client_count = client_config.pop("client_count", 1)
    client_index = client_config.pop("client_index", 0)

144
145
146
147
148
    try:
        async_llm = AsyncLLM.from_vllm_config(
            vllm_config=vllm_config,
            usage_context=usage_context,
            enable_log_requests=engine_args.enable_log_requests,
149
            aggregate_engine_logging=engine_args.aggregate_engine_logging,
150
151
152
            disable_log_stats=engine_args.disable_log_stats,
            client_addresses=client_config,
            client_count=client_count,
153
154
            client_index=client_index,
        )
155
156

        # Don't keep the dummy data in memory
157
        assert async_llm is not None
158
159
160
161
162
163
        await async_llm.reset_mm_cache()

        yield async_llm
    finally:
        if async_llm:
            async_llm.shutdown()
164
165


166
167
168
169
170
171
172
173
174
175
176
177
178
def build_app(
    args: Namespace, supported_tasks: tuple["SupportedTask", ...] | None = None
) -> FastAPI:
    if supported_tasks is None:
        warnings.warn(
            "The 'supported_tasks' parameter was not provided to "
            "build_app and will be required in a future version. "
            "Defaulting to ('generate',).",
            DeprecationWarning,
            stacklevel=2,
        )
        supported_tasks = _FALLBACK_SUPPORTED_TASKS

179
    if args.disable_fastapi_docs:
180
181
182
        app = FastAPI(
            openapi_url=None, docs_url=None, redoc_url=None, lifespan=lifespan
        )
183
184
    elif args.enable_offline_docs:
        app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)
185
186
    else:
        app = FastAPI(lifespan=lifespan)
187
    app.state.args = args
188

189
    from vllm.entrypoints.serve import register_vllm_serve_api_routers
190

191
    register_vllm_serve_api_routers(app)
192

193
194
    from vllm.entrypoints.openai.models.api_router import (
        attach_router as register_models_api_router,
195
196
    )

197
    register_models_api_router(app)
198

199
200
    from vllm.entrypoints.sagemaker.api_router import (
        attach_router as register_sagemaker_api_router,
201
202
    )

203
    register_sagemaker_api_router(app, supported_tasks)
204

205
    if "generate" in supported_tasks:
206
207
208
        from vllm.entrypoints.openai.generate.api_router import (
            register_generate_api_routers,
        )
209

210
        register_generate_api_routers(app)
211

212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
        from vllm.entrypoints.serve.disagg.api_router import (
            attach_router as attach_disagg_router,
        )

        attach_disagg_router(app)

        from vllm.entrypoints.serve.rlhf.api_router import (
            attach_router as attach_rlhf_router,
        )

        attach_rlhf_router(app)

        from vllm.entrypoints.serve.elastic_ep.api_router import (
            attach_router as elastic_ep_attach_router,
        )

        elastic_ep_attach_router(app)

230
231
232
233
234
235
236
    if "generate" in supported_tasks or "render" in supported_tasks:
        from vllm.entrypoints.serve.render.api_router import (
            attach_router as attach_render_router,
        )

        attach_render_router(app)

237
    if "transcription" in supported_tasks:
238
239
        from vllm.entrypoints.openai.speech_to_text.api_router import (
            attach_router as register_speech_to_text_api_router,
240
        )
241

242
        register_speech_to_text_api_router(app)
Zhuohan Li's avatar
Zhuohan Li committed
243

244
245
246
247
248
249
250
    if "realtime" in supported_tasks:
        from vllm.entrypoints.openai.realtime.api_router import (
            attach_router as register_realtime_api_router,
        )

        register_realtime_api_router(app)

251
252
    if any(task in POOLING_TASKS for task in supported_tasks):
        from vllm.entrypoints.pooling import register_pooling_api_routers
253

254
        register_pooling_api_routers(app, supported_tasks)
255

256
    app.root_path = args.root_path
Zhuohan Li's avatar
Zhuohan Li committed
257
258
259
260
261
262
263
264
    app.add_middleware(
        CORSMiddleware,
        allow_origins=args.allowed_origins,
        allow_credentials=args.allow_credentials,
        allow_methods=args.allowed_methods,
        allow_headers=args.allowed_headers,
    )

265
266
    app.exception_handler(HTTPException)(http_exception_handler)
    app.exception_handler(RequestValidationError)(validation_exception_handler)
267
268
    app.exception_handler(EngineGenerateError)(engine_error_handler)
    app.exception_handler(EngineDeadError)(engine_error_handler)
269
    app.exception_handler(GenerationError)(generation_error_handler)
270
    app.exception_handler(Exception)(exception_handler)
Ethan Xu's avatar
Ethan Xu committed
271

272
    # Ensure --api-key option from CLI takes precedence over VLLM_API_KEY
273
    if tokens := [key for key in (args.api_key or [envs.VLLM_API_KEY]) if key]:
274
275
        from vllm.entrypoints.openai.server_utils import AuthenticationMiddleware

276
        app.add_middleware(AuthenticationMiddleware, tokens=tokens)
277

278
    if args.enable_request_id_headers:
279
280
        from vllm.entrypoints.openai.server_utils import XRequestIdMiddleware

281
        app.add_middleware(XRequestIdMiddleware)
282

283
284
285
    # Add scaling middleware to check for scaling state
    app.add_middleware(ScalingMiddleware)

286
287
288
289
290
291
292
293
    if "realtime" in supported_tasks:
        # Add WebSocket metrics middleware
        from vllm.entrypoints.openai.realtime.metrics import (
            WebSocketMetricsMiddleware,
        )

        app.add_middleware(WebSocketMetricsMiddleware)

294
    if envs.VLLM_DEBUG_LOG_API_SERVER_RESPONSE:
295
296
297
298
299
        logger.warning(
            "CAUTION: Enabling log response in the API Server. "
            "This can include sensitive information and should be "
            "avoided in production."
        )
300
        app.middleware("http")(log_response)
301

302
303
304
305
    for middleware in args.middleware:
        module_path, object_name = middleware.rsplit(".", 1)
        imported = getattr(importlib.import_module(module_path), object_name)
        if inspect.isclass(imported):
306
            app.add_middleware(imported)  # type: ignore[arg-type]
307
308
309
        elif inspect.iscoroutinefunction(imported):
            app.middleware("http")(imported)
        else:
310
311
312
            raise ValueError(
                f"Invalid middleware {middleware}. Must be a function or a class."
            )
313

314
    app = sagemaker_standards_bootstrap(app)
Ethan Xu's avatar
Ethan Xu committed
315
316
317
    return app


318
async def init_app_state(
319
    engine_client: EngineClient,
320
    state: State,
321
    args: Namespace,
322
    supported_tasks: tuple["SupportedTask", ...] | None = None,
323
) -> None:
324
    vllm_config = engine_client.vllm_config
325
326
327
328
329
330
331
332
333
    if supported_tasks is None:
        warnings.warn(
            "The 'supported_tasks' parameter was not provided to "
            "init_app_state and will be required in a future version. "
            "Please pass 'supported_tasks' explicitly.",
            DeprecationWarning,
            stacklevel=2,
        )
        supported_tasks = _FALLBACK_SUPPORTED_TASKS
334

335
    if args.served_model_name is not None:
336
        served_model_names = args.served_model_name
337
    else:
338
        served_model_names = [args.model]
339

340
    if args.enable_log_requests:
341
        request_logger = RequestLogger(max_log_len=args.max_log_len)
342
343
    else:
        request_logger = None
344

345
    base_model_paths = [
346
        BaseModelPath(name=name, model_path=args.model) for name in served_model_names
347
348
    ]

349
    state.engine_client = engine_client
350
    state.log_stats = not args.disable_log_stats
351
    state.vllm_config = vllm_config
352
    state.args = args
353
    resolved_chat_template = load_chat_template(args.chat_template)
354

355
    # Merge default_mm_loras into the static lora_modules
356
357
358
359
360
361
    default_mm_loras = (
        vllm_config.lora_config.default_mm_loras
        if vllm_config.lora_config is not None
        else {}
    )
    lora_modules = process_lora_modules(args.lora_modules, default_mm_loras)
362

363
    state.openai_serving_models = OpenAIServingModels(
364
        engine_client=engine_client,
365
        base_model_paths=base_model_paths,
366
        lora_modules=lora_modules,
367
    )
368
    await state.openai_serving_models.init_static_loras()
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385

    state.openai_serving_render = OpenAIServingRender(
        model_config=engine_client.model_config,
        renderer=engine_client.renderer,
        io_processor=engine_client.io_processor,
        model_registry=state.openai_serving_models.registry,
        request_logger=request_logger,
        chat_template=resolved_chat_template,
        chat_template_content_format=args.chat_template_content_format,
        trust_request_chat_template=args.trust_request_chat_template,
        enable_auto_tools=args.enable_auto_tool_choice,
        exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
        tool_parser=args.tool_call_parser,
        default_chat_template_kwargs=args.default_chat_template_kwargs,
        log_error_stack=args.log_error_stack,
    )

386
    state.openai_serving_tokenization = OpenAIServingTokenization(
387
        engine_client,
388
        state.openai_serving_models,
389
        state.openai_serving_render,
390
        request_logger=request_logger,
391
392
        chat_template=resolved_chat_template,
        chat_template_content_format=args.chat_template_content_format,
393
        default_chat_template_kwargs=args.default_chat_template_kwargs,
394
        trust_request_chat_template=args.trust_request_chat_template,
395
    )
396

397
    if "generate" in supported_tasks:
398
399
400
401
        from vllm.entrypoints.openai.generate.api_router import init_generate_state

        await init_generate_state(
            engine_client, state, args, request_logger, supported_tasks
402
        )
403
404

    if "transcription" in supported_tasks:
405
        from vllm.entrypoints.openai.speech_to_text.api_router import (
406
            init_transcription_state,
407
        )
408
409
410

        init_transcription_state(
            engine_client, state, args, request_logger, supported_tasks
411
        )
412

413
414
415
416
417
    if "realtime" in supported_tasks:
        from vllm.entrypoints.openai.realtime.api_router import init_realtime_state

        init_realtime_state(engine_client, state, args, request_logger, supported_tasks)

418
419
    if any(task in POOLING_TASKS for task in supported_tasks):
        from vllm.entrypoints.pooling import init_pooling_state
420

421
        init_pooling_state(engine_client, state, args, request_logger, supported_tasks)
422

423
424
425
    state.enable_server_load_tracking = args.enable_server_load_tracking
    state.server_load_metrics = 0

426

427
428
429
430
431
432
433
434
435
436
437
438
439
async def init_render_app_state(
    vllm_config: VllmConfig,
    state: State,
    args: Namespace,
) -> None:
    """Initialise FastAPI app state for a CPU-only render server.

    Unlike :func:`init_app_state` this function does not require an
    :class:`~vllm.engine.protocol.EngineClient`; it bootstraps the
    preprocessing pipeline (renderer, io_processor, input_processor)
    directly from the :class:`~vllm.config.VllmConfig`.
    """
    from vllm.entrypoints.chat_utils import load_chat_template
440
    from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
441
442
443
444
445
    from vllm.entrypoints.serve.render.serving import OpenAIServingRender
    from vllm.plugins.io_processors import get_io_processor
    from vllm.renderers import renderer_from_config

    served_model_names = args.served_model_name or [args.model]
446
447
448
449
450
451
452
    model_registry = OpenAIModelRegistry(
        model_config=vllm_config.model_config,
        base_model_paths=[
            BaseModelPath(name=name, model_path=args.model)
            for name in served_model_names
        ],
    )
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468

    if args.enable_log_requests:
        request_logger = RequestLogger(max_log_len=args.max_log_len)
    else:
        request_logger = None

    renderer = renderer_from_config(vllm_config)
    io_processor = get_io_processor(
        vllm_config, renderer, vllm_config.model_config.io_processor_plugin
    )
    resolved_chat_template = load_chat_template(args.chat_template)

    state.openai_serving_render = OpenAIServingRender(
        model_config=vllm_config.model_config,
        renderer=renderer,
        io_processor=io_processor,
469
        model_registry=model_registry,
470
471
472
473
474
475
476
477
478
479
480
        request_logger=request_logger,
        chat_template=resolved_chat_template,
        chat_template_content_format=args.chat_template_content_format,
        trust_request_chat_template=args.trust_request_chat_template,
        enable_auto_tools=args.enable_auto_tool_choice,
        exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
        tool_parser=args.tool_call_parser,
        default_chat_template_kwargs=args.default_chat_template_kwargs,
        log_error_stack=args.log_error_stack,
    )

481
    state.openai_serving_models = model_registry
482

483
484
485
    # Expose tokenization via the render handler (no engine required).
    state.openai_serving_tokenization = state.openai_serving_render

486
487
488
489
490
491
492
493
494
    state.vllm_config = vllm_config
    # Disable stats logging — there is no engine to poll.
    state.log_stats = False
    state.engine_client = None
    state.args = args
    state.enable_server_load_tracking = False
    state.server_load_metrics = 0


495
def create_server_socket(addr: tuple[str, int]) -> socket.socket:
496
497
498
499
500
501
    family = socket.AF_INET
    if is_valid_ipv6_address(addr[0]):
        family = socket.AF_INET6

    sock = socket.socket(family=family, type=socket.SOCK_STREAM)
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
502
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
503
504
505
506
507
    sock.bind(addr)

    return sock


508
509
510
511
512
513
def create_server_unix_socket(path: str) -> socket.socket:
    sock = socket.socket(family=socket.AF_UNIX, type=socket.SOCK_STREAM)
    sock.bind(path)
    return sock


514
def validate_api_server_args(args):
515
    valid_tool_parses = ToolParserManager.list_registered()
516
517
518
519
520
    if args.enable_auto_tool_choice and args.tool_call_parser not in valid_tool_parses:
        raise KeyError(
            f"invalid tool call parser: {args.tool_call_parser} "
            f"(chose from {{ {','.join(valid_tool_parses)} }})"
        )
521

522
    valid_reasoning_parsers = ReasoningParserManager.list_registered()
523
524
    if (
        reasoning_parser := args.structured_outputs_config.reasoning_parser
525
    ) and reasoning_parser not in valid_reasoning_parsers:
526
        raise KeyError(
527
            f"invalid reasoning parser: {reasoning_parser} "
528
            f"(chose from {{ {','.join(valid_reasoning_parsers)} }})"
529
        )
530

531

532
@instrument(span_name="API server setup")
533
534
535
536
def setup_server(args):
    """Validate API server args, set up signal handler, create socket
    ready to serve."""

537
    log_version_and_model(logger, VLLM_VERSION, args.model)
538
539
540
541
542
    log_non_default_args(args)

    if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
        ToolParserManager.import_tool_parser(args.tool_parser_plugin)

543
544
545
    if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:
        ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)

546
547
    validate_api_server_args(args)

548
549
550
    # workaround to make sure that we bind the port before the engine is set up.
    # This avoids race conditions with ray.
    # see https://github.com/vllm-project/vllm/issues/8204
551
552
553
554
555
    if args.uds:
        sock = create_server_unix_socket(args.uds)
    else:
        sock_addr = (args.host or "", args.port)
        sock = create_server_socket(sock_addr)
556

557
558
559
560
    # workaround to avoid footguns where uvicorn drops requests with too
    # many concurrent requests active
    set_ulimit()

561
562
563
564
565
566
    def signal_handler(*_) -> None:
        # Interrupt server on sigterm while initializing
        raise KeyboardInterrupt("terminated")

    signal.signal(signal.SIGTERM, signal_handler)

567
568
569
570
571
    if args.uds:
        listen_address = f"unix:{args.uds}"
    else:
        addr, port = sock_addr
        is_ssl = args.ssl_keyfile and args.ssl_certfile
572
        host_part = f"[{addr}]" if is_valid_ipv6_address(addr) else addr or "0.0.0.0"
573
        listen_address = f"http{'s' if is_ssl else ''}://{host_part}:{port}"
574
575
576
    return listen_address, sock


577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
async def build_and_serve(
    engine_client: EngineClient,
    listen_address: str,
    sock: socket.socket,
    args: Namespace,
    **uvicorn_kwargs,
) -> asyncio.Task:
    """Build FastAPI app, initialize state, and start serving.

    Returns the shutdown task for the caller to await.
    """

    # Get uvicorn log config (from file or with endpoint filter)
    log_config = get_uvicorn_log_config(args)
    if log_config is not None:
        uvicorn_kwargs["log_config"] = log_config

    supported_tasks = await engine_client.get_supported_tasks()
    logger.info("Supported tasks: %s", supported_tasks)
    app = build_app(args, supported_tasks)
    await init_app_state(engine_client, app.state, args, supported_tasks)

    logger.info("Starting vLLM server on %s", listen_address)

    return await serve_http(
        app,
        sock=sock,
        enable_ssl_refresh=args.enable_ssl_refresh,
        host=args.host,
        port=args.port,
        log_level=args.uvicorn_log_level,
        # NOTE: When the 'disable_uvicorn_access_log' value is True,
        # no access log will be output.
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
        access_log=not args.disable_uvicorn_access_log,
        timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
        ssl_keyfile=args.ssl_keyfile,
        ssl_certfile=args.ssl_certfile,
        ssl_ca_certs=args.ssl_ca_certs,
        ssl_cert_reqs=args.ssl_cert_reqs,
        ssl_ciphers=args.ssl_ciphers,
        h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
        h11_max_header_count=args.h11_max_header_count,
        **uvicorn_kwargs,
    )


async def build_and_serve_renderer(
    vllm_config: VllmConfig,
    listen_address: str,
    sock: socket.socket,
    args: Namespace,
    **uvicorn_kwargs,
) -> asyncio.Task:
    """Build FastAPI app for a CPU-only render server, initialize state, and
    start serving.

    Returns the shutdown task for the caller to await.
    """

    # Get uvicorn log config (from file or with endpoint filter)
    log_config = get_uvicorn_log_config(args)
    if log_config is not None:
        uvicorn_kwargs["log_config"] = log_config

    app = build_app(args, ("render",))
    await init_render_app_state(vllm_config, app.state, args)

    logger.info("Starting vLLM server on %s", listen_address)

    return await serve_http(
        app,
        sock=sock,
        enable_ssl_refresh=args.enable_ssl_refresh,
        host=args.host,
        port=args.port,
        log_level=args.uvicorn_log_level,
        # NOTE: When the 'disable_uvicorn_access_log' value is True,
        # no access log will be output.
655
656
657
658
659
660
661
662
663
664
665
666
667
        access_log=not args.disable_uvicorn_access_log,
        timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
        ssl_keyfile=args.ssl_keyfile,
        ssl_certfile=args.ssl_certfile,
        ssl_ca_certs=args.ssl_ca_certs,
        ssl_cert_reqs=args.ssl_cert_reqs,
        ssl_ciphers=args.ssl_ciphers,
        h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
        h11_max_header_count=args.h11_max_header_count,
        **uvicorn_kwargs,
    )


668
669
async def run_server(args, **uvicorn_kwargs) -> None:
    """Run a single-worker API server."""
670
671

    # Add process-specific prefix to stdout and stderr.
672
    decorate_logs("APIServer")
673

674
675
676
677
    listen_address, sock = setup_server(args)
    await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)


678
679
680
async def run_server_worker(
    listen_address, sock, args, client_config=None, **uvicorn_kwargs
) -> None:
681
682
683
684
685
    """Run a single API server worker."""

    if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
        ToolParserManager.import_tool_parser(args.tool_parser_plugin)

686
687
688
    if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:
        ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)

689
    async with build_async_engine_client(
690
691
        args,
        client_config=client_config,
692
    ) as engine_client:
693
694
        shutdown_task = await build_and_serve(
            engine_client, listen_address, sock, args, **uvicorn_kwargs
695
        )
696
    # NB: Await server shutdown only after the backend context is exited
697
698
699
700
    try:
        await shutdown_task
    finally:
        sock.close()
701

Ethan Xu's avatar
Ethan Xu committed
702
703
704

if __name__ == "__main__":
    # NOTE(simon):
705
706
    # This section should be in sync with vllm/entrypoints/cli/main.py for CLI
    # entrypoints.
707
    cli_env_setup()
Ethan Xu's avatar
Ethan Xu committed
708
    parser = FlexibleArgumentParser(
709
710
        description="vLLM OpenAI-Compatible RESTful API server."
    )
Ethan Xu's avatar
Ethan Xu committed
711
712
    parser = make_arg_parser(parser)
    args = parser.parse_args()
713
    validate_parsed_serve_args(args)
714

715
    uvloop.run(run_server(args))