api_server.py 23.8 KB
Newer Older
1
import asyncio
2
3
import importlib
import inspect
4
import multiprocessing
5
import os
6
import re
7
import signal
8
import socket
9
import tempfile
10
import uuid
11
from argparse import Namespace
12
from contextlib import asynccontextmanager
13
from functools import partial
14
from http import HTTPStatus
15
from typing import AsyncIterator, Optional, Set, Tuple
16

17
import uvloop
18
from fastapi import APIRouter, FastAPI, Request
Zhuohan Li's avatar
Zhuohan Li committed
19
20
from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware
21
from fastapi.responses import JSONResponse, Response, StreamingResponse
22
from starlette.datastructures import State
23
from starlette.routing import Mount
24
from typing_extensions import assert_never
Zhuohan Li's avatar
Zhuohan Li committed
25

26
import vllm.envs as envs
27
from vllm.config import ModelConfig
Woosuk Kwon's avatar
Woosuk Kwon committed
28
from vllm.engine.arg_utils import AsyncEngineArgs
29
30
31
from vllm.engine.multiprocessing.client import MQLLMEngineClient
from vllm.engine.multiprocessing.engine import run_mp_engine
from vllm.engine.protocol import EngineClient
32
from vllm.entrypoints.launcher import serve_http
33
from vllm.entrypoints.logger import RequestLogger
34
35
from vllm.entrypoints.openai.cli_args import (make_arg_parser,
                                              validate_parsed_serve_args)
36
37
# yapf conflicts with isort for this block
# yapf: disable
38
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
39
                                              ChatCompletionResponse,
40
                                              CompletionRequest,
41
                                              CompletionResponse,
42
43
                                              DetokenizeRequest,
                                              DetokenizeResponse,
44
45
                                              EmbeddingRequest,
                                              EmbeddingResponse, ErrorResponse,
46
                                              LoadLoraAdapterRequest,
47
                                              TokenizeRequest,
48
49
50
                                              TokenizeResponse,
                                              UnloadLoraAdapterRequest)
# yapf: enable
51
52
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
53
from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
54
from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
55
56
from vllm.entrypoints.openai.serving_tokenization import (
    OpenAIServingTokenization)
57
from vllm.entrypoints.openai.tool_parsers import ToolParserManager
58
from vllm.logger import init_logger
yhu422's avatar
yhu422 committed
59
from vllm.usage.usage_lib import UsageContext
60
61
from vllm.utils import (FlexibleArgumentParser, get_open_zmq_ipc_path,
                        is_valid_ipv6_address)
62
from vllm.version import __version__ as VLLM_VERSION
Zhuohan Li's avatar
Zhuohan Li committed
63

64
65
66
67
68
if envs.VLLM_USE_V1:
    from vllm.v1.engine.async_llm import AsyncLLMEngine  # type: ignore
else:
    from vllm.engine.async_llm_engine import AsyncLLMEngine  # type: ignore

69
TIMEOUT_KEEP_ALIVE = 5  # seconds
Zhuohan Li's avatar
Zhuohan Li committed
70

71
prometheus_multiproc_dir: tempfile.TemporaryDirectory
72

73
# Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765)
74
logger = init_logger('vllm.entrypoints.openai.api_server')
75

76
_running_tasks: Set[asyncio.Task] = set()
77

78

79
@asynccontextmanager
80
async def lifespan(app: FastAPI):
81
82
    try:
        if app.state.log_stats:
83
            engine_client: EngineClient = app.state.engine_client
84
85
86

            async def _force_log():
                while True:
87
88
                    await asyncio.sleep(10.)
                    await engine_client.do_log_stats()
89
90
91
92
93
94
95
96
97
98
99
100
101
102

            task = asyncio.create_task(_force_log())
            _running_tasks.add(task)
            task.add_done_callback(_running_tasks.remove)
        else:
            task = None
        try:
            yield
        finally:
            if task is not None:
                task.cancel()
    finally:
        # Ensure app state including engine ref is gc'd
        del app.state
103
104


105
@asynccontextmanager
106
async def build_async_engine_client(
107
        args: Namespace) -> AsyncIterator[EngineClient]:
108

109
    # Context manager to handle engine_client lifecycle
110
111
112
    # Ensures everything is shutdown and cleaned up on error/exit
    engine_args = AsyncEngineArgs.from_cli_args(args)

113
114
115
116
117
118
119
120
121
    async with build_async_engine_client_from_engine_args(
            engine_args, args.disable_frontend_multiprocessing) as engine:
        yield engine


@asynccontextmanager
async def build_async_engine_client_from_engine_args(
    engine_args: AsyncEngineArgs,
    disable_frontend_multiprocessing: bool = False,
122
) -> AsyncIterator[EngineClient]:
123
    """
124
    Create EngineClient, either:
125
126
127
128
129
130
        - in-process using the AsyncLLMEngine Directly
        - multiprocess using AsyncLLMEngine RPC

    Returns the Client or None if the creation failed.
    """

131
132
133
    # Fall back
    # TODO: fill out feature matrix.
    if (MQLLMEngineClient.is_unsupported_config(engine_args)
134
135
            or envs.VLLM_USE_V1 or disable_frontend_multiprocessing):

136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
        engine_config = engine_args.create_engine_config()
        uses_ray = getattr(AsyncLLMEngine._get_executor_cls(engine_config),
                           "uses_ray", False)

        build_engine = partial(AsyncLLMEngine.from_engine_args,
                               engine_args=engine_args,
                               engine_config=engine_config,
                               usage_context=UsageContext.OPENAI_API_SERVER)
        if uses_ray:
            # Must run in main thread with ray for its signal handlers to work
            engine_client = build_engine()
        else:
            engine_client = await asyncio.get_running_loop().run_in_executor(
                None, build_engine)

        yield engine_client
152
153
        if hasattr(engine_client, "shutdown"):
            engine_client.shutdown()
154
155
156
157
        return

    # Otherwise, use the multiprocessing AsyncLLMEngine.
    else:
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
        if "PROMETHEUS_MULTIPROC_DIR" not in os.environ:
            # Make TemporaryDirectory for prometheus multiprocessing
            # Note: global TemporaryDirectory will be automatically
            #   cleaned up upon exit.
            global prometheus_multiproc_dir
            prometheus_multiproc_dir = tempfile.TemporaryDirectory()
            os.environ[
                "PROMETHEUS_MULTIPROC_DIR"] = prometheus_multiproc_dir.name
        else:
            logger.warning(
                "Found PROMETHEUS_MULTIPROC_DIR was set by user. "
                "This directory must be wiped between vLLM runs or "
                "you will find inaccurate metrics. Unset the variable "
                "and vLLM will properly handle cleanup.")

173
        # Select random path for IPC.
174
175
176
        ipc_path = get_open_zmq_ipc_path()
        logger.info("Multiprocessing frontend to use %s for IPC Path.",
                    ipc_path)
177

178
        # Start RPCServer in separate process (holds the LLMEngine).
179
180
        # the current process might have CUDA context,
        # so we need to spawn a new process
181
182
        context = multiprocessing.get_context("spawn")

183
184
185
186
        # The Process can raise an exception during startup, which may
        # not actually result in an exitcode being reported. As a result
        # we use a shared variable to communicate the information.
        engine_alive = multiprocessing.Value('b', True, lock=False)
187
188
189
        engine_process = context.Process(target=run_mp_engine,
                                         args=(engine_args,
                                               UsageContext.OPENAI_API_SERVER,
190
                                               ipc_path, engine_alive))
191
        engine_process.start()
192
        engine_pid = engine_process.pid
193
        assert engine_pid is not None, "Engine process failed to start."
194
        logger.info("Started engine process with PID %d", engine_pid)
195
196
197

        # Build RPCClient, which conforms to EngineClient Protocol.
        engine_config = engine_args.create_engine_config()
198
199
200
201
        build_client = partial(MQLLMEngineClient, ipc_path, engine_config,
                               engine_pid)
        mq_engine_client = await asyncio.get_running_loop().run_in_executor(
            None, build_client)
202
        try:
203
204
            while True:
                try:
205
                    await mq_engine_client.setup()
206
                    break
207
                except TimeoutError:
208
209
                    if (not engine_process.is_alive()
                            or not engine_alive.value):
210
                        raise RuntimeError(
211
212
                            "Engine process failed to start. See stack "
                            "trace for the root cause.") from None
213

214
            yield mq_engine_client  # type: ignore[misc]
215
216
        finally:
            # Ensure rpc server process was terminated
217
            engine_process.terminate()
218
219

            # Close all open connections to the backend
220
            mq_engine_client.close()
221

222
223
224
225
226
            # Wait for engine process to join
            engine_process.join(4)
            if engine_process.exitcode is None:
                # Kill if taking longer than 5 seconds to stop
                engine_process.kill()
227

228
229
230
231
232
            # Lazy import for prometheus multiprocessing.
            # We need to set PROMETHEUS_MULTIPROC_DIR environment variable
            # before prometheus_client is imported.
            # See https://prometheus.github.io/client_python/multiprocess/
            from prometheus_client import multiprocess
233
            multiprocess.mark_process_dead(engine_process.pid)
234

235

Ethan Xu's avatar
Ethan Xu committed
236
router = APIRouter()
Zhuohan Li's avatar
Zhuohan Li committed
237

238

239
def mount_metrics(app: FastAPI):
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
    # Lazy import for prometheus multiprocessing.
    # We need to set PROMETHEUS_MULTIPROC_DIR environment variable
    # before prometheus_client is imported.
    # See https://prometheus.github.io/client_python/multiprocess/
    from prometheus_client import (CollectorRegistry, make_asgi_app,
                                   multiprocess)

    prometheus_multiproc_dir_path = os.getenv("PROMETHEUS_MULTIPROC_DIR", None)
    if prometheus_multiproc_dir_path is not None:
        logger.info("vLLM to use %s as PROMETHEUS_MULTIPROC_DIR",
                    prometheus_multiproc_dir_path)
        registry = CollectorRegistry()
        multiprocess.MultiProcessCollector(registry)

        # Add prometheus asgi middleware to route /metrics requests
        metrics_route = Mount("/metrics", make_asgi_app(registry=registry))
    else:
        # Add prometheus asgi middleware to route /metrics requests
        metrics_route = Mount("/metrics", make_asgi_app())

260
    # Workaround for 307 Redirect for /metrics
261
    metrics_route.path_regex = re.compile("^/metrics(?P<path>.*)$")
262
    app.routes.append(metrics_route)
263
264


265
266
267
268
269
270
def base(request: Request) -> OpenAIServing:
    # Reuse the existing instance
    return tokenization(request)


def chat(request: Request) -> Optional[OpenAIServingChat]:
271
272
273
    return request.app.state.openai_serving_chat


274
def completion(request: Request) -> Optional[OpenAIServingCompletion]:
275
276
277
    return request.app.state.openai_serving_completion


278
279
def embedding(request: Request) -> Optional[OpenAIServingEmbedding]:
    return request.app.state.openai_serving_embedding
280
281


282
283
def tokenization(request: Request) -> OpenAIServingTokenization:
    return request.app.state.openai_serving_tokenization
284
285


286
def engine_client(request: Request) -> EngineClient:
287
288
289
    return request.app.state.engine_client


Ethan Xu's avatar
Ethan Xu committed
290
@router.get("/health")
291
async def health(raw_request: Request) -> Response:
292
    """Health check."""
293
    await engine_client(raw_request).check_health()
294
295
296
    return Response(status_code=200)


Ethan Xu's avatar
Ethan Xu committed
297
@router.post("/tokenize")
298
async def tokenize(request: TokenizeRequest, raw_request: Request):
299
300
301
    handler = tokenization(raw_request)

    generator = await handler.create_tokenize(request)
302
303
304
    if isinstance(generator, ErrorResponse):
        return JSONResponse(content=generator.model_dump(),
                            status_code=generator.code)
305
    elif isinstance(generator, TokenizeResponse):
306
307
        return JSONResponse(content=generator.model_dump())

308
309
    assert_never(generator)

310

Ethan Xu's avatar
Ethan Xu committed
311
@router.post("/detokenize")
312
async def detokenize(request: DetokenizeRequest, raw_request: Request):
313
314
315
    handler = tokenization(raw_request)

    generator = await handler.create_detokenize(request)
316
317
318
    if isinstance(generator, ErrorResponse):
        return JSONResponse(content=generator.model_dump(),
                            status_code=generator.code)
319
    elif isinstance(generator, DetokenizeResponse):
320
321
        return JSONResponse(content=generator.model_dump())

322
323
    assert_never(generator)

324

Ethan Xu's avatar
Ethan Xu committed
325
@router.get("/v1/models")
326
async def show_available_models(raw_request: Request):
327
328
329
    handler = base(raw_request)

    models = await handler.show_available_models()
330
    return JSONResponse(content=models.model_dump())
Zhuohan Li's avatar
Zhuohan Li committed
331
332


Ethan Xu's avatar
Ethan Xu committed
333
@router.get("/version")
334
async def show_version():
335
    ver = {"version": VLLM_VERSION}
336
337
338
    return JSONResponse(content=ver)


Ethan Xu's avatar
Ethan Xu committed
339
@router.post("/v1/chat/completions")
340
341
async def create_chat_completion(request: ChatCompletionRequest,
                                 raw_request: Request):
342
343
344
345
    handler = chat(raw_request)
    if handler is None:
        return base(raw_request).create_error_response(
            message="The model does not support Chat Completions API")
346

347
    generator = await handler.create_chat_completion(request, raw_request)
348

349
350
351
    if isinstance(generator, ErrorResponse):
        return JSONResponse(content=generator.model_dump(),
                            status_code=generator.code)
352

353
    elif isinstance(generator, ChatCompletionResponse):
354
        return JSONResponse(content=generator.model_dump())
355

356
357
    return StreamingResponse(content=generator, media_type="text/event-stream")

358

Ethan Xu's avatar
Ethan Xu committed
359
@router.post("/v1/completions")
360
async def create_completion(request: CompletionRequest, raw_request: Request):
361
362
363
364
365
366
    handler = completion(raw_request)
    if handler is None:
        return base(raw_request).create_error_response(
            message="The model does not support Completions API")

    generator = await handler.create_completion(request, raw_request)
367
368
369
    if isinstance(generator, ErrorResponse):
        return JSONResponse(content=generator.model_dump(),
                            status_code=generator.code)
370
    elif isinstance(generator, CompletionResponse):
371
        return JSONResponse(content=generator.model_dump())
Zhuohan Li's avatar
Zhuohan Li committed
372

373
374
    return StreamingResponse(content=generator, media_type="text/event-stream")

Zhuohan Li's avatar
Zhuohan Li committed
375

Ethan Xu's avatar
Ethan Xu committed
376
@router.post("/v1/embeddings")
377
async def create_embedding(request: EmbeddingRequest, raw_request: Request):
378
379
380
381
382
383
    handler = embedding(raw_request)
    if handler is None:
        return base(raw_request).create_error_response(
            message="The model does not support Embeddings API")

    generator = await handler.create_embedding(request, raw_request)
384
385
386
    if isinstance(generator, ErrorResponse):
        return JSONResponse(content=generator.model_dump(),
                            status_code=generator.code)
387
    elif isinstance(generator, EmbeddingResponse):
388
389
        return JSONResponse(content=generator.model_dump())

390
391
    assert_never(generator)

392

393
394
395
396
397
398
if envs.VLLM_TORCH_PROFILER_DIR:
    logger.warning(
        "Torch Profiler is enabled in the API server. This should ONLY be "
        "used for local development!")

    @router.post("/start_profile")
399
    async def start_profile(raw_request: Request):
400
        logger.info("Starting profiler...")
401
        await engine_client(raw_request).start_profile()
402
403
404
405
        logger.info("Profiler started.")
        return Response(status_code=200)

    @router.post("/stop_profile")
406
    async def stop_profile(raw_request: Request):
407
        logger.info("Stopping profiler...")
408
        await engine_client(raw_request).stop_profile()
409
410
411
412
        logger.info("Profiler stopped.")
        return Response(status_code=200)


413
414
415
416
417
418
if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
    logger.warning(
        "Lora dynamic loading & unloading is enabled in the API server. "
        "This should ONLY be used for local development!")

    @router.post("/v1/load_lora_adapter")
419
420
    async def load_lora_adapter(request: LoadLoraAdapterRequest,
                                raw_request: Request):
421
422
423
424
425
426
427
        for route in [chat, completion, embedding]:
            handler = route(raw_request)
            if handler is not None:
                response = await handler.load_lora_adapter(request)
                if isinstance(response, ErrorResponse):
                    return JSONResponse(content=response.model_dump(),
                                        status_code=response.code)
428
429
430
431

        return Response(status_code=200, content=response)

    @router.post("/v1/unload_lora_adapter")
432
433
    async def unload_lora_adapter(request: UnloadLoraAdapterRequest,
                                  raw_request: Request):
434
435
436
437
438
439
440
        for route in [chat, completion, embedding]:
            handler = route(raw_request)
            if handler is not None:
                response = await handler.unload_lora_adapter(request)
                if isinstance(response, ErrorResponse):
                    return JSONResponse(content=response.model_dump(),
                                        status_code=response.code)
441
442
443
444

        return Response(status_code=200, content=response)


445
def build_app(args: Namespace) -> FastAPI:
446
447
448
449
450
451
452
    if args.disable_fastapi_docs:
        app = FastAPI(openapi_url=None,
                      docs_url=None,
                      redoc_url=None,
                      lifespan=lifespan)
    else:
        app = FastAPI(lifespan=lifespan)
Ethan Xu's avatar
Ethan Xu committed
453
454
    app.include_router(router)
    app.root_path = args.root_path
Zhuohan Li's avatar
Zhuohan Li committed
455

456
457
    mount_metrics(app)

Zhuohan Li's avatar
Zhuohan Li committed
458
459
460
461
462
463
464
465
    app.add_middleware(
        CORSMiddleware,
        allow_origins=args.allowed_origins,
        allow_credentials=args.allow_credentials,
        allow_methods=args.allowed_methods,
        allow_headers=args.allowed_headers,
    )

Ethan Xu's avatar
Ethan Xu committed
466
467
    @app.exception_handler(RequestValidationError)
    async def validation_exception_handler(_, exc):
468
469
        chat = app.state.openai_serving_chat
        err = chat.create_error_response(message=str(exc))
Ethan Xu's avatar
Ethan Xu committed
470
471
472
        return JSONResponse(err.model_dump(),
                            status_code=HTTPStatus.BAD_REQUEST)

473
    if token := envs.VLLM_API_KEY or args.api_key:
474
475
476

        @app.middleware("http")
        async def authentication(request: Request, call_next):
477
            root_path = "" if args.root_path is None else args.root_path
478
479
            if request.method == "OPTIONS":
                return await call_next(request)
480
            if not request.url.path.startswith(f"{root_path}/v1"):
481
482
483
484
485
486
                return await call_next(request)
            if request.headers.get("Authorization") != "Bearer " + token:
                return JSONResponse(content={"error": "Unauthorized"},
                                    status_code=401)
            return await call_next(request)

487
488
489
490
491
492
493
    @app.middleware("http")
    async def add_request_id(request: Request, call_next):
        request_id = request.headers.get("X-Request-Id") or uuid.uuid4().hex
        response = await call_next(request)
        response.headers["X-Request-Id"] = request_id
        return response

494
495
496
497
498
499
500
501
    for middleware in args.middleware:
        module_path, object_name = middleware.rsplit(".", 1)
        imported = getattr(importlib.import_module(module_path), object_name)
        if inspect.isclass(imported):
            app.add_middleware(imported)
        elif inspect.iscoroutinefunction(imported):
            app.middleware("http")(imported)
        else:
502
503
            raise ValueError(f"Invalid middleware {middleware}. "
                             f"Must be a function or a class.")
504

Ethan Xu's avatar
Ethan Xu committed
505
506
507
    return app


508
def init_app_state(
509
    engine_client: EngineClient,
510
511
    model_config: ModelConfig,
    state: State,
512
    args: Namespace,
513
) -> None:
514
    if args.served_model_name is not None:
515
        served_model_names = args.served_model_name
516
    else:
517
        served_model_names = [args.model]
518

519
520
521
522
523
    if args.disable_log_requests:
        request_logger = None
    else:
        request_logger = RequestLogger(max_log_len=args.max_log_len)

524
525
526
527
528
    base_model_paths = [
        BaseModelPath(name=name, model_path=args.model)
        for name in served_model_names
    ]

529
    state.engine_client = engine_client
530
    state.log_stats = not args.disable_log_stats
Ethan Xu's avatar
Ethan Xu committed
531

532
    state.openai_serving_chat = OpenAIServingChat(
533
        engine_client,
534
        model_config,
535
        base_model_paths,
536
537
538
539
540
        args.response_role,
        lora_modules=args.lora_modules,
        prompt_adapters=args.prompt_adapters,
        request_logger=request_logger,
        chat_template=args.chat_template,
541
        return_tokens_as_token_ids=args.return_tokens_as_token_ids,
542
        enable_auto_tools=args.enable_auto_tool_choice,
543
        tool_parser=args.tool_call_parser,
544
        enable_prompt_tokens_details=args.enable_prompt_tokens_details,
545
    ) if model_config.task == "generate" else None
546
    state.openai_serving_completion = OpenAIServingCompletion(
547
        engine_client,
548
        model_config,
549
        base_model_paths,
550
551
552
        lora_modules=args.lora_modules,
        prompt_adapters=args.prompt_adapters,
        request_logger=request_logger,
553
        return_tokens_as_token_ids=args.return_tokens_as_token_ids,
554
    ) if model_config.task == "generate" else None
555
    state.openai_serving_embedding = OpenAIServingEmbedding(
556
        engine_client,
557
        model_config,
558
        base_model_paths,
559
        request_logger=request_logger,
560
561
        chat_template=args.chat_template,
    ) if model_config.task == "embedding" else None
562
    state.openai_serving_tokenization = OpenAIServingTokenization(
563
        engine_client,
564
        model_config,
565
        base_model_paths,
566
567
568
569
        lora_modules=args.lora_modules,
        request_logger=request_logger,
        chat_template=args.chat_template,
    )
570
571


572
573
574
575
576
577
578
579
580
581
582
583
def create_server_socket(addr: Tuple[str, int]) -> socket.socket:
    family = socket.AF_INET
    if is_valid_ipv6_address(addr[0]):
        family = socket.AF_INET6

    sock = socket.socket(family=family, type=socket.SOCK_STREAM)
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    sock.bind(addr)

    return sock


584
async def run_server(args, **uvicorn_kwargs) -> None:
585
586
587
    logger.info("vLLM API server version %s", VLLM_VERSION)
    logger.info("args: %s", args)

588
589
590
591
592
593
594
595
596
    if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
        ToolParserManager.import_tool_parser(args.tool_parser_plugin)

    valide_tool_parses = ToolParserManager.tool_parsers.keys()
    if args.enable_auto_tool_choice \
        and args.tool_call_parser not in valide_tool_parses:
        raise KeyError(f"invalid tool call parser: {args.tool_call_parser} "
                       f"(chose from {{ {','.join(valide_tool_parses)} }})")

597
598
599
    # workaround to make sure that we bind the port before the engine is set up.
    # This avoids race conditions with ray.
    # see https://github.com/vllm-project/vllm/issues/8204
600
601
    sock_addr = (args.host or "", args.port)
    sock = create_server_socket(sock_addr)
602

603
604
605
606
607
608
    def signal_handler(*_) -> None:
        # Interrupt server on sigterm while initializing
        raise KeyboardInterrupt("terminated")

    signal.signal(signal.SIGTERM, signal_handler)

609
    async with build_async_engine_client(args) as engine_client:
610
611
        app = build_app(args)

612
613
        model_config = await engine_client.get_model_config()
        init_app_state(engine_client, model_config, app.state, args)
614
615
616
617
618
619
620
621
622
623
624

        shutdown_task = await serve_http(
            app,
            host=args.host,
            port=args.port,
            log_level=args.uvicorn_log_level,
            timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
            ssl_keyfile=args.ssl_keyfile,
            ssl_certfile=args.ssl_certfile,
            ssl_ca_certs=args.ssl_ca_certs,
            ssl_cert_reqs=args.ssl_cert_reqs,
625
626
627
            **uvicorn_kwargs,
        )

628
629
    # NB: Await server shutdown only after the backend context is exited
    await shutdown_task
630

631
632
    sock.close()

Ethan Xu's avatar
Ethan Xu committed
633
634
635
636
637
638
639
640

if __name__ == "__main__":
    # NOTE(simon):
    # This section should be in sync with vllm/scripts.py for CLI entrypoints.
    parser = FlexibleArgumentParser(
        description="vLLM OpenAI-Compatible RESTful API server.")
    parser = make_arg_parser(parser)
    args = parser.parse_args()
641
    validate_parsed_serve_args(args)
642

643
    uvloop.run(run_server(args))