serve.py 8.83 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4

import argparse
5
import signal
6
from typing import Optional
7
8
9

import uvloop

10
import vllm
11
import vllm.envs as envs
12
from vllm.entrypoints.cli.types import CLISubcommand
13
14
from vllm.entrypoints.openai.api_server import (run_server, run_server_worker,
                                                setup_server)
15
16
from vllm.entrypoints.openai.cli_args import (make_arg_parser,
                                              validate_parsed_serve_args)
17
from vllm.entrypoints.utils import (VLLM_SUBCMD_PARSER_EPILOG,
18
                                    show_filtered_argument_or_group_from_help)
19
20
from vllm.logger import init_logger
from vllm.usage.usage_lib import UsageContext
21
22
from vllm.utils import (FlexibleArgumentParser, decorate_logs, get_tcp_uri,
                        set_process_title)
23
from vllm.v1.engine.core import EngineCoreProc
24
from vllm.v1.engine.utils import CoreEngineProcManager, launch_core_engines
25
from vllm.v1.executor.abstract import Executor
26
from vllm.v1.metrics.prometheus import setup_multiprocess_prometheus
27
28
from vllm.v1.utils import (APIServerProcessManager,
                           wait_for_completion_or_failure)
29
30

logger = init_logger(__name__)
31
32
33
34


class ServeSubcommand(CLISubcommand):
    """The `serve` subcommand for the vLLM CLI. """
35
    name = "serve"
36
37
38

    @staticmethod
    def cmd(args: argparse.Namespace) -> None:
39
40
41
        # If model is specified in CLI (as positional arg), it takes precedence
        if hasattr(args, 'model_tag') and args.model_tag is not None:
            args.model = args.model_tag
42

43
        if args.headless or args.api_server_count < 1:
44
45
            run_headless(args)
        else:
46
47
48
49
50
            if args.api_server_count > 1:
                run_multi_api_server(args)
            else:
                # Single API server (this process).
                uvloop.run(run_server(args))
51
52
53
54
55
56
57
58
59

    def validate(self, args: argparse.Namespace) -> None:
        validate_parsed_serve_args(args)

    def subparser_init(
            self,
            subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
        serve_parser = subparsers.add_parser(
            "serve",
60
61
            help="Start the vLLM OpenAI Compatible API server.",
            description="Start the vLLM OpenAI Compatible API server.",
62
            usage="vllm serve [model_tag] [options]")
63

64
        serve_parser = make_arg_parser(serve_parser)
65
        show_filtered_argument_or_group_from_help(serve_parser, ["serve"])
66
        serve_parser.epilog = VLLM_SUBCMD_PARSER_EPILOG
67
        return serve_parser
68
69


70
def cmd_init() -> list[CLISubcommand]:
71
    return [ServeSubcommand()]
72
73
74
75


def run_headless(args: argparse.Namespace):

76
77
    if args.api_server_count > 1:
        raise ValueError("api_server_count can't be set in headless mode")
78

79
    # Create the EngineConfig.
80
    engine_args = vllm.AsyncEngineArgs.from_cli_args(args)
81
    usage_context = UsageContext.OPENAI_API_SERVER
82
83
    vllm_config = engine_args.create_engine_config(usage_context=usage_context,
                                                   headless=True)
84
85

    if not envs.VLLM_USE_V1:
86
        raise ValueError("Headless mode is only supported for V1")
87

88
89
    if engine_args.data_parallel_hybrid_lb:
        raise ValueError("data_parallel_hybrid_lb is not applicable in "
90
91
                         "headless mode")

92
93
94
95
    parallel_config = vllm_config.parallel_config
    local_engine_count = parallel_config.data_parallel_size_local

    if local_engine_count <= 0:
96
97
        raise ValueError("data_parallel_size_local must be > 0 in "
                         "headless mode")
98

99
100
101
102
    host = parallel_config.data_parallel_master_ip
    port = engine_args.data_parallel_rpc_port  # add to config too
    handshake_address = get_tcp_uri(host, port)

103
104
105
106
107
108
109
110
111
112
    # Catch SIGTERM and SIGINT to allow graceful shutdown.
    def signal_handler(signum, frame):
        logger.debug("Received %d signal.", signum)
        raise SystemExit

    signal.signal(signal.SIGTERM, signal_handler)
    signal.signal(signal.SIGINT, signal_handler)

    logger.info(
        "Launching %d data parallel engine(s) in headless mode, "
113
        "with head node address %s.", local_engine_count, handshake_address)
114
115
116
117
118

    # Create the engines.
    engine_manager = CoreEngineProcManager(
        target_fn=EngineCoreProc.run_engine_core,
        local_engine_count=local_engine_count,
119
        start_index=vllm_config.parallel_config.data_parallel_rank,
120
121
        local_start_index=0,
        vllm_config=vllm_config,
122
        local_client=False,
123
        handshake_address=handshake_address,
124
125
126
127
128
129
130
131
132
        executor_class=Executor.get_class(vllm_config),
        log_stats=not engine_args.disable_log_stats,
    )

    try:
        engine_manager.join_first()
    finally:
        logger.info("Shutting down.")
        engine_manager.close()
133
134
135
136
137
138
139
140


def run_multi_api_server(args: argparse.Namespace):

    assert not args.headless
    num_api_servers = args.api_server_count
    assert num_api_servers > 0

141
    orig_mm_processor_cache_gb = args.mm_processor_cache_gb
142

143
144
145
    if num_api_servers > 1:
        setup_multiprocess_prometheus()

146
        # Not compatible with API server scale-out
147
        args.mm_processor_cache_gb = 0
148

149
150
    listen_address, sock = setup_server(args)

151
    engine_args = vllm.AsyncEngineArgs.from_cli_args(args)
152
153
154
155
156
157
158
159
160
161
162
163
    usage_context = UsageContext.OPENAI_API_SERVER
    vllm_config = engine_args.create_engine_config(usage_context=usage_context)
    model_config = vllm_config.model_config

    if num_api_servers > 1:
        if not envs.VLLM_USE_V1:
            raise ValueError("api_server_count > 1 is only supported for V1")

        if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
            raise ValueError("VLLM_ALLOW_RUNTIME_LORA_UPDATING cannot be used "
                             "with api_server_count > 1")

164
        if model_config.is_multimodal_model and orig_mm_processor_cache_gb > 0:
165
166
            logger.warning("Multi-modal processor cache is disabled because "
                           "it is not compatible with `api_server_count > 1`.")
167

168
169
170
    executor_class = Executor.get_class(vllm_config)
    log_stats = not engine_args.disable_log_stats

171
    parallel_config = vllm_config.parallel_config
172
173
    dp_rank = parallel_config.data_parallel_rank
    external_dp_lb = parallel_config.data_parallel_external_lb
174
175
    hybrid_dp_lb = parallel_config.data_parallel_hybrid_lb
    assert external_dp_lb or hybrid_dp_lb or dp_rank == 0
176

177
    api_server_manager: Optional[APIServerProcessManager] = None
178

179
180
181
    with launch_core_engines(vllm_config, executor_class, log_stats,
                             num_api_servers) as (local_engine_manager,
                                                  coordinator, addresses):
182

183
184
        # Construct common args for the APIServerProcessManager up-front.
        api_server_manager_kwargs = dict(
Rui Qiao's avatar
Rui Qiao committed
185
186
187
188
189
            target_server_fn=run_api_server_worker_proc,
            listen_address=listen_address,
            sock=sock,
            args=args,
            num_servers=num_api_servers,
190
191
192
193
194
            input_addresses=addresses.inputs,
            output_addresses=addresses.outputs,
            stats_update_address=coordinator.get_stats_publish_address()
            if coordinator else None)

195
        # For dp ranks > 0 in external/hybrid DP LB modes, we must delay the
196
197
198
199
        # start of the API servers until the local engine is started
        # (after the launcher context manager exits),
        # since we get the front-end stats update address from the coordinator
        # via the handshake with the local engine.
200
        if dp_rank == 0 or not (external_dp_lb or hybrid_dp_lb):
201
202
203
204
205
206
207
208
            # Start API servers using the manager.
            api_server_manager = APIServerProcessManager(
                **api_server_manager_kwargs)

    # Start API servers now if they weren't already started.
    if api_server_manager is None:
        api_server_manager_kwargs["stats_update_address"] = (
            addresses.frontend_stats_publish_address)
209
        api_server_manager = APIServerProcessManager(
210
211
212
213
214
215
            **api_server_manager_kwargs)

    # Wait for API servers
    wait_for_completion_or_failure(api_server_manager=api_server_manager,
                                   engine_manager=local_engine_manager,
                                   coordinator=coordinator)
216
217
218
219
220
221
222
223
224


def run_api_server_worker_proc(listen_address,
                               sock,
                               args,
                               client_config=None,
                               **uvicorn_kwargs) -> None:
    """Entrypoint for individual API server worker processes."""

225
226
227
    # Set process title and add process-specific prefix to stdout and stderr.
    server_index = client_config.get("client_index", 0) if client_config else 0
    set_process_title("APIServer", str(server_index))
228
    decorate_logs()
229
230
231
232

    uvloop.run(
        run_server_worker(listen_address, sock, args, client_config,
                          **uvicorn_kwargs))