async_llm.py 15.3 KB
Newer Older
1
2
# SPDX-License-Identifier: Apache-2.0

3
import asyncio
4
import os
5
from typing import AsyncGenerator, List, Mapping, Optional, Type, Union
6

7
8
import numpy as np

9
10
11
from vllm.config import ModelConfig, VllmConfig
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.engine.protocol import EngineClient
12
from vllm.envs import VLLM_V1_OUTPUT_PROC_CHUNK_SIZE
13
from vllm.inputs import INPUT_REGISTRY, InputRegistry, PromptType
14
from vllm.inputs.preprocess import InputPreprocessor
15
16
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
17
from vllm.outputs import RequestOutput
18
19
from vllm.pooling_params import PoolingParams
from vllm.prompt_adapter.request import PromptAdapterRequest
20
from vllm.sampling_params import RequestOutputKind, SamplingParams
21
22
23
from vllm.transformers_utils.tokenizer import AnyTokenizer
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
from vllm.usage.usage_lib import UsageContext
24
from vllm.utils import cdiv, kill_process_tree
25
from vllm.v1.engine.core_client import EngineCoreClient
26
from vllm.v1.engine.output_processor import OutputProcessor
27
from vllm.v1.engine.parallel_sampling import generate_parallel_sampling_async
28
from vllm.v1.engine.processor import Processor
29
from vllm.v1.executor.abstract import Executor
30
31
from vllm.v1.metrics.loggers import (LoggingStatLogger, PrometheusStatLogger,
                                     StatLoggerBase)
32
from vllm.v1.metrics.stats import IterationStats, SchedulerStats
33
34
35
36
37
38
39
40
41

logger = init_logger(__name__)


class AsyncLLM(EngineClient):

    def __init__(
        self,
        vllm_config: VllmConfig,
42
        executor_class: Type[Executor],
43
44
45
46
47
48
49
        log_stats: bool,
        usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
        input_registry: InputRegistry = INPUT_REGISTRY,
        use_cached_outputs: bool = False,
        log_requests: bool = True,
        start_engine_loop: bool = True,
    ) -> None:
50

51
52
        assert start_engine_loop

53
54
        self.model_config = vllm_config.model_config

55
56
        self.log_requests = log_requests
        self.log_stats = log_stats
57
58
59
60
        self.stat_loggers: List[StatLoggerBase] = []
        if self.log_stats:
            self.stat_loggers.extend([
                LoggingStatLogger(),
61
                PrometheusStatLogger(vllm_config),
62
            ])
63
64
65
66
67
68

        # Tokenizer (+ ensure liveness if running in another process).
        self.tokenizer = init_tokenizer_from_configs(
            model_config=vllm_config.model_config,
            scheduler_config=vllm_config.scheduler_config,
            parallel_config=vllm_config.parallel_config,
69
            lora_config=vllm_config.lora_config)
70
71
72
        self.tokenizer.ping()

        # Processor (converts Inputs --> EngineCoreRequests).
73
74
75
76
77
78
79
        self.processor = Processor(
            model_config=vllm_config.model_config,
            cache_config=vllm_config.cache_config,
            lora_config=vllm_config.lora_config,
            tokenizer=self.tokenizer,
            input_registry=input_registry,
        )
80

81
82
83
        # OutputProcessor (converts EngineCoreOutputs --> RequestOutput).
        self.output_processor = OutputProcessor(self.tokenizer,
                                                log_stats=self.log_stats)
84
85
86
87
88

        # EngineCore (starts the engine in background process).
        self.engine_core = EngineCoreClient.make_client(
            multiprocess_mode=True,
            asyncio_mode=True,
89
90
            vllm_config=vllm_config,
            executor_class=executor_class,
91
            log_stats=self.log_stats,
92
93
        )

94
        self.output_handler: Optional[asyncio.Task] = None
95
96
97
98
99
100
101
102

    @classmethod
    def from_engine_args(
        cls,
        engine_args: AsyncEngineArgs,
        engine_config: Optional[VllmConfig] = None,
        start_engine_loop: bool = True,
        usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
103
    ) -> "AsyncLLM":
104
105
106
107
        """Create an AsyncLLM from the EngineArgs."""

        # Create the engine configs.
        if engine_config is None:
108
            vllm_config = engine_args.create_engine_config(usage_context)
109
110
111
        else:
            vllm_config = engine_config

112
        executor_class = Executor.get_class(vllm_config)
113
114
115
116
117
118
119
120
121
122
123
124
125
126

        # Create the AsyncLLM.
        return cls(
            vllm_config=vllm_config,
            executor_class=executor_class,
            log_requests=not engine_args.disable_log_requests,
            log_stats=not engine_args.disable_log_stats,
            start_engine_loop=start_engine_loop,
            usage_context=usage_context,
        )

    def shutdown(self):
        """Shutdown, cleaning up the background proc and IPC."""

127
128
        if engine_core := getattr(self, "engine_core", None):
            engine_core.shutdown()
129
130
131
132
133
134
135
136
137
138
139
140
141
142

        if handler := getattr(self, "output_handler", None):
            handler.cancel()

    async def add_request(
        self,
        request_id: str,
        prompt: PromptType,
        params: Union[SamplingParams, PoolingParams],
        arrival_time: Optional[float] = None,
        lora_request: Optional[LoRARequest] = None,
        trace_headers: Optional[Mapping[str, str]] = None,
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        priority: int = 0,
143
    ) -> asyncio.Queue[RequestOutput]:
144
145
        """Add new request to the AsyncLLM."""

146
        # 1) Create a new output queue for the request.
147
        if self.output_processor.is_request_active(request_id):
148
            raise ValueError(f"Request id {request_id} already running.")
149
        queue: asyncio.Queue[RequestOutput] = asyncio.Queue()
150

151
152
153
154
155
156
        # 2) Convert Input --> Request.
        request = self.processor.process_inputs(request_id, prompt, params,
                                                arrival_time, lora_request,
                                                trace_headers,
                                                prompt_adapter_request,
                                                priority)
157

158
159
        # 3) Add the request to OutputProcessor (this process).
        self.output_processor.add_request(request, queue)
160
161

        # 4) Add the EngineCoreRequest to EngineCore (separate process).
162
        await self.engine_core.add_request_async(request)
163

164
165
166
        if self.log_requests:
            logger.info("Added request %s.", request_id)

167
        return queue
168
169
170
171
172
173

    # TODO: we should support multiple prompts in one call, as you
    # can do with LLM.generate. So that for multi-prompt completion
    # requests we don't need to send multiple messages to core proc,
    # and so we don't need multiple streams which then get
    # re-multiplexed in the API server anyhow.
174
    async def _generate(
175
176
177
178
179
180
181
182
183
184
185
186
        self,
        prompt: PromptType,
        sampling_params: SamplingParams,
        request_id: str,
        lora_request: Optional[LoRARequest] = None,
        trace_headers: Optional[Mapping[str, str]] = None,
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        priority: int = 0,
    ) -> AsyncGenerator[RequestOutput, None]:
        """
        Main function called by the API server to kick off a request
            * 1) Making an AsyncStream corresponding to the Request.
187
            * 2) Processing the Input.
188
189
190
191
192
193
194
195
196
197
198
            * 3) Adding the Request to the Detokenizer.
            * 4) Adding the Request to the EngineCore (separate process).

        A separate output_handler loop runs in a background AsyncIO task, 
        pulling outputs from EngineCore and putting them into the 
        per-request AsyncStream.

        The caller of generate() iterates the returned AsyncGenerator,
        returning the RequestOutput back to the caller.
        """

199
200
201
202
203
204
205
206
207
        try:
            # We start the output_handler on the first call to generate() so
            # we can call __init__ before the event loop, which enables us
            # to handle startup failure gracefully in the OpenAI server.
            if self.output_handler is None:
                self.output_handler = asyncio.create_task(
                    self._run_output_handler())

            q = await self.add_request(
208
209
210
211
212
213
214
                request_id,
                prompt,
                sampling_params,
                lora_request=lora_request,
                trace_headers=trace_headers,
                prompt_adapter_request=prompt_adapter_request,
                priority=priority,
215
            )
216

217
218
            # The output_handler task pushes items into the queue.
            # This task pulls from the queue and yields to caller.
219
220
            finished = False
            while not finished:
221
222
                # Note: drain queue without await if possible (avoids
                # task switching under load which helps performance).
223
                out = q.get_nowait() if not q.empty() else await q.get()
224

225
226
227
228
229
230
231
232
                # Coalesce any additional queued outputs
                while not q.empty():
                    next_out = q.get_nowait()
                    if sampling_params.output_kind == RequestOutputKind.DELTA:
                        out.add(next_out)
                    else:
                        out = next_out

233
                # Note: both OutputProcessor and EngineCore handle their
234
                # own request cleanup based on finished.
235
                finished = out.finished
236
237
238
239
240
241
242
243
                yield out

        # If the request is disconnected by the client, the
        # generate() task will be canceled. So, we abort the
        # request if we end up here.
        except asyncio.CancelledError:
            await self.abort(request_id)
            raise
244

245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
    def generate(
        self,
        prompt: PromptType,
        sampling_params: SamplingParams,
        request_id: str,
        lora_request: Optional[LoRARequest] = None,
        trace_headers: Optional[Mapping[str, str]] = None,
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        priority: int = 0,
    ) -> AsyncGenerator[RequestOutput, None]:
        kwargs = dict(prompt=prompt,
                      sampling_params=sampling_params,
                      request_id=request_id,
                      lora_request=lora_request,
                      trace_headers=trace_headers,
                      prompt_adapter_request=prompt_adapter_request,
                      priority=priority)
        if sampling_params.n is None or sampling_params.n == 1:
            return self._generate(**kwargs)
        else:
            # Special handling for parallel sampling requests
            return generate_parallel_sampling_async(generate=self._generate,
                                                    **kwargs)

269
270
271
272
273
    async def _run_output_handler(self):
        """Background loop: pulls from EngineCore and pushes to AsyncStreams."""

        try:
            while True:
274
                # 1) Pull EngineCoreOutputs from the EngineCore.
275
276
                outputs = await self.engine_core.get_output_async()

277
278
                iteration_stats = IterationStats() if self.log_stats else None

279
280
281
282
283
284
285
286
287
288
289
290
291
292
                # Split outputs into chunks of at most
                # VLLM_V1_OUTPUT_PROC_CHUNK_SIZE, so that we don't block the
                # event loop for too long.
                num_outputs = len(outputs.outputs)
                if num_outputs <= VLLM_V1_OUTPUT_PROC_CHUNK_SIZE:
                    slices = (outputs.outputs, )
                else:
                    slices = np.array_split(
                        outputs.outputs,
                        cdiv(num_outputs, VLLM_V1_OUTPUT_PROC_CHUNK_SIZE))

                for i, outputs_slice in enumerate(slices):
                    # 2) Process EngineCoreOutputs.
                    processed_outputs = self.output_processor.process_outputs(
293
                        outputs_slice, outputs.timestamp, iteration_stats)
294
295
296
297
298
299
300
301
302
303
                    # NOTE: RequestOutputs are pushed to their queues.
                    assert not processed_outputs.request_outputs

                    # Allow other asyncio tasks to run between chunks
                    if i + 1 < len(slices):
                        await asyncio.sleep(0)

                    # 3) Abort any reqs that finished due to stop strings.
                    await self.engine_core.abort_requests_async(
                        processed_outputs.reqs_to_abort)
304

305
306
                # 4) Logging.
                # TODO(rob): make into a coroutine and launch it in
307
                # background thread once Prometheus overhead is non-trivial.
308
309
                self._log_stats(
                    scheduler_stats=outputs.scheduler_stats,
310
                    iteration_stats=iteration_stats,
311
                )
312

313
314
315
        except Exception as e:
            logger.exception("EngineCore output handler hit an error: %s", e)
            kill_process_tree(os.getpid())
316
317

    async def abort(self, request_id: str) -> None:
318
        """Abort RequestId in OutputProcessor and EngineCore."""
319
320
321

        request_ids = [request_id]
        await self.engine_core.abort_requests_async(request_ids)
322
        self.output_processor.abort_requests(request_ids)
323

324
325
        if self.log_requests:
            logger.info("Aborted request %s.", request_id)
326

327
328
    def _log_stats(
        self,
329
330
        scheduler_stats: Optional[SchedulerStats],
        iteration_stats: Optional[IterationStats],
331
    ):
332
333
334
        if not self.log_stats:
            return

335
336
        assert scheduler_stats is not None
        assert iteration_stats is not None
337
        for logger in self.stat_loggers:
338
339
            logger.log(scheduler_stats=scheduler_stats,
                       iteration_stats=iteration_stats)
340

341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
    def encode(
        self,
        prompt: PromptType,
        pooling_params: PoolingParams,
        request_id: str,
        lora_request: Optional[LoRARequest] = None,
        trace_headers: Optional[Mapping[str, str]] = None,
        priority: int = 0,
    ):
        raise ValueError("Not Supported on V1 yet.")

    async def get_model_config(self) -> ModelConfig:
        return self.model_config

    async def get_decoding_config(self):
        raise ValueError("Not Supported on V1 yet.")

358
359
360
    async def get_input_preprocessor(self) -> InputPreprocessor:
        return self.processor.input_preprocessor

361
362
363
364
    async def get_tokenizer(
        self,
        lora_request: Optional[LoRARequest] = None,
    ) -> AnyTokenizer:
365
        return self.tokenizer.get_lora_tokenizer(lora_request)
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380

    async def is_tracing_enabled(self) -> bool:
        return False

    async def do_log_stats(
        self,
        scheduler_outputs=None,
        model_output=None,
    ) -> None:
        logger.debug("Called do_log_stats.")

    async def check_health(self) -> None:
        logger.debug("Called check_health.")

    async def start_profile(self) -> None:
381
        await self.engine_core.profile_async(True)
382
383

    async def stop_profile(self) -> None:
384
        await self.engine_core.profile_async(False)
385

386
387
388
    async def reset_prefix_cache(self) -> None:
        await self.engine_core.reset_prefix_cache_async()

389
390
391
392
393
394
    async def sleep(self, level: int = 1) -> None:
        await self.engine_core.sleep_async(level)

    async def wake_up(self) -> None:
        await self.engine_core.wake_up_async()

395
396
397
398
    async def add_lora(self, lora_request: LoRARequest) -> None:
        """Load a new LoRA adapter into the engine for future requests."""
        await self.engine_core.add_lora_async(lora_request)

399
400
401
402
403
404
405
406
407
408
409
410
411
412
    @property
    def is_running(self) -> bool:
        return True

    @property
    def is_stopped(self) -> bool:
        return False

    @property
    def errored(self) -> bool:
        return False

    @property
    def dead_error(self) -> BaseException:
413
        return Exception()  # TODO: implement