async_llm.py 16.6 KB
Newer Older
1
2
# SPDX-License-Identifier: Apache-2.0

3
import asyncio
4
import logging
5
import os
6
7
from collections.abc import AsyncGenerator, Mapping
from typing import Optional, Union
8

9
10
import numpy as np

11
import vllm.envs as envs
12
13
14
from vllm.config import ModelConfig, VllmConfig
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.engine.protocol import EngineClient
15
from vllm.envs import VLLM_V1_OUTPUT_PROC_CHUNK_SIZE
16
from vllm.inputs import INPUT_REGISTRY, InputRegistry, PromptType
17
from vllm.inputs.preprocess import InputPreprocessor
18
19
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
20
from vllm.outputs import RequestOutput
21
22
from vllm.pooling_params import PoolingParams
from vllm.prompt_adapter.request import PromptAdapterRequest
23
from vllm.sampling_params import RequestOutputKind, SamplingParams
24
25
26
from vllm.transformers_utils.tokenizer import AnyTokenizer
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
from vllm.usage.usage_lib import UsageContext
27
from vllm.utils import cdiv, kill_process_tree
28
from vllm.v1.engine.core_client import EngineCoreClient
29
from vllm.v1.engine.output_processor import OutputProcessor
30
from vllm.v1.engine.parallel_sampling import ParentRequest
31
from vllm.v1.engine.processor import Processor
32
from vllm.v1.executor.abstract import Executor
33
34
from vllm.v1.metrics.loggers import (LoggingStatLogger, PrometheusStatLogger,
                                     StatLoggerBase)
35
from vllm.v1.metrics.stats import IterationStats, SchedulerStats
36
37
38
39
40
41
42
43
44

logger = init_logger(__name__)


class AsyncLLM(EngineClient):

    def __init__(
        self,
        vllm_config: VllmConfig,
45
        executor_class: type[Executor],
46
47
48
49
50
51
52
        log_stats: bool,
        usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
        input_registry: InputRegistry = INPUT_REGISTRY,
        use_cached_outputs: bool = False,
        log_requests: bool = True,
        start_engine_loop: bool = True,
    ) -> None:
53
54
55
56
57
58
        if not envs.VLLM_USE_V1:
            raise ValueError(
                "Using V1 AsyncLLMEngine, but envs.VLLM_USE_V1=False. "
                "This should not happen. As a workaround, try using "
                "AsyncLLMEngine.from_vllm_config(...) or explicitly set "
                "VLLM_USE_V1=0 or 1 and report this issue on Github.")
59

60
61
        assert start_engine_loop

62
63
        self.model_config = vllm_config.model_config

64
65
        self.log_requests = log_requests
        self.log_stats = log_stats
66
        self.stat_loggers: list[StatLoggerBase] = []
67
        if self.log_stats:
68
69
70
            if logger.isEnabledFor(logging.INFO):
                self.stat_loggers.append(LoggingStatLogger())
            self.stat_loggers.append(PrometheusStatLogger(vllm_config))
71
72
73
74
75
76

        # Tokenizer (+ ensure liveness if running in another process).
        self.tokenizer = init_tokenizer_from_configs(
            model_config=vllm_config.model_config,
            scheduler_config=vllm_config.scheduler_config,
            parallel_config=vllm_config.parallel_config,
77
            lora_config=vllm_config.lora_config)
78
79
80
        self.tokenizer.ping()

        # Processor (converts Inputs --> EngineCoreRequests).
81
        self.processor = Processor(
82
            vllm_config=vllm_config,
83
84
85
            tokenizer=self.tokenizer,
            input_registry=input_registry,
        )
86

87
88
89
        # OutputProcessor (converts EngineCoreOutputs --> RequestOutput).
        self.output_processor = OutputProcessor(self.tokenizer,
                                                log_stats=self.log_stats)
90
91
92
93
94

        # EngineCore (starts the engine in background process).
        self.engine_core = EngineCoreClient.make_client(
            multiprocess_mode=True,
            asyncio_mode=True,
95
96
            vllm_config=vllm_config,
            executor_class=executor_class,
97
            log_stats=self.log_stats,
98
99
        )

100
        self.output_handler: Optional[asyncio.Task] = None
101

102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
    @classmethod
    def from_vllm_config(
        cls,
        vllm_config: VllmConfig,
        start_engine_loop: bool = True,
        usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
        stat_loggers: Optional[dict[str, StatLoggerBase]] = None,
        disable_log_requests: bool = False,
        disable_log_stats: bool = False,
    ) -> "AsyncLLM":
        if not envs.VLLM_USE_V1:
            raise ValueError(
                "Using V1 AsyncLLMEngine, but envs.VLLM_USE_V1=False. "
                "This should not happen. As a workaround, try using "
                "AsyncLLMEngine.from_vllm_config(...) or explicitly set "
                "VLLM_USE_V1=0 or 1 and report this issue on Github.")

        # FIXME(rob): refactor VllmConfig to include the StatLoggers
        # include StatLogger in the Oracle decision.
        if stat_loggers is not None:
            raise ValueError("Custom StatLoggers are not yet supported on V1. "
                             "Explicitly set VLLM_USE_V1=0 to disable V1.")

        # Create the LLMEngine.
        return cls(
            vllm_config=vllm_config,
            executor_class=Executor.get_class(vllm_config),
            start_engine_loop=start_engine_loop,
            log_requests=not disable_log_requests,
            log_stats=not disable_log_stats,
            usage_context=usage_context,
        )

135
136
137
138
139
140
    @classmethod
    def from_engine_args(
        cls,
        engine_args: AsyncEngineArgs,
        start_engine_loop: bool = True,
        usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
141
    ) -> "AsyncLLM":
142
143
144
        """Create an AsyncLLM from the EngineArgs."""

        # Create the engine configs.
145
        vllm_config = engine_args.create_engine_config(usage_context)
146
        executor_class = Executor.get_class(vllm_config)
147
148
149
150
151
152
153
154
155
156
157
158
159
160

        # Create the AsyncLLM.
        return cls(
            vllm_config=vllm_config,
            executor_class=executor_class,
            log_requests=not engine_args.disable_log_requests,
            log_stats=not engine_args.disable_log_stats,
            start_engine_loop=start_engine_loop,
            usage_context=usage_context,
        )

    def shutdown(self):
        """Shutdown, cleaning up the background proc and IPC."""

161
162
        if engine_core := getattr(self, "engine_core", None):
            engine_core.shutdown()
163
164
165
166
167
168
169
170
171
172
173
174
175
176

        if handler := getattr(self, "output_handler", None):
            handler.cancel()

    async def add_request(
        self,
        request_id: str,
        prompt: PromptType,
        params: Union[SamplingParams, PoolingParams],
        arrival_time: Optional[float] = None,
        lora_request: Optional[LoRARequest] = None,
        trace_headers: Optional[Mapping[str, str]] = None,
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        priority: int = 0,
177
    ) -> asyncio.Queue[RequestOutput]:
178
179
        """Add new request to the AsyncLLM."""

180
        # 1) Create a new output queue for the request.
181
        queue: asyncio.Queue[RequestOutput] = asyncio.Queue()
182

183
184
185
186
187
188
        # 2) Fan out child requests (for n>1)
        parent_req = ParentRequest.from_params(request_id, params)
        n = params.n if isinstance(params, SamplingParams) else 1
        for idx in range(n):
            if parent_req is not None:
                request_id, params = parent_req.get_child_info(idx)
189

190
191
192
193
194
195
            # 3) Convert Input --> Request.
            request = self.processor.process_inputs(request_id, prompt, params,
                                                    arrival_time, lora_request,
                                                    trace_headers,
                                                    prompt_adapter_request,
                                                    priority)
196

197
198
            # 4) Add the request to OutputProcessor (this process).
            self.output_processor.add_request(request, parent_req, idx, queue)
199

200
201
202
203
204
            # 5) Add the EngineCoreRequest to EngineCore (separate process).
            await self.engine_core.add_request_async(request)

            if self.log_requests:
                logger.info("Added request %s.", request_id)
205

206
        return queue
207
208
209
210
211
212

    # TODO: we should support multiple prompts in one call, as you
    # can do with LLM.generate. So that for multi-prompt completion
    # requests we don't need to send multiple messages to core proc,
    # and so we don't need multiple streams which then get
    # re-multiplexed in the API server anyhow.
213
    async def generate(
214
215
216
217
218
219
220
221
222
223
224
225
        self,
        prompt: PromptType,
        sampling_params: SamplingParams,
        request_id: str,
        lora_request: Optional[LoRARequest] = None,
        trace_headers: Optional[Mapping[str, str]] = None,
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        priority: int = 0,
    ) -> AsyncGenerator[RequestOutput, None]:
        """
        Main function called by the API server to kick off a request
            * 1) Making an AsyncStream corresponding to the Request.
226
            * 2) Processing the Input.
227
228
229
            * 3) Adding the Request to the Detokenizer.
            * 4) Adding the Request to the EngineCore (separate process).

230
231
        A separate output_handler loop runs in a background AsyncIO task,
        pulling outputs from EngineCore and putting them into the
232
233
234
235
236
237
        per-request AsyncStream.

        The caller of generate() iterates the returned AsyncGenerator,
        returning the RequestOutput back to the caller.
        """

238
239
240
241
242
243
244
245
246
        try:
            # We start the output_handler on the first call to generate() so
            # we can call __init__ before the event loop, which enables us
            # to handle startup failure gracefully in the OpenAI server.
            if self.output_handler is None:
                self.output_handler = asyncio.create_task(
                    self._run_output_handler())

            q = await self.add_request(
247
248
249
250
251
252
253
                request_id,
                prompt,
                sampling_params,
                lora_request=lora_request,
                trace_headers=trace_headers,
                prompt_adapter_request=prompt_adapter_request,
                priority=priority,
254
            )
255

256
257
            # The output_handler task pushes items into the queue.
            # This task pulls from the queue and yields to caller.
258
259
            finished = False
            while not finished:
260
261
                # Note: drain queue without await if possible (avoids
                # task switching under load which helps performance).
262
                out = q.get_nowait() if not q.empty() else await q.get()
263

264
265
266
267
268
269
270
271
                # Coalesce any additional queued outputs
                while not q.empty():
                    next_out = q.get_nowait()
                    if sampling_params.output_kind == RequestOutputKind.DELTA:
                        out.add(next_out)
                    else:
                        out = next_out

272
                # Note: both OutputProcessor and EngineCore handle their
273
                # own request cleanup based on finished.
274
                finished = out.finished
275
276
277
278
279
280
281
282
                yield out

        # If the request is disconnected by the client, the
        # generate() task will be canceled. So, we abort the
        # request if we end up here.
        except asyncio.CancelledError:
            await self.abort(request_id)
            raise
283
284
285
286
287
288

    async def _run_output_handler(self):
        """Background loop: pulls from EngineCore and pushes to AsyncStreams."""

        try:
            while True:
289
                # 1) Pull EngineCoreOutputs from the EngineCore.
290
                outputs = await self.engine_core.get_output_async()
291
                num_outputs = len(outputs.outputs)
292

293
294
                iteration_stats = IterationStats() if (
                    self.log_stats and num_outputs) else None
295

296
297
298
299
300
301
302
303
304
305
306
307
308
                # Split outputs into chunks of at most
                # VLLM_V1_OUTPUT_PROC_CHUNK_SIZE, so that we don't block the
                # event loop for too long.
                if num_outputs <= VLLM_V1_OUTPUT_PROC_CHUNK_SIZE:
                    slices = (outputs.outputs, )
                else:
                    slices = np.array_split(
                        outputs.outputs,
                        cdiv(num_outputs, VLLM_V1_OUTPUT_PROC_CHUNK_SIZE))

                for i, outputs_slice in enumerate(slices):
                    # 2) Process EngineCoreOutputs.
                    processed_outputs = self.output_processor.process_outputs(
309
                        outputs_slice, outputs.timestamp, iteration_stats)
310
311
312
313
314
315
316
317
318
319
                    # NOTE: RequestOutputs are pushed to their queues.
                    assert not processed_outputs.request_outputs

                    # Allow other asyncio tasks to run between chunks
                    if i + 1 < len(slices):
                        await asyncio.sleep(0)

                    # 3) Abort any reqs that finished due to stop strings.
                    await self.engine_core.abort_requests_async(
                        processed_outputs.reqs_to_abort)
320

321
322
                # 4) Logging.
                # TODO(rob): make into a coroutine and launch it in
323
                # background thread once Prometheus overhead is non-trivial.
324
                self._record_stats(
325
                    scheduler_stats=outputs.scheduler_stats,
326
                    iteration_stats=iteration_stats,
327
                )
328

329
330
331
        except Exception as e:
            logger.exception("EngineCore output handler hit an error: %s", e)
            kill_process_tree(os.getpid())
332
333

    async def abort(self, request_id: str) -> None:
334
        """Abort RequestId in OutputProcessor and EngineCore."""
335

336
        request_ids = self.output_processor.abort_requests((request_id, ))
337
338
        await self.engine_core.abort_requests_async(request_ids)

339
340
        if self.log_requests:
            logger.info("Aborted request %s.", request_id)
341

342
    def _record_stats(
343
        self,
344
345
        scheduler_stats: Optional[SchedulerStats],
        iteration_stats: Optional[IterationStats],
346
    ):
347
348
349
        if not self.log_stats:
            return

350
        assert scheduler_stats is not None
351
352
353
        for stat_logger in self.stat_loggers:
            stat_logger.record(scheduler_stats=scheduler_stats,
                               iteration_stats=iteration_stats)
354

355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
    def encode(
        self,
        prompt: PromptType,
        pooling_params: PoolingParams,
        request_id: str,
        lora_request: Optional[LoRARequest] = None,
        trace_headers: Optional[Mapping[str, str]] = None,
        priority: int = 0,
    ):
        raise ValueError("Not Supported on V1 yet.")

    async def get_model_config(self) -> ModelConfig:
        return self.model_config

    async def get_decoding_config(self):
        raise ValueError("Not Supported on V1 yet.")

372
373
374
    async def get_input_preprocessor(self) -> InputPreprocessor:
        return self.processor.input_preprocessor

375
376
377
378
    async def get_tokenizer(
        self,
        lora_request: Optional[LoRARequest] = None,
    ) -> AnyTokenizer:
379
        return self.tokenizer.get_lora_tokenizer(lora_request)
380
381
382
383
384
385
386
387
388

    async def is_tracing_enabled(self) -> bool:
        return False

    async def do_log_stats(
        self,
        scheduler_outputs=None,
        model_output=None,
    ) -> None:
389
390
        for stat_logger in self.stat_loggers:
            stat_logger.log()
391
392
393
394
395

    async def check_health(self) -> None:
        logger.debug("Called check_health.")

    async def start_profile(self) -> None:
396
        await self.engine_core.profile_async(True)
397
398

    async def stop_profile(self) -> None:
399
        await self.engine_core.profile_async(False)
400

401
402
403
    async def reset_prefix_cache(self) -> None:
        await self.engine_core.reset_prefix_cache_async()

404
405
406
407
408
409
    async def sleep(self, level: int = 1) -> None:
        await self.engine_core.sleep_async(level)

    async def wake_up(self) -> None:
        await self.engine_core.wake_up_async()

410
    async def add_lora(self, lora_request: LoRARequest) -> bool:
411
        """Load a new LoRA adapter into the engine for future requests."""
412
413
414
415
416
417
        return await self.engine_core.add_lora_async(lora_request)

    async def remove_lora(self, lora_id: int) -> bool:
        """Remove an already loaded LoRA adapter."""
        return await self.engine_core.remove_lora_async(lora_id)

418
    async def list_loras(self) -> set[int]:
419
420
421
422
423
424
        """List all registered adapters."""
        return await self.engine_core.list_loras_async()

    async def pin_lora(self, lora_id: int) -> bool:
        """Prevent an adapter from being evicted."""
        return await self.engine_core.pin_lora_async(lora_id)
425

426
427
428
429
430
431
432
433
434
435
436
437
438
439
    @property
    def is_running(self) -> bool:
        return True

    @property
    def is_stopped(self) -> bool:
        return False

    @property
    def errored(self) -> bool:
        return False

    @property
    def dead_error(self) -> BaseException:
440
        return Exception()  # TODO: implement