llm.py 75.3 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import itertools
5
import warnings
6
from collections.abc import Sequence
7
from contextlib import contextmanager
8
9
from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Optional, Union,
                    cast, overload)
10

11
import cloudpickle
12
import torch.nn as nn
13
from pydantic import ValidationError
14
from tqdm.auto import tqdm
15
from typing_extensions import TypeVar, deprecated
16

17
import vllm.envs as envs
18
from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
19
20
                              BeamSearchSequence,
                              create_sort_beams_key_function)
21
22
from vllm.config import (CompilationConfig, ModelDType, TokenizerMode,
                         is_init_field)
23
24
from vllm.engine.arg_utils import (EngineArgs, HfOverrides, PoolerConfig,
                                   TaskOption)
Joe Runde's avatar
Joe Runde committed
25
from vllm.engine.llm_engine import LLMEngine
nunjunj's avatar
nunjunj committed
26
from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
27
                                         ChatTemplateContentFormatOption,
28
29
                                         apply_hf_chat_template,
                                         apply_mistral_chat_template,
30
31
                                         parse_chat_messages,
                                         resolve_chat_template_content_format)
32
33
34
35
36
from vllm.entrypoints.score_utils import (ScoreContentPartParam,
                                          ScoreMultiModalParam,
                                          _cosine_similarity,
                                          _validate_score_input_lens,
                                          get_score_prompt)
37
from vllm.entrypoints.utils import _validate_truncation_size
38
from vllm.inputs import PromptType, SingletonPrompt, TextPrompt, TokensPrompt
39
from vllm.inputs.parse import parse_and_batch_prompt
40
from vllm.logger import init_logger
41
from vllm.lora.request import LoRARequest
42
43
from vllm.model_executor.guided_decoding.guided_fields import (
    GuidedDecodingRequest, LLMGuidedOptions)
44
from vllm.model_executor.layers.quantization import QuantizationMethods
45
46
47
from vllm.outputs import (ClassificationRequestOutput, EmbeddingRequestOutput,
                          PoolingRequestOutput, RequestOutput,
                          ScoringRequestOutput)
48
from vllm.pooling_params import PoolingParams
49
50
from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
                                  RequestOutputKind, SamplingParams)
51
from vllm.tasks import PoolingTask
52
from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
53
                                               get_cached_tokenizer)
yhu422's avatar
yhu422 committed
54
from vllm.usage.usage_lib import UsageContext
55
from vllm.utils import Counter, Device, deprecate_kwargs, is_list_of
56

57
58
59
if TYPE_CHECKING:
    from vllm.v1.metrics.reader import Metric

60
61
logger = init_logger(__name__)

62
63
_R = TypeVar("_R", default=Any)

64
65

class LLM:
Woosuk Kwon's avatar
Woosuk Kwon committed
66
67
68
69
70
71
72
73
74
75
    """An LLM for generating texts from given prompts and sampling parameters.

    This class includes a tokenizer, a language model (possibly distributed
    across multiple GPUs), and GPU memory space allocated for intermediate
    states (aka KV cache). Given a batch of prompts and sampling parameters,
    this class generates texts from the model, using an intelligent batching
    mechanism and efficient memory management.

    Args:
        model: The name or path of a HuggingFace Transformers model.
76
        tokenizer: The name or path of a HuggingFace Transformers tokenizer.
77
78
        tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer
            if available, and "slow" will always use the slow tokenizer.
79
80
81
        skip_tokenizer_init: If true, skip initialization of tokenizer and
            detokenizer. Expect valid prompt_token_ids and None for prompt
            from the input.
82
83
        trust_remote_code: Trust remote code (e.g., from HuggingFace) when
            downloading the model and tokenizer.
84
85
86
87
        allowed_local_media_path: Allowing API requests to read local images
            or videos from directories specified by the server file system.
            This is a security risk. Should only be enabled in trusted
            environments.
Woosuk Kwon's avatar
Woosuk Kwon committed
88
89
90
        tensor_parallel_size: The number of GPUs to use for distributed
            execution with tensor parallelism.
        dtype: The data type for the model weights and activations. Currently,
Woosuk Kwon's avatar
Woosuk Kwon committed
91
92
93
94
            we support `float32`, `float16`, and `bfloat16`. If `auto`, we use
            the `torch_dtype` attribute specified in the model config file.
            However, if the `torch_dtype` in the config is `float32`, we will
            use `float16` instead.
95
        quantization: The method used to quantize the model weights. Currently,
96
            we support "awq", "gptq", and "fp8" (experimental).
97
98
99
100
            If None, we first check the `quantization_config` attribute in the
            model config file. If that is None, we assume the model weights are
            not quantized and use `dtype` to determine the data type of
            the weights.
Jasmond L's avatar
Jasmond L committed
101
102
        revision: The specific model version to use. It can be a branch name,
            a tag name, or a commit id.
103
104
        tokenizer_revision: The specific tokenizer version to use. It can be a
            branch name, a tag name, or a commit id.
105
106
107
108
109
110
111
        seed: The seed to initialize the random number generator for sampling.
        gpu_memory_utilization: The ratio (between 0 and 1) of GPU memory to
            reserve for the model weights, activations, and KV cache. Higher
            values will increase the KV cache size and thus improve the model's
            throughput. However, if the value is too high, it may cause out-of-
            memory (OOM) errors.
        swap_space: The size (GiB) of CPU memory per GPU to use as swap space.
112
113
114
115
116
            This can be used for temporarily storing the states of the requests
            when their `best_of` sampling parameters are larger than 1. If all
            requests will have `best_of=1`, you can safely set this to 0.
            Noting that `best_of` is only supported in V0. Otherwise, too small
            values may cause out-of-memory (OOM) errors.
117
118
119
120
        cpu_offload_gb: The size (GiB) of CPU memory to use for offloading
            the model weights. This virtually increases the GPU memory space
            you can use to hold the model weights, at the cost of CPU-GPU data
            transfer for every forward pass.
121
122
123
        enforce_eager: Whether to enforce eager execution. If True, we will
            disable CUDA graph and always execute the model in eager mode.
            If False, we will use CUDA graph and eager execution in hybrid.
124
        max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs.
125
            When a sequence has context length larger than this, we fall back
126
127
128
            to eager mode. Additionally for encoder-decoder models, if the
            sequence length of the encoder input is larger than this, we fall
            back to the eager mode.
129
130
        disable_custom_all_reduce: See
            [ParallelConfig][vllm.config.ParallelConfig].
131
132
        disable_async_output_proc: Disable async output processing.
            This may result in lower performance.
133
        hf_token: The token to use as HTTP bearer authorization for remote files
134
            . If `True`, will use the token generated when running
135
            `huggingface-cli login` (stored in `~/.huggingface`).
136
137
138
        hf_overrides: If a dictionary, contains arguments to be forwarded to the
            HuggingFace config. If a callable, it is called to update the
            HuggingFace config.
139
140
141
142
143
144
145
146
        mm_processor_kwargs: Arguments to be forwarded to the model's processor
            for multi-modal data, e.g., image processor. Overrides for the
            multi-modal processor obtained from `AutoProcessor.from_pretrained`.
            The available overrides depend on the model that is being run.
            For example, for Phi-3-Vision: `{"num_crops": 4}`.
        override_pooler_config: Initialize non-default pooling config or
            override default pooling config for the pooling model.
            e.g. `PoolerConfig(pooling_type="mean", normalize=False)`.
147
148
149
        compilation_config: Either an integer or a dictionary. If it is an
            integer, it is used as the level of compilation optimization. If it
            is a dictionary, it can specify the full compilation configuration.
150
        **kwargs: Arguments for [`EngineArgs`][vllm.EngineArgs].
nunjunj's avatar
nunjunj committed
151

152
153
    Note:
        This class is intended to be used for offline inference. For online
154
        serving, use the [AsyncLLMEngine][vllm.AsyncLLMEngine] class instead.
155
    """
156

157
    DEPRECATE_LEGACY: ClassVar[bool] = True
158
159
160
161
162
163
164
165
166
167
168
    """A flag to toggle whether to deprecate the legacy generate/encode API."""

    @classmethod
    @contextmanager
    def deprecate_legacy_api(cls):
        cls.DEPRECATE_LEGACY = True

        yield

        cls.DEPRECATE_LEGACY = False

169
170
171
    def __init__(
        self,
        model: str,
172
173
        *,
        task: TaskOption = "auto",
174
        tokenizer: Optional[str] = None,
175
        tokenizer_mode: TokenizerMode = "auto",
176
        skip_tokenizer_init: bool = False,
177
        trust_remote_code: bool = False,
178
        allowed_local_media_path: str = "",
179
        tensor_parallel_size: int = 1,
180
181
        dtype: ModelDType = "auto",
        quantization: Optional[QuantizationMethods] = None,
182
        revision: Optional[str] = None,
183
        tokenizer_revision: Optional[str] = None,
184
        seed: Optional[int] = None,
185
        gpu_memory_utilization: float = 0.9,
186
        swap_space: float = 4,
187
        cpu_offload_gb: float = 0,
188
        enforce_eager: bool = False,
189
        max_seq_len_to_capture: int = 8192,
190
        disable_custom_all_reduce: bool = False,
191
        disable_async_output_proc: bool = False,
192
        hf_token: Optional[Union[bool, str]] = None,
193
        hf_overrides: Optional[HfOverrides] = None,
194
        mm_processor_kwargs: Optional[dict[str, Any]] = None,
195
        override_pooler_config: Optional[PoolerConfig] = None,
196
197
        compilation_config: Optional[Union[int, dict[str, Any],
                                           CompilationConfig]] = None,
198
199
        **kwargs,
    ) -> None:
200
        """LLM constructor."""
201

202
203
        if "disable_log_stats" not in kwargs:
            kwargs["disable_log_stats"] = True
204

205
206
207
208
209
210
211
        if "worker_cls" in kwargs:
            worker_cls = kwargs["worker_cls"]
            # if the worker_cls is not qualified string name,
            # we serialize it using cloudpickle to avoid pickling issues
            if isinstance(worker_cls, type):
                kwargs["worker_cls"] = cloudpickle.dumps(worker_cls)

212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
        if "kv_transfer_config" in kwargs and isinstance(
                kwargs["kv_transfer_config"], dict):
            from vllm.config import KVTransferConfig
            raw_config_dict = kwargs["kv_transfer_config"]
            try:
                kwargs["kv_transfer_config"] = KVTransferConfig(
                    **raw_config_dict)
            except ValidationError as e:
                logger.error(
                    "Failed to convert 'kv_transfer_config' dict to "
                    "KVTransferConfig object. Dict: %s. Error: %s",
                    raw_config_dict, e)
                # Consider re-raising a more specific vLLM error or ValueError
                # to provide better context to the user.
                raise ValueError(
                    f"Invalid 'kv_transfer_config' provided: {e}") from e

229
230
231
        if hf_overrides is None:
            hf_overrides = {}

232
        if compilation_config is not None:
233
234
235
236
237
238
239
            if isinstance(compilation_config, int):
                compilation_config_instance = CompilationConfig(
                    level=compilation_config)
            elif isinstance(compilation_config, dict):
                predicate = lambda x: is_init_field(CompilationConfig, x[0])
                compilation_config_instance = CompilationConfig(
                    **dict(filter(predicate, compilation_config.items())))
240
241
            else:
                compilation_config_instance = compilation_config
242
        else:
243
            compilation_config_instance = CompilationConfig()
244

Zhuohan Li's avatar
Zhuohan Li committed
245
        engine_args = EngineArgs(
246
            model=model,
247
            task=task,
248
            tokenizer=tokenizer,
249
            tokenizer_mode=tokenizer_mode,
250
            skip_tokenizer_init=skip_tokenizer_init,
251
            trust_remote_code=trust_remote_code,
252
            allowed_local_media_path=allowed_local_media_path,
253
254
            tensor_parallel_size=tensor_parallel_size,
            dtype=dtype,
255
            quantization=quantization,
256
            revision=revision,
257
            tokenizer_revision=tokenizer_revision,
258
259
260
            seed=seed,
            gpu_memory_utilization=gpu_memory_utilization,
            swap_space=swap_space,
261
            cpu_offload_gb=cpu_offload_gb,
262
            enforce_eager=enforce_eager,
263
            max_seq_len_to_capture=max_seq_len_to_capture,
264
            disable_custom_all_reduce=disable_custom_all_reduce,
265
            disable_async_output_proc=disable_async_output_proc,
266
            hf_token=hf_token,
267
            hf_overrides=hf_overrides,
268
            mm_processor_kwargs=mm_processor_kwargs,
269
            override_pooler_config=override_pooler_config,
270
            compilation_config=compilation_config_instance,
271
272
            **kwargs,
        )
273
274
275
276
277

        # Create the Engine (autoselects V0 vs V1)
        self.llm_engine = LLMEngine.from_engine_args(
            engine_args=engine_args, usage_context=UsageContext.LLM_CLASS)
        self.engine_class = type(self.llm_engine)
278

279
        self.request_counter = Counter()
280
        self.default_sampling_params: Union[dict[str, Any], None] = None
281

282
283
284
285
286
287
288
289
290
291
        if envs.VLLM_USE_V1:
            supported_tasks = self.llm_engine \
                .get_supported_tasks()  # type: ignore
        else:
            supported_tasks = self.llm_engine.model_config.supported_tasks

        logger.info("Supported_tasks: %s", supported_tasks)

        self.supported_tasks = supported_tasks

292
293
294
295
296
297
    def get_tokenizer(
        self,
        lora_request: Optional[LoRARequest] = None,
    ) -> AnyTokenizer:
        return self.llm_engine.get_tokenizer_group().get_lora_tokenizer(
            lora_request)
298
299

    def set_tokenizer(self, tokenizer: AnyTokenizer) -> None:
300
        tokenizer_group = self.llm_engine.get_tokenizer_group()
301

302
303
304
305
        # While CachedTokenizer is dynamic, have no choice but
        # compare class name. Misjudgment will arise from
        # user-defined tokenizer started with 'Cached'
        if tokenizer.__class__.__name__.startswith("Cached"):
306
            tokenizer_group.tokenizer = tokenizer
307
        else:
308
            tokenizer_group.tokenizer = get_cached_tokenizer(tokenizer)
309

310
    def get_default_sampling_params(self) -> SamplingParams:
311
312
313
314
315
        if self.default_sampling_params is None:
            self.default_sampling_params = (
                self.llm_engine.model_config.get_diff_sampling_param())
        if self.default_sampling_params:
            return SamplingParams.from_optional(**self.default_sampling_params)
316
317
        return SamplingParams()

318
319
320
321
322
323
324
    @overload
    def generate(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        sampling_params: Optional[Union[SamplingParams,
                                        Sequence[SamplingParams]]] = None,
325
        *,
326
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
327
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
328
329
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
330
    ) -> list[RequestOutput]:
331
332
        ...

333
    @overload  # LEGACY: single (prompt + optional token ids)
334
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
335
336
337
338
    def generate(
        self,
        prompts: str,
        sampling_params: Optional[Union[SamplingParams,
339
340
                                        list[SamplingParams]]] = None,
        prompt_token_ids: Optional[list[int]] = None,
341
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
342
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
343
344
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
345
    ) -> list[RequestOutput]:
346
347
348
        ...

    @overload  # LEGACY: multi (prompt + optional token ids)
349
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
350
351
    def generate(
        self,
352
        prompts: list[str],
353
        sampling_params: Optional[Union[SamplingParams,
354
355
                                        list[SamplingParams]]] = None,
        prompt_token_ids: Optional[list[list[int]]] = None,
356
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
357
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
358
359
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
360
    ) -> list[RequestOutput]:
361
362
363
        ...

    @overload  # LEGACY: single (token ids + optional prompt)
364
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
365
366
367
368
    def generate(
        self,
        prompts: Optional[str] = None,
        sampling_params: Optional[Union[SamplingParams,
369
                                        list[SamplingParams]]] = None,
370
        *,
371
        prompt_token_ids: list[int],
372
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
373
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
374
375
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
376
    ) -> list[RequestOutput]:
377
378
379
        ...

    @overload  # LEGACY: multi (token ids + optional prompt)
380
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
381
382
    def generate(
        self,
383
        prompts: Optional[list[str]] = None,
384
        sampling_params: Optional[Union[SamplingParams,
385
                                        list[SamplingParams]]] = None,
386
        *,
387
        prompt_token_ids: list[list[int]],
388
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
389
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
390
391
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
392
    ) -> list[RequestOutput]:
393
394
395
        ...

    @overload  # LEGACY: single or multi token ids [pos-only]
396
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
397
398
399
400
    def generate(
        self,
        prompts: None,
        sampling_params: None,
401
        prompt_token_ids: Union[list[int], list[list[int]]],
402
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
403
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
404
405
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
406
    ) -> list[RequestOutput]:
407
408
        ...

nunjunj's avatar
nunjunj committed
409
410
411
    @deprecate_kwargs(
        "prompt_token_ids",
        is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
412
        additional_message="Please use the 'prompts' parameter instead.",
nunjunj's avatar
nunjunj committed
413
    )
414
415
    def generate(
        self,
416
        prompts: Union[Union[PromptType, Sequence[PromptType]],
417
                       Optional[Union[str, list[str]]]] = None,
418
419
        sampling_params: Optional[Union[SamplingParams,
                                        Sequence[SamplingParams]]] = None,
420
        prompt_token_ids: Optional[Union[list[int], list[list[int]]]] = None,
421
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
422
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
423
        guided_options_request: Optional[Union[LLMGuidedOptions,
424
                                               GuidedDecodingRequest]] = None,
425
426
        priority: Optional[list[int]] = None,
    ) -> list[RequestOutput]:
Woosuk Kwon's avatar
Woosuk Kwon committed
427
428
        """Generates the completions for the input prompts.

429
        This class automatically batches the given prompts, considering
Woosuk Kwon's avatar
Woosuk Kwon committed
430
431
432
433
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
434
            prompts: The prompts to the LLM. You may pass a sequence of prompts
435
                for batch inference. See [PromptType][vllm.inputs.PromptType]
436
                for more details about the format of each prompts.
Woosuk Kwon's avatar
Woosuk Kwon committed
437
            sampling_params: The sampling parameters for text generation. If
nunjunj's avatar
nunjunj committed
438
439
440
                None, we use the default sampling parameters.
                When it is a single value, it is applied to every prompt.
                When it is a list, the list must have the same length as the
441
                prompts and it is paired one by one with the prompt.
442
443
444
445
            use_tqdm: If `True`, shows a tqdm progress bar.
                If a callable (e.g., `functools.partial(tqdm, leave=False)`),
                it is used to create the progress bar.
                If `False`, no progress bar is created.
446
            lora_request: LoRA request to use for generation, if any.
447
448
            priority: The priority of the requests, if any.
                Only applicable when priority scheduling policy is enabled.
Woosuk Kwon's avatar
Woosuk Kwon committed
449
450

        Returns:
451
            A list of `RequestOutput` objects containing the
452
            generated completions in the same order as the input prompts.
453

454
455
456
457
        Note:
            Using `prompts` and `prompt_token_ids` as keyword parameters is
            considered legacy and may be deprecated in the future. You should
            instead pass them via the `inputs` parameter.
458
        """
459
460
461
        model_config = self.llm_engine.model_config
        runner_type = model_config.runner_type
        if runner_type != "generate":
462
            messages = [
463
                "LLM.generate() is only supported for generative models."
464
465
            ]

466
            if "generate" in model_config.supported_runner_types:
467
                messages.append(
468
469
                    "Your model supports the 'generate' runner, but is "
                    f"currently initialized for the '{runner_type}' runner. "
470
471
                    "Please initialize vLLM using `--task generate` or "
                    "`--task transcription`.")
472
473

            raise ValueError(" ".join(messages))
474

475
        if prompt_token_ids is not None:
476
            parsed_prompts = self._convert_v1_inputs(
477
                prompts=cast(Optional[Union[str, list[str]]], prompts),
478
479
480
                prompt_token_ids=prompt_token_ids,
            )
        else:
481
482
            parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
                                  prompts)
483

484
485
486
487
488
489
490
491
        if isinstance(guided_options_request, dict):
            if len(guided_options_request) > 1:
                raise ValueError(
                    "You can only use one guided decoding but multiple is "
                    f"specified: {guided_options_request}")
            guided_options_request = GuidedDecodingRequest(
                **guided_options_request)

492
493
        if sampling_params is None:
            # Use default sampling params.
494
            sampling_params = self.get_default_sampling_params()
495

496
497
498
499
500
501
502
        tokenization_kwargs: dict[str, Any] = {}
        truncate_prompt_tokens = None
        if isinstance(sampling_params, SamplingParams):
            truncate_prompt_tokens = sampling_params.truncate_prompt_tokens
        _validate_truncation_size(self.llm_engine.model_config.max_model_len,
                                  truncate_prompt_tokens, tokenization_kwargs)

503
504
505
506
        # Add any modality specific loras to the corresponding prompts
        lora_request = self._get_modality_specific_lora_reqs(
            parsed_prompts, lora_request)

507
        self._validate_and_add_requests(
508
            prompts=parsed_prompts,
509
            params=sampling_params,
510
            use_tqdm=use_tqdm,
511
            lora_request=lora_request,
512
            guided_options=guided_options_request,
513
            tokenization_kwargs=tokenization_kwargs,
514
515
            priority=priority,
        )
516

517
        outputs = self._run_engine(use_tqdm=use_tqdm)
Joe Runde's avatar
Joe Runde committed
518
        return self.engine_class.validate_outputs(outputs, RequestOutput)
519

520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
    def _get_modality_specific_lora_reqs(
            self, parsed_prompts: Union[PromptType, Sequence[PromptType]],
            lora_request: Optional[Union[list[LoRARequest], LoRARequest]]):
        # Grab the lora config off the vllm config on the engine,
        # since this is the same for both v0 & v1.
        lora_config = self.llm_engine.vllm_config.lora_config

        # If there's no lora config / default_mm_loras, or the model
        # isn't multimodal, leave the lora as is.
        if (lora_config is None
                or not self.llm_engine.model_config.is_multimodal_model
                or (lora_config and lora_config.default_mm_loras is None)):
            return lora_request

        if not isinstance(parsed_prompts, Sequence):
            parsed_prompts = [parsed_prompts]

        optional_loras = ([lora_request] * len(parsed_prompts)
                          if not isinstance(lora_request, Sequence) else
                          lora_request)

        return [
            self._resolve_single_prompt_mm_lora(
                parsed_prompt,
                opt_lora_req,
                lora_config.default_mm_loras,
            ) for parsed_prompt, opt_lora_req in zip(parsed_prompts,
                                                     optional_loras)
        ]

    def _resolve_single_prompt_mm_lora(self, parsed_prompt: PromptType,
                                       lora_request: Optional[LoRARequest],
                                       default_mm_loras: Optional[dict[str,
                                                                       str]]):
        if (not default_mm_loras or not isinstance(parsed_prompt, dict)
                or "multi_modal_data" not in parsed_prompt):
            return lora_request

        parsed_prompt = cast(Union[TextPrompt, TokensPrompt], parsed_prompt)

        intersection = set(
            parsed_prompt["multi_modal_data"].keys()).intersection(
                default_mm_loras.keys())
        if not intersection:
            return lora_request
        if len(intersection) > 1:
            # TODO: Would be nice to be able to have multiple loras per prompt
            logger.warning(
                "Multiple modality specific loras were registered and would be"
                " used by a single prompt consuming several modalities; "
                " currently we only support one lora per request; as such,"
                " lora(s) registered with modalities: %s"
                " will be skipped", intersection)
            return lora_request

        # Build the LoRA request; the ID of the default mm lora is the
        # index of the modality name sorted alphabetically + 1.
        modality_name = intersection.pop()
        modality_lora_path = default_mm_loras[modality_name]
        modality_lora_id = sorted(default_mm_loras).index(modality_name) + 1

        # If we have a collision, warn if there is a collision,
        # but always send the explicitly provided request.
        if lora_request:
            if lora_request.lora_int_id != modality_lora_id:
                logger.warning(
                    "A modality with a registered lora and a lora_request "
                    "with a different ID were provided; falling back to the "
                    "lora_request as we only apply one LoRARequest per prompt")
            return lora_request

        return LoRARequest(
            modality_name,
            modality_lora_id,
            modality_lora_path,
        )

597
    def collective_rpc(self,
598
                       method: Union[str, Callable[..., _R]],
599
                       timeout: Optional[float] = None,
600
601
                       args: tuple = (),
                       kwargs: Optional[dict[str, Any]] = None) -> list[_R]:
602
603
604
605
606
607
608
609
610
611
612
        """
        Execute an RPC call on all workers.

        Args:
            method: Name of the worker method to execute, or a callable that
                is serialized and sent to all workers to execute.

                If the method is a callable, it should accept an additional
                `self` argument, in addition to the arguments passed in `args`
                and `kwargs`. The `self` argument will be the worker object.
            timeout: Maximum time in seconds to wait for execution. Raises a
613
                [`TimeoutError`][] on timeout. `None` means wait indefinitely.
614
615
616
617
618
            args: Positional arguments to pass to the worker method.
            kwargs: Keyword arguments to pass to the worker method.

        Returns:
            A list containing the results from each worker.
619

620
621
622
        Note:
            It is recommended to use this API to only pass control messages,
            and set up data-plane communication to pass data.
623
        """
624
625

        return self.llm_engine.collective_rpc(method, timeout, args, kwargs)
626
627

    def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]:
628
        """
629
630
        Run a function directly on the model inside each worker,
        returning the result for each of them.
631
        """
632
633
        executor = self.llm_engine.model_executor
        return executor.apply_model(func)
634

635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
    def _get_beam_search_lora_requests(
        self,
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]],
        prompts: list[Union[TokensPrompt, TextPrompt]],
    ) -> list[Optional[LoRARequest]]:
        """Get the optional lora request corresponding to each prompt."""
        if isinstance(lora_request,
                      Sequence) and len(lora_request) != len(prompts):
            raise ValueError(
                "Lora request list should be the same length as the prompts")

        if lora_request is None or isinstance(lora_request, LoRARequest):
            return [lora_request] * len(prompts)

        raise TypeError(f"Invalid lora_request type {type(lora_request)}")

651
652
    def beam_search(
        self,
653
        prompts: list[Union[TokensPrompt, TextPrompt]],
654
        params: BeamSearchParams,
655
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
656
        use_tqdm: bool = False,
657
    ) -> list[BeamSearchOutput]:
658
659
660
661
662
663
        """
        Generate sequences using beam search.

        Args:
            prompts: A list of prompts. Each prompt can be a string or a list
                of token IDs.
664
            params: The beam search parameters.
665
            lora_request: LoRA request to use for generation, if any.
666
            use_tqdm: Whether to use tqdm to display the progress bar.
667
        """
668
669
        # TODO: how does beam search work together with length penalty,
        # frequency, penalty, and stopping criteria, etc.?
670
671
672
673
        beam_width = params.beam_width
        max_tokens = params.max_tokens
        temperature = params.temperature
        ignore_eos = params.ignore_eos
674
675
        length_penalty = params.length_penalty

676
677
678
        lora_requests = self._get_beam_search_lora_requests(
            lora_request, prompts)

679
680
681
682
683
        tokenizer = self.get_tokenizer()
        sort_beams_key = create_sort_beams_key_function(
            tokenizer.eos_token_id,
            length_penalty,
        )
684

685
686
687
688
689
690
691
692
693
694
695
696
        def create_tokens_prompt_from_beam(
                beam: BeamSearchSequence) -> TokensPrompt:
            token_prompt_kwargs: TokensPrompt = {
                "prompt_token_ids": beam.tokens
            }
            if beam.multi_modal_data is not None:
                token_prompt_kwargs["multi_modal_data"] = beam.multi_modal_data

            if beam.mm_processor_kwargs is not None:
                token_prompt_kwargs[
                    "mm_processor_kwargs"] = beam.mm_processor_kwargs
            return TokensPrompt(**token_prompt_kwargs)
697

698
699
700
701
702
        # generate 2 * beam_width candidates at each step
        # following the huggingface transformers implementation
        # at https://github.com/huggingface/transformers/blob/e15687fffe5c9d20598a19aeab721ae0a7580f8a/src/transformers/generation/beam_search.py#L534 # noqa
        beam_search_params = SamplingParams(logprobs=2 * beam_width,
                                            max_tokens=1,
703
                                            temperature=temperature)
704
        instances: list[BeamSearchInstance] = []
705

706
        for lora_req, prompt in zip(lora_requests, prompts):
707
708
709
710
711
712
713
714
            # Add multimodal processor kwargs & data
            mm_kwargs = {}
            if "multi_modal_data" in prompt:
                mm_kwargs["multi_modal_data"] = prompt["multi_modal_data"]
            if "mm_processor_kwargs" in prompt:
                mm_kwargs["mm_processor_kwargs"] = prompt[
                    "mm_processor_kwargs"]

715
716
            if "prompt_token_ids" in prompt:
                prompt = cast(TokensPrompt, prompt)  # Needed for mypy
717
718
719
                prompt_tokens = prompt["prompt_token_ids"]
            else:
                prompt_tokens = tokenizer.encode(prompt["prompt"])
720

721
            instances.append(
722
723
724
725
726
727
                BeamSearchInstance(
                    prompt_tokens,
                    lora_request=lora_req,
                    logprobs=None,
                    **mm_kwargs,
                ), )
728

729
730
731
732
733
734
735
736
737
738
739
740
        token_iter = range(max_tokens)
        if use_tqdm:
            token_iter = tqdm(token_iter,
                              desc="Beam search",
                              unit="token",
                              unit_scale=False)
            logger.warning(
                "The progress bar shows the upper bound on token steps and "
                "may finish early due to stopping conditions. It does not "
                "reflect instance-level progress.")

        for _ in token_iter:
741
            all_beams: list[BeamSearchSequence] = list(
742
743
744
745
                sum((instance.beams for instance in instances), []))
            pos = [0] + list(
                itertools.accumulate(
                    len(instance.beams) for instance in instances))
746
            instance_start_and_end: list[tuple[int, int]] = list(
747
748
749
750
751
                zip(pos[:-1], pos[1:]))

            if len(all_beams) == 0:
                break

752
753
754
755
            # create the corresponding batch entries for prompt & optional lora
            prompts_batch, lora_req_batch = zip(
                *[(create_tokens_prompt_from_beam(beam), beam.lora_request)
                  for beam in all_beams])
756
757
758
759
760

            # only runs for one step
            # we don't need to use tqdm here
            output = self.generate(prompts_batch,
                                   sampling_params=beam_search_params,
761
762
                                   use_tqdm=False,
                                   lora_request=lora_req_batch)
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778

            for (start, end), instance in zip(instance_start_and_end,
                                              instances):
                instance_new_beams = []
                for i in range(start, end):
                    current_beam = all_beams[i]
                    result = output[i]

                    if result.outputs[0].logprobs is not None:
                        # if `result.outputs[0].logprobs` is None, it means
                        # the sequence is completed because of the max-model-len
                        # or abortion. we don't need to add it to the new beams.
                        logprobs = result.outputs[0].logprobs[0]
                        for token_id, logprob_obj in logprobs.items():
                            new_beam = BeamSearchSequence(
                                tokens=current_beam.tokens + [token_id],
779
                                logprobs=current_beam.logprobs + [logprobs],
780
                                lora_request=current_beam.lora_request,
781
                                cum_logprob=current_beam.cum_logprob +
782
783
784
785
                                logprob_obj.logprob,
                                multi_modal_data=current_beam.multi_modal_data,
                                mm_processor_kwargs=current_beam.
                                mm_processor_kwargs)
786
787
788
789
790
791
792

                            if token_id == tokenizer.eos_token_id and \
                                not ignore_eos:
                                instance.completed.append(new_beam)
                            else:
                                instance_new_beams.append(new_beam)
                sorted_beams = sorted(instance_new_beams,
793
                                      key=sort_beams_key,
794
795
796
797
798
799
800
                                      reverse=True)
                instance.beams = sorted_beams[:beam_width]

        outputs = []
        for instance in instances:
            instance.completed.extend(instance.beams)
            sorted_completed = sorted(instance.completed,
801
                                      key=sort_beams_key,
802
803
804
805
806
807
808
809
810
                                      reverse=True)
            best_beams = sorted_completed[:beam_width]

            for beam in best_beams:
                beam.text = tokenizer.decode(beam.tokens)
            outputs.append(BeamSearchOutput(sequences=best_beams))

        return outputs

nunjunj's avatar
nunjunj committed
811
812
    def chat(
        self,
813
814
        messages: Union[list[ChatCompletionMessageParam],
                        list[list[ChatCompletionMessageParam]]],
nunjunj's avatar
nunjunj committed
815
        sampling_params: Optional[Union[SamplingParams,
816
                                        list[SamplingParams]]] = None,
817
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
nunjunj's avatar
nunjunj committed
818
819
        lora_request: Optional[LoRARequest] = None,
        chat_template: Optional[str] = None,
820
        chat_template_content_format: ChatTemplateContentFormatOption = "auto",
821
        add_generation_prompt: bool = True,
822
        continue_final_message: bool = False,
823
        tools: Optional[list[dict[str, Any]]] = None,
824
        chat_template_kwargs: Optional[dict[str, Any]] = None,
825
826
        mm_processor_kwargs: Optional[dict[str, Any]] = None,
    ) -> list[RequestOutput]:
nunjunj's avatar
nunjunj committed
827
        """
828
        Generate responses for a chat conversation.
nunjunj's avatar
nunjunj committed
829

830
        The chat conversation is converted into a text prompt using the
831
        tokenizer and calls the [generate][] method to generate the
832
833
834
835
        responses.

        Multi-modal inputs can be passed in the same way you would pass them
        to the OpenAI API.
nunjunj's avatar
nunjunj committed
836
837

        Args:
838
839
            messages: A list of conversations or a single conversation.

840
841
                - Each conversation is represented as a list of messages.
                - Each message is a dictionary with 'role' and 'content' keys.
842

nunjunj's avatar
nunjunj committed
843
844
845
846
847
            sampling_params: The sampling parameters for text generation.
                If None, we use the default sampling parameters. When it
                is a single value, it is applied to every prompt. When it
                is a list, the list must have the same length as the
                prompts and it is paired one by one with the prompt.
848
849
850
851
            use_tqdm: If `True`, shows a tqdm progress bar.
                If a callable (e.g., `functools.partial(tqdm, leave=False)`),
                it is used to create the progress bar.
                If `False`, no progress bar is created.
nunjunj's avatar
nunjunj committed
852
853
            lora_request: LoRA request to use for generation, if any.
            chat_template: The template to use for structuring the chat.
854
                If not provided, the model's default chat template will be used.
855
856
            chat_template_content_format: The format to render message content.

857
858
859
860
861
                - "string" will render the content as a string.
                  Example: `"Who are you?"`
                - "openai" will render the content as a list of dictionaries,
                  similar to OpenAI schema.
                  Example: `[{"type": "text", "text": "Who are you?"}]`
862

863
            add_generation_prompt: If True, adds a generation template
nunjunj's avatar
nunjunj committed
864
                to each message.
865
            continue_final_message: If True, continues the final message in
866
                the conversation instead of starting a new one. Cannot be
867
                `True` if `add_generation_prompt` is also `True`.
868
869
            chat_template_kwargs: Additional kwargs to pass to the chat
                template.
870
871
            mm_processor_kwargs: Multimodal processor kwarg overrides for this
                chat request. Only used for offline requests.
nunjunj's avatar
nunjunj committed
872
873

        Returns:
874
            A list of `RequestOutput` objects containing the generated
nunjunj's avatar
nunjunj committed
875
876
            responses in the same order as the input messages.
        """
877
        list_of_messages: list[list[ChatCompletionMessageParam]]
nunjunj's avatar
nunjunj committed
878

879
880
        # Handle multi and single conversations
        if is_list_of(messages, list):
881
882
            # messages is list[list[...]]
            list_of_messages = cast(list[list[ChatCompletionMessageParam]],
883
                                    messages)
884
        else:
885
            # messages is list[...]
886
            list_of_messages = [
887
                cast(list[ChatCompletionMessageParam], messages)
888
            ]
889

890
        tokenizer = self.get_tokenizer(lora_request)
891
892
893
        model_config = self.llm_engine.get_model_config()
        resolved_content_format = resolve_chat_template_content_format(
            chat_template,
894
            tools,
895
896
            chat_template_content_format,
            tokenizer,
897
            model_config=model_config,
898
899
        )

900
901
902
903
904
905
906
907
        _chat_template_kwargs: dict[str, Any] = dict(
            chat_template=chat_template,
            add_generation_prompt=add_generation_prompt,
            continue_final_message=continue_final_message,
            tools=tools,
        )
        _chat_template_kwargs.update(chat_template_kwargs or {})

908
        prompts: list[Union[TokensPrompt, TextPrompt]] = []
909
910

        for msgs in list_of_messages:
911
912
913
            # NOTE: _parse_chat_message_content_parts() currently doesn't
            # handle mm_processor_kwargs, since there is no implementation in
            # the chat message parsing for it.
914
            conversation, mm_data = parse_chat_messages(
915
916
917
918
919
                msgs,
                model_config,
                tokenizer,
                content_format=resolved_content_format,
            )
920
921

            if isinstance(tokenizer, MistralTokenizer):
922
                prompt_token_ids = apply_mistral_chat_template(
923
924
                    tokenizer,
                    messages=msgs,
925
                    **_chat_template_kwargs,
926
927
                )
            else:
928
                prompt_str = apply_hf_chat_template(
929
                    tokenizer=tokenizer,
930
                    conversation=conversation,
931
                    model_config=model_config,
932
                    **_chat_template_kwargs,
933
                )
934
935
936
937
                # Special tokens are already included in chat templates so
                # should not be added by the tokenizer in this case.
                prompt_token_ids = tokenizer.encode(prompt_str,
                                                    add_special_tokens=False)
938

939
            prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
940
941
942
943

            if mm_data is not None:
                prompt["multi_modal_data"] = mm_data

944
945
946
            if mm_processor_kwargs is not None:
                prompt["mm_processor_kwargs"] = mm_processor_kwargs

947
            prompts.append(prompt)
948

nunjunj's avatar
nunjunj committed
949
        return self.generate(
950
            prompts,
951
            sampling_params=sampling_params,
nunjunj's avatar
nunjunj committed
952
953
954
955
            use_tqdm=use_tqdm,
            lora_request=lora_request,
        )

956
957
958
959
960
961
962
    @overload
    def encode(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
963
        *,
964
        truncate_prompt_tokens: Optional[int] = None,
965
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
966
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
967
        pooling_task: PoolingTask = "encode",
968
        tokenization_kwargs: Optional[dict[str, Any]] = None,
969
    ) -> list[PoolingRequestOutput]:
970
971
        ...

972
    @overload  # LEGACY: single (prompt + optional token ids)
973
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
974
975
976
977
978
    def encode(
        self,
        prompts: str,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
979
        prompt_token_ids: Optional[list[int]] = None,
980
        truncate_prompt_tokens: Optional[int] = None,
981
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
982
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
983
        pooling_task: PoolingTask = "encode",
984
        tokenization_kwargs: Optional[dict[str, Any]] = None,
985
    ) -> list[PoolingRequestOutput]:
986
        ...
987

988
    @overload  # LEGACY: multi (prompt + optional token ids)
989
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
990
991
    def encode(
        self,
992
        prompts: list[str],
993
        pooling_params: Optional[Union[PoolingParams,
994
                                       Sequence[PoolingParams]]] = None,
995
        prompt_token_ids: Optional[list[list[int]]] = None,
996
        truncate_prompt_tokens: Optional[int] = None,
997
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
998
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
999
        pooling_task: PoolingTask = "encode",
1000
        tokenization_kwargs: Optional[dict[str, Any]] = None,
1001
    ) -> list[PoolingRequestOutput]:
1002
1003
1004
        ...

    @overload  # LEGACY: single (token ids + optional prompt)
1005
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
1006
1007
1008
1009
1010
1011
    def encode(
        self,
        prompts: Optional[str] = None,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
        *,
1012
        prompt_token_ids: list[int],
1013
        truncate_prompt_tokens: Optional[int] = None,
1014
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
1015
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
1016
        pooling_task: PoolingTask = "encode",
1017
        tokenization_kwargs: Optional[dict[str, Any]] = None,
1018
    ) -> list[PoolingRequestOutput]:
1019
1020
1021
        ...

    @overload  # LEGACY: multi (token ids + optional prompt)
1022
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
1023
1024
    def encode(
        self,
1025
        prompts: Optional[list[str]] = None,
1026
1027
1028
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
        *,
1029
        prompt_token_ids: list[list[int]],
1030
        truncate_prompt_tokens: Optional[int] = None,
1031
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
1032
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
1033
        pooling_task: PoolingTask = "encode",
1034
        tokenization_kwargs: Optional[dict[str, Any]] = None,
1035
    ) -> list[PoolingRequestOutput]:
1036
1037
1038
        ...

    @overload  # LEGACY: single or multi token ids [pos-only]
1039
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
1040
1041
1042
1043
    def encode(
        self,
        prompts: None,
        pooling_params: None,
1044
        prompt_token_ids: Union[list[int], list[list[int]]],
1045
        truncate_prompt_tokens: Optional[int] = None,
1046
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
1047
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
1048
        pooling_task: PoolingTask = "encode",
1049
        tokenization_kwargs: Optional[dict[str, Any]] = None,
1050
    ) -> list[PoolingRequestOutput]:
1051
1052
        ...

nunjunj's avatar
nunjunj committed
1053
1054
1055
    @deprecate_kwargs(
        "prompt_token_ids",
        is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
1056
        additional_message="Please use the 'prompts' parameter instead.",
nunjunj's avatar
nunjunj committed
1057
    )
1058
1059
    def encode(
        self,
1060
        prompts: Union[Union[PromptType, Sequence[PromptType]],
1061
                       Optional[Union[str, list[str]]]] = None,
1062
1063
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
1064
        prompt_token_ids: Optional[Union[list[int], list[list[int]]]] = None,
1065
        truncate_prompt_tokens: Optional[int] = None,
1066
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
1067
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
1068
        pooling_task: PoolingTask = "encode",
1069
        tokenization_kwargs: Optional[dict[str, Any]] = None,
1070
    ) -> list[PoolingRequestOutput]:
1071
1072
        """Apply pooling to the hidden states corresponding to the input
        prompts.
1073

1074
        This class automatically batches the given prompts, considering
1075
1076
1077
1078
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
1079
            prompts: The prompts to the LLM. You may pass a sequence of prompts
1080
                for batch inference. See [PromptType][vllm.inputs.PromptType]
1081
                for more details about the format of each prompts.
1082
1083
            pooling_params: The pooling parameters for pooling. If None, we
                use the default pooling parameters.
1084
1085
1086
1087
            use_tqdm: If `True`, shows a tqdm progress bar.
                If a callable (e.g., `functools.partial(tqdm, leave=False)`),
                it is used to create the progress bar.
                If `False`, no progress bar is created.
1088
            lora_request: LoRA request to use for generation, if any.
1089
            pooling_task: Override the pooling task to use.
1090
1091

        Returns:
1092
            A list of `PoolingRequestOutput` objects containing the
1093
            pooled hidden states in the same order as the input prompts.
1094

1095
1096
1097
1098
        Note:
            Using `prompts` and `prompt_token_ids` as keyword parameters is
            considered legacy and may be deprecated in the future. You should
            instead pass them via the `inputs` parameter.
1099
        """
1100
1101
        model_config = self.llm_engine.model_config
        runner_type = model_config.runner_type
1102
1103
        if runner_type != "pooling":
            messages = ["LLM.encode() is only supported for pooling models."]
1104

1105
            if "pooling" in model_config.supported_runner_types:
1106
                messages.append(
1107
1108
1109
1110
                    "Your model supports the 'pooling' runner, but is "
                    f"currently initialized for the '{runner_type}' runner. "
                    "Please initialize vLLM using `--task embed`, "
                    "`--task classify`, `--task score` etc.")
1111
1112

            raise ValueError(" ".join(messages))
1113

1114
        if prompt_token_ids is not None:
1115
            parsed_prompts = self._convert_v1_inputs(
1116
                prompts=cast(Optional[Union[str, list[str]]], prompts),
1117
1118
1119
                prompt_token_ids=prompt_token_ids,
            )
        else:
1120
1121
            parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
                                  prompts)
1122

1123
1124
1125
        if pooling_params is None:
            # Use default pooling params.
            pooling_params = PoolingParams()
1126
1127
1128

        if isinstance(pooling_params, PoolingParams):
            pooling_params.verify(pooling_task, model_config)
1129
1130
        else:
            for pooling_param in pooling_params:
1131
                pooling_param.verify(pooling_task, model_config)
1132

1133
1134
1135
1136
1137
        if tokenization_kwargs is None:
            tokenization_kwargs = dict[str, Any]()
            _validate_truncation_size(model_config.max_model_len,
                                      truncate_prompt_tokens,
                                      tokenization_kwargs)
1138

1139
        self._validate_and_add_requests(
1140
            prompts=parsed_prompts,
1141
            params=pooling_params,
1142
            use_tqdm=use_tqdm,
1143
            lora_request=lora_request,
1144
            tokenization_kwargs=tokenization_kwargs,
1145
1146
        )

1147
        outputs = self._run_engine(use_tqdm=use_tqdm)
Joe Runde's avatar
Joe Runde committed
1148
        return self.engine_class.validate_outputs(outputs,
1149
                                                  PoolingRequestOutput)
1150

1151
1152
1153
1154
1155
    def embed(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        *,
1156
        truncate_prompt_tokens: Optional[int] = None,
1157
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
1158
1159
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
1160
1161
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
    ) -> list[EmbeddingRequestOutput]:
1162
1163
1164
1165
1166
1167
1168
1169
1170
        """
        Generate an embedding vector for each prompt.

        This class automatically batches the given prompts, considering
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
            prompts: The prompts to the LLM. You may pass a sequence of prompts
1171
                for batch inference. See [PromptType][vllm.inputs.PromptType]
1172
                for more details about the format of each prompts.
1173
1174
            pooling_params: The pooling parameters for pooling. If None, we
                use the default pooling parameters.
1175
1176
1177
1178
            use_tqdm: If `True`, shows a tqdm progress bar.
                If a callable (e.g., `functools.partial(tqdm, leave=False)`),
                it is used to create the progress bar.
                If `False`, no progress bar is created.
1179
1180
1181
            lora_request: LoRA request to use for generation, if any.

        Returns:
1182
            A list of `EmbeddingRequestOutput` objects containing the
1183
1184
            embedding vectors in the same order as the input prompts.
        """
1185
        if "embed" not in self.supported_tasks:
1186
1187
            raise ValueError("Embedding API is not supported by this model. "
                             "Please set `--task embed`.")
1188

1189
1190
1191
1192
1193
1194
1195
1196
        items = self.encode(
            prompts,
            truncate_prompt_tokens=truncate_prompt_tokens,
            use_tqdm=use_tqdm,
            pooling_params=pooling_params,
            lora_request=lora_request,
            pooling_task="embed",
        )
1197
1198
1199
1200
1201
1202
1203
1204

        return [EmbeddingRequestOutput.from_base(item) for item in items]

    def classify(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        *,
1205
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
1206
1207
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
    ) -> list[ClassificationRequestOutput]:
1208
1209
1210
1211
1212
1213
1214
1215
1216
        """
        Generate class logits for each prompt.

        This class automatically batches the given prompts, considering
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
            prompts: The prompts to the LLM. You may pass a sequence of prompts
1217
                for batch inference. See [PromptType][vllm.inputs.PromptType]
1218
                for more details about the format of each prompts.
1219
1220
1221
1222
            use_tqdm: If `True`, shows a tqdm progress bar.
                If a callable (e.g., `functools.partial(tqdm, leave=False)`),
                it is used to create the progress bar.
                If `False`, no progress bar is created.
1223
1224
1225
            lora_request: LoRA request to use for generation, if any.

        Returns:
1226
            A list of `ClassificationRequestOutput` objects containing the
1227
1228
            embedding vectors in the same order as the input prompts.
        """
1229
        if "classify" not in self.supported_tasks:
1230
            raise ValueError(
1231
1232
                "Classification API is not supported by this model. "
                "Please set `--task classify`.")
1233

1234
1235
1236
1237
1238
1239
        items = self.encode(
            prompts,
            use_tqdm=use_tqdm,
            lora_request=lora_request,
            pooling_task="classify",
        )
1240
1241
1242

        return [ClassificationRequestOutput.from_base(item) for item in items]

1243
1244
1245
    def _embedding_score(
        self,
        tokenizer: AnyTokenizer,
1246
1247
        text_1: list[Union[str, TextPrompt, TokensPrompt]],
        text_2: list[Union[str, TextPrompt, TokensPrompt]],
1248
        truncate_prompt_tokens: Optional[int] = None,
1249
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
1250
1251
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
    ) -> list[ScoringRequestOutput]:
1252

1253
        encoded_output: list[PoolingRequestOutput] = self.encode(
1254
            text_1 + text_2,
1255
            truncate_prompt_tokens=truncate_prompt_tokens,
1256
1257
            use_tqdm=use_tqdm,
            lora_request=lora_request,
1258
1259
            pooling_task="embed",
        )
1260

1261
        encoded_output_1: list[PoolingRequestOutput] = encoded_output[
1262
            0:len(text_1)]
1263
        encoded_output_2: list[PoolingRequestOutput] = encoded_output[
1264
            len(text_1):]
1265
1266
1267
1268

        if len(encoded_output_1) == 1:
            encoded_output_1 = encoded_output_1 * len(encoded_output_2)

1269
1270
1271
        scores = _cosine_similarity(tokenizer=tokenizer,
                                    embed_1=encoded_output_1,
                                    embed_2=encoded_output_2)
1272
1273
1274
1275
1276
1277
1278

        items = self.engine_class.validate_outputs(scores,
                                                   PoolingRequestOutput)
        return [ScoringRequestOutput.from_base(item) for item in items]

    def _cross_encoding_score(
        self,
1279
        tokenizer: AnyTokenizer,
1280
1281
        data_1: Union[list[str], list[ScoreContentPartParam]],
        data_2: Union[list[str], list[ScoreContentPartParam]],
1282
        truncate_prompt_tokens: Optional[int] = None,
1283
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
1284
1285
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
    ) -> list[ScoringRequestOutput]:
1286
1287
1288
1289
1290

        if isinstance(tokenizer, MistralTokenizer):
            raise ValueError(
                "Score API is only enabled for `--task embed or score`")

1291
1292
        if len(data_1) == 1:
            data_1 = data_1 * len(data_2)
1293

1294
        pooling_params = PoolingParams(task="score")
1295
        tokenization_kwargs: dict[str, Any] = {}
1296
1297
        _validate_truncation_size(self.llm_engine.model_config.max_model_len,
                                  truncate_prompt_tokens, tokenization_kwargs)
1298
1299
1300

        parsed_prompts = []

1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
        input_pairs = [(t1, t2) for t1, t2 in zip(data_1, data_2)]

        if self.llm_engine.model_config.is_multimodal_model:

            model_config = self.llm_engine.model_config

            for q, d in input_pairs:
                _, engine_prompt = get_score_prompt(
                    model_config=model_config,
                    data_1=q,
                    data_2=d,
                    tokenizer=tokenizer,
                    tokenization_kwargs=tokenization_kwargs,
                )

                parsed_prompts.append(engine_prompt)

        else:

            for q, t in input_pairs:
                if self.llm_engine.model_config.use_pad_token:
                    # cross_encoder models defaults to using pad_token.
                    prompt_inputs = tokenizer(
                        text=q,  # type: ignore[arg-type]
                        text_pair=t,  # type: ignore[arg-type]
                        **tokenization_kwargs)
                else:
                    # `llm as reranker` models defaults to not using pad_token.
                    prompt_inputs = tokenizer(
                        text=q + t,  # type: ignore[operator]
                        **tokenization_kwargs)
                engine_prompt = TokensPrompt(
                    prompt_token_ids=prompt_inputs["input_ids"],
                    token_type_ids=prompt_inputs.get("token_type_ids"))
                parsed_prompts.append(engine_prompt)
1336
1337
1338
1339

        self._validate_and_add_requests(
            prompts=parsed_prompts,
            params=pooling_params,
1340
            use_tqdm=use_tqdm,
1341
1342
1343
1344
1345
1346
1347
1348
1349
            lora_request=lora_request,
        )

        outputs = self._run_engine(use_tqdm=use_tqdm)
        items = self.engine_class.validate_outputs(outputs,
                                                   PoolingRequestOutput)

        return [ScoringRequestOutput.from_base(item) for item in items]

1350
1351
    def score(
        self,
1352
1353
1354
1355
        data_1: Union[SingletonPrompt, Sequence[SingletonPrompt],
                      ScoreMultiModalParam],
        data_2: Union[SingletonPrompt, Sequence[SingletonPrompt],
                      ScoreMultiModalParam],
1356
        /,
1357
        *,
1358
        truncate_prompt_tokens: Optional[int] = None,
1359
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
1360
1361
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
    ) -> list[ScoringRequestOutput]:
1362
1363
        """Generate similarity scores for all pairs `<text,text_pair>` or
          `<multi-modal data, multi-modal data pair>`.
1364

1365
        The inputs can be `1 -> 1`, `1 -> N` or `N -> N`.
1366
1367
        In the `1 - N` case the `data_1` input will be replicated `N`
        times to pair with the `data_2` inputs.
1368
        The input pairs are used to build a list of prompts for the
1369
1370
        cross encoder model. This class automatically batches the prompts,
        considering the memory constraint. For the best performance, put all
1371
1372
1373
1374
1375
        of your inputs into a single list and pass it to this method.

        Supports both text and multi-modal data (images, etc.) when used with
        appropriate multi-modal models. For multi-modal inputs, ensure the 
        prompt structure matches the model's expected input format.
1376
1377

        Args:
1378
1379
1380
1381
1382
1383
1384
1385
            data_1: Can be a single prompt, a list of prompts or 
                `ScoreMultiModalParam`, which can contain either text or 
                multi-modal data. When a list, it must have the same length as 
                the `data_2` list.
            data_2: The data to pair with the query to form the input to 
                the LLM. Can be text or multi-modal data. See [PromptType]
                [vllm.inputs.PromptType] for more details about the format of 
                each prompt.
1386
1387
1388
1389
            use_tqdm: If `True`, shows a tqdm progress bar.
                If a callable (e.g., `functools.partial(tqdm, leave=False)`),
                it is used to create the progress bar.
                If `False`, no progress bar is created.
1390
1391
1392
            lora_request: LoRA request to use for generation, if any.

        Returns:
1393
            A list of `ScoringRequestOutput` objects containing the
1394
1395
            generated scores in the same order as the input prompts.
        """
1396
1397
        model_config = self.llm_engine.model_config
        runner_type = model_config.runner_type
1398
1399
        if runner_type != "pooling":
            messages = ["LLM.score() is only supported for pooling models."]
1400

1401
            if "pooling" in model_config.supported_runner_types:
1402
                messages.append(
1403
1404
1405
1406
                    "Your model supports the 'pooling' runner, but is "
                    f"currently initialized for the '{runner_type}' runner. "
                    "Please initialize vLLM using `--task embed`, "
                    "`--task classify`, `--task score` etc.")
1407
1408
1409

            raise ValueError(" ".join(messages))

1410
1411
        supported_tasks = self.supported_tasks
        if all(t not in supported_tasks for t in ("embed", "classify")):
1412
1413
            raise ValueError("Score API is not supported by this model. "
                             "Please set `--task embed` or `--task classify`.")
1414

1415
1416
        if (model_config.task == "classify"
                and getattr(model_config.hf_config, "num_labels", 0) != 1):
1417
            raise ValueError("Score API is only enabled for num_labels == 1.")
1418
1419
1420
1421

        # the tokenizer for models such as
        # "cross-encoder/ms-marco-MiniLM-L-6-v2" doesn't support passing
        # lists of tokens to the `text` and `text_pair` kwargs
1422
        tokenizer = self.get_tokenizer()
1423

1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
        if not self.llm_engine.model_config.is_multimodal_model:

            def check_data_type(data: Union[SingletonPrompt,
                                            Sequence[SingletonPrompt],
                                            ScoreMultiModalParam]):
                if isinstance(data, dict) and "content" in data:
                    raise ValueError(
                        f"ScoreMultiModalParam is not supported for {self.llm_engine.model_config.architecture}",  # noqa: E501
                    )

            check_data_type(data_1)
            check_data_type(data_2)

            def ensure_str(prompt: SingletonPrompt):
                if isinstance(prompt, dict):
                    if "multi_modal_data" in prompt:
                        raise ValueError("Multi-modal prompt is not "
                                         "supported for scoring")
                    elif "prompt_token_ids" in prompt:
                        prompt = tokenizer.decode(
                            cast(TokensPrompt, prompt)["prompt_token_ids"])
                    elif "prompt" in prompt:
                        prompt = cast(TextPrompt, prompt)["prompt"]
                assert type(prompt) is str
                return prompt

            if isinstance(data_1, (str, dict)):
                # Convert a single prompt to a list.
                data_1 = [data_1]  # type: ignore[list-item]

            data_1 = [ensure_str(t) for t in data_1]

            if isinstance(data_2, (str, dict)):
                # Convert a single prompt to a list.
                data_2 = [data_2]  # type: ignore[list-item]

            data_2 = [ensure_str(t) for t in data_2]

        if isinstance(data_1, dict) and "content" in data_1:
            data_1 = data_1.get("content")  # type: ignore[assignment]
        elif isinstance(data_1, str):
            data_1 = [data_1]

        if isinstance(data_2, dict) and "content" in data_2:
            data_2 = data_2.get("content")  # type: ignore[assignment]
        elif isinstance(data_2, str):
            data_2 = [data_2]

        _validate_score_input_lens(data_1, data_2)  # type: ignore[arg-type]
1473

1474
        if self.llm_engine.model_config.is_cross_encoder:
1475
1476
1477
1478
1479
1480
            return self._cross_encoding_score(
                tokenizer,
                data_1,  # type: ignore[arg-type]
                data_2,  # type: ignore[arg-type]
                truncate_prompt_tokens,
                use_tqdm,
1481
                lora_request)
1482
        else:
1483
1484
            return self._embedding_score(
                tokenizer,
1485
1486
                data_1,  # type: ignore[arg-type]
                data_2,  # type: ignore[arg-type]
1487
1488
                truncate_prompt_tokens,
                use_tqdm,
1489
                lora_request)
1490

1491
1492
1493
1494
1495
1496
    def start_profile(self) -> None:
        self.llm_engine.start_profile()

    def stop_profile(self) -> None:
        self.llm_engine.stop_profile()

1497
1498
    def reset_prefix_cache(self, device: Optional[Device] = None) -> bool:
        return self.llm_engine.reset_prefix_cache(device)
1499

1500
1501
1502
1503
1504
1505
    def sleep(self, level: int = 1):
        """
        Put the engine to sleep. The engine should not process any requests.
        The caller should guarantee that no requests are being processed
        during the sleep period, before `wake_up` is called.

1506
        Args:
1507
1508
            level: The sleep level. Level 1 sleep will offload the model
                weights and discard the kv cache. The content of kv cache
1509
                is forgotten. Level 1 sleep is good for sleeping and waking
1510
1511
1512
1513
1514
                up the engine to run the same model again. The model weights
                are backed up in CPU memory. Please make sure there's enough
                CPU memory to store the model weights. Level 2 sleep will
                discard both the model weights and the kv cache. The content
                of both the model weights and kv cache is forgotten. Level 2
1515
                sleep is good for sleeping and waking up the engine to run a
1516
                different model or update the model, where previous model
1517
                weights are not needed. It reduces CPU memory pressure.
1518
        """
1519
        self.reset_prefix_cache()
1520
1521
        self.llm_engine.sleep(level=level)

1522
    def wake_up(self, tags: Optional[list[str]] = None):
1523
        """
1524
        Wake up the engine from sleep mode. See the [sleep][] method
1525
        for more details.
1526

1527
        Args:
1528
1529
            tags: An optional list of tags to reallocate the engine memory
                for specific memory allocations. Values must be in
1530
                `("weights", "kv_cache")`. If None, all memory is reallocated.
1531
                wake_up should be called with all tags (or None) before the
1532
1533
1534
                engine is used again.
        """
        self.llm_engine.wake_up(tags)
1535

1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
    def get_metrics(self) -> list["Metric"]:
        """Return a snapshot of aggregated metrics from Prometheus.

        Returns:
            A ``MetricSnapshot`` instance capturing the current state
            of all aggregated metrics from Prometheus.

        Note:
            This method is only available with the V1 LLM engine.
        """
        from vllm.v1.engine.llm_engine import LLMEngine as V1LLMEngine
        assert isinstance(self.llm_engine, V1LLMEngine)
        return self.llm_engine.get_metrics()

1550
1551
    # LEGACY
    def _convert_v1_inputs(
1552
        self,
1553
1554
        prompts: Optional[Union[str, list[str]]],
        prompt_token_ids: Optional[Union[list[int], list[list[int]]]],
1555
1556
    ):
        # skip_tokenizer_init is now checked in engine
1557

1558
1559
1560
1561
1562
1563
1564
1565
1566
        if prompts is None and prompt_token_ids is None:
            raise ValueError(
                "Either prompts or prompt_token_ids must be provided.")
        if prompts is not None and prompt_token_ids is not None \
                and len(prompts) != len(prompt_token_ids):
            raise ValueError(
                "The lengths of prompts and prompt_token_ids must be the same."
            )

1567
1568
1569
1570
1571
1572
        if prompts is not None:
            prompts = [p["content"] for p in parse_and_batch_prompt(prompts)]
        if prompt_token_ids is not None:
            prompt_token_ids = [
                p["content"] for p in parse_and_batch_prompt(prompt_token_ids)
            ]
1573
1574
        if prompts is not None:
            num_requests = len(prompts)
1575
        elif prompt_token_ids is not None:
1576
            num_requests = len(prompt_token_ids)
1577
        parsed_prompts: list[PromptType] = []
1578
        for i in range(num_requests):
1579
            item: PromptType
1580

1581
            if prompts is not None:
1582
1583
1584
                item = TextPrompt(prompt=prompts[i])
            elif prompt_token_ids is not None:
                item = TokensPrompt(prompt_token_ids=prompt_token_ids[i])
1585
            else:
1586
                raise AssertionError
1587

1588
            parsed_prompts.append(item)
1589

1590
        return parsed_prompts
1591
1592
1593

    def _validate_and_add_requests(
        self,
1594
        prompts: Union[PromptType, Sequence[PromptType]],
1595
1596
        params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
                      Sequence[PoolingParams]],
1597
        *,
1598
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
1599
        lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
1600
        tokenization_kwargs: Optional[dict[str, Any]] = None,
1601
        guided_options: Optional[GuidedDecodingRequest] = None,
1602
        priority: Optional[list[int]] = None,
1603
    ) -> None:
1604
1605
1606
1607
1608
1609
1610
1611
        if guided_options is not None:
            warnings.warn(
                "guided_options_request is deprecated, use "
                "SamplingParams.guided_decoding instead",
                DeprecationWarning,
                stacklevel=2,
            )

1612
        if isinstance(prompts, (str, dict)):
1613
            # Convert a single prompt to a list.
1614
            prompts = [prompts]
1615

1616
        num_requests = len(prompts)
1617
        if isinstance(params, Sequence) and len(params) != num_requests:
1618
            raise ValueError("The lengths of prompts and params "
1619
                             "must be the same.")
1620
        if isinstance(lora_request,
1621
                      Sequence) and len(lora_request) != num_requests:
1622
1623
            raise ValueError("The lengths of prompts and lora_request "
                             "must be the same.")
1624

1625
        for sp in params if isinstance(params, Sequence) else (params, ):
1626
            if isinstance(sp, SamplingParams):
1627
                self._add_guided_params(sp, guided_options)
1628
1629
1630

                # We only care about the final output
                sp.output_kind = RequestOutputKind.FINAL_ONLY
1631

Zhuohan Li's avatar
Zhuohan Li committed
1632
        # Add requests to the engine.
1633
1634
        it = prompts
        if use_tqdm:
1635
1636
            tqdm_func = use_tqdm if callable(use_tqdm) else tqdm
            it = tqdm_func(it, desc="Adding requests")
1637
1638

        for i, prompt in enumerate(it):
1639
            self._add_request(
1640
                prompt,
1641
                params[i] if isinstance(params, Sequence) else params,
1642
                tokenization_kwargs=tokenization_kwargs,
1643
1644
                lora_request=lora_request[i] if isinstance(
                    lora_request, Sequence) else lora_request,
1645
                priority=priority[i] if priority else 0,
nunjunj's avatar
nunjunj committed
1646
            )
1647

1648
    def _add_request(
nunjunj's avatar
nunjunj committed
1649
        self,
1650
        prompt: PromptType,
nunjunj's avatar
nunjunj committed
1651
        params: Union[SamplingParams, PoolingParams],
1652
        tokenization_kwargs: Optional[dict[str, Any]] = None,
1653
        lora_request: Optional[LoRARequest] = None,
1654
        priority: int = 0,
1655
1656
    ) -> None:
        request_id = str(next(self.request_counter))
1657
1658
        self.llm_engine.add_request(
            request_id,
1659
            prompt,
1660
1661
            params,
            lora_request=lora_request,
1662
            tokenization_kwargs=tokenization_kwargs,
1663
            priority=priority,
nunjunj's avatar
nunjunj committed
1664
        )
1665

1666
    def _add_guided_params(
1667
1668
1669
            self,
            params: SamplingParams,
            guided_options: Optional[GuidedDecodingRequest] = None):
1670
1671
1672
1673
        if guided_options is None:
            return params

        if params.guided_decoding is not None:
1674
            raise ValueError("Cannot set both guided_options_request and "
1675
1676
1677
1678
1679
1680
1681
1682
1683
                             "params.guided_decoding.")

        params.guided_decoding = GuidedDecodingParams(
            json=guided_options.guided_json,
            regex=guided_options.guided_regex,
            choice=guided_options.guided_choice,
            grammar=guided_options.guided_grammar,
            json_object=guided_options.guided_json_object,
            backend=guided_options.guided_decoding_backend,
1684
1685
1686
            whitespace_pattern=guided_options.guided_whitespace_pattern,
            structural_tag=guided_options.structural_tag,
        )
1687
1688
        return params

1689
    def _run_engine(
1690
1691
1692
        self,
        *,
        use_tqdm: Union[bool, Callable[..., tqdm]] = True
1693
    ) -> list[Union[RequestOutput, PoolingRequestOutput]]:
1694
1695
        # Initialize tqdm.
        if use_tqdm:
Zhuohan Li's avatar
Zhuohan Li committed
1696
            num_requests = self.llm_engine.get_num_unfinished_requests()
1697
1698
            tqdm_func = use_tqdm if callable(use_tqdm) else tqdm
            pbar = tqdm_func(
1699
1700
1701
                total=num_requests,
                desc="Processed prompts",
                dynamic_ncols=True,
1702
1703
                postfix=(f"est. speed input: {0:.2f} toks/s, "
                         f"output: {0:.2f} toks/s"),
1704
            )
1705

Zhuohan Li's avatar
Zhuohan Li committed
1706
        # Run the engine.
1707
        outputs: list[Union[RequestOutput, PoolingRequestOutput]] = []
1708
1709
        total_in_toks = 0
        total_out_toks = 0
Zhuohan Li's avatar
Zhuohan Li committed
1710
1711
        while self.llm_engine.has_unfinished_requests():
            step_outputs = self.llm_engine.step()
1712
            for output in step_outputs:
1713
                if output.finished:
1714
1715
                    outputs.append(output)
                    if use_tqdm:
1716
1717
                        if isinstance(output, RequestOutput):
                            # Calculate tokens only for RequestOutput
1718
                            n = len(output.outputs)
1719
                            assert output.prompt_token_ids is not None
1720
                            total_in_toks += len(output.prompt_token_ids) * n
1721
1722
                            in_spd = total_in_toks / pbar.format_dict["elapsed"]
                            total_out_toks += sum(
1723
                                len(stp.token_ids) for stp in output.outputs)
nunjunj's avatar
nunjunj committed
1724
1725
                            out_spd = (total_out_toks /
                                       pbar.format_dict["elapsed"])
1726
1727
1728
                            pbar.postfix = (
                                f"est. speed input: {in_spd:.2f} toks/s, "
                                f"output: {out_spd:.2f} toks/s")
1729
                            pbar.update(n)
1730
1731
                        else:
                            pbar.update(1)
1732
1733
                        if pbar.n == num_requests:
                            pbar.refresh()
1734

1735
1736
        if use_tqdm:
            pbar.close()
1737
1738
1739
        # Sort the outputs by request ID.
        # This is necessary because some requests may be finished earlier than
        # its previous requests.
1740
        return sorted(outputs, key=lambda x: int(x.request_id))