llm.py 62.9 KB
Newer Older
1
2
# SPDX-License-Identifier: Apache-2.0

3
import itertools
4
import warnings
5
from collections.abc import Sequence
6
from contextlib import contextmanager
7
from typing import Any, Callable, ClassVar, Optional, Union, cast, overload
8

9
import cloudpickle
10
import torch.nn as nn
11
from tqdm.auto import tqdm
12
from typing_extensions import TypeVar, deprecated
13

14
15
from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
                              BeamSearchSequence, get_beam_search_score)
16
from vllm.config import CompilationConfig
17
18
from vllm.engine.arg_utils import (EngineArgs, HfOverrides, PoolerConfig,
                                   TaskOption)
Joe Runde's avatar
Joe Runde committed
19
from vllm.engine.llm_engine import LLMEngine
nunjunj's avatar
nunjunj committed
20
from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
21
                                         ChatTemplateContentFormatOption,
22
23
                                         apply_hf_chat_template,
                                         apply_mistral_chat_template,
24
25
                                         parse_chat_messages,
                                         resolve_chat_template_content_format)
26
27
from vllm.entrypoints.score_utils import (_cosine_similarity,
                                          _validate_score_input_lens)
28
from vllm.inputs import PromptType, SingletonPrompt, TextPrompt, TokensPrompt
29
from vllm.inputs.parse import is_token_prompt, parse_and_batch_prompt
30
from vllm.logger import init_logger
31
from vllm.lora.request import LoRARequest
32
33
from vllm.model_executor.guided_decoding.guided_fields import (
    GuidedDecodingRequest, LLMGuidedOptions)
34
35
36
from vllm.outputs import (ClassificationRequestOutput, EmbeddingRequestOutput,
                          PoolingRequestOutput, RequestOutput,
                          ScoringRequestOutput)
37
from vllm.pooling_params import PoolingParams
38
from vllm.prompt_adapter.request import PromptAdapterRequest
39
40
from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
                                  RequestOutputKind, SamplingParams)
41
from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
42
                                               get_cached_tokenizer)
yhu422's avatar
yhu422 committed
43
from vllm.usage.usage_lib import UsageContext
44
45
from vllm.utils import (Counter, Device, deprecate_args, deprecate_kwargs,
                        is_list_of)
lizhigong's avatar
lizhigong committed
46
from vllm.zero_overhead.v0.llm_engine import ZeroOverheadEngine
lizhigong's avatar
lizhigong committed
47
from vllm.zero_overhead.v0.utils import is_zero_overhead
48

49
50
logger = init_logger(__name__)

51
52
_R = TypeVar("_R", default=Any)

53
54

class LLM:
Woosuk Kwon's avatar
Woosuk Kwon committed
55
56
57
58
59
60
61
62
63
64
    """An LLM for generating texts from given prompts and sampling parameters.

    This class includes a tokenizer, a language model (possibly distributed
    across multiple GPUs), and GPU memory space allocated for intermediate
    states (aka KV cache). Given a batch of prompts and sampling parameters,
    this class generates texts from the model, using an intelligent batching
    mechanism and efficient memory management.

    Args:
        model: The name or path of a HuggingFace Transformers model.
65
        tokenizer: The name or path of a HuggingFace Transformers tokenizer.
66
67
        tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer
            if available, and "slow" will always use the slow tokenizer.
68
69
70
        skip_tokenizer_init: If true, skip initialization of tokenizer and
            detokenizer. Expect valid prompt_token_ids and None for prompt
            from the input.
71
72
        trust_remote_code: Trust remote code (e.g., from HuggingFace) when
            downloading the model and tokenizer.
73
74
75
76
        allowed_local_media_path: Allowing API requests to read local images
            or videos from directories specified by the server file system.
            This is a security risk. Should only be enabled in trusted
            environments.
Woosuk Kwon's avatar
Woosuk Kwon committed
77
78
79
        tensor_parallel_size: The number of GPUs to use for distributed
            execution with tensor parallelism.
        dtype: The data type for the model weights and activations. Currently,
Woosuk Kwon's avatar
Woosuk Kwon committed
80
81
82
83
            we support `float32`, `float16`, and `bfloat16`. If `auto`, we use
            the `torch_dtype` attribute specified in the model config file.
            However, if the `torch_dtype` in the config is `float32`, we will
            use `float16` instead.
84
        quantization: The method used to quantize the model weights. Currently,
85
            we support "awq", "gptq", and "fp8" (experimental).
86
87
88
89
            If None, we first check the `quantization_config` attribute in the
            model config file. If that is None, we assume the model weights are
            not quantized and use `dtype` to determine the data type of
            the weights.
Jasmond L's avatar
Jasmond L committed
90
91
        revision: The specific model version to use. It can be a branch name,
            a tag name, or a commit id.
92
93
        tokenizer_revision: The specific tokenizer version to use. It can be a
            branch name, a tag name, or a commit id.
94
95
96
97
98
99
100
        seed: The seed to initialize the random number generator for sampling.
        gpu_memory_utilization: The ratio (between 0 and 1) of GPU memory to
            reserve for the model weights, activations, and KV cache. Higher
            values will increase the KV cache size and thus improve the model's
            throughput. However, if the value is too high, it may cause out-of-
            memory (OOM) errors.
        swap_space: The size (GiB) of CPU memory per GPU to use as swap space.
101
102
103
104
105
            This can be used for temporarily storing the states of the requests
            when their `best_of` sampling parameters are larger than 1. If all
            requests will have `best_of=1`, you can safely set this to 0.
            Noting that `best_of` is only supported in V0. Otherwise, too small
            values may cause out-of-memory (OOM) errors.
106
107
108
109
        cpu_offload_gb: The size (GiB) of CPU memory to use for offloading
            the model weights. This virtually increases the GPU memory space
            you can use to hold the model weights, at the cost of CPU-GPU data
            transfer for every forward pass.
110
111
112
        enforce_eager: Whether to enforce eager execution. If True, we will
            disable CUDA graph and always execute the model in eager mode.
            If False, we will use CUDA graph and eager execution in hybrid.
113
        max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs.
114
            When a sequence has context length larger than this, we fall back
115
116
117
            to eager mode. Additionally for encoder-decoder models, if the
            sequence length of the encoder input is larger than this, we fall
            back to the eager mode.
118
119
120
        disable_custom_all_reduce: See :class:`~vllm.config.ParallelConfig`
        disable_async_output_proc: Disable async output processing.
            This may result in lower performance.
121
        hf_token: The token to use as HTTP bearer authorization for remote files
122
            . If `True`, will use the token generated when running
123
            `huggingface-cli login` (stored in `~/.huggingface`).
124
125
126
        hf_overrides: If a dictionary, contains arguments to be forwarded to the
            HuggingFace config. If a callable, it is called to update the
            HuggingFace config.
127
128
129
        compilation_config: Either an integer or a dictionary. If it is an
            integer, it is used as the level of compilation optimization. If it
            is a dictionary, it can specify the full compilation configuration.
130
        **kwargs: Arguments for :class:`~vllm.EngineArgs`. (See
131
            :ref:`engine-args`)
nunjunj's avatar
nunjunj committed
132

133
134
135
    Note:
        This class is intended to be used for offline inference. For online
        serving, use the :class:`~vllm.AsyncLLMEngine` class instead.
136
    """
137

138
    DEPRECATE_LEGACY: ClassVar[bool] = True
139
140
    """A flag to toggle whether to deprecate the legacy generate/encode API."""

141
142
143
144
145
146
    DEPRECATE_INIT_POSARGS: ClassVar[bool] = True
    """
    A flag to toggle whether to deprecate positional arguments in
    :meth:`LLM.__init__`.
    """

147
148
149
150
151
152
153
154
155
    @classmethod
    @contextmanager
    def deprecate_legacy_api(cls):
        cls.DEPRECATE_LEGACY = True

        yield

        cls.DEPRECATE_LEGACY = False

156
157
158
159
160
161
162
    @deprecate_args(
        start_index=2,  # Ignore self and model
        is_deprecated=lambda: LLM.DEPRECATE_INIT_POSARGS,
        additional_message=(
            "All positional arguments other than `model` will be "
            "replaced with keyword arguments in an upcoming version."),
    )
163
164
165
    def __init__(
        self,
        model: str,
166
        tokenizer: Optional[str] = None,
167
        tokenizer_mode: str = "auto",
168
        skip_tokenizer_init: bool = False,
169
        trust_remote_code: bool = False,
170
        allowed_local_media_path: str = "",
171
        tensor_parallel_size: int = 1,
Woosuk Kwon's avatar
Woosuk Kwon committed
172
        dtype: str = "auto",
173
        quantization: Optional[str] = None,
174
        revision: Optional[str] = None,
175
        tokenizer_revision: Optional[str] = None,
176
        seed: Optional[int] = None,
177
        gpu_memory_utilization: float = 0.9,
178
        swap_space: float = 4,
179
        cpu_offload_gb: float = 0,
180
        enforce_eager: Optional[bool] = None,
181
        max_seq_len_to_capture: int = 8192,
182
        disable_custom_all_reduce: bool = False,
183
        disable_async_output_proc: bool = False,
184
        hf_token: Optional[Union[bool, str]] = None,
185
        hf_overrides: Optional[HfOverrides] = None,
186
        mm_processor_kwargs: Optional[dict[str, Any]] = None,
187
188
        # After positional args are removed, move this right below `model`
        task: TaskOption = "auto",
189
        override_pooler_config: Optional[PoolerConfig] = None,
190
        compilation_config: Optional[Union[int, dict[str, Any]]] = None,
191
192
        **kwargs,
    ) -> None:
193
194
195
196
        '''
        LLM constructor.

        Note: if enforce_eager is unset (enforce_eager is None)
197
        it defaults to False.
198
199
        '''

200
201
        if "disable_log_stats" not in kwargs:
            kwargs["disable_log_stats"] = True
202

203
204
205
206
207
208
209
        if "worker_cls" in kwargs:
            worker_cls = kwargs["worker_cls"]
            # if the worker_cls is not qualified string name,
            # we serialize it using cloudpickle to avoid pickling issues
            if isinstance(worker_cls, type):
                kwargs["worker_cls"] = cloudpickle.dumps(worker_cls)

210
        if compilation_config is not None:
211
            if isinstance(compilation_config, (int, dict)):
212
213
214
215
                compilation_config_instance = CompilationConfig.from_cli(
                    str(compilation_config))
            else:
                compilation_config_instance = compilation_config
216
217
218
        else:
            compilation_config_instance = None

Zhuohan Li's avatar
Zhuohan Li committed
219
        engine_args = EngineArgs(
220
            model=model,
221
            task=task,
222
            tokenizer=tokenizer,
223
            tokenizer_mode=tokenizer_mode,
224
            skip_tokenizer_init=skip_tokenizer_init,
225
            trust_remote_code=trust_remote_code,
226
            allowed_local_media_path=allowed_local_media_path,
227
228
            tensor_parallel_size=tensor_parallel_size,
            dtype=dtype,
229
            quantization=quantization,
230
            revision=revision,
231
            tokenizer_revision=tokenizer_revision,
232
233
234
            seed=seed,
            gpu_memory_utilization=gpu_memory_utilization,
            swap_space=swap_space,
235
            cpu_offload_gb=cpu_offload_gb,
236
            enforce_eager=enforce_eager,
237
            max_seq_len_to_capture=max_seq_len_to_capture,
238
            disable_custom_all_reduce=disable_custom_all_reduce,
239
            disable_async_output_proc=disable_async_output_proc,
240
            hf_token=hf_token,
241
            hf_overrides=hf_overrides,
242
            mm_processor_kwargs=mm_processor_kwargs,
243
            override_pooler_config=override_pooler_config,
244
            compilation_config=compilation_config_instance,
245
246
            **kwargs,
        )
247
248

        # Create the Engine (autoselects V0 vs V1)
lizhigong's avatar
lizhigong committed
249
250
251
252
253
254
        if is_zero_overhead():
            self.llm_engine = ZeroOverheadEngine.from_engine_args(
                engine_args=engine_args, usage_context=UsageContext.LLM_CLASS)
        else:
            self.llm_engine = LLMEngine.from_engine_args(
                engine_args=engine_args, usage_context=UsageContext.LLM_CLASS)
255
        self.engine_class = type(self.llm_engine)
256

257
        self.request_counter = Counter()
258
        self.default_sampling_params: Union[dict[str, Any], None] = None
259

260
261
262
263
264
265
    def get_tokenizer(
        self,
        lora_request: Optional[LoRARequest] = None,
    ) -> AnyTokenizer:
        return self.llm_engine.get_tokenizer_group().get_lora_tokenizer(
            lora_request)
266
267

    def set_tokenizer(self, tokenizer: AnyTokenizer) -> None:
268
        tokenizer_group = self.llm_engine.get_tokenizer_group()
269

270
271
272
273
        # While CachedTokenizer is dynamic, have no choice but
        # compare class name. Misjudgment will arise from
        # user-defined tokenizer started with 'Cached'
        if tokenizer.__class__.__name__.startswith("Cached"):
274
            tokenizer_group.tokenizer = tokenizer
275
        else:
276
            tokenizer_group.tokenizer = get_cached_tokenizer(tokenizer)
277

278
    def get_default_sampling_params(self) -> SamplingParams:
279
280
281
282
283
        if self.default_sampling_params is None:
            self.default_sampling_params = (
                self.llm_engine.model_config.get_diff_sampling_param())
        if self.default_sampling_params:
            return SamplingParams.from_optional(**self.default_sampling_params)
284
285
        return SamplingParams()

286
287
288
289
290
291
292
    @overload
    def generate(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        sampling_params: Optional[Union[SamplingParams,
                                        Sequence[SamplingParams]]] = None,
293
        *,
294
        use_tqdm: bool = True,
295
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
296
297
298
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
299
    ) -> list[RequestOutput]:
300
301
        ...

302
    @overload  # LEGACY: single (prompt + optional token ids)
303
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
304
305
306
307
    def generate(
        self,
        prompts: str,
        sampling_params: Optional[Union[SamplingParams,
308
309
                                        list[SamplingParams]]] = None,
        prompt_token_ids: Optional[list[int]] = None,
310
        use_tqdm: bool = True,
311
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
312
313
314
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
315
    ) -> list[RequestOutput]:
316
317
318
        ...

    @overload  # LEGACY: multi (prompt + optional token ids)
319
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
320
321
    def generate(
        self,
322
        prompts: list[str],
323
        sampling_params: Optional[Union[SamplingParams,
324
325
                                        list[SamplingParams]]] = None,
        prompt_token_ids: Optional[list[list[int]]] = None,
326
        use_tqdm: bool = True,
327
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
328
329
330
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
331
    ) -> list[RequestOutput]:
332
333
334
        ...

    @overload  # LEGACY: single (token ids + optional prompt)
335
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
336
337
338
339
    def generate(
        self,
        prompts: Optional[str] = None,
        sampling_params: Optional[Union[SamplingParams,
340
                                        list[SamplingParams]]] = None,
341
        *,
342
        prompt_token_ids: list[int],
343
        use_tqdm: bool = True,
344
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
345
346
347
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
348
    ) -> list[RequestOutput]:
349
350
351
        ...

    @overload  # LEGACY: multi (token ids + optional prompt)
352
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
353
354
    def generate(
        self,
355
        prompts: Optional[list[str]] = None,
356
        sampling_params: Optional[Union[SamplingParams,
357
                                        list[SamplingParams]]] = None,
358
        *,
359
        prompt_token_ids: list[list[int]],
360
        use_tqdm: bool = True,
361
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
362
363
364
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
365
    ) -> list[RequestOutput]:
366
367
368
        ...

    @overload  # LEGACY: single or multi token ids [pos-only]
369
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
370
371
372
373
    def generate(
        self,
        prompts: None,
        sampling_params: None,
374
        prompt_token_ids: Union[list[int], list[list[int]]],
375
        use_tqdm: bool = True,
376
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
377
378
379
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
380
    ) -> list[RequestOutput]:
381
382
        ...

nunjunj's avatar
nunjunj committed
383
384
385
    @deprecate_kwargs(
        "prompt_token_ids",
        is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
386
        additional_message="Please use the 'prompts' parameter instead.",
nunjunj's avatar
nunjunj committed
387
    )
388
389
    def generate(
        self,
390
        prompts: Union[Union[PromptType, Sequence[PromptType]],
391
                       Optional[Union[str, list[str]]]] = None,
392
393
        sampling_params: Optional[Union[SamplingParams,
                                        Sequence[SamplingParams]]] = None,
394
        prompt_token_ids: Optional[Union[list[int], list[list[int]]]] = None,
395
        use_tqdm: bool = True,
396
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
397
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
398
        guided_options_request: Optional[Union[LLMGuidedOptions,
399
                                               GuidedDecodingRequest]] = None,
400
401
        priority: Optional[list[int]] = None,
    ) -> list[RequestOutput]:
Woosuk Kwon's avatar
Woosuk Kwon committed
402
403
        """Generates the completions for the input prompts.

404
        This class automatically batches the given prompts, considering
Woosuk Kwon's avatar
Woosuk Kwon committed
405
406
407
408
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
409
410
411
            prompts: The prompts to the LLM. You may pass a sequence of prompts
                for batch inference. See :class:`~vllm.inputs.PromptType`
                for more details about the format of each prompts.
Woosuk Kwon's avatar
Woosuk Kwon committed
412
            sampling_params: The sampling parameters for text generation. If
nunjunj's avatar
nunjunj committed
413
414
415
                None, we use the default sampling parameters.
                When it is a single value, it is applied to every prompt.
                When it is a list, the list must have the same length as the
416
                prompts and it is paired one by one with the prompt.
Woosuk Kwon's avatar
Woosuk Kwon committed
417
            use_tqdm: Whether to use tqdm to display the progress bar.
418
            lora_request: LoRA request to use for generation, if any.
nunjunj's avatar
nunjunj committed
419
            prompt_adapter_request: Prompt Adapter request to use for
420
                generation, if any.
421
422
            priority: The priority of the requests, if any.
                Only applicable when priority scheduling policy is enabled.
Woosuk Kwon's avatar
Woosuk Kwon committed
423
424

        Returns:
nunjunj's avatar
nunjunj committed
425
            A list of ``RequestOutput`` objects containing the
426
            generated completions in the same order as the input prompts.
427
428
429
430
431

        Note:
            Using ``prompts`` and ``prompt_token_ids`` as keyword parameters is
            considered legacy and may be deprecated in the future. You should
            instead pass them via the ``inputs`` parameter.
432
        """
433
        runner_type = self.llm_engine.model_config.runner_type
434
        if runner_type not in ["generate", "transcription"]:
435
            messages = [
436
                "LLM.generate() is only supported for (conditional) generation "
437
438
439
                "models (XForCausalLM, XForConditionalGeneration).",
            ]

440
441
442
            supported_runner_types = self.llm_engine.model_config \
                .supported_runner_types
            if "generate" in supported_runner_types:
443
                messages.append(
444
445
446
                    "Your model supports the 'generate' runner, but is "
                    f"currently initialized for the '{runner_type}' runner. "
                    "Please initialize vLLM using `--task generate`.")
447
448

            raise ValueError(" ".join(messages))
449

450
        if prompt_token_ids is not None:
451
            parsed_prompts = self._convert_v1_inputs(
452
                prompts=cast(Optional[Union[str, list[str]]], prompts),
453
454
455
                prompt_token_ids=prompt_token_ids,
            )
        else:
456
457
            parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
                                  prompts)
458

459
460
461
462
463
464
465
466
        if isinstance(guided_options_request, dict):
            if len(guided_options_request) > 1:
                raise ValueError(
                    "You can only use one guided decoding but multiple is "
                    f"specified: {guided_options_request}")
            guided_options_request = GuidedDecodingRequest(
                **guided_options_request)

467
468
        if sampling_params is None:
            # Use default sampling params.
469
            sampling_params = self.get_default_sampling_params()
470

471
        self._validate_and_add_requests(
472
            prompts=parsed_prompts,
473
474
            params=sampling_params,
            lora_request=lora_request,
475
            prompt_adapter_request=prompt_adapter_request,
476
477
            guided_options=guided_options_request,
            priority=priority)
478

479
        outputs = self._run_engine(use_tqdm=use_tqdm)
Joe Runde's avatar
Joe Runde committed
480
        return self.engine_class.validate_outputs(outputs, RequestOutput)
481

482
    def collective_rpc(self,
483
                       method: Union[str, Callable[..., _R]],
484
                       timeout: Optional[float] = None,
485
486
                       args: tuple = (),
                       kwargs: Optional[dict[str, Any]] = None) -> list[_R]:
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
        """
        Execute an RPC call on all workers.

        Args:
            method: Name of the worker method to execute, or a callable that
                is serialized and sent to all workers to execute.

                If the method is a callable, it should accept an additional
                `self` argument, in addition to the arguments passed in `args`
                and `kwargs`. The `self` argument will be the worker object.
            timeout: Maximum time in seconds to wait for execution. Raises a
                :exc:`TimeoutError` on timeout. `None` means wait indefinitely.
            args: Positional arguments to pass to the worker method.
            kwargs: Keyword arguments to pass to the worker method.

        Returns:
            A list containing the results from each worker.
        
        Note:
            It is recommended to use this API to only pass control messages,
            and set up data-plane communication to pass data.
        """
509
510

        return self.llm_engine.collective_rpc(method, timeout, args, kwargs)
511
512

    def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]:
513
        """
514
515
        Run a function directly on the model inside each worker,
        returning the result for each of them.
516
        """
517
518
        executor = self.llm_engine.model_executor
        return executor.apply_model(func)
519

520
521
    def beam_search(
        self,
522
        prompts: list[Union[TokensPrompt, TextPrompt]],
523
        params: BeamSearchParams,
524
    ) -> list[BeamSearchOutput]:
525
526
527
528
529
530
        """
        Generate sequences using beam search.

        Args:
            prompts: A list of prompts. Each prompt can be a string or a list
                of token IDs.
531
            params: The beam search parameters.
532
        """
533
534
        # TODO: how does beam search work together with length penalty,
        # frequency, penalty, and stopping criteria, etc.?
535
536
537
538
        beam_width = params.beam_width
        max_tokens = params.max_tokens
        temperature = params.temperature
        ignore_eos = params.ignore_eos
539
540
541
542
543
544
        length_penalty = params.length_penalty

        def sort_beams_key(x: BeamSearchSequence) -> float:
            return get_beam_search_score(x.tokens, x.cum_logprob,
                                         tokenizer.eos_token_id,
                                         length_penalty)
545

546
547
548
549
550
551
552
553
554
555
556
557
        def create_tokens_prompt_from_beam(
                beam: BeamSearchSequence) -> TokensPrompt:
            token_prompt_kwargs: TokensPrompt = {
                "prompt_token_ids": beam.tokens
            }
            if beam.multi_modal_data is not None:
                token_prompt_kwargs["multi_modal_data"] = beam.multi_modal_data

            if beam.mm_processor_kwargs is not None:
                token_prompt_kwargs[
                    "mm_processor_kwargs"] = beam.mm_processor_kwargs
            return TokensPrompt(**token_prompt_kwargs)
558

559
560
561
562
563
564
        tokenizer = self.get_tokenizer()
        # generate 2 * beam_width candidates at each step
        # following the huggingface transformers implementation
        # at https://github.com/huggingface/transformers/blob/e15687fffe5c9d20598a19aeab721ae0a7580f8a/src/transformers/generation/beam_search.py#L534 # noqa
        beam_search_params = SamplingParams(logprobs=2 * beam_width,
                                            max_tokens=1,
565
                                            temperature=temperature)
566
        instances: list[BeamSearchInstance] = []
567
568

        for prompt in prompts:
569
570
571
572
573
574
575
576
            # Add multimodal processor kwargs & data
            mm_kwargs = {}
            if "multi_modal_data" in prompt:
                mm_kwargs["multi_modal_data"] = prompt["multi_modal_data"]
            if "mm_processor_kwargs" in prompt:
                mm_kwargs["mm_processor_kwargs"] = prompt[
                    "mm_processor_kwargs"]

577
578
579
580
            if is_token_prompt(prompt):
                prompt_tokens = prompt["prompt_token_ids"]
            else:
                prompt_tokens = tokenizer.encode(prompt["prompt"])
581
582
            instances.append(
                BeamSearchInstance(prompt_tokens, logprobs=None, **mm_kwargs))
583
584

        for _ in range(max_tokens):
585
            all_beams: list[BeamSearchSequence] = list(
586
587
588
589
                sum((instance.beams for instance in instances), []))
            pos = [0] + list(
                itertools.accumulate(
                    len(instance.beams) for instance in instances))
590
            instance_start_and_end: list[tuple[int, int]] = list(
591
592
593
594
595
596
                zip(pos[:-1], pos[1:]))

            if len(all_beams) == 0:
                break

            prompts_batch = [
597
                create_tokens_prompt_from_beam(beam) for beam in all_beams
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
            ]

            # only runs for one step
            # we don't need to use tqdm here
            output = self.generate(prompts_batch,
                                   sampling_params=beam_search_params,
                                   use_tqdm=False)

            for (start, end), instance in zip(instance_start_and_end,
                                              instances):
                instance_new_beams = []
                for i in range(start, end):
                    current_beam = all_beams[i]
                    result = output[i]

                    if result.outputs[0].logprobs is not None:
                        # if `result.outputs[0].logprobs` is None, it means
                        # the sequence is completed because of the max-model-len
                        # or abortion. we don't need to add it to the new beams.
                        logprobs = result.outputs[0].logprobs[0]
                        for token_id, logprob_obj in logprobs.items():
                            new_beam = BeamSearchSequence(
                                tokens=current_beam.tokens + [token_id],
621
                                logprobs=current_beam.logprobs + [logprobs],
622
                                cum_logprob=current_beam.cum_logprob +
623
624
625
626
                                logprob_obj.logprob,
                                multi_modal_data=current_beam.multi_modal_data,
                                mm_processor_kwargs=current_beam.
                                mm_processor_kwargs)
627
628
629
630
631
632
633

                            if token_id == tokenizer.eos_token_id and \
                                not ignore_eos:
                                instance.completed.append(new_beam)
                            else:
                                instance_new_beams.append(new_beam)
                sorted_beams = sorted(instance_new_beams,
634
                                      key=sort_beams_key,
635
636
637
638
639
640
641
                                      reverse=True)
                instance.beams = sorted_beams[:beam_width]

        outputs = []
        for instance in instances:
            instance.completed.extend(instance.beams)
            sorted_completed = sorted(instance.completed,
642
                                      key=sort_beams_key,
643
644
645
646
647
648
649
650
651
                                      reverse=True)
            best_beams = sorted_completed[:beam_width]

            for beam in best_beams:
                beam.text = tokenizer.decode(beam.tokens)
            outputs.append(BeamSearchOutput(sequences=best_beams))

        return outputs

nunjunj's avatar
nunjunj committed
652
653
    def chat(
        self,
654
655
        messages: Union[list[ChatCompletionMessageParam],
                        list[list[ChatCompletionMessageParam]]],
nunjunj's avatar
nunjunj committed
656
        sampling_params: Optional[Union[SamplingParams,
657
                                        list[SamplingParams]]] = None,
nunjunj's avatar
nunjunj committed
658
659
660
        use_tqdm: bool = True,
        lora_request: Optional[LoRARequest] = None,
        chat_template: Optional[str] = None,
661
        chat_template_content_format: ChatTemplateContentFormatOption = "auto",
662
        add_generation_prompt: bool = True,
663
        continue_final_message: bool = False,
664
665
666
        tools: Optional[list[dict[str, Any]]] = None,
        mm_processor_kwargs: Optional[dict[str, Any]] = None,
    ) -> list[RequestOutput]:
nunjunj's avatar
nunjunj committed
667
        """
668
        Generate responses for a chat conversation.
nunjunj's avatar
nunjunj committed
669

670
671
672
673
674
675
        The chat conversation is converted into a text prompt using the
        tokenizer and calls the :meth:`generate` method to generate the
        responses.

        Multi-modal inputs can be passed in the same way you would pass them
        to the OpenAI API.
nunjunj's avatar
nunjunj committed
676
677

        Args:
678
679
680
681
682
            messages: A list of conversations or a single conversation.

              - Each conversation is represented as a list of messages.
              - Each message is a dictionary with 'role' and 'content' keys.

nunjunj's avatar
nunjunj committed
683
684
685
686
687
688
689
690
691
            sampling_params: The sampling parameters for text generation.
                If None, we use the default sampling parameters. When it
                is a single value, it is applied to every prompt. When it
                is a list, the list must have the same length as the
                prompts and it is paired one by one with the prompt.
            use_tqdm: Whether to use tqdm to display the progress bar.
            lora_request: LoRA request to use for generation, if any.
            chat_template: The template to use for structuring the chat.
              If not provided, the model's default chat template will be used.
692
693
694
695
696
697
698
699
            chat_template_content_format: The format to render message content.

              - "string" will render the content as a string.
                Example: ``"Who are you?"``
              - "openai" will render the content as a list of dictionaries,
                similar to OpenAI schema.
                Example: ``[{"type": "text", "text": "Who are you?"}]``

700
            add_generation_prompt: If True, adds a generation template
nunjunj's avatar
nunjunj committed
701
                to each message.
702
            continue_final_message: If True, continues the final message in
703
704
                the conversation instead of starting a new one. Cannot be
                ``True`` if ``add_generation_prompt`` is also ``True``.
705
706
            mm_processor_kwargs: Multimodal processor kwarg overrides for this
                chat request. Only used for offline requests.
nunjunj's avatar
nunjunj committed
707
708
709
710
711

        Returns:
            A list of ``RequestOutput`` objects containing the generated
            responses in the same order as the input messages.
        """
712
        list_of_messages: list[list[ChatCompletionMessageParam]]
nunjunj's avatar
nunjunj committed
713

714
715
        # Handle multi and single conversations
        if is_list_of(messages, list):
716
717
            # messages is list[list[...]]
            list_of_messages = cast(list[list[ChatCompletionMessageParam]],
718
                                    messages)
719
        else:
720
            # messages is list[...]
721
            list_of_messages = [
722
                cast(list[ChatCompletionMessageParam], messages)
723
            ]
724

725
        tokenizer = self.get_tokenizer(lora_request)
726
727
728
        model_config = self.llm_engine.get_model_config()
        resolved_content_format = resolve_chat_template_content_format(
            chat_template,
729
            tools,
730
731
            chat_template_content_format,
            tokenizer,
732
            trust_remote_code=model_config.trust_remote_code,
733
734
        )

735
        prompts: list[Union[TokensPrompt, TextPrompt]] = []
736
737

        for msgs in list_of_messages:
738
739
740
            # NOTE: _parse_chat_message_content_parts() currently doesn't
            # handle mm_processor_kwargs, since there is no implementation in
            # the chat message parsing for it.
741
            conversation, mm_data = parse_chat_messages(
742
743
744
745
746
                msgs,
                model_config,
                tokenizer,
                content_format=resolved_content_format,
            )
747
748

            if isinstance(tokenizer, MistralTokenizer):
749
                prompt_token_ids = apply_mistral_chat_template(
750
751
752
                    tokenizer,
                    messages=msgs,
                    chat_template=chat_template,
753
                    tools=tools,
754
                    add_generation_prompt=add_generation_prompt,
755
                    continue_final_message=continue_final_message,
756
757
                )
            else:
758
                prompt_str = apply_hf_chat_template(
759
                    tokenizer,
760
                    trust_remote_code=model_config.trust_remote_code,
761
762
                    conversation=conversation,
                    chat_template=chat_template,
763
                    tools=tools,
764
                    add_generation_prompt=add_generation_prompt,
765
                    continue_final_message=continue_final_message,
766
                )
767
768
769
770
                # Special tokens are already included in chat templates so
                # should not be added by the tokenizer in this case.
                prompt_token_ids = tokenizer.encode(prompt_str,
                                                    add_special_tokens=False)
771

772
            prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
773
774
775
776

            if mm_data is not None:
                prompt["multi_modal_data"] = mm_data

777
778
779
            if mm_processor_kwargs is not None:
                prompt["mm_processor_kwargs"] = mm_processor_kwargs

780
            prompts.append(prompt)
781

nunjunj's avatar
nunjunj committed
782
        return self.generate(
783
            prompts,
784
            sampling_params=sampling_params,
nunjunj's avatar
nunjunj committed
785
786
787
788
            use_tqdm=use_tqdm,
            lora_request=lora_request,
        )

789
790
791
792
793
794
795
    @overload
    def encode(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
796
        *,
797
        use_tqdm: bool = True,
798
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
799
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
800
    ) -> list[PoolingRequestOutput]:
801
802
        ...

803
    @overload  # LEGACY: single (prompt + optional token ids)
804
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
805
806
807
808
809
    def encode(
        self,
        prompts: str,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
810
        prompt_token_ids: Optional[list[int]] = None,
811
        use_tqdm: bool = True,
812
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
813
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
814
    ) -> list[PoolingRequestOutput]:
815
        ...
816

817
    @overload  # LEGACY: multi (prompt + optional token ids)
818
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
819
820
    def encode(
        self,
821
        prompts: list[str],
822
        pooling_params: Optional[Union[PoolingParams,
823
                                       Sequence[PoolingParams]]] = None,
824
        prompt_token_ids: Optional[list[list[int]]] = None,
825
        use_tqdm: bool = True,
826
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
827
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
828
    ) -> list[PoolingRequestOutput]:
829
830
831
        ...

    @overload  # LEGACY: single (token ids + optional prompt)
832
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
833
834
835
836
837
838
    def encode(
        self,
        prompts: Optional[str] = None,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
        *,
839
        prompt_token_ids: list[int],
840
        use_tqdm: bool = True,
841
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
842
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
843
    ) -> list[PoolingRequestOutput]:
844
845
846
        ...

    @overload  # LEGACY: multi (token ids + optional prompt)
847
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
848
849
    def encode(
        self,
850
        prompts: Optional[list[str]] = None,
851
852
853
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
        *,
854
        prompt_token_ids: list[list[int]],
855
        use_tqdm: bool = True,
856
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
857
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
858
    ) -> list[PoolingRequestOutput]:
859
860
861
        ...

    @overload  # LEGACY: single or multi token ids [pos-only]
862
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
863
864
865
866
    def encode(
        self,
        prompts: None,
        pooling_params: None,
867
        prompt_token_ids: Union[list[int], list[list[int]]],
868
        use_tqdm: bool = True,
869
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
870
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
871
    ) -> list[PoolingRequestOutput]:
872
873
        ...

nunjunj's avatar
nunjunj committed
874
875
876
    @deprecate_kwargs(
        "prompt_token_ids",
        is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
877
        additional_message="Please use the 'prompts' parameter instead.",
nunjunj's avatar
nunjunj committed
878
    )
879
880
    def encode(
        self,
881
        prompts: Union[Union[PromptType, Sequence[PromptType]],
882
                       Optional[Union[str, list[str]]]] = None,
883
884
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
885
        prompt_token_ids: Optional[Union[list[int], list[list[int]]]] = None,
886
        use_tqdm: bool = True,
887
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
888
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
889
    ) -> list[PoolingRequestOutput]:
890
891
        """Apply pooling to the hidden states corresponding to the input
        prompts.
892

893
        This class automatically batches the given prompts, considering
894
895
896
897
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
898
899
900
            prompts: The prompts to the LLM. You may pass a sequence of prompts
                for batch inference. See :class:`~vllm.inputs.PromptType`
                for more details about the format of each prompts.
901
902
903
904
            pooling_params: The pooling parameters for pooling. If None, we
                use the default pooling parameters.
            use_tqdm: Whether to use tqdm to display the progress bar.
            lora_request: LoRA request to use for generation, if any.
nunjunj's avatar
nunjunj committed
905
            prompt_adapter_request: Prompt Adapter request to use for
906
                generation, if any.
907
908

        Returns:
909
            A list of ``PoolingRequestOutput`` objects containing the
910
            pooled hidden states in the same order as the input prompts.
911
912
913
914
915

        Note:
            Using ``prompts`` and ``prompt_token_ids`` as keyword parameters is
            considered legacy and may be deprecated in the future. You should
            instead pass them via the ``inputs`` parameter.
916
        """
917
918
919
        runner_type = self.llm_engine.model_config.runner_type
        if runner_type != "pooling":
            messages = ["LLM.encode() is only supported for pooling models."]
920

921
922
923
            supported_runner_types = self.llm_engine.model_config \
                .supported_runner_types
            if "pooling" in supported_runner_types:
924
                messages.append(
925
926
927
928
                    "Your model supports the 'pooling' runner, but is "
                    f"currently initialized for the '{runner_type}' runner. "
                    "Please initialize vLLM using `--task embed`, "
                    "`--task classify`, `--task score` etc.")
929
930

            raise ValueError(" ".join(messages))
931

932
        if prompt_token_ids is not None:
933
            parsed_prompts = self._convert_v1_inputs(
934
                prompts=cast(Optional[Union[str, list[str]]], prompts),
935
936
937
                prompt_token_ids=prompt_token_ids,
            )
        else:
938
939
            parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
                                  prompts)
940

941
942
943
        if pooling_params is None:
            # Use default pooling params.
            pooling_params = PoolingParams()
944
945
946
947
948
        elif isinstance(pooling_params, PoolingParams):
            pooling_params.verify(self.llm_engine.model_config)
        else:
            for pooling_param in pooling_params:
                pooling_param.verify(self.llm_engine.model_config)
949

950
        self._validate_and_add_requests(
951
            prompts=parsed_prompts,
952
953
            params=pooling_params,
            lora_request=lora_request,
954
            prompt_adapter_request=prompt_adapter_request,
955
956
        )

957
        outputs = self._run_engine(use_tqdm=use_tqdm)
Joe Runde's avatar
Joe Runde committed
958
        return self.engine_class.validate_outputs(outputs,
959
                                                  PoolingRequestOutput)
960

961
962
963
964
965
966
    def embed(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        *,
        use_tqdm: bool = True,
967
968
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
969
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
970
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
971
    ) -> list[EmbeddingRequestOutput]:
972
973
974
975
976
977
978
979
980
981
982
        """
        Generate an embedding vector for each prompt.

        This class automatically batches the given prompts, considering
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
            prompts: The prompts to the LLM. You may pass a sequence of prompts
                for batch inference. See :class:`~vllm.inputs.PromptType`
                for more details about the format of each prompts.
983
984
            pooling_params: The pooling parameters for pooling. If None, we
                use the default pooling parameters.
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
            use_tqdm: Whether to use tqdm to display the progress bar.
            lora_request: LoRA request to use for generation, if any.
            prompt_adapter_request: Prompt Adapter request to use for
                generation, if any.

        Returns:
            A list of ``EmbeddingRequestOutput`` objects containing the
            embedding vectors in the same order as the input prompts.
        """
        if self.llm_engine.model_config.task != "embed":
            raise ValueError(
                "Embedding API is only enabled for `--task embed`")

        items = self.encode(prompts,
                            use_tqdm=use_tqdm,
1000
                            pooling_params=pooling_params,
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
                            lora_request=lora_request,
                            prompt_adapter_request=prompt_adapter_request)

        return [EmbeddingRequestOutput.from_base(item) for item in items]

    def classify(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        *,
        use_tqdm: bool = True,
1012
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
1013
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
1014
    ) -> list[ClassificationRequestOutput]:
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
        """
        Generate class logits for each prompt.

        This class automatically batches the given prompts, considering
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
            prompts: The prompts to the LLM. You may pass a sequence of prompts
                for batch inference. See :class:`~vllm.inputs.PromptType`
                for more details about the format of each prompts.
            use_tqdm: Whether to use tqdm to display the progress bar.
            lora_request: LoRA request to use for generation, if any.
            prompt_adapter_request: Prompt Adapter request to use for
                generation, if any.

        Returns:
            A list of ``ClassificationRequestOutput`` objects containing the
            embedding vectors in the same order as the input prompts.
        """
        if self.llm_engine.model_config.task != "classify":
            raise ValueError(
                "Classification API is only enabled for `--task classify`")

        items = self.encode(prompts,
                            use_tqdm=use_tqdm,
                            lora_request=lora_request,
                            prompt_adapter_request=prompt_adapter_request)

        return [ClassificationRequestOutput.from_base(item) for item in items]

1046
1047
1048
    def _embedding_score(
        self,
        tokenizer: AnyTokenizer,
1049
1050
        text_1: list[Union[str, TextPrompt, TokensPrompt]],
        text_2: list[Union[str, TextPrompt, TokensPrompt]],
1051
1052
        truncate_prompt_tokens: Optional[int] = None,
        use_tqdm: bool = True,
1053
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
1054
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
1055
    ) -> list[ScoringRequestOutput]:
1056

1057
        encoded_output: list[PoolingRequestOutput] = self.encode(
1058
1059
1060
1061
            text_1 + text_2,
            use_tqdm=use_tqdm,
            lora_request=lora_request,
            prompt_adapter_request=prompt_adapter_request)
1062

1063
        encoded_output_1: list[PoolingRequestOutput] = encoded_output[
1064
            0:len(text_1)]
1065
        encoded_output_2: list[PoolingRequestOutput] = encoded_output[
1066
            len(text_1):]
1067
1068
1069
1070

        if len(encoded_output_1) == 1:
            encoded_output_1 = encoded_output_1 * len(encoded_output_2)

1071
1072
1073
        scores = _cosine_similarity(tokenizer=tokenizer,
                                    embed_1=encoded_output_1,
                                    embed_2=encoded_output_2)
1074
1075
1076
1077
1078
1079
1080

        items = self.engine_class.validate_outputs(scores,
                                                   PoolingRequestOutput)
        return [ScoringRequestOutput.from_base(item) for item in items]

    def _cross_encoding_score(
        self,
1081
        tokenizer: AnyTokenizer,
1082
1083
        text_1: list[str],
        text_2: list[str],
1084
1085
        truncate_prompt_tokens: Optional[int] = None,
        use_tqdm: bool = True,
1086
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
1087
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
1088
    ) -> list[ScoringRequestOutput]:
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100

        if isinstance(tokenizer, MistralTokenizer):
            raise ValueError(
                "Score API is only enabled for `--task embed or score`")

        if len(text_1) == 1:
            text_1 = text_1 * len(text_2)

        input_pairs = [(t1, t2) for t1, t2 in zip(text_1, text_2)]

        pooling_params = PoolingParams()

1101
        tokenization_kwargs: dict[str, Any] = {}
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
        if truncate_prompt_tokens is not None:
            tokenization_kwargs["truncation"] = True
            tokenization_kwargs["max_length"] = truncate_prompt_tokens

        parsed_prompts = []

        for q, t in input_pairs:
            prompt_inputs = tokenizer(text=q,
                                      text_pair=t,
                                      **tokenization_kwargs)
            engine_prompt = TokensPrompt(
                prompt_token_ids=prompt_inputs["input_ids"],
                token_type_ids=prompt_inputs.get("token_type_ids"))
            parsed_prompts.append(engine_prompt)

        self._validate_and_add_requests(
            prompts=parsed_prompts,
            params=pooling_params,
            lora_request=lora_request,
            prompt_adapter_request=prompt_adapter_request,
        )

        outputs = self._run_engine(use_tqdm=use_tqdm)
        items = self.engine_class.validate_outputs(outputs,
                                                   PoolingRequestOutput)

        return [ScoringRequestOutput.from_base(item) for item in items]

1130
1131
1132
1133
1134
    def score(
        self,
        text_1: Union[SingletonPrompt, Sequence[SingletonPrompt]],
        text_2: Union[SingletonPrompt, Sequence[SingletonPrompt]],
        /,
1135
        *,
1136
1137
        truncate_prompt_tokens: Optional[int] = None,
        use_tqdm: bool = True,
1138
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
1139
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
1140
    ) -> list[ScoringRequestOutput]:
1141
        """Generate similarity scores for all pairs ``<text,text_pair>``.
1142

1143
1144
1145
1146
        The inputs can be ``1 -> 1``, ``1 -> N`` or ``N -> N``.
        In the ``1 - N`` case the ``text_1`` sentence will be replicated ``N``
        times to pair with the ``text_2`` sentences.
        The input pairs are used to build a list of prompts for the
1147
1148
1149
1150
1151
1152
        cross encoder model. This class automatically batches the prompts,
        considering the memory constraint. For the best performance, put all
        of your texts into a single list and pass it to this method.

        Args:
            text_1: can be a single prompt or a list of prompts, in which
1153
                case it has to have the same length as the ``text_2`` list
1154
1155
1156
1157
1158
1159
1160
1161
1162
            text_2: The texts to pair with the query to form the input
                to the LLM. See :class:`~vllm.inputs.PromptType` for
                more details about the format of each prompts.
            use_tqdm: Whether to use tqdm to display the progress bar.
            lora_request: LoRA request to use for generation, if any.
            prompt_adapter_request: Prompt Adapter request to use for
                generation, if any.

        Returns:
1163
            A list of ``ScoringRequestOutput`` objects containing the
1164
1165
            generated scores in the same order as the input prompts.
        """
1166
1167
1168
        runner_type = self.llm_engine.model_config.runner_type
        if runner_type != "pooling":
            messages = ["LLM.score() is only supported for pooling models."]
1169

1170
1171
1172
            supported_runner_types = self.llm_engine.model_config \
                .supported_runner_types
            if "pooling" in supported_runner_types:
1173
                messages.append(
1174
1175
1176
1177
                    "Your model supports the 'pooling' runner, but is "
                    f"currently initialized for the '{runner_type}' runner. "
                    "Please initialize vLLM using `--task embed`, "
                    "`--task classify`, `--task score` etc.")
1178
1179
1180

            raise ValueError(" ".join(messages))

1181
        if self.llm_engine.model_config.task not in ("embed", "score"):
1182
            raise ValueError(
1183
                "Score API is only enabled for `--task embed or --task score`")
1184
1185
1186
1187

        # the tokenizer for models such as
        # "cross-encoder/ms-marco-MiniLM-L-6-v2" doesn't support passing
        # lists of tokens to the `text` and `text_pair` kwargs
1188
1189
        tokenizer = self.llm_engine.get_tokenizer()

1190
1191
1192
1193
        def ensure_str(prompt: SingletonPrompt):
            if isinstance(prompt, dict):
                if "multi_modal_data" in prompt:
                    raise ValueError("Multi-modal prompt is not "
1194
                                     "supported for scoring")
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
                elif "prompt_token_ids" in prompt:
                    prompt = tokenizer.decode(
                        cast(TokensPrompt, prompt)["prompt_token_ids"])
                elif "prompt" in prompt:
                    prompt = cast(TextPrompt, prompt)["prompt"]
            assert type(prompt) is str
            return prompt

        if isinstance(text_1, (str, dict)):
            # Convert a single prompt to a list.
            text_1 = [text_1]
1206
        input_text_1: list[str] = [ensure_str(t) for t in text_1]
1207
1208
1209
1210

        if isinstance(text_2, (str, dict)):
            # Convert a single prompt to a list.
            text_2 = [text_2]
1211
        input_text_2: list[str] = [ensure_str(t) for t in text_2]
1212

1213
        _validate_score_input_lens(input_text_1, input_text_2)
1214

1215
        if self.llm_engine.model_config.is_cross_encoder:
1216
1217
            return self._cross_encoding_score(tokenizer, input_text_1,
                                              input_text_2,
1218
1219
1220
1221
                                              truncate_prompt_tokens, use_tqdm,
                                              lora_request,
                                              prompt_adapter_request)
        else:
1222
1223
1224
1225
1226
1227
1228
1229
            return self._embedding_score(
                tokenizer,
                input_text_1,  # type: ignore[arg-type]
                input_text_2,  # type: ignore[arg-type]
                truncate_prompt_tokens,
                use_tqdm,
                lora_request,
                prompt_adapter_request)
1230

1231
1232
1233
1234
1235
1236
    def start_profile(self) -> None:
        self.llm_engine.start_profile()

    def stop_profile(self) -> None:
        self.llm_engine.stop_profile()

1237
1238
    def reset_prefix_cache(self, device: Optional[Device] = None) -> bool:
        return self.llm_engine.reset_prefix_cache(device)
1239

1240
1241
1242
1243
1244
1245
    def sleep(self, level: int = 1):
        """
        Put the engine to sleep. The engine should not process any requests.
        The caller should guarantee that no requests are being processed
        during the sleep period, before `wake_up` is called.

1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
        Args:
            level: The sleep level. Level 1 sleep will offload the model 
                weights and discard the kv cache. The content of kv cache 
                is forgotten. Level 1 sleep is good for sleeping and waking
                up the engine to run the same model again. The model weights 
                are backed up in CPU memory. Please make sure there's enough 
                CPU memory to store the model weights. Level 2 sleep will 
                discard both the model weights and the kv cache. The content 
                of both the model weights and kv cache is forgotten. Level 2 
                sleep is good for sleeping and waking up the engine to run a
                different model or update the model, where previous model 
                weights are not needed. It reduces CPU memory pressure.
1258
        """
1259
        self.reset_prefix_cache()
1260
1261
        self.llm_engine.sleep(level=level)

1262
    def wake_up(self, tags: Optional[list[str]] = None):
1263
1264
        """
        Wake up the engine from sleep mode. See the :meth:`sleep` method
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
        for more details.
        
        Args:
            tags: An optional list of tags to reallocate the engine memory 
                for specific memory allocations. Values must be in 
                ("weights", "kv_cache",). If None, all memory is reallocated.
                wake_up should be called with all tags (or None) before the 
                engine is used again.
        """
        self.llm_engine.wake_up(tags)
1275

1276
1277
    # LEGACY
    def _convert_v1_inputs(
1278
        self,
1279
1280
        prompts: Optional[Union[str, list[str]]],
        prompt_token_ids: Optional[Union[list[int], list[list[int]]]],
1281
1282
    ):
        # skip_tokenizer_init is now checked in engine
1283

1284
1285
1286
1287
1288
1289
        if prompts is not None:
            prompts = [p["content"] for p in parse_and_batch_prompt(prompts)]
        if prompt_token_ids is not None:
            prompt_token_ids = [
                p["content"] for p in parse_and_batch_prompt(prompt_token_ids)
            ]
1290

1291
        num_requests = None
1292
1293
        if prompts is not None:
            num_requests = len(prompts)
1294
1295
1296
1297
1298
1299
        if prompt_token_ids is not None:
            if (num_requests is not None
                    and num_requests != len(prompt_token_ids)):
                raise ValueError("The lengths of prompts and prompt_token_ids "
                                 "must be the same.")

1300
            num_requests = len(prompt_token_ids)
1301
1302
1303
1304
        if num_requests is None:
            raise ValueError("Either prompts or prompt_token_ids must be "
                             "provided.")

1305
        parsed_prompts: list[PromptType] = []
1306
        for i in range(num_requests):
1307
            item: PromptType
1308

1309
            if prompts is not None:
1310
1311
1312
                item = TextPrompt(prompt=prompts[i])
            elif prompt_token_ids is not None:
                item = TokensPrompt(prompt_token_ids=prompt_token_ids[i])
1313
            else:
1314
                raise AssertionError
1315

1316
            parsed_prompts.append(item)
1317

1318
        return parsed_prompts
1319
1320
1321

    def _validate_and_add_requests(
        self,
1322
        prompts: Union[PromptType, Sequence[PromptType]],
1323
1324
        params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
                      Sequence[PoolingParams]],
1325
        lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
1326
        prompt_adapter_request: Optional[PromptAdapterRequest],
1327
        guided_options: Optional[GuidedDecodingRequest] = None,
1328
        priority: Optional[list[int]] = None,
1329
    ) -> None:
1330
1331
1332
1333
1334
1335
1336
1337
        if guided_options is not None:
            warnings.warn(
                "guided_options_request is deprecated, use "
                "SamplingParams.guided_decoding instead",
                DeprecationWarning,
                stacklevel=2,
            )

1338
        if isinstance(prompts, (str, dict)):
1339
            # Convert a single prompt to a list.
1340
            prompts = [prompts]
1341

1342
        num_requests = len(prompts)
1343
1344
        if isinstance(params, list) and len(params) != num_requests:
            raise ValueError("The lengths of prompts and params "
1345
                             "must be the same.")
1346
1347
1348
1349
        if isinstance(lora_request,
                      list) and len(lora_request) != num_requests:
            raise ValueError("The lengths of prompts and lora_request "
                             "must be the same.")
1350

1351
1352
        for sp in params if isinstance(params, list) else (params, ):
            if isinstance(sp, SamplingParams):
1353
                self._add_guided_params(sp, guided_options)
1354
1355
1356

                # We only care about the final output
                sp.output_kind = RequestOutputKind.FINAL_ONLY
1357

Zhuohan Li's avatar
Zhuohan Li committed
1358
        # Add requests to the engine.
1359
        for i, prompt in enumerate(prompts):
1360
            self._add_request(
1361
                prompt,
1362
                params[i] if isinstance(params, Sequence) else params,
1363
1364
                lora_request=lora_request[i] if isinstance(
                    lora_request, Sequence) else lora_request,
nunjunj's avatar
nunjunj committed
1365
                prompt_adapter_request=prompt_adapter_request,
1366
                priority=priority[i] if priority else 0,
nunjunj's avatar
nunjunj committed
1367
            )
1368

1369
    def _add_request(
nunjunj's avatar
nunjunj committed
1370
        self,
1371
        prompt: PromptType,
nunjunj's avatar
nunjunj committed
1372
        params: Union[SamplingParams, PoolingParams],
1373
        lora_request: Optional[LoRARequest] = None,
nunjunj's avatar
nunjunj committed
1374
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
1375
        priority: int = 0,
1376
1377
    ) -> None:
        request_id = str(next(self.request_counter))
1378
1379
        self.llm_engine.add_request(
            request_id,
1380
            prompt,
1381
1382
            params,
            lora_request=lora_request,
nunjunj's avatar
nunjunj committed
1383
            prompt_adapter_request=prompt_adapter_request,
1384
            priority=priority,
nunjunj's avatar
nunjunj committed
1385
        )
1386

1387
    def _add_guided_params(
1388
1389
1390
            self,
            params: SamplingParams,
            guided_options: Optional[GuidedDecodingRequest] = None):
1391
1392
1393
1394
        if guided_options is None:
            return params

        if params.guided_decoding is not None:
1395
            raise ValueError("Cannot set both guided_options_request and "
1396
1397
1398
1399
1400
1401
1402
1403
1404
                             "params.guided_decoding.")

        params.guided_decoding = GuidedDecodingParams(
            json=guided_options.guided_json,
            regex=guided_options.guided_regex,
            choice=guided_options.guided_choice,
            grammar=guided_options.guided_grammar,
            json_object=guided_options.guided_json_object,
            backend=guided_options.guided_decoding_backend,
1405
1406
1407
            whitespace_pattern=guided_options.guided_whitespace_pattern,
            structural_tag=guided_options.structural_tag,
        )
1408
1409
        return params

1410
    def _run_engine(
1411
            self, *, use_tqdm: bool
1412
    ) -> list[Union[RequestOutput, PoolingRequestOutput]]:
1413
1414
        # Initialize tqdm.
        if use_tqdm:
Zhuohan Li's avatar
Zhuohan Li committed
1415
            num_requests = self.llm_engine.get_num_unfinished_requests()
1416
1417
1418
1419
            pbar = tqdm(
                total=num_requests,
                desc="Processed prompts",
                dynamic_ncols=True,
1420
1421
                postfix=(f"est. speed input: {0:.2f} toks/s, "
                         f"output: {0:.2f} toks/s"),
1422
            )
1423

Zhuohan Li's avatar
Zhuohan Li committed
1424
        # Run the engine.
1425
        outputs: list[Union[RequestOutput, PoolingRequestOutput]] = []
1426
1427
        total_in_toks = 0
        total_out_toks = 0
Zhuohan Li's avatar
Zhuohan Li committed
1428
1429
        while self.llm_engine.has_unfinished_requests():
            step_outputs = self.llm_engine.step()
1430
            for output in step_outputs:
1431
                if output.finished:
1432
1433
                    outputs.append(output)
                    if use_tqdm:
1434
1435
                        if isinstance(output, RequestOutput):
                            # Calculate tokens only for RequestOutput
1436
                            n = len(output.outputs)
1437
                            assert output.prompt_token_ids is not None
1438
                            total_in_toks += len(output.prompt_token_ids) * n
1439
1440
                            in_spd = total_in_toks / pbar.format_dict["elapsed"]
                            total_out_toks += sum(
1441
                                len(stp.token_ids) for stp in output.outputs)
nunjunj's avatar
nunjunj committed
1442
1443
                            out_spd = (total_out_toks /
                                       pbar.format_dict["elapsed"])
1444
1445
1446
                            pbar.postfix = (
                                f"est. speed input: {in_spd:.2f} toks/s, "
                                f"output: {out_spd:.2f} toks/s")
1447
                            pbar.update(n)
1448
1449
                        else:
                            pbar.update(1)
1450

1451
1452
        if use_tqdm:
            pbar.close()
lizhigong's avatar
lizhigong committed
1453

1454
1455
1456
        # Sort the outputs by request ID.
        # This is necessary because some requests may be finished earlier than
        # its previous requests.
1457
        return sorted(outputs, key=lambda x: int(x.request_id))