llm.py 59.8 KB
Newer Older
1
2
# SPDX-License-Identifier: Apache-2.0

3
import itertools
4
import warnings
5
from collections.abc import Sequence
6
from contextlib import contextmanager
7
from typing import Any, Callable, ClassVar, Optional, Union, cast, overload
8

9
import cloudpickle
10
import torch.nn as nn
11
from tqdm import tqdm
12
from typing_extensions import TypeVar, deprecated
13

14
from vllm import envs
15
16
from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
                              BeamSearchSequence, get_beam_search_score)
17
from vllm.config import CompilationConfig
18
19
from vllm.engine.arg_utils import (EngineArgs, HfOverrides, PoolerConfig,
                                   TaskOption)
Joe Runde's avatar
Joe Runde committed
20
from vllm.engine.llm_engine import LLMEngine
nunjunj's avatar
nunjunj committed
21
from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
22
                                         ChatTemplateContentFormatOption,
23
24
                                         apply_hf_chat_template,
                                         apply_mistral_chat_template,
25
26
                                         parse_chat_messages,
                                         resolve_chat_template_content_format)
27
28
from vllm.entrypoints.score_utils import (_cosine_similarity,
                                          _validate_score_input_lens)
29
from vllm.inputs import PromptType, SingletonPrompt, TextPrompt, TokensPrompt
30
from vllm.inputs.parse import is_token_prompt, parse_and_batch_prompt
31
from vllm.logger import init_logger
32
from vllm.lora.request import LoRARequest
33
34
from vllm.model_executor.guided_decoding.guided_fields import (
    GuidedDecodingRequest, LLMGuidedOptions)
35
36
37
from vllm.outputs import (ClassificationRequestOutput, EmbeddingRequestOutput,
                          PoolingRequestOutput, RequestOutput,
                          ScoringRequestOutput)
38
from vllm.pooling_params import PoolingParams
39
from vllm.prompt_adapter.request import PromptAdapterRequest
40
41
from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
                                  RequestOutputKind, SamplingParams)
42
from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
43
44
                                               get_cached_tokenizer)
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
yhu422's avatar
yhu422 committed
45
from vllm.usage.usage_lib import UsageContext
46
from vllm.utils import Counter, deprecate_args, deprecate_kwargs, is_list_of
47

48
49
logger = init_logger(__name__)

50
51
_R = TypeVar("_R", default=Any)

52
53

class LLM:
Woosuk Kwon's avatar
Woosuk Kwon committed
54
55
56
57
58
59
60
61
62
63
    """An LLM for generating texts from given prompts and sampling parameters.

    This class includes a tokenizer, a language model (possibly distributed
    across multiple GPUs), and GPU memory space allocated for intermediate
    states (aka KV cache). Given a batch of prompts and sampling parameters,
    this class generates texts from the model, using an intelligent batching
    mechanism and efficient memory management.

    Args:
        model: The name or path of a HuggingFace Transformers model.
64
        tokenizer: The name or path of a HuggingFace Transformers tokenizer.
65
66
        tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer
            if available, and "slow" will always use the slow tokenizer.
67
68
69
        skip_tokenizer_init: If true, skip initialization of tokenizer and
            detokenizer. Expect valid prompt_token_ids and None for prompt
            from the input.
70
71
        trust_remote_code: Trust remote code (e.g., from HuggingFace) when
            downloading the model and tokenizer.
72
73
74
75
        allowed_local_media_path: Allowing API requests to read local images
            or videos from directories specified by the server file system.
            This is a security risk. Should only be enabled in trusted
            environments.
Woosuk Kwon's avatar
Woosuk Kwon committed
76
77
78
        tensor_parallel_size: The number of GPUs to use for distributed
            execution with tensor parallelism.
        dtype: The data type for the model weights and activations. Currently,
Woosuk Kwon's avatar
Woosuk Kwon committed
79
80
81
82
            we support `float32`, `float16`, and `bfloat16`. If `auto`, we use
            the `torch_dtype` attribute specified in the model config file.
            However, if the `torch_dtype` in the config is `float32`, we will
            use `float16` instead.
83
        quantization: The method used to quantize the model weights. Currently,
84
            we support "awq", "gptq", and "fp8" (experimental).
85
86
87
88
            If None, we first check the `quantization_config` attribute in the
            model config file. If that is None, we assume the model weights are
            not quantized and use `dtype` to determine the data type of
            the weights.
Jasmond L's avatar
Jasmond L committed
89
90
        revision: The specific model version to use. It can be a branch name,
            a tag name, or a commit id.
91
92
        tokenizer_revision: The specific tokenizer version to use. It can be a
            branch name, a tag name, or a commit id.
93
94
95
96
97
98
99
        seed: The seed to initialize the random number generator for sampling.
        gpu_memory_utilization: The ratio (between 0 and 1) of GPU memory to
            reserve for the model weights, activations, and KV cache. Higher
            values will increase the KV cache size and thus improve the model's
            throughput. However, if the value is too high, it may cause out-of-
            memory (OOM) errors.
        swap_space: The size (GiB) of CPU memory per GPU to use as swap space.
100
           Too small values may cause out-of-memory (OOM) errors.
101
102
103
104
        cpu_offload_gb: The size (GiB) of CPU memory to use for offloading
            the model weights. This virtually increases the GPU memory space
            you can use to hold the model weights, at the cost of CPU-GPU data
            transfer for every forward pass.
105
106
107
        enforce_eager: Whether to enforce eager execution. If True, we will
            disable CUDA graph and always execute the model in eager mode.
            If False, we will use CUDA graph and eager execution in hybrid.
108
        max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs.
109
            When a sequence has context length larger than this, we fall back
110
111
112
            to eager mode. Additionally for encoder-decoder models, if the
            sequence length of the encoder input is larger than this, we fall
            back to the eager mode.
113
114
115
        disable_custom_all_reduce: See :class:`~vllm.config.ParallelConfig`
        disable_async_output_proc: Disable async output processing.
            This may result in lower performance.
116
117
118
        hf_overrides: If a dictionary, contains arguments to be forwarded to the
            HuggingFace config. If a callable, it is called to update the
            HuggingFace config.
119
120
121
        compilation_config: Either an integer or a dictionary. If it is an
            integer, it is used as the level of compilation optimization. If it
            is a dictionary, it can specify the full compilation configuration.
122
        **kwargs: Arguments for :class:`~vllm.EngineArgs`. (See
123
            :ref:`engine-args`)
nunjunj's avatar
nunjunj committed
124

125
126
127
    Note:
        This class is intended to be used for offline inference. For online
        serving, use the :class:`~vllm.AsyncLLMEngine` class instead.
128
    """
129

130
    DEPRECATE_LEGACY: ClassVar[bool] = True
131
132
    """A flag to toggle whether to deprecate the legacy generate/encode API."""

133
134
135
136
137
138
    DEPRECATE_INIT_POSARGS: ClassVar[bool] = True
    """
    A flag to toggle whether to deprecate positional arguments in
    :meth:`LLM.__init__`.
    """

139
140
141
142
143
144
145
146
147
    @classmethod
    @contextmanager
    def deprecate_legacy_api(cls):
        cls.DEPRECATE_LEGACY = True

        yield

        cls.DEPRECATE_LEGACY = False

148
149
150
151
152
153
154
    @deprecate_args(
        start_index=2,  # Ignore self and model
        is_deprecated=lambda: LLM.DEPRECATE_INIT_POSARGS,
        additional_message=(
            "All positional arguments other than `model` will be "
            "replaced with keyword arguments in an upcoming version."),
    )
155
156
157
    def __init__(
        self,
        model: str,
158
        tokenizer: Optional[str] = None,
159
        tokenizer_mode: str = "auto",
160
        skip_tokenizer_init: bool = False,
161
        trust_remote_code: bool = False,
162
        allowed_local_media_path: str = "",
163
        tensor_parallel_size: int = 1,
Woosuk Kwon's avatar
Woosuk Kwon committed
164
        dtype: str = "auto",
165
        quantization: Optional[str] = None,
166
        revision: Optional[str] = None,
167
        tokenizer_revision: Optional[str] = None,
168
169
        seed: int = 0,
        gpu_memory_utilization: float = 0.9,
170
        swap_space: float = 4,
171
        cpu_offload_gb: float = 0,
172
        enforce_eager: Optional[bool] = None,
173
        max_seq_len_to_capture: int = 8192,
174
        disable_custom_all_reduce: bool = False,
175
        disable_async_output_proc: bool = False,
176
        hf_overrides: Optional[HfOverrides] = None,
177
        mm_processor_kwargs: Optional[dict[str, Any]] = None,
178
179
        # After positional args are removed, move this right below `model`
        task: TaskOption = "auto",
180
        override_pooler_config: Optional[PoolerConfig] = None,
181
        compilation_config: Optional[Union[int, dict[str, Any]]] = None,
182
183
        **kwargs,
    ) -> None:
184
185
186
187
        '''
        LLM constructor.

        Note: if enforce_eager is unset (enforce_eager is None)
188
        it defaults to False.
189
190
        '''

191
192
        if "disable_log_stats" not in kwargs:
            kwargs["disable_log_stats"] = True
193

194
195
196
197
198
199
200
        if "worker_cls" in kwargs:
            worker_cls = kwargs["worker_cls"]
            # if the worker_cls is not qualified string name,
            # we serialize it using cloudpickle to avoid pickling issues
            if isinstance(worker_cls, type):
                kwargs["worker_cls"] = cloudpickle.dumps(worker_cls)

201
        if compilation_config is not None:
202
            if isinstance(compilation_config, (int, dict)):
203
204
205
206
                compilation_config_instance = CompilationConfig.from_cli(
                    str(compilation_config))
            else:
                compilation_config_instance = compilation_config
207
208
209
        else:
            compilation_config_instance = None

Zhuohan Li's avatar
Zhuohan Li committed
210
        engine_args = EngineArgs(
211
            model=model,
212
            task=task,
213
            tokenizer=tokenizer,
214
            tokenizer_mode=tokenizer_mode,
215
            skip_tokenizer_init=skip_tokenizer_init,
216
            trust_remote_code=trust_remote_code,
217
            allowed_local_media_path=allowed_local_media_path,
218
219
            tensor_parallel_size=tensor_parallel_size,
            dtype=dtype,
220
            quantization=quantization,
221
            revision=revision,
222
            tokenizer_revision=tokenizer_revision,
223
224
225
            seed=seed,
            gpu_memory_utilization=gpu_memory_utilization,
            swap_space=swap_space,
226
            cpu_offload_gb=cpu_offload_gb,
227
            enforce_eager=enforce_eager,
228
            max_seq_len_to_capture=max_seq_len_to_capture,
229
            disable_custom_all_reduce=disable_custom_all_reduce,
230
            disable_async_output_proc=disable_async_output_proc,
231
            hf_overrides=hf_overrides,
232
            mm_processor_kwargs=mm_processor_kwargs,
233
            override_pooler_config=override_pooler_config,
234
            compilation_config=compilation_config_instance,
235
236
            **kwargs,
        )
Joe Runde's avatar
Joe Runde committed
237
238
239
240
        # Logic to switch between engines is done at runtime instead of import
        # to avoid import order issues
        self.engine_class = self.get_engine_class()
        self.llm_engine = self.engine_class.from_engine_args(
yhu422's avatar
yhu422 committed
241
            engine_args, usage_context=UsageContext.LLM_CLASS)
242

243
        self.request_counter = Counter()
244
        self.default_sampling_params: Union[dict[str, Any], None] = None
245

Joe Runde's avatar
Joe Runde committed
246
    @staticmethod
247
    def get_engine_class() -> type[LLMEngine]:
Joe Runde's avatar
Joe Runde committed
248
249
250
251
252
253
        if envs.VLLM_USE_V1:
            # Lazy import: the v1 package isn't distributed
            from vllm.v1.engine.llm_engine import LLMEngine as V1LLMEngine
            return V1LLMEngine  # type: ignore
        return LLMEngine

254
255
256
257
258
    def get_tokenizer(self) -> AnyTokenizer:
        return self.llm_engine.get_tokenizer_group(TokenizerGroup).tokenizer

    def set_tokenizer(self, tokenizer: AnyTokenizer) -> None:
        tokenizer_group = self.llm_engine.get_tokenizer_group(TokenizerGroup)
259

260
261
262
263
        # While CachedTokenizer is dynamic, have no choice but
        # compare class name. Misjudgment will arise from
        # user-defined tokenizer started with 'Cached'
        if tokenizer.__class__.__name__.startswith("Cached"):
264
            tokenizer_group.tokenizer = tokenizer
265
        else:
266
            tokenizer_group.tokenizer = get_cached_tokenizer(tokenizer)
267

268
    def get_default_sampling_params(self) -> SamplingParams:
269
270
271
272
273
        if self.default_sampling_params is None:
            self.default_sampling_params = (
                self.llm_engine.model_config.get_diff_sampling_param())
        if self.default_sampling_params:
            return SamplingParams.from_optional(**self.default_sampling_params)
274
275
        return SamplingParams()

276
277
278
279
280
281
282
    @overload
    def generate(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        sampling_params: Optional[Union[SamplingParams,
                                        Sequence[SamplingParams]]] = None,
283
        *,
284
        use_tqdm: bool = True,
285
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
286
287
288
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
289
    ) -> list[RequestOutput]:
290
291
        ...

292
    @overload  # LEGACY: single (prompt + optional token ids)
293
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
294
295
296
297
    def generate(
        self,
        prompts: str,
        sampling_params: Optional[Union[SamplingParams,
298
299
                                        list[SamplingParams]]] = None,
        prompt_token_ids: Optional[list[int]] = None,
300
        use_tqdm: bool = True,
301
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
302
303
304
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
305
    ) -> list[RequestOutput]:
306
307
308
        ...

    @overload  # LEGACY: multi (prompt + optional token ids)
309
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
310
311
    def generate(
        self,
312
        prompts: list[str],
313
        sampling_params: Optional[Union[SamplingParams,
314
315
                                        list[SamplingParams]]] = None,
        prompt_token_ids: Optional[list[list[int]]] = None,
316
        use_tqdm: bool = True,
317
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
318
319
320
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
321
    ) -> list[RequestOutput]:
322
323
324
        ...

    @overload  # LEGACY: single (token ids + optional prompt)
325
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
326
327
328
329
    def generate(
        self,
        prompts: Optional[str] = None,
        sampling_params: Optional[Union[SamplingParams,
330
                                        list[SamplingParams]]] = None,
331
        *,
332
        prompt_token_ids: list[int],
333
        use_tqdm: bool = True,
334
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
335
336
337
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
338
    ) -> list[RequestOutput]:
339
340
341
        ...

    @overload  # LEGACY: multi (token ids + optional prompt)
342
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
343
344
    def generate(
        self,
345
        prompts: Optional[list[str]] = None,
346
        sampling_params: Optional[Union[SamplingParams,
347
                                        list[SamplingParams]]] = None,
348
        *,
349
        prompt_token_ids: list[list[int]],
350
        use_tqdm: bool = True,
351
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
352
353
354
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
355
    ) -> list[RequestOutput]:
356
357
358
        ...

    @overload  # LEGACY: single or multi token ids [pos-only]
359
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
360
361
362
363
    def generate(
        self,
        prompts: None,
        sampling_params: None,
364
        prompt_token_ids: Union[list[int], list[list[int]]],
365
        use_tqdm: bool = True,
366
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
367
368
369
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
370
    ) -> list[RequestOutput]:
371
372
        ...

nunjunj's avatar
nunjunj committed
373
374
375
    @deprecate_kwargs(
        "prompt_token_ids",
        is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
376
        additional_message="Please use the 'prompts' parameter instead.",
nunjunj's avatar
nunjunj committed
377
    )
378
379
    def generate(
        self,
380
        prompts: Union[Union[PromptType, Sequence[PromptType]],
381
                       Optional[Union[str, list[str]]]] = None,
382
383
        sampling_params: Optional[Union[SamplingParams,
                                        Sequence[SamplingParams]]] = None,
384
        prompt_token_ids: Optional[Union[list[int], list[list[int]]]] = None,
385
        use_tqdm: bool = True,
386
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
387
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
388
        guided_options_request: Optional[Union[LLMGuidedOptions,
389
                                               GuidedDecodingRequest]] = None,
390
391
        priority: Optional[list[int]] = None,
    ) -> list[RequestOutput]:
Woosuk Kwon's avatar
Woosuk Kwon committed
392
393
        """Generates the completions for the input prompts.

394
        This class automatically batches the given prompts, considering
Woosuk Kwon's avatar
Woosuk Kwon committed
395
396
397
398
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
399
400
401
            prompts: The prompts to the LLM. You may pass a sequence of prompts
                for batch inference. See :class:`~vllm.inputs.PromptType`
                for more details about the format of each prompts.
Woosuk Kwon's avatar
Woosuk Kwon committed
402
            sampling_params: The sampling parameters for text generation. If
nunjunj's avatar
nunjunj committed
403
404
405
                None, we use the default sampling parameters.
                When it is a single value, it is applied to every prompt.
                When it is a list, the list must have the same length as the
406
                prompts and it is paired one by one with the prompt.
Woosuk Kwon's avatar
Woosuk Kwon committed
407
            use_tqdm: Whether to use tqdm to display the progress bar.
408
            lora_request: LoRA request to use for generation, if any.
nunjunj's avatar
nunjunj committed
409
            prompt_adapter_request: Prompt Adapter request to use for
410
                generation, if any.
411
412
            priority: The priority of the requests, if any.
                Only applicable when priority scheduling policy is enabled.
Woosuk Kwon's avatar
Woosuk Kwon committed
413
414

        Returns:
nunjunj's avatar
nunjunj committed
415
            A list of ``RequestOutput`` objects containing the
416
            generated completions in the same order as the input prompts.
417
418
419
420
421

        Note:
            Using ``prompts`` and ``prompt_token_ids`` as keyword parameters is
            considered legacy and may be deprecated in the future. You should
            instead pass them via the ``inputs`` parameter.
422
        """
423
        runner_type = self.llm_engine.model_config.runner_type
424
        if runner_type not in ["generate", "transcription"]:
425
            messages = [
426
                "LLM.generate() is only supported for (conditional) generation "
427
428
429
                "models (XForCausalLM, XForConditionalGeneration).",
            ]

430
431
432
            supported_runner_types = self.llm_engine.model_config \
                .supported_runner_types
            if "generate" in supported_runner_types:
433
                messages.append(
434
435
436
                    "Your model supports the 'generate' runner, but is "
                    f"currently initialized for the '{runner_type}' runner. "
                    "Please initialize vLLM using `--task generate`.")
437
438

            raise ValueError(" ".join(messages))
439

440
        if prompt_token_ids is not None:
441
            parsed_prompts = self._convert_v1_inputs(
442
                prompts=cast(Optional[Union[str, list[str]]], prompts),
443
444
445
                prompt_token_ids=prompt_token_ids,
            )
        else:
446
447
            parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
                                  prompts)
448

449
450
451
452
453
454
455
456
        if isinstance(guided_options_request, dict):
            if len(guided_options_request) > 1:
                raise ValueError(
                    "You can only use one guided decoding but multiple is "
                    f"specified: {guided_options_request}")
            guided_options_request = GuidedDecodingRequest(
                **guided_options_request)

457
458
        if sampling_params is None:
            # Use default sampling params.
459
            sampling_params = self.get_default_sampling_params()
460

461
        self._validate_and_add_requests(
462
            prompts=parsed_prompts,
463
464
            params=sampling_params,
            lora_request=lora_request,
465
            prompt_adapter_request=prompt_adapter_request,
466
467
            guided_options=guided_options_request,
            priority=priority)
468

469
        outputs = self._run_engine(use_tqdm=use_tqdm)
Joe Runde's avatar
Joe Runde committed
470
        return self.engine_class.validate_outputs(outputs, RequestOutput)
471

472
    def collective_rpc(self,
473
                       method: Union[str, Callable[..., _R]],
474
                       timeout: Optional[float] = None,
475
476
                       args: tuple = (),
                       kwargs: Optional[dict[str, Any]] = None) -> list[_R]:
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
        """
        Execute an RPC call on all workers.

        Args:
            method: Name of the worker method to execute, or a callable that
                is serialized and sent to all workers to execute.

                If the method is a callable, it should accept an additional
                `self` argument, in addition to the arguments passed in `args`
                and `kwargs`. The `self` argument will be the worker object.
            timeout: Maximum time in seconds to wait for execution. Raises a
                :exc:`TimeoutError` on timeout. `None` means wait indefinitely.
            args: Positional arguments to pass to the worker method.
            kwargs: Keyword arguments to pass to the worker method.

        Returns:
            A list containing the results from each worker.
        
        Note:
            It is recommended to use this API to only pass control messages,
            and set up data-plane communication to pass data.
        """
        executor = self.llm_engine.model_executor
        return executor.collective_rpc(method, timeout, args, kwargs)

    def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]:
503
        """
504
505
        Run a function directly on the model inside each worker,
        returning the result for each of them.
506
        """
507
508
        executor = self.llm_engine.model_executor
        return executor.apply_model(func)
509

510
511
    def beam_search(
        self,
512
        prompts: list[Union[TokensPrompt, TextPrompt]],
513
        params: BeamSearchParams,
514
    ) -> list[BeamSearchOutput]:
515
516
517
518
519
520
        """
        Generate sequences using beam search.

        Args:
            prompts: A list of prompts. Each prompt can be a string or a list
                of token IDs.
521
522
            params: The beam search parameters.

523
524
525
526
        TODO: how does beam search work together with length penalty, frequency
        penalty, and stopping criteria, etc.?
        """

527
528
529
530
        beam_width = params.beam_width
        max_tokens = params.max_tokens
        temperature = params.temperature
        ignore_eos = params.ignore_eos
531
532
533
534
535
536
        length_penalty = params.length_penalty

        def sort_beams_key(x: BeamSearchSequence) -> float:
            return get_beam_search_score(x.tokens, x.cum_logprob,
                                         tokenizer.eos_token_id,
                                         length_penalty)
537

538
539
540
541
542
543
        tokenizer = self.get_tokenizer()
        # generate 2 * beam_width candidates at each step
        # following the huggingface transformers implementation
        # at https://github.com/huggingface/transformers/blob/e15687fffe5c9d20598a19aeab721ae0a7580f8a/src/transformers/generation/beam_search.py#L534 # noqa
        beam_search_params = SamplingParams(logprobs=2 * beam_width,
                                            max_tokens=1,
544
                                            temperature=temperature)
545
        instances: list[BeamSearchInstance] = []
546
547

        for prompt in prompts:
548
549
550
551
            if is_token_prompt(prompt):
                prompt_tokens = prompt["prompt_token_ids"]
            else:
                prompt_tokens = tokenizer.encode(prompt["prompt"])
552
553
554
            instances.append(BeamSearchInstance(prompt_tokens))

        for _ in range(max_tokens):
555
            all_beams: list[BeamSearchSequence] = list(
556
557
558
559
                sum((instance.beams for instance in instances), []))
            pos = [0] + list(
                itertools.accumulate(
                    len(instance.beams) for instance in instances))
560
            instance_start_and_end: list[tuple[int, int]] = list(
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
                zip(pos[:-1], pos[1:]))

            if len(all_beams) == 0:
                break

            prompts_batch = [
                TokensPrompt(prompt_token_ids=beam.tokens)
                for beam in all_beams
            ]

            # only runs for one step
            # we don't need to use tqdm here
            output = self.generate(prompts_batch,
                                   sampling_params=beam_search_params,
                                   use_tqdm=False)

            for (start, end), instance in zip(instance_start_and_end,
                                              instances):
                instance_new_beams = []
                for i in range(start, end):
                    current_beam = all_beams[i]
                    result = output[i]

                    if result.outputs[0].logprobs is not None:
                        # if `result.outputs[0].logprobs` is None, it means
                        # the sequence is completed because of the max-model-len
                        # or abortion. we don't need to add it to the new beams.
                        logprobs = result.outputs[0].logprobs[0]
                        for token_id, logprob_obj in logprobs.items():
                            new_beam = BeamSearchSequence(
                                tokens=current_beam.tokens + [token_id],
592
                                logprobs=current_beam.logprobs + [logprobs],
593
594
595
596
597
598
599
600
601
                                cum_logprob=current_beam.cum_logprob +
                                logprob_obj.logprob)

                            if token_id == tokenizer.eos_token_id and \
                                not ignore_eos:
                                instance.completed.append(new_beam)
                            else:
                                instance_new_beams.append(new_beam)
                sorted_beams = sorted(instance_new_beams,
602
                                      key=sort_beams_key,
603
604
605
606
607
608
609
                                      reverse=True)
                instance.beams = sorted_beams[:beam_width]

        outputs = []
        for instance in instances:
            instance.completed.extend(instance.beams)
            sorted_completed = sorted(instance.completed,
610
                                      key=sort_beams_key,
611
612
613
614
615
616
617
618
619
                                      reverse=True)
            best_beams = sorted_completed[:beam_width]

            for beam in best_beams:
                beam.text = tokenizer.decode(beam.tokens)
            outputs.append(BeamSearchOutput(sequences=best_beams))

        return outputs

nunjunj's avatar
nunjunj committed
620
621
    def chat(
        self,
622
623
        messages: Union[list[ChatCompletionMessageParam],
                        list[list[ChatCompletionMessageParam]]],
nunjunj's avatar
nunjunj committed
624
        sampling_params: Optional[Union[SamplingParams,
625
                                        list[SamplingParams]]] = None,
nunjunj's avatar
nunjunj committed
626
627
628
        use_tqdm: bool = True,
        lora_request: Optional[LoRARequest] = None,
        chat_template: Optional[str] = None,
629
        chat_template_content_format: ChatTemplateContentFormatOption = "auto",
630
        add_generation_prompt: bool = True,
631
        continue_final_message: bool = False,
632
633
634
        tools: Optional[list[dict[str, Any]]] = None,
        mm_processor_kwargs: Optional[dict[str, Any]] = None,
    ) -> list[RequestOutput]:
nunjunj's avatar
nunjunj committed
635
        """
636
        Generate responses for a chat conversation.
nunjunj's avatar
nunjunj committed
637

638
639
640
641
642
643
        The chat conversation is converted into a text prompt using the
        tokenizer and calls the :meth:`generate` method to generate the
        responses.

        Multi-modal inputs can be passed in the same way you would pass them
        to the OpenAI API.
nunjunj's avatar
nunjunj committed
644
645

        Args:
646
647
648
649
650
            messages: A list of conversations or a single conversation.

              - Each conversation is represented as a list of messages.
              - Each message is a dictionary with 'role' and 'content' keys.

nunjunj's avatar
nunjunj committed
651
652
653
654
655
656
657
658
659
            sampling_params: The sampling parameters for text generation.
                If None, we use the default sampling parameters. When it
                is a single value, it is applied to every prompt. When it
                is a list, the list must have the same length as the
                prompts and it is paired one by one with the prompt.
            use_tqdm: Whether to use tqdm to display the progress bar.
            lora_request: LoRA request to use for generation, if any.
            chat_template: The template to use for structuring the chat.
              If not provided, the model's default chat template will be used.
660
661
662
663
664
665
666
667
            chat_template_content_format: The format to render message content.

              - "string" will render the content as a string.
                Example: ``"Who are you?"``
              - "openai" will render the content as a list of dictionaries,
                similar to OpenAI schema.
                Example: ``[{"type": "text", "text": "Who are you?"}]``

668
            add_generation_prompt: If True, adds a generation template
nunjunj's avatar
nunjunj committed
669
                to each message.
670
            continue_final_message: If True, continues the final message in
671
672
                the conversation instead of starting a new one. Cannot be
                ``True`` if ``add_generation_prompt`` is also ``True``.
673
674
            mm_processor_kwargs: Multimodal processor kwarg overrides for this
                chat request. Only used for offline requests.
nunjunj's avatar
nunjunj committed
675
676
677
678
679

        Returns:
            A list of ``RequestOutput`` objects containing the generated
            responses in the same order as the input messages.
        """
680
        list_of_messages: list[list[ChatCompletionMessageParam]]
nunjunj's avatar
nunjunj committed
681

682
683
        # Handle multi and single conversations
        if is_list_of(messages, list):
684
685
            # messages is list[list[...]]
            list_of_messages = cast(list[list[ChatCompletionMessageParam]],
686
                                    messages)
687
        else:
688
            # messages is list[...]
689
            list_of_messages = [
690
                cast(list[ChatCompletionMessageParam], messages)
691
            ]
692

693
694
695
696
697
698
699
700
        tokenizer = self.get_tokenizer()
        model_config = self.llm_engine.get_model_config()
        resolved_content_format = resolve_chat_template_content_format(
            chat_template,
            chat_template_content_format,
            tokenizer,
        )

701
        prompts: list[Union[TokensPrompt, TextPrompt]] = []
702
703

        for msgs in list_of_messages:
704
705
706
            # NOTE: _parse_chat_message_content_parts() currently doesn't
            # handle mm_processor_kwargs, since there is no implementation in
            # the chat message parsing for it.
707
            conversation, mm_data = parse_chat_messages(
708
709
710
711
712
                msgs,
                model_config,
                tokenizer,
                content_format=resolved_content_format,
            )
713

714
            prompt_data: Union[str, list[int]]
715
716
717
718
719
720
            if isinstance(tokenizer, MistralTokenizer):
                prompt_data = apply_mistral_chat_template(
                    tokenizer,
                    messages=msgs,
                    chat_template=chat_template,
                    add_generation_prompt=add_generation_prompt,
721
                    continue_final_message=continue_final_message,
722
723
724
725
726
727
728
729
                    tools=tools,
                )
            else:
                prompt_data = apply_hf_chat_template(
                    tokenizer,
                    conversation=conversation,
                    chat_template=chat_template,
                    add_generation_prompt=add_generation_prompt,
730
                    continue_final_message=continue_final_message,
731
732
733
734
735
736
737
738
739
740
741
742
                    tools=tools,
                )

            prompt: Union[TokensPrompt, TextPrompt]
            if is_list_of(prompt_data, int):
                prompt = TokensPrompt(prompt_token_ids=prompt_data)
            else:
                prompt = TextPrompt(prompt=prompt_data)

            if mm_data is not None:
                prompt["multi_modal_data"] = mm_data

743
744
745
            if mm_processor_kwargs is not None:
                prompt["mm_processor_kwargs"] = mm_processor_kwargs

746
            prompts.append(prompt)
747

nunjunj's avatar
nunjunj committed
748
        return self.generate(
749
            prompts,
750
            sampling_params=sampling_params,
nunjunj's avatar
nunjunj committed
751
752
753
754
            use_tqdm=use_tqdm,
            lora_request=lora_request,
        )

755
756
757
758
759
760
761
    @overload
    def encode(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
762
        *,
763
        use_tqdm: bool = True,
764
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
765
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
766
    ) -> list[PoolingRequestOutput]:
767
768
        ...

769
    @overload  # LEGACY: single (prompt + optional token ids)
770
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
771
772
773
774
775
    def encode(
        self,
        prompts: str,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
776
        prompt_token_ids: Optional[list[int]] = None,
777
        use_tqdm: bool = True,
778
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
779
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
780
    ) -> list[PoolingRequestOutput]:
781
        ...
782

783
    @overload  # LEGACY: multi (prompt + optional token ids)
784
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
785
786
    def encode(
        self,
787
        prompts: list[str],
788
        pooling_params: Optional[Union[PoolingParams,
789
                                       Sequence[PoolingParams]]] = None,
790
        prompt_token_ids: Optional[list[list[int]]] = None,
791
        use_tqdm: bool = True,
792
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
793
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
794
    ) -> list[PoolingRequestOutput]:
795
796
797
        ...

    @overload  # LEGACY: single (token ids + optional prompt)
798
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
799
800
801
802
803
804
    def encode(
        self,
        prompts: Optional[str] = None,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
        *,
805
        prompt_token_ids: list[int],
806
        use_tqdm: bool = True,
807
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
808
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
809
    ) -> list[PoolingRequestOutput]:
810
811
812
        ...

    @overload  # LEGACY: multi (token ids + optional prompt)
813
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
814
815
    def encode(
        self,
816
        prompts: Optional[list[str]] = None,
817
818
819
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
        *,
820
        prompt_token_ids: list[list[int]],
821
        use_tqdm: bool = True,
822
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
823
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
824
    ) -> list[PoolingRequestOutput]:
825
826
827
        ...

    @overload  # LEGACY: single or multi token ids [pos-only]
828
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
829
830
831
832
    def encode(
        self,
        prompts: None,
        pooling_params: None,
833
        prompt_token_ids: Union[list[int], list[list[int]]],
834
        use_tqdm: bool = True,
835
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
836
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
837
    ) -> list[PoolingRequestOutput]:
838
839
        ...

nunjunj's avatar
nunjunj committed
840
841
842
    @deprecate_kwargs(
        "prompt_token_ids",
        is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
843
        additional_message="Please use the 'prompts' parameter instead.",
nunjunj's avatar
nunjunj committed
844
    )
845
846
    def encode(
        self,
847
        prompts: Union[Union[PromptType, Sequence[PromptType]],
848
                       Optional[Union[str, list[str]]]] = None,
849
850
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
851
        prompt_token_ids: Optional[Union[list[int], list[list[int]]]] = None,
852
        use_tqdm: bool = True,
853
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
854
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
855
    ) -> list[PoolingRequestOutput]:
856
857
        """Apply pooling to the hidden states corresponding to the input
        prompts.
858

859
        This class automatically batches the given prompts, considering
860
861
862
863
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
864
865
866
            prompts: The prompts to the LLM. You may pass a sequence of prompts
                for batch inference. See :class:`~vllm.inputs.PromptType`
                for more details about the format of each prompts.
867
868
869
870
            pooling_params: The pooling parameters for pooling. If None, we
                use the default pooling parameters.
            use_tqdm: Whether to use tqdm to display the progress bar.
            lora_request: LoRA request to use for generation, if any.
nunjunj's avatar
nunjunj committed
871
            prompt_adapter_request: Prompt Adapter request to use for
872
                generation, if any.
873
874

        Returns:
875
            A list of ``PoolingRequestOutput`` objects containing the
876
            pooled hidden states in the same order as the input prompts.
877
878
879
880
881

        Note:
            Using ``prompts`` and ``prompt_token_ids`` as keyword parameters is
            considered legacy and may be deprecated in the future. You should
            instead pass them via the ``inputs`` parameter.
882
        """
883
884
885
        runner_type = self.llm_engine.model_config.runner_type
        if runner_type != "pooling":
            messages = ["LLM.encode() is only supported for pooling models."]
886

887
888
889
            supported_runner_types = self.llm_engine.model_config \
                .supported_runner_types
            if "pooling" in supported_runner_types:
890
                messages.append(
891
892
893
894
                    "Your model supports the 'pooling' runner, but is "
                    f"currently initialized for the '{runner_type}' runner. "
                    "Please initialize vLLM using `--task embed`, "
                    "`--task classify`, `--task score` etc.")
895
896

            raise ValueError(" ".join(messages))
897

898
        if prompt_token_ids is not None:
899
            parsed_prompts = self._convert_v1_inputs(
900
                prompts=cast(Optional[Union[str, list[str]]], prompts),
901
902
903
                prompt_token_ids=prompt_token_ids,
            )
        else:
904
905
            parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
                                  prompts)
906

907
908
909
910
        if pooling_params is None:
            # Use default pooling params.
            pooling_params = PoolingParams()

911
        self._validate_and_add_requests(
912
            prompts=parsed_prompts,
913
914
            params=pooling_params,
            lora_request=lora_request,
915
            prompt_adapter_request=prompt_adapter_request,
916
917
        )

918
        outputs = self._run_engine(use_tqdm=use_tqdm)
Joe Runde's avatar
Joe Runde committed
919
        return self.engine_class.validate_outputs(outputs,
920
                                                  PoolingRequestOutput)
921

922
923
924
925
926
927
    def embed(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        *,
        use_tqdm: bool = True,
928
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
929
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
930
    ) -> list[EmbeddingRequestOutput]:
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
        """
        Generate an embedding vector for each prompt.

        This class automatically batches the given prompts, considering
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
            prompts: The prompts to the LLM. You may pass a sequence of prompts
                for batch inference. See :class:`~vllm.inputs.PromptType`
                for more details about the format of each prompts.
            use_tqdm: Whether to use tqdm to display the progress bar.
            lora_request: LoRA request to use for generation, if any.
            prompt_adapter_request: Prompt Adapter request to use for
                generation, if any.

        Returns:
            A list of ``EmbeddingRequestOutput`` objects containing the
            embedding vectors in the same order as the input prompts.
        """
        if self.llm_engine.model_config.task != "embed":
            raise ValueError(
                "Embedding API is only enabled for `--task embed`")

        items = self.encode(prompts,
                            use_tqdm=use_tqdm,
                            lora_request=lora_request,
                            prompt_adapter_request=prompt_adapter_request)

        return [EmbeddingRequestOutput.from_base(item) for item in items]

    def classify(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        *,
        use_tqdm: bool = True,
968
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
969
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
970
    ) -> list[ClassificationRequestOutput]:
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
        """
        Generate class logits for each prompt.

        This class automatically batches the given prompts, considering
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
            prompts: The prompts to the LLM. You may pass a sequence of prompts
                for batch inference. See :class:`~vllm.inputs.PromptType`
                for more details about the format of each prompts.
            use_tqdm: Whether to use tqdm to display the progress bar.
            lora_request: LoRA request to use for generation, if any.
            prompt_adapter_request: Prompt Adapter request to use for
                generation, if any.

        Returns:
            A list of ``ClassificationRequestOutput`` objects containing the
            embedding vectors in the same order as the input prompts.
        """
        if self.llm_engine.model_config.task != "classify":
            raise ValueError(
                "Classification API is only enabled for `--task classify`")

        items = self.encode(prompts,
                            use_tqdm=use_tqdm,
                            lora_request=lora_request,
                            prompt_adapter_request=prompt_adapter_request)

        return [ClassificationRequestOutput.from_base(item) for item in items]

1002
1003
1004
    def _embedding_score(
        self,
        tokenizer: AnyTokenizer,
1005
1006
        text_1: list[Union[str, TextPrompt, TokensPrompt]],
        text_2: list[Union[str, TextPrompt, TokensPrompt]],
1007
1008
        truncate_prompt_tokens: Optional[int] = None,
        use_tqdm: bool = True,
1009
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
1010
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
1011
    ) -> list[ScoringRequestOutput]:
1012

1013
        encoded_output: list[PoolingRequestOutput] = self.encode(
1014
1015
1016
1017
            text_1 + text_2,
            use_tqdm=use_tqdm,
            lora_request=lora_request,
            prompt_adapter_request=prompt_adapter_request)
1018

1019
        encoded_output_1: list[PoolingRequestOutput] = encoded_output[
1020
            0:len(text_1)]
1021
        encoded_output_2: list[PoolingRequestOutput] = encoded_output[
1022
            len(text_1):]
1023
1024
1025
1026

        if len(encoded_output_1) == 1:
            encoded_output_1 = encoded_output_1 * len(encoded_output_2)

1027
        scores: list[PoolingRequestOutput] = []
1028

1029
1030
1031
        scores = _cosine_similarity(tokenizer=tokenizer,
                                    embed_1=encoded_output_1,
                                    embed_2=encoded_output_2)
1032
1033
1034
1035
1036
1037
1038

        items = self.engine_class.validate_outputs(scores,
                                                   PoolingRequestOutput)
        return [ScoringRequestOutput.from_base(item) for item in items]

    def _cross_encoding_score(
        self,
1039
        tokenizer: AnyTokenizer,
1040
1041
        text_1: list[str],
        text_2: list[str],
1042
1043
        truncate_prompt_tokens: Optional[int] = None,
        use_tqdm: bool = True,
1044
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
1045
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
1046
    ) -> list[ScoringRequestOutput]:
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058

        if isinstance(tokenizer, MistralTokenizer):
            raise ValueError(
                "Score API is only enabled for `--task embed or score`")

        if len(text_1) == 1:
            text_1 = text_1 * len(text_2)

        input_pairs = [(t1, t2) for t1, t2 in zip(text_1, text_2)]

        pooling_params = PoolingParams()

1059
        tokenization_kwargs: dict[str, Any] = {}
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
        if truncate_prompt_tokens is not None:
            tokenization_kwargs["truncation"] = True
            tokenization_kwargs["max_length"] = truncate_prompt_tokens

        parsed_prompts = []

        for q, t in input_pairs:
            prompt_inputs = tokenizer(text=q,
                                      text_pair=t,
                                      **tokenization_kwargs)
            engine_prompt = TokensPrompt(
                prompt_token_ids=prompt_inputs["input_ids"],
                token_type_ids=prompt_inputs.get("token_type_ids"))
            parsed_prompts.append(engine_prompt)

        self._validate_and_add_requests(
            prompts=parsed_prompts,
            params=pooling_params,
            lora_request=lora_request,
            prompt_adapter_request=prompt_adapter_request,
        )

        outputs = self._run_engine(use_tqdm=use_tqdm)
        items = self.engine_class.validate_outputs(outputs,
                                                   PoolingRequestOutput)

        return [ScoringRequestOutput.from_base(item) for item in items]

1088
1089
1090
1091
1092
    def score(
        self,
        text_1: Union[SingletonPrompt, Sequence[SingletonPrompt]],
        text_2: Union[SingletonPrompt, Sequence[SingletonPrompt]],
        /,
1093
        *,
1094
1095
        truncate_prompt_tokens: Optional[int] = None,
        use_tqdm: bool = True,
1096
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
1097
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
1098
    ) -> list[ScoringRequestOutput]:
1099
        """Generate similarity scores for all pairs ``<text,text_pair>``.
1100

1101
1102
1103
1104
        The inputs can be ``1 -> 1``, ``1 -> N`` or ``N -> N``.
        In the ``1 - N`` case the ``text_1`` sentence will be replicated ``N``
        times to pair with the ``text_2`` sentences.
        The input pairs are used to build a list of prompts for the
1105
1106
1107
1108
1109
1110
        cross encoder model. This class automatically batches the prompts,
        considering the memory constraint. For the best performance, put all
        of your texts into a single list and pass it to this method.

        Args:
            text_1: can be a single prompt or a list of prompts, in which
1111
                case it has to have the same length as the ``text_2`` list
1112
1113
1114
1115
1116
1117
1118
1119
1120
            text_2: The texts to pair with the query to form the input
                to the LLM. See :class:`~vllm.inputs.PromptType` for
                more details about the format of each prompts.
            use_tqdm: Whether to use tqdm to display the progress bar.
            lora_request: LoRA request to use for generation, if any.
            prompt_adapter_request: Prompt Adapter request to use for
                generation, if any.

        Returns:
1121
            A list of ``ScoringRequestOutput`` objects containing the
1122
1123
            generated scores in the same order as the input prompts.
        """
1124
1125
1126
        runner_type = self.llm_engine.model_config.runner_type
        if runner_type != "pooling":
            messages = ["LLM.score() is only supported for pooling models."]
1127

1128
1129
1130
            supported_runner_types = self.llm_engine.model_config \
                .supported_runner_types
            if "pooling" in supported_runner_types:
1131
                messages.append(
1132
1133
1134
1135
                    "Your model supports the 'pooling' runner, but is "
                    f"currently initialized for the '{runner_type}' runner. "
                    "Please initialize vLLM using `--task embed`, "
                    "`--task classify`, `--task score` etc.")
1136
1137
1138

            raise ValueError(" ".join(messages))

1139
        if self.llm_engine.model_config.task not in ("embed", "score"):
1140
            raise ValueError(
1141
                "Score API is only enabled for `--task embed or --task score`")
1142
1143
1144
1145

        # the tokenizer for models such as
        # "cross-encoder/ms-marco-MiniLM-L-6-v2" doesn't support passing
        # lists of tokens to the `text` and `text_pair` kwargs
1146
1147
        tokenizer = self.llm_engine.get_tokenizer()

1148
1149
1150
1151
        def ensure_str(prompt: SingletonPrompt):
            if isinstance(prompt, dict):
                if "multi_modal_data" in prompt:
                    raise ValueError("Multi-modal prompt is not "
1152
                                     "supported for scoring")
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
                elif "prompt_token_ids" in prompt:
                    prompt = tokenizer.decode(
                        cast(TokensPrompt, prompt)["prompt_token_ids"])
                elif "prompt" in prompt:
                    prompt = cast(TextPrompt, prompt)["prompt"]
            assert type(prompt) is str
            return prompt

        if isinstance(text_1, (str, dict)):
            # Convert a single prompt to a list.
            text_1 = [text_1]
1164
        input_text_1: list[str] = [ensure_str(t) for t in text_1]
1165
1166
1167
1168

        if isinstance(text_2, (str, dict)):
            # Convert a single prompt to a list.
            text_2 = [text_2]
1169
        input_text_2: list[str] = [ensure_str(t) for t in text_2]
1170

1171
        _validate_score_input_lens(input_text_1, input_text_2)
1172

1173
        if self.llm_engine.model_config.is_cross_encoder:
1174
1175
            return self._cross_encoding_score(tokenizer, input_text_1,
                                              input_text_2,
1176
1177
1178
1179
                                              truncate_prompt_tokens, use_tqdm,
                                              lora_request,
                                              prompt_adapter_request)
        else:
1180
1181
1182
1183
1184
1185
1186
1187
            return self._embedding_score(
                tokenizer,
                input_text_1,  # type: ignore[arg-type]
                input_text_2,  # type: ignore[arg-type]
                truncate_prompt_tokens,
                use_tqdm,
                lora_request,
                prompt_adapter_request)
1188

1189
1190
1191
1192
1193
1194
    def start_profile(self) -> None:
        self.llm_engine.start_profile()

    def stop_profile(self) -> None:
        self.llm_engine.stop_profile()

1195
1196
1197
    def reset_prefix_cache(self) -> bool:
        return self.llm_engine.reset_prefix_cache()

1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
    def sleep(self, level: int = 1):
        """
        Put the engine to sleep. The engine should not process any requests.
        The caller should guarantee that no requests are being processed
        during the sleep period, before `wake_up` is called.

        :param level: The sleep level. Level 1 sleep will offload the model 
            weights and discard the kv cache. The content of kv cache is 
            forgotten. Level 1 sleep is good for sleeping and waking up the 
            engine to run the same model again. The model weights are backed 
            up in CPU memory. Please make sure there's enough CPU memory to 
            store the model weights. Level 2 sleep will discard both the model 
            weights and the kv cache. The content of both the model weights 
            and kv cache is forgotten. Level 2 sleep is good for sleeping and 
            waking up the engine to run a different model or update the model, 
            where previous model weights are not needed. It reduces CPU memory 
            pressure.
        """
1216
        self.reset_prefix_cache()
1217
1218
1219
        self.llm_engine.sleep(level=level)

    def wake_up(self):
1220
1221
1222
        """
        Wake up the engine from sleep mode. See the :meth:`sleep` method
        for more details."""
1223
1224
        self.llm_engine.wake_up()

1225
1226
    # LEGACY
    def _convert_v1_inputs(
1227
        self,
1228
1229
        prompts: Optional[Union[str, list[str]]],
        prompt_token_ids: Optional[Union[list[int], list[list[int]]]],
1230
1231
    ):
        # skip_tokenizer_init is now checked in engine
1232

1233
1234
1235
1236
1237
1238
        if prompts is not None:
            prompts = [p["content"] for p in parse_and_batch_prompt(prompts)]
        if prompt_token_ids is not None:
            prompt_token_ids = [
                p["content"] for p in parse_and_batch_prompt(prompt_token_ids)
            ]
1239

1240
        num_requests = None
1241
1242
        if prompts is not None:
            num_requests = len(prompts)
1243
1244
1245
1246
1247
1248
        if prompt_token_ids is not None:
            if (num_requests is not None
                    and num_requests != len(prompt_token_ids)):
                raise ValueError("The lengths of prompts and prompt_token_ids "
                                 "must be the same.")

1249
            num_requests = len(prompt_token_ids)
1250
1251
1252
1253
        if num_requests is None:
            raise ValueError("Either prompts or prompt_token_ids must be "
                             "provided.")

1254
        parsed_prompts: list[PromptType] = []
1255
        for i in range(num_requests):
1256
            item: PromptType
1257

1258
            if prompts is not None:
1259
1260
1261
                item = TextPrompt(prompt=prompts[i])
            elif prompt_token_ids is not None:
                item = TokensPrompt(prompt_token_ids=prompt_token_ids[i])
1262
            else:
1263
                raise AssertionError
1264

1265
            parsed_prompts.append(item)
1266

1267
        return parsed_prompts
1268
1269
1270

    def _validate_and_add_requests(
        self,
1271
        prompts: Union[PromptType, Sequence[PromptType]],
1272
1273
        params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
                      Sequence[PoolingParams]],
1274
        lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
1275
        prompt_adapter_request: Optional[PromptAdapterRequest],
1276
        guided_options: Optional[GuidedDecodingRequest] = None,
1277
        priority: Optional[list[int]] = None,
1278
    ) -> None:
1279
1280
1281
1282
1283
1284
1285
1286
        if guided_options is not None:
            warnings.warn(
                "guided_options_request is deprecated, use "
                "SamplingParams.guided_decoding instead",
                DeprecationWarning,
                stacklevel=2,
            )

1287
        if isinstance(prompts, (str, dict)):
1288
            # Convert a single prompt to a list.
1289
            prompts = [prompts]
1290

1291
        num_requests = len(prompts)
1292
1293
        if isinstance(params, list) and len(params) != num_requests:
            raise ValueError("The lengths of prompts and params "
1294
                             "must be the same.")
1295
1296
1297
1298
        if isinstance(lora_request,
                      list) and len(lora_request) != num_requests:
            raise ValueError("The lengths of prompts and lora_request "
                             "must be the same.")
1299

1300
1301
        for sp in params if isinstance(params, list) else (params, ):
            if isinstance(sp, SamplingParams):
1302
                self._add_guided_params(sp, guided_options)
1303
1304
1305

                # We only care about the final output
                sp.output_kind = RequestOutputKind.FINAL_ONLY
1306

Zhuohan Li's avatar
Zhuohan Li committed
1307
        # Add requests to the engine.
1308
        for i, prompt in enumerate(prompts):
1309
            self._add_request(
1310
                prompt,
1311
                params[i] if isinstance(params, Sequence) else params,
1312
1313
                lora_request=lora_request[i] if isinstance(
                    lora_request, Sequence) else lora_request,
nunjunj's avatar
nunjunj committed
1314
                prompt_adapter_request=prompt_adapter_request,
1315
                priority=priority[i] if priority else 0,
nunjunj's avatar
nunjunj committed
1316
            )
1317

1318
    def _add_request(
nunjunj's avatar
nunjunj committed
1319
        self,
1320
        prompt: PromptType,
nunjunj's avatar
nunjunj committed
1321
        params: Union[SamplingParams, PoolingParams],
1322
        lora_request: Optional[LoRARequest] = None,
nunjunj's avatar
nunjunj committed
1323
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
1324
        priority: int = 0,
1325
1326
    ) -> None:
        request_id = str(next(self.request_counter))
1327
1328
        self.llm_engine.add_request(
            request_id,
1329
            prompt,
1330
1331
            params,
            lora_request=lora_request,
nunjunj's avatar
nunjunj committed
1332
            prompt_adapter_request=prompt_adapter_request,
1333
            priority=priority,
nunjunj's avatar
nunjunj committed
1334
        )
1335

1336
    def _add_guided_params(
1337
1338
1339
            self,
            params: SamplingParams,
            guided_options: Optional[GuidedDecodingRequest] = None):
1340
1341
1342
1343
        if guided_options is None:
            return params

        if params.guided_decoding is not None:
1344
            raise ValueError("Cannot set both guided_options_request and "
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
                             "params.guided_decoding.")

        params.guided_decoding = GuidedDecodingParams(
            json=guided_options.guided_json,
            regex=guided_options.guided_regex,
            choice=guided_options.guided_choice,
            grammar=guided_options.guided_grammar,
            json_object=guided_options.guided_json_object,
            backend=guided_options.guided_decoding_backend,
            whitespace_pattern=guided_options.guided_whitespace_pattern)
1355
1356
        return params

1357
    def _run_engine(
1358
            self, *, use_tqdm: bool
1359
    ) -> list[Union[RequestOutput, PoolingRequestOutput]]:
1360
1361
        # Initialize tqdm.
        if use_tqdm:
Zhuohan Li's avatar
Zhuohan Li committed
1362
            num_requests = self.llm_engine.get_num_unfinished_requests()
1363
1364
1365
1366
            pbar = tqdm(
                total=num_requests,
                desc="Processed prompts",
                dynamic_ncols=True,
1367
1368
                postfix=(f"est. speed input: {0:.2f} toks/s, "
                         f"output: {0:.2f} toks/s"),
1369
            )
1370

Zhuohan Li's avatar
Zhuohan Li committed
1371
        # Run the engine.
1372
        outputs: list[Union[RequestOutput, PoolingRequestOutput]] = []
1373
1374
        total_in_toks = 0
        total_out_toks = 0
Zhuohan Li's avatar
Zhuohan Li committed
1375
1376
        while self.llm_engine.has_unfinished_requests():
            step_outputs = self.llm_engine.step()
1377
            for output in step_outputs:
1378
                if output.finished:
1379
1380
                    outputs.append(output)
                    if use_tqdm:
1381
1382
                        if isinstance(output, RequestOutput):
                            # Calculate tokens only for RequestOutput
1383
                            assert output.prompt_token_ids is not None
1384
1385
1386
                            total_in_toks += len(output.prompt_token_ids)
                            in_spd = total_in_toks / pbar.format_dict["elapsed"]
                            total_out_toks += sum(
1387
                                len(stp.token_ids) for stp in output.outputs)
nunjunj's avatar
nunjunj committed
1388
1389
                            out_spd = (total_out_toks /
                                       pbar.format_dict["elapsed"])
1390
1391
1392
                            pbar.postfix = (
                                f"est. speed input: {in_spd:.2f} toks/s, "
                                f"output: {out_spd:.2f} toks/s")
1393
                        pbar.update(1)
1394

1395
1396
        if use_tqdm:
            pbar.close()
1397
1398
1399
        # Sort the outputs by request ID.
        # This is necessary because some requests may be finished earlier than
        # its previous requests.
1400
        return sorted(outputs, key=lambda x: int(x.request_id))