llm.py 71.7 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import itertools
5
from collections.abc import Sequence
6
from contextlib import contextmanager
7
8
from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Optional, Union,
                    cast, overload)
9

10
import cloudpickle
11
import torch.nn as nn
12
from pydantic import ValidationError
13
from tqdm.auto import tqdm
14
from typing_extensions import TypeVar, deprecated
15

16
import vllm.envs as envs
17
from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
18
19
                              BeamSearchSequence,
                              create_sort_beams_key_function)
20
21
from vllm.config import (CompilationConfig, ModelDType, TokenizerMode,
                         is_init_field)
22
23
from vllm.engine.arg_utils import (ConvertOption, EngineArgs, HfOverrides,
                                   PoolerConfig, RunnerOption)
Joe Runde's avatar
Joe Runde committed
24
from vllm.engine.llm_engine import LLMEngine
nunjunj's avatar
nunjunj committed
25
from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
26
                                         ChatTemplateContentFormatOption,
27
28
                                         apply_hf_chat_template,
                                         apply_mistral_chat_template,
29
30
                                         parse_chat_messages,
                                         resolve_chat_template_content_format)
31
32
33
34
35
from vllm.entrypoints.score_utils import (ScoreContentPartParam,
                                          ScoreMultiModalParam,
                                          _cosine_similarity,
                                          _validate_score_input_lens,
                                          get_score_prompt)
36
37
from vllm.entrypoints.utils import (_validate_truncation_size,
                                    log_non_default_args)
38
from vllm.inputs import PromptType, SingletonPrompt, TextPrompt, TokensPrompt
39
from vllm.inputs.parse import parse_and_batch_prompt
40
from vllm.logger import init_logger
41
from vllm.lora.request import LoRARequest
42
from vllm.model_executor.layers.quantization import QuantizationMethods
43
44
45
from vllm.outputs import (ClassificationRequestOutput, EmbeddingRequestOutput,
                          PoolingRequestOutput, RequestOutput,
                          ScoringRequestOutput)
46
from vllm.pooling_params import PoolingParams
47
48
from vllm.sampling_params import (BeamSearchParams, RequestOutputKind,
                                  SamplingParams)
49
from vllm.tasks import PoolingTask
50
from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
51
                                               get_cached_tokenizer)
yhu422's avatar
yhu422 committed
52
from vllm.usage.usage_lib import UsageContext
53
from vllm.utils import Counter, Device, deprecate_kwargs, is_list_of
54

55
56
57
if TYPE_CHECKING:
    from vllm.v1.metrics.reader import Metric

58
59
logger = init_logger(__name__)

60
61
_R = TypeVar("_R", default=Any)

62
63

class LLM:
Woosuk Kwon's avatar
Woosuk Kwon committed
64
65
66
67
68
69
70
71
72
73
    """An LLM for generating texts from given prompts and sampling parameters.

    This class includes a tokenizer, a language model (possibly distributed
    across multiple GPUs), and GPU memory space allocated for intermediate
    states (aka KV cache). Given a batch of prompts and sampling parameters,
    this class generates texts from the model, using an intelligent batching
    mechanism and efficient memory management.

    Args:
        model: The name or path of a HuggingFace Transformers model.
74
        tokenizer: The name or path of a HuggingFace Transformers tokenizer.
75
76
        tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer
            if available, and "slow" will always use the slow tokenizer.
77
78
79
        skip_tokenizer_init: If true, skip initialization of tokenizer and
            detokenizer. Expect valid prompt_token_ids and None for prompt
            from the input.
80
81
        trust_remote_code: Trust remote code (e.g., from HuggingFace) when
            downloading the model and tokenizer.
82
83
84
85
        allowed_local_media_path: Allowing API requests to read local images
            or videos from directories specified by the server file system.
            This is a security risk. Should only be enabled in trusted
            environments.
Woosuk Kwon's avatar
Woosuk Kwon committed
86
87
88
        tensor_parallel_size: The number of GPUs to use for distributed
            execution with tensor parallelism.
        dtype: The data type for the model weights and activations. Currently,
Woosuk Kwon's avatar
Woosuk Kwon committed
89
90
91
92
            we support `float32`, `float16`, and `bfloat16`. If `auto`, we use
            the `torch_dtype` attribute specified in the model config file.
            However, if the `torch_dtype` in the config is `float32`, we will
            use `float16` instead.
93
        quantization: The method used to quantize the model weights. Currently,
94
            we support "awq", "gptq", and "fp8" (experimental).
95
96
97
98
            If None, we first check the `quantization_config` attribute in the
            model config file. If that is None, we assume the model weights are
            not quantized and use `dtype` to determine the data type of
            the weights.
Jasmond L's avatar
Jasmond L committed
99
100
        revision: The specific model version to use. It can be a branch name,
            a tag name, or a commit id.
101
102
        tokenizer_revision: The specific tokenizer version to use. It can be a
            branch name, a tag name, or a commit id.
103
104
105
106
107
108
109
        seed: The seed to initialize the random number generator for sampling.
        gpu_memory_utilization: The ratio (between 0 and 1) of GPU memory to
            reserve for the model weights, activations, and KV cache. Higher
            values will increase the KV cache size and thus improve the model's
            throughput. However, if the value is too high, it may cause out-of-
            memory (OOM) errors.
        swap_space: The size (GiB) of CPU memory per GPU to use as swap space.
110
111
112
113
114
            This can be used for temporarily storing the states of the requests
            when their `best_of` sampling parameters are larger than 1. If all
            requests will have `best_of=1`, you can safely set this to 0.
            Noting that `best_of` is only supported in V0. Otherwise, too small
            values may cause out-of-memory (OOM) errors.
115
116
117
118
        cpu_offload_gb: The size (GiB) of CPU memory to use for offloading
            the model weights. This virtually increases the GPU memory space
            you can use to hold the model weights, at the cost of CPU-GPU data
            transfer for every forward pass.
119
120
121
        enforce_eager: Whether to enforce eager execution. If True, we will
            disable CUDA graph and always execute the model in eager mode.
            If False, we will use CUDA graph and eager execution in hybrid.
122
        max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs.
123
            When a sequence has context length larger than this, we fall back
124
125
126
            to eager mode. Additionally for encoder-decoder models, if the
            sequence length of the encoder input is larger than this, we fall
            back to the eager mode.
127
128
        disable_custom_all_reduce: See
            [ParallelConfig][vllm.config.ParallelConfig].
129
130
        disable_async_output_proc: Disable async output processing.
            This may result in lower performance.
131
        hf_token: The token to use as HTTP bearer authorization for remote files
132
            . If `True`, will use the token generated when running
133
            `huggingface-cli login` (stored in `~/.huggingface`).
134
135
136
        hf_overrides: If a dictionary, contains arguments to be forwarded to the
            HuggingFace config. If a callable, it is called to update the
            HuggingFace config.
137
138
139
140
141
142
143
144
        mm_processor_kwargs: Arguments to be forwarded to the model's processor
            for multi-modal data, e.g., image processor. Overrides for the
            multi-modal processor obtained from `AutoProcessor.from_pretrained`.
            The available overrides depend on the model that is being run.
            For example, for Phi-3-Vision: `{"num_crops": 4}`.
        override_pooler_config: Initialize non-default pooling config or
            override default pooling config for the pooling model.
            e.g. `PoolerConfig(pooling_type="mean", normalize=False)`.
145
146
147
        compilation_config: Either an integer or a dictionary. If it is an
            integer, it is used as the level of compilation optimization. If it
            is a dictionary, it can specify the full compilation configuration.
148
        **kwargs: Arguments for [`EngineArgs`][vllm.EngineArgs].
nunjunj's avatar
nunjunj committed
149

150
151
    Note:
        This class is intended to be used for offline inference. For online
152
        serving, use the [AsyncLLMEngine][vllm.AsyncLLMEngine] class instead.
153
    """
154

155
    DEPRECATE_LEGACY: ClassVar[bool] = True
156
157
158
159
160
161
162
163
164
165
166
    """A flag to toggle whether to deprecate the legacy generate/encode API."""

    @classmethod
    @contextmanager
    def deprecate_legacy_api(cls):
        cls.DEPRECATE_LEGACY = True

        yield

        cls.DEPRECATE_LEGACY = False

167
168
169
    def __init__(
        self,
        model: str,
170
        *,
171
172
        runner: RunnerOption = "auto",
        convert: ConvertOption = "auto",
173
        tokenizer: Optional[str] = None,
174
        tokenizer_mode: TokenizerMode = "auto",
175
        skip_tokenizer_init: bool = False,
176
        trust_remote_code: bool = False,
177
        allowed_local_media_path: str = "",
178
        tensor_parallel_size: int = 1,
179
180
        dtype: ModelDType = "auto",
        quantization: Optional[QuantizationMethods] = None,
181
        revision: Optional[str] = None,
182
        tokenizer_revision: Optional[str] = None,
183
        seed: Optional[int] = None,
184
        gpu_memory_utilization: float = 0.9,
185
        swap_space: float = 4,
186
        cpu_offload_gb: float = 0,
187
        enforce_eager: bool = False,
188
        max_seq_len_to_capture: int = 8192,
189
        disable_custom_all_reduce: bool = False,
190
        disable_async_output_proc: bool = False,
191
        hf_token: Optional[Union[bool, str]] = None,
192
        hf_overrides: Optional[HfOverrides] = None,
193
        mm_processor_kwargs: Optional[dict[str, Any]] = None,
194
        override_pooler_config: Optional[PoolerConfig] = None,
195
196
        compilation_config: Optional[Union[int, dict[str, Any],
                                           CompilationConfig]] = None,
197
198
        **kwargs,
    ) -> None:
199
        """LLM constructor."""
200

201
202
        if "disable_log_stats" not in kwargs:
            kwargs["disable_log_stats"] = True
203

204
205
206
207
208
209
210
        if "worker_cls" in kwargs:
            worker_cls = kwargs["worker_cls"]
            # if the worker_cls is not qualified string name,
            # we serialize it using cloudpickle to avoid pickling issues
            if isinstance(worker_cls, type):
                kwargs["worker_cls"] = cloudpickle.dumps(worker_cls)

211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
        if "kv_transfer_config" in kwargs and isinstance(
                kwargs["kv_transfer_config"], dict):
            from vllm.config import KVTransferConfig
            raw_config_dict = kwargs["kv_transfer_config"]
            try:
                kwargs["kv_transfer_config"] = KVTransferConfig(
                    **raw_config_dict)
            except ValidationError as e:
                logger.error(
                    "Failed to convert 'kv_transfer_config' dict to "
                    "KVTransferConfig object. Dict: %s. Error: %s",
                    raw_config_dict, e)
                # Consider re-raising a more specific vLLM error or ValueError
                # to provide better context to the user.
                raise ValueError(
                    f"Invalid 'kv_transfer_config' provided: {e}") from e

228
229
230
        if hf_overrides is None:
            hf_overrides = {}

231
        if compilation_config is not None:
232
233
234
235
236
237
238
            if isinstance(compilation_config, int):
                compilation_config_instance = CompilationConfig(
                    level=compilation_config)
            elif isinstance(compilation_config, dict):
                predicate = lambda x: is_init_field(CompilationConfig, x[0])
                compilation_config_instance = CompilationConfig(
                    **dict(filter(predicate, compilation_config.items())))
239
240
            else:
                compilation_config_instance = compilation_config
241
        else:
242
            compilation_config_instance = CompilationConfig()
243

Zhuohan Li's avatar
Zhuohan Li committed
244
        engine_args = EngineArgs(
245
            model=model,
246
247
            runner=runner,
            convert=convert,
248
            tokenizer=tokenizer,
249
            tokenizer_mode=tokenizer_mode,
250
            skip_tokenizer_init=skip_tokenizer_init,
251
            trust_remote_code=trust_remote_code,
252
            allowed_local_media_path=allowed_local_media_path,
253
254
            tensor_parallel_size=tensor_parallel_size,
            dtype=dtype,
255
            quantization=quantization,
256
            revision=revision,
257
            tokenizer_revision=tokenizer_revision,
258
259
260
            seed=seed,
            gpu_memory_utilization=gpu_memory_utilization,
            swap_space=swap_space,
261
            cpu_offload_gb=cpu_offload_gb,
262
            enforce_eager=enforce_eager,
263
            max_seq_len_to_capture=max_seq_len_to_capture,
264
            disable_custom_all_reduce=disable_custom_all_reduce,
265
            disable_async_output_proc=disable_async_output_proc,
266
            hf_token=hf_token,
267
            hf_overrides=hf_overrides,
268
            mm_processor_kwargs=mm_processor_kwargs,
269
            override_pooler_config=override_pooler_config,
270
            compilation_config=compilation_config_instance,
271
272
            **kwargs,
        )
273

274
275
        log_non_default_args(engine_args)

276
277
278
279
        # Create the Engine (autoselects V0 vs V1)
        self.llm_engine = LLMEngine.from_engine_args(
            engine_args=engine_args, usage_context=UsageContext.LLM_CLASS)
        self.engine_class = type(self.llm_engine)
280

281
        self.request_counter = Counter()
282
        self.default_sampling_params: Union[dict[str, Any], None] = None
283

284
285
286
287
288
289
290
291
292
293
        if envs.VLLM_USE_V1:
            supported_tasks = self.llm_engine \
                .get_supported_tasks()  # type: ignore
        else:
            supported_tasks = self.llm_engine.model_config.supported_tasks

        logger.info("Supported_tasks: %s", supported_tasks)

        self.supported_tasks = supported_tasks

294
295
296
297
298
299
    def get_tokenizer(
        self,
        lora_request: Optional[LoRARequest] = None,
    ) -> AnyTokenizer:
        return self.llm_engine.get_tokenizer_group().get_lora_tokenizer(
            lora_request)
300
301

    def set_tokenizer(self, tokenizer: AnyTokenizer) -> None:
302
        tokenizer_group = self.llm_engine.get_tokenizer_group()
303

304
305
306
307
        # While CachedTokenizer is dynamic, have no choice but
        # compare class name. Misjudgment will arise from
        # user-defined tokenizer started with 'Cached'
        if tokenizer.__class__.__name__.startswith("Cached"):
308
            tokenizer_group.tokenizer = tokenizer
309
        else:
310
            tokenizer_group.tokenizer = get_cached_tokenizer(tokenizer)
311

312
    def get_default_sampling_params(self) -> SamplingParams:
313
314
315
316
317
        if self.default_sampling_params is None:
            self.default_sampling_params = (
                self.llm_engine.model_config.get_diff_sampling_param())
        if self.default_sampling_params:
            return SamplingParams.from_optional(**self.default_sampling_params)
318
319
        return SamplingParams()

320
321
322
323
324
325
326
    @overload
    def generate(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        sampling_params: Optional[Union[SamplingParams,
                                        Sequence[SamplingParams]]] = None,
327
        *,
328
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
329
330
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
    ) -> list[RequestOutput]:
331
332
        ...

333
    @overload  # LEGACY: single (prompt + optional token ids)
334
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
335
336
337
338
    def generate(
        self,
        prompts: str,
        sampling_params: Optional[Union[SamplingParams,
339
340
                                        list[SamplingParams]]] = None,
        prompt_token_ids: Optional[list[int]] = None,
341
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
342
343
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
    ) -> list[RequestOutput]:
344
345
346
        ...

    @overload  # LEGACY: multi (prompt + optional token ids)
347
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
348
349
    def generate(
        self,
350
        prompts: list[str],
351
        sampling_params: Optional[Union[SamplingParams,
352
353
                                        list[SamplingParams]]] = None,
        prompt_token_ids: Optional[list[list[int]]] = None,
354
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
355
356
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
    ) -> list[RequestOutput]:
357
358
359
        ...

    @overload  # LEGACY: single (token ids + optional prompt)
360
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
361
362
363
364
    def generate(
        self,
        prompts: Optional[str] = None,
        sampling_params: Optional[Union[SamplingParams,
365
                                        list[SamplingParams]]] = None,
366
        *,
367
        prompt_token_ids: list[int],
368
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
369
370
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
    ) -> list[RequestOutput]:
371
372
373
        ...

    @overload  # LEGACY: multi (token ids + optional prompt)
374
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
375
376
    def generate(
        self,
377
        prompts: Optional[list[str]] = None,
378
        sampling_params: Optional[Union[SamplingParams,
379
                                        list[SamplingParams]]] = None,
380
        *,
381
        prompt_token_ids: list[list[int]],
382
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
383
384
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
    ) -> list[RequestOutput]:
385
386
387
        ...

    @overload  # LEGACY: single or multi token ids [pos-only]
388
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
389
390
391
392
    def generate(
        self,
        prompts: None,
        sampling_params: None,
393
        prompt_token_ids: Union[list[int], list[list[int]]],
394
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
395
396
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
    ) -> list[RequestOutput]:
397
398
        ...

nunjunj's avatar
nunjunj committed
399
400
401
    @deprecate_kwargs(
        "prompt_token_ids",
        is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
402
        additional_message="Please use the 'prompts' parameter instead.",
nunjunj's avatar
nunjunj committed
403
    )
404
405
    def generate(
        self,
406
        prompts: Union[Union[PromptType, Sequence[PromptType]],
407
                       Optional[Union[str, list[str]]]] = None,
408
409
        sampling_params: Optional[Union[SamplingParams,
                                        Sequence[SamplingParams]]] = None,
410
        prompt_token_ids: Optional[Union[list[int], list[list[int]]]] = None,
411
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
412
413
414
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
        priority: Optional[list[int]] = None,
    ) -> list[RequestOutput]:
Woosuk Kwon's avatar
Woosuk Kwon committed
415
416
        """Generates the completions for the input prompts.

417
        This class automatically batches the given prompts, considering
Woosuk Kwon's avatar
Woosuk Kwon committed
418
419
420
421
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
422
            prompts: The prompts to the LLM. You may pass a sequence of prompts
423
                for batch inference. See [PromptType][vllm.inputs.PromptType]
424
                for more details about the format of each prompts.
Woosuk Kwon's avatar
Woosuk Kwon committed
425
            sampling_params: The sampling parameters for text generation. If
nunjunj's avatar
nunjunj committed
426
427
428
                None, we use the default sampling parameters.
                When it is a single value, it is applied to every prompt.
                When it is a list, the list must have the same length as the
429
                prompts and it is paired one by one with the prompt.
430
431
432
433
            use_tqdm: If `True`, shows a tqdm progress bar.
                If a callable (e.g., `functools.partial(tqdm, leave=False)`),
                it is used to create the progress bar.
                If `False`, no progress bar is created.
434
            lora_request: LoRA request to use for generation, if any.
435
436
            priority: The priority of the requests, if any.
                Only applicable when priority scheduling policy is enabled.
Woosuk Kwon's avatar
Woosuk Kwon committed
437
438

        Returns:
439
            A list of `RequestOutput` objects containing the
440
            generated completions in the same order as the input prompts.
441

442
443
444
445
        Note:
            Using `prompts` and `prompt_token_ids` as keyword parameters is
            considered legacy and may be deprecated in the future. You should
            instead pass them via the `inputs` parameter.
446
        """
447
448
449
        model_config = self.llm_engine.model_config
        runner_type = model_config.runner_type
        if runner_type != "generate":
450
451
452
453
            raise ValueError(
                "LLM.generate() is only supported for generative models. "
                "Try passing `--runner generate` to use the model as a "
                "generative model.")
454

455
        if prompt_token_ids is not None:
456
            parsed_prompts = self._convert_v1_inputs(
457
                prompts=cast(Optional[Union[str, list[str]]], prompts),
458
459
460
                prompt_token_ids=prompt_token_ids,
            )
        else:
461
462
            parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
                                  prompts)
463

464
465
        if sampling_params is None:
            # Use default sampling params.
466
            sampling_params = self.get_default_sampling_params()
467

468
469
470
471
        tokenization_kwargs: dict[str, Any] = {}
        truncate_prompt_tokens = None
        if isinstance(sampling_params, SamplingParams):
            truncate_prompt_tokens = sampling_params.truncate_prompt_tokens
472
473

        _validate_truncation_size(model_config.max_model_len,
474
475
                                  truncate_prompt_tokens, tokenization_kwargs)

476
477
478
479
        # Add any modality specific loras to the corresponding prompts
        lora_request = self._get_modality_specific_lora_reqs(
            parsed_prompts, lora_request)

480
        self._validate_and_add_requests(
481
            prompts=parsed_prompts,
482
            params=sampling_params,
483
            use_tqdm=use_tqdm,
484
            lora_request=lora_request,
485
            tokenization_kwargs=tokenization_kwargs,
486
487
            priority=priority,
        )
488

489
        outputs = self._run_engine(use_tqdm=use_tqdm)
Joe Runde's avatar
Joe Runde committed
490
        return self.engine_class.validate_outputs(outputs, RequestOutput)
491

492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
    def _get_modality_specific_lora_reqs(
            self, parsed_prompts: Union[PromptType, Sequence[PromptType]],
            lora_request: Optional[Union[list[LoRARequest], LoRARequest]]):
        # Grab the lora config off the vllm config on the engine,
        # since this is the same for both v0 & v1.
        lora_config = self.llm_engine.vllm_config.lora_config

        # If there's no lora config / default_mm_loras, or the model
        # isn't multimodal, leave the lora as is.
        if (lora_config is None
                or not self.llm_engine.model_config.is_multimodal_model
                or (lora_config and lora_config.default_mm_loras is None)):
            return lora_request

        if not isinstance(parsed_prompts, Sequence):
            parsed_prompts = [parsed_prompts]

        optional_loras = ([lora_request] * len(parsed_prompts)
                          if not isinstance(lora_request, Sequence) else
                          lora_request)

        return [
            self._resolve_single_prompt_mm_lora(
                parsed_prompt,
                opt_lora_req,
                lora_config.default_mm_loras,
            ) for parsed_prompt, opt_lora_req in zip(parsed_prompts,
                                                     optional_loras)
        ]

    def _resolve_single_prompt_mm_lora(self, parsed_prompt: PromptType,
                                       lora_request: Optional[LoRARequest],
                                       default_mm_loras: Optional[dict[str,
                                                                       str]]):
        if (not default_mm_loras or not isinstance(parsed_prompt, dict)
                or "multi_modal_data" not in parsed_prompt):
            return lora_request

        parsed_prompt = cast(Union[TextPrompt, TokensPrompt], parsed_prompt)

        intersection = set(
            parsed_prompt["multi_modal_data"].keys()).intersection(
                default_mm_loras.keys())
        if not intersection:
            return lora_request
        if len(intersection) > 1:
            # TODO: Would be nice to be able to have multiple loras per prompt
            logger.warning(
                "Multiple modality specific loras were registered and would be"
                " used by a single prompt consuming several modalities; "
                " currently we only support one lora per request; as such,"
                " lora(s) registered with modalities: %s"
                " will be skipped", intersection)
            return lora_request

        # Build the LoRA request; the ID of the default mm lora is the
        # index of the modality name sorted alphabetically + 1.
        modality_name = intersection.pop()
        modality_lora_path = default_mm_loras[modality_name]
        modality_lora_id = sorted(default_mm_loras).index(modality_name) + 1

        # If we have a collision, warn if there is a collision,
        # but always send the explicitly provided request.
        if lora_request:
            if lora_request.lora_int_id != modality_lora_id:
                logger.warning(
                    "A modality with a registered lora and a lora_request "
                    "with a different ID were provided; falling back to the "
                    "lora_request as we only apply one LoRARequest per prompt")
            return lora_request

        return LoRARequest(
            modality_name,
            modality_lora_id,
            modality_lora_path,
        )

569
    def collective_rpc(self,
570
                       method: Union[str, Callable[..., _R]],
571
                       timeout: Optional[float] = None,
572
573
                       args: tuple = (),
                       kwargs: Optional[dict[str, Any]] = None) -> list[_R]:
574
575
576
577
578
579
580
581
582
583
584
        """
        Execute an RPC call on all workers.

        Args:
            method: Name of the worker method to execute, or a callable that
                is serialized and sent to all workers to execute.

                If the method is a callable, it should accept an additional
                `self` argument, in addition to the arguments passed in `args`
                and `kwargs`. The `self` argument will be the worker object.
            timeout: Maximum time in seconds to wait for execution. Raises a
585
                [`TimeoutError`][] on timeout. `None` means wait indefinitely.
586
587
588
589
590
            args: Positional arguments to pass to the worker method.
            kwargs: Keyword arguments to pass to the worker method.

        Returns:
            A list containing the results from each worker.
591

592
593
594
        Note:
            It is recommended to use this API to only pass control messages,
            and set up data-plane communication to pass data.
595
        """
596
597

        return self.llm_engine.collective_rpc(method, timeout, args, kwargs)
598
599

    def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]:
600
        """
601
602
        Run a function directly on the model inside each worker,
        returning the result for each of them.
603
        """
604
605
        executor = self.llm_engine.model_executor
        return executor.apply_model(func)
606

607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
    def _get_beam_search_lora_requests(
        self,
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]],
        prompts: list[Union[TokensPrompt, TextPrompt]],
    ) -> list[Optional[LoRARequest]]:
        """Get the optional lora request corresponding to each prompt."""
        if isinstance(lora_request,
                      Sequence) and len(lora_request) != len(prompts):
            raise ValueError(
                "Lora request list should be the same length as the prompts")

        if lora_request is None or isinstance(lora_request, LoRARequest):
            return [lora_request] * len(prompts)

        raise TypeError(f"Invalid lora_request type {type(lora_request)}")

623
624
    def beam_search(
        self,
625
        prompts: list[Union[TokensPrompt, TextPrompt]],
626
        params: BeamSearchParams,
627
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
628
        use_tqdm: bool = False,
629
    ) -> list[BeamSearchOutput]:
630
631
632
633
634
635
        """
        Generate sequences using beam search.

        Args:
            prompts: A list of prompts. Each prompt can be a string or a list
                of token IDs.
636
            params: The beam search parameters.
637
            lora_request: LoRA request to use for generation, if any.
638
            use_tqdm: Whether to use tqdm to display the progress bar.
639
        """
640
641
        # TODO: how does beam search work together with length penalty,
        # frequency, penalty, and stopping criteria, etc.?
642
643
644
645
        beam_width = params.beam_width
        max_tokens = params.max_tokens
        temperature = params.temperature
        ignore_eos = params.ignore_eos
646
647
        length_penalty = params.length_penalty

648
649
650
        lora_requests = self._get_beam_search_lora_requests(
            lora_request, prompts)

651
652
653
654
655
        tokenizer = self.get_tokenizer()
        sort_beams_key = create_sort_beams_key_function(
            tokenizer.eos_token_id,
            length_penalty,
        )
656

657
658
659
660
661
662
663
664
665
666
667
668
        def create_tokens_prompt_from_beam(
                beam: BeamSearchSequence) -> TokensPrompt:
            token_prompt_kwargs: TokensPrompt = {
                "prompt_token_ids": beam.tokens
            }
            if beam.multi_modal_data is not None:
                token_prompt_kwargs["multi_modal_data"] = beam.multi_modal_data

            if beam.mm_processor_kwargs is not None:
                token_prompt_kwargs[
                    "mm_processor_kwargs"] = beam.mm_processor_kwargs
            return TokensPrompt(**token_prompt_kwargs)
669

670
671
672
673
674
        # generate 2 * beam_width candidates at each step
        # following the huggingface transformers implementation
        # at https://github.com/huggingface/transformers/blob/e15687fffe5c9d20598a19aeab721ae0a7580f8a/src/transformers/generation/beam_search.py#L534 # noqa
        beam_search_params = SamplingParams(logprobs=2 * beam_width,
                                            max_tokens=1,
675
                                            temperature=temperature)
676
        instances: list[BeamSearchInstance] = []
677

678
        for lora_req, prompt in zip(lora_requests, prompts):
679
680
681
682
683
684
685
686
            # Add multimodal processor kwargs & data
            mm_kwargs = {}
            if "multi_modal_data" in prompt:
                mm_kwargs["multi_modal_data"] = prompt["multi_modal_data"]
            if "mm_processor_kwargs" in prompt:
                mm_kwargs["mm_processor_kwargs"] = prompt[
                    "mm_processor_kwargs"]

687
688
            if "prompt_token_ids" in prompt:
                prompt = cast(TokensPrompt, prompt)  # Needed for mypy
689
690
691
                prompt_tokens = prompt["prompt_token_ids"]
            else:
                prompt_tokens = tokenizer.encode(prompt["prompt"])
692

693
            instances.append(
694
695
696
697
698
699
                BeamSearchInstance(
                    prompt_tokens,
                    lora_request=lora_req,
                    logprobs=None,
                    **mm_kwargs,
                ), )
700

701
702
703
704
705
706
707
708
709
710
711
712
        token_iter = range(max_tokens)
        if use_tqdm:
            token_iter = tqdm(token_iter,
                              desc="Beam search",
                              unit="token",
                              unit_scale=False)
            logger.warning(
                "The progress bar shows the upper bound on token steps and "
                "may finish early due to stopping conditions. It does not "
                "reflect instance-level progress.")

        for _ in token_iter:
713
            all_beams: list[BeamSearchSequence] = list(
714
715
716
717
                sum((instance.beams for instance in instances), []))
            pos = [0] + list(
                itertools.accumulate(
                    len(instance.beams) for instance in instances))
718
            instance_start_and_end: list[tuple[int, int]] = list(
719
720
721
722
723
                zip(pos[:-1], pos[1:]))

            if len(all_beams) == 0:
                break

724
725
726
727
            # create the corresponding batch entries for prompt & optional lora
            prompts_batch, lora_req_batch = zip(
                *[(create_tokens_prompt_from_beam(beam), beam.lora_request)
                  for beam in all_beams])
728
729
730
731
732

            # only runs for one step
            # we don't need to use tqdm here
            output = self.generate(prompts_batch,
                                   sampling_params=beam_search_params,
733
734
                                   use_tqdm=False,
                                   lora_request=lora_req_batch)
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750

            for (start, end), instance in zip(instance_start_and_end,
                                              instances):
                instance_new_beams = []
                for i in range(start, end):
                    current_beam = all_beams[i]
                    result = output[i]

                    if result.outputs[0].logprobs is not None:
                        # if `result.outputs[0].logprobs` is None, it means
                        # the sequence is completed because of the max-model-len
                        # or abortion. we don't need to add it to the new beams.
                        logprobs = result.outputs[0].logprobs[0]
                        for token_id, logprob_obj in logprobs.items():
                            new_beam = BeamSearchSequence(
                                tokens=current_beam.tokens + [token_id],
751
                                logprobs=current_beam.logprobs + [logprobs],
752
                                lora_request=current_beam.lora_request,
753
                                cum_logprob=current_beam.cum_logprob +
754
755
756
757
                                logprob_obj.logprob,
                                multi_modal_data=current_beam.multi_modal_data,
                                mm_processor_kwargs=current_beam.
                                mm_processor_kwargs)
758
759
760
761
762
763
764

                            if token_id == tokenizer.eos_token_id and \
                                not ignore_eos:
                                instance.completed.append(new_beam)
                            else:
                                instance_new_beams.append(new_beam)
                sorted_beams = sorted(instance_new_beams,
765
                                      key=sort_beams_key,
766
767
768
769
770
771
772
                                      reverse=True)
                instance.beams = sorted_beams[:beam_width]

        outputs = []
        for instance in instances:
            instance.completed.extend(instance.beams)
            sorted_completed = sorted(instance.completed,
773
                                      key=sort_beams_key,
774
775
776
777
778
779
780
781
782
                                      reverse=True)
            best_beams = sorted_completed[:beam_width]

            for beam in best_beams:
                beam.text = tokenizer.decode(beam.tokens)
            outputs.append(BeamSearchOutput(sequences=best_beams))

        return outputs

nunjunj's avatar
nunjunj committed
783
784
    def chat(
        self,
785
786
        messages: Union[list[ChatCompletionMessageParam],
                        list[list[ChatCompletionMessageParam]]],
nunjunj's avatar
nunjunj committed
787
        sampling_params: Optional[Union[SamplingParams,
788
                                        list[SamplingParams]]] = None,
789
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
nunjunj's avatar
nunjunj committed
790
791
        lora_request: Optional[LoRARequest] = None,
        chat_template: Optional[str] = None,
792
        chat_template_content_format: ChatTemplateContentFormatOption = "auto",
793
        add_generation_prompt: bool = True,
794
        continue_final_message: bool = False,
795
        tools: Optional[list[dict[str, Any]]] = None,
796
        chat_template_kwargs: Optional[dict[str, Any]] = None,
797
798
        mm_processor_kwargs: Optional[dict[str, Any]] = None,
    ) -> list[RequestOutput]:
nunjunj's avatar
nunjunj committed
799
        """
800
        Generate responses for a chat conversation.
nunjunj's avatar
nunjunj committed
801

802
        The chat conversation is converted into a text prompt using the
803
        tokenizer and calls the [generate][] method to generate the
804
805
806
807
        responses.

        Multi-modal inputs can be passed in the same way you would pass them
        to the OpenAI API.
nunjunj's avatar
nunjunj committed
808
809

        Args:
810
811
            messages: A list of conversations or a single conversation.

812
813
                - Each conversation is represented as a list of messages.
                - Each message is a dictionary with 'role' and 'content' keys.
814

nunjunj's avatar
nunjunj committed
815
816
817
818
819
            sampling_params: The sampling parameters for text generation.
                If None, we use the default sampling parameters. When it
                is a single value, it is applied to every prompt. When it
                is a list, the list must have the same length as the
                prompts and it is paired one by one with the prompt.
820
821
822
823
            use_tqdm: If `True`, shows a tqdm progress bar.
                If a callable (e.g., `functools.partial(tqdm, leave=False)`),
                it is used to create the progress bar.
                If `False`, no progress bar is created.
nunjunj's avatar
nunjunj committed
824
825
            lora_request: LoRA request to use for generation, if any.
            chat_template: The template to use for structuring the chat.
826
                If not provided, the model's default chat template will be used.
827
828
            chat_template_content_format: The format to render message content.

829
830
831
832
833
                - "string" will render the content as a string.
                  Example: `"Who are you?"`
                - "openai" will render the content as a list of dictionaries,
                  similar to OpenAI schema.
                  Example: `[{"type": "text", "text": "Who are you?"}]`
834

835
            add_generation_prompt: If True, adds a generation template
nunjunj's avatar
nunjunj committed
836
                to each message.
837
            continue_final_message: If True, continues the final message in
838
                the conversation instead of starting a new one. Cannot be
839
                `True` if `add_generation_prompt` is also `True`.
840
841
            chat_template_kwargs: Additional kwargs to pass to the chat
                template.
842
843
            mm_processor_kwargs: Multimodal processor kwarg overrides for this
                chat request. Only used for offline requests.
nunjunj's avatar
nunjunj committed
844
845

        Returns:
846
            A list of `RequestOutput` objects containing the generated
nunjunj's avatar
nunjunj committed
847
848
            responses in the same order as the input messages.
        """
849
        list_of_messages: list[list[ChatCompletionMessageParam]]
nunjunj's avatar
nunjunj committed
850

851
852
        # Handle multi and single conversations
        if is_list_of(messages, list):
853
854
            # messages is list[list[...]]
            list_of_messages = cast(list[list[ChatCompletionMessageParam]],
855
                                    messages)
856
        else:
857
            # messages is list[...]
858
            list_of_messages = [
859
                cast(list[ChatCompletionMessageParam], messages)
860
            ]
861

862
        tokenizer = self.get_tokenizer(lora_request)
863
864
865
        model_config = self.llm_engine.get_model_config()
        resolved_content_format = resolve_chat_template_content_format(
            chat_template,
866
            tools,
867
868
            chat_template_content_format,
            tokenizer,
869
            model_config=model_config,
870
871
        )

872
873
874
875
876
877
878
879
        _chat_template_kwargs: dict[str, Any] = dict(
            chat_template=chat_template,
            add_generation_prompt=add_generation_prompt,
            continue_final_message=continue_final_message,
            tools=tools,
        )
        _chat_template_kwargs.update(chat_template_kwargs or {})

880
        prompts: list[Union[TokensPrompt, TextPrompt]] = []
881
882

        for msgs in list_of_messages:
883
884
885
            # NOTE: _parse_chat_message_content_parts() currently doesn't
            # handle mm_processor_kwargs, since there is no implementation in
            # the chat message parsing for it.
886
            conversation, mm_data = parse_chat_messages(
887
888
889
890
891
                msgs,
                model_config,
                tokenizer,
                content_format=resolved_content_format,
            )
892
893

            if isinstance(tokenizer, MistralTokenizer):
894
                prompt_token_ids = apply_mistral_chat_template(
895
896
                    tokenizer,
                    messages=msgs,
897
                    **_chat_template_kwargs,
898
899
                )
            else:
900
                prompt_str = apply_hf_chat_template(
901
                    tokenizer=tokenizer,
902
                    conversation=conversation,
903
                    model_config=model_config,
904
                    **_chat_template_kwargs,
905
                )
906
907
908
909
                # Special tokens are already included in chat templates so
                # should not be added by the tokenizer in this case.
                prompt_token_ids = tokenizer.encode(prompt_str,
                                                    add_special_tokens=False)
910

911
            prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
912
913
914
915

            if mm_data is not None:
                prompt["multi_modal_data"] = mm_data

916
917
918
            if mm_processor_kwargs is not None:
                prompt["mm_processor_kwargs"] = mm_processor_kwargs

919
            prompts.append(prompt)
920

nunjunj's avatar
nunjunj committed
921
        return self.generate(
922
            prompts,
923
            sampling_params=sampling_params,
nunjunj's avatar
nunjunj committed
924
925
926
927
            use_tqdm=use_tqdm,
            lora_request=lora_request,
        )

928
929
930
931
932
933
934
    @overload
    def encode(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
935
        *,
936
        truncate_prompt_tokens: Optional[int] = None,
937
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
938
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
939
        pooling_task: PoolingTask = "encode",
940
        tokenization_kwargs: Optional[dict[str, Any]] = None,
941
    ) -> list[PoolingRequestOutput]:
942
943
        ...

944
    @overload  # LEGACY: single (prompt + optional token ids)
945
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
946
947
948
949
950
    def encode(
        self,
        prompts: str,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
951
        prompt_token_ids: Optional[list[int]] = None,
952
        truncate_prompt_tokens: Optional[int] = None,
953
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
954
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
955
        pooling_task: PoolingTask = "encode",
956
        tokenization_kwargs: Optional[dict[str, Any]] = None,
957
    ) -> list[PoolingRequestOutput]:
958
        ...
959

960
    @overload  # LEGACY: multi (prompt + optional token ids)
961
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
962
963
    def encode(
        self,
964
        prompts: list[str],
965
        pooling_params: Optional[Union[PoolingParams,
966
                                       Sequence[PoolingParams]]] = None,
967
        prompt_token_ids: Optional[list[list[int]]] = None,
968
        truncate_prompt_tokens: Optional[int] = None,
969
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
970
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
971
        pooling_task: PoolingTask = "encode",
972
        tokenization_kwargs: Optional[dict[str, Any]] = None,
973
    ) -> list[PoolingRequestOutput]:
974
975
976
        ...

    @overload  # LEGACY: single (token ids + optional prompt)
977
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
978
979
980
981
982
983
    def encode(
        self,
        prompts: Optional[str] = None,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
        *,
984
        prompt_token_ids: list[int],
985
        truncate_prompt_tokens: Optional[int] = None,
986
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
987
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
988
        pooling_task: PoolingTask = "encode",
989
        tokenization_kwargs: Optional[dict[str, Any]] = None,
990
    ) -> list[PoolingRequestOutput]:
991
992
993
        ...

    @overload  # LEGACY: multi (token ids + optional prompt)
994
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
995
996
    def encode(
        self,
997
        prompts: Optional[list[str]] = None,
998
999
1000
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
        *,
1001
        prompt_token_ids: list[list[int]],
1002
        truncate_prompt_tokens: Optional[int] = None,
1003
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
1004
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
1005
        pooling_task: PoolingTask = "encode",
1006
        tokenization_kwargs: Optional[dict[str, Any]] = None,
1007
    ) -> list[PoolingRequestOutput]:
1008
1009
1010
        ...

    @overload  # LEGACY: single or multi token ids [pos-only]
1011
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
1012
1013
1014
1015
    def encode(
        self,
        prompts: None,
        pooling_params: None,
1016
        prompt_token_ids: Union[list[int], list[list[int]]],
1017
        truncate_prompt_tokens: Optional[int] = None,
1018
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
1019
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
1020
        pooling_task: PoolingTask = "encode",
1021
        tokenization_kwargs: Optional[dict[str, Any]] = None,
1022
    ) -> list[PoolingRequestOutput]:
1023
1024
        ...

nunjunj's avatar
nunjunj committed
1025
1026
1027
    @deprecate_kwargs(
        "prompt_token_ids",
        is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
1028
        additional_message="Please use the 'prompts' parameter instead.",
nunjunj's avatar
nunjunj committed
1029
    )
1030
1031
    def encode(
        self,
1032
        prompts: Union[Union[PromptType, Sequence[PromptType]],
1033
                       Optional[Union[str, list[str]]]] = None,
1034
1035
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
1036
        prompt_token_ids: Optional[Union[list[int], list[list[int]]]] = None,
1037
        truncate_prompt_tokens: Optional[int] = None,
1038
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
1039
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
1040
        pooling_task: PoolingTask = "encode",
1041
        tokenization_kwargs: Optional[dict[str, Any]] = None,
1042
    ) -> list[PoolingRequestOutput]:
1043
1044
        """Apply pooling to the hidden states corresponding to the input
        prompts.
1045

1046
        This class automatically batches the given prompts, considering
1047
1048
1049
1050
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
1051
            prompts: The prompts to the LLM. You may pass a sequence of prompts
1052
                for batch inference. See [PromptType][vllm.inputs.PromptType]
1053
                for more details about the format of each prompts.
1054
1055
            pooling_params: The pooling parameters for pooling. If None, we
                use the default pooling parameters.
1056
1057
1058
1059
            use_tqdm: If `True`, shows a tqdm progress bar.
                If a callable (e.g., `functools.partial(tqdm, leave=False)`),
                it is used to create the progress bar.
                If `False`, no progress bar is created.
1060
            lora_request: LoRA request to use for generation, if any.
1061
            pooling_task: Override the pooling task to use.
1062
1063

        Returns:
1064
            A list of `PoolingRequestOutput` objects containing the
1065
            pooled hidden states in the same order as the input prompts.
1066

1067
1068
1069
1070
        Note:
            Using `prompts` and `prompt_token_ids` as keyword parameters is
            considered legacy and may be deprecated in the future. You should
            instead pass them via the `inputs` parameter.
1071
        """
1072
1073
        model_config = self.llm_engine.model_config
        runner_type = model_config.runner_type
1074
        if runner_type != "pooling":
1075
1076
1077
1078
            raise ValueError(
                "LLM.encode() is only supported for pooling models. "
                "Try passing `--runner pooling` to use the model as a "
                "pooling model.")
1079

1080
        if prompt_token_ids is not None:
1081
            parsed_prompts = self._convert_v1_inputs(
1082
                prompts=cast(Optional[Union[str, list[str]]], prompts),
1083
1084
1085
                prompt_token_ids=prompt_token_ids,
            )
        else:
1086
1087
            parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
                                  prompts)
1088

1089
1090
1091
        if pooling_params is None:
            # Use default pooling params.
            pooling_params = PoolingParams()
1092
1093
1094

        if isinstance(pooling_params, PoolingParams):
            pooling_params.verify(pooling_task, model_config)
1095
1096
        else:
            for pooling_param in pooling_params:
1097
                pooling_param.verify(pooling_task, model_config)
1098

1099
1100
1101
1102
1103
        if tokenization_kwargs is None:
            tokenization_kwargs = dict[str, Any]()
            _validate_truncation_size(model_config.max_model_len,
                                      truncate_prompt_tokens,
                                      tokenization_kwargs)
1104

1105
        self._validate_and_add_requests(
1106
            prompts=parsed_prompts,
1107
            params=pooling_params,
1108
            use_tqdm=use_tqdm,
1109
            lora_request=lora_request,
1110
            tokenization_kwargs=tokenization_kwargs,
1111
1112
        )

1113
        outputs = self._run_engine(use_tqdm=use_tqdm)
Joe Runde's avatar
Joe Runde committed
1114
        return self.engine_class.validate_outputs(outputs,
1115
                                                  PoolingRequestOutput)
1116

1117
1118
1119
1120
1121
    def embed(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        *,
1122
        truncate_prompt_tokens: Optional[int] = None,
1123
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
1124
1125
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
1126
1127
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
    ) -> list[EmbeddingRequestOutput]:
1128
1129
1130
1131
1132
1133
1134
1135
1136
        """
        Generate an embedding vector for each prompt.

        This class automatically batches the given prompts, considering
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
            prompts: The prompts to the LLM. You may pass a sequence of prompts
1137
                for batch inference. See [PromptType][vllm.inputs.PromptType]
1138
                for more details about the format of each prompts.
1139
1140
            pooling_params: The pooling parameters for pooling. If None, we
                use the default pooling parameters.
1141
1142
1143
1144
            use_tqdm: If `True`, shows a tqdm progress bar.
                If a callable (e.g., `functools.partial(tqdm, leave=False)`),
                it is used to create the progress bar.
                If `False`, no progress bar is created.
1145
1146
1147
            lora_request: LoRA request to use for generation, if any.

        Returns:
1148
            A list of `EmbeddingRequestOutput` objects containing the
1149
1150
            embedding vectors in the same order as the input prompts.
        """
1151
        if "embed" not in self.supported_tasks:
1152
1153
1154
            raise ValueError(
                "Embedding API is not supported by this model. "
                "Try converting the model using `--convert embed`.")
1155

1156
1157
1158
1159
1160
1161
1162
1163
        items = self.encode(
            prompts,
            truncate_prompt_tokens=truncate_prompt_tokens,
            use_tqdm=use_tqdm,
            pooling_params=pooling_params,
            lora_request=lora_request,
            pooling_task="embed",
        )
1164
1165
1166
1167
1168
1169
1170
1171

        return [EmbeddingRequestOutput.from_base(item) for item in items]

    def classify(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        *,
1172
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
1173
1174
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
    ) -> list[ClassificationRequestOutput]:
1175
1176
1177
1178
1179
1180
1181
1182
1183
        """
        Generate class logits for each prompt.

        This class automatically batches the given prompts, considering
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
            prompts: The prompts to the LLM. You may pass a sequence of prompts
1184
                for batch inference. See [PromptType][vllm.inputs.PromptType]
1185
                for more details about the format of each prompts.
1186
1187
1188
1189
            use_tqdm: If `True`, shows a tqdm progress bar.
                If a callable (e.g., `functools.partial(tqdm, leave=False)`),
                it is used to create the progress bar.
                If `False`, no progress bar is created.
1190
1191
1192
            lora_request: LoRA request to use for generation, if any.

        Returns:
1193
            A list of `ClassificationRequestOutput` objects containing the
1194
1195
            embedding vectors in the same order as the input prompts.
        """
1196
        if "classify" not in self.supported_tasks:
1197
            raise ValueError(
1198
                "Classification API is not supported by this model. "
1199
                "Try converting the model using `--convert classify`.")
1200

1201
1202
1203
1204
1205
1206
        items = self.encode(
            prompts,
            use_tqdm=use_tqdm,
            lora_request=lora_request,
            pooling_task="classify",
        )
1207
1208
1209

        return [ClassificationRequestOutput.from_base(item) for item in items]

1210
1211
1212
    def _embedding_score(
        self,
        tokenizer: AnyTokenizer,
1213
1214
        text_1: list[Union[str, TextPrompt, TokensPrompt]],
        text_2: list[Union[str, TextPrompt, TokensPrompt]],
1215
        truncate_prompt_tokens: Optional[int] = None,
1216
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
1217
1218
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
    ) -> list[ScoringRequestOutput]:
1219

1220
        encoded_output: list[PoolingRequestOutput] = self.encode(
1221
            text_1 + text_2,
1222
            truncate_prompt_tokens=truncate_prompt_tokens,
1223
1224
            use_tqdm=use_tqdm,
            lora_request=lora_request,
1225
1226
            pooling_task="embed",
        )
1227

1228
        encoded_output_1: list[PoolingRequestOutput] = encoded_output[
1229
            0:len(text_1)]
1230
        encoded_output_2: list[PoolingRequestOutput] = encoded_output[
1231
            len(text_1):]
1232
1233
1234
1235

        if len(encoded_output_1) == 1:
            encoded_output_1 = encoded_output_1 * len(encoded_output_2)

1236
1237
1238
        scores = _cosine_similarity(tokenizer=tokenizer,
                                    embed_1=encoded_output_1,
                                    embed_2=encoded_output_2)
1239
1240
1241
1242
1243
1244
1245

        items = self.engine_class.validate_outputs(scores,
                                                   PoolingRequestOutput)
        return [ScoringRequestOutput.from_base(item) for item in items]

    def _cross_encoding_score(
        self,
1246
        tokenizer: AnyTokenizer,
1247
1248
        data_1: Union[list[str], list[ScoreContentPartParam]],
        data_2: Union[list[str], list[ScoreContentPartParam]],
1249
        truncate_prompt_tokens: Optional[int] = None,
1250
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
1251
1252
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
    ) -> list[ScoringRequestOutput]:
1253
        model_config = self.llm_engine.model_config
1254
1255
1256

        if isinstance(tokenizer, MistralTokenizer):
            raise ValueError(
1257
                "Score API is not supported for Mistral tokenizer")
1258

1259
1260
        if len(data_1) == 1:
            data_1 = data_1 * len(data_2)
1261

1262
        pooling_params = PoolingParams(task="score")
1263
        tokenization_kwargs: dict[str, Any] = {}
1264
1265

        _validate_truncation_size(model_config.max_model_len,
1266
                                  truncate_prompt_tokens, tokenization_kwargs)
1267
1268
1269

        parsed_prompts = []

1270
1271
        input_pairs = [(t1, t2) for t1, t2 in zip(data_1, data_2)]

1272
        if model_config.is_multimodal_model:
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
            for q, d in input_pairs:
                _, engine_prompt = get_score_prompt(
                    model_config=model_config,
                    data_1=q,
                    data_2=d,
                    tokenizer=tokenizer,
                    tokenization_kwargs=tokenization_kwargs,
                )

                parsed_prompts.append(engine_prompt)
        else:
            for q, t in input_pairs:
1285
                if model_config.use_pad_token:
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
                    # cross_encoder models defaults to using pad_token.
                    prompt_inputs = tokenizer(
                        text=q,  # type: ignore[arg-type]
                        text_pair=t,  # type: ignore[arg-type]
                        **tokenization_kwargs)
                else:
                    # `llm as reranker` models defaults to not using pad_token.
                    prompt_inputs = tokenizer(
                        text=q + t,  # type: ignore[operator]
                        **tokenization_kwargs)
                engine_prompt = TokensPrompt(
                    prompt_token_ids=prompt_inputs["input_ids"],
                    token_type_ids=prompt_inputs.get("token_type_ids"))
                parsed_prompts.append(engine_prompt)
1300
1301
1302
1303

        self._validate_and_add_requests(
            prompts=parsed_prompts,
            params=pooling_params,
1304
            use_tqdm=use_tqdm,
1305
1306
1307
1308
1309
1310
1311
1312
1313
            lora_request=lora_request,
        )

        outputs = self._run_engine(use_tqdm=use_tqdm)
        items = self.engine_class.validate_outputs(outputs,
                                                   PoolingRequestOutput)

        return [ScoringRequestOutput.from_base(item) for item in items]

1314
1315
    def score(
        self,
1316
1317
1318
1319
        data_1: Union[SingletonPrompt, Sequence[SingletonPrompt],
                      ScoreMultiModalParam],
        data_2: Union[SingletonPrompt, Sequence[SingletonPrompt],
                      ScoreMultiModalParam],
1320
        /,
1321
        *,
1322
        truncate_prompt_tokens: Optional[int] = None,
1323
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
1324
1325
        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
    ) -> list[ScoringRequestOutput]:
1326
1327
        """Generate similarity scores for all pairs `<text,text_pair>` or
          `<multi-modal data, multi-modal data pair>`.
1328

1329
        The inputs can be `1 -> 1`, `1 -> N` or `N -> N`.
1330
1331
        In the `1 - N` case the `data_1` input will be replicated `N`
        times to pair with the `data_2` inputs.
1332
        The input pairs are used to build a list of prompts for the
1333
1334
        cross encoder model. This class automatically batches the prompts,
        considering the memory constraint. For the best performance, put all
1335
1336
1337
        of your inputs into a single list and pass it to this method.

        Supports both text and multi-modal data (images, etc.) when used with
1338
        appropriate multi-modal models. For multi-modal inputs, ensure the
1339
        prompt structure matches the model's expected input format.
1340
1341

        Args:
1342
1343
1344
            data_1: Can be a single prompt, a list of prompts or
                `ScoreMultiModalParam`, which can contain either text or
                multi-modal data. When a list, it must have the same length as
1345
                the `data_2` list.
1346
            data_2: The data to pair with the query to form the input to
1347
                the LLM. Can be text or multi-modal data. See [PromptType]
1348
                [vllm.inputs.PromptType] for more details about the format of
1349
                each prompt.
1350
1351
1352
1353
            use_tqdm: If `True`, shows a tqdm progress bar.
                If a callable (e.g., `functools.partial(tqdm, leave=False)`),
                it is used to create the progress bar.
                If `False`, no progress bar is created.
1354
1355
1356
            lora_request: LoRA request to use for generation, if any.

        Returns:
1357
            A list of `ScoringRequestOutput` objects containing the
1358
1359
            generated scores in the same order as the input prompts.
        """
1360
1361
        model_config = self.llm_engine.model_config
        runner_type = model_config.runner_type
1362
        if runner_type != "pooling":
1363
1364
1365
1366
            raise ValueError(
                "LLM.score() is only supported for pooling models. "
                "Try passing `--runner pooling` to use the model as a "
                "pooling model.")
1367

1368
1369
        supported_tasks = self.supported_tasks
        if all(t not in supported_tasks for t in ("embed", "classify")):
1370
            raise ValueError("Score API is not supported by this model. "
1371
1372
                             "Try converting the model using "
                             "`--convert embed` or `--convert classify`.")
1373

1374
        if (model_config.is_cross_encoder
1375
                and getattr(model_config.hf_config, "num_labels", 0) != 1):
1376
            raise ValueError("Score API is only enabled for num_labels == 1.")
1377
1378
1379
1380

        # the tokenizer for models such as
        # "cross-encoder/ms-marco-MiniLM-L-6-v2" doesn't support passing
        # lists of tokens to the `text` and `text_pair` kwargs
1381
        tokenizer = self.get_tokenizer()
1382

1383
        if not model_config.is_multimodal_model:
1384
1385
1386
1387
1388

            def check_data_type(data: Union[SingletonPrompt,
                                            Sequence[SingletonPrompt],
                                            ScoreMultiModalParam]):
                if isinstance(data, dict) and "content" in data:
1389
1390
                    raise ValueError("ScoreMultiModalParam is not supported "
                                     f"for {model_config.architecture}")
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430

            check_data_type(data_1)
            check_data_type(data_2)

            def ensure_str(prompt: SingletonPrompt):
                if isinstance(prompt, dict):
                    if "multi_modal_data" in prompt:
                        raise ValueError("Multi-modal prompt is not "
                                         "supported for scoring")
                    elif "prompt_token_ids" in prompt:
                        prompt = tokenizer.decode(
                            cast(TokensPrompt, prompt)["prompt_token_ids"])
                    elif "prompt" in prompt:
                        prompt = cast(TextPrompt, prompt)["prompt"]
                assert type(prompt) is str
                return prompt

            if isinstance(data_1, (str, dict)):
                # Convert a single prompt to a list.
                data_1 = [data_1]  # type: ignore[list-item]

            data_1 = [ensure_str(t) for t in data_1]

            if isinstance(data_2, (str, dict)):
                # Convert a single prompt to a list.
                data_2 = [data_2]  # type: ignore[list-item]

            data_2 = [ensure_str(t) for t in data_2]

        if isinstance(data_1, dict) and "content" in data_1:
            data_1 = data_1.get("content")  # type: ignore[assignment]
        elif isinstance(data_1, str):
            data_1 = [data_1]

        if isinstance(data_2, dict) and "content" in data_2:
            data_2 = data_2.get("content")  # type: ignore[assignment]
        elif isinstance(data_2, str):
            data_2 = [data_2]

        _validate_score_input_lens(data_1, data_2)  # type: ignore[arg-type]
1431

1432
        if model_config.is_cross_encoder:
1433
1434
1435
1436
1437
1438
            return self._cross_encoding_score(
                tokenizer,
                data_1,  # type: ignore[arg-type]
                data_2,  # type: ignore[arg-type]
                truncate_prompt_tokens,
                use_tqdm,
1439
                lora_request)
1440
        else:
1441
1442
            return self._embedding_score(
                tokenizer,
1443
1444
                data_1,  # type: ignore[arg-type]
                data_2,  # type: ignore[arg-type]
1445
1446
                truncate_prompt_tokens,
                use_tqdm,
1447
                lora_request)
1448

1449
1450
1451
1452
1453
1454
    def start_profile(self) -> None:
        self.llm_engine.start_profile()

    def stop_profile(self) -> None:
        self.llm_engine.stop_profile()

1455
1456
    def reset_prefix_cache(self, device: Optional[Device] = None) -> bool:
        return self.llm_engine.reset_prefix_cache(device)
1457

1458
1459
1460
1461
1462
1463
    def sleep(self, level: int = 1):
        """
        Put the engine to sleep. The engine should not process any requests.
        The caller should guarantee that no requests are being processed
        during the sleep period, before `wake_up` is called.

1464
        Args:
1465
1466
            level: The sleep level. Level 1 sleep will offload the model
                weights and discard the kv cache. The content of kv cache
1467
                is forgotten. Level 1 sleep is good for sleeping and waking
1468
1469
1470
1471
1472
                up the engine to run the same model again. The model weights
                are backed up in CPU memory. Please make sure there's enough
                CPU memory to store the model weights. Level 2 sleep will
                discard both the model weights and the kv cache. The content
                of both the model weights and kv cache is forgotten. Level 2
1473
                sleep is good for sleeping and waking up the engine to run a
1474
                different model or update the model, where previous model
1475
                weights are not needed. It reduces CPU memory pressure.
1476
        """
1477
        self.reset_prefix_cache()
1478
1479
        self.llm_engine.sleep(level=level)

1480
    def wake_up(self, tags: Optional[list[str]] = None):
1481
        """
1482
        Wake up the engine from sleep mode. See the [sleep][] method
1483
        for more details.
1484

1485
        Args:
1486
1487
            tags: An optional list of tags to reallocate the engine memory
                for specific memory allocations. Values must be in
1488
                `("weights", "kv_cache")`. If None, all memory is reallocated.
1489
                wake_up should be called with all tags (or None) before the
1490
1491
1492
                engine is used again.
        """
        self.llm_engine.wake_up(tags)
1493

1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
    def get_metrics(self) -> list["Metric"]:
        """Return a snapshot of aggregated metrics from Prometheus.

        Returns:
            A ``MetricSnapshot`` instance capturing the current state
            of all aggregated metrics from Prometheus.

        Note:
            This method is only available with the V1 LLM engine.
        """
        from vllm.v1.engine.llm_engine import LLMEngine as V1LLMEngine
        assert isinstance(self.llm_engine, V1LLMEngine)
        return self.llm_engine.get_metrics()

1508
1509
    # LEGACY
    def _convert_v1_inputs(
1510
        self,
1511
1512
        prompts: Optional[Union[str, list[str]]],
        prompt_token_ids: Optional[Union[list[int], list[list[int]]]],
1513
1514
    ):
        # skip_tokenizer_init is now checked in engine
1515

1516
1517
1518
1519
1520
1521
1522
1523
1524
        if prompts is None and prompt_token_ids is None:
            raise ValueError(
                "Either prompts or prompt_token_ids must be provided.")
        if prompts is not None and prompt_token_ids is not None \
                and len(prompts) != len(prompt_token_ids):
            raise ValueError(
                "The lengths of prompts and prompt_token_ids must be the same."
            )

1525
1526
1527
1528
1529
1530
        if prompts is not None:
            prompts = [p["content"] for p in parse_and_batch_prompt(prompts)]
        if prompt_token_ids is not None:
            prompt_token_ids = [
                p["content"] for p in parse_and_batch_prompt(prompt_token_ids)
            ]
1531
1532
        if prompts is not None:
            num_requests = len(prompts)
1533
        elif prompt_token_ids is not None:
1534
            num_requests = len(prompt_token_ids)
1535
        parsed_prompts: list[PromptType] = []
1536
        for i in range(num_requests):
1537
            item: PromptType
1538

1539
            if prompts is not None:
1540
1541
1542
                item = TextPrompt(prompt=prompts[i])
            elif prompt_token_ids is not None:
                item = TokensPrompt(prompt_token_ids=prompt_token_ids[i])
1543
            else:
1544
                raise AssertionError
1545

1546
            parsed_prompts.append(item)
1547

1548
        return parsed_prompts
1549
1550
1551

    def _validate_and_add_requests(
        self,
1552
        prompts: Union[PromptType, Sequence[PromptType]],
1553
1554
        params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
                      Sequence[PoolingParams]],
1555
        *,
1556
        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
1557
        lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
1558
        tokenization_kwargs: Optional[dict[str, Any]] = None,
1559
        priority: Optional[list[int]] = None,
1560
    ) -> None:
1561
        if isinstance(prompts, (str, dict)):
1562
            # Convert a single prompt to a list.
1563
            prompts = [prompts]
1564

1565
        num_requests = len(prompts)
1566
        if isinstance(params, Sequence) and len(params) != num_requests:
1567
            raise ValueError("The lengths of prompts and params "
1568
                             "must be the same.")
1569
        if isinstance(lora_request,
1570
                      Sequence) and len(lora_request) != num_requests:
1571
1572
            raise ValueError("The lengths of prompts and lora_request "
                             "must be the same.")
1573

1574
        for sp in params if isinstance(params, Sequence) else (params, ):
1575
1576
1577
            if isinstance(sp, SamplingParams):
                # We only care about the final output
                sp.output_kind = RequestOutputKind.FINAL_ONLY
1578

Zhuohan Li's avatar
Zhuohan Li committed
1579
        # Add requests to the engine.
1580
1581
        it = prompts
        if use_tqdm:
1582
1583
            tqdm_func = use_tqdm if callable(use_tqdm) else tqdm
            it = tqdm_func(it, desc="Adding requests")
1584
1585

        for i, prompt in enumerate(it):
1586
            self._add_request(
1587
                prompt,
1588
                params[i] if isinstance(params, Sequence) else params,
1589
                tokenization_kwargs=tokenization_kwargs,
1590
1591
                lora_request=lora_request[i] if isinstance(
                    lora_request, Sequence) else lora_request,
1592
                priority=priority[i] if priority else 0,
nunjunj's avatar
nunjunj committed
1593
            )
1594

1595
    def _add_request(
nunjunj's avatar
nunjunj committed
1596
        self,
1597
        prompt: PromptType,
nunjunj's avatar
nunjunj committed
1598
        params: Union[SamplingParams, PoolingParams],
1599
        tokenization_kwargs: Optional[dict[str, Any]] = None,
1600
        lora_request: Optional[LoRARequest] = None,
1601
        priority: int = 0,
1602
1603
    ) -> None:
        request_id = str(next(self.request_counter))
1604
1605
        self.llm_engine.add_request(
            request_id,
1606
            prompt,
1607
1608
            params,
            lora_request=lora_request,
1609
            tokenization_kwargs=tokenization_kwargs,
1610
            priority=priority,
nunjunj's avatar
nunjunj committed
1611
        )
1612

1613
    def _run_engine(
1614
1615
1616
        self,
        *,
        use_tqdm: Union[bool, Callable[..., tqdm]] = True
1617
    ) -> list[Union[RequestOutput, PoolingRequestOutput]]:
1618
1619
        # Initialize tqdm.
        if use_tqdm:
Zhuohan Li's avatar
Zhuohan Li committed
1620
            num_requests = self.llm_engine.get_num_unfinished_requests()
1621
1622
            tqdm_func = use_tqdm if callable(use_tqdm) else tqdm
            pbar = tqdm_func(
1623
1624
1625
                total=num_requests,
                desc="Processed prompts",
                dynamic_ncols=True,
1626
1627
                postfix=(f"est. speed input: {0:.2f} toks/s, "
                         f"output: {0:.2f} toks/s"),
1628
            )
1629

Zhuohan Li's avatar
Zhuohan Li committed
1630
        # Run the engine.
1631
        outputs: list[Union[RequestOutput, PoolingRequestOutput]] = []
1632
1633
        total_in_toks = 0
        total_out_toks = 0
Zhuohan Li's avatar
Zhuohan Li committed
1634
1635
        while self.llm_engine.has_unfinished_requests():
            step_outputs = self.llm_engine.step()
1636
            for output in step_outputs:
1637
                if output.finished:
1638
1639
                    outputs.append(output)
                    if use_tqdm:
1640
1641
                        if isinstance(output, RequestOutput):
                            # Calculate tokens only for RequestOutput
1642
                            n = len(output.outputs)
1643
                            assert output.prompt_token_ids is not None
1644
                            total_in_toks += len(output.prompt_token_ids) * n
1645
1646
                            in_spd = total_in_toks / pbar.format_dict["elapsed"]
                            total_out_toks += sum(
1647
                                len(stp.token_ids) for stp in output.outputs)
nunjunj's avatar
nunjunj committed
1648
1649
                            out_spd = (total_out_toks /
                                       pbar.format_dict["elapsed"])
1650
1651
1652
                            pbar.postfix = (
                                f"est. speed input: {in_spd:.2f} toks/s, "
                                f"output: {out_spd:.2f} toks/s")
1653
                            pbar.update(n)
1654
1655
                        else:
                            pbar.update(1)
1656
1657
                        if pbar.n == num_requests:
                            pbar.refresh()
1658

1659
1660
        if use_tqdm:
            pbar.close()
1661
1662
1663
        # Sort the outputs by request ID.
        # This is necessary because some requests may be finished earlier than
        # its previous requests.
1664
        return sorted(outputs, key=lambda x: int(x.request_id))