llm.py 37.9 KB
Newer Older
1
import itertools
2
import warnings
3
from contextlib import contextmanager
4
5
from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Tuple,
                    Union, cast, overload)
6

7
from tqdm import tqdm
8

9
10
from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
                              BeamSearchSequence, get_beam_search_score)
Woosuk Kwon's avatar
Woosuk Kwon committed
11
12
from vllm.engine.arg_utils import EngineArgs
from vllm.engine.llm_engine import LLMEngine
nunjunj's avatar
nunjunj committed
13
from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
14
15
                                         apply_hf_chat_template,
                                         apply_mistral_chat_template,
nunjunj's avatar
nunjunj committed
16
                                         parse_chat_messages)
17
from vllm.inputs import PromptType, TextPrompt, TokensPrompt
18
from vllm.inputs.parse import parse_and_batch_prompt
19
from vllm.logger import init_logger
20
from vllm.lora.request import LoRARequest
21
22
from vllm.model_executor.guided_decoding.guided_fields import (
    GuidedDecodingRequest, LLMGuidedOptions)
23
24
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
from vllm.pooling_params import PoolingParams
25
from vllm.prompt_adapter.request import PromptAdapterRequest
26
27
from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
                                  RequestOutputKind, SamplingParams)
28
from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
29
30
                                               get_cached_tokenizer)
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
yhu422's avatar
yhu422 committed
31
from vllm.usage.usage_lib import UsageContext
32
from vllm.utils import Counter, deprecate_kwargs, is_list_of
33

34
35
logger = init_logger(__name__)

36
37

class LLM:
Woosuk Kwon's avatar
Woosuk Kwon committed
38
39
40
41
42
43
44
45
46
47
    """An LLM for generating texts from given prompts and sampling parameters.

    This class includes a tokenizer, a language model (possibly distributed
    across multiple GPUs), and GPU memory space allocated for intermediate
    states (aka KV cache). Given a batch of prompts and sampling parameters,
    this class generates texts from the model, using an intelligent batching
    mechanism and efficient memory management.

    Args:
        model: The name or path of a HuggingFace Transformers model.
48
        tokenizer: The name or path of a HuggingFace Transformers tokenizer.
49
50
        tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer
            if available, and "slow" will always use the slow tokenizer.
51
52
53
        skip_tokenizer_init: If true, skip initialization of tokenizer and
            detokenizer. Expect valid prompt_token_ids and None for prompt
            from the input.
54
55
        trust_remote_code: Trust remote code (e.g., from HuggingFace) when
            downloading the model and tokenizer.
Woosuk Kwon's avatar
Woosuk Kwon committed
56
57
58
        tensor_parallel_size: The number of GPUs to use for distributed
            execution with tensor parallelism.
        dtype: The data type for the model weights and activations. Currently,
Woosuk Kwon's avatar
Woosuk Kwon committed
59
60
61
62
            we support `float32`, `float16`, and `bfloat16`. If `auto`, we use
            the `torch_dtype` attribute specified in the model config file.
            However, if the `torch_dtype` in the config is `float32`, we will
            use `float16` instead.
63
        quantization: The method used to quantize the model weights. Currently,
64
            we support "awq", "gptq", and "fp8" (experimental).
65
66
67
68
            If None, we first check the `quantization_config` attribute in the
            model config file. If that is None, we assume the model weights are
            not quantized and use `dtype` to determine the data type of
            the weights.
Jasmond L's avatar
Jasmond L committed
69
70
        revision: The specific model version to use. It can be a branch name,
            a tag name, or a commit id.
71
72
        tokenizer_revision: The specific tokenizer version to use. It can be a
            branch name, a tag name, or a commit id.
73
74
75
76
77
78
79
80
81
82
83
        seed: The seed to initialize the random number generator for sampling.
        gpu_memory_utilization: The ratio (between 0 and 1) of GPU memory to
            reserve for the model weights, activations, and KV cache. Higher
            values will increase the KV cache size and thus improve the model's
            throughput. However, if the value is too high, it may cause out-of-
            memory (OOM) errors.
        swap_space: The size (GiB) of CPU memory per GPU to use as swap space.
            This can be used for temporarily storing the states of the requests
            when their `best_of` sampling parameters are larger than 1. If all
            requests will have `best_of=1`, you can safely set this to 0.
            Otherwise, too small values may cause out-of-memory (OOM) errors.
84
85
86
87
        cpu_offload_gb: The size (GiB) of CPU memory to use for offloading
            the model weights. This virtually increases the GPU memory space
            you can use to hold the model weights, at the cost of CPU-GPU data
            transfer for every forward pass.
88
89
90
91
        enforce_eager: Whether to enforce eager execution. If True, we will
            disable CUDA graph and always execute the model in eager mode.
            If False, we will use CUDA graph and eager execution in hybrid.
        max_context_len_to_capture: Maximum context len covered by CUDA graphs.
92
93
94
            When a sequence has context length larger than this, we fall back
            to eager mode (DEPRECATED. Use `max_seq_len_to_capture` instead).
        max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs.
95
            When a sequence has context length larger than this, we fall back
96
97
98
            to eager mode. Additionally for encoder-decoder models, if the
            sequence length of the encoder input is larger than this, we fall
            back to the eager mode.
99
        disable_custom_all_reduce: See ParallelConfig
100
101
        **kwargs: Arguments for :class:`~vllm.EngineArgs`. (See
            :ref:`engine_args`)
nunjunj's avatar
nunjunj committed
102

103
104
105
    Note:
        This class is intended to be used for offline inference. For online
        serving, use the :class:`~vllm.AsyncLLMEngine` class instead.
Woosuk Kwon's avatar
Woosuk Kwon committed
106
    """
107

108
109
110
111
112
113
114
115
116
117
118
119
    DEPRECATE_LEGACY: ClassVar[bool] = False
    """A flag to toggle whether to deprecate the legacy generate/encode API."""

    @classmethod
    @contextmanager
    def deprecate_legacy_api(cls):
        cls.DEPRECATE_LEGACY = True

        yield

        cls.DEPRECATE_LEGACY = False

120
121
122
    def __init__(
        self,
        model: str,
123
        tokenizer: Optional[str] = None,
124
        tokenizer_mode: str = "auto",
125
        skip_tokenizer_init: bool = False,
126
        trust_remote_code: bool = False,
127
        tensor_parallel_size: int = 1,
Woosuk Kwon's avatar
Woosuk Kwon committed
128
        dtype: str = "auto",
129
        quantization: Optional[str] = None,
130
        revision: Optional[str] = None,
131
        tokenizer_revision: Optional[str] = None,
132
133
        seed: int = 0,
        gpu_memory_utilization: float = 0.9,
134
        swap_space: float = 4,
135
        cpu_offload_gb: float = 0,
136
        enforce_eager: Optional[bool] = None,
137
138
        max_context_len_to_capture: Optional[int] = None,
        max_seq_len_to_capture: int = 8192,
139
        disable_custom_all_reduce: bool = False,
140
        disable_async_output_proc: bool = False,
141
        mm_processor_kwargs: Optional[Dict[str, Any]] = None,
142
143
        **kwargs,
    ) -> None:
144
145
146
147
        '''
        LLM constructor.

        Note: if enforce_eager is unset (enforce_eager is None)
148
        it defaults to False.
149
150
        '''

151
152
        if "disable_log_stats" not in kwargs:
            kwargs["disable_log_stats"] = True
153

Zhuohan Li's avatar
Zhuohan Li committed
154
        engine_args = EngineArgs(
155
            model=model,
156
            tokenizer=tokenizer,
157
            tokenizer_mode=tokenizer_mode,
158
            skip_tokenizer_init=skip_tokenizer_init,
159
            trust_remote_code=trust_remote_code,
160
161
            tensor_parallel_size=tensor_parallel_size,
            dtype=dtype,
162
            quantization=quantization,
163
            revision=revision,
164
            tokenizer_revision=tokenizer_revision,
165
166
167
            seed=seed,
            gpu_memory_utilization=gpu_memory_utilization,
            swap_space=swap_space,
168
            cpu_offload_gb=cpu_offload_gb,
169
170
            enforce_eager=enforce_eager,
            max_context_len_to_capture=max_context_len_to_capture,
171
            max_seq_len_to_capture=max_seq_len_to_capture,
172
            disable_custom_all_reduce=disable_custom_all_reduce,
173
            disable_async_output_proc=disable_async_output_proc,
174
            mm_processor_kwargs=mm_processor_kwargs,
175
176
            **kwargs,
        )
yhu422's avatar
yhu422 committed
177
178
        self.llm_engine = LLMEngine.from_engine_args(
            engine_args, usage_context=UsageContext.LLM_CLASS)
179
180
        self.request_counter = Counter()

181
182
183
184
185
    def get_tokenizer(self) -> AnyTokenizer:
        return self.llm_engine.get_tokenizer_group(TokenizerGroup).tokenizer

    def set_tokenizer(self, tokenizer: AnyTokenizer) -> None:
        tokenizer_group = self.llm_engine.get_tokenizer_group(TokenizerGroup)
186

187
188
189
190
        # While CachedTokenizer is dynamic, have no choice but
        # compare class name. Misjudgment will arise from
        # user-defined tokenizer started with 'Cached'
        if tokenizer.__class__.__name__.startswith("Cached"):
191
            tokenizer_group.tokenizer = tokenizer
192
        else:
193
            tokenizer_group.tokenizer = get_cached_tokenizer(tokenizer)
194

195
196
197
198
199
200
201
202
    @overload  # LEGACY: single (prompt + optional token ids)
    def generate(
        self,
        prompts: str,
        sampling_params: Optional[Union[SamplingParams,
                                        List[SamplingParams]]] = None,
        prompt_token_ids: Optional[List[int]] = None,
        use_tqdm: bool = True,
203
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
204
205
206
207
    ) -> List[RequestOutput]:
        ...

    @overload  # LEGACY: multi (prompt + optional token ids)
208
209
    def generate(
        self,
210
        prompts: List[str],
211
212
        sampling_params: Optional[Union[SamplingParams,
                                        List[SamplingParams]]] = None,
213
        prompt_token_ids: Optional[List[List[int]]] = None,
214
        use_tqdm: bool = True,
215
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
216
217
218
219
220
221
222
223
224
225
226
227
    ) -> List[RequestOutput]:
        ...

    @overload  # LEGACY: single (token ids + optional prompt)
    def generate(
        self,
        prompts: Optional[str] = None,
        sampling_params: Optional[Union[SamplingParams,
                                        List[SamplingParams]]] = None,
        *,
        prompt_token_ids: List[int],
        use_tqdm: bool = True,
228
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
229
230
231
232
233
234
235
236
237
238
239
240
    ) -> List[RequestOutput]:
        ...

    @overload  # LEGACY: multi (token ids + optional prompt)
    def generate(
        self,
        prompts: Optional[List[str]] = None,
        sampling_params: Optional[Union[SamplingParams,
                                        List[SamplingParams]]] = None,
        *,
        prompt_token_ids: List[List[int]],
        use_tqdm: bool = True,
241
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
242
243
244
245
246
247
248
249
250
251
    ) -> List[RequestOutput]:
        ...

    @overload  # LEGACY: single or multi token ids [pos-only]
    def generate(
        self,
        prompts: None,
        sampling_params: None,
        prompt_token_ids: Union[List[int], List[List[int]]],
        use_tqdm: bool = True,
252
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
253
254
255
256
257
258
    ) -> List[RequestOutput]:
        ...

    @overload
    def generate(
        self,
259
260
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
261
262
263
264
        *,
        sampling_params: Optional[Union[SamplingParams,
                                        Sequence[SamplingParams]]] = None,
        use_tqdm: bool = True,
265
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
266
267
268
    ) -> List[RequestOutput]:
        ...

nunjunj's avatar
nunjunj committed
269
270
271
    @deprecate_kwargs(
        "prompt_token_ids",
        is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
272
        additional_message="Please use the 'prompts' parameter instead.",
nunjunj's avatar
nunjunj committed
273
    )
274
275
    def generate(
        self,
276
        prompts: Union[Union[PromptType, Sequence[PromptType]],
277
278
279
280
281
                       Optional[Union[str, List[str]]]] = None,
        sampling_params: Optional[Union[SamplingParams,
                                        Sequence[SamplingParams]]] = None,
        prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
        use_tqdm: bool = True,
282
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
283
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
284
        guided_options_request: Optional[Union[LLMGuidedOptions,
285
286
                                               GuidedDecodingRequest]] = None,
        priority: Optional[List[int]] = None,
287
    ) -> List[RequestOutput]:
Woosuk Kwon's avatar
Woosuk Kwon committed
288
289
        """Generates the completions for the input prompts.

290
        This class automatically batches the given prompts, considering
Woosuk Kwon's avatar
Woosuk Kwon committed
291
292
293
294
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
295
296
297
            prompts: The prompts to the LLM. You may pass a sequence of prompts
                for batch inference. See :class:`~vllm.inputs.PromptType`
                for more details about the format of each prompts.
Woosuk Kwon's avatar
Woosuk Kwon committed
298
            sampling_params: The sampling parameters for text generation. If
nunjunj's avatar
nunjunj committed
299
300
301
                None, we use the default sampling parameters.
                When it is a single value, it is applied to every prompt.
                When it is a list, the list must have the same length as the
302
                prompts and it is paired one by one with the prompt.
Woosuk Kwon's avatar
Woosuk Kwon committed
303
            use_tqdm: Whether to use tqdm to display the progress bar.
304
            lora_request: LoRA request to use for generation, if any.
nunjunj's avatar
nunjunj committed
305
            prompt_adapter_request: Prompt Adapter request to use for
306
                generation, if any.
307
308
            priority: The priority of the requests, if any.
                Only applicable when priority scheduling policy is enabled.
Woosuk Kwon's avatar
Woosuk Kwon committed
309
310

        Returns:
nunjunj's avatar
nunjunj committed
311
            A list of ``RequestOutput`` objects containing the
312
            generated completions in the same order as the input prompts.
313
314
315
316
317

        Note:
            Using ``prompts`` and ``prompt_token_ids`` as keyword parameters is
            considered legacy and may be deprecated in the future. You should
            instead pass them via the ``inputs`` parameter.
318
        """
319
320
        if self.llm_engine.model_config.embedding_mode:
            raise ValueError(
321
322
                "LLM.generate() is only supported for (conditional) generation "
                "models (XForCausalLM, XForConditionalGeneration).")
323

324
        if prompt_token_ids is not None:
325
            parsed_prompts = self._convert_v1_inputs(
326
327
328
329
                prompts=cast(Optional[Union[str, List[str]]], prompts),
                prompt_token_ids=prompt_token_ids,
            )
        else:
330
331
            parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
                                  prompts)
332

333
334
335
336
337
338
339
340
        if isinstance(guided_options_request, dict):
            if len(guided_options_request) > 1:
                raise ValueError(
                    "You can only use one guided decoding but multiple is "
                    f"specified: {guided_options_request}")
            guided_options_request = GuidedDecodingRequest(
                **guided_options_request)

341
342
343
344
        if sampling_params is None:
            # Use default sampling params.
            sampling_params = SamplingParams()

345
        self._validate_and_add_requests(
346
            prompts=parsed_prompts,
347
348
            params=sampling_params,
            lora_request=lora_request,
349
            prompt_adapter_request=prompt_adapter_request,
350
351
            guided_options=guided_options_request,
            priority=priority)
352

353
354
        outputs = self._run_engine(use_tqdm=use_tqdm)
        return LLMEngine.validate_outputs(outputs, RequestOutput)
355

356
357
358
    def beam_search(
        self,
        prompts: List[Union[str, List[int]]],
359
        params: BeamSearchParams,
360
361
362
363
364
365
366
    ) -> List[BeamSearchOutput]:
        """
        Generate sequences using beam search.

        Args:
            prompts: A list of prompts. Each prompt can be a string or a list
                of token IDs.
367
368
            params: The beam search parameters.

369
370
371
372
        TODO: how does beam search work together with length penalty, frequency
        penalty, and stopping criteria, etc.?
        """

373
374
375
376
        beam_width = params.beam_width
        max_tokens = params.max_tokens
        temperature = params.temperature
        ignore_eos = params.ignore_eos
377
378
379
380
381
382
        length_penalty = params.length_penalty

        def sort_beams_key(x: BeamSearchSequence) -> float:
            return get_beam_search_score(x.tokens, x.cum_logprob,
                                         tokenizer.eos_token_id,
                                         length_penalty)
383

384
385
386
387
388
389
        tokenizer = self.get_tokenizer()
        # generate 2 * beam_width candidates at each step
        # following the huggingface transformers implementation
        # at https://github.com/huggingface/transformers/blob/e15687fffe5c9d20598a19aeab721ae0a7580f8a/src/transformers/generation/beam_search.py#L534 # noqa
        beam_search_params = SamplingParams(logprobs=2 * beam_width,
                                            max_tokens=1,
390
                                            temperature=temperature)
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
        instances: List[BeamSearchInstance] = []

        for prompt in prompts:
            prompt_tokens = prompt if isinstance(
                prompt, list) else tokenizer.encode(prompt)
            instances.append(BeamSearchInstance(prompt_tokens))

        for _ in range(max_tokens):
            all_beams: List[BeamSearchSequence] = list(
                sum((instance.beams for instance in instances), []))
            pos = [0] + list(
                itertools.accumulate(
                    len(instance.beams) for instance in instances))
            instance_start_and_end: List[Tuple[int, int]] = list(
                zip(pos[:-1], pos[1:]))

            if len(all_beams) == 0:
                break

            prompts_batch = [
                TokensPrompt(prompt_token_ids=beam.tokens)
                for beam in all_beams
            ]

            # only runs for one step
            # we don't need to use tqdm here
            output = self.generate(prompts_batch,
                                   sampling_params=beam_search_params,
                                   use_tqdm=False)

            for (start, end), instance in zip(instance_start_and_end,
                                              instances):
                instance_new_beams = []
                for i in range(start, end):
                    current_beam = all_beams[i]
                    result = output[i]

                    if result.outputs[0].logprobs is not None:
                        # if `result.outputs[0].logprobs` is None, it means
                        # the sequence is completed because of the max-model-len
                        # or abortion. we don't need to add it to the new beams.
                        logprobs = result.outputs[0].logprobs[0]
                        for token_id, logprob_obj in logprobs.items():
                            new_beam = BeamSearchSequence(
                                tokens=current_beam.tokens + [token_id],
436
                                logprobs=current_beam.logprobs + [logprobs],
437
438
439
440
441
442
443
444
445
                                cum_logprob=current_beam.cum_logprob +
                                logprob_obj.logprob)

                            if token_id == tokenizer.eos_token_id and \
                                not ignore_eos:
                                instance.completed.append(new_beam)
                            else:
                                instance_new_beams.append(new_beam)
                sorted_beams = sorted(instance_new_beams,
446
                                      key=sort_beams_key,
447
448
449
450
451
452
453
                                      reverse=True)
                instance.beams = sorted_beams[:beam_width]

        outputs = []
        for instance in instances:
            instance.completed.extend(instance.beams)
            sorted_completed = sorted(instance.completed,
454
                                      key=sort_beams_key,
455
456
457
458
459
460
461
462
463
                                      reverse=True)
            best_beams = sorted_completed[:beam_width]

            for beam in best_beams:
                beam.text = tokenizer.decode(beam.tokens)
            outputs.append(BeamSearchOutput(sequences=best_beams))

        return outputs

nunjunj's avatar
nunjunj committed
464
465
    def chat(
        self,
466
467
        messages: Union[List[ChatCompletionMessageParam],
                        List[List[ChatCompletionMessageParam]]],
nunjunj's avatar
nunjunj committed
468
469
470
471
472
        sampling_params: Optional[Union[SamplingParams,
                                        List[SamplingParams]]] = None,
        use_tqdm: bool = True,
        lora_request: Optional[LoRARequest] = None,
        chat_template: Optional[str] = None,
473
        add_generation_prompt: bool = True,
474
        continue_final_message: bool = False,
475
        tools: Optional[List[Dict[str, Any]]] = None,
476
        mm_processor_kwargs: Optional[Dict[str, Any]] = None,
nunjunj's avatar
nunjunj committed
477
478
    ) -> List[RequestOutput]:
        """
479
        Generate responses for a chat conversation.
nunjunj's avatar
nunjunj committed
480

481
482
483
484
485
486
        The chat conversation is converted into a text prompt using the
        tokenizer and calls the :meth:`generate` method to generate the
        responses.

        Multi-modal inputs can be passed in the same way you would pass them
        to the OpenAI API.
nunjunj's avatar
nunjunj committed
487
488

        Args:
489
490
491
            messages: A list of conversations or a single conversation. 
                - Each conversation is represented as a list of messages.
                - Each message is a dictionary with 'role' and 'content' keys.
nunjunj's avatar
nunjunj committed
492
493
494
495
496
497
498
499
500
            sampling_params: The sampling parameters for text generation.
                If None, we use the default sampling parameters. When it
                is a single value, it is applied to every prompt. When it
                is a list, the list must have the same length as the
                prompts and it is paired one by one with the prompt.
            use_tqdm: Whether to use tqdm to display the progress bar.
            lora_request: LoRA request to use for generation, if any.
            chat_template: The template to use for structuring the chat.
              If not provided, the model's default chat template will be used.
501
            add_generation_prompt: If True, adds a generation template
nunjunj's avatar
nunjunj committed
502
                to each message.
503
504
505
            continue_final_message: If True, continues the final message in
                the conversation instead of starting a new one. Cannot be `True`
                if `add_generation_prompt` is also `True`.
506
507
            mm_processor_kwargs: Multimodal processor kwarg overrides for this
                chat request. Only used for offline requests.
nunjunj's avatar
nunjunj committed
508
509
510
511
512

        Returns:
            A list of ``RequestOutput`` objects containing the generated
            responses in the same order as the input messages.
        """
513
        list_of_messages: List[List[ChatCompletionMessageParam]]
nunjunj's avatar
nunjunj committed
514

515
516
517
        # Handle multi and single conversations
        if is_list_of(messages, list):
            # messages is List[List[...]]
518
519
            list_of_messages = cast(List[List[ChatCompletionMessageParam]],
                                    messages)
520
        else:
521
            # messages is List[...]
522
523
524
            list_of_messages = [
                cast(List[ChatCompletionMessageParam], messages)
            ]
525
526
527
528
529
530
531

        prompts: List[Union[TokensPrompt, TextPrompt]] = []

        for msgs in list_of_messages:
            tokenizer = self.get_tokenizer()
            model_config = self.llm_engine.get_model_config()

532
533
534
            # NOTE: _parse_chat_message_content_parts() currently doesn't
            # handle mm_processor_kwargs, since there is no implementation in
            # the chat message parsing for it.
535
536
537
538
539
540
541
542
543
544
            conversation, mm_data = parse_chat_messages(
                msgs, model_config, tokenizer)

            prompt_data: Union[str, List[int]]
            if isinstance(tokenizer, MistralTokenizer):
                prompt_data = apply_mistral_chat_template(
                    tokenizer,
                    messages=msgs,
                    chat_template=chat_template,
                    add_generation_prompt=add_generation_prompt,
545
                    continue_final_message=continue_final_message,
546
547
548
549
550
551
552
553
                    tools=tools,
                )
            else:
                prompt_data = apply_hf_chat_template(
                    tokenizer,
                    conversation=conversation,
                    chat_template=chat_template,
                    add_generation_prompt=add_generation_prompt,
554
                    continue_final_message=continue_final_message,
555
556
557
558
559
560
561
562
563
564
565
566
                    tools=tools,
                )

            prompt: Union[TokensPrompt, TextPrompt]
            if is_list_of(prompt_data, int):
                prompt = TokensPrompt(prompt_token_ids=prompt_data)
            else:
                prompt = TextPrompt(prompt=prompt_data)

            if mm_data is not None:
                prompt["multi_modal_data"] = mm_data

567
568
569
            if mm_processor_kwargs is not None:
                prompt["mm_processor_kwargs"] = mm_processor_kwargs

570
            prompts.append(prompt)
571

nunjunj's avatar
nunjunj committed
572
        return self.generate(
573
            prompts,
574
            sampling_params=sampling_params,
nunjunj's avatar
nunjunj committed
575
576
577
578
            use_tqdm=use_tqdm,
            lora_request=lora_request,
        )

579
580
581
582
583
584
585
586
    @overload  # LEGACY: single (prompt + optional token ids)
    def encode(
        self,
        prompts: str,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
        prompt_token_ids: Optional[List[int]] = None,
        use_tqdm: bool = True,
587
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
588
589
    ) -> List[EmbeddingRequestOutput]:
        ...
590

591
    @overload  # LEGACY: multi (prompt + optional token ids)
592
593
    def encode(
        self,
594
        prompts: List[str],
595
        pooling_params: Optional[Union[PoolingParams,
596
                                       Sequence[PoolingParams]]] = None,
597
598
        prompt_token_ids: Optional[List[List[int]]] = None,
        use_tqdm: bool = True,
599
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
600
601
602
603
604
605
606
607
608
609
610
611
    ) -> List[EmbeddingRequestOutput]:
        ...

    @overload  # LEGACY: single (token ids + optional prompt)
    def encode(
        self,
        prompts: Optional[str] = None,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
        *,
        prompt_token_ids: List[int],
        use_tqdm: bool = True,
612
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
613
614
615
616
617
618
619
620
621
622
623
624
    ) -> List[EmbeddingRequestOutput]:
        ...

    @overload  # LEGACY: multi (token ids + optional prompt)
    def encode(
        self,
        prompts: Optional[List[str]] = None,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
        *,
        prompt_token_ids: List[List[int]],
        use_tqdm: bool = True,
625
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
626
627
628
629
630
631
632
633
634
635
    ) -> List[EmbeddingRequestOutput]:
        ...

    @overload  # LEGACY: single or multi token ids [pos-only]
    def encode(
        self,
        prompts: None,
        pooling_params: None,
        prompt_token_ids: Union[List[int], List[List[int]]],
        use_tqdm: bool = True,
636
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
637
638
639
640
641
642
    ) -> List[EmbeddingRequestOutput]:
        ...

    @overload
    def encode(
        self,
643
644
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
645
646
647
648
        *,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
        use_tqdm: bool = True,
649
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
650
651
652
    ) -> List[EmbeddingRequestOutput]:
        ...

nunjunj's avatar
nunjunj committed
653
654
655
    @deprecate_kwargs(
        "prompt_token_ids",
        is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
656
        additional_message="Please use the 'prompts' parameter instead.",
nunjunj's avatar
nunjunj committed
657
    )
658
659
    def encode(
        self,
660
        prompts: Union[Union[PromptType, Sequence[PromptType]],
661
662
663
664
665
                       Optional[Union[str, List[str]]]] = None,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
        prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
        use_tqdm: bool = True,
666
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
667
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
668
669
670
    ) -> List[EmbeddingRequestOutput]:
        """Generates the completions for the input prompts.

671
        This class automatically batches the given prompts, considering
672
673
674
675
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
676
677
678
            prompts: The prompts to the LLM. You may pass a sequence of prompts
                for batch inference. See :class:`~vllm.inputs.PromptType`
                for more details about the format of each prompts.
679
680
681
682
            pooling_params: The pooling parameters for pooling. If None, we
                use the default pooling parameters.
            use_tqdm: Whether to use tqdm to display the progress bar.
            lora_request: LoRA request to use for generation, if any.
nunjunj's avatar
nunjunj committed
683
            prompt_adapter_request: Prompt Adapter request to use for
684
                generation, if any.
685
686
687
688

        Returns:
            A list of `EmbeddingRequestOutput` objects containing the
            generated embeddings in the same order as the input prompts.
689
690
691
692
693

        Note:
            Using ``prompts`` and ``prompt_token_ids`` as keyword parameters is
            considered legacy and may be deprecated in the future. You should
            instead pass them via the ``inputs`` parameter.
694
        """
695
696
697
698
699
        if not self.llm_engine.model_config.embedding_mode:
            raise ValueError(
                "LLM.encode() is only supported for embedding models (XModel)."
            )

700
        if prompt_token_ids is not None:
701
            parsed_prompts = self._convert_v1_inputs(
702
703
704
705
                prompts=cast(Optional[Union[str, List[str]]], prompts),
                prompt_token_ids=prompt_token_ids,
            )
        else:
706
707
            parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
                                  prompts)
708

709
710
711
712
        if pooling_params is None:
            # Use default pooling params.
            pooling_params = PoolingParams()

713
        self._validate_and_add_requests(
714
            prompts=parsed_prompts,
715
716
            params=pooling_params,
            lora_request=lora_request,
717
            prompt_adapter_request=prompt_adapter_request,
718
719
        )

720
721
        outputs = self._run_engine(use_tqdm=use_tqdm)
        return LLMEngine.validate_outputs(outputs, EmbeddingRequestOutput)
722

723
724
725
726
727
728
    def start_profile(self) -> None:
        self.llm_engine.start_profile()

    def stop_profile(self) -> None:
        self.llm_engine.stop_profile()

729
730
    # LEGACY
    def _convert_v1_inputs(
731
732
        self,
        prompts: Optional[Union[str, List[str]]],
733
734
735
        prompt_token_ids: Optional[Union[List[int], List[List[int]]]],
    ):
        # skip_tokenizer_init is now checked in engine
736

737
738
739
740
741
742
        if prompts is not None:
            prompts = [p["content"] for p in parse_and_batch_prompt(prompts)]
        if prompt_token_ids is not None:
            prompt_token_ids = [
                p["content"] for p in parse_and_batch_prompt(prompt_token_ids)
            ]
743

744
        num_requests = None
745
746
        if prompts is not None:
            num_requests = len(prompts)
747
748
749
750
751
752
        if prompt_token_ids is not None:
            if (num_requests is not None
                    and num_requests != len(prompt_token_ids)):
                raise ValueError("The lengths of prompts and prompt_token_ids "
                                 "must be the same.")

753
            num_requests = len(prompt_token_ids)
754
755
756
757
        if num_requests is None:
            raise ValueError("Either prompts or prompt_token_ids must be "
                             "provided.")

758
        parsed_prompts: List[PromptType] = []
759
        for i in range(num_requests):
760
            item: PromptType
761

762
            if prompts is not None:
763
764
765
                item = TextPrompt(prompt=prompts[i])
            elif prompt_token_ids is not None:
                item = TokensPrompt(prompt_token_ids=prompt_token_ids[i])
766
            else:
767
                raise AssertionError
768

769
            parsed_prompts.append(item)
770

771
        return parsed_prompts
772
773
774

    def _validate_and_add_requests(
        self,
775
        prompts: Union[PromptType, Sequence[PromptType]],
776
777
        params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
                      Sequence[PoolingParams]],
778
        lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
779
        prompt_adapter_request: Optional[PromptAdapterRequest],
780
        guided_options: Optional[GuidedDecodingRequest] = None,
781
        priority: Optional[List[int]] = None,
782
    ) -> None:
783
784
785
786
787
788
789
790
        if guided_options is not None:
            warnings.warn(
                "guided_options_request is deprecated, use "
                "SamplingParams.guided_decoding instead",
                DeprecationWarning,
                stacklevel=2,
            )

791
        if isinstance(prompts, (str, dict)):
792
            # Convert a single prompt to a list.
793
            prompts = [prompts]
794

795
        num_requests = len(prompts)
796
797
        if isinstance(params, list) and len(params) != num_requests:
            raise ValueError("The lengths of prompts and params "
798
                             "must be the same.")
799
800
801
802
        if isinstance(lora_request,
                      list) and len(lora_request) != num_requests:
            raise ValueError("The lengths of prompts and lora_request "
                             "must be the same.")
803

804
805
        for sp in params if isinstance(params, list) else (params, ):
            if isinstance(sp, SamplingParams):
806
                self._add_guided_params(sp, guided_options)
807
808
809

                # We only care about the final output
                sp.output_kind = RequestOutputKind.FINAL_ONLY
810

Zhuohan Li's avatar
Zhuohan Li committed
811
        # Add requests to the engine.
812
        for i, prompt in enumerate(prompts):
813
            self._add_request(
814
                prompt,
815
                params[i] if isinstance(params, Sequence) else params,
816
817
                lora_request=lora_request[i] if isinstance(
                    lora_request, Sequence) else lora_request,
nunjunj's avatar
nunjunj committed
818
                prompt_adapter_request=prompt_adapter_request,
819
                priority=priority[i] if priority else 0,
nunjunj's avatar
nunjunj committed
820
            )
821

822
    def _add_request(
nunjunj's avatar
nunjunj committed
823
        self,
824
        prompt: PromptType,
nunjunj's avatar
nunjunj committed
825
        params: Union[SamplingParams, PoolingParams],
826
        lora_request: Optional[LoRARequest] = None,
nunjunj's avatar
nunjunj committed
827
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
828
        priority: int = 0,
829
830
    ) -> None:
        request_id = str(next(self.request_counter))
831
832
        self.llm_engine.add_request(
            request_id,
833
            prompt,
834
835
            params,
            lora_request=lora_request,
nunjunj's avatar
nunjunj committed
836
            prompt_adapter_request=prompt_adapter_request,
837
            priority=priority,
nunjunj's avatar
nunjunj committed
838
        )
839

840
    def _add_guided_params(
841
842
843
            self,
            params: SamplingParams,
            guided_options: Optional[GuidedDecodingRequest] = None):
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
        if guided_options is None:
            return params

        if params.guided_decoding is not None:
            raise ValueError("Cannot set both guided_options_request and"
                             "params.guided_decoding.")

        params.guided_decoding = GuidedDecodingParams(
            json=guided_options.guided_json,
            regex=guided_options.guided_regex,
            choice=guided_options.guided_choice,
            grammar=guided_options.guided_grammar,
            json_object=guided_options.guided_json_object,
            backend=guided_options.guided_decoding_backend,
            whitespace_pattern=guided_options.guided_whitespace_pattern)
859
860
        return params

861
    def _run_engine(
862
            self, *, use_tqdm: bool
863
    ) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
864
865
        # Initialize tqdm.
        if use_tqdm:
Zhuohan Li's avatar
Zhuohan Li committed
866
            num_requests = self.llm_engine.get_num_unfinished_requests()
867
868
869
870
            pbar = tqdm(
                total=num_requests,
                desc="Processed prompts",
                dynamic_ncols=True,
871
872
                postfix=(f"est. speed input: {0:.2f} toks/s, "
                         f"output: {0:.2f} toks/s"),
873
            )
874

Zhuohan Li's avatar
Zhuohan Li committed
875
        # Run the engine.
876
        outputs: List[Union[RequestOutput, EmbeddingRequestOutput]] = []
877
878
        total_in_toks = 0
        total_out_toks = 0
Zhuohan Li's avatar
Zhuohan Li committed
879
880
        while self.llm_engine.has_unfinished_requests():
            step_outputs = self.llm_engine.step()
881
            for output in step_outputs:
882
                if output.finished:
883
884
                    outputs.append(output)
                    if use_tqdm:
885
886
                        if isinstance(output, RequestOutput):
                            # Calculate tokens only for RequestOutput
887
                            assert output.prompt_token_ids is not None
888
889
890
                            total_in_toks += len(output.prompt_token_ids)
                            in_spd = total_in_toks / pbar.format_dict["elapsed"]
                            total_out_toks += sum(
891
                                len(stp.token_ids) for stp in output.outputs)
nunjunj's avatar
nunjunj committed
892
893
                            out_spd = (total_out_toks /
                                       pbar.format_dict["elapsed"])
894
895
896
                            pbar.postfix = (
                                f"est. speed input: {in_spd:.2f} toks/s, "
                                f"output: {out_spd:.2f} toks/s")
897
                        pbar.update(1)
898

899
900
        if use_tqdm:
            pbar.close()
901
902
903
        # Sort the outputs by request ID.
        # This is necessary because some requests may be finished earlier than
        # its previous requests.
904
        return sorted(outputs, key=lambda x: int(x.request_id))
905
906
907
908
909
910

    def _is_encoder_decoder_model(self):
        return self.llm_engine.is_encoder_decoder_model()

    def _is_embedding_model(self):
        return self.llm_engine.is_embedding_model()