llm.py 55.5 KB
Newer Older
1
import itertools
2
import warnings
3
from contextlib import contextmanager
4
5
from typing import (Any, Callable, ClassVar, Dict, List, Optional, Sequence,
                    Tuple, Type, Union, cast, overload)
6

7
import cloudpickle
8
from tqdm import tqdm
9
from typing_extensions import deprecated
10

11
from vllm import envs
12
13
from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
                              BeamSearchSequence, get_beam_search_score)
14
from vllm.config import CompilationConfig
15
16
from vllm.engine.arg_utils import (EngineArgs, HfOverrides, PoolerConfig,
                                   TaskOption)
Joe Runde's avatar
Joe Runde committed
17
from vllm.engine.llm_engine import LLMEngine
nunjunj's avatar
nunjunj committed
18
from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
19
                                         ChatTemplateContentFormatOption,
20
21
                                         apply_hf_chat_template,
                                         apply_mistral_chat_template,
22
23
                                         parse_chat_messages,
                                         resolve_chat_template_content_format)
24
from vllm.inputs import PromptType, SingletonPrompt, TextPrompt, TokensPrompt
25
from vllm.inputs.parse import is_token_prompt, parse_and_batch_prompt
26
from vllm.logger import init_logger
27
from vllm.lora.request import LoRARequest
28
29
from vllm.model_executor.guided_decoding.guided_fields import (
    GuidedDecodingRequest, LLMGuidedOptions)
30
31
32
from vllm.outputs import (ClassificationRequestOutput, EmbeddingRequestOutput,
                          PoolingRequestOutput, RequestOutput,
                          ScoringRequestOutput)
33
from vllm.pooling_params import PoolingParams
34
from vllm.prompt_adapter.request import PromptAdapterRequest
35
36
from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
                                  RequestOutputKind, SamplingParams)
37
from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
38
39
                                               get_cached_tokenizer)
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
yhu422's avatar
yhu422 committed
40
from vllm.usage.usage_lib import UsageContext
41
from vllm.utils import Counter, deprecate_args, deprecate_kwargs, is_list_of
42

43
44
logger = init_logger(__name__)

45
46

class LLM:
Woosuk Kwon's avatar
Woosuk Kwon committed
47
48
49
50
51
52
53
54
55
56
    """An LLM for generating texts from given prompts and sampling parameters.

    This class includes a tokenizer, a language model (possibly distributed
    across multiple GPUs), and GPU memory space allocated for intermediate
    states (aka KV cache). Given a batch of prompts and sampling parameters,
    this class generates texts from the model, using an intelligent batching
    mechanism and efficient memory management.

    Args:
        model: The name or path of a HuggingFace Transformers model.
57
        tokenizer: The name or path of a HuggingFace Transformers tokenizer.
58
59
        tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer
            if available, and "slow" will always use the slow tokenizer.
60
61
62
        skip_tokenizer_init: If true, skip initialization of tokenizer and
            detokenizer. Expect valid prompt_token_ids and None for prompt
            from the input.
63
64
        trust_remote_code: Trust remote code (e.g., from HuggingFace) when
            downloading the model and tokenizer.
65
66
67
68
        allowed_local_media_path: Allowing API requests to read local images
            or videos from directories specified by the server file system.
            This is a security risk. Should only be enabled in trusted
            environments.
Woosuk Kwon's avatar
Woosuk Kwon committed
69
70
71
        tensor_parallel_size: The number of GPUs to use for distributed
            execution with tensor parallelism.
        dtype: The data type for the model weights and activations. Currently,
Woosuk Kwon's avatar
Woosuk Kwon committed
72
73
74
75
            we support `float32`, `float16`, and `bfloat16`. If `auto`, we use
            the `torch_dtype` attribute specified in the model config file.
            However, if the `torch_dtype` in the config is `float32`, we will
            use `float16` instead.
76
        quantization: The method used to quantize the model weights. Currently,
77
            we support "awq", "gptq", and "fp8" (experimental).
78
79
80
81
            If None, we first check the `quantization_config` attribute in the
            model config file. If that is None, we assume the model weights are
            not quantized and use `dtype` to determine the data type of
            the weights.
Jasmond L's avatar
Jasmond L committed
82
83
        revision: The specific model version to use. It can be a branch name,
            a tag name, or a commit id.
84
85
        tokenizer_revision: The specific tokenizer version to use. It can be a
            branch name, a tag name, or a commit id.
86
87
88
89
90
91
92
93
94
95
96
        seed: The seed to initialize the random number generator for sampling.
        gpu_memory_utilization: The ratio (between 0 and 1) of GPU memory to
            reserve for the model weights, activations, and KV cache. Higher
            values will increase the KV cache size and thus improve the model's
            throughput. However, if the value is too high, it may cause out-of-
            memory (OOM) errors.
        swap_space: The size (GiB) of CPU memory per GPU to use as swap space.
            This can be used for temporarily storing the states of the requests
            when their `best_of` sampling parameters are larger than 1. If all
            requests will have `best_of=1`, you can safely set this to 0.
            Otherwise, too small values may cause out-of-memory (OOM) errors.
97
98
99
100
        cpu_offload_gb: The size (GiB) of CPU memory to use for offloading
            the model weights. This virtually increases the GPU memory space
            you can use to hold the model weights, at the cost of CPU-GPU data
            transfer for every forward pass.
101
102
103
        enforce_eager: Whether to enforce eager execution. If True, we will
            disable CUDA graph and always execute the model in eager mode.
            If False, we will use CUDA graph and eager execution in hybrid.
104
        max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs.
105
            When a sequence has context length larger than this, we fall back
106
107
108
            to eager mode. Additionally for encoder-decoder models, if the
            sequence length of the encoder input is larger than this, we fall
            back to the eager mode.
109
110
111
        disable_custom_all_reduce: See :class:`~vllm.config.ParallelConfig`
        disable_async_output_proc: Disable async output processing.
            This may result in lower performance.
112
113
114
        hf_overrides: If a dictionary, contains arguments to be forwarded to the
            HuggingFace config. If a callable, it is called to update the
            HuggingFace config.
115
116
117
        compilation_config: Either an integer or a dictionary. If it is an
            integer, it is used as the level of compilation optimization. If it
            is a dictionary, it can specify the full compilation configuration.
118
        **kwargs: Arguments for :class:`~vllm.EngineArgs`. (See
119
            :ref:`engine-args`)
nunjunj's avatar
nunjunj committed
120

121
122
123
    Note:
        This class is intended to be used for offline inference. For online
        serving, use the :class:`~vllm.AsyncLLMEngine` class instead.
124
    """
125

126
    DEPRECATE_LEGACY: ClassVar[bool] = True
127
128
    """A flag to toggle whether to deprecate the legacy generate/encode API."""

129
130
131
132
133
134
    DEPRECATE_INIT_POSARGS: ClassVar[bool] = True
    """
    A flag to toggle whether to deprecate positional arguments in
    :meth:`LLM.__init__`.
    """

135
136
137
138
139
140
141
142
143
    @classmethod
    @contextmanager
    def deprecate_legacy_api(cls):
        cls.DEPRECATE_LEGACY = True

        yield

        cls.DEPRECATE_LEGACY = False

144
145
146
147
148
149
150
    @deprecate_args(
        start_index=2,  # Ignore self and model
        is_deprecated=lambda: LLM.DEPRECATE_INIT_POSARGS,
        additional_message=(
            "All positional arguments other than `model` will be "
            "replaced with keyword arguments in an upcoming version."),
    )
151
152
153
    def __init__(
        self,
        model: str,
154
        tokenizer: Optional[str] = None,
155
        tokenizer_mode: str = "auto",
156
        skip_tokenizer_init: bool = False,
157
        trust_remote_code: bool = False,
158
        allowed_local_media_path: str = "",
159
        tensor_parallel_size: int = 1,
Woosuk Kwon's avatar
Woosuk Kwon committed
160
        dtype: str = "auto",
161
        quantization: Optional[str] = None,
162
        revision: Optional[str] = None,
163
        tokenizer_revision: Optional[str] = None,
164
165
        seed: int = 0,
        gpu_memory_utilization: float = 0.9,
166
        swap_space: float = 4,
167
        cpu_offload_gb: float = 0,
168
        enforce_eager: Optional[bool] = None,
169
        max_seq_len_to_capture: int = 8192,
170
        disable_custom_all_reduce: bool = False,
171
        disable_async_output_proc: bool = False,
172
        hf_overrides: Optional[HfOverrides] = None,
173
        mm_processor_kwargs: Optional[Dict[str, Any]] = None,
174
175
        # After positional args are removed, move this right below `model`
        task: TaskOption = "auto",
176
        override_pooler_config: Optional[PoolerConfig] = None,
177
        compilation_config: Optional[Union[int, Dict[str, Any]]] = None,
178
179
        **kwargs,
    ) -> None:
180
181
182
183
        '''
        LLM constructor.

        Note: if enforce_eager is unset (enforce_eager is None)
184
        it defaults to False.
185
186
        '''

187
188
        if "disable_log_stats" not in kwargs:
            kwargs["disable_log_stats"] = True
189

190
191
192
193
194
195
196
        if "worker_cls" in kwargs:
            worker_cls = kwargs["worker_cls"]
            # if the worker_cls is not qualified string name,
            # we serialize it using cloudpickle to avoid pickling issues
            if isinstance(worker_cls, type):
                kwargs["worker_cls"] = cloudpickle.dumps(worker_cls)

197
        if compilation_config is not None:
198
            if isinstance(compilation_config, (int, dict)):
199
200
201
202
                compilation_config_instance = CompilationConfig.from_cli(
                    str(compilation_config))
            else:
                compilation_config_instance = compilation_config
203
204
205
        else:
            compilation_config_instance = None

Zhuohan Li's avatar
Zhuohan Li committed
206
        engine_args = EngineArgs(
207
            model=model,
208
            task=task,
209
            tokenizer=tokenizer,
210
            tokenizer_mode=tokenizer_mode,
211
            skip_tokenizer_init=skip_tokenizer_init,
212
            trust_remote_code=trust_remote_code,
213
            allowed_local_media_path=allowed_local_media_path,
214
215
            tensor_parallel_size=tensor_parallel_size,
            dtype=dtype,
216
            quantization=quantization,
217
            revision=revision,
218
            tokenizer_revision=tokenizer_revision,
219
220
221
            seed=seed,
            gpu_memory_utilization=gpu_memory_utilization,
            swap_space=swap_space,
222
            cpu_offload_gb=cpu_offload_gb,
223
            enforce_eager=enforce_eager,
224
            max_seq_len_to_capture=max_seq_len_to_capture,
225
            disable_custom_all_reduce=disable_custom_all_reduce,
226
            disable_async_output_proc=disable_async_output_proc,
227
            hf_overrides=hf_overrides,
228
            mm_processor_kwargs=mm_processor_kwargs,
229
            override_pooler_config=override_pooler_config,
230
            compilation_config=compilation_config_instance,
231
232
            **kwargs,
        )
Joe Runde's avatar
Joe Runde committed
233
234
235
236
        # Logic to switch between engines is done at runtime instead of import
        # to avoid import order issues
        self.engine_class = self.get_engine_class()
        self.llm_engine = self.engine_class.from_engine_args(
yhu422's avatar
yhu422 committed
237
            engine_args, usage_context=UsageContext.LLM_CLASS)
238

239
240
        self.request_counter = Counter()

Joe Runde's avatar
Joe Runde committed
241
242
243
244
245
246
247
248
    @staticmethod
    def get_engine_class() -> Type[LLMEngine]:
        if envs.VLLM_USE_V1:
            # Lazy import: the v1 package isn't distributed
            from vllm.v1.engine.llm_engine import LLMEngine as V1LLMEngine
            return V1LLMEngine  # type: ignore
        return LLMEngine

249
250
251
252
253
    def get_tokenizer(self) -> AnyTokenizer:
        return self.llm_engine.get_tokenizer_group(TokenizerGroup).tokenizer

    def set_tokenizer(self, tokenizer: AnyTokenizer) -> None:
        tokenizer_group = self.llm_engine.get_tokenizer_group(TokenizerGroup)
254

255
256
257
258
        # While CachedTokenizer is dynamic, have no choice but
        # compare class name. Misjudgment will arise from
        # user-defined tokenizer started with 'Cached'
        if tokenizer.__class__.__name__.startswith("Cached"):
259
            tokenizer_group.tokenizer = tokenizer
260
        else:
261
            tokenizer_group.tokenizer = get_cached_tokenizer(tokenizer)
262

263
264
265
266
267
268
269
    def get_default_sampling_params(self) -> SamplingParams:
        diff_sampling_param = (
            self.llm_engine.model_config.get_diff_sampling_param())
        if diff_sampling_param:
            return SamplingParams.from_optional(**diff_sampling_param)
        return SamplingParams()

270
271
272
273
274
275
276
    @overload
    def generate(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        sampling_params: Optional[Union[SamplingParams,
                                        Sequence[SamplingParams]]] = None,
277
        *,
278
279
        use_tqdm: bool = True,
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
280
281
282
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
283
284
285
    ) -> List[RequestOutput]:
        ...

286
    @overload  # LEGACY: single (prompt + optional token ids)
287
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
288
289
290
291
292
293
294
    def generate(
        self,
        prompts: str,
        sampling_params: Optional[Union[SamplingParams,
                                        List[SamplingParams]]] = None,
        prompt_token_ids: Optional[List[int]] = None,
        use_tqdm: bool = True,
295
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
296
297
298
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
299
300
301
302
    ) -> List[RequestOutput]:
        ...

    @overload  # LEGACY: multi (prompt + optional token ids)
303
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
304
305
    def generate(
        self,
306
        prompts: List[str],
307
308
        sampling_params: Optional[Union[SamplingParams,
                                        List[SamplingParams]]] = None,
309
        prompt_token_ids: Optional[List[List[int]]] = None,
310
        use_tqdm: bool = True,
311
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
312
313
314
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
315
316
317
318
    ) -> List[RequestOutput]:
        ...

    @overload  # LEGACY: single (token ids + optional prompt)
319
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
320
321
322
323
324
325
326
327
    def generate(
        self,
        prompts: Optional[str] = None,
        sampling_params: Optional[Union[SamplingParams,
                                        List[SamplingParams]]] = None,
        *,
        prompt_token_ids: List[int],
        use_tqdm: bool = True,
328
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
329
330
331
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
332
333
334
335
    ) -> List[RequestOutput]:
        ...

    @overload  # LEGACY: multi (token ids + optional prompt)
336
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
337
338
339
340
341
342
343
344
    def generate(
        self,
        prompts: Optional[List[str]] = None,
        sampling_params: Optional[Union[SamplingParams,
                                        List[SamplingParams]]] = None,
        *,
        prompt_token_ids: List[List[int]],
        use_tqdm: bool = True,
345
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
346
347
348
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
349
350
351
352
    ) -> List[RequestOutput]:
        ...

    @overload  # LEGACY: single or multi token ids [pos-only]
353
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
354
355
356
357
358
359
    def generate(
        self,
        prompts: None,
        sampling_params: None,
        prompt_token_ids: Union[List[int], List[List[int]]],
        use_tqdm: bool = True,
360
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
361
362
363
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
364
365
366
    ) -> List[RequestOutput]:
        ...

nunjunj's avatar
nunjunj committed
367
368
369
    @deprecate_kwargs(
        "prompt_token_ids",
        is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
370
        additional_message="Please use the 'prompts' parameter instead.",
nunjunj's avatar
nunjunj committed
371
    )
372
373
    def generate(
        self,
374
        prompts: Union[Union[PromptType, Sequence[PromptType]],
375
376
377
378
379
                       Optional[Union[str, List[str]]]] = None,
        sampling_params: Optional[Union[SamplingParams,
                                        Sequence[SamplingParams]]] = None,
        prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
        use_tqdm: bool = True,
380
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
381
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
382
        guided_options_request: Optional[Union[LLMGuidedOptions,
383
384
                                               GuidedDecodingRequest]] = None,
        priority: Optional[List[int]] = None,
385
    ) -> List[RequestOutput]:
Woosuk Kwon's avatar
Woosuk Kwon committed
386
387
        """Generates the completions for the input prompts.

388
        This class automatically batches the given prompts, considering
Woosuk Kwon's avatar
Woosuk Kwon committed
389
390
391
392
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
393
394
395
            prompts: The prompts to the LLM. You may pass a sequence of prompts
                for batch inference. See :class:`~vllm.inputs.PromptType`
                for more details about the format of each prompts.
Woosuk Kwon's avatar
Woosuk Kwon committed
396
            sampling_params: The sampling parameters for text generation. If
nunjunj's avatar
nunjunj committed
397
398
399
                None, we use the default sampling parameters.
                When it is a single value, it is applied to every prompt.
                When it is a list, the list must have the same length as the
400
                prompts and it is paired one by one with the prompt.
Woosuk Kwon's avatar
Woosuk Kwon committed
401
            use_tqdm: Whether to use tqdm to display the progress bar.
402
            lora_request: LoRA request to use for generation, if any.
nunjunj's avatar
nunjunj committed
403
            prompt_adapter_request: Prompt Adapter request to use for
404
                generation, if any.
405
406
            priority: The priority of the requests, if any.
                Only applicable when priority scheduling policy is enabled.
Woosuk Kwon's avatar
Woosuk Kwon committed
407
408

        Returns:
nunjunj's avatar
nunjunj committed
409
            A list of ``RequestOutput`` objects containing the
410
            generated completions in the same order as the input prompts.
411
412
413
414
415

        Note:
            Using ``prompts`` and ``prompt_token_ids`` as keyword parameters is
            considered legacy and may be deprecated in the future. You should
            instead pass them via the ``inputs`` parameter.
416
        """
417
418
        runner_type = self.llm_engine.model_config.runner_type
        if runner_type != "generate":
419
            messages = [
420
                "LLM.generate() is only supported for (conditional) generation "
421
422
423
                "models (XForCausalLM, XForConditionalGeneration).",
            ]

424
425
426
            supported_runner_types = self.llm_engine.model_config \
                .supported_runner_types
            if "generate" in supported_runner_types:
427
                messages.append(
428
429
430
                    "Your model supports the 'generate' runner, but is "
                    f"currently initialized for the '{runner_type}' runner. "
                    "Please initialize vLLM using `--task generate`.")
431
432

            raise ValueError(" ".join(messages))
433

434
        if prompt_token_ids is not None:
435
            parsed_prompts = self._convert_v1_inputs(
436
437
438
439
                prompts=cast(Optional[Union[str, List[str]]], prompts),
                prompt_token_ids=prompt_token_ids,
            )
        else:
440
441
            parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
                                  prompts)
442

443
444
445
446
447
448
449
450
        if isinstance(guided_options_request, dict):
            if len(guided_options_request) > 1:
                raise ValueError(
                    "You can only use one guided decoding but multiple is "
                    f"specified: {guided_options_request}")
            guided_options_request = GuidedDecodingRequest(
                **guided_options_request)

451
452
        if sampling_params is None:
            # Use default sampling params.
453
            sampling_params = self.get_default_sampling_params()
454

455
        self._validate_and_add_requests(
456
            prompts=parsed_prompts,
457
458
            params=sampling_params,
            lora_request=lora_request,
459
            prompt_adapter_request=prompt_adapter_request,
460
461
            guided_options=guided_options_request,
            priority=priority)
462

463
        outputs = self._run_engine(use_tqdm=use_tqdm)
Joe Runde's avatar
Joe Runde committed
464
        return self.engine_class.validate_outputs(outputs, RequestOutput)
465

466
    def collective_rpc(self,
467
                       method: Union[str, Callable],
468
469
470
471
472
473
474
475
476
477
478
                       timeout: Optional[float] = None,
                       args: Tuple = (),
                       kwargs: Optional[Dict] = None) -> List[Any]:
        """
        Run a method on all workers, with homogeneous arguments.
        The main extension point for the LLM entrypoint.
        Users can provide custom worker class through `worker_cls`
        argument, and implement new methods in the worker class.
        Then, users can call the new methods through this API.
        It is recommended to use this API to only pass control messages,
        and set up data-plane communication to pass data.
479
480
481
482
483
        The method can also be a callable, which will be serialized
        and sent to all workers to execute.
        If the method is a callable, it should accept an additional
        `self` argument, in addition to the arguments passed in `args`
        and `kwargs`. The `self` argument will be the worker object.
484
        """
485
        return self.llm_engine.collective_rpc(method, timeout, args, kwargs)
486

487
488
    def beam_search(
        self,
489
        prompts: List[Union[TokensPrompt, TextPrompt]],
490
        params: BeamSearchParams,
491
492
493
494
495
496
497
    ) -> List[BeamSearchOutput]:
        """
        Generate sequences using beam search.

        Args:
            prompts: A list of prompts. Each prompt can be a string or a list
                of token IDs.
498
499
            params: The beam search parameters.

500
501
502
503
        TODO: how does beam search work together with length penalty, frequency
        penalty, and stopping criteria, etc.?
        """

504
505
506
507
        beam_width = params.beam_width
        max_tokens = params.max_tokens
        temperature = params.temperature
        ignore_eos = params.ignore_eos
508
509
510
511
512
513
        length_penalty = params.length_penalty

        def sort_beams_key(x: BeamSearchSequence) -> float:
            return get_beam_search_score(x.tokens, x.cum_logprob,
                                         tokenizer.eos_token_id,
                                         length_penalty)
514

515
516
517
518
519
520
        tokenizer = self.get_tokenizer()
        # generate 2 * beam_width candidates at each step
        # following the huggingface transformers implementation
        # at https://github.com/huggingface/transformers/blob/e15687fffe5c9d20598a19aeab721ae0a7580f8a/src/transformers/generation/beam_search.py#L534 # noqa
        beam_search_params = SamplingParams(logprobs=2 * beam_width,
                                            max_tokens=1,
521
                                            temperature=temperature)
522
523
524
        instances: List[BeamSearchInstance] = []

        for prompt in prompts:
525
526
527
528
            if is_token_prompt(prompt):
                prompt_tokens = prompt["prompt_token_ids"]
            else:
                prompt_tokens = tokenizer.encode(prompt["prompt"])
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
            instances.append(BeamSearchInstance(prompt_tokens))

        for _ in range(max_tokens):
            all_beams: List[BeamSearchSequence] = list(
                sum((instance.beams for instance in instances), []))
            pos = [0] + list(
                itertools.accumulate(
                    len(instance.beams) for instance in instances))
            instance_start_and_end: List[Tuple[int, int]] = list(
                zip(pos[:-1], pos[1:]))

            if len(all_beams) == 0:
                break

            prompts_batch = [
                TokensPrompt(prompt_token_ids=beam.tokens)
                for beam in all_beams
            ]

            # only runs for one step
            # we don't need to use tqdm here
            output = self.generate(prompts_batch,
                                   sampling_params=beam_search_params,
                                   use_tqdm=False)

            for (start, end), instance in zip(instance_start_and_end,
                                              instances):
                instance_new_beams = []
                for i in range(start, end):
                    current_beam = all_beams[i]
                    result = output[i]

                    if result.outputs[0].logprobs is not None:
                        # if `result.outputs[0].logprobs` is None, it means
                        # the sequence is completed because of the max-model-len
                        # or abortion. we don't need to add it to the new beams.
                        logprobs = result.outputs[0].logprobs[0]
                        for token_id, logprob_obj in logprobs.items():
                            new_beam = BeamSearchSequence(
                                tokens=current_beam.tokens + [token_id],
569
                                logprobs=current_beam.logprobs + [logprobs],
570
571
572
573
574
575
576
577
578
                                cum_logprob=current_beam.cum_logprob +
                                logprob_obj.logprob)

                            if token_id == tokenizer.eos_token_id and \
                                not ignore_eos:
                                instance.completed.append(new_beam)
                            else:
                                instance_new_beams.append(new_beam)
                sorted_beams = sorted(instance_new_beams,
579
                                      key=sort_beams_key,
580
581
582
583
584
585
586
                                      reverse=True)
                instance.beams = sorted_beams[:beam_width]

        outputs = []
        for instance in instances:
            instance.completed.extend(instance.beams)
            sorted_completed = sorted(instance.completed,
587
                                      key=sort_beams_key,
588
589
590
591
592
593
594
595
596
                                      reverse=True)
            best_beams = sorted_completed[:beam_width]

            for beam in best_beams:
                beam.text = tokenizer.decode(beam.tokens)
            outputs.append(BeamSearchOutput(sequences=best_beams))

        return outputs

nunjunj's avatar
nunjunj committed
597
598
    def chat(
        self,
599
600
        messages: Union[List[ChatCompletionMessageParam],
                        List[List[ChatCompletionMessageParam]]],
nunjunj's avatar
nunjunj committed
601
602
603
604
605
        sampling_params: Optional[Union[SamplingParams,
                                        List[SamplingParams]]] = None,
        use_tqdm: bool = True,
        lora_request: Optional[LoRARequest] = None,
        chat_template: Optional[str] = None,
606
        chat_template_content_format: ChatTemplateContentFormatOption = "auto",
607
        add_generation_prompt: bool = True,
608
        continue_final_message: bool = False,
609
        tools: Optional[List[Dict[str, Any]]] = None,
610
        mm_processor_kwargs: Optional[Dict[str, Any]] = None,
nunjunj's avatar
nunjunj committed
611
612
    ) -> List[RequestOutput]:
        """
613
        Generate responses for a chat conversation.
nunjunj's avatar
nunjunj committed
614

615
616
617
618
619
620
        The chat conversation is converted into a text prompt using the
        tokenizer and calls the :meth:`generate` method to generate the
        responses.

        Multi-modal inputs can be passed in the same way you would pass them
        to the OpenAI API.
nunjunj's avatar
nunjunj committed
621
622

        Args:
623
624
625
626
627
            messages: A list of conversations or a single conversation.

              - Each conversation is represented as a list of messages.
              - Each message is a dictionary with 'role' and 'content' keys.

nunjunj's avatar
nunjunj committed
628
629
630
631
632
633
634
635
636
            sampling_params: The sampling parameters for text generation.
                If None, we use the default sampling parameters. When it
                is a single value, it is applied to every prompt. When it
                is a list, the list must have the same length as the
                prompts and it is paired one by one with the prompt.
            use_tqdm: Whether to use tqdm to display the progress bar.
            lora_request: LoRA request to use for generation, if any.
            chat_template: The template to use for structuring the chat.
              If not provided, the model's default chat template will be used.
637
638
639
640
641
642
643
644
            chat_template_content_format: The format to render message content.

              - "string" will render the content as a string.
                Example: ``"Who are you?"``
              - "openai" will render the content as a list of dictionaries,
                similar to OpenAI schema.
                Example: ``[{"type": "text", "text": "Who are you?"}]``

645
            add_generation_prompt: If True, adds a generation template
nunjunj's avatar
nunjunj committed
646
                to each message.
647
            continue_final_message: If True, continues the final message in
648
649
                the conversation instead of starting a new one. Cannot be
                ``True`` if ``add_generation_prompt`` is also ``True``.
650
651
            mm_processor_kwargs: Multimodal processor kwarg overrides for this
                chat request. Only used for offline requests.
nunjunj's avatar
nunjunj committed
652
653
654
655
656

        Returns:
            A list of ``RequestOutput`` objects containing the generated
            responses in the same order as the input messages.
        """
657
        list_of_messages: List[List[ChatCompletionMessageParam]]
nunjunj's avatar
nunjunj committed
658

659
660
661
        # Handle multi and single conversations
        if is_list_of(messages, list):
            # messages is List[List[...]]
662
663
            list_of_messages = cast(List[List[ChatCompletionMessageParam]],
                                    messages)
664
        else:
665
            # messages is List[...]
666
667
668
            list_of_messages = [
                cast(List[ChatCompletionMessageParam], messages)
            ]
669

670
671
672
673
674
675
676
677
        tokenizer = self.get_tokenizer()
        model_config = self.llm_engine.get_model_config()
        resolved_content_format = resolve_chat_template_content_format(
            chat_template,
            chat_template_content_format,
            tokenizer,
        )

678
679
680
        prompts: List[Union[TokensPrompt, TextPrompt]] = []

        for msgs in list_of_messages:
681
682
683
            # NOTE: _parse_chat_message_content_parts() currently doesn't
            # handle mm_processor_kwargs, since there is no implementation in
            # the chat message parsing for it.
684
            conversation, mm_data = parse_chat_messages(
685
686
687
688
689
                msgs,
                model_config,
                tokenizer,
                content_format=resolved_content_format,
            )
690
691
692
693
694
695
696
697

            prompt_data: Union[str, List[int]]
            if isinstance(tokenizer, MistralTokenizer):
                prompt_data = apply_mistral_chat_template(
                    tokenizer,
                    messages=msgs,
                    chat_template=chat_template,
                    add_generation_prompt=add_generation_prompt,
698
                    continue_final_message=continue_final_message,
699
700
701
702
703
704
705
706
                    tools=tools,
                )
            else:
                prompt_data = apply_hf_chat_template(
                    tokenizer,
                    conversation=conversation,
                    chat_template=chat_template,
                    add_generation_prompt=add_generation_prompt,
707
                    continue_final_message=continue_final_message,
708
709
710
711
712
713
714
715
716
717
718
719
                    tools=tools,
                )

            prompt: Union[TokensPrompt, TextPrompt]
            if is_list_of(prompt_data, int):
                prompt = TokensPrompt(prompt_token_ids=prompt_data)
            else:
                prompt = TextPrompt(prompt=prompt_data)

            if mm_data is not None:
                prompt["multi_modal_data"] = mm_data

720
721
722
            if mm_processor_kwargs is not None:
                prompt["mm_processor_kwargs"] = mm_processor_kwargs

723
            prompts.append(prompt)
724

nunjunj's avatar
nunjunj committed
725
        return self.generate(
726
            prompts,
727
            sampling_params=sampling_params,
nunjunj's avatar
nunjunj committed
728
729
730
731
            use_tqdm=use_tqdm,
            lora_request=lora_request,
        )

732
733
734
735
736
737
738
    @overload
    def encode(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
739
        *,
740
741
        use_tqdm: bool = True,
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
742
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
743
744
745
    ) -> List[PoolingRequestOutput]:
        ...

746
    @overload  # LEGACY: single (prompt + optional token ids)
747
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
748
749
750
751
752
753
754
    def encode(
        self,
        prompts: str,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
        prompt_token_ids: Optional[List[int]] = None,
        use_tqdm: bool = True,
755
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
756
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
757
    ) -> List[PoolingRequestOutput]:
758
        ...
759

760
    @overload  # LEGACY: multi (prompt + optional token ids)
761
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
762
763
    def encode(
        self,
764
        prompts: List[str],
765
        pooling_params: Optional[Union[PoolingParams,
766
                                       Sequence[PoolingParams]]] = None,
767
768
        prompt_token_ids: Optional[List[List[int]]] = None,
        use_tqdm: bool = True,
769
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
770
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
771
    ) -> List[PoolingRequestOutput]:
772
773
774
        ...

    @overload  # LEGACY: single (token ids + optional prompt)
775
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
776
777
778
779
780
781
782
783
    def encode(
        self,
        prompts: Optional[str] = None,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
        *,
        prompt_token_ids: List[int],
        use_tqdm: bool = True,
784
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
785
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
786
    ) -> List[PoolingRequestOutput]:
787
788
789
        ...

    @overload  # LEGACY: multi (token ids + optional prompt)
790
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
791
792
793
794
795
796
797
798
    def encode(
        self,
        prompts: Optional[List[str]] = None,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
        *,
        prompt_token_ids: List[List[int]],
        use_tqdm: bool = True,
799
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
800
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
801
    ) -> List[PoolingRequestOutput]:
802
803
804
        ...

    @overload  # LEGACY: single or multi token ids [pos-only]
805
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
806
807
808
809
810
811
    def encode(
        self,
        prompts: None,
        pooling_params: None,
        prompt_token_ids: Union[List[int], List[List[int]]],
        use_tqdm: bool = True,
812
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
813
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
814
    ) -> List[PoolingRequestOutput]:
815
816
        ...

nunjunj's avatar
nunjunj committed
817
818
819
    @deprecate_kwargs(
        "prompt_token_ids",
        is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
820
        additional_message="Please use the 'prompts' parameter instead.",
nunjunj's avatar
nunjunj committed
821
    )
822
823
    def encode(
        self,
824
        prompts: Union[Union[PromptType, Sequence[PromptType]],
825
826
827
828
829
                       Optional[Union[str, List[str]]]] = None,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
        prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
        use_tqdm: bool = True,
830
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
831
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
832
    ) -> List[PoolingRequestOutput]:
833
834
        """Apply pooling to the hidden states corresponding to the input
        prompts.
835

836
        This class automatically batches the given prompts, considering
837
838
839
840
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
841
842
843
            prompts: The prompts to the LLM. You may pass a sequence of prompts
                for batch inference. See :class:`~vllm.inputs.PromptType`
                for more details about the format of each prompts.
844
845
846
847
            pooling_params: The pooling parameters for pooling. If None, we
                use the default pooling parameters.
            use_tqdm: Whether to use tqdm to display the progress bar.
            lora_request: LoRA request to use for generation, if any.
nunjunj's avatar
nunjunj committed
848
            prompt_adapter_request: Prompt Adapter request to use for
849
                generation, if any.
850
851

        Returns:
852
            A list of ``PoolingRequestOutput`` objects containing the
853
            pooled hidden states in the same order as the input prompts.
854
855
856
857
858

        Note:
            Using ``prompts`` and ``prompt_token_ids`` as keyword parameters is
            considered legacy and may be deprecated in the future. You should
            instead pass them via the ``inputs`` parameter.
859
        """
860
861
862
        runner_type = self.llm_engine.model_config.runner_type
        if runner_type != "pooling":
            messages = ["LLM.encode() is only supported for pooling models."]
863

864
865
866
            supported_runner_types = self.llm_engine.model_config \
                .supported_runner_types
            if "pooling" in supported_runner_types:
867
                messages.append(
868
869
870
871
                    "Your model supports the 'pooling' runner, but is "
                    f"currently initialized for the '{runner_type}' runner. "
                    "Please initialize vLLM using `--task embed`, "
                    "`--task classify`, `--task score` etc.")
872
873

            raise ValueError(" ".join(messages))
874

875
        if prompt_token_ids is not None:
876
            parsed_prompts = self._convert_v1_inputs(
877
878
879
880
                prompts=cast(Optional[Union[str, List[str]]], prompts),
                prompt_token_ids=prompt_token_ids,
            )
        else:
881
882
            parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
                                  prompts)
883

884
885
886
887
        if pooling_params is None:
            # Use default pooling params.
            pooling_params = PoolingParams()

888
        self._validate_and_add_requests(
889
            prompts=parsed_prompts,
890
891
            params=pooling_params,
            lora_request=lora_request,
892
            prompt_adapter_request=prompt_adapter_request,
893
894
        )

895
        outputs = self._run_engine(use_tqdm=use_tqdm)
Joe Runde's avatar
Joe Runde committed
896
        return self.engine_class.validate_outputs(outputs,
897
                                                  PoolingRequestOutput)
898

899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
    def embed(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        *,
        use_tqdm: bool = True,
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
    ) -> List[EmbeddingRequestOutput]:
        """
        Generate an embedding vector for each prompt.

        This class automatically batches the given prompts, considering
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
            prompts: The prompts to the LLM. You may pass a sequence of prompts
                for batch inference. See :class:`~vllm.inputs.PromptType`
                for more details about the format of each prompts.
            use_tqdm: Whether to use tqdm to display the progress bar.
            lora_request: LoRA request to use for generation, if any.
            prompt_adapter_request: Prompt Adapter request to use for
                generation, if any.

        Returns:
            A list of ``EmbeddingRequestOutput`` objects containing the
            embedding vectors in the same order as the input prompts.
        """
        if self.llm_engine.model_config.task != "embed":
            raise ValueError(
                "Embedding API is only enabled for `--task embed`")

        items = self.encode(prompts,
                            use_tqdm=use_tqdm,
                            lora_request=lora_request,
                            prompt_adapter_request=prompt_adapter_request)

        return [EmbeddingRequestOutput.from_base(item) for item in items]

    def classify(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        *,
        use_tqdm: bool = True,
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
    ) -> List[ClassificationRequestOutput]:
        """
        Generate class logits for each prompt.

        This class automatically batches the given prompts, considering
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
            prompts: The prompts to the LLM. You may pass a sequence of prompts
                for batch inference. See :class:`~vllm.inputs.PromptType`
                for more details about the format of each prompts.
            use_tqdm: Whether to use tqdm to display the progress bar.
            lora_request: LoRA request to use for generation, if any.
            prompt_adapter_request: Prompt Adapter request to use for
                generation, if any.

        Returns:
            A list of ``ClassificationRequestOutput`` objects containing the
            embedding vectors in the same order as the input prompts.
        """
        if self.llm_engine.model_config.task != "classify":
            raise ValueError(
                "Classification API is only enabled for `--task classify`")

        items = self.encode(prompts,
                            use_tqdm=use_tqdm,
                            lora_request=lora_request,
                            prompt_adapter_request=prompt_adapter_request)

        return [ClassificationRequestOutput.from_base(item) for item in items]

979
980
981
982
983
    def score(
        self,
        text_1: Union[SingletonPrompt, Sequence[SingletonPrompt]],
        text_2: Union[SingletonPrompt, Sequence[SingletonPrompt]],
        /,
984
        *,
985
986
987
988
        truncate_prompt_tokens: Optional[int] = None,
        use_tqdm: bool = True,
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
989
990
    ) -> List[ScoringRequestOutput]:
        """Generate similarity scores for all pairs ``<text,text_pair>``.
991

992
993
994
995
        The inputs can be ``1 -> 1``, ``1 -> N`` or ``N -> N``.
        In the ``1 - N`` case the ``text_1`` sentence will be replicated ``N``
        times to pair with the ``text_2`` sentences.
        The input pairs are used to build a list of prompts for the
996
997
998
999
1000
1001
        cross encoder model. This class automatically batches the prompts,
        considering the memory constraint. For the best performance, put all
        of your texts into a single list and pass it to this method.

        Args:
            text_1: can be a single prompt or a list of prompts, in which
1002
                case it has to have the same length as the ``text_2`` list
1003
1004
1005
1006
1007
1008
1009
1010
1011
            text_2: The texts to pair with the query to form the input
                to the LLM. See :class:`~vllm.inputs.PromptType` for
                more details about the format of each prompts.
            use_tqdm: Whether to use tqdm to display the progress bar.
            lora_request: LoRA request to use for generation, if any.
            prompt_adapter_request: Prompt Adapter request to use for
                generation, if any.

        Returns:
1012
            A list of ``ScoringRequestOutput`` objects containing the
1013
1014
            generated scores in the same order as the input prompts.
        """
1015
1016
1017
        runner_type = self.llm_engine.model_config.runner_type
        if runner_type != "pooling":
            messages = ["LLM.score() is only supported for pooling models."]
1018

1019
1020
1021
            supported_runner_types = self.llm_engine.model_config \
                .supported_runner_types
            if "pooling" in supported_runner_types:
1022
                messages.append(
1023
1024
1025
1026
                    "Your model supports the 'pooling' runner, but is "
                    f"currently initialized for the '{runner_type}' runner. "
                    "Please initialize vLLM using `--task embed`, "
                    "`--task classify`, `--task score` etc.")
1027
1028
1029
1030

            raise ValueError(" ".join(messages))

        if not self.llm_engine.model_config.is_cross_encoder:
1031
            raise ValueError("Your model does not support cross encoding")
1032
1033
        if self.llm_engine.model_config.task != "score":
            raise ValueError("Score API is only enabled for `--task score`")
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103

        tokenizer = self.llm_engine.get_tokenizer()

        if isinstance(tokenizer, MistralTokenizer):
            raise ValueError(
                "MistralTokenizer not supported for cross-encoding")

        # the tokenizer for models such as
        # "cross-encoder/ms-marco-MiniLM-L-6-v2" doesn't support passing
        # lists of tokens to the `text` and `text_pair` kwargs
        def ensure_str(prompt: SingletonPrompt):
            if isinstance(prompt, dict):
                if "multi_modal_data" in prompt:
                    raise ValueError("Multi-modal prompt is not "
                                     "supported for cross encoding")
                elif "prompt_token_ids" in prompt:
                    prompt = tokenizer.decode(
                        cast(TokensPrompt, prompt)["prompt_token_ids"])
                elif "prompt" in prompt:
                    prompt = cast(TextPrompt, prompt)["prompt"]
            assert type(prompt) is str
            return prompt

        if isinstance(text_1, (str, dict)):
            # Convert a single prompt to a list.
            text_1 = [text_1]
        text_1 = [ensure_str(t) for t in text_1]

        if isinstance(text_2, (str, dict)):
            # Convert a single prompt to a list.
            text_2 = [text_2]
        text_2 = [ensure_str(t) for t in text_2]

        if len(text_1) > 1 and len(text_1) != len(text_2):
            raise ValueError("Input lengths must be either 1:1, 1:N or N:N")
        if len(text_1) == 0:
            raise ValueError("At least one text element must be given")
        if len(text_2) == 0:
            raise ValueError("At least one text_pair element must be given")

        if len(text_1) == 1:
            text_1 = text_1 * len(text_2)

        input_pairs = [(t1, t2) for t1, t2 in zip(text_1, text_2)]
        pooling_params = PoolingParams()

        tokenization_kwargs: Dict[str, Any] = {}
        if truncate_prompt_tokens is not None:
            tokenization_kwargs["truncation"] = True
            tokenization_kwargs["max_length"] = truncate_prompt_tokens

        parsed_prompts = []

        for q, t in input_pairs:
            prompt_inputs = tokenizer(text=q,
                                      text_pair=t,
                                      **tokenization_kwargs)
            engine_prompt = TokensPrompt(
                prompt_token_ids=prompt_inputs["input_ids"],
                token_type_ids=prompt_inputs.get("token_type_ids"))
            parsed_prompts.append(engine_prompt)

        self._validate_and_add_requests(
            prompts=parsed_prompts,
            params=pooling_params,
            lora_request=lora_request,
            prompt_adapter_request=prompt_adapter_request,
        )

        outputs = self._run_engine(use_tqdm=use_tqdm)
1104
1105
1106
1107
        items = self.engine_class.validate_outputs(outputs,
                                                   PoolingRequestOutput)

        return [ScoringRequestOutput.from_base(item) for item in items]
1108

1109
1110
1111
1112
1113
1114
    def start_profile(self) -> None:
        self.llm_engine.start_profile()

    def stop_profile(self) -> None:
        self.llm_engine.stop_profile()

1115
1116
    # LEGACY
    def _convert_v1_inputs(
1117
1118
        self,
        prompts: Optional[Union[str, List[str]]],
1119
1120
1121
        prompt_token_ids: Optional[Union[List[int], List[List[int]]]],
    ):
        # skip_tokenizer_init is now checked in engine
1122

1123
1124
1125
1126
1127
1128
        if prompts is not None:
            prompts = [p["content"] for p in parse_and_batch_prompt(prompts)]
        if prompt_token_ids is not None:
            prompt_token_ids = [
                p["content"] for p in parse_and_batch_prompt(prompt_token_ids)
            ]
1129

1130
        num_requests = None
1131
1132
        if prompts is not None:
            num_requests = len(prompts)
1133
1134
1135
1136
1137
1138
        if prompt_token_ids is not None:
            if (num_requests is not None
                    and num_requests != len(prompt_token_ids)):
                raise ValueError("The lengths of prompts and prompt_token_ids "
                                 "must be the same.")

1139
            num_requests = len(prompt_token_ids)
1140
1141
1142
1143
        if num_requests is None:
            raise ValueError("Either prompts or prompt_token_ids must be "
                             "provided.")

1144
        parsed_prompts: List[PromptType] = []
1145
        for i in range(num_requests):
1146
            item: PromptType
1147

1148
            if prompts is not None:
1149
1150
1151
                item = TextPrompt(prompt=prompts[i])
            elif prompt_token_ids is not None:
                item = TokensPrompt(prompt_token_ids=prompt_token_ids[i])
1152
            else:
1153
                raise AssertionError
1154

1155
            parsed_prompts.append(item)
1156

1157
        return parsed_prompts
1158
1159
1160

    def _validate_and_add_requests(
        self,
1161
        prompts: Union[PromptType, Sequence[PromptType]],
1162
1163
        params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
                      Sequence[PoolingParams]],
1164
        lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
1165
        prompt_adapter_request: Optional[PromptAdapterRequest],
1166
        guided_options: Optional[GuidedDecodingRequest] = None,
1167
        priority: Optional[List[int]] = None,
1168
    ) -> None:
1169
1170
1171
1172
1173
1174
1175
1176
        if guided_options is not None:
            warnings.warn(
                "guided_options_request is deprecated, use "
                "SamplingParams.guided_decoding instead",
                DeprecationWarning,
                stacklevel=2,
            )

1177
        if isinstance(prompts, (str, dict)):
1178
            # Convert a single prompt to a list.
1179
            prompts = [prompts]
1180

1181
        num_requests = len(prompts)
1182
1183
        if isinstance(params, list) and len(params) != num_requests:
            raise ValueError("The lengths of prompts and params "
1184
                             "must be the same.")
1185
1186
1187
1188
        if isinstance(lora_request,
                      list) and len(lora_request) != num_requests:
            raise ValueError("The lengths of prompts and lora_request "
                             "must be the same.")
1189

1190
1191
        for sp in params if isinstance(params, list) else (params, ):
            if isinstance(sp, SamplingParams):
1192
                self._add_guided_params(sp, guided_options)
1193
1194
1195

                # We only care about the final output
                sp.output_kind = RequestOutputKind.FINAL_ONLY
1196

Zhuohan Li's avatar
Zhuohan Li committed
1197
        # Add requests to the engine.
1198
        for i, prompt in enumerate(prompts):
1199
            self._add_request(
1200
                prompt,
1201
                params[i] if isinstance(params, Sequence) else params,
1202
1203
                lora_request=lora_request[i] if isinstance(
                    lora_request, Sequence) else lora_request,
nunjunj's avatar
nunjunj committed
1204
                prompt_adapter_request=prompt_adapter_request,
1205
                priority=priority[i] if priority else 0,
nunjunj's avatar
nunjunj committed
1206
            )
1207

1208
    def _add_request(
nunjunj's avatar
nunjunj committed
1209
        self,
1210
        prompt: PromptType,
nunjunj's avatar
nunjunj committed
1211
        params: Union[SamplingParams, PoolingParams],
1212
        lora_request: Optional[LoRARequest] = None,
nunjunj's avatar
nunjunj committed
1213
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
1214
        priority: int = 0,
1215
1216
    ) -> None:
        request_id = str(next(self.request_counter))
1217
1218
        self.llm_engine.add_request(
            request_id,
1219
            prompt,
1220
1221
            params,
            lora_request=lora_request,
nunjunj's avatar
nunjunj committed
1222
            prompt_adapter_request=prompt_adapter_request,
1223
            priority=priority,
nunjunj's avatar
nunjunj committed
1224
        )
1225

1226
    def _add_guided_params(
1227
1228
1229
            self,
            params: SamplingParams,
            guided_options: Optional[GuidedDecodingRequest] = None):
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
        if guided_options is None:
            return params

        if params.guided_decoding is not None:
            raise ValueError("Cannot set both guided_options_request and"
                             "params.guided_decoding.")

        params.guided_decoding = GuidedDecodingParams(
            json=guided_options.guided_json,
            regex=guided_options.guided_regex,
            choice=guided_options.guided_choice,
            grammar=guided_options.guided_grammar,
            json_object=guided_options.guided_json_object,
            backend=guided_options.guided_decoding_backend,
            whitespace_pattern=guided_options.guided_whitespace_pattern)
1245
1246
        return params

1247
    def _run_engine(
1248
            self, *, use_tqdm: bool
1249
    ) -> List[Union[RequestOutput, PoolingRequestOutput]]:
1250
1251
        # Initialize tqdm.
        if use_tqdm:
Zhuohan Li's avatar
Zhuohan Li committed
1252
            num_requests = self.llm_engine.get_num_unfinished_requests()
1253
1254
1255
1256
            pbar = tqdm(
                total=num_requests,
                desc="Processed prompts",
                dynamic_ncols=True,
1257
1258
                postfix=(f"est. speed input: {0:.2f} toks/s, "
                         f"output: {0:.2f} toks/s"),
1259
            )
1260

Zhuohan Li's avatar
Zhuohan Li committed
1261
        # Run the engine.
1262
        outputs: List[Union[RequestOutput, PoolingRequestOutput]] = []
1263
1264
        total_in_toks = 0
        total_out_toks = 0
Zhuohan Li's avatar
Zhuohan Li committed
1265
1266
        while self.llm_engine.has_unfinished_requests():
            step_outputs = self.llm_engine.step()
1267
            for output in step_outputs:
1268
                if output.finished:
1269
1270
                    outputs.append(output)
                    if use_tqdm:
1271
1272
                        if isinstance(output, RequestOutput):
                            # Calculate tokens only for RequestOutput
1273
                            assert output.prompt_token_ids is not None
1274
1275
1276
                            total_in_toks += len(output.prompt_token_ids)
                            in_spd = total_in_toks / pbar.format_dict["elapsed"]
                            total_out_toks += sum(
1277
                                len(stp.token_ids) for stp in output.outputs)
nunjunj's avatar
nunjunj committed
1278
1279
                            out_spd = (total_out_toks /
                                       pbar.format_dict["elapsed"])
1280
1281
1282
                            pbar.postfix = (
                                f"est. speed input: {in_spd:.2f} toks/s, "
                                f"output: {out_spd:.2f} toks/s")
1283
                        pbar.update(1)
1284

1285
1286
        if use_tqdm:
            pbar.close()
1287
1288
1289
        # Sort the outputs by request ID.
        # This is necessary because some requests may be finished earlier than
        # its previous requests.
1290
        return sorted(outputs, key=lambda x: int(x.request_id))