"tests/kernels/mamba/test_causal_conv1d.py" did not exist on "78029b34ed1be46baf06f92c9e971ea1961d0867"
llm.py 55.2 KB
Newer Older
1
import itertools
2
import warnings
3
from contextlib import contextmanager
Joe Runde's avatar
Joe Runde committed
4
from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Tuple, Type,
5
                    Union, cast, overload)
6

7
import cloudpickle
8
from tqdm import tqdm
9
from typing_extensions import deprecated
10

11
from vllm import envs
12
13
from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
                              BeamSearchSequence, get_beam_search_score)
14
from vllm.config import CompilationConfig
15
16
from vllm.engine.arg_utils import (EngineArgs, HfOverrides, PoolerConfig,
                                   TaskOption)
Joe Runde's avatar
Joe Runde committed
17
from vllm.engine.llm_engine import LLMEngine
nunjunj's avatar
nunjunj committed
18
from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
19
                                         ChatTemplateContentFormatOption,
20
21
                                         apply_hf_chat_template,
                                         apply_mistral_chat_template,
22
23
                                         parse_chat_messages,
                                         resolve_chat_template_content_format)
24
from vllm.inputs import PromptType, SingletonPrompt, TextPrompt, TokensPrompt
25
from vllm.inputs.parse import is_token_prompt, parse_and_batch_prompt
26
from vllm.logger import init_logger
27
from vllm.lora.request import LoRARequest
28
29
from vllm.model_executor.guided_decoding.guided_fields import (
    GuidedDecodingRequest, LLMGuidedOptions)
30
31
32
from vllm.outputs import (ClassificationRequestOutput, EmbeddingRequestOutput,
                          PoolingRequestOutput, RequestOutput,
                          ScoringRequestOutput)
33
from vllm.pooling_params import PoolingParams
34
from vllm.prompt_adapter.request import PromptAdapterRequest
35
36
from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
                                  RequestOutputKind, SamplingParams)
37
from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
38
39
                                               get_cached_tokenizer)
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
yhu422's avatar
yhu422 committed
40
from vllm.usage.usage_lib import UsageContext
41
from vllm.utils import Counter, deprecate_args, deprecate_kwargs, is_list_of
42

43
44
logger = init_logger(__name__)

45
46

class LLM:
Woosuk Kwon's avatar
Woosuk Kwon committed
47
48
49
50
51
52
53
54
55
56
    """An LLM for generating texts from given prompts and sampling parameters.

    This class includes a tokenizer, a language model (possibly distributed
    across multiple GPUs), and GPU memory space allocated for intermediate
    states (aka KV cache). Given a batch of prompts and sampling parameters,
    this class generates texts from the model, using an intelligent batching
    mechanism and efficient memory management.

    Args:
        model: The name or path of a HuggingFace Transformers model.
57
        tokenizer: The name or path of a HuggingFace Transformers tokenizer.
58
59
        tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer
            if available, and "slow" will always use the slow tokenizer.
60
61
62
        skip_tokenizer_init: If true, skip initialization of tokenizer and
            detokenizer. Expect valid prompt_token_ids and None for prompt
            from the input.
63
64
        trust_remote_code: Trust remote code (e.g., from HuggingFace) when
            downloading the model and tokenizer.
65
66
67
68
        allowed_local_media_path: Allowing API requests to read local images
            or videos from directories specified by the server file system.
            This is a security risk. Should only be enabled in trusted
            environments.
Woosuk Kwon's avatar
Woosuk Kwon committed
69
70
71
        tensor_parallel_size: The number of GPUs to use for distributed
            execution with tensor parallelism.
        dtype: The data type for the model weights and activations. Currently,
Woosuk Kwon's avatar
Woosuk Kwon committed
72
73
74
75
            we support `float32`, `float16`, and `bfloat16`. If `auto`, we use
            the `torch_dtype` attribute specified in the model config file.
            However, if the `torch_dtype` in the config is `float32`, we will
            use `float16` instead.
76
        quantization: The method used to quantize the model weights. Currently,
77
            we support "awq", "gptq", and "fp8" (experimental).
78
79
80
81
            If None, we first check the `quantization_config` attribute in the
            model config file. If that is None, we assume the model weights are
            not quantized and use `dtype` to determine the data type of
            the weights.
Jasmond L's avatar
Jasmond L committed
82
83
        revision: The specific model version to use. It can be a branch name,
            a tag name, or a commit id.
84
85
        tokenizer_revision: The specific tokenizer version to use. It can be a
            branch name, a tag name, or a commit id.
86
87
88
89
90
91
92
93
94
95
96
        seed: The seed to initialize the random number generator for sampling.
        gpu_memory_utilization: The ratio (between 0 and 1) of GPU memory to
            reserve for the model weights, activations, and KV cache. Higher
            values will increase the KV cache size and thus improve the model's
            throughput. However, if the value is too high, it may cause out-of-
            memory (OOM) errors.
        swap_space: The size (GiB) of CPU memory per GPU to use as swap space.
            This can be used for temporarily storing the states of the requests
            when their `best_of` sampling parameters are larger than 1. If all
            requests will have `best_of=1`, you can safely set this to 0.
            Otherwise, too small values may cause out-of-memory (OOM) errors.
97
98
99
100
        cpu_offload_gb: The size (GiB) of CPU memory to use for offloading
            the model weights. This virtually increases the GPU memory space
            you can use to hold the model weights, at the cost of CPU-GPU data
            transfer for every forward pass.
101
102
103
        enforce_eager: Whether to enforce eager execution. If True, we will
            disable CUDA graph and always execute the model in eager mode.
            If False, we will use CUDA graph and eager execution in hybrid.
104
        max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs.
105
            When a sequence has context length larger than this, we fall back
106
107
108
            to eager mode. Additionally for encoder-decoder models, if the
            sequence length of the encoder input is larger than this, we fall
            back to the eager mode.
109
110
111
        disable_custom_all_reduce: See :class:`~vllm.config.ParallelConfig`
        disable_async_output_proc: Disable async output processing.
            This may result in lower performance.
112
113
114
        hf_overrides: If a dictionary, contains arguments to be forwarded to the
            HuggingFace config. If a callable, it is called to update the
            HuggingFace config.
115
116
117
        compilation_config: Either an integer or a dictionary. If it is an
            integer, it is used as the level of compilation optimization. If it
            is a dictionary, it can specify the full compilation configuration.
118
        **kwargs: Arguments for :class:`~vllm.EngineArgs`. (See
119
            :ref:`engine-args`)
nunjunj's avatar
nunjunj committed
120

121
122
123
    Note:
        This class is intended to be used for offline inference. For online
        serving, use the :class:`~vllm.AsyncLLMEngine` class instead.
124
    """
125

126
    DEPRECATE_LEGACY: ClassVar[bool] = True
127
128
    """A flag to toggle whether to deprecate the legacy generate/encode API."""

129
130
131
132
133
134
    DEPRECATE_INIT_POSARGS: ClassVar[bool] = True
    """
    A flag to toggle whether to deprecate positional arguments in
    :meth:`LLM.__init__`.
    """

135
136
137
138
139
140
141
142
143
    @classmethod
    @contextmanager
    def deprecate_legacy_api(cls):
        cls.DEPRECATE_LEGACY = True

        yield

        cls.DEPRECATE_LEGACY = False

144
145
146
147
148
149
150
    @deprecate_args(
        start_index=2,  # Ignore self and model
        is_deprecated=lambda: LLM.DEPRECATE_INIT_POSARGS,
        additional_message=(
            "All positional arguments other than `model` will be "
            "replaced with keyword arguments in an upcoming version."),
    )
151
152
153
    def __init__(
        self,
        model: str,
154
        tokenizer: Optional[str] = None,
155
        tokenizer_mode: str = "auto",
156
        skip_tokenizer_init: bool = False,
157
        trust_remote_code: bool = False,
158
        allowed_local_media_path: str = "",
159
        tensor_parallel_size: int = 1,
Woosuk Kwon's avatar
Woosuk Kwon committed
160
        dtype: str = "auto",
161
        quantization: Optional[str] = None,
162
        revision: Optional[str] = None,
163
        tokenizer_revision: Optional[str] = None,
164
165
        seed: int = 0,
        gpu_memory_utilization: float = 0.9,
166
        swap_space: float = 4,
167
        cpu_offload_gb: float = 0,
168
        enforce_eager: Optional[bool] = None,
169
        max_seq_len_to_capture: int = 8192,
170
        disable_custom_all_reduce: bool = False,
171
        disable_async_output_proc: bool = False,
172
        hf_overrides: Optional[HfOverrides] = None,
173
        mm_processor_kwargs: Optional[Dict[str, Any]] = None,
174
175
        # After positional args are removed, move this right below `model`
        task: TaskOption = "auto",
176
        override_pooler_config: Optional[PoolerConfig] = None,
177
        compilation_config: Optional[Union[int, Dict[str, Any]]] = None,
178
179
        **kwargs,
    ) -> None:
180
181
182
183
        '''
        LLM constructor.

        Note: if enforce_eager is unset (enforce_eager is None)
184
        it defaults to False.
185
186
        '''

187
188
        if "disable_log_stats" not in kwargs:
            kwargs["disable_log_stats"] = True
189

190
191
192
193
194
195
196
        if "worker_cls" in kwargs:
            worker_cls = kwargs["worker_cls"]
            # if the worker_cls is not qualified string name,
            # we serialize it using cloudpickle to avoid pickling issues
            if isinstance(worker_cls, type):
                kwargs["worker_cls"] = cloudpickle.dumps(worker_cls)

197
        if compilation_config is not None:
198
            if isinstance(compilation_config, (int, dict)):
199
200
201
202
                compilation_config_instance = CompilationConfig.from_cli(
                    str(compilation_config))
            else:
                compilation_config_instance = compilation_config
203
204
205
        else:
            compilation_config_instance = None

Zhuohan Li's avatar
Zhuohan Li committed
206
        engine_args = EngineArgs(
207
            model=model,
208
            task=task,
209
            tokenizer=tokenizer,
210
            tokenizer_mode=tokenizer_mode,
211
            skip_tokenizer_init=skip_tokenizer_init,
212
            trust_remote_code=trust_remote_code,
213
            allowed_local_media_path=allowed_local_media_path,
214
215
            tensor_parallel_size=tensor_parallel_size,
            dtype=dtype,
216
            quantization=quantization,
217
            revision=revision,
218
            tokenizer_revision=tokenizer_revision,
219
220
221
            seed=seed,
            gpu_memory_utilization=gpu_memory_utilization,
            swap_space=swap_space,
222
            cpu_offload_gb=cpu_offload_gb,
223
            enforce_eager=enforce_eager,
224
            max_seq_len_to_capture=max_seq_len_to_capture,
225
            disable_custom_all_reduce=disable_custom_all_reduce,
226
            disable_async_output_proc=disable_async_output_proc,
227
            hf_overrides=hf_overrides,
228
            mm_processor_kwargs=mm_processor_kwargs,
229
            override_pooler_config=override_pooler_config,
230
            compilation_config=compilation_config_instance,
231
232
            **kwargs,
        )
Joe Runde's avatar
Joe Runde committed
233
234
235
236
        # Logic to switch between engines is done at runtime instead of import
        # to avoid import order issues
        self.engine_class = self.get_engine_class()
        self.llm_engine = self.engine_class.from_engine_args(
yhu422's avatar
yhu422 committed
237
            engine_args, usage_context=UsageContext.LLM_CLASS)
238

239
240
        self.request_counter = Counter()

Joe Runde's avatar
Joe Runde committed
241
242
243
244
245
246
247
248
    @staticmethod
    def get_engine_class() -> Type[LLMEngine]:
        if envs.VLLM_USE_V1:
            # Lazy import: the v1 package isn't distributed
            from vllm.v1.engine.llm_engine import LLMEngine as V1LLMEngine
            return V1LLMEngine  # type: ignore
        return LLMEngine

249
250
251
252
253
    def get_tokenizer(self) -> AnyTokenizer:
        return self.llm_engine.get_tokenizer_group(TokenizerGroup).tokenizer

    def set_tokenizer(self, tokenizer: AnyTokenizer) -> None:
        tokenizer_group = self.llm_engine.get_tokenizer_group(TokenizerGroup)
254

255
256
257
258
        # While CachedTokenizer is dynamic, have no choice but
        # compare class name. Misjudgment will arise from
        # user-defined tokenizer started with 'Cached'
        if tokenizer.__class__.__name__.startswith("Cached"):
259
            tokenizer_group.tokenizer = tokenizer
260
        else:
261
            tokenizer_group.tokenizer = get_cached_tokenizer(tokenizer)
262

263
264
265
266
267
268
269
    def get_default_sampling_params(self) -> SamplingParams:
        diff_sampling_param = (
            self.llm_engine.model_config.get_diff_sampling_param())
        if diff_sampling_param:
            return SamplingParams.from_optional(**diff_sampling_param)
        return SamplingParams()

270
271
272
273
274
275
276
    @overload
    def generate(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        sampling_params: Optional[Union[SamplingParams,
                                        Sequence[SamplingParams]]] = None,
277
        *,
278
279
        use_tqdm: bool = True,
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
280
281
282
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
283
284
285
    ) -> List[RequestOutput]:
        ...

286
    @overload  # LEGACY: single (prompt + optional token ids)
287
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
288
289
290
291
292
293
294
    def generate(
        self,
        prompts: str,
        sampling_params: Optional[Union[SamplingParams,
                                        List[SamplingParams]]] = None,
        prompt_token_ids: Optional[List[int]] = None,
        use_tqdm: bool = True,
295
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
296
297
298
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
299
300
301
302
    ) -> List[RequestOutput]:
        ...

    @overload  # LEGACY: multi (prompt + optional token ids)
303
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
304
305
    def generate(
        self,
306
        prompts: List[str],
307
308
        sampling_params: Optional[Union[SamplingParams,
                                        List[SamplingParams]]] = None,
309
        prompt_token_ids: Optional[List[List[int]]] = None,
310
        use_tqdm: bool = True,
311
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
312
313
314
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
315
316
317
318
    ) -> List[RequestOutput]:
        ...

    @overload  # LEGACY: single (token ids + optional prompt)
319
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
320
321
322
323
324
325
326
327
    def generate(
        self,
        prompts: Optional[str] = None,
        sampling_params: Optional[Union[SamplingParams,
                                        List[SamplingParams]]] = None,
        *,
        prompt_token_ids: List[int],
        use_tqdm: bool = True,
328
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
329
330
331
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
332
333
334
335
    ) -> List[RequestOutput]:
        ...

    @overload  # LEGACY: multi (token ids + optional prompt)
336
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
337
338
339
340
341
342
343
344
    def generate(
        self,
        prompts: Optional[List[str]] = None,
        sampling_params: Optional[Union[SamplingParams,
                                        List[SamplingParams]]] = None,
        *,
        prompt_token_ids: List[List[int]],
        use_tqdm: bool = True,
345
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
346
347
348
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
349
350
351
352
    ) -> List[RequestOutput]:
        ...

    @overload  # LEGACY: single or multi token ids [pos-only]
353
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
354
355
356
357
358
359
    def generate(
        self,
        prompts: None,
        sampling_params: None,
        prompt_token_ids: Union[List[int], List[List[int]]],
        use_tqdm: bool = True,
360
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
361
362
363
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
        guided_options_request: Optional[Union[LLMGuidedOptions,
                                               GuidedDecodingRequest]] = None,
364
365
366
    ) -> List[RequestOutput]:
        ...

nunjunj's avatar
nunjunj committed
367
368
369
    @deprecate_kwargs(
        "prompt_token_ids",
        is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
370
        additional_message="Please use the 'prompts' parameter instead.",
nunjunj's avatar
nunjunj committed
371
    )
372
373
    def generate(
        self,
374
        prompts: Union[Union[PromptType, Sequence[PromptType]],
375
376
377
378
379
                       Optional[Union[str, List[str]]]] = None,
        sampling_params: Optional[Union[SamplingParams,
                                        Sequence[SamplingParams]]] = None,
        prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
        use_tqdm: bool = True,
380
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
381
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
382
        guided_options_request: Optional[Union[LLMGuidedOptions,
383
384
                                               GuidedDecodingRequest]] = None,
        priority: Optional[List[int]] = None,
385
    ) -> List[RequestOutput]:
Woosuk Kwon's avatar
Woosuk Kwon committed
386
387
        """Generates the completions for the input prompts.

388
        This class automatically batches the given prompts, considering
Woosuk Kwon's avatar
Woosuk Kwon committed
389
390
391
392
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
393
394
395
            prompts: The prompts to the LLM. You may pass a sequence of prompts
                for batch inference. See :class:`~vllm.inputs.PromptType`
                for more details about the format of each prompts.
Woosuk Kwon's avatar
Woosuk Kwon committed
396
            sampling_params: The sampling parameters for text generation. If
nunjunj's avatar
nunjunj committed
397
398
399
                None, we use the default sampling parameters.
                When it is a single value, it is applied to every prompt.
                When it is a list, the list must have the same length as the
400
                prompts and it is paired one by one with the prompt.
Woosuk Kwon's avatar
Woosuk Kwon committed
401
            use_tqdm: Whether to use tqdm to display the progress bar.
402
            lora_request: LoRA request to use for generation, if any.
nunjunj's avatar
nunjunj committed
403
            prompt_adapter_request: Prompt Adapter request to use for
404
                generation, if any.
405
406
            priority: The priority of the requests, if any.
                Only applicable when priority scheduling policy is enabled.
Woosuk Kwon's avatar
Woosuk Kwon committed
407
408

        Returns:
nunjunj's avatar
nunjunj committed
409
            A list of ``RequestOutput`` objects containing the
410
            generated completions in the same order as the input prompts.
411
412
413
414
415

        Note:
            Using ``prompts`` and ``prompt_token_ids`` as keyword parameters is
            considered legacy and may be deprecated in the future. You should
            instead pass them via the ``inputs`` parameter.
416
        """
417
418
        runner_type = self.llm_engine.model_config.runner_type
        if runner_type != "generate":
419
            messages = [
420
                "LLM.generate() is only supported for (conditional) generation "
421
422
423
                "models (XForCausalLM, XForConditionalGeneration).",
            ]

424
425
426
            supported_runner_types = self.llm_engine.model_config \
                .supported_runner_types
            if "generate" in supported_runner_types:
427
                messages.append(
428
429
430
                    "Your model supports the 'generate' runner, but is "
                    f"currently initialized for the '{runner_type}' runner. "
                    "Please initialize vLLM using `--task generate`.")
431
432

            raise ValueError(" ".join(messages))
433

434
        if prompt_token_ids is not None:
435
            parsed_prompts = self._convert_v1_inputs(
436
437
438
439
                prompts=cast(Optional[Union[str, List[str]]], prompts),
                prompt_token_ids=prompt_token_ids,
            )
        else:
440
441
            parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
                                  prompts)
442

443
444
445
446
447
448
449
450
        if isinstance(guided_options_request, dict):
            if len(guided_options_request) > 1:
                raise ValueError(
                    "You can only use one guided decoding but multiple is "
                    f"specified: {guided_options_request}")
            guided_options_request = GuidedDecodingRequest(
                **guided_options_request)

451
452
        if sampling_params is None:
            # Use default sampling params.
453
            sampling_params = self.get_default_sampling_params()
454

455
        self._validate_and_add_requests(
456
            prompts=parsed_prompts,
457
458
            params=sampling_params,
            lora_request=lora_request,
459
            prompt_adapter_request=prompt_adapter_request,
460
461
            guided_options=guided_options_request,
            priority=priority)
462

463
        outputs = self._run_engine(use_tqdm=use_tqdm)
Joe Runde's avatar
Joe Runde committed
464
        return self.engine_class.validate_outputs(outputs, RequestOutput)
465

466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
    def collective_rpc(self,
                       method: str,
                       timeout: Optional[float] = None,
                       args: Tuple = (),
                       kwargs: Optional[Dict] = None) -> List[Any]:
        """
        Run a method on all workers, with homogeneous arguments.
        The main extension point for the LLM entrypoint.
        Users can provide custom worker class through `worker_cls`
        argument, and implement new methods in the worker class.
        Then, users can call the new methods through this API.
        It is recommended to use this API to only pass control messages,
        and set up data-plane communication to pass data.
        """
        return self.llm_engine.model_executor.collective_rpc(
            method, timeout, args, kwargs)

483
484
    def beam_search(
        self,
485
        prompts: List[Union[TokensPrompt, TextPrompt]],
486
        params: BeamSearchParams,
487
488
489
490
491
492
493
    ) -> List[BeamSearchOutput]:
        """
        Generate sequences using beam search.

        Args:
            prompts: A list of prompts. Each prompt can be a string or a list
                of token IDs.
494
495
            params: The beam search parameters.

496
497
498
499
        TODO: how does beam search work together with length penalty, frequency
        penalty, and stopping criteria, etc.?
        """

500
501
502
503
        beam_width = params.beam_width
        max_tokens = params.max_tokens
        temperature = params.temperature
        ignore_eos = params.ignore_eos
504
505
506
507
508
509
        length_penalty = params.length_penalty

        def sort_beams_key(x: BeamSearchSequence) -> float:
            return get_beam_search_score(x.tokens, x.cum_logprob,
                                         tokenizer.eos_token_id,
                                         length_penalty)
510

511
512
513
514
515
516
        tokenizer = self.get_tokenizer()
        # generate 2 * beam_width candidates at each step
        # following the huggingface transformers implementation
        # at https://github.com/huggingface/transformers/blob/e15687fffe5c9d20598a19aeab721ae0a7580f8a/src/transformers/generation/beam_search.py#L534 # noqa
        beam_search_params = SamplingParams(logprobs=2 * beam_width,
                                            max_tokens=1,
517
                                            temperature=temperature)
518
519
520
        instances: List[BeamSearchInstance] = []

        for prompt in prompts:
521
522
523
524
            if is_token_prompt(prompt):
                prompt_tokens = prompt["prompt_token_ids"]
            else:
                prompt_tokens = tokenizer.encode(prompt["prompt"])
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
            instances.append(BeamSearchInstance(prompt_tokens))

        for _ in range(max_tokens):
            all_beams: List[BeamSearchSequence] = list(
                sum((instance.beams for instance in instances), []))
            pos = [0] + list(
                itertools.accumulate(
                    len(instance.beams) for instance in instances))
            instance_start_and_end: List[Tuple[int, int]] = list(
                zip(pos[:-1], pos[1:]))

            if len(all_beams) == 0:
                break

            prompts_batch = [
                TokensPrompt(prompt_token_ids=beam.tokens)
                for beam in all_beams
            ]

            # only runs for one step
            # we don't need to use tqdm here
            output = self.generate(prompts_batch,
                                   sampling_params=beam_search_params,
                                   use_tqdm=False)

            for (start, end), instance in zip(instance_start_and_end,
                                              instances):
                instance_new_beams = []
                for i in range(start, end):
                    current_beam = all_beams[i]
                    result = output[i]

                    if result.outputs[0].logprobs is not None:
                        # if `result.outputs[0].logprobs` is None, it means
                        # the sequence is completed because of the max-model-len
                        # or abortion. we don't need to add it to the new beams.
                        logprobs = result.outputs[0].logprobs[0]
                        for token_id, logprob_obj in logprobs.items():
                            new_beam = BeamSearchSequence(
                                tokens=current_beam.tokens + [token_id],
565
                                logprobs=current_beam.logprobs + [logprobs],
566
567
568
569
570
571
572
573
574
                                cum_logprob=current_beam.cum_logprob +
                                logprob_obj.logprob)

                            if token_id == tokenizer.eos_token_id and \
                                not ignore_eos:
                                instance.completed.append(new_beam)
                            else:
                                instance_new_beams.append(new_beam)
                sorted_beams = sorted(instance_new_beams,
575
                                      key=sort_beams_key,
576
577
578
579
580
581
582
                                      reverse=True)
                instance.beams = sorted_beams[:beam_width]

        outputs = []
        for instance in instances:
            instance.completed.extend(instance.beams)
            sorted_completed = sorted(instance.completed,
583
                                      key=sort_beams_key,
584
585
586
587
588
589
590
591
592
                                      reverse=True)
            best_beams = sorted_completed[:beam_width]

            for beam in best_beams:
                beam.text = tokenizer.decode(beam.tokens)
            outputs.append(BeamSearchOutput(sequences=best_beams))

        return outputs

nunjunj's avatar
nunjunj committed
593
594
    def chat(
        self,
595
596
        messages: Union[List[ChatCompletionMessageParam],
                        List[List[ChatCompletionMessageParam]]],
nunjunj's avatar
nunjunj committed
597
598
599
600
601
        sampling_params: Optional[Union[SamplingParams,
                                        List[SamplingParams]]] = None,
        use_tqdm: bool = True,
        lora_request: Optional[LoRARequest] = None,
        chat_template: Optional[str] = None,
602
        chat_template_content_format: ChatTemplateContentFormatOption = "auto",
603
        add_generation_prompt: bool = True,
604
        continue_final_message: bool = False,
605
        tools: Optional[List[Dict[str, Any]]] = None,
606
        mm_processor_kwargs: Optional[Dict[str, Any]] = None,
nunjunj's avatar
nunjunj committed
607
608
    ) -> List[RequestOutput]:
        """
609
        Generate responses for a chat conversation.
nunjunj's avatar
nunjunj committed
610

611
612
613
614
615
616
        The chat conversation is converted into a text prompt using the
        tokenizer and calls the :meth:`generate` method to generate the
        responses.

        Multi-modal inputs can be passed in the same way you would pass them
        to the OpenAI API.
nunjunj's avatar
nunjunj committed
617
618

        Args:
619
620
621
622
623
            messages: A list of conversations or a single conversation.

              - Each conversation is represented as a list of messages.
              - Each message is a dictionary with 'role' and 'content' keys.

nunjunj's avatar
nunjunj committed
624
625
626
627
628
629
630
631
632
            sampling_params: The sampling parameters for text generation.
                If None, we use the default sampling parameters. When it
                is a single value, it is applied to every prompt. When it
                is a list, the list must have the same length as the
                prompts and it is paired one by one with the prompt.
            use_tqdm: Whether to use tqdm to display the progress bar.
            lora_request: LoRA request to use for generation, if any.
            chat_template: The template to use for structuring the chat.
              If not provided, the model's default chat template will be used.
633
634
635
636
637
638
639
640
            chat_template_content_format: The format to render message content.

              - "string" will render the content as a string.
                Example: ``"Who are you?"``
              - "openai" will render the content as a list of dictionaries,
                similar to OpenAI schema.
                Example: ``[{"type": "text", "text": "Who are you?"}]``

641
            add_generation_prompt: If True, adds a generation template
nunjunj's avatar
nunjunj committed
642
                to each message.
643
            continue_final_message: If True, continues the final message in
644
645
                the conversation instead of starting a new one. Cannot be
                ``True`` if ``add_generation_prompt`` is also ``True``.
646
647
            mm_processor_kwargs: Multimodal processor kwarg overrides for this
                chat request. Only used for offline requests.
nunjunj's avatar
nunjunj committed
648
649
650
651
652

        Returns:
            A list of ``RequestOutput`` objects containing the generated
            responses in the same order as the input messages.
        """
653
        list_of_messages: List[List[ChatCompletionMessageParam]]
nunjunj's avatar
nunjunj committed
654

655
656
657
        # Handle multi and single conversations
        if is_list_of(messages, list):
            # messages is List[List[...]]
658
659
            list_of_messages = cast(List[List[ChatCompletionMessageParam]],
                                    messages)
660
        else:
661
            # messages is List[...]
662
663
664
            list_of_messages = [
                cast(List[ChatCompletionMessageParam], messages)
            ]
665

666
667
668
669
670
671
672
673
        tokenizer = self.get_tokenizer()
        model_config = self.llm_engine.get_model_config()
        resolved_content_format = resolve_chat_template_content_format(
            chat_template,
            chat_template_content_format,
            tokenizer,
        )

674
675
676
        prompts: List[Union[TokensPrompt, TextPrompt]] = []

        for msgs in list_of_messages:
677
678
679
            # NOTE: _parse_chat_message_content_parts() currently doesn't
            # handle mm_processor_kwargs, since there is no implementation in
            # the chat message parsing for it.
680
            conversation, mm_data = parse_chat_messages(
681
682
683
684
685
                msgs,
                model_config,
                tokenizer,
                content_format=resolved_content_format,
            )
686
687
688
689
690
691
692
693

            prompt_data: Union[str, List[int]]
            if isinstance(tokenizer, MistralTokenizer):
                prompt_data = apply_mistral_chat_template(
                    tokenizer,
                    messages=msgs,
                    chat_template=chat_template,
                    add_generation_prompt=add_generation_prompt,
694
                    continue_final_message=continue_final_message,
695
696
697
698
699
700
701
702
                    tools=tools,
                )
            else:
                prompt_data = apply_hf_chat_template(
                    tokenizer,
                    conversation=conversation,
                    chat_template=chat_template,
                    add_generation_prompt=add_generation_prompt,
703
                    continue_final_message=continue_final_message,
704
705
706
707
708
709
710
711
712
713
714
715
                    tools=tools,
                )

            prompt: Union[TokensPrompt, TextPrompt]
            if is_list_of(prompt_data, int):
                prompt = TokensPrompt(prompt_token_ids=prompt_data)
            else:
                prompt = TextPrompt(prompt=prompt_data)

            if mm_data is not None:
                prompt["multi_modal_data"] = mm_data

716
717
718
            if mm_processor_kwargs is not None:
                prompt["mm_processor_kwargs"] = mm_processor_kwargs

719
            prompts.append(prompt)
720

nunjunj's avatar
nunjunj committed
721
        return self.generate(
722
            prompts,
723
            sampling_params=sampling_params,
nunjunj's avatar
nunjunj committed
724
725
726
727
            use_tqdm=use_tqdm,
            lora_request=lora_request,
        )

728
729
730
731
732
733
734
    @overload
    def encode(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
735
        *,
736
737
        use_tqdm: bool = True,
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
738
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
739
740
741
    ) -> List[PoolingRequestOutput]:
        ...

742
    @overload  # LEGACY: single (prompt + optional token ids)
743
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
744
745
746
747
748
749
750
    def encode(
        self,
        prompts: str,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
        prompt_token_ids: Optional[List[int]] = None,
        use_tqdm: bool = True,
751
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
752
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
753
    ) -> List[PoolingRequestOutput]:
754
        ...
755

756
    @overload  # LEGACY: multi (prompt + optional token ids)
757
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
758
759
    def encode(
        self,
760
        prompts: List[str],
761
        pooling_params: Optional[Union[PoolingParams,
762
                                       Sequence[PoolingParams]]] = None,
763
764
        prompt_token_ids: Optional[List[List[int]]] = None,
        use_tqdm: bool = True,
765
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
766
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
767
    ) -> List[PoolingRequestOutput]:
768
769
770
        ...

    @overload  # LEGACY: single (token ids + optional prompt)
771
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
772
773
774
775
776
777
778
779
    def encode(
        self,
        prompts: Optional[str] = None,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
        *,
        prompt_token_ids: List[int],
        use_tqdm: bool = True,
780
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
781
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
782
    ) -> List[PoolingRequestOutput]:
783
784
785
        ...

    @overload  # LEGACY: multi (token ids + optional prompt)
786
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
787
788
789
790
791
792
793
794
    def encode(
        self,
        prompts: Optional[List[str]] = None,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
        *,
        prompt_token_ids: List[List[int]],
        use_tqdm: bool = True,
795
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
796
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
797
    ) -> List[PoolingRequestOutput]:
798
799
800
        ...

    @overload  # LEGACY: single or multi token ids [pos-only]
801
    @deprecated("'prompt_token_ids' will become part of 'prompts'")
802
803
804
805
806
807
    def encode(
        self,
        prompts: None,
        pooling_params: None,
        prompt_token_ids: Union[List[int], List[List[int]]],
        use_tqdm: bool = True,
808
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
809
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
810
    ) -> List[PoolingRequestOutput]:
811
812
        ...

nunjunj's avatar
nunjunj committed
813
814
815
    @deprecate_kwargs(
        "prompt_token_ids",
        is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
816
        additional_message="Please use the 'prompts' parameter instead.",
nunjunj's avatar
nunjunj committed
817
    )
818
819
    def encode(
        self,
820
        prompts: Union[Union[PromptType, Sequence[PromptType]],
821
822
823
824
825
                       Optional[Union[str, List[str]]]] = None,
        pooling_params: Optional[Union[PoolingParams,
                                       Sequence[PoolingParams]]] = None,
        prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
        use_tqdm: bool = True,
826
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
827
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
828
    ) -> List[PoolingRequestOutput]:
829
830
        """Apply pooling to the hidden states corresponding to the input
        prompts.
831

832
        This class automatically batches the given prompts, considering
833
834
835
836
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
837
838
839
            prompts: The prompts to the LLM. You may pass a sequence of prompts
                for batch inference. See :class:`~vllm.inputs.PromptType`
                for more details about the format of each prompts.
840
841
842
843
            pooling_params: The pooling parameters for pooling. If None, we
                use the default pooling parameters.
            use_tqdm: Whether to use tqdm to display the progress bar.
            lora_request: LoRA request to use for generation, if any.
nunjunj's avatar
nunjunj committed
844
            prompt_adapter_request: Prompt Adapter request to use for
845
                generation, if any.
846
847

        Returns:
848
            A list of ``PoolingRequestOutput`` objects containing the
849
            pooled hidden states in the same order as the input prompts.
850
851
852
853
854

        Note:
            Using ``prompts`` and ``prompt_token_ids`` as keyword parameters is
            considered legacy and may be deprecated in the future. You should
            instead pass them via the ``inputs`` parameter.
855
        """
856
857
858
        runner_type = self.llm_engine.model_config.runner_type
        if runner_type != "pooling":
            messages = ["LLM.encode() is only supported for pooling models."]
859

860
861
862
            supported_runner_types = self.llm_engine.model_config \
                .supported_runner_types
            if "pooling" in supported_runner_types:
863
                messages.append(
864
865
866
867
                    "Your model supports the 'pooling' runner, but is "
                    f"currently initialized for the '{runner_type}' runner. "
                    "Please initialize vLLM using `--task embed`, "
                    "`--task classify`, `--task score` etc.")
868
869

            raise ValueError(" ".join(messages))
870

871
        if prompt_token_ids is not None:
872
            parsed_prompts = self._convert_v1_inputs(
873
874
875
876
                prompts=cast(Optional[Union[str, List[str]]], prompts),
                prompt_token_ids=prompt_token_ids,
            )
        else:
877
878
            parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
                                  prompts)
879

880
881
882
883
        if pooling_params is None:
            # Use default pooling params.
            pooling_params = PoolingParams()

884
        self._validate_and_add_requests(
885
            prompts=parsed_prompts,
886
887
            params=pooling_params,
            lora_request=lora_request,
888
            prompt_adapter_request=prompt_adapter_request,
889
890
        )

891
        outputs = self._run_engine(use_tqdm=use_tqdm)
Joe Runde's avatar
Joe Runde committed
892
        return self.engine_class.validate_outputs(outputs,
893
                                                  PoolingRequestOutput)
894

895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
    def embed(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        *,
        use_tqdm: bool = True,
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
    ) -> List[EmbeddingRequestOutput]:
        """
        Generate an embedding vector for each prompt.

        This class automatically batches the given prompts, considering
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
            prompts: The prompts to the LLM. You may pass a sequence of prompts
                for batch inference. See :class:`~vllm.inputs.PromptType`
                for more details about the format of each prompts.
            use_tqdm: Whether to use tqdm to display the progress bar.
            lora_request: LoRA request to use for generation, if any.
            prompt_adapter_request: Prompt Adapter request to use for
                generation, if any.

        Returns:
            A list of ``EmbeddingRequestOutput`` objects containing the
            embedding vectors in the same order as the input prompts.
        """
        if self.llm_engine.model_config.task != "embed":
            raise ValueError(
                "Embedding API is only enabled for `--task embed`")

        items = self.encode(prompts,
                            use_tqdm=use_tqdm,
                            lora_request=lora_request,
                            prompt_adapter_request=prompt_adapter_request)

        return [EmbeddingRequestOutput.from_base(item) for item in items]

    def classify(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
        /,
        *,
        use_tqdm: bool = True,
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
    ) -> List[ClassificationRequestOutput]:
        """
        Generate class logits for each prompt.

        This class automatically batches the given prompts, considering
        the memory constraint. For the best performance, put all of your prompts
        into a single list and pass it to this method.

        Args:
            prompts: The prompts to the LLM. You may pass a sequence of prompts
                for batch inference. See :class:`~vllm.inputs.PromptType`
                for more details about the format of each prompts.
            use_tqdm: Whether to use tqdm to display the progress bar.
            lora_request: LoRA request to use for generation, if any.
            prompt_adapter_request: Prompt Adapter request to use for
                generation, if any.

        Returns:
            A list of ``ClassificationRequestOutput`` objects containing the
            embedding vectors in the same order as the input prompts.
        """
        if self.llm_engine.model_config.task != "classify":
            raise ValueError(
                "Classification API is only enabled for `--task classify`")

        items = self.encode(prompts,
                            use_tqdm=use_tqdm,
                            lora_request=lora_request,
                            prompt_adapter_request=prompt_adapter_request)

        return [ClassificationRequestOutput.from_base(item) for item in items]

975
976
977
978
979
    def score(
        self,
        text_1: Union[SingletonPrompt, Sequence[SingletonPrompt]],
        text_2: Union[SingletonPrompt, Sequence[SingletonPrompt]],
        /,
980
        *,
981
982
983
984
        truncate_prompt_tokens: Optional[int] = None,
        use_tqdm: bool = True,
        lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
985
986
    ) -> List[ScoringRequestOutput]:
        """Generate similarity scores for all pairs ``<text,text_pair>``.
987

988
989
990
991
        The inputs can be ``1 -> 1``, ``1 -> N`` or ``N -> N``.
        In the ``1 - N`` case the ``text_1`` sentence will be replicated ``N``
        times to pair with the ``text_2`` sentences.
        The input pairs are used to build a list of prompts for the
992
993
994
995
996
997
        cross encoder model. This class automatically batches the prompts,
        considering the memory constraint. For the best performance, put all
        of your texts into a single list and pass it to this method.

        Args:
            text_1: can be a single prompt or a list of prompts, in which
998
                case it has to have the same length as the ``text_2`` list
999
1000
1001
1002
1003
1004
1005
1006
1007
            text_2: The texts to pair with the query to form the input
                to the LLM. See :class:`~vllm.inputs.PromptType` for
                more details about the format of each prompts.
            use_tqdm: Whether to use tqdm to display the progress bar.
            lora_request: LoRA request to use for generation, if any.
            prompt_adapter_request: Prompt Adapter request to use for
                generation, if any.

        Returns:
1008
            A list of ``ScoringRequestOutput`` objects containing the
1009
1010
            generated scores in the same order as the input prompts.
        """
1011
1012
1013
        runner_type = self.llm_engine.model_config.runner_type
        if runner_type != "pooling":
            messages = ["LLM.score() is only supported for pooling models."]
1014

1015
1016
1017
            supported_runner_types = self.llm_engine.model_config \
                .supported_runner_types
            if "pooling" in supported_runner_types:
1018
                messages.append(
1019
1020
1021
1022
                    "Your model supports the 'pooling' runner, but is "
                    f"currently initialized for the '{runner_type}' runner. "
                    "Please initialize vLLM using `--task embed`, "
                    "`--task classify`, `--task score` etc.")
1023
1024
1025
1026

            raise ValueError(" ".join(messages))

        if not self.llm_engine.model_config.is_cross_encoder:
1027
            raise ValueError("Your model does not support cross encoding")
1028
1029
        if self.llm_engine.model_config.task != "score":
            raise ValueError("Score API is only enabled for `--task score`")
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099

        tokenizer = self.llm_engine.get_tokenizer()

        if isinstance(tokenizer, MistralTokenizer):
            raise ValueError(
                "MistralTokenizer not supported for cross-encoding")

        # the tokenizer for models such as
        # "cross-encoder/ms-marco-MiniLM-L-6-v2" doesn't support passing
        # lists of tokens to the `text` and `text_pair` kwargs
        def ensure_str(prompt: SingletonPrompt):
            if isinstance(prompt, dict):
                if "multi_modal_data" in prompt:
                    raise ValueError("Multi-modal prompt is not "
                                     "supported for cross encoding")
                elif "prompt_token_ids" in prompt:
                    prompt = tokenizer.decode(
                        cast(TokensPrompt, prompt)["prompt_token_ids"])
                elif "prompt" in prompt:
                    prompt = cast(TextPrompt, prompt)["prompt"]
            assert type(prompt) is str
            return prompt

        if isinstance(text_1, (str, dict)):
            # Convert a single prompt to a list.
            text_1 = [text_1]
        text_1 = [ensure_str(t) for t in text_1]

        if isinstance(text_2, (str, dict)):
            # Convert a single prompt to a list.
            text_2 = [text_2]
        text_2 = [ensure_str(t) for t in text_2]

        if len(text_1) > 1 and len(text_1) != len(text_2):
            raise ValueError("Input lengths must be either 1:1, 1:N or N:N")
        if len(text_1) == 0:
            raise ValueError("At least one text element must be given")
        if len(text_2) == 0:
            raise ValueError("At least one text_pair element must be given")

        if len(text_1) == 1:
            text_1 = text_1 * len(text_2)

        input_pairs = [(t1, t2) for t1, t2 in zip(text_1, text_2)]
        pooling_params = PoolingParams()

        tokenization_kwargs: Dict[str, Any] = {}
        if truncate_prompt_tokens is not None:
            tokenization_kwargs["truncation"] = True
            tokenization_kwargs["max_length"] = truncate_prompt_tokens

        parsed_prompts = []

        for q, t in input_pairs:
            prompt_inputs = tokenizer(text=q,
                                      text_pair=t,
                                      **tokenization_kwargs)
            engine_prompt = TokensPrompt(
                prompt_token_ids=prompt_inputs["input_ids"],
                token_type_ids=prompt_inputs.get("token_type_ids"))
            parsed_prompts.append(engine_prompt)

        self._validate_and_add_requests(
            prompts=parsed_prompts,
            params=pooling_params,
            lora_request=lora_request,
            prompt_adapter_request=prompt_adapter_request,
        )

        outputs = self._run_engine(use_tqdm=use_tqdm)
1100
1101
1102
1103
        items = self.engine_class.validate_outputs(outputs,
                                                   PoolingRequestOutput)

        return [ScoringRequestOutput.from_base(item) for item in items]
1104

1105
1106
1107
1108
1109
1110
    def start_profile(self) -> None:
        self.llm_engine.start_profile()

    def stop_profile(self) -> None:
        self.llm_engine.stop_profile()

1111
1112
    # LEGACY
    def _convert_v1_inputs(
1113
1114
        self,
        prompts: Optional[Union[str, List[str]]],
1115
1116
1117
        prompt_token_ids: Optional[Union[List[int], List[List[int]]]],
    ):
        # skip_tokenizer_init is now checked in engine
1118

1119
1120
1121
1122
1123
1124
        if prompts is not None:
            prompts = [p["content"] for p in parse_and_batch_prompt(prompts)]
        if prompt_token_ids is not None:
            prompt_token_ids = [
                p["content"] for p in parse_and_batch_prompt(prompt_token_ids)
            ]
1125

1126
        num_requests = None
1127
1128
        if prompts is not None:
            num_requests = len(prompts)
1129
1130
1131
1132
1133
1134
        if prompt_token_ids is not None:
            if (num_requests is not None
                    and num_requests != len(prompt_token_ids)):
                raise ValueError("The lengths of prompts and prompt_token_ids "
                                 "must be the same.")

1135
            num_requests = len(prompt_token_ids)
1136
1137
1138
1139
        if num_requests is None:
            raise ValueError("Either prompts or prompt_token_ids must be "
                             "provided.")

1140
        parsed_prompts: List[PromptType] = []
1141
        for i in range(num_requests):
1142
            item: PromptType
1143

1144
            if prompts is not None:
1145
1146
1147
                item = TextPrompt(prompt=prompts[i])
            elif prompt_token_ids is not None:
                item = TokensPrompt(prompt_token_ids=prompt_token_ids[i])
1148
            else:
1149
                raise AssertionError
1150

1151
            parsed_prompts.append(item)
1152

1153
        return parsed_prompts
1154
1155
1156

    def _validate_and_add_requests(
        self,
1157
        prompts: Union[PromptType, Sequence[PromptType]],
1158
1159
        params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
                      Sequence[PoolingParams]],
1160
        lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
1161
        prompt_adapter_request: Optional[PromptAdapterRequest],
1162
        guided_options: Optional[GuidedDecodingRequest] = None,
1163
        priority: Optional[List[int]] = None,
1164
    ) -> None:
1165
1166
1167
1168
1169
1170
1171
1172
        if guided_options is not None:
            warnings.warn(
                "guided_options_request is deprecated, use "
                "SamplingParams.guided_decoding instead",
                DeprecationWarning,
                stacklevel=2,
            )

1173
        if isinstance(prompts, (str, dict)):
1174
            # Convert a single prompt to a list.
1175
            prompts = [prompts]
1176

1177
        num_requests = len(prompts)
1178
1179
        if isinstance(params, list) and len(params) != num_requests:
            raise ValueError("The lengths of prompts and params "
1180
                             "must be the same.")
1181
1182
1183
1184
        if isinstance(lora_request,
                      list) and len(lora_request) != num_requests:
            raise ValueError("The lengths of prompts and lora_request "
                             "must be the same.")
1185

1186
1187
        for sp in params if isinstance(params, list) else (params, ):
            if isinstance(sp, SamplingParams):
1188
                self._add_guided_params(sp, guided_options)
1189
1190
1191

                # We only care about the final output
                sp.output_kind = RequestOutputKind.FINAL_ONLY
1192

Zhuohan Li's avatar
Zhuohan Li committed
1193
        # Add requests to the engine.
1194
        for i, prompt in enumerate(prompts):
1195
            self._add_request(
1196
                prompt,
1197
                params[i] if isinstance(params, Sequence) else params,
1198
1199
                lora_request=lora_request[i] if isinstance(
                    lora_request, Sequence) else lora_request,
nunjunj's avatar
nunjunj committed
1200
                prompt_adapter_request=prompt_adapter_request,
1201
                priority=priority[i] if priority else 0,
nunjunj's avatar
nunjunj committed
1202
            )
1203

1204
    def _add_request(
nunjunj's avatar
nunjunj committed
1205
        self,
1206
        prompt: PromptType,
nunjunj's avatar
nunjunj committed
1207
        params: Union[SamplingParams, PoolingParams],
1208
        lora_request: Optional[LoRARequest] = None,
nunjunj's avatar
nunjunj committed
1209
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
1210
        priority: int = 0,
1211
1212
    ) -> None:
        request_id = str(next(self.request_counter))
1213
1214
        self.llm_engine.add_request(
            request_id,
1215
            prompt,
1216
1217
            params,
            lora_request=lora_request,
nunjunj's avatar
nunjunj committed
1218
            prompt_adapter_request=prompt_adapter_request,
1219
            priority=priority,
nunjunj's avatar
nunjunj committed
1220
        )
1221

1222
    def _add_guided_params(
1223
1224
1225
            self,
            params: SamplingParams,
            guided_options: Optional[GuidedDecodingRequest] = None):
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
        if guided_options is None:
            return params

        if params.guided_decoding is not None:
            raise ValueError("Cannot set both guided_options_request and"
                             "params.guided_decoding.")

        params.guided_decoding = GuidedDecodingParams(
            json=guided_options.guided_json,
            regex=guided_options.guided_regex,
            choice=guided_options.guided_choice,
            grammar=guided_options.guided_grammar,
            json_object=guided_options.guided_json_object,
            backend=guided_options.guided_decoding_backend,
            whitespace_pattern=guided_options.guided_whitespace_pattern)
1241
1242
        return params

1243
    def _run_engine(
1244
            self, *, use_tqdm: bool
1245
    ) -> List[Union[RequestOutput, PoolingRequestOutput]]:
1246
1247
        # Initialize tqdm.
        if use_tqdm:
Zhuohan Li's avatar
Zhuohan Li committed
1248
            num_requests = self.llm_engine.get_num_unfinished_requests()
1249
1250
1251
1252
            pbar = tqdm(
                total=num_requests,
                desc="Processed prompts",
                dynamic_ncols=True,
1253
1254
                postfix=(f"est. speed input: {0:.2f} toks/s, "
                         f"output: {0:.2f} toks/s"),
1255
            )
1256

Zhuohan Li's avatar
Zhuohan Li committed
1257
        # Run the engine.
1258
        outputs: List[Union[RequestOutput, PoolingRequestOutput]] = []
1259
1260
        total_in_toks = 0
        total_out_toks = 0
Zhuohan Li's avatar
Zhuohan Li committed
1261
1262
        while self.llm_engine.has_unfinished_requests():
            step_outputs = self.llm_engine.step()
1263
            for output in step_outputs:
1264
                if output.finished:
1265
1266
                    outputs.append(output)
                    if use_tqdm:
1267
1268
                        if isinstance(output, RequestOutput):
                            # Calculate tokens only for RequestOutput
1269
                            assert output.prompt_token_ids is not None
1270
1271
1272
                            total_in_toks += len(output.prompt_token_ids)
                            in_spd = total_in_toks / pbar.format_dict["elapsed"]
                            total_out_toks += sum(
1273
                                len(stp.token_ids) for stp in output.outputs)
nunjunj's avatar
nunjunj committed
1274
1275
                            out_spd = (total_out_toks /
                                       pbar.format_dict["elapsed"])
1276
1277
1278
                            pbar.postfix = (
                                f"est. speed input: {in_spd:.2f} toks/s, "
                                f"output: {out_spd:.2f} toks/s")
1279
                        pbar.update(1)
1280

1281
1282
        if use_tqdm:
            pbar.close()
1283
1284
1285
        # Sort the outputs by request ID.
        # This is necessary because some requests may be finished earlier than
        # its previous requests.
1286
        return sorted(outputs, key=lambda x: int(x.request_id))