preprocess.py 33.6 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
from collections.abc import Mapping
6
from typing import Any, Optional, Union, cast
7
8
9
10
11
12

from typing_extensions import assert_never

from vllm.config import ModelConfig
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
13
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
14
15
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalEncDecInputs,
                                    MultiModalInputs)
16
from vllm.prompt_adapter.request import PromptAdapterRequest
17
from vllm.transformers_utils.tokenizer import AnyTokenizer
18
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
19

20
21
22
23
24
from .data import (DecoderOnlyInputs, EmbedsInputs, EmbedsPrompt,
                   EncoderDecoderInputs, ProcessorInputs, PromptType,
                   SingletonInputs, SingletonPrompt, TextPrompt, TokenInputs,
                   TokensPrompt, embeds_inputs, token_inputs)
from .parse import is_explicit_encoder_decoder_prompt, parse_singleton_prompt
25
26
27
28
29
30
31
32
33

logger = init_logger(__name__)


class InputPreprocessor:

    def __init__(
        self,
        model_config: ModelConfig,
34
        tokenizer: Optional[TokenizerGroup],
35
        mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
36
37
38
39
40
    ) -> None:
        super().__init__()

        self.model_config = model_config
        self.tokenizer = tokenizer
41
        self.mm_registry = mm_registry
42

43
    def get_tokenizer_group(self) -> TokenizerGroup:
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
        if self.tokenizer is None:
            raise ValueError("You cannot pass text prompts when "
                             "`skip_tokenizer_init` is True")

        return self.tokenizer

    def get_bos_token_id(self,
                         lora_request: Optional[LoRARequest] = None
                         ) -> Optional[int]:
        if self.tokenizer is None:
            logger.warning("Using None for BOS token id because tokenizer "
                           "is not initialized")
            return None

        return self.tokenizer.get_lora_tokenizer(lora_request).bos_token_id

    def get_eos_token_id(self,
                         lora_request: Optional[LoRARequest] = None
                         ) -> Optional[int]:
        if self.tokenizer is None:
            logger.warning("Using None for EOS token id because tokenizer "
                           "is not initialized")
            return None

        return self.tokenizer.get_lora_tokenizer(lora_request).eos_token_id

    def get_decoder_start_token_id(self) -> Optional[int]:
71
        """
72
73
74
        Obtain the decoder start token id employed by an encoder/decoder
        model. Returns None for non-encoder/decoder models or if the
        model config is unavailable.
75
        """
76

77
        if not self.model_config.is_encoder_decoder:
78
79
80
            logger.warning_once(
                "Using None for decoder start token id because "
                "this is not an encoder/decoder model.")
81
82
            return None

83
        if self.model_config is None or self.model_config.hf_config is None:
84
85
86
            logger.warning_once(
                "Using None for decoder start token id because "
                "model config is not available.")
87
88
89
            return None

        dec_start_token_id = getattr(self.model_config.hf_config,
90
                                     "decoder_start_token_id", None)
91
        if dec_start_token_id is None:
92
93
94
95
            logger.warning_once(
                "Falling back on <BOS> for decoder start token "
                "id because decoder start token id is not "
                "available.")
96
97
98
99
            dec_start_token_id = self.get_bos_token_id()

        return dec_start_token_id

100
    def _get_default_enc_dec_decoder_prompt(self) -> list[int]:
101
        """
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
        Specifically for encoder/decoder models:
        generate a default decoder prompt for when
        the user specifies only the encoder prompt.

        Encoder/decoder models utilize the decoder
        prompt in different ways; as new models are
        added, it is intended that this function
        will be extended to produce differing
        default decoder prompts, depending on the
        model variety.

        Absent a special case, the default behavior
        of this method is to mirror the behavior of
        the HuggingFace (HF) GenerationMixin for a None
        decoder prompt, which is to employ a logit processor
        setting to force the first decoded token to be <BOS>.
        Here, this behavior is approximated by having the
        "default" decoder prompt be <BOS>.

        However, it is possible that in the future
122
        other models may have different or more
123
124
125
126
127
128
129
        complex logic for the default decoder prompt.
        This motivates having a special helper method
        for default decoder prompts.

        Returns:

        * prompt_token_ids
130
        """
131
132
133
134
135
136
137

        bos_token_id = self.get_bos_token_id()
        assert bos_token_id is not None
        return [bos_token_id]

    def _prepare_decoder_input_ids_for_generation(
        self,
138
139
        decoder_input_ids: Optional[list[int]],
    ) -> list[int]:
140
141
142
        """
        Prepares `decoder_input_ids` for generation with encoder-decoder models.

143
144
145
146
        Based on:
        https://github.com/huggingface/transformers/blob/4037a2b5b1278736e566aec12e169100275545ea/src/transformers/generation/utils.py
        specifically,
        `GenerationMixin._prepare_decoder_input_ids_for_generation()`.
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164

        Arguments:

        * decoder_input_ids: input token ids to preprocess

        Returns:

        * Processed token list
        """

        decoder_start_token_id = self.get_decoder_start_token_id()
        assert decoder_start_token_id is not None

        if decoder_input_ids is None:
            # no decoder prompt input ->
            # use decoder_start_token_id as decoder_input_ids
            decoder_input_ids = self._get_default_enc_dec_decoder_prompt()

165
166
        if (len(decoder_input_ids) == 0
                or decoder_input_ids[0] != decoder_start_token_id):
167
168
169
170
171
172
            decoder_input_ids = [decoder_start_token_id] + decoder_input_ids

        return decoder_input_ids

    def _apply_prompt_adapter(
        self,
173
        prompt_token_ids: list[int],
174
        prompt_adapter_request: Optional[PromptAdapterRequest],
175
    ) -> list[int]:
176
177
178
179
180
181
182
        if prompt_adapter_request:
            prompt_token_ids = (
                [0] * prompt_adapter_request.prompt_adapter_num_virtual_tokens
                + prompt_token_ids)

        return prompt_token_ids

183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
    def _get_tokenization_kw(
        self,
        overrides: Optional[dict[str, Any]] = None,
    ) -> dict[str, Any]:
        kwargs = dict[str, Any]()

        if self.model_config.hf_config.model_type == "whisper":
            # For Whisper, special tokens should be provided by the user based
            # on the task and language of their request. Also needed to avoid
            # appending an EOS token to the prompt which disrupts generation.
            kwargs["add_special_tokens"] = False

        if overrides:
            kwargs.update(overrides)

        return kwargs

200
201
202
203
    def _tokenize_prompt(
        self,
        prompt: str,
        lora_request: Optional[LoRARequest],
204
        tokenization_kwargs: Optional[dict[str, Any]] = None,
205
    ) -> list[int]:
206
207
208
209
210
        """
        Apply the model's tokenizer to a text prompt, returning the
        corresponding token IDs.
        """
        tokenizer = self.get_tokenizer_group()
211
        tokenization_kwargs = self._get_tokenization_kw(tokenization_kwargs)
212

213
        encoder_config = self.model_config.encoder_config
214

215
        if encoder_config and encoder_config.get("do_lower_case", False):
216
217
            prompt = prompt.lower()

218
        return tokenizer.encode(prompt=prompt,
219
                                lora_request=lora_request,
220
                                **tokenization_kwargs)
221
222
223
224
225

    async def _tokenize_prompt_async(
        self,
        prompt: str,
        lora_request: Optional[LoRARequest],
226
        tokenization_kwargs: Optional[dict[str, Any]] = None,
227
    ) -> list[int]:
228
229
230
231
        """
        Async version of
        [`_tokenize_prompt`][vllm.inputs.preprocess.InputPreprocessor._tokenize_prompt].
        """
232
        tokenizer = self.get_tokenizer_group()
233
        tokenization_kwargs = self._get_tokenization_kw(tokenization_kwargs)
234
235
236
237

        return await tokenizer.encode_async(prompt=prompt,
                                            lora_request=lora_request,
                                            **tokenization_kwargs)
238

239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
    def _get_mm_tokenizer(
        self,
        lora_request: Optional[LoRARequest],
    ) -> AnyTokenizer:
        # PrithviGeoSpatialMAE needs to be initialized without a tokenizer
        # while using also multi-modal input
        if not self.tokenizer:
            return cast(AnyTokenizer, object())  # Dummy

        tokenizer_group = self.get_tokenizer_group()
        return tokenizer_group.get_lora_tokenizer(lora_request)

    async def _get_mm_tokenizer_async(
        self,
        lora_request: Optional[LoRARequest],
    ) -> AnyTokenizer:
        # PrithviGeoSpatialMAE needs to be initialized without a tokenizer
        # while using also multi-modal input
        if not self.tokenizer:
            return cast(AnyTokenizer, object())  # Dummy

        tokenizer_group = self.get_tokenizer_group()
        return await tokenizer_group.get_lora_tokenizer_async(lora_request)

263
264
    def _process_multimodal(
        self,
265
        prompt: Union[str, list[int]],
266
267
268
        mm_data: MultiModalDataDict,
        mm_processor_kwargs: Optional[Mapping[str, object]],
        lora_request: Optional[LoRARequest],
269
        return_mm_hashes: bool = False,
270
    ) -> MultiModalInputs:
271
272
273
274
        """
        Apply the model's multi-modal processor to a multi-modal prompt,
        returning the corresponding token IDs and metadata.
        """
275
        tokenizer = self._get_mm_tokenizer(lora_request)
276

277
278
        mm_processor = self.mm_registry.create_processor(self.model_config,
                                                         tokenizer=tokenizer)
279
280
281
282

        if mm_processor_kwargs is None:
            mm_processor_kwargs = {}

283
284
        return mm_processor.apply(prompt, mm_data, mm_processor_kwargs,
                                  return_mm_hashes)
285
286
287

    async def _process_multimodal_async(
        self,
288
        prompt: Union[str, list[int]],
289
290
291
        mm_data: MultiModalDataDict,
        mm_processor_kwargs: Optional[Mapping[str, object]],
        lora_request: Optional[LoRARequest],
292
        return_mm_hashes: bool = False,
293
    ) -> MultiModalInputs:
294
295
296
297
        """
        Async version of
        [`_process_multimodal`][vllm.inputs.preprocess.InputPreprocessor._process_multimodal].
        """
298
        tokenizer = await self._get_mm_tokenizer_async(lora_request)
299

300
301
        mm_processor = self.mm_registry.create_processor(self.model_config,
                                                         tokenizer=tokenizer)
302
303
304
        if mm_processor_kwargs is None:
            mm_processor_kwargs = {}

305
306
        return mm_processor.apply(prompt, mm_data, mm_processor_kwargs,
                                  return_mm_hashes)
307

308
309
310
311
    def _process_embeds(
        self,
        parsed_content: EmbedsPrompt,
    ) -> EmbedsInputs:
312
313
314
        if not self.model_config.enable_prompt_embeds:
            raise ValueError("You must set `--enable-prompt-embeds` to input "
                             "`prompt_embeds`.")
315
316

        prompt_embeds = parsed_content["prompt_embeds"]
317

318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
        # prompt_embeds must be (seq_len, hidden_size), but if the user
        # passes in a batch of size 1, i.e. (1, seq_len, hidden_size),
        # we can unambiguously process the intent by squeezing the batch
        # dimension.
        if prompt_embeds.ndim == 3:
            prompt_embeds = prompt_embeds.squeeze(dim=0)

        if prompt_embeds.ndim != 2:
            raise ValueError(
                "prompt_embeds must be of shape (seq_len, hidden_size).")

        return embeds_inputs(prompt_embeds=prompt_embeds,
                             cache_salt=parsed_content.get("cache_salt"))

    async def _process_embeds_async(
        self,
        parsed_content: EmbedsPrompt,
    ) -> EmbedsInputs:
        return self._process_embeds(parsed_content)

    def _process_tokens(
        self,
        parsed_content: TokensPrompt,
        lora_request: Optional[LoRARequest] = None,
        return_mm_hashes: bool = False,
    ) -> Union[TokenInputs, MultiModalInputs]:
        prompt_token_ids = parsed_content["prompt_token_ids"]
        token_type_ids = parsed_content.get("token_type_ids")

        inputs: Union[TokenInputs, MultiModalInputs]
        if multi_modal_data := parsed_content.get("multi_modal_data"):
            inputs = self._process_multimodal(
                prompt_token_ids,
                multi_modal_data,
                parsed_content.get("mm_processor_kwargs"),
                lora_request=lora_request,
                return_mm_hashes=return_mm_hashes,
            )
356
        else:
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
            inputs = token_inputs(
                prompt_token_ids=prompt_token_ids,
                token_type_ids=token_type_ids,
            )

        if cache_salt := parsed_content.get("cache_salt"):
            inputs["cache_salt"] = cache_salt

        return inputs

    async def _process_tokens_async(
        self,
        parsed_content: TokensPrompt,
        lora_request: Optional[LoRARequest] = None,
        return_mm_hashes: bool = False,
    ) -> Union[TokenInputs, MultiModalInputs]:
        prompt_token_ids = parsed_content["prompt_token_ids"]
        token_type_ids = parsed_content.get("token_type_ids")

        inputs: Union[TokenInputs, MultiModalInputs]
        if multi_modal_data := parsed_content.get("multi_modal_data"):
            inputs = await self._process_multimodal_async(
                prompt_token_ids,
                multi_modal_data,
                parsed_content.get("mm_processor_kwargs"),
                lora_request=lora_request,
                return_mm_hashes=return_mm_hashes,
            )
        else:
            inputs = token_inputs(
                prompt_token_ids=prompt_token_ids,
                token_type_ids=token_type_ids,
            )

        if cache_salt := parsed_content.get("cache_salt"):
            inputs["cache_salt"] = cache_salt

        return inputs

    def _process_text(
        self,
        parsed_content: TextPrompt,
        tokenization_kwargs: Optional[dict[str, Any]] = None,
        lora_request: Optional[LoRARequest] = None,
        return_mm_hashes: bool = False,
    ) -> Union[TokenInputs, MultiModalInputs]:
        prompt_text = parsed_content["prompt"]

        inputs: Union[TokenInputs, MultiModalInputs]
        if multi_modal_data := parsed_content.get("multi_modal_data"):
            inputs = self._process_multimodal(
                prompt_text,
                multi_modal_data,
                parsed_content.get("mm_processor_kwargs"),
                lora_request=lora_request,
                return_mm_hashes=return_mm_hashes,
            )
        else:
            prompt_token_ids = self._tokenize_prompt(
                prompt_text,
                lora_request=lora_request,
                tokenization_kwargs=tokenization_kwargs,
            )
            inputs = token_inputs(
                prompt=prompt_text,
                prompt_token_ids=prompt_token_ids,
            )

        if cache_salt := parsed_content.get("cache_salt"):
            inputs["cache_salt"] = cache_salt

        return inputs
429

430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
    async def _process_text_async(
        self,
        parsed_content: TextPrompt,
        tokenization_kwargs: Optional[dict[str, Any]] = None,
        lora_request: Optional[LoRARequest] = None,
        return_mm_hashes: bool = False,
    ) -> Union[TokenInputs, MultiModalInputs]:
        prompt_text = parsed_content["prompt"]

        inputs: Union[TokenInputs, MultiModalInputs]
        if multi_modal_data := parsed_content.get("multi_modal_data"):
            inputs = await self._process_multimodal_async(
                prompt_text,
                multi_modal_data,
                parsed_content.get("mm_processor_kwargs"),
                lora_request=lora_request,
                return_mm_hashes=return_mm_hashes,
            )
        else:
            prompt_token_ids = await self._tokenize_prompt_async(
                prompt_text,
                lora_request=lora_request,
                tokenization_kwargs=tokenization_kwargs,
            )
            inputs = token_inputs(
                prompt=prompt_text,
                prompt_token_ids=prompt_token_ids,
            )

        if cache_salt := parsed_content.get("cache_salt"):
            inputs["cache_salt"] = cache_salt

        return inputs
463

464
    def _prompt_to_llm_inputs(
465
        self,
466
        prompt: SingletonPrompt,
467
        tokenization_kwargs: Optional[dict[str, Any]] = None,
468
        lora_request: Optional[LoRARequest] = None,
469
        return_mm_hashes: bool = False,
470
    ) -> SingletonInputs:
471
472
        """
        Extract the singleton inputs from a prompt.
473
474
475

        Arguments:

476
        * prompt: single encoder or decoder input prompt
477
        * lora_request: this is only valid for decoder prompts
478
        * return_mm_hashes: whether to return multimodal hashes
479
480
481

        Returns:

482
        * [`SingletonInputs`][vllm.inputs.data.SingletonInputs] instance
483
        """
484
        parsed = parse_singleton_prompt(prompt)
485
486

        if parsed["type"] == "embeds":
487
488
489
490
            return self._process_embeds(parsed["content"])
        if parsed["type"] == "tokens":
            return self._process_tokens(
                parsed["content"],
491
                lora_request=lora_request,
492
                return_mm_hashes=return_mm_hashes,
493
            )
494
495
496
497
        if parsed["type"] == "text":
            return self._process_text(
                parsed["content"],
                tokenization_kwargs=tokenization_kwargs,
498
                lora_request=lora_request,
499
500
501
502
503
                return_mm_hashes=return_mm_hashes,
            )
        if parsed["type"] == "str":
            return self._process_text(
                TextPrompt(prompt=parsed["content"]),
504
                tokenization_kwargs=tokenization_kwargs,
505
506
                lora_request=lora_request,
                return_mm_hashes=return_mm_hashes,
507
            )
508

509
510
        assert_never(parsed)

511
    async def _prompt_to_llm_inputs_async(
512
        self,
513
        prompt: SingletonPrompt,
514
        tokenization_kwargs: Optional[dict[str, Any]] = None,
515
        lora_request: Optional[LoRARequest] = None,
516
        return_mm_hashes: bool = False,
517
    ) -> SingletonInputs:
518
519
520
521
        """
        Async version of
        [`_prompt_to_llm_inputs`][vllm.inputs.preprocess.InputPreprocessor._prompt_to_llm_inputs].
        """
522
        parsed = parse_singleton_prompt(prompt)
523

524
        if parsed["type"] == "embeds":
525
526
527
528
            return await self._process_embeds_async(parsed["content"])
        if parsed["type"] == "tokens":
            return await self._process_tokens_async(
                parsed["content"],
529
530
                lora_request=lora_request,
                return_mm_hashes=return_mm_hashes,
531
            )
532
533
534
535
        if parsed["type"] == "text":
            return await self._process_text_async(
                parsed["content"],
                tokenization_kwargs=tokenization_kwargs,
536
                lora_request=lora_request,
537
538
539
540
541
                return_mm_hashes=return_mm_hashes,
            )
        if parsed["type"] == "str":
            return await self._process_text_async(
                TextPrompt(prompt=parsed["content"]),
542
                tokenization_kwargs=tokenization_kwargs,
543
544
                lora_request=lora_request,
                return_mm_hashes=return_mm_hashes,
545
            )
546

547
548
        assert_never(parsed)

549
550
    def _build_enc_dec_llm_inputs(
        self,
551
552
        encoder_inputs: SingletonInputs,
        decoder_inputs: Optional[SingletonInputs],
553
    ) -> EncoderDecoderInputs:
554
555
556
557
        if (encoder_inputs["type"] == "embeds"
                or decoder_inputs and decoder_inputs["type"] == "embeds"):
            raise ValueError("Embedding inputs are not supported for encoder-"
                             "decoder models")
558

559
560
561
562
563
        # Needed for mypy
        encoder_inputs = cast(Union[TokenInputs, MultiModalInputs],
                              encoder_inputs)
        decoder_inputs = cast(Optional[Union[TokenInputs, MultiModalInputs]],
                              decoder_inputs)
564

565
        if decoder_inputs is None:
566
567
568
569
570
571
572
573
574
            if self.model_config.hf_config.model_type == "whisper":
                # For Whisper models, the text prompt should go to the decoder.
                # If no explicit encoder/decoder inputs, then copy the prompt
                # from the encoder to the decoder. The encoder tokens are later
                # overridden by the audio features.
                dec_token_ids = encoder_inputs["prompt_token_ids"].copy()
            else:
                dec_token_ids = self._prepare_decoder_input_ids_for_generation(
                    None)
575
            decoder_inputs = token_inputs(dec_token_ids)
576
        else:
577
578
579
            if "multi_modal_data" in decoder_inputs:
                raise ValueError("Multi-modal decoder inputs of encoder-"
                                 "decoder models are not supported yet")
580
581
582
583

            dec_token_ids = self._prepare_decoder_input_ids_for_generation(
                decoder_inputs["prompt_token_ids"])
            decoder_inputs["prompt_token_ids"] = dec_token_ids
584

585
        return EncoderDecoderInputs(
586
587
            encoder=encoder_inputs,
            decoder=decoder_inputs,
588
589
        )

590
    def _split_enc_dec_mm_inputs(
591
        self,
592
593
        inputs: Union[SingletonInputs, MultiModalEncDecInputs],
        decoder_inputs_to_override: Optional[SingletonInputs] = None,
594
    ) -> tuple[SingletonInputs, SingletonInputs]:
595
596
597
598
        """
        For encoder/decoder models only:
        Separate Encoder/Decoder inputs from a MultiModalEncDecInputs
        """
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
        if (inputs["type"] == "embeds" or decoder_inputs_to_override
                and decoder_inputs_to_override["type"] == "embeds"):
            raise ValueError("Embedding inputs are not supported for encoder-"
                             "decoder models")

        # Needed for mypy
        inputs = cast(
            Union[TokenInputs, MultiModalInputs, MultiModalEncDecInputs],
            inputs,
        )
        decoder_inputs_to_override = cast(
            Optional[Union[TokenInputs, MultiModalInputs]],
            decoder_inputs_to_override,
        )

614
615
        encoder_inputs: SingletonInputs
        decoder_inputs: SingletonInputs
616
617
618
619
620
621
622

        if inputs["type"] == "multimodal":  # Multimodal data inputs
            if not ("encoder_prompt" in inputs
                    and "encoder_prompt_token_ids" in inputs):
                raise RuntimeError("You should register an encoder-decoder "
                                   "multi-modal processor for encoder-decoder "
                                   "models.")
623
            inputs = cast(MultiModalEncDecInputs, inputs)
624

625
626
627
628
            encoder_inputs = token_inputs(
                prompt=inputs["encoder_prompt"],
                prompt_token_ids=inputs["encoder_prompt_token_ids"],
            )
629

630
631
632
633
634
635
636
637
638
639
            decoder_prompt_inputs = decoder_inputs_to_override or inputs
            decoder_inputs = MultiModalInputs(
                type="multimodal",
                prompt=decoder_prompt_inputs.get("prompt", ""),
                prompt_token_ids=decoder_prompt_inputs["prompt_token_ids"],
                mm_kwargs=inputs["mm_kwargs"],
                mm_hashes=inputs["mm_hashes"],
                mm_placeholders=inputs["mm_placeholders"],
            )
            if cache_salt := inputs.get("cache_salt"):
640
641
                decoder_inputs["cache_salt"] = cache_salt

642
        elif inputs["type"] == "token":  # Text-only inputs
643
644
645
646
            encoder_inputs = token_inputs(prompt="", prompt_token_ids=[])
            decoder_inputs = decoder_inputs_to_override or inputs
        else:
            assert_never(inputs)  # type: ignore[arg-type]
647

648
649
        return encoder_inputs, decoder_inputs

650
651
    def _process_encoder_decoder_prompt(
        self,
652
        prompt: PromptType,
653
        tokenization_kwargs: Optional[dict[str, Any]] = None,
654
    ) -> EncoderDecoderInputs:
655
        """
656
        For encoder/decoder models only:
657
658
659
        Process an input prompt into an
        [`EncoderDecoderInputs`][vllm.inputs.data.EncoderDecoderInputs]
        instance.
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677

        There are two types of input prompts:
        singleton prompts which carry only the
        encoder prompt, and explicit encoder/decoder
        prompts which carry both the encoder and the
        decoder prompts as member variables.

        This function handles the following scenarios:
        * Singleton encoder prompt: extract encoder prompt
          token ids & infer default decoder prompt token ids
        * Explicit encoder/decoder prompt: extract encoder
          and decoder prompt token ids

        Note that for Explicit encoder/decoder prompts,
        each sub-prompt (encoder or decoder prompt) can
        have any possible singleton type; thus this
        method relies on helper functions to obtain
        token ids for the sub-prompts.
678

679
680
        Arguments:

681
        * prompt: an input prompt
682
683
684

        Returns:

685
686
        * [`EncoderDecoderInputs`][vllm.inputs.data.EncoderDecoderInputs]
          instance
687
        """
688
689
        encoder_inputs: SingletonInputs
        decoder_inputs: Optional[SingletonInputs]
690

691
        if is_explicit_encoder_decoder_prompt(prompt):
692
            encoder_inputs = self._prompt_to_llm_inputs(
693
694
695
                prompt["encoder_prompt"],
                tokenization_kwargs=tokenization_kwargs,
            )
696
            if (decoder_input := prompt["decoder_prompt"]) is None:
697
                decoder_inputs = None
698
            else:
699
                decoder_inputs = self._prompt_to_llm_inputs(decoder_input)
700
701
            # For multimodal model, override decoder prompt from processor
            # with explicit decoder prompt.
702
            if self.model_config.is_multimodal_model:
703
                encoder_inputs, decoder_inputs = (
704
705
                    self._split_enc_dec_mm_inputs(encoder_inputs,
                                                  decoder_inputs))
706
        else:
707
708
709
710
            inputs = self._prompt_to_llm_inputs(
                prompt,
                tokenization_kwargs=tokenization_kwargs,
            )
711
            if self.model_config.is_multimodal_model:
712
713
                # Encoder-Decoder Multimodal model
                encoder_inputs, decoder_inputs = (
714
                    self._split_enc_dec_mm_inputs(inputs))
715
716
717
            else:
                encoder_inputs = inputs
                decoder_inputs = None
718
719

        return self._build_enc_dec_llm_inputs(encoder_inputs, decoder_inputs)
720
721
722

    async def _process_encoder_decoder_prompt_async(
        self,
723
        prompt: PromptType,
724
        tokenization_kwargs: Optional[dict[str, Any]] = None,
725
    ) -> EncoderDecoderInputs:
726
727
728
729
        """
        Async version of
        [`_process_encoder_decoder_prompt`][vllm.inputs.preprocess.InputPreprocessor._process_encoder_decoder_prompt].
        """
730
731
        encoder_inputs: SingletonInputs
        decoder_inputs: Optional[SingletonInputs]
732

733
        if is_explicit_encoder_decoder_prompt(prompt):
734
            encoder_task = self._prompt_to_llm_inputs_async(
735
736
737
                prompt["encoder_prompt"],
                tokenization_kwargs=tokenization_kwargs,
            )
738

739
            if (decoder_input := prompt["decoder_prompt"]) is None:
740
741
                encoder_inputs = await encoder_task
                decoder_inputs = None
742
            else:
743
744
745
746
                decoder_task = self._prompt_to_llm_inputs_async(
                    decoder_input,
                    tokenization_kwargs=tokenization_kwargs,
                )
747

748
                encoder_inputs, decoder_inputs = await asyncio.gather(
749
                    encoder_task, decoder_task)
750
751
752

            # For multimodal model, override decoder prompt from processor
            # with explicit decoder prompt.
753
            if self.model_config.is_multimodal_model:
754
                encoder_inputs, decoder_inputs = (
755
756
                    self._split_enc_dec_mm_inputs(encoder_inputs,
                                                  decoder_inputs))
757
        else:
758
759
760
761
            inputs = await self._prompt_to_llm_inputs_async(
                prompt,
                tokenization_kwargs=tokenization_kwargs,
            )
762
            if self.model_config.is_multimodal_model:
763
764
                # Encoder-Decoder Multimodal model
                encoder_inputs, decoder_inputs = (
765
                    self._split_enc_dec_mm_inputs(inputs))
766
767
768
            else:
                encoder_inputs = inputs
                decoder_inputs = None
769
770

        return self._build_enc_dec_llm_inputs(encoder_inputs, decoder_inputs)
771
772
773

    def _build_decoder_only_llm_inputs(
        self,
774
        prompt_inputs: DecoderOnlyInputs,
775
        prompt_adapter_request: Optional[PromptAdapterRequest],
776
    ) -> DecoderOnlyInputs:
777
778
779
        if "prompt_token_ids" in prompt_inputs:
            prompt_inputs = cast(Union[TokenInputs, MultiModalInputs],
                                 prompt_inputs)  # Needed for mypy
780
781
782
783
            prompt_inputs["prompt_token_ids"] = self._apply_prompt_adapter(
                prompt_inputs["prompt_token_ids"],
                prompt_adapter_request=prompt_adapter_request,
            )
784

785
        return prompt_inputs
786
787
788

    def _process_decoder_only_prompt(
        self,
789
        prompt: SingletonPrompt,
790
        tokenization_kwargs: Optional[dict[str, Any]] = None,
791
792
        lora_request: Optional[LoRARequest] = None,
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
793
        return_mm_hashes: bool = False,
794
    ) -> DecoderOnlyInputs:
795
        """
796
        For decoder-only models:
797
798
        Process an input prompt into a
        [`DecoderOnlyInputs`][vllm.inputs.data.DecoderOnlyInputs] instance.
799
800
801

        Arguments:

802
        * prompt: input prompt
803
804
        * lora_request
        * prompt_adapter_request
805
        * return_mm_hashes
806
807
808

        Returns:

809
        * [`DecoderOnlyInputs`][vllm.inputs.data.DecoderOnlyInputs] instance
810
        """
811

812
        prompt_comps = self._prompt_to_llm_inputs(
813
            prompt,
814
            tokenization_kwargs=tokenization_kwargs,
815
            lora_request=lora_request,
816
            return_mm_hashes=return_mm_hashes,
817
818
819
820
821
822
823
824
825
        )

        return self._build_decoder_only_llm_inputs(
            prompt_comps,
            prompt_adapter_request=prompt_adapter_request,
        )

    async def _process_decoder_only_prompt_async(
        self,
826
        prompt: SingletonPrompt,
827
        tokenization_kwargs: Optional[dict[str, Any]] = None,
828
829
        lora_request: Optional[LoRARequest] = None,
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
830
        return_mm_hashes: bool = False,
831
    ) -> DecoderOnlyInputs:
832
833
834
835
        """
        Async version of
        [`_process_decoder_only_prompt`][vllm.inputs.preprocess.InputPreprocessor._process_decoder_only_prompt].
        """
836
        prompt_comps = await self._prompt_to_llm_inputs_async(
837
            prompt,
838
            tokenization_kwargs=tokenization_kwargs,
839
            lora_request=lora_request,
840
            return_mm_hashes=return_mm_hashes,
841
842
843
844
845
846
847
848
849
        )

        return self._build_decoder_only_llm_inputs(
            prompt_comps,
            prompt_adapter_request=prompt_adapter_request,
        )

    def preprocess(
        self,
850
        prompt: PromptType,
851
        tokenization_kwargs: Optional[dict[str, Any]] = None,
852
853
        lora_request: Optional[LoRARequest] = None,
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
854
        return_mm_hashes: bool = False,
855
    ) -> ProcessorInputs:
856
        """Preprocess the input prompt."""
857
        if self.model_config.is_encoder_decoder:
858
859
860
            assert not return_mm_hashes, (
                "Multimodal hashes for encoder-decoder models should not be ",
                "returned until they are supported on vLLM V1.")
861
862
            # Encoder-decoder model requires special mapping of
            # input prompts to encoder & decoder
863
            return self._process_encoder_decoder_prompt(prompt)
864

865
        if is_explicit_encoder_decoder_prompt(prompt):
866
867
868
869
870
            raise ValueError("Cannot pass encoder-decoder prompt "
                             "to decoder-only models")

        # Decoder-only operation
        return self._process_decoder_only_prompt(
871
            prompt,
872
            tokenization_kwargs=tokenization_kwargs,
873
874
            lora_request=lora_request,
            prompt_adapter_request=prompt_adapter_request,
875
            return_mm_hashes=return_mm_hashes,
876
877
878
879
        )

    async def preprocess_async(
        self,
880
        prompt: PromptType,
881
        tokenization_kwargs: Optional[dict[str, Any]] = None,
882
883
        lora_request: Optional[LoRARequest] = None,
        prompt_adapter_request: Optional[PromptAdapterRequest] = None,
884
        return_mm_hashes: bool = False,
885
    ) -> ProcessorInputs:
886
887
888
889
        """
        Async version of
        [`preprocess`][vllm.inputs.preprocess.InputPreprocessor.preprocess].
        """
890
        if self.model_config.is_encoder_decoder:
891
892
893
            assert not return_mm_hashes, (
                "Multimodal hashes for encoder-decoder models should not be ",
                "returned until they are supported on vLLM V1.")
894
895
            # Encoder-decoder model requires special mapping of
            # input prompts to encoder & decoder
896
            return await self._process_encoder_decoder_prompt_async(prompt)
897

898
        if is_explicit_encoder_decoder_prompt(prompt):
899
900
901
902
903
            raise ValueError("Cannot pass encoder-decoder prompt "
                             "to decoder-only models")

        # Decoder-only operation
        return await self._process_decoder_only_prompt_async(
904
            prompt,
905
            tokenization_kwargs=tokenization_kwargs,
906
907
            lora_request=lora_request,
            prompt_adapter_request=prompt_adapter_request,
908
            return_mm_hashes=return_mm_hashes,
909
        )