preprocess.py 33.2 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
from collections.abc import Mapping
6
from typing import Any, Optional, Union, cast
7
8
9
10
11
12

from typing_extensions import assert_never

from vllm.config import ModelConfig
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
13
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
14
15
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalEncDecInputs,
                                    MultiModalInputs)
16
from vllm.transformers_utils.tokenizer import AnyTokenizer
17
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
18

19
20
21
22
23
from .data import (DecoderOnlyInputs, EmbedsInputs, EmbedsPrompt,
                   EncoderDecoderInputs, ProcessorInputs, PromptType,
                   SingletonInputs, SingletonPrompt, TextPrompt, TokenInputs,
                   TokensPrompt, embeds_inputs, token_inputs)
from .parse import is_explicit_encoder_decoder_prompt, parse_singleton_prompt
24
25
26
27
28
29
30
31
32

logger = init_logger(__name__)


class InputPreprocessor:

    def __init__(
        self,
        model_config: ModelConfig,
33
        tokenizer: Optional[TokenizerGroup],
34
        mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
35
36
37
38
39
    ) -> None:
        super().__init__()

        self.model_config = model_config
        self.tokenizer = tokenizer
40
        self.mm_registry = mm_registry
41

42
    def get_tokenizer_group(self) -> TokenizerGroup:
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
        if self.tokenizer is None:
            raise ValueError("You cannot pass text prompts when "
                             "`skip_tokenizer_init` is True")

        return self.tokenizer

    def get_bos_token_id(self,
                         lora_request: Optional[LoRARequest] = None
                         ) -> Optional[int]:
        if self.tokenizer is None:
            logger.warning("Using None for BOS token id because tokenizer "
                           "is not initialized")
            return None

        return self.tokenizer.get_lora_tokenizer(lora_request).bos_token_id

    def get_eos_token_id(self,
                         lora_request: Optional[LoRARequest] = None
                         ) -> Optional[int]:
        if self.tokenizer is None:
            logger.warning("Using None for EOS token id because tokenizer "
                           "is not initialized")
            return None

        return self.tokenizer.get_lora_tokenizer(lora_request).eos_token_id

    def get_decoder_start_token_id(self) -> Optional[int]:
70
        """
71
72
73
        Obtain the decoder start token id employed by an encoder/decoder
        model. Returns None for non-encoder/decoder models or if the
        model config is unavailable.
74
        """
75

76
        if not self.model_config.is_encoder_decoder:
77
78
79
            logger.warning_once(
                "Using None for decoder start token id because "
                "this is not an encoder/decoder model.")
80
81
            return None

82
        if self.model_config is None or self.model_config.hf_config is None:
83
84
85
            logger.warning_once(
                "Using None for decoder start token id because "
                "model config is not available.")
86
87
88
            return None

        dec_start_token_id = getattr(self.model_config.hf_config,
89
                                     "decoder_start_token_id", None)
90
        if dec_start_token_id is None:
91
92
93
94
            logger.warning_once(
                "Falling back on <BOS> for decoder start token "
                "id because decoder start token id is not "
                "available.")
95
96
97
98
            dec_start_token_id = self.get_bos_token_id()

        return dec_start_token_id

99
    def _get_default_enc_dec_decoder_prompt(self) -> list[int]:
100
        """
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
        Specifically for encoder/decoder models:
        generate a default decoder prompt for when
        the user specifies only the encoder prompt.

        Encoder/decoder models utilize the decoder
        prompt in different ways; as new models are
        added, it is intended that this function
        will be extended to produce differing
        default decoder prompts, depending on the
        model variety.

        Absent a special case, the default behavior
        of this method is to mirror the behavior of
        the HuggingFace (HF) GenerationMixin for a None
        decoder prompt, which is to employ a logit processor
        setting to force the first decoded token to be <BOS>.
        Here, this behavior is approximated by having the
        "default" decoder prompt be <BOS>.

        However, it is possible that in the future
121
        other models may have different or more
122
123
124
125
126
127
128
        complex logic for the default decoder prompt.
        This motivates having a special helper method
        for default decoder prompts.

        Returns:

        * prompt_token_ids
129
        """
130
131
132
133
134
135
136

        bos_token_id = self.get_bos_token_id()
        assert bos_token_id is not None
        return [bos_token_id]

    def _prepare_decoder_input_ids_for_generation(
        self,
137
138
        decoder_input_ids: Optional[list[int]],
    ) -> list[int]:
139
140
141
        """
        Prepares `decoder_input_ids` for generation with encoder-decoder models.

142
143
144
145
        Based on:
        https://github.com/huggingface/transformers/blob/4037a2b5b1278736e566aec12e169100275545ea/src/transformers/generation/utils.py
        specifically,
        `GenerationMixin._prepare_decoder_input_ids_for_generation()`.
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163

        Arguments:

        * decoder_input_ids: input token ids to preprocess

        Returns:

        * Processed token list
        """

        decoder_start_token_id = self.get_decoder_start_token_id()
        assert decoder_start_token_id is not None

        if decoder_input_ids is None:
            # no decoder prompt input ->
            # use decoder_start_token_id as decoder_input_ids
            decoder_input_ids = self._get_default_enc_dec_decoder_prompt()

164
165
        if (len(decoder_input_ids) == 0
                or decoder_input_ids[0] != decoder_start_token_id):
166
167
168
169
            decoder_input_ids = [decoder_start_token_id] + decoder_input_ids

        return decoder_input_ids

170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
    def _get_tokenization_kw(
        self,
        overrides: Optional[dict[str, Any]] = None,
    ) -> dict[str, Any]:
        kwargs = dict[str, Any]()

        if self.model_config.hf_config.model_type == "whisper":
            # For Whisper, special tokens should be provided by the user based
            # on the task and language of their request. Also needed to avoid
            # appending an EOS token to the prompt which disrupts generation.
            kwargs["add_special_tokens"] = False

        if overrides:
            kwargs.update(overrides)

        return kwargs

187
188
189
190
    def _tokenize_prompt(
        self,
        prompt: str,
        lora_request: Optional[LoRARequest],
191
        tokenization_kwargs: Optional[dict[str, Any]] = None,
192
    ) -> list[int]:
193
194
195
196
197
        """
        Apply the model's tokenizer to a text prompt, returning the
        corresponding token IDs.
        """
        tokenizer = self.get_tokenizer_group()
198
        tokenization_kwargs = self._get_tokenization_kw(tokenization_kwargs)
199

200
        encoder_config = self.model_config.encoder_config
201

202
        if encoder_config and encoder_config.get("do_lower_case", False):
203
204
            prompt = prompt.lower()

205
        return tokenizer.encode(prompt=prompt,
206
                                lora_request=lora_request,
207
                                **tokenization_kwargs)
208
209
210
211
212

    async def _tokenize_prompt_async(
        self,
        prompt: str,
        lora_request: Optional[LoRARequest],
213
        tokenization_kwargs: Optional[dict[str, Any]] = None,
214
    ) -> list[int]:
215
216
217
218
        """
        Async version of
        [`_tokenize_prompt`][vllm.inputs.preprocess.InputPreprocessor._tokenize_prompt].
        """
219
        tokenizer = self.get_tokenizer_group()
220
        tokenization_kwargs = self._get_tokenization_kw(tokenization_kwargs)
221
222
223
224

        return await tokenizer.encode_async(prompt=prompt,
                                            lora_request=lora_request,
                                            **tokenization_kwargs)
225

226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
    def _get_mm_tokenizer(
        self,
        lora_request: Optional[LoRARequest],
    ) -> AnyTokenizer:
        # PrithviGeoSpatialMAE needs to be initialized without a tokenizer
        # while using also multi-modal input
        if not self.tokenizer:
            return cast(AnyTokenizer, object())  # Dummy

        tokenizer_group = self.get_tokenizer_group()
        return tokenizer_group.get_lora_tokenizer(lora_request)

    async def _get_mm_tokenizer_async(
        self,
        lora_request: Optional[LoRARequest],
    ) -> AnyTokenizer:
        # PrithviGeoSpatialMAE needs to be initialized without a tokenizer
        # while using also multi-modal input
        if not self.tokenizer:
            return cast(AnyTokenizer, object())  # Dummy

        tokenizer_group = self.get_tokenizer_group()
        return await tokenizer_group.get_lora_tokenizer_async(lora_request)

250
251
    def _process_multimodal(
        self,
252
        prompt: Union[str, list[int]],
253
254
        mm_data: MultiModalDataDict,
        mm_processor_kwargs: Optional[Mapping[str, object]],
255
256
        tokenization_kwargs: Optional[dict[str, Any]] = None,
        lora_request: Optional[LoRARequest] = None,
257
        return_mm_hashes: bool = False,
258
    ) -> MultiModalInputs:
259
260
261
262
        """
        Apply the model's multi-modal processor to a multi-modal prompt,
        returning the corresponding token IDs and metadata.
        """
263
        tokenizer = self._get_mm_tokenizer(lora_request)
264

265
266
        mm_processor = self.mm_registry.create_processor(self.model_config,
                                                         tokenizer=tokenizer)
267
268
269
270

        if mm_processor_kwargs is None:
            mm_processor_kwargs = {}

271
272
273
274
275
        return mm_processor.apply(prompt,
                                  mm_data,
                                  hf_processor_mm_kwargs=mm_processor_kwargs,
                                  tokenization_kwargs=tokenization_kwargs,
                                  return_mm_hashes=return_mm_hashes)
276
277
278

    async def _process_multimodal_async(
        self,
279
        prompt: Union[str, list[int]],
280
281
        mm_data: MultiModalDataDict,
        mm_processor_kwargs: Optional[Mapping[str, object]],
282
283
        tokenization_kwargs: Optional[dict[str, Any]] = None,
        lora_request: Optional[LoRARequest] = None,
284
        return_mm_hashes: bool = False,
285
    ) -> MultiModalInputs:
286
287
288
289
        """
        Async version of
        [`_process_multimodal`][vllm.inputs.preprocess.InputPreprocessor._process_multimodal].
        """
290
        tokenizer = await self._get_mm_tokenizer_async(lora_request)
291

292
293
        mm_processor = self.mm_registry.create_processor(self.model_config,
                                                         tokenizer=tokenizer)
294
295
296
        if mm_processor_kwargs is None:
            mm_processor_kwargs = {}

297
298
299
300
301
        return mm_processor.apply(prompt,
                                  mm_data,
                                  hf_processor_mm_kwargs=mm_processor_kwargs,
                                  tokenization_kwargs=tokenization_kwargs,
                                  return_mm_hashes=return_mm_hashes)
302

303
304
305
306
    def _process_embeds(
        self,
        parsed_content: EmbedsPrompt,
    ) -> EmbedsInputs:
307
308
309
        if not self.model_config.enable_prompt_embeds:
            raise ValueError("You must set `--enable-prompt-embeds` to input "
                             "`prompt_embeds`.")
310
311

        prompt_embeds = parsed_content["prompt_embeds"]
312

313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
        # prompt_embeds must be (seq_len, hidden_size), but if the user
        # passes in a batch of size 1, i.e. (1, seq_len, hidden_size),
        # we can unambiguously process the intent by squeezing the batch
        # dimension.
        if prompt_embeds.ndim == 3:
            prompt_embeds = prompt_embeds.squeeze(dim=0)

        if prompt_embeds.ndim != 2:
            raise ValueError(
                "prompt_embeds must be of shape (seq_len, hidden_size).")

        return embeds_inputs(prompt_embeds=prompt_embeds,
                             cache_salt=parsed_content.get("cache_salt"))

    async def _process_embeds_async(
        self,
        parsed_content: EmbedsPrompt,
    ) -> EmbedsInputs:
        return self._process_embeds(parsed_content)

    def _process_tokens(
        self,
        parsed_content: TokensPrompt,
336
        tokenization_kwargs: Optional[dict[str, Any]] = None,
337
338
339
340
341
342
343
344
345
346
347
348
        lora_request: Optional[LoRARequest] = None,
        return_mm_hashes: bool = False,
    ) -> Union[TokenInputs, MultiModalInputs]:
        prompt_token_ids = parsed_content["prompt_token_ids"]
        token_type_ids = parsed_content.get("token_type_ids")

        inputs: Union[TokenInputs, MultiModalInputs]
        if multi_modal_data := parsed_content.get("multi_modal_data"):
            inputs = self._process_multimodal(
                prompt_token_ids,
                multi_modal_data,
                parsed_content.get("mm_processor_kwargs"),
349
                tokenization_kwargs=tokenization_kwargs,
350
351
352
                lora_request=lora_request,
                return_mm_hashes=return_mm_hashes,
            )
353
        else:
354
355
356
357
358
359
360
361
362
363
364
365
366
            inputs = token_inputs(
                prompt_token_ids=prompt_token_ids,
                token_type_ids=token_type_ids,
            )

        if cache_salt := parsed_content.get("cache_salt"):
            inputs["cache_salt"] = cache_salt

        return inputs

    async def _process_tokens_async(
        self,
        parsed_content: TokensPrompt,
367
        tokenization_kwargs: Optional[dict[str, Any]] = None,
368
369
370
371
372
373
374
375
376
377
378
379
        lora_request: Optional[LoRARequest] = None,
        return_mm_hashes: bool = False,
    ) -> Union[TokenInputs, MultiModalInputs]:
        prompt_token_ids = parsed_content["prompt_token_ids"]
        token_type_ids = parsed_content.get("token_type_ids")

        inputs: Union[TokenInputs, MultiModalInputs]
        if multi_modal_data := parsed_content.get("multi_modal_data"):
            inputs = await self._process_multimodal_async(
                prompt_token_ids,
                multi_modal_data,
                parsed_content.get("mm_processor_kwargs"),
380
                tokenization_kwargs=tokenization_kwargs,
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
                lora_request=lora_request,
                return_mm_hashes=return_mm_hashes,
            )
        else:
            inputs = token_inputs(
                prompt_token_ids=prompt_token_ids,
                token_type_ids=token_type_ids,
            )

        if cache_salt := parsed_content.get("cache_salt"):
            inputs["cache_salt"] = cache_salt

        return inputs

    def _process_text(
        self,
        parsed_content: TextPrompt,
        tokenization_kwargs: Optional[dict[str, Any]] = None,
        lora_request: Optional[LoRARequest] = None,
        return_mm_hashes: bool = False,
    ) -> Union[TokenInputs, MultiModalInputs]:
        prompt_text = parsed_content["prompt"]

        inputs: Union[TokenInputs, MultiModalInputs]
        if multi_modal_data := parsed_content.get("multi_modal_data"):
            inputs = self._process_multimodal(
                prompt_text,
                multi_modal_data,
                parsed_content.get("mm_processor_kwargs"),
410
                tokenization_kwargs=tokenization_kwargs,
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
                lora_request=lora_request,
                return_mm_hashes=return_mm_hashes,
            )
        else:
            prompt_token_ids = self._tokenize_prompt(
                prompt_text,
                lora_request=lora_request,
                tokenization_kwargs=tokenization_kwargs,
            )
            inputs = token_inputs(
                prompt=prompt_text,
                prompt_token_ids=prompt_token_ids,
            )

        if cache_salt := parsed_content.get("cache_salt"):
            inputs["cache_salt"] = cache_salt

        return inputs
429

430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
    async def _process_text_async(
        self,
        parsed_content: TextPrompt,
        tokenization_kwargs: Optional[dict[str, Any]] = None,
        lora_request: Optional[LoRARequest] = None,
        return_mm_hashes: bool = False,
    ) -> Union[TokenInputs, MultiModalInputs]:
        prompt_text = parsed_content["prompt"]

        inputs: Union[TokenInputs, MultiModalInputs]
        if multi_modal_data := parsed_content.get("multi_modal_data"):
            inputs = await self._process_multimodal_async(
                prompt_text,
                multi_modal_data,
                parsed_content.get("mm_processor_kwargs"),
445
                tokenization_kwargs=tokenization_kwargs,
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
                lora_request=lora_request,
                return_mm_hashes=return_mm_hashes,
            )
        else:
            prompt_token_ids = await self._tokenize_prompt_async(
                prompt_text,
                lora_request=lora_request,
                tokenization_kwargs=tokenization_kwargs,
            )
            inputs = token_inputs(
                prompt=prompt_text,
                prompt_token_ids=prompt_token_ids,
            )

        if cache_salt := parsed_content.get("cache_salt"):
            inputs["cache_salt"] = cache_salt

        return inputs
464

465
    def _prompt_to_llm_inputs(
466
        self,
467
        prompt: SingletonPrompt,
468
        tokenization_kwargs: Optional[dict[str, Any]] = None,
469
        lora_request: Optional[LoRARequest] = None,
470
        return_mm_hashes: bool = False,
471
    ) -> SingletonInputs:
472
473
        """
        Extract the singleton inputs from a prompt.
474
475
476

        Arguments:

477
        * prompt: single encoder or decoder input prompt
478
        * lora_request: this is only valid for decoder prompts
479
        * return_mm_hashes: whether to return multimodal hashes
480
481
482

        Returns:

483
        * [`SingletonInputs`][vllm.inputs.data.SingletonInputs] instance
484
        """
485
        parsed = parse_singleton_prompt(prompt)
486
487

        if parsed["type"] == "embeds":
488
489
490
491
            return self._process_embeds(parsed["content"])
        if parsed["type"] == "tokens":
            return self._process_tokens(
                parsed["content"],
492
                lora_request=lora_request,
493
                return_mm_hashes=return_mm_hashes,
494
            )
495
496
497
498
        if parsed["type"] == "text":
            return self._process_text(
                parsed["content"],
                tokenization_kwargs=tokenization_kwargs,
499
                lora_request=lora_request,
500
501
502
503
504
                return_mm_hashes=return_mm_hashes,
            )
        if parsed["type"] == "str":
            return self._process_text(
                TextPrompt(prompt=parsed["content"]),
505
                tokenization_kwargs=tokenization_kwargs,
506
507
                lora_request=lora_request,
                return_mm_hashes=return_mm_hashes,
508
            )
509

510
511
        assert_never(parsed)

512
    async def _prompt_to_llm_inputs_async(
513
        self,
514
        prompt: SingletonPrompt,
515
        tokenization_kwargs: Optional[dict[str, Any]] = None,
516
        lora_request: Optional[LoRARequest] = None,
517
        return_mm_hashes: bool = False,
518
    ) -> SingletonInputs:
519
520
521
522
        """
        Async version of
        [`_prompt_to_llm_inputs`][vllm.inputs.preprocess.InputPreprocessor._prompt_to_llm_inputs].
        """
523
        parsed = parse_singleton_prompt(prompt)
524

525
        if parsed["type"] == "embeds":
526
527
528
529
            return await self._process_embeds_async(parsed["content"])
        if parsed["type"] == "tokens":
            return await self._process_tokens_async(
                parsed["content"],
530
531
                lora_request=lora_request,
                return_mm_hashes=return_mm_hashes,
532
            )
533
534
535
536
        if parsed["type"] == "text":
            return await self._process_text_async(
                parsed["content"],
                tokenization_kwargs=tokenization_kwargs,
537
                lora_request=lora_request,
538
539
540
541
542
                return_mm_hashes=return_mm_hashes,
            )
        if parsed["type"] == "str":
            return await self._process_text_async(
                TextPrompt(prompt=parsed["content"]),
543
                tokenization_kwargs=tokenization_kwargs,
544
545
                lora_request=lora_request,
                return_mm_hashes=return_mm_hashes,
546
            )
547

548
549
        assert_never(parsed)

550
551
    def _build_enc_dec_llm_inputs(
        self,
552
553
        encoder_inputs: SingletonInputs,
        decoder_inputs: Optional[SingletonInputs],
554
    ) -> EncoderDecoderInputs:
555
556
557
558
        if (encoder_inputs["type"] == "embeds"
                or decoder_inputs and decoder_inputs["type"] == "embeds"):
            raise ValueError("Embedding inputs are not supported for encoder-"
                             "decoder models")
559

560
561
562
563
564
        # Needed for mypy
        encoder_inputs = cast(Union[TokenInputs, MultiModalInputs],
                              encoder_inputs)
        decoder_inputs = cast(Optional[Union[TokenInputs, MultiModalInputs]],
                              decoder_inputs)
565

566
        if decoder_inputs is None:
567
568
569
570
571
572
573
574
575
            if self.model_config.hf_config.model_type == "whisper":
                # For Whisper models, the text prompt should go to the decoder.
                # If no explicit encoder/decoder inputs, then copy the prompt
                # from the encoder to the decoder. The encoder tokens are later
                # overridden by the audio features.
                dec_token_ids = encoder_inputs["prompt_token_ids"].copy()
            else:
                dec_token_ids = self._prepare_decoder_input_ids_for_generation(
                    None)
576
            decoder_inputs = token_inputs(dec_token_ids)
577
        else:
578
579
580
            if "multi_modal_data" in decoder_inputs:
                raise ValueError("Multi-modal decoder inputs of encoder-"
                                 "decoder models are not supported yet")
581
582
583
584

            dec_token_ids = self._prepare_decoder_input_ids_for_generation(
                decoder_inputs["prompt_token_ids"])
            decoder_inputs["prompt_token_ids"] = dec_token_ids
585

586
        return EncoderDecoderInputs(
587
588
            encoder=encoder_inputs,
            decoder=decoder_inputs,
589
590
        )

591
    def _split_enc_dec_mm_inputs(
592
        self,
593
594
        inputs: Union[SingletonInputs, MultiModalEncDecInputs],
        decoder_inputs_to_override: Optional[SingletonInputs] = None,
595
    ) -> tuple[SingletonInputs, SingletonInputs]:
596
597
598
599
        """
        For encoder/decoder models only:
        Separate Encoder/Decoder inputs from a MultiModalEncDecInputs
        """
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
        if (inputs["type"] == "embeds" or decoder_inputs_to_override
                and decoder_inputs_to_override["type"] == "embeds"):
            raise ValueError("Embedding inputs are not supported for encoder-"
                             "decoder models")

        # Needed for mypy
        inputs = cast(
            Union[TokenInputs, MultiModalInputs, MultiModalEncDecInputs],
            inputs,
        )
        decoder_inputs_to_override = cast(
            Optional[Union[TokenInputs, MultiModalInputs]],
            decoder_inputs_to_override,
        )

615
616
        encoder_inputs: SingletonInputs
        decoder_inputs: SingletonInputs
617
618
619
620
621
622
623

        if inputs["type"] == "multimodal":  # Multimodal data inputs
            if not ("encoder_prompt" in inputs
                    and "encoder_prompt_token_ids" in inputs):
                raise RuntimeError("You should register an encoder-decoder "
                                   "multi-modal processor for encoder-decoder "
                                   "models.")
624
            inputs = cast(MultiModalEncDecInputs, inputs)
625

626
627
628
629
            encoder_inputs = token_inputs(
                prompt=inputs["encoder_prompt"],
                prompt_token_ids=inputs["encoder_prompt_token_ids"],
            )
630

631
632
633
634
635
636
637
638
639
640
            decoder_prompt_inputs = decoder_inputs_to_override or inputs
            decoder_inputs = MultiModalInputs(
                type="multimodal",
                prompt=decoder_prompt_inputs.get("prompt", ""),
                prompt_token_ids=decoder_prompt_inputs["prompt_token_ids"],
                mm_kwargs=inputs["mm_kwargs"],
                mm_hashes=inputs["mm_hashes"],
                mm_placeholders=inputs["mm_placeholders"],
            )
            if cache_salt := inputs.get("cache_salt"):
641
642
                decoder_inputs["cache_salt"] = cache_salt

643
        elif inputs["type"] == "token":  # Text-only inputs
644
645
646
647
            encoder_inputs = token_inputs(prompt="", prompt_token_ids=[])
            decoder_inputs = decoder_inputs_to_override or inputs
        else:
            assert_never(inputs)  # type: ignore[arg-type]
648

649
650
        return encoder_inputs, decoder_inputs

651
652
    def _process_encoder_decoder_prompt(
        self,
653
        prompt: PromptType,
654
        tokenization_kwargs: Optional[dict[str, Any]] = None,
655
    ) -> EncoderDecoderInputs:
656
        """
657
        For encoder/decoder models only:
658
659
660
        Process an input prompt into an
        [`EncoderDecoderInputs`][vllm.inputs.data.EncoderDecoderInputs]
        instance.
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678

        There are two types of input prompts:
        singleton prompts which carry only the
        encoder prompt, and explicit encoder/decoder
        prompts which carry both the encoder and the
        decoder prompts as member variables.

        This function handles the following scenarios:
        * Singleton encoder prompt: extract encoder prompt
          token ids & infer default decoder prompt token ids
        * Explicit encoder/decoder prompt: extract encoder
          and decoder prompt token ids

        Note that for Explicit encoder/decoder prompts,
        each sub-prompt (encoder or decoder prompt) can
        have any possible singleton type; thus this
        method relies on helper functions to obtain
        token ids for the sub-prompts.
679

680
681
        Arguments:

682
        * prompt: an input prompt
683
684
685

        Returns:

686
687
        * [`EncoderDecoderInputs`][vllm.inputs.data.EncoderDecoderInputs]
          instance
688
        """
689
690
        encoder_inputs: SingletonInputs
        decoder_inputs: Optional[SingletonInputs]
691

692
        if is_explicit_encoder_decoder_prompt(prompt):
693
            encoder_inputs = self._prompt_to_llm_inputs(
694
695
696
                prompt["encoder_prompt"],
                tokenization_kwargs=tokenization_kwargs,
            )
697
            if (decoder_input := prompt["decoder_prompt"]) is None:
698
                decoder_inputs = None
699
            else:
700
                decoder_inputs = self._prompt_to_llm_inputs(decoder_input)
701
702
            # For multimodal model, override decoder prompt from processor
            # with explicit decoder prompt.
703
            if self.model_config.is_multimodal_model:
704
                encoder_inputs, decoder_inputs = (
705
706
                    self._split_enc_dec_mm_inputs(encoder_inputs,
                                                  decoder_inputs))
707
        else:
708
709
710
711
            inputs = self._prompt_to_llm_inputs(
                prompt,
                tokenization_kwargs=tokenization_kwargs,
            )
712
            if self.model_config.is_multimodal_model:
713
714
                # Encoder-Decoder Multimodal model
                encoder_inputs, decoder_inputs = (
715
                    self._split_enc_dec_mm_inputs(inputs))
716
717
718
            else:
                encoder_inputs = inputs
                decoder_inputs = None
719
720

        return self._build_enc_dec_llm_inputs(encoder_inputs, decoder_inputs)
721
722
723

    async def _process_encoder_decoder_prompt_async(
        self,
724
        prompt: PromptType,
725
        tokenization_kwargs: Optional[dict[str, Any]] = None,
726
    ) -> EncoderDecoderInputs:
727
728
729
730
        """
        Async version of
        [`_process_encoder_decoder_prompt`][vllm.inputs.preprocess.InputPreprocessor._process_encoder_decoder_prompt].
        """
731
732
        encoder_inputs: SingletonInputs
        decoder_inputs: Optional[SingletonInputs]
733

734
        if is_explicit_encoder_decoder_prompt(prompt):
735
            encoder_task = self._prompt_to_llm_inputs_async(
736
737
738
                prompt["encoder_prompt"],
                tokenization_kwargs=tokenization_kwargs,
            )
739

740
            if (decoder_input := prompt["decoder_prompt"]) is None:
741
742
                encoder_inputs = await encoder_task
                decoder_inputs = None
743
            else:
744
745
746
747
                decoder_task = self._prompt_to_llm_inputs_async(
                    decoder_input,
                    tokenization_kwargs=tokenization_kwargs,
                )
748

749
                encoder_inputs, decoder_inputs = await asyncio.gather(
750
                    encoder_task, decoder_task)
751
752
753

            # For multimodal model, override decoder prompt from processor
            # with explicit decoder prompt.
754
            if self.model_config.is_multimodal_model:
755
                encoder_inputs, decoder_inputs = (
756
757
                    self._split_enc_dec_mm_inputs(encoder_inputs,
                                                  decoder_inputs))
758
        else:
759
760
761
762
            inputs = await self._prompt_to_llm_inputs_async(
                prompt,
                tokenization_kwargs=tokenization_kwargs,
            )
763
            if self.model_config.is_multimodal_model:
764
765
                # Encoder-Decoder Multimodal model
                encoder_inputs, decoder_inputs = (
766
                    self._split_enc_dec_mm_inputs(inputs))
767
768
769
            else:
                encoder_inputs = inputs
                decoder_inputs = None
770
771

        return self._build_enc_dec_llm_inputs(encoder_inputs, decoder_inputs)
772
773
774

    def _build_decoder_only_llm_inputs(
        self,
775
        prompt_inputs: DecoderOnlyInputs,
776
    ) -> DecoderOnlyInputs:
777
778
779
        if "prompt_token_ids" in prompt_inputs:
            prompt_inputs = cast(Union[TokenInputs, MultiModalInputs],
                                 prompt_inputs)  # Needed for mypy
780

781
        return prompt_inputs
782
783
784

    def _process_decoder_only_prompt(
        self,
785
        prompt: SingletonPrompt,
786
        tokenization_kwargs: Optional[dict[str, Any]] = None,
787
        lora_request: Optional[LoRARequest] = None,
788
        return_mm_hashes: bool = False,
789
    ) -> DecoderOnlyInputs:
790
        """
791
        For decoder-only models:
792
793
        Process an input prompt into a
        [`DecoderOnlyInputs`][vllm.inputs.data.DecoderOnlyInputs] instance.
794
795
796

        Arguments:

797
        * prompt: input prompt
798
        * lora_request
799
        * return_mm_hashes
800
801
802

        Returns:

803
        * [`DecoderOnlyInputs`][vllm.inputs.data.DecoderOnlyInputs] instance
804
        """
805

806
        prompt_comps = self._prompt_to_llm_inputs(
807
            prompt,
808
            tokenization_kwargs=tokenization_kwargs,
809
            lora_request=lora_request,
810
            return_mm_hashes=return_mm_hashes,
811
812
        )

813
        return self._build_decoder_only_llm_inputs(prompt_comps)
814
815
816

    async def _process_decoder_only_prompt_async(
        self,
817
        prompt: SingletonPrompt,
818
        tokenization_kwargs: Optional[dict[str, Any]] = None,
819
        lora_request: Optional[LoRARequest] = None,
820
        return_mm_hashes: bool = False,
821
    ) -> DecoderOnlyInputs:
822
823
824
825
        """
        Async version of
        [`_process_decoder_only_prompt`][vllm.inputs.preprocess.InputPreprocessor._process_decoder_only_prompt].
        """
826
        prompt_comps = await self._prompt_to_llm_inputs_async(
827
            prompt,
828
            tokenization_kwargs=tokenization_kwargs,
829
            lora_request=lora_request,
830
            return_mm_hashes=return_mm_hashes,
831
832
        )

833
        return self._build_decoder_only_llm_inputs(prompt_comps)
834
835
836

    def preprocess(
        self,
837
        prompt: PromptType,
838
        tokenization_kwargs: Optional[dict[str, Any]] = None,
839
        lora_request: Optional[LoRARequest] = None,
840
        return_mm_hashes: bool = False,
841
    ) -> ProcessorInputs:
842
        """Preprocess the input prompt."""
843
        if self.model_config.is_encoder_decoder:
844
845
846
            assert not return_mm_hashes, (
                "Multimodal hashes for encoder-decoder models should not be ",
                "returned until they are supported on vLLM V1.")
847
848
            # Encoder-decoder model requires special mapping of
            # input prompts to encoder & decoder
849
850
            return self._process_encoder_decoder_prompt(
                prompt, tokenization_kwargs)
851

852
        if is_explicit_encoder_decoder_prompt(prompt):
853
854
855
856
857
            raise ValueError("Cannot pass encoder-decoder prompt "
                             "to decoder-only models")

        # Decoder-only operation
        return self._process_decoder_only_prompt(
858
            prompt,
859
            tokenization_kwargs=tokenization_kwargs,
860
            lora_request=lora_request,
861
            return_mm_hashes=return_mm_hashes,
862
863
864
865
        )

    async def preprocess_async(
        self,
866
        prompt: PromptType,
867
        tokenization_kwargs: Optional[dict[str, Any]] = None,
868
        lora_request: Optional[LoRARequest] = None,
869
        return_mm_hashes: bool = False,
870
    ) -> ProcessorInputs:
871
872
873
874
        """
        Async version of
        [`preprocess`][vllm.inputs.preprocess.InputPreprocessor.preprocess].
        """
875
        if self.model_config.is_encoder_decoder:
876
877
878
            assert not return_mm_hashes, (
                "Multimodal hashes for encoder-decoder models should not be ",
                "returned until they are supported on vLLM V1.")
879
880
            # Encoder-decoder model requires special mapping of
            # input prompts to encoder & decoder
881
            return await self._process_encoder_decoder_prompt_async(prompt)
882

883
        if is_explicit_encoder_decoder_prompt(prompt):
884
885
886
887
888
            raise ValueError("Cannot pass encoder-decoder prompt "
                             "to decoder-only models")

        # Decoder-only operation
        return await self._process_decoder_only_prompt_async(
889
            prompt,
890
            tokenization_kwargs=tokenization_kwargs,
891
            lora_request=lora_request,
892
            return_mm_hashes=return_mm_hashes,
893
        )