preprocess.py 33.8 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
from collections.abc import Mapping
6
from typing import Any, Optional, Union, cast
7
8
9
10
11
12

from typing_extensions import assert_never

from vllm.config import ModelConfig
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
13
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
14
from vllm.multimodal.cache import BaseMultiModalProcessorCache
15
16
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalEncDecInputs,
                                    MultiModalInputs)
17
from vllm.transformers_utils.tokenizer import AnyTokenizer
18
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
19

20
21
22
23
24
from .data import (DecoderOnlyInputs, EmbedsInputs, EmbedsPrompt,
                   EncoderDecoderInputs, ProcessorInputs, PromptType,
                   SingletonInputs, SingletonPrompt, TextPrompt, TokenInputs,
                   TokensPrompt, embeds_inputs, token_inputs)
from .parse import is_explicit_encoder_decoder_prompt, parse_singleton_prompt
25
26
27
28
29
30
31
32
33

logger = init_logger(__name__)


class InputPreprocessor:

    def __init__(
        self,
        model_config: ModelConfig,
34
        tokenizer: Optional[TokenizerGroup],
35
        mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
36
        mm_processor_cache: Optional[BaseMultiModalProcessorCache] = None,
37
38
39
40
41
    ) -> None:
        super().__init__()

        self.model_config = model_config
        self.tokenizer = tokenizer
42
        self.mm_registry = mm_registry
43
        self.mm_processor_cache = mm_processor_cache
44

45
    def get_tokenizer_group(self) -> TokenizerGroup:
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
        if self.tokenizer is None:
            raise ValueError("You cannot pass text prompts when "
                             "`skip_tokenizer_init` is True")

        return self.tokenizer

    def get_bos_token_id(self,
                         lora_request: Optional[LoRARequest] = None
                         ) -> Optional[int]:
        if self.tokenizer is None:
            logger.warning("Using None for BOS token id because tokenizer "
                           "is not initialized")
            return None

        return self.tokenizer.get_lora_tokenizer(lora_request).bos_token_id

    def get_eos_token_id(self,
                         lora_request: Optional[LoRARequest] = None
                         ) -> Optional[int]:
        if self.tokenizer is None:
            logger.warning("Using None for EOS token id because tokenizer "
                           "is not initialized")
            return None

        return self.tokenizer.get_lora_tokenizer(lora_request).eos_token_id

    def get_decoder_start_token_id(self) -> Optional[int]:
73
        """
74
75
76
        Obtain the decoder start token id employed by an encoder/decoder
        model. Returns None for non-encoder/decoder models or if the
        model config is unavailable.
77
        """
78

79
        if not self.model_config.is_encoder_decoder:
80
81
82
            logger.warning_once(
                "Using None for decoder start token id because "
                "this is not an encoder/decoder model.")
83
84
            return None

85
        if self.model_config is None or self.model_config.hf_config is None:
86
87
88
            logger.warning_once(
                "Using None for decoder start token id because "
                "model config is not available.")
89
90
91
            return None

        dec_start_token_id = getattr(self.model_config.hf_config,
92
                                     "decoder_start_token_id", None)
93
        if dec_start_token_id is None:
94
95
96
97
            logger.warning_once(
                "Falling back on <BOS> for decoder start token "
                "id because decoder start token id is not "
                "available.")
98
99
100
101
            dec_start_token_id = self.get_bos_token_id()

        return dec_start_token_id

102
    def _get_default_enc_dec_decoder_prompt(self) -> list[int]:
103
        """
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
        Specifically for encoder/decoder models:
        generate a default decoder prompt for when
        the user specifies only the encoder prompt.

        Encoder/decoder models utilize the decoder
        prompt in different ways; as new models are
        added, it is intended that this function
        will be extended to produce differing
        default decoder prompts, depending on the
        model variety.

        Absent a special case, the default behavior
        of this method is to mirror the behavior of
        the HuggingFace (HF) GenerationMixin for a None
        decoder prompt, which is to employ a logit processor
        setting to force the first decoded token to be <BOS>.
        Here, this behavior is approximated by having the
        "default" decoder prompt be <BOS>.

        However, it is possible that in the future
124
        other models may have different or more
125
126
127
128
129
130
131
        complex logic for the default decoder prompt.
        This motivates having a special helper method
        for default decoder prompts.

        Returns:

        * prompt_token_ids
132
        """
133
134
135
136
137
138
139

        bos_token_id = self.get_bos_token_id()
        assert bos_token_id is not None
        return [bos_token_id]

    def _prepare_decoder_input_ids_for_generation(
        self,
140
141
        decoder_input_ids: Optional[list[int]],
    ) -> list[int]:
142
143
144
        """
        Prepares `decoder_input_ids` for generation with encoder-decoder models.

145
146
147
148
        Based on:
        https://github.com/huggingface/transformers/blob/4037a2b5b1278736e566aec12e169100275545ea/src/transformers/generation/utils.py
        specifically,
        `GenerationMixin._prepare_decoder_input_ids_for_generation()`.
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166

        Arguments:

        * decoder_input_ids: input token ids to preprocess

        Returns:

        * Processed token list
        """

        decoder_start_token_id = self.get_decoder_start_token_id()
        assert decoder_start_token_id is not None

        if decoder_input_ids is None:
            # no decoder prompt input ->
            # use decoder_start_token_id as decoder_input_ids
            decoder_input_ids = self._get_default_enc_dec_decoder_prompt()

167
168
        if (len(decoder_input_ids) == 0
                or decoder_input_ids[0] != decoder_start_token_id):
169
170
171
172
            decoder_input_ids = [decoder_start_token_id] + decoder_input_ids

        return decoder_input_ids

173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
    def _get_tokenization_kw(
        self,
        overrides: Optional[dict[str, Any]] = None,
    ) -> dict[str, Any]:
        kwargs = dict[str, Any]()

        if self.model_config.hf_config.model_type == "whisper":
            # For Whisper, special tokens should be provided by the user based
            # on the task and language of their request. Also needed to avoid
            # appending an EOS token to the prompt which disrupts generation.
            kwargs["add_special_tokens"] = False

        if overrides:
            kwargs.update(overrides)

        return kwargs

190
191
192
193
    def _tokenize_prompt(
        self,
        prompt: str,
        lora_request: Optional[LoRARequest],
194
        tokenization_kwargs: Optional[dict[str, Any]] = None,
195
    ) -> list[int]:
196
197
198
199
200
        """
        Apply the model's tokenizer to a text prompt, returning the
        corresponding token IDs.
        """
        tokenizer = self.get_tokenizer_group()
201
        tokenization_kwargs = self._get_tokenization_kw(tokenization_kwargs)
202

203
        encoder_config = self.model_config.encoder_config
204

205
        if encoder_config and encoder_config.get("do_lower_case", False):
206
207
            prompt = prompt.lower()

208
        return tokenizer.encode(prompt=prompt,
209
                                lora_request=lora_request,
210
                                **tokenization_kwargs)
211
212
213
214
215

    async def _tokenize_prompt_async(
        self,
        prompt: str,
        lora_request: Optional[LoRARequest],
216
        tokenization_kwargs: Optional[dict[str, Any]] = None,
217
    ) -> list[int]:
218
219
220
221
        """
        Async version of
        [`_tokenize_prompt`][vllm.inputs.preprocess.InputPreprocessor._tokenize_prompt].
        """
222
        tokenizer = self.get_tokenizer_group()
223
        tokenization_kwargs = self._get_tokenization_kw(tokenization_kwargs)
224
225
226
227

        return await tokenizer.encode_async(prompt=prompt,
                                            lora_request=lora_request,
                                            **tokenization_kwargs)
228

229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
    def _get_mm_tokenizer(
        self,
        lora_request: Optional[LoRARequest],
    ) -> AnyTokenizer:
        # PrithviGeoSpatialMAE needs to be initialized without a tokenizer
        # while using also multi-modal input
        if not self.tokenizer:
            return cast(AnyTokenizer, object())  # Dummy

        tokenizer_group = self.get_tokenizer_group()
        return tokenizer_group.get_lora_tokenizer(lora_request)

    async def _get_mm_tokenizer_async(
        self,
        lora_request: Optional[LoRARequest],
    ) -> AnyTokenizer:
        # PrithviGeoSpatialMAE needs to be initialized without a tokenizer
        # while using also multi-modal input
        if not self.tokenizer:
            return cast(AnyTokenizer, object())  # Dummy

        tokenizer_group = self.get_tokenizer_group()
        return await tokenizer_group.get_lora_tokenizer_async(lora_request)

253
254
    def _process_multimodal(
        self,
255
        prompt: Union[str, list[int]],
256
257
        mm_data: MultiModalDataDict,
        mm_processor_kwargs: Optional[Mapping[str, object]],
258
259
        tokenization_kwargs: Optional[dict[str, Any]] = None,
        lora_request: Optional[LoRARequest] = None,
260
261
        *,
        mm_hash_overrides: Optional[dict[str, list[str]]] = None,
262
    ) -> MultiModalInputs:
263
264
265
266
        """
        Apply the model's multi-modal processor to a multi-modal prompt,
        returning the corresponding token IDs and metadata.
        """
267
        tokenizer = self._get_mm_tokenizer(lora_request)
268

269
270
271
272
273
        mm_processor = self.mm_registry.create_processor(
            self.model_config,
            tokenizer=tokenizer,
            cache=self.mm_processor_cache,
        )
274
275
276
277

        if mm_processor_kwargs is None:
            mm_processor_kwargs = {}

278
279
280
281
282
283
284
        return mm_processor.apply(
            prompt,
            mm_data,
            hf_processor_mm_kwargs=mm_processor_kwargs,
            tokenization_kwargs=tokenization_kwargs,
            mm_hash_overrides=mm_hash_overrides,
        )
285
286
287

    async def _process_multimodal_async(
        self,
288
        prompt: Union[str, list[int]],
289
290
        mm_data: MultiModalDataDict,
        mm_processor_kwargs: Optional[Mapping[str, object]],
291
292
        tokenization_kwargs: Optional[dict[str, Any]] = None,
        lora_request: Optional[LoRARequest] = None,
293
294
        *,
        mm_hash_overrides: Optional[dict[str, list[str]]] = None,
295
    ) -> MultiModalInputs:
296
297
298
299
        """
        Async version of
        [`_process_multimodal`][vllm.inputs.preprocess.InputPreprocessor._process_multimodal].
        """
300
        tokenizer = await self._get_mm_tokenizer_async(lora_request)
301

302
303
304
305
306
307
        mm_processor = self.mm_registry.create_processor(
            self.model_config,
            tokenizer=tokenizer,
            cache=self.mm_processor_cache,
        )

308
309
310
        if mm_processor_kwargs is None:
            mm_processor_kwargs = {}

311
312
313
314
315
316
317
        return mm_processor.apply(
            prompt,
            mm_data,
            hf_processor_mm_kwargs=mm_processor_kwargs,
            tokenization_kwargs=tokenization_kwargs,
            mm_hash_overrides=mm_hash_overrides,
        )
318

319
320
321
322
    def _process_embeds(
        self,
        parsed_content: EmbedsPrompt,
    ) -> EmbedsInputs:
323
324
325
        if not self.model_config.enable_prompt_embeds:
            raise ValueError("You must set `--enable-prompt-embeds` to input "
                             "`prompt_embeds`.")
326
327

        prompt_embeds = parsed_content["prompt_embeds"]
328

329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
        # prompt_embeds must be (seq_len, hidden_size), but if the user
        # passes in a batch of size 1, i.e. (1, seq_len, hidden_size),
        # we can unambiguously process the intent by squeezing the batch
        # dimension.
        if prompt_embeds.ndim == 3:
            prompt_embeds = prompt_embeds.squeeze(dim=0)

        if prompt_embeds.ndim != 2:
            raise ValueError(
                "prompt_embeds must be of shape (seq_len, hidden_size).")

        return embeds_inputs(prompt_embeds=prompt_embeds,
                             cache_salt=parsed_content.get("cache_salt"))

    async def _process_embeds_async(
        self,
        parsed_content: EmbedsPrompt,
    ) -> EmbedsInputs:
        return self._process_embeds(parsed_content)

    def _process_tokens(
        self,
        parsed_content: TokensPrompt,
352
        tokenization_kwargs: Optional[dict[str, Any]] = None,
353
        lora_request: Optional[LoRARequest] = None,
354
355
        *,
        mm_hash_overrides: Optional[dict[str, list[str]]] = None,
356
357
358
359
360
361
362
363
364
    ) -> Union[TokenInputs, MultiModalInputs]:
        prompt_token_ids = parsed_content["prompt_token_ids"]

        inputs: Union[TokenInputs, MultiModalInputs]
        if multi_modal_data := parsed_content.get("multi_modal_data"):
            inputs = self._process_multimodal(
                prompt_token_ids,
                multi_modal_data,
                parsed_content.get("mm_processor_kwargs"),
365
                tokenization_kwargs=tokenization_kwargs,
366
                lora_request=lora_request,
367
                mm_hash_overrides=mm_hash_overrides,
368
            )
369
        else:
370
            inputs = token_inputs(prompt_token_ids=prompt_token_ids)
371
372
373
374
375
376
377
378
379

        if cache_salt := parsed_content.get("cache_salt"):
            inputs["cache_salt"] = cache_salt

        return inputs

    async def _process_tokens_async(
        self,
        parsed_content: TokensPrompt,
380
        tokenization_kwargs: Optional[dict[str, Any]] = None,
381
        lora_request: Optional[LoRARequest] = None,
382
383
        *,
        mm_hash_overrides: Optional[dict[str, list[str]]] = None,
384
385
386
387
388
389
390
391
392
    ) -> Union[TokenInputs, MultiModalInputs]:
        prompt_token_ids = parsed_content["prompt_token_ids"]

        inputs: Union[TokenInputs, MultiModalInputs]
        if multi_modal_data := parsed_content.get("multi_modal_data"):
            inputs = await self._process_multimodal_async(
                prompt_token_ids,
                multi_modal_data,
                parsed_content.get("mm_processor_kwargs"),
393
                tokenization_kwargs=tokenization_kwargs,
394
                lora_request=lora_request,
395
                mm_hash_overrides=mm_hash_overrides,
396
397
            )
        else:
398
            inputs = token_inputs(prompt_token_ids=prompt_token_ids, )
399
400
401
402
403
404
405
406
407
408
409

        if cache_salt := parsed_content.get("cache_salt"):
            inputs["cache_salt"] = cache_salt

        return inputs

    def _process_text(
        self,
        parsed_content: TextPrompt,
        tokenization_kwargs: Optional[dict[str, Any]] = None,
        lora_request: Optional[LoRARequest] = None,
410
411
        *,
        mm_hash_overrides: Optional[dict[str, list[str]]] = None,
412
413
414
415
416
417
418
419
420
    ) -> Union[TokenInputs, MultiModalInputs]:
        prompt_text = parsed_content["prompt"]

        inputs: Union[TokenInputs, MultiModalInputs]
        if multi_modal_data := parsed_content.get("multi_modal_data"):
            inputs = self._process_multimodal(
                prompt_text,
                multi_modal_data,
                parsed_content.get("mm_processor_kwargs"),
421
                tokenization_kwargs=tokenization_kwargs,
422
                lora_request=lora_request,
423
                mm_hash_overrides=mm_hash_overrides,
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
            )
        else:
            prompt_token_ids = self._tokenize_prompt(
                prompt_text,
                lora_request=lora_request,
                tokenization_kwargs=tokenization_kwargs,
            )
            inputs = token_inputs(
                prompt=prompt_text,
                prompt_token_ids=prompt_token_ids,
            )

        if cache_salt := parsed_content.get("cache_salt"):
            inputs["cache_salt"] = cache_salt

        return inputs
440

441
442
443
444
445
    async def _process_text_async(
        self,
        parsed_content: TextPrompt,
        tokenization_kwargs: Optional[dict[str, Any]] = None,
        lora_request: Optional[LoRARequest] = None,
446
447
        *,
        mm_hash_overrides: Optional[dict[str, list[str]]] = None,
448
449
450
451
452
453
454
455
456
    ) -> Union[TokenInputs, MultiModalInputs]:
        prompt_text = parsed_content["prompt"]

        inputs: Union[TokenInputs, MultiModalInputs]
        if multi_modal_data := parsed_content.get("multi_modal_data"):
            inputs = await self._process_multimodal_async(
                prompt_text,
                multi_modal_data,
                parsed_content.get("mm_processor_kwargs"),
457
                tokenization_kwargs=tokenization_kwargs,
458
                lora_request=lora_request,
459
                mm_hash_overrides=mm_hash_overrides,
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
            )
        else:
            prompt_token_ids = await self._tokenize_prompt_async(
                prompt_text,
                lora_request=lora_request,
                tokenization_kwargs=tokenization_kwargs,
            )
            inputs = token_inputs(
                prompt=prompt_text,
                prompt_token_ids=prompt_token_ids,
            )

        if cache_salt := parsed_content.get("cache_salt"):
            inputs["cache_salt"] = cache_salt

        return inputs
476

477
    def _prompt_to_llm_inputs(
478
        self,
479
        prompt: SingletonPrompt,
480
        tokenization_kwargs: Optional[dict[str, Any]] = None,
481
        lora_request: Optional[LoRARequest] = None,
482
483
        *,
        mm_hash_overrides: Optional[dict[str, list[str]]] = None,
484
    ) -> SingletonInputs:
485
486
        """
        Extract the singleton inputs from a prompt.
487
488
489

        Arguments:

490
        * prompt: single encoder or decoder input prompt
491
492
493
494
        * lora_request: this is only valid for decoder prompts

        Returns:

495
        * [`SingletonInputs`][vllm.inputs.data.SingletonInputs] instance
496
        """
497
        parsed = parse_singleton_prompt(prompt)
498
499

        if parsed["type"] == "embeds":
500
501
502
503
            return self._process_embeds(parsed["content"])
        if parsed["type"] == "tokens":
            return self._process_tokens(
                parsed["content"],
504
                lora_request=lora_request,
505
                mm_hash_overrides=mm_hash_overrides,
506
            )
507
508
509
510
        if parsed["type"] == "text":
            return self._process_text(
                parsed["content"],
                tokenization_kwargs=tokenization_kwargs,
511
                lora_request=lora_request,
512
                mm_hash_overrides=mm_hash_overrides,
513
514
515
516
            )
        if parsed["type"] == "str":
            return self._process_text(
                TextPrompt(prompt=parsed["content"]),
517
                tokenization_kwargs=tokenization_kwargs,
518
                lora_request=lora_request,
519
                mm_hash_overrides=mm_hash_overrides,
520
            )
521

522
523
        assert_never(parsed)

524
    async def _prompt_to_llm_inputs_async(
525
        self,
526
        prompt: SingletonPrompt,
527
        tokenization_kwargs: Optional[dict[str, Any]] = None,
528
        lora_request: Optional[LoRARequest] = None,
529
530
        *,
        mm_hash_overrides: Optional[dict[str, list[str]]] = None,
531
    ) -> SingletonInputs:
532
533
534
535
        """
        Async version of
        [`_prompt_to_llm_inputs`][vllm.inputs.preprocess.InputPreprocessor._prompt_to_llm_inputs].
        """
536
        parsed = parse_singleton_prompt(prompt)
537

538
        if parsed["type"] == "embeds":
539
540
541
542
            return await self._process_embeds_async(parsed["content"])
        if parsed["type"] == "tokens":
            return await self._process_tokens_async(
                parsed["content"],
543
                lora_request=lora_request,
544
                mm_hash_overrides=mm_hash_overrides,
545
            )
546
547
548
549
        if parsed["type"] == "text":
            return await self._process_text_async(
                parsed["content"],
                tokenization_kwargs=tokenization_kwargs,
550
                lora_request=lora_request,
551
                mm_hash_overrides=mm_hash_overrides,
552
553
554
555
            )
        if parsed["type"] == "str":
            return await self._process_text_async(
                TextPrompt(prompt=parsed["content"]),
556
                tokenization_kwargs=tokenization_kwargs,
557
                lora_request=lora_request,
558
                mm_hash_overrides=mm_hash_overrides,
559
            )
560

561
562
        assert_never(parsed)

563
564
    def _build_enc_dec_llm_inputs(
        self,
565
566
        encoder_inputs: SingletonInputs,
        decoder_inputs: Optional[SingletonInputs],
567
    ) -> EncoderDecoderInputs:
568
569
570
571
        if (encoder_inputs["type"] == "embeds"
                or decoder_inputs and decoder_inputs["type"] == "embeds"):
            raise ValueError("Embedding inputs are not supported for encoder-"
                             "decoder models")
572

573
574
575
576
577
        # Needed for mypy
        encoder_inputs = cast(Union[TokenInputs, MultiModalInputs],
                              encoder_inputs)
        decoder_inputs = cast(Optional[Union[TokenInputs, MultiModalInputs]],
                              decoder_inputs)
578

579
        if decoder_inputs is None:
580
581
582
583
584
585
586
587
588
            if self.model_config.hf_config.model_type == "whisper":
                # For Whisper models, the text prompt should go to the decoder.
                # If no explicit encoder/decoder inputs, then copy the prompt
                # from the encoder to the decoder. The encoder tokens are later
                # overridden by the audio features.
                dec_token_ids = encoder_inputs["prompt_token_ids"].copy()
            else:
                dec_token_ids = self._prepare_decoder_input_ids_for_generation(
                    None)
589
            decoder_inputs = token_inputs(dec_token_ids)
590
        else:
591
592
593
            if "multi_modal_data" in decoder_inputs:
                raise ValueError("Multi-modal decoder inputs of encoder-"
                                 "decoder models are not supported yet")
594
595
596
597

            dec_token_ids = self._prepare_decoder_input_ids_for_generation(
                decoder_inputs["prompt_token_ids"])
            decoder_inputs["prompt_token_ids"] = dec_token_ids
598

599
        return EncoderDecoderInputs(
600
601
            encoder=encoder_inputs,
            decoder=decoder_inputs,
602
603
        )

604
    def _split_enc_dec_mm_inputs(
605
        self,
606
607
        inputs: Union[SingletonInputs, MultiModalEncDecInputs],
        decoder_inputs_to_override: Optional[SingletonInputs] = None,
608
    ) -> tuple[SingletonInputs, SingletonInputs]:
609
610
611
612
        """
        For encoder/decoder models only:
        Separate Encoder/Decoder inputs from a MultiModalEncDecInputs
        """
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
        if (inputs["type"] == "embeds" or decoder_inputs_to_override
                and decoder_inputs_to_override["type"] == "embeds"):
            raise ValueError("Embedding inputs are not supported for encoder-"
                             "decoder models")

        # Needed for mypy
        inputs = cast(
            Union[TokenInputs, MultiModalInputs, MultiModalEncDecInputs],
            inputs,
        )
        decoder_inputs_to_override = cast(
            Optional[Union[TokenInputs, MultiModalInputs]],
            decoder_inputs_to_override,
        )

628
629
        encoder_inputs: SingletonInputs
        decoder_inputs: SingletonInputs
630
631
632
633
634
635
636

        if inputs["type"] == "multimodal":  # Multimodal data inputs
            if not ("encoder_prompt" in inputs
                    and "encoder_prompt_token_ids" in inputs):
                raise RuntimeError("You should register an encoder-decoder "
                                   "multi-modal processor for encoder-decoder "
                                   "models.")
637
            inputs = cast(MultiModalEncDecInputs, inputs)
638

639
640
641
642
            encoder_inputs = token_inputs(
                prompt=inputs["encoder_prompt"],
                prompt_token_ids=inputs["encoder_prompt_token_ids"],
            )
643

644
645
646
647
648
649
650
651
652
653
            decoder_prompt_inputs = decoder_inputs_to_override or inputs
            decoder_inputs = MultiModalInputs(
                type="multimodal",
                prompt=decoder_prompt_inputs.get("prompt", ""),
                prompt_token_ids=decoder_prompt_inputs["prompt_token_ids"],
                mm_kwargs=inputs["mm_kwargs"],
                mm_hashes=inputs["mm_hashes"],
                mm_placeholders=inputs["mm_placeholders"],
            )
            if cache_salt := inputs.get("cache_salt"):
654
655
                decoder_inputs["cache_salt"] = cache_salt

656
        elif inputs["type"] == "token":  # Text-only inputs
657
658
659
660
            encoder_inputs = token_inputs(prompt="", prompt_token_ids=[])
            decoder_inputs = decoder_inputs_to_override or inputs
        else:
            assert_never(inputs)  # type: ignore[arg-type]
661

662
663
        return encoder_inputs, decoder_inputs

664
665
    def _process_encoder_decoder_prompt(
        self,
666
        prompt: PromptType,
667
        tokenization_kwargs: Optional[dict[str, Any]] = None,
668
669
        *,
        mm_hash_overrides: Optional[dict[str, list[str]]] = None,
670
    ) -> EncoderDecoderInputs:
671
        """
672
        For encoder/decoder models only:
673
674
675
        Process an input prompt into an
        [`EncoderDecoderInputs`][vllm.inputs.data.EncoderDecoderInputs]
        instance.
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693

        There are two types of input prompts:
        singleton prompts which carry only the
        encoder prompt, and explicit encoder/decoder
        prompts which carry both the encoder and the
        decoder prompts as member variables.

        This function handles the following scenarios:
        * Singleton encoder prompt: extract encoder prompt
          token ids & infer default decoder prompt token ids
        * Explicit encoder/decoder prompt: extract encoder
          and decoder prompt token ids

        Note that for Explicit encoder/decoder prompts,
        each sub-prompt (encoder or decoder prompt) can
        have any possible singleton type; thus this
        method relies on helper functions to obtain
        token ids for the sub-prompts.
694

695
696
        Arguments:

697
        * prompt: an input prompt
698
699
700

        Returns:

701
702
        * [`EncoderDecoderInputs`][vllm.inputs.data.EncoderDecoderInputs]
          instance
703
        """
704
705
        encoder_inputs: SingletonInputs
        decoder_inputs: Optional[SingletonInputs]
706

707
        if is_explicit_encoder_decoder_prompt(prompt):
708
            encoder_inputs = self._prompt_to_llm_inputs(
709
710
                prompt["encoder_prompt"],
                tokenization_kwargs=tokenization_kwargs,
711
                mm_hash_overrides=mm_hash_overrides,
712
            )
713
            if (decoder_input := prompt["decoder_prompt"]) is None:
714
                decoder_inputs = None
715
            else:
716
                decoder_inputs = self._prompt_to_llm_inputs(decoder_input)
717
718
            # For multimodal model, override decoder prompt from processor
            # with explicit decoder prompt.
719
            if self.model_config.is_multimodal_model:
720
                encoder_inputs, decoder_inputs = (
721
722
                    self._split_enc_dec_mm_inputs(encoder_inputs,
                                                  decoder_inputs))
723
        else:
724
725
726
            inputs = self._prompt_to_llm_inputs(
                prompt,
                tokenization_kwargs=tokenization_kwargs,
727
                mm_hash_overrides=mm_hash_overrides,
728
            )
729
            if self.model_config.is_multimodal_model:
730
731
                # Encoder-Decoder Multimodal model
                encoder_inputs, decoder_inputs = (
732
                    self._split_enc_dec_mm_inputs(inputs))
733
734
735
            else:
                encoder_inputs = inputs
                decoder_inputs = None
736
737

        return self._build_enc_dec_llm_inputs(encoder_inputs, decoder_inputs)
738
739
740

    async def _process_encoder_decoder_prompt_async(
        self,
741
        prompt: PromptType,
742
        tokenization_kwargs: Optional[dict[str, Any]] = None,
743
744
        *,
        mm_hash_overrides: Optional[dict[str, list[str]]] = None,
745
    ) -> EncoderDecoderInputs:
746
747
748
749
        """
        Async version of
        [`_process_encoder_decoder_prompt`][vllm.inputs.preprocess.InputPreprocessor._process_encoder_decoder_prompt].
        """
750
751
        encoder_inputs: SingletonInputs
        decoder_inputs: Optional[SingletonInputs]
752

753
        if is_explicit_encoder_decoder_prompt(prompt):
754
            encoder_task = self._prompt_to_llm_inputs_async(
755
756
                prompt["encoder_prompt"],
                tokenization_kwargs=tokenization_kwargs,
757
                mm_hash_overrides=mm_hash_overrides,
758
            )
759

760
            if (decoder_input := prompt["decoder_prompt"]) is None:
761
762
                encoder_inputs = await encoder_task
                decoder_inputs = None
763
            else:
764
765
766
                decoder_task = self._prompt_to_llm_inputs_async(
                    decoder_input,
                    tokenization_kwargs=tokenization_kwargs,
767
                    mm_hash_overrides=mm_hash_overrides,
768
                )
769

770
                encoder_inputs, decoder_inputs = await asyncio.gather(
771
                    encoder_task, decoder_task)
772
773
774

            # For multimodal model, override decoder prompt from processor
            # with explicit decoder prompt.
775
            if self.model_config.is_multimodal_model:
776
                encoder_inputs, decoder_inputs = (
777
778
                    self._split_enc_dec_mm_inputs(encoder_inputs,
                                                  decoder_inputs))
779
        else:
780
781
782
            inputs = await self._prompt_to_llm_inputs_async(
                prompt,
                tokenization_kwargs=tokenization_kwargs,
783
                mm_hash_overrides=mm_hash_overrides,
784
            )
785
            if self.model_config.is_multimodal_model:
786
787
                # Encoder-Decoder Multimodal model
                encoder_inputs, decoder_inputs = (
788
                    self._split_enc_dec_mm_inputs(inputs))
789
790
791
            else:
                encoder_inputs = inputs
                decoder_inputs = None
792
793

        return self._build_enc_dec_llm_inputs(encoder_inputs, decoder_inputs)
794
795
796

    def _build_decoder_only_llm_inputs(
        self,
797
        prompt_inputs: DecoderOnlyInputs,
798
    ) -> DecoderOnlyInputs:
799
800
801
        if "prompt_token_ids" in prompt_inputs:
            prompt_inputs = cast(Union[TokenInputs, MultiModalInputs],
                                 prompt_inputs)  # Needed for mypy
802

803
        return prompt_inputs
804
805
806

    def _process_decoder_only_prompt(
        self,
807
        prompt: SingletonPrompt,
808
        tokenization_kwargs: Optional[dict[str, Any]] = None,
809
        lora_request: Optional[LoRARequest] = None,
810
811
        *,
        mm_hash_overrides: Optional[dict[str, list[str]]] = None,
812
    ) -> DecoderOnlyInputs:
813
        """
814
        For decoder-only models:
815
816
        Process an input prompt into a
        [`DecoderOnlyInputs`][vllm.inputs.data.DecoderOnlyInputs] instance.
817
818
819

        Arguments:

820
        * prompt: input prompt
821
822
823
824
        * lora_request

        Returns:

825
        * [`DecoderOnlyInputs`][vllm.inputs.data.DecoderOnlyInputs] instance
826
        """
827

828
        prompt_comps = self._prompt_to_llm_inputs(
829
            prompt,
830
            tokenization_kwargs=tokenization_kwargs,
831
            lora_request=lora_request,
832
            mm_hash_overrides=mm_hash_overrides,
833
834
        )

835
        return self._build_decoder_only_llm_inputs(prompt_comps)
836
837
838

    async def _process_decoder_only_prompt_async(
        self,
839
        prompt: SingletonPrompt,
840
        tokenization_kwargs: Optional[dict[str, Any]] = None,
841
        lora_request: Optional[LoRARequest] = None,
842
843
        *,
        mm_hash_overrides: Optional[dict[str, list[str]]] = None,
844
    ) -> DecoderOnlyInputs:
845
846
847
848
        """
        Async version of
        [`_process_decoder_only_prompt`][vllm.inputs.preprocess.InputPreprocessor._process_decoder_only_prompt].
        """
849
        prompt_comps = await self._prompt_to_llm_inputs_async(
850
            prompt,
851
            tokenization_kwargs=tokenization_kwargs,
852
            lora_request=lora_request,
853
            mm_hash_overrides=mm_hash_overrides,
854
855
        )

856
        return self._build_decoder_only_llm_inputs(prompt_comps)
857
858
859

    def preprocess(
        self,
860
        prompt: PromptType,
861
        tokenization_kwargs: Optional[dict[str, Any]] = None,
862
        lora_request: Optional[LoRARequest] = None,
863
864
        *,
        mm_hash_overrides: Optional[dict[str, list[str]]] = None,
865
    ) -> ProcessorInputs:
866
        """Preprocess the input prompt."""
867
        if self.model_config.is_encoder_decoder:
868
            # Encoder-decoder model requires special mapping of
869
            # input prompts to encoder & decoder.
870
            return self._process_encoder_decoder_prompt(
871
872
                prompt,
                tokenization_kwargs,
873
                mm_hash_overrides=mm_hash_overrides,
874
            )
875

876
        if is_explicit_encoder_decoder_prompt(prompt):
877
878
879
880
881
            raise ValueError("Cannot pass encoder-decoder prompt "
                             "to decoder-only models")

        # Decoder-only operation
        return self._process_decoder_only_prompt(
882
            prompt,
883
            tokenization_kwargs=tokenization_kwargs,
884
            lora_request=lora_request,
885
            mm_hash_overrides=mm_hash_overrides,
886
887
888
889
        )

    async def preprocess_async(
        self,
890
        prompt: PromptType,
891
        tokenization_kwargs: Optional[dict[str, Any]] = None,
892
        lora_request: Optional[LoRARequest] = None,
893
894
        *,
        mm_hash_overrides: Optional[dict[str, list[str]]] = None,
895
    ) -> ProcessorInputs:
896
897
898
899
        """
        Async version of
        [`preprocess`][vllm.inputs.preprocess.InputPreprocessor.preprocess].
        """
900
        if self.model_config.is_encoder_decoder:
901
            # Encoder-decoder model requires special mapping of
902
903
904
905
            # input prompts to encoder & decoder.
            return await self._process_encoder_decoder_prompt_async(
                prompt,
                tokenization_kwargs,
906
                mm_hash_overrides=mm_hash_overrides,
907
            )
908

909
        if is_explicit_encoder_decoder_prompt(prompt):
910
911
912
913
914
            raise ValueError("Cannot pass encoder-decoder prompt "
                             "to decoder-only models")

        # Decoder-only operation
        return await self._process_decoder_only_prompt_async(
915
            prompt,
916
            tokenization_kwargs=tokenization_kwargs,
917
            lora_request=lora_request,
918
            mm_hash_overrides=mm_hash_overrides,
919
        )
920
921
922
923

    def clear_cache(self) -> None:
        if self.mm_processor_cache is not None:
            self.mm_processor_cache.clear_cache()