processing.py 69.5 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import time
4
from abc import ABC, abstractmethod
5
from collections import defaultdict
6
from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence
7
from dataclasses import dataclass, field, replace
8
from enum import Enum
9
from functools import lru_cache
10
11
12
13
14
15
16
17
18
19
20
from typing import (
    TYPE_CHECKING,
    Any,
    Generic,
    NamedTuple,
    Optional,
    Protocol,
    Union,
    cast,
    overload,
)
21

22
import regex as re
23
import torch
24
from typing_extensions import TypeVar, assert_never
25

26
from vllm.logger import init_logger
27
from vllm.transformers_utils.processor import cached_processor_from_config
28
29
from vllm.transformers_utils.tokenizer import AnyTokenizer, decode_tokens, encode_tokens
from vllm.utils import flatten_2d_lists, full_groupby, get_allowed_kwarg_only_overrides
30
from vllm.utils.jsontree import JSONTree, json_map_leaves
31

32
from .hasher import MultiModalHasher
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
from .inputs import (
    MultiModalDataDict,
    MultiModalEncDecInputs,
    MultiModalFieldConfig,
    MultiModalInputs,
    MultiModalKwargsItem,
    MultiModalKwargsItems,
    MultiModalKwargsOptionalItems,
    MultiModalUUIDDict,
    PlaceholderRange,
)
from .parse import (
    DictEmbeddingItems,
    EmbeddingItems,
    MultiModalDataItems,
    MultiModalDataParser,
)
50
51

if TYPE_CHECKING:
52
53
54
55
    from transformers.configuration_utils import PretrainedConfig
    from transformers.feature_extraction_utils import BatchFeature
    from transformers.processing_utils import ProcessorMixin

56
57
    from vllm.config import ModelConfig

58
    from .cache import BaseMultiModalProcessorCache
59
    from .profiling import BaseDummyInputsBuilder
60

61
logger = init_logger(__name__)
62
63

_S = TypeVar("_S", str, list[int])
64
65
66

PromptSeq = Union[str, list[int]]
"""A token sequence (list of token IDs) or text."""
67

68

69
70
71
72
73
74
75
@lru_cache(maxsize=2048)
def _cached_encode(
    tokenizer: AnyTokenizer,
    text: str,
    *,
    add_special_tokens: Optional[bool] = None,
) -> list[int]:
76
    return encode_tokens(tokenizer, text, add_special_tokens=add_special_tokens)
77
78
79
80
81
82
83
84
85


@lru_cache(maxsize=2048)
def _cached_decode(
    tokenizer: AnyTokenizer,
    token_ids: tuple[int, ...],
    *,
    skip_special_tokens: Optional[bool] = None,
) -> str:
86
87
88
    return decode_tokens(
        tokenizer, list(token_ids), skip_special_tokens=skip_special_tokens
    )
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104


def _seq2text(tokenizer: AnyTokenizer, seq: PromptSeq) -> str:
    if isinstance(seq, str):
        return seq

    return _cached_decode(tokenizer, tuple(seq))


def _seq2tokens(tokenizer: AnyTokenizer, seq: PromptSeq) -> list[int]:
    if isinstance(seq, str):
        return _cached_encode(tokenizer, seq, add_special_tokens=False)

    return seq


105
106
107
108
109
110
class _GetMatchIndex(Protocol):
    def __call__(
        self,
        tokenizer: AnyTokenizer,
        prompt: PromptSeq,
        start_idx: int = 0,
111
    ) -> Optional[int]: ...
112
113


114
115
116
@dataclass
class PromptIndex:
    """Resolves to an index in the prompt."""
117

118
    get_match_index: _GetMatchIndex
119
120
121
122
123
124
125
126
127
128


class PromptIndexTargets:
    @staticmethod
    def start() -> PromptIndex:
        """
        Resolves to the start of the prompt (before the first token).

        This results in a match even if the prompt is empty.
        """
129
        return PromptIndex(lambda tokenizer, prompt, start_idx=0: 0)
130
131
132
133
134
135
136
137
138
139

    @staticmethod
    def prefix(seq: PromptSeq) -> PromptIndex:
        """
        Resolves to a location in the prompt after the given prefix.
        """

        def get_match_index(
            tokenizer: AnyTokenizer,
            prompt: PromptSeq,
140
            start_idx: int = 0,
141
        ) -> Optional[int]:
142
143
144
            if start_idx != 0:
                return None

145
146
147
148
149
150
151
152
153
            prefix = seq

            if isinstance(prompt, str):
                if not isinstance(prefix, str):
                    # Make both `str`
                    prefix = decode_tokens(tokenizer, prefix)
            else:
                if isinstance(prefix, str):
                    # Make both `list[int]`
154
                    prefix = encode_tokens(tokenizer, prefix, add_special_tokens=False)
155
156
157
158
159
160
161
162
163
164
165
166
167

            match_idx = len(prefix)
            return match_idx if prompt[:match_idx] == prefix else None

        return PromptIndex(get_match_index)

    @staticmethod
    def end() -> PromptIndex:
        """
        Resolves to the end of the prompt (after the last token).

        This results in a match even if the prompt is empty.
        """
168
        return PromptIndex(lambda tokenizer, prompt, start_idx=0: len(prompt))
169
170


171
UpdateTarget = Union[PromptSeq, PromptIndex]
172
173
174
175
"""
The token sequence or text to update.
"""

176
177
178
179
180
181
182
183
184
185
PromptUpdateTarget = Union[Callable[[int], UpdateTarget], UpdateTarget]
"""
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
output the corresponding token sequence (or text).

For convenience, you can directly pass in the token sequence (or text)
instead of a function if it does not depend on the input.
"""

186

187
@dataclass
188
class PromptUpdateDetails(Generic[_S]):
189
    """Details about the token sequence or text that are part of the update."""
190

191
    full: _S
192
    """The full content."""
193

194
    is_embed: Optional[Callable[[AnyTokenizer, PromptSeq], torch.Tensor]] = None
195
    """
196
197
198
    Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full],
    return a boolean mask of shape `(len(full),)` indicating which positions
    of `full` to assign embeddings to.
199
200
201
202

    `None` (default) means to assign embeddings to all positions of `full`.

    The embeddings are obtained by calling
203
    [`SupportsMultiModal.get_multimodal_embeddings`][vllm.model_executor.models.interfaces.SupportsMultiModal.get_multimodal_embeddings].
204
205
206
    """

    @staticmethod
207
    def from_seq(seq: _S) -> "PromptUpdateDetails[_S]":
208
209
210
211
212
213
214
        return PromptUpdateDetails(full=seq)

    @staticmethod
    def select_text(
        seq: _S,
        embed_text: str,
    ) -> "PromptUpdateDetails[_S]":
215
216
217
        def is_embed(tokenizer: AnyTokenizer, full: PromptSeq) -> torch.Tensor:
            embed_token_ids = encode_tokens(tokenizer, embed_text)
            token_ids = _seq2tokens(tokenizer, full)
218
219

            return torch.isin(
220
                torch.tensor(token_ids),
221
222
223
224
225
226
227
228
229
230
                torch.tensor(embed_token_ids),
            )

        return PromptUpdateDetails(full=seq, is_embed=is_embed)

    @staticmethod
    def select_token_id(
        seq: _S,
        embed_token_id: int,
    ) -> "PromptUpdateDetails[_S]":
231
232
233
234
235
236
        def is_embed(tokenizer: AnyTokenizer, full: PromptSeq) -> torch.Tensor:
            token_ids = _seq2tokens(tokenizer, full)

            return torch.tensor(token_ids) == embed_token_id

        return PromptUpdateDetails(full=seq, is_embed=is_embed)
237
238


239
PromptUpdateInfo = Union[PromptSeq, PromptUpdateDetails]
240
"""
241
The token sequence or text that are part of the update.
242

243
If only part of the content corresponds to feature placeholders, you can
244
245
use [`PromptUpdateDetails`][vllm.multimodal.processing.PromptUpdateDetails] to
specify which part.
246
"""
247

248
PromptUpdateContent = Union[Callable[[int], PromptUpdateInfo], PromptUpdateInfo]
249
"""
250
251
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
252
253
254
255
256
257
258
259
260
261
262
263
264
output the corresponding token sequence (or text).

For convenience, you can directly pass in the token sequence (or text)
instead of a function if it does not depend on the input.
"""


class UpdateMode(str, Enum):
    INSERT = "insert"
    REPLACE = "replace"


@dataclass
265
class PromptUpdate(ABC):
266
267
268
269
270
271
272
    """
    Defines how to update a prompt with placeholder tokens.
    """

    modality: str
    """The modality for which the update is made."""

273
    target: PromptUpdateTarget
274
275
276
277
278
279
280
281
282
283
284
285
286
287
    """The token sequence (or text) to update."""

    @property
    @abstractmethod
    def content(self) -> PromptUpdateContent:
        """The placeholder tokens that are part of the update."""
        raise NotImplementedError

    @property
    @abstractmethod
    def mode(self) -> UpdateMode:
        """Defines how to update the prompt."""
        raise NotImplementedError

288
    def _resolve_target(self, item_idx: int) -> UpdateTarget:
289
290
291
292
        target = self.target
        if callable(target):
            target = target(item_idx)

293
        return target
294

295
    def _resolve_content(self, item_idx: int) -> PromptUpdateDetails:
296
297
298
299
300
301
302
        content = self.content
        if callable(content):
            content = content(item_idx)

        if not isinstance(content, PromptUpdateDetails):
            content = PromptUpdateDetails.from_seq(content)

303
        return content
304

305
    def resolve(self, item_idx: int) -> "ResolvedPromptUpdate":
306
307
308
309
310
311
312
313
314
        """
        Given the index of the processed item within
        [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
        output a copy of this object with its lazy attributes resolved.
        """
        return ResolvedPromptUpdate(
            modality=self.modality,
            item_idx=item_idx,
            mode=self.mode,
315
316
            target=self._resolve_target(item_idx),
            content=self._resolve_content(item_idx),
317
318
        )

319

320
@dataclass
321
322
323
324
325
326
class PromptInsertion(PromptUpdate):
    """
    Defines how to insert placeholder tokens into a prompt.

    Example:

327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
    For each image, insert a number of ``<image>`` feature placeholders
    equal to the feature size of the vision encoder after the ``<s>`` token:

    ```python
    PromptInsertion(
        modality="image",
        target="<s>",
        insertion="<image>" * image_feature_size,
    )
    ```

    Insert these tokens at the start of the prompt:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.start(),
        insertion="<image>" * image_feature_size,
    )
    ```

    Insert these tokens after a prefix ``Images:``:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.prefix("Images:"),
        insertion="<image>" * image_feature_size,
    )
    ```

    Insert these tokens at the end of the prompt:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.end(),
        insertion="<image>" * image_feature_size,
    )
    ```
367
368
369
370
    """

    insertion: PromptUpdateContent = field(repr=False)
    """
371
372
373
374
    Given the index of the processed item within
    [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
    output the token sequence (or text) to insert right after
    [`target`][vllm.multimodal.processing.PromptUpdate.target].
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390

    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
    """

    @property
    def content(self) -> PromptUpdateContent:
        return self.insertion

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.INSERT


@dataclass
class PromptReplacement(PromptUpdate):
391
392
    """
    Defines how to replace portions of an input prompt with placeholder tokens.
393
394
395

    Example:

396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
    For each image, replace one ``<image>`` input placeholder in the prompt
    with a number of ``<image>`` feature placeholders
    equal to the feature size of the vision encoder:

    ```python
    PromptReplacement(
        modality="image",
        target="<image>",
        replacement="<image>" * image_feature_size,
    )
    ```

    As above, but further pad the feature placeholders with ``<image_bos>``
    and `<image_eos>``, which are not supposed to be passed to the vision
    encoder:

    ```python
    PromptReplacement(
        modality="image",
        target="<image>",
        replacement=PromptUpdateDetails(
417
418
419
420
421
422
423
            full="".join(
                [
                    "<image_bos>",
                    "<image>" * image_feature_size,
                    "<image_eos>",
                ]
            ),
424
425
426
427
428
429
430
431
432
433
434
435
436
            features="<image>" * image_feature_size,
        ),
    )
    ```

    To avoid unnecessary tokenization during prompt replacement,
    we recommended passing token sequences instead of text:

    ```python
    PromptReplacement(
        modality="image",
        target=[image_token_id],
        replacement=PromptUpdateDetails(
437
438
439
            full=(
                [image_bos_id] + [image_token_id] * image_feature_size + [image_eos_id]
            ),
440
441
442
443
            features=[image_token_id] * image_feature_size,
        ),
    )
    ```
444
445
    """

446
    replacement: PromptUpdateContent = field(repr=False)
447
    """
448
449
450
451
    Given the index of the processed item within
    [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
    output the token sequence (or text) to replace
    [`target`][vllm.multimodal.processing.PromptUpdate.target].
452

453
454
    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
455
456
    """

457
458
459
460
461
462
463
    @property
    def content(self) -> PromptUpdateContent:
        return self.replacement

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.REPLACE
464
465


466
467
468
class _HasModalityAttr(Protocol):
    modality: str

469

470
471
class _HasModalityProp(Protocol):
    @property
472
    def modality(self) -> str: ...
473
474
475
476
477
478


_M = TypeVar("_M", bound=Union[_HasModalityAttr, _HasModalityProp])


def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]:
479
480
    """Convenience function to apply [`full_groupby`][vllm.utils.full_groupby]
    based on modality."""
481
482
483
    return full_groupby(values, key=lambda x: x.modality)


484
485
486
487
488
489
490
class PromptTargetMatch(NamedTuple):
    start_idx: int
    end_idx: int


@dataclass(frozen=True)
class ResolvedPromptUpdate:
491
    """
492
493
    A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] with its
    lazy attributes resolved, apart from those related to tokenization.
494
    """
495

496
497
    modality: str
    """The modality for which the update is made."""
498

499
500
    item_idx: int
    """The index within `modality` of the item this update pertains to."""
501

502
503
    mode: UpdateMode
    """Defines how to update the prompt."""
504

505
    target: UpdateTarget
506
    """The token sequence (or text) to update."""
507

508
    content: PromptUpdateDetails = field(repr=False)
509
    """The placeholder tokens that are part of the update."""
510

511
512
513
514
515
516
517
518
519
    def iter_token_matches(
        self,
        prompt: list[int],
        tokenizer: AnyTokenizer,
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        target = self.target
520

521
522
523
524
        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(tokenizer, prompt, start_idx)
            if match_idx is not None:
                yield PromptTargetMatch(match_idx, match_idx)
525

526
            return
527

528
529
        target_token_ids = _seq2tokens(tokenizer, target)

530
        for match in iter_token_matches(prompt, target_token_ids, start_idx=start_idx):
531
            yield PromptTargetMatch(match.start_idx, match.end_idx)
532

533
534
535
536
537
538
539
540
541
    def iter_text_matches(
        self,
        prompt: str,
        tokenizer: AnyTokenizer,
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        target = self.target
542

543
544
545
546
        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(tokenizer, prompt, start_idx)
            if match_idx is not None:
                yield PromptTargetMatch(match_idx, match_idx)
547

548
            return
549

550
551
        target_text = _seq2text(tokenizer, target)

552
        for match in re.finditer(re.escape(target_text), prompt, pos=start_idx):
553
554
555
556
557
558
559
560
561
562
563
            yield PromptTargetMatch(match.start(), match.end())

    def iter_matches(
        self,
        prompt: Union[list[int], str],
        tokenizer: AnyTokenizer,
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        if isinstance(prompt, str):
564
            return self.iter_text_matches(prompt, tokenizer, start_idx=start_idx)
565
566

        return self.iter_token_matches(prompt, tokenizer, start_idx=start_idx)
567

568
569
570
571
572
573
574
575
576
    def with_target(self, target: UpdateTarget):
        return replace(self, target=target)

    def with_content(self, content: PromptUpdateInfo):
        if not isinstance(content, PromptUpdateDetails):
            content = PromptUpdateDetails.from_seq(content)

        return replace(self, content=content)

577

578
579
580
class _TokenMatch(NamedTuple):
    start_idx: int
    end_idx: int
581
582


583
584
585
def iter_token_matches(
    token_ids: list[int],
    match_ids: list[int],
586
587
    *,
    start_idx: int = 0,
588
) -> Generator[_TokenMatch]:
589
    """
590
    Yield each occurrence of `match_ids` in `token_ids`.
591
592
593
594

    Note that empty matches are ignored.
    """
    prompt_len = len(token_ids)
595
    match_len = len(match_ids)
596

597
598
    if match_len == 0:
        return
599

600
    while start_idx < prompt_len - match_len + 1:
601
        end_idx = start_idx + match_len
602

603
604
        if token_ids[start_idx:end_idx] == match_ids:
            yield _TokenMatch(start_idx=start_idx, end_idx=end_idx)
605
606
607
608
609

            # Exclude overlapping matches
            start_idx = end_idx
        else:
            start_idx += 1
610
611


612
613
614
615
616
617
def replace_token_matches(
    token_ids: list[int],
    match_ids: list[int],
    new_ids: list[int],
) -> list[int]:
    """
618
619
    Replace each occurrence of `match_ids` in `token_ids`
    with `new_ids`.
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638

    Note that empty matches are ignored.
    """
    out_seqs = list[list[int]]()
    prev_end_idx = 0

    for match in iter_token_matches(token_ids, match_ids):
        start_idx = match.start_idx
        end_idx = match.end_idx

        out_seqs.append(token_ids[prev_end_idx:start_idx])
        out_seqs.append(new_ids)
        prev_end_idx = end_idx

    out_seqs.append(token_ids[prev_end_idx:])

    return flatten_2d_lists(out_seqs)


639
@dataclass
640
class PlaceholderFeaturesInfo:
641
    modality: str
642
    item_idx: int
643
    start_idx: int
644
    tokens: list[int]
645
    is_embed: Optional[torch.Tensor]
646
647
648

    @property
    def length(self) -> int:
649
        return len(self.tokens)
650
651

    def to_range(self) -> PlaceholderRange:
652
653
        # TODO: Is it worth it to optimize this by stripping the
        # leading and ending positions where `is_embed=False`?
654
655
656
        return PlaceholderRange(
            offset=self.start_idx,
            length=self.length,
657
            is_embed=self.is_embed,
658
        )
659
660


661
_MatchToApply = tuple[tuple[str, int], tuple[PromptTargetMatch, int]]
662
663


664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
def _find_matches(
    prompt: _S,
    mm_prompt_updates: "MultiModalPromptUpdates",
    tokenizer: AnyTokenizer,
    *,
    prev_end_idx: int = 0,
    current_result: "MultiModalPromptUpdatesApplyResult",
) -> tuple[Optional[UpdateMode], list[_MatchToApply]]:
    mode: Optional[UpdateMode] = None
    mm_matches = dict[tuple[str, int], tuple[PromptTargetMatch, int]]()

    for modality, modality_updates in mm_prompt_updates.items():
        for item_idx, item_updates in enumerate(modality_updates):
            if current_result[modality][item_idx] is not None:
                continue  # Updates have already been applied for this item

            for update_idx, update in enumerate(item_updates):
                if (modality, item_idx) in mm_matches:
                    break  # Already found a match for this item

                for match in update.iter_matches(
685
686
687
                    prompt,
                    tokenizer,
                    start_idx=prev_end_idx,
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
                ):
                    # All matches should share the same mode
                    if mode is None:
                        mode = update.mode
                    elif mode != update.mode:
                        continue

                    mm_matches[(modality, item_idx)] = match, update_idx
                    break  # Get only the first valid match per item

    # Prioritize earlier matches
    matches_to_apply = sorted(mm_matches.items(), key=lambda item: item[1][0])

    # To avoid conflicts, only replace one non-empty item at a time
    if mode == UpdateMode.REPLACE:
        matches_to_apply_ = list[_MatchToApply]()
        has_non_empty_matches = False

        for item in matches_to_apply:
            _, (match, _) = item
            if match.start_idx == match.end_idx:
                matches_to_apply_.append(item)
            elif not has_non_empty_matches:
                has_non_empty_matches = True
                matches_to_apply_.append(item)

        matches_to_apply = matches_to_apply_

    return mode, matches_to_apply
717
718


719
def _apply_matches(
720
    prompt: _S,
721
722
723
724
725
    mm_prompt_updates: "MultiModalPromptUpdates",
    tokenizer: AnyTokenizer,
) -> tuple[list[_S], "MultiModalPromptUpdatesApplyResult"]:
    prompt_len = len(prompt)

726
    out_seqs = list[Union[str, list[int]]]()
727
    out_result: MultiModalPromptUpdatesApplyResult = {
728
        m: [None] * len(items) for m, items in mm_prompt_updates.items()
729
    }
730

731
732
733
    start_idx = prev_end_idx = 0
    while start_idx < max(prompt_len, 1):  # Allow inserts into empty prompt
        found = False
734

735
736
737
738
739
740
741
        mode, matches_to_apply = _find_matches(
            prompt,
            mm_prompt_updates,
            tokenizer,
            prev_end_idx=prev_end_idx,
            current_result=out_result,
        )
742

743
744
745
        if mode is not None:
            for (modality, item_idx), (match, update_idx) in matches_to_apply:
                found = True
746

747
                matched_update = mm_prompt_updates[modality][item_idx][update_idx]
748
                matched_content = matched_update.content.full
749

750
751
752
753
754
755
                if mode == UpdateMode.INSERT:
                    end_idx_to_insert = match.end_idx
                elif mode == UpdateMode.REPLACE:
                    end_idx_to_insert = match.start_idx
                else:
                    assert_never(mode)
756

757
                out_seqs.append(prompt[prev_end_idx:end_idx_to_insert])
758
                out_seqs.append(
759
760
761
762
                    _seq2text(tokenizer, matched_content)
                    if isinstance(prompt, str)
                    else _seq2tokens(tokenizer, matched_content)
                )
763
                out_result[modality][item_idx] = update_idx
764

765
766
767
768
769
                # Exclude overlapping matches
                start_idx = prev_end_idx = match.end_idx

        if not found:
            start_idx += 1
770
771
772

    out_seqs.append(prompt[prev_end_idx:])

773
    return cast(list[_S], out_seqs), out_result
774
775


776
def apply_token_matches(
777
    prompt: list[int],
778
779
780
781
782
    mm_prompt_updates: "MultiModalPromptUpdates",
    tokenizer: AnyTokenizer,
) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
    """
    Apply the updates in `mm_prompt_updates` to `prompt`.
783

784
785
786
787
    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
    appears earlier in `mm_prompt_updates` takes priority.
    """
788
    token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
789

790
    return flatten_2d_lists(token_id_seqs), result
791
792


793
def apply_text_matches(
794
    prompt: str,
795
796
797
798
799
    mm_prompt_updates: "MultiModalPromptUpdates",
    tokenizer: AnyTokenizer,
) -> tuple[str, "MultiModalPromptUpdatesApplyResult"]:
    """
    Apply the updates in `mm_prompt_updates` to `prompt`.
800

801
802
803
804
805
    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
    appears earlier in `mm_prompt_updates` takes priority.
    """
    texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
806

807
    return "".join(texts), result
808
809


810
def _iter_placeholders(
811
    prompt: list[int],
812
    mm_prompt_updates: "MultiModalPromptUpdates",
813
    tokenizer: AnyTokenizer,
814
) -> Iterable[PlaceholderFeaturesInfo]:
815
    """
816
    Yield each set of placeholder tokens found in `prompt`.
817
818
819

    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
820
    appears earlier in `mm_prompt_updates` takes priority.
821

822
823
    Note that empty matches are ignored.
    """
824
    prompt_len = len(prompt)
825
826
    mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()}

827
    item_idx_by_modality = defaultdict[str, int](lambda: 0)
828
829
830
831
832

    start_idx = 0
    while start_idx < prompt_len:
        found = False

833
        for modality, modality_updates in mm_prompt_updates.items():
834
835
            item_idx = item_idx_by_modality[modality]
            if item_idx >= mm_item_counts.get(modality, 0):
836
                continue
837

838
839
            for update in modality_updates[item_idx]:
                content = update.content
840
                content_tokens_full = _seq2tokens(tokenizer, content.full)
841
842
                content_len_full = len(content_tokens_full)
                end_idx_full = start_idx + content_len_full
843

844
                if content_len_full == 0 or end_idx_full > prompt_len:
845
846
                    continue

847
                if prompt[start_idx:end_idx_full] == content_tokens_full:
848
849
                    content_is_embed = content.is_embed
                    if content_is_embed is not None:
850
                        content_is_embed = content_is_embed(tokenizer, content.full)
851
852
853
854
855
856
857
858

                    yield PlaceholderFeaturesInfo(
                        modality=modality,
                        item_idx=item_idx,
                        start_idx=start_idx,
                        tokens=content_tokens_full,
                        is_embed=content_is_embed,
                    )
859

860
                    # Exclude overlapping matches
861
                    start_idx = end_idx_full
862
863
864
                    item_idx_by_modality[modality] += 1
                    found = True
                    break
865

866
867
            if found:
                break  # Go back to the outer while loop
868
869
870

        if not found:
            start_idx += 1
871
872


873
874
def find_mm_placeholders(
    prompt: list[int],
875
    mm_prompt_updates: "MultiModalPromptUpdates",
876
    tokenizer: AnyTokenizer,
877
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
878
    it = _iter_placeholders(prompt, mm_prompt_updates, tokenizer)
879
880
881
    return dict(full_groupby_modality(it))


882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
_T = TypeVar("_T")
_C = TypeVar("_C", bound="PretrainedConfig", default="PretrainedConfig")
_P = TypeVar("_P", bound="ProcessorMixin", default="ProcessorMixin")


@dataclass(frozen=True)
class InputProcessingContext:
    """
    Contains information about the model which may be used to
    modify the inputs.
    """

    model_config: "ModelConfig"
    """The configuration of the model."""

    tokenizer: AnyTokenizer
    """The tokenizer used to tokenize the inputs."""

    @overload
901
    def get_hf_config(self, /) -> "PretrainedConfig": ...
902
903
904
905
906
907

    @overload
    def get_hf_config(
        self,
        typ: Union[type[_C], tuple[type[_C], ...]],
        /,
908
    ) -> _C: ...
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929

    def get_hf_config(
        self,
        typ: Optional[Union[type[Any], tuple[type[Any], ...]]] = None,
        /,
    ) -> Any:
        """
        Get the HuggingFace configuration
        (`transformers.PretrainedConfig`) of the model,
        additionally checking its type.

        Raises:
            TypeError: If the configuration is not of the specified type.
        """
        if typ is None:
            from transformers.configuration_utils import PretrainedConfig

            typ = PretrainedConfig

        hf_config = self.model_config.hf_config
        if not isinstance(hf_config, typ):
930
931
932
933
934
            raise TypeError(
                "Invalid type of HuggingFace config. "
                f"Expected type: {typ}, but "
                f"found type: {type(hf_config)}"
            )
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957

        return hf_config

    def get_hf_image_processor_config(self) -> dict[str, Any]:
        """
        Get the HuggingFace image processor configuration of the model.
        """
        return self.model_config.hf_image_processor_config

    def get_mm_config(self):
        """
        Get the multimodal config of the model.

        Raises:
            RuntimeError: If the model is not a multimodal model.
        """
        mm_config = self.model_config.multimodal_config
        if mm_config is None:
            raise RuntimeError("Not a multimodal model")

        return mm_config

    @overload
958
    def get_hf_processor(self, /, **kwargs: object) -> "ProcessorMixin": ...
959
960
961
962
963
964
965

    @overload
    def get_hf_processor(
        self,
        typ: Union[type[_P], tuple[type[_P], ...]],
        /,
        **kwargs: object,
966
    ) -> _P: ...
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052

    def get_hf_processor(
        self,
        typ: Optional[Union[type[Any], tuple[type[Any], ...]]] = None,
        /,
        **kwargs: object,
    ) -> Any:
        """
        Get the HuggingFace processor
        (`transformers.ProcessorMixin`) of the model,
        additionally checking its type.

        Raises:
            TypeError: If the processor is not of the specified type.
        """
        if typ is None:
            from transformers.processing_utils import ProcessorMixin

            typ = ProcessorMixin

        return cached_processor_from_config(
            self.model_config,
            processor_cls=typ,
            tokenizer=self.tokenizer,
            **kwargs,
        )

    def init_processor(
        self,
        typ: type[_T],
        /,
        **kwargs: object,
    ) -> _T:
        """
        Initialize a HuggingFace-like processor class, merging the
        keyword arguments with those in the model's configuration.
        """
        mm_config = self.model_config.get_multimodal_config()
        base_kwargs = mm_config.mm_processor_kwargs
        if base_kwargs is None:
            base_kwargs = {}

        merged_kwargs = {**base_kwargs, **kwargs}

        return typ(**merged_kwargs)

    def _postprocess_output(
        self,
        output: JSONTree,
    ) -> JSONTree:
        def _postprocess_one(x: object):
            if isinstance(x, torch.Tensor):  # noqa: SIM102
                # This mimics the behavior of transformers.BatchFeature
                if x.is_floating_point():
                    x = x.to(dtype=self.model_config.dtype)

            return x

        return json_map_leaves(_postprocess_one, output)

    def call_hf_processor(
        self,
        hf_processor: "ProcessorMixin",
        data: Mapping[str, object],
        kwargs: Mapping[str, object] = {},
        *,
        num_tries: int = 1,
        max_tries: int = 5,
    ) -> Union["BatchFeature", JSONTree]:
        """
        Call `hf_processor` on the prompt `data`
        (text, image, audio...) with configurable options `kwargs`.
        """
        assert callable(hf_processor)

        mm_config = self.model_config.get_multimodal_config()
        merged_kwargs = mm_config.merge_mm_processor_kwargs(kwargs)

        allowed_kwargs = get_allowed_kwarg_only_overrides(
            hf_processor,
            merged_kwargs,
            requires_kw_only=False,
            allow_var_kwargs=True,
        )

        try:
1053
            output = hf_processor(**data, **allowed_kwargs, return_tensors="pt")
1054
1055
        except Exception as exc:
            # See https://github.com/huggingface/tokenizers/issues/537
1056
1057
1058
1059
1060
1061
            if (
                isinstance(exc, RuntimeError)
                and exc
                and exc.args[0] == "Already borrowed"
                and num_tries < max_tries
            ):
1062
1063
                logger.warning(
                    "Failed to acquire tokenizer in current thread. "
1064
1065
1066
1067
                    "Retrying (%d/%d)...",
                    num_tries,
                    max_tries,
                )
1068
1069
1070
1071
1072
1073
1074
1075
1076
                time.sleep(0.5)
                return self.call_hf_processor(
                    hf_processor,
                    data,
                    kwargs,
                    num_tries=num_tries + 1,
                    max_tries=max_tries,
                )

1077
1078
1079
1080
            msg = (
                f"Failed to apply {type(hf_processor).__name__} "
                f"on data={data} with kwargs={allowed_kwargs}"
            )
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100

            raise ValueError(msg) from exc

        # this emulates output.to(dtype=self.model_config.dtype)
        from transformers.feature_extraction_utils import BatchFeature

        if isinstance(output, BatchFeature):
            output_ = self._postprocess_output(output.data)
            return BatchFeature(output_)

        logger.warning_once(
            "%s did not return `BatchFeature`. "
            "Make sure to match the behaviour of `ProcessorMixin` when "
            "implementing custom processors.",
            type(hf_processor).__name__,
        )

        return self._postprocess_output(output)


1101
class BaseProcessingInfo:
1102
    """Base class to provide the information necessary for data processing."""
1103

1104
1105
    def __init__(self, ctx: InputProcessingContext) -> None:
        super().__init__()
1106

1107
1108
1109
1110
1111
1112
1113
        self.ctx = ctx

    @property
    def model_id(self) -> str:
        return self.ctx.model_config.model

    def get_tokenizer(self) -> AnyTokenizer:
1114
1115
        return self.ctx.tokenizer

1116
    def get_hf_config(self) -> "PretrainedConfig":
1117
1118
        return self.ctx.get_hf_config()

1119
    def get_hf_processor(self, **kwargs: object) -> "ProcessorMixin":
1120
1121
1122
1123
1124
1125
        """
        Subclasses can override this method to handle
        specific kwargs from model config or user inputs.
        """
        return self.ctx.get_hf_processor(**kwargs)

1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
    @abstractmethod
    def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
        """
        Return the maximum supported number of items for each modality.

        A value of `None` means unlimited number of items.

        Omitting a modality from the returned dictionary means that
        it is not supported at all.
        """
        raise NotImplementedError

1138
1139
1140
1141
1142
1143
1144
1145
1146
    def get_allowed_mm_limits(self) -> Mapping[str, int]:
        """Return the maximum allowed number of items for each modality."""
        supported_mm_limits = self.get_supported_mm_limits()
        mm_config = self.ctx.get_mm_config()

        allowed_limits = dict[str, int]()
        for modality, supported_limit in supported_mm_limits.items():
            user_limit = mm_config.get_limit_per_prompt(modality)

1147
1148
1149
1150
1151
            allowed_limits[modality] = (
                user_limit
                if supported_limit is None
                else min(user_limit, supported_limit)
            )
1152
1153
1154

        return allowed_limits

1155
1156
1157
1158
1159
1160
1161
    def get_mm_max_tokens_per_item(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
    ) -> Optional[Mapping[str, int]]:
        """
        Return the maximum number of tokens per item of for each modality.
1162

1163
1164
1165
1166
        When `None` (the default) is returned, vLLM will generate dummy inputs
        (images/videos) at maximum possible sizes and process them to determine
        the maximum token count per modality.

1167
1168
1169
1170
1171
        This approach works but can be very slow for certain models (e.g.,
        Qwen2.5-VL), leading to very long startup time. For better performance,
        each model can override this method to return pre-computed maximum token
        counts, avoiding the need for dummy input generation and processing.

1172
        Note:
1173
            The maximum number of tokens per item of each modality returned
1174
1175
1176
1177
            from this function should respect the model's maximum sequence
            length and the maximum number of items of each modality allowed,
            and agree with dummy inputs (images/videos) at maximum possible
            sizes.
1178
1179
1180
        """
        return None

1181
1182

_I = TypeVar("_I", bound=BaseProcessingInfo)
1183

1184
1185
MultiModalHashes = dict[str, list[str]]
"""
1186
A collection of hashes with a similar structure as
1187
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
1188
1189
"""

1190
MultiModalPromptUpdates = Mapping[str, list[Sequence[ResolvedPromptUpdate]]]
1191
1192
1193
1194
1195
"""
A collection of prompt updates with a similar structure as
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
"""

1196
1197
1198
1199
1200
1201
1202
1203
MultiModalPromptUpdatesApplyResult = Mapping[str, list[Optional[int]]]
"""
For an item `MultiModalPromptUpdates[k][i]`,
`MultiModalPromptUpdatesApplyResult[k][i]` represents the index of the
`ResolvedPromptUpdate` instance that has been applied, or `None` if none of the
`ResolvedPromptUpdate` instances have been applied.
"""

1204
1205

class MultiModalProcessingInfo(NamedTuple):
1206
    kwargs: MultiModalKwargsOptionalItems
1207
    hashes: MultiModalHashes
1208
1209
    prompt_updates: MultiModalPromptUpdates

1210
1211

class BaseMultiModalProcessor(ABC, Generic[_I]):
1212
    """
1213
    Abstract base class to process multi-modal inputs to be used in vLLM.
1214

1215
    Not to be confused with `transformers.ProcessorMixin`.
1216
1217
    """

1218
1219
1220
1221
1222
1223
1224
    def __init__(
        self,
        info: _I,
        dummy_inputs: "BaseDummyInputsBuilder[_I]",
        *,
        cache: Optional["BaseMultiModalProcessorCache"] = None,
    ) -> None:
1225
1226
        super().__init__()

1227
1228
        self.info = info
        self.dummy_inputs = dummy_inputs
1229
        self.cache = cache
1230

1231
1232
        self.data_parser = self._get_data_parser()

1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
        # Avoid unnecessary recomputation
        self._supported_mm_limits = self.info.get_supported_mm_limits()
        self._allowed_mm_limits = self.info.get_allowed_mm_limits()

    @property
    def supported_mm_limits(self):
        return self._supported_mm_limits

    @property
    def allowed_mm_limits(self):
        return self._allowed_mm_limits

1245
    def __call__(
1246
        self,
1247
1248
        prompt: str,
        mm_data: MultiModalDataDict,
1249
        hf_processor_mm_kwargs: Mapping[str, object],
1250
        *,
1251
        mm_uuids: Optional[MultiModalUUIDDict] = None,
1252
    ) -> MultiModalInputs:
1253
        return self.apply(prompt, mm_data, hf_processor_mm_kwargs, mm_uuids=mm_uuids)
1254

1255
1256
    def _get_data_parser(self) -> MultiModalDataParser:
        """
1257
        Construct a parser to preprocess multi-modal data items
1258
1259
        before passing them to
        [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
1260
1261

        You can support additional modalities by creating a subclass
1262
1263
        of [`MultiModalDataParser`][vllm.multimodal.parse.MultiModalDataParser]
        that has additional subparsers.
1264
1265
1266
        """
        return MultiModalDataParser()

1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
    def validate_num_items(
        self,
        modality: str,
        num_items: int,
    ) -> None:
        supported_limit = self.supported_mm_limits.get(modality, 0)
        allowed_limit = self.allowed_mm_limits.get(modality, 0)

        if supported_limit is None:
            supported_limit = allowed_limit

        limit = min(supported_limit, allowed_limit)

        if num_items > limit:
1281
            msg = f"At most {limit} {modality}(s) may be provided in one prompt."
1282
1283
1284
1285
1286
1287

            if num_items <= supported_limit:
                msg += " Set `--limit-mm-per-prompt` to increase this limit."

            raise ValueError(msg)

1288
    def _to_mm_items(
1289
1290
1291
        self,
        mm_data: MultiModalDataDict,
    ) -> MultiModalDataItems:
1292
        """
1293
1294
1295
1296
1297
        Normalize
        [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict]
        to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]
        before passing them to
        [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
1298
        """
1299
        mm_items = self.data_parser.parse_mm_data(mm_data)
1300
        for modality, items in mm_items.items():
1301
            self.validate_num_items(modality, len(items))
1302
1303

        return mm_items
1304

1305
1306
1307
    @abstractmethod
    def _get_mm_fields_config(
        self,
1308
        hf_inputs: "BatchFeature",
1309
1310
1311
1312
1313
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, MultiModalFieldConfig]:
        """Given the HF-processed data, output the metadata of each field."""
        raise NotImplementedError

1314
    @abstractmethod
1315
    def _get_prompt_updates(
1316
        self,
1317
        mm_items: MultiModalDataItems,
1318
        hf_processor_mm_kwargs: Mapping[str, object],
1319
        out_mm_kwargs: MultiModalKwargsItems,
1320
    ) -> Sequence[PromptUpdate]:
1321
1322
        """
        Given the original multi-modal items for this modality
1323
        and HF-processed data, output the updates to perform.
1324

1325
1326
1327
1328
1329
1330
        The information returned by this method is used to update token inputs
        which bypass the HF processor. It is also used to update the output of
        HF processor if the HF process does not apply prompt updates to text
        inputs.

        Moreover, this information is critical to determine the token positions
1331
1332
        in order to construct
        [`PlaceholderRange`][vllm.multimodal.inputs.PlaceholderRange]
1333
        for each multi-modal item.
1334
1335
        """
        raise NotImplementedError
1336

1337
1338
1339
1340
1341
1342
    def _bind_and_group_updates(
        self,
        prompt_updates: Sequence[PromptUpdate],
        mm_item_counts: Mapping[str, int],
    ) -> MultiModalPromptUpdates:
        return {
1343
1344
1345
1346
            modality: [
                [update.resolve(item_idx) for update in updates]
                for item_idx in range(mm_item_counts.get(modality, 0))
            ]
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
            for modality, updates in full_groupby_modality(prompt_updates)
        }

    def _get_mm_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        out_mm_kwargs: MultiModalKwargsItems,
    ) -> MultiModalPromptUpdates:
        unbound_prompt_updates = self._get_prompt_updates(
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
            out_mm_kwargs=out_mm_kwargs,
        )

        mm_prompt_updates = self._bind_and_group_updates(
            unbound_prompt_updates,
            mm_items.get_all_counts(),
        )

        for modality, prompt_updates in mm_prompt_updates.items():
            for item_idx, item_prompt_updates in enumerate(prompt_updates):
                if len(item_prompt_updates) > 1:
                    logger.warning_once(
                        "Detected %d prompt updates for `mm_items[%r][%s]`. "
                        "Multiple prompt updates per item is now "
                        "deprecated and may be removed in v0.13. "
                        "Instead, please specify dynamic update targets "
                        "in the same prompt update definition by passing "
                        "a function to `PromptUpdate.target`.",
                        len(prompt_updates),
                        modality,
                        item_idx,
                    )

        return mm_prompt_updates

1384
    def _find_mm_placeholders(
1385
1386
        self,
        new_token_ids: list[int],
1387
        mm_prompt_updates: MultiModalPromptUpdates,
1388
    ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
1389
1390
        tokenizer = self.info.get_tokenizer()

1391
        return find_mm_placeholders(new_token_ids, mm_prompt_updates, tokenizer)
1392

1393
    def _get_hf_mm_data(
1394
        self,
1395
        mm_items: MultiModalDataItems,
1396
1397
1398
    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
        processor_data = dict[str, object]()
        passthrough_data = dict[str, object]()
1399

1400
1401
1402
        for items in mm_items.values():
            processor_data.update(items.get_processor_data())
            passthrough_data.update(items.get_passthrough_data())
1403

1404
1405
        return processor_data, passthrough_data

1406
1407
1408
    def _call_hf_processor(
        self,
        prompt: str,
1409
1410
1411
1412
        # Not to be confused with `mm_data` in `self.apply`.
        # This refers to the data to be passed to HF processor.
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
1413
        tok_kwargs: Mapping[str, object],
1414
    ) -> "BatchFeature":
1415
1416
1417
1418
        """
        Call the HF processor on the prompt text and
        associated multi-modal data.
        """
1419
1420
        return self.info.ctx.call_hf_processor(
            self.info.get_hf_processor(**mm_kwargs),
1421
            dict(text=prompt, **mm_data),
1422
            dict(**mm_kwargs, **tok_kwargs),
1423
1424
        )

1425
    def _hf_processor_applies_updates(
1426
1427
1428
1429
        self,
        prompt_text: str,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1430
        tokenization_kwargs: Mapping[str, object],
1431
1432
    ) -> bool:
        """
1433
        Return whether the HF processor applies prompt updates.
1434

1435
1436
        For most HF processors, this should be `True` when multi-modal
        data items are passed, but `False` when multi-modal embeddings
1437
1438
1439
1440
        are passed.
        """
        return not any(
            isinstance(items, (EmbeddingItems, DictEmbeddingItems))
1441
1442
            for items in mm_items.values()
        )
1443

1444
    def _apply_hf_processor_text_mm(
1445
        self,
1446
        prompt_text: str,
1447
        mm_items: MultiModalDataItems,
1448
        hf_processor_mm_kwargs: Mapping[str, object],
1449
        tokenization_kwargs: Mapping[str, object],
1450
    ) -> tuple[list[int], "BatchFeature", bool]:
1451
        """
1452
1453
        Apply the HF processor on the prompt text and multi-modal data
        together.
1454

1455
        In addition, return whether prompt updates have been applied.
1456
1457
1458
1459
1460
1461
1462
        """
        processor_data, passthrough_data = self._get_hf_mm_data(mm_items)

        processed_data = self._call_hf_processor(
            prompt=prompt_text,
            mm_data=processor_data,
            mm_kwargs=hf_processor_mm_kwargs,
1463
            tok_kwargs=tokenization_kwargs,
1464
1465
        )
        processed_data.update(passthrough_data)
1466

1467
        (prompt_ids,) = processed_data.pop("input_ids").tolist()
1468

1469
        is_update_applied = self._hf_processor_applies_updates(
1470
1471
1472
            prompt_text=prompt_text,
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1473
            tokenization_kwargs=tokenization_kwargs,
1474
1475
        )

1476
        return prompt_ids, processed_data, is_update_applied
1477

1478
    def _apply_hf_processor_text_only(
1479
1480
1481
1482
        self,
        prompt_text: str,
        tokenization_kwargs: Mapping[str, object],
    ) -> list[int]:
1483
        """
1484
        Apply the HF processor on the prompt text only.
1485

1486
1487
1488
        Since HF processor requires that text and multi-modal items
        correspond to each other, we create dummy multi-modal items
        to go along with the text.
1489
        """
1490
        prompt_ids, _, _ = self._apply_hf_processor_text_mm(
1491
1492
1493
            prompt_text=prompt_text,
            mm_items=MultiModalDataItems({}),
            hf_processor_mm_kwargs={},
1494
            tokenization_kwargs=tokenization_kwargs,
1495
1496
        )

1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
        return prompt_ids

    def _apply_hf_processor_tokens_only(
        self,
        prompt_tokens: list[int],
    ) -> list[int]:
        """
        Apply the HF processor on the prompt tokens only.

        Most HF processors accept prompt text but not prompt tokens.
        If the HF processor adds or removes tokens that are not related to
        multi-modal data, you should override this method so it is consistent
1509
1510
1511
        with the output of
        [`_apply_hf_processor_text_only`][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_text_only]
        on the
1512
1513
1514
1515
1516
1517
1518
1519
        corresponding text.
        """
        return prompt_tokens

    def _apply_hf_processor_mm_only(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1520
        tokenization_kwargs: Mapping[str, object],
1521
    ) -> "BatchFeature":
1522
1523
1524
1525
1526
        """
        Apply the HF processor on the multi-modal data only.

        Since HF processor requires that text and multi-modal items
        correspond to each other, we generate dummy text using
1527
1528
        [`DummyInputsBuilder`][vllm.multimodal.profiling.BaseDummyInputsBuilder]
        to go along with the multi-modal data.
1529
1530
1531
        """
        mm_counts = mm_items.get_all_counts()

1532
        _, mm_processed_data, _ = self._apply_hf_processor_text_mm(
1533
            prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
1534
1535
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1536
            tokenization_kwargs=tokenization_kwargs,
1537
1538
        )

1539
        return mm_processed_data
1540
1541
1542
1543
1544
1545

    def _apply_hf_processor_main(
        self,
        prompt: Union[str, list[int]],
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1546
        tokenization_kwargs: Mapping[str, object],
1547
        *,
1548
        enable_hf_prompt_update: bool,
1549
    ) -> tuple[list[int], "BatchFeature", bool]:
1550
1551
1552
        """
        Apply the HF processor on the prompt text and multi-modal data.

1553
        In addition, return whether prompt updates have been applied
1554
        (for most HF processors, this should be `True`).
1555

1556
        Note:
1557
            If `enable_hf_prompt_update=False`, we use HF processor
1558
            to perform prompt updates if available; HF processor requires
1559
            that the prompt corresponds to multi-modal items.
1560
1561
        """
        if isinstance(prompt, str):
1562
            if enable_hf_prompt_update:
1563
1564
1565
1566
                return self._apply_hf_processor_text_mm(
                    prompt_text=prompt,
                    mm_items=mm_items,
                    hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1567
                    tokenization_kwargs=tokenization_kwargs,
1568
1569
                )

1570
            prompt_ids = self._apply_hf_processor_text_only(prompt, tokenization_kwargs)
1571
1572
1573
        else:
            prompt_ids = self._apply_hf_processor_tokens_only(prompt)

1574
        mm_processed_data = self._apply_hf_processor_mm_only(
1575
            mm_items=mm_items,
1576
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1577
            tokenization_kwargs=tokenization_kwargs,
1578
1579
        )

1580
        return prompt_ids, mm_processed_data, False
1581

1582
    def _hash_mm_items(
1583
1584
1585
1586
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        tokenization_kwargs: Mapping[str, object],
1587
        *,
1588
        mm_uuids: Optional[MultiModalUUIDDict] = None,
1589
    ) -> MultiModalHashes:
1590
        """Create MM hashes to be returned.
1591

1592

1593
1594
1595
        Note: When overrides are provided via callers of `apply`,
        `_hash_mm_items` will be bypassed and the overrides will be used.
        """
1596
1597
        model_id = self.info.model_id

1598
        hashes: MultiModalHashes = {}
1599
        mm_uuids = mm_uuids or {}
1600
1601

        for modality, items in mm_items.items():
1602
1603
1604
1605
            if modality in mm_uuids:
                mm_uuids_per_modality = mm_uuids[modality]
                if isinstance(mm_uuids_per_modality, str):
                    mm_uuids_per_modality = [mm_uuids_per_modality]
1606
1607
1608
1609

                # For None entries, compute a hash; otherwise, use provided ID.
                computed: list[str] = []
                for i, item in enumerate(items):
1610
                    item_uuid = mm_uuids_per_modality[i]
1611

1612
                    # NOTE: Even if a item_uuid is provided, we still compute a
1613
1614
1615
                    # hash if `hf_processor_mm_kwargs` or `tokenization_kwargs`
                    # are provided. This is because the processed multimodal
                    # inputs can be different depending on the processor kwargs.
1616
1617
1618
1619
1620
                    if (
                        item_uuid is None
                        or hf_processor_mm_kwargs
                        or tokenization_kwargs
                    ):
1621
1622
                        # NOTE: use provided hash string to hash with kwargs
                        # if available for better performance.
1623
                        item = item_uuid if item_uuid is not None else item
1624
1625
1626
1627
1628
                        computed.append(
                            MultiModalHasher.hash_kwargs(
                                model_id=model_id,
                                **{modality: item},
                                **hf_processor_mm_kwargs,
1629
1630
1631
                                **tokenization_kwargs,
                            )
                        )
1632
                    else:
1633
                        computed.append(item_uuid)
1634
1635
1636
                hashes[modality] = computed
            else:
                hashes[modality] = [
1637
1638
1639
1640
1641
1642
                    MultiModalHasher.hash_kwargs(
                        model_id=model_id,
                        **{modality: item},
                        **hf_processor_mm_kwargs,
                        **tokenization_kwargs,
                    )
1643
1644
1645
1646
                    for item in items
                ]

        return hashes
1647

1648
1649
1650
1651
1652
1653
1654
    def _get_cache_missing_items(
        self,
        cache: "BaseMultiModalProcessorCache",
        mm_data_items: MultiModalDataItems,
        mm_hashes: MultiModalHashes,
    ) -> MultiModalDataItems:
        mm_is_cached = {
1655
            modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
1656
1657
1658
1659
        }

        mm_missing_idxs = {
            modality: [
1660
1661
                idx
                for idx, item_is_cached in enumerate(items_is_cached)
1662
1663
1664
1665
                if not item_is_cached
            ]
            for modality, items_is_cached in mm_is_cached.items()
        }
1666
1667
1668
1669
1670
1671
1672
1673
        mm_missing_data = {}
        for modality, idxs in mm_missing_idxs.items():
            missing_modality_data = []
            for idx in idxs:
                data = mm_data_items[modality][idx]
                if data is None:
                    raise ValueError(
                        f"Cache miss for {modality} at index {idx} "
1674
1675
                        f"but data is not provided."
                    )
1676
1677
1678
                else:
                    missing_modality_data.append(data)
            mm_missing_data[modality] = missing_modality_data
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692

        return self._to_mm_items(mm_missing_data)

    def _recompute_cached_prompt_update(
        self,
        cached_update: ResolvedPromptUpdate,
        new_item_idx: int,
    ) -> ResolvedPromptUpdate:
        """
        Override this if other attributes of `ResolvedPromptUpdate`
        also need to be recomputed after retrieving from the cache.
        """
        return replace(cached_update, item_idx=new_item_idx)

1693
1694
    def _merge_mm_kwargs(
        self,
1695
1696
        cache: "BaseMultiModalProcessorCache",
        mm_hashes: MultiModalHashes,
1697
        mm_missing_kwargs: MultiModalKwargsItems,
1698
1699
1700
1701
1702
        mm_missing_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[MultiModalKwargsOptionalItems, MultiModalPromptUpdates]:
        # Need to calculate this at the beginning to avoid skipping cache logic
        # for subsequently repeated items in the same modality
        mm_is_cached = {
1703
            modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
1704
1705
        }

1706
        mm_missing_next_idx = defaultdict[str, int](lambda: 0)
1707

1708
1709
1710
1711
        merged_kwargs = defaultdict[str, list[Optional[MultiModalKwargsItem]]](list)
        merged_prompt_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](
            list
        )
1712
1713
        for modality, hashes in mm_hashes.items():
            missing_kwargs = mm_missing_kwargs.get(modality, [])
1714
            missing_prompt_updates = mm_missing_prompt_updates.get(modality, [])
1715
1716
1717
1718
1719
1720
1721
1722

            for item_idx, item_hash in enumerate(hashes):
                kwargs: Optional[MultiModalKwargsItem]
                if not mm_is_cached[modality][item_idx]:
                    missing_next_idx = mm_missing_next_idx[modality]
                    kwargs = missing_kwargs[missing_next_idx]
                    updates = missing_prompt_updates[missing_next_idx]

1723
                    mm_missing_next_idx[modality] += 1
1724
1725

                    item = kwargs, updates
1726
                else:
1727
1728
1729
1730
1731
                    item = None

                kwargs, updates = cache.get_and_update_item(item, item_hash)

                merged_kwargs[modality].append(kwargs)
1732
1733
1734
1735
1736
1737
                merged_prompt_updates[modality].append(
                    [
                        self._recompute_cached_prompt_update(update, item_idx)
                        for update in updates
                    ]
                )
1738

1739
1740
        mm_kwargs = MultiModalKwargsItems(merged_kwargs)
        mm_prompt_updates = dict(merged_prompt_updates)
1741

1742
        return mm_kwargs, mm_prompt_updates
1743
1744
1745
1746
1747
1748

    def _apply_hf_processor(
        self,
        prompt: Union[str, list[int]],
        mm_data_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1749
        tokenization_kwargs: Mapping[str, object],
1750
        *,
1751
        mm_uuids: Optional[MultiModalUUIDDict] = None,
1752
    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1753
1754
        (
            prompt_ids,
1755
            mm_processed_data,
1756
1757
1758
1759
1760
            is_update_applied,
        ) = self._apply_hf_processor_main(
            prompt=prompt,
            mm_items=mm_data_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1761
            tokenization_kwargs=tokenization_kwargs,
1762
1763
1764
            enable_hf_prompt_update=True,
        )

1765
        mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
1766
            mm_processed_data,
1767
            self._get_mm_fields_config(mm_processed_data, hf_processor_mm_kwargs),
1768
1769
        )

1770
        # Use overrides if provided; fallback to data-dependent hashing.
1771
1772
1773
1774
1775
1776
        mm_hashes = self._hash_mm_items(
            mm_data_items,
            hf_processor_mm_kwargs,
            tokenization_kwargs,
            mm_uuids=mm_uuids,
        )
1777

1778
        mm_prompt_updates = self._get_mm_prompt_updates(
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
            mm_data_items,
            hf_processor_mm_kwargs,
            mm_kwargs,
        )

        mm_info = MultiModalProcessingInfo(
            kwargs=mm_kwargs,
            hashes=mm_hashes,
            prompt_updates=mm_prompt_updates,
        )

        return prompt_ids, mm_info, is_update_applied
1791

1792
1793
    def _cached_apply_hf_processor(
        self,
1794
        prompt: Union[str, list[int]],
1795
1796
        mm_data_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1797
        tokenization_kwargs: Mapping[str, object],
1798
        *,
1799
        mm_uuids: Optional[MultiModalUUIDDict] = None,
1800
    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1801
1802
1803
1804
1805
1806
        """
        Apply the HF processor on the full prompt text,
        caching the results and reusing cached results.
        """
        cache = self.cache

1807
1808
        _, passthrough_data = self._get_hf_mm_data(mm_data_items)
        if cache is None or passthrough_data:
1809
            return self._apply_hf_processor(
1810
                prompt=prompt,
1811
                mm_data_items=mm_data_items,
1812
                hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1813
                tokenization_kwargs=tokenization_kwargs,
1814
                mm_uuids=mm_uuids,
1815
1816
            )

1817
1818
1819
1820
1821
1822
        mm_hashes = self._hash_mm_items(
            mm_data_items,
            hf_processor_mm_kwargs,
            tokenization_kwargs,
            mm_uuids=mm_uuids,
        )
1823
1824

        mm_missing_data_items = self._get_cache_missing_items(
1825
1826
            cache=cache,
            mm_data_items=mm_data_items,
1827
            mm_hashes=mm_hashes,
1828
        )
1829

1830
        # NOTE: `prompt` does not correspond to `mm_missing_data_items`,
1831
        # so we can't apply prompt updates until the new multimodal
1832
1833
1834
        # items are combined with the cached multimodal items
        (
            prompt_ids,
1835
            mm_missing_processed_data,
1836
            is_update_applied,
1837
        ) = self._apply_hf_processor_main(
1838
            prompt=prompt,
1839
            mm_items=mm_missing_data_items,
1840
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1841
            tokenization_kwargs=tokenization_kwargs,
1842
            enable_hf_prompt_update=False,
1843
1844
        )

1845
        mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
1846
            mm_missing_processed_data,
1847
1848
1849
            self._get_mm_fields_config(
                mm_missing_processed_data, hf_processor_mm_kwargs
            ),
1850
1851
        )

1852
1853
1854
1855
        mm_missing_prompt_updates = self._get_mm_prompt_updates(
            mm_missing_data_items,
            hf_processor_mm_kwargs,
            mm_missing_kwargs,
1856
        )
1857

1858
1859
1860
1861
1862
        mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
            cache,
            mm_hashes=mm_hashes,
            mm_missing_kwargs=mm_missing_kwargs,
            mm_missing_prompt_updates=mm_missing_prompt_updates,
1863
1864
1865
1866
        )

        mm_info = MultiModalProcessingInfo(
            kwargs=mm_kwargs,
1867
            hashes=mm_hashes,
1868
1869
            prompt_updates=mm_prompt_updates,
        )
1870

1871
        return prompt_ids, mm_info, is_update_applied
1872

1873
1874
1875
    def _apply_token_matches(
        self,
        prompt: list[int],
1876
1877
1878
1879
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
        tokenizer = self.info.get_tokenizer()
        return apply_token_matches(prompt, mm_prompt_updates, tokenizer)
1880
1881
1882
1883

    def _apply_text_matches(
        self,
        prompt: str,
1884
1885
1886
1887
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[str, MultiModalPromptUpdatesApplyResult]:
        tokenizer = self.info.get_tokenizer()
        return apply_text_matches(prompt, mm_prompt_updates, tokenizer)
1888

1889
    def _apply_prompt_updates(
1890
1891
        self,
        token_ids: list[int],
1892
        mm_prompt_updates: MultiModalPromptUpdates,
1893
    ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
1894
        tokenizer = self.info.get_tokenizer()
1895

1896
1897
1898
1899
        new_token_ids, match_result = self._apply_token_matches(
            token_ids,
            mm_prompt_updates,
        )
1900
1901
1902
1903
1904
1905
1906
1907
1908

        # If the search text does not represent a special token,
        # it may have different token IDs in the prompt, because
        # the tokens may go across the boundaries of the search text.
        # ----
        # e.g. when searching for "foo" in "food", if "food" itself makes
        # up a token, then the token ID of "foo" will not appear at all
        # ----
        # Since it is inefficient to search for all possible tokenizations
1909
1910
        # of the search text in the prompt, we instead perform string-based
        # updates on the decoded token IDs, then encode them back.
1911
        if not all(
1912
1913
1914
            all(update_idx is not None for update_idx in update_idxs)
            for update_idxs in match_result.values()
        ):
1915
1916
1917
            new_text, match_result = self._apply_text_matches(
                decode_tokens(tokenizer, token_ids),
                mm_prompt_updates,
1918
1919
            )

1920
1921
1922
1923
            new_token_ids = encode_tokens(
                tokenizer,
                new_text,
                add_special_tokens=False,
1924
1925
            )

1926
        matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
1927
1928
1929
1930
        for modality, update_idxs in match_result.items():
            for item_idx, update_idx in enumerate(update_idxs):
                assert update_idx is not None, (
                    "Failed to apply prompt replacement for "
1931
1932
                    f"mm_items[{modality!r}][{item_idx}]"
                )
1933
1934

                matched_updates[modality].append(
1935
1936
                    [mm_prompt_updates[modality][item_idx][update_idx]]
                )
1937
1938

        placeholders = self._find_mm_placeholders(
1939
1940
            new_token_ids,
            dict(matched_updates),
1941
        )
1942

1943
        return new_token_ids, placeholders
1944

1945
1946
    def _validate_mm_kwargs(
        self,
1947
        mm_kwargs: MultiModalKwargsOptionalItems,
1948
1949
1950
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
1951
            items = mm_kwargs.get(modality, [])
1952
1953
1954
1955
1956
1957
1958
1959
1960

            if len(items) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} {modality} items in "
                    f"keyword arguments corresponding to {item_count} "
                    f"{modality} data items, but only found {len(items)}! "
                    "There is likely a problem with your "
                    "implementation of merged multi-modal processor for this "
                    "model (usually arising from an inconsistency between "
1961
1962
                    "`_call_hf_processor` and `_get_mm_fields_config`)."
                )
1963

1964
    def _validate_mm_updates(
1965
        self,
1966
        mm_updates: MultiModalPromptUpdates,
1967
        mm_item_counts: Mapping[str, int],
1968
    ) -> None:
1969
        for modality, item_count in mm_item_counts.items():
1970
            placeholders = mm_updates.get(modality, [])
1971

1972
            if len(placeholders) != item_count:
1973
                raise RuntimeError(
1974
                    f"Expected there to be {item_count} prompt updates "
1975
                    f"corresponding to {item_count} {modality} items, but "
1976
                    f"instead found {len(placeholders)} prompt updates! "
1977
1978
1979
                    "This is likely because you forgot to include input "
                    "placeholder tokens (e.g., `<image>`, `<|image_pad|>`) "
                    "in the prompt. If the model has a chat template, make "
1980
1981
                    "sure you have applied it before calling `LLM.generate`."
                )
1982

1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
    def _validate_mm_placeholders(
        self,
        mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
            placeholders = mm_placeholders.get(modality, [])

            if len(placeholders) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} prompt placeholders "
                    f"corresponding to {item_count} {modality} items, but "
                    f"instead found {len(placeholders)} prompt placeholders! "
                    "Make sure the implementation of `_call_hf_processor` and "
1997
1998
                    "`_get_mm_fields_config` are consistent with each other."
                )
1999

2000
2001
2002
2003
    def _maybe_apply_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        prompt_ids: list[int],
2004
        mm_kwargs: MultiModalKwargsOptionalItems,
2005
        mm_prompt_updates: MultiModalPromptUpdates,
2006
        is_update_applied: bool,
2007
    ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
2008
        mm_item_counts = mm_items.get_all_counts()
2009
        self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
2010
        self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
2011

2012
        if is_update_applied:
2013
2014
            mm_placeholders = self._find_mm_placeholders(
                prompt_ids,
2015
                mm_prompt_updates,
2016
            )
2017
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
2018
        else:
2019
            prompt_ids, mm_placeholders = self._apply_prompt_updates(
2020
                prompt_ids,
2021
                mm_prompt_updates,
2022
            )
2023
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
2024

2025
        return prompt_ids, mm_placeholders
2026
2027
2028
2029
2030
2031

    def apply(
        self,
        prompt: Union[str, list[int]],
        mm_data: MultiModalDataDict,
        hf_processor_mm_kwargs: Mapping[str, object],
2032
        tokenization_kwargs: Optional[Mapping[str, object]] = None,
2033
        *,
2034
        mm_uuids: Optional[MultiModalUUIDDict] = None,
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
    ) -> MultiModalInputs:
        """
        Process multi-modal inputs to be used in vLLM.

        The main steps are:

        1. Apply HF Processor on prompt text and multi-modal data together,
           outputting token IDs and processed tensors.
        2. Find and update sequences in the token IDs with placeholder tokens.
           The number of placeholder tokens equals the feature size of the
           multi-modal data outputted by the multi-modal encoder.
        3. Extract information about the placeholder tokens from the
           processed token IDs.
        """
        mm_items = self._to_mm_items(mm_data)

2051
2052
2053
        if tokenization_kwargs is None:
            tokenization_kwargs = {}

2054
2055
        (
            prompt_ids,
2056
            mm_info,
2057
2058
2059
2060
2061
            is_update_applied,
        ) = self._cached_apply_hf_processor(
            prompt,
            mm_items,
            hf_processor_mm_kwargs,
2062
            tokenization_kwargs=tokenization_kwargs,
2063
            mm_uuids=mm_uuids,
2064
2065
        )

2066
        # NOTE: tokenization_kwargs are not required to init processor
2067
        prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
2068
2069
            mm_items=mm_items,
            prompt_ids=prompt_ids,
2070
2071
            mm_kwargs=mm_info.kwargs,
            mm_prompt_updates=mm_info.prompt_updates,
2072
2073
2074
            is_update_applied=is_update_applied,
        )

2075
2076
2077
2078
        mm_placeholder_ranges = {
            modality: [item.to_range() for item in placeholders]
            for modality, placeholders in mm_placeholders.items()
        }
2079

2080
        return MultiModalInputs(
2081
            type="multimodal",
2082
            prompt_token_ids=prompt_ids,
2083
2084
            mm_kwargs=mm_info.kwargs,
            mm_hashes=mm_info.hashes,
2085
            mm_placeholders=mm_placeholder_ranges,
2086
        )
2087
2088
2089
2090
2091
2092
2093
2094
2095


class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
    @abstractmethod
    def create_encoder_prompt(
        self,
        prompt: Union[str, list[int]],
        mm_data: MultiModalDataDict,
    ) -> Union[str, list[int]]:
2096
        """
2097
        Create input prompt for the encoder. HF processor will be applied on
2098
2099
        this prompt during profiling and generation.
        """
2100
2101
        raise NotImplementedError

2102
2103
2104
2105
    @property
    def pad_dummy_encoder_prompt(self) -> bool:
        return False

2106
2107
2108
2109
2110
2111
2112
2113
    def create_decoder_prompt(
        self,
        prompt: Union[str, list[int]],
        mm_data: MultiModalDataDict,
    ) -> Union[str, list[int]]:
        """Create input prompt for the decoder."""
        return prompt

2114
    def _get_enc_dec_inputs(
2115
2116
2117
        self,
        prompt: Union[str, list[int]],
        mm_data: MultiModalDataDict,
2118
2119
        encoder_inputs: MultiModalInputs,
    ):
2120
        tokenizer = self.info.get_tokenizer()
2121
2122
        decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_data)
        if isinstance(decoder_prompt_raw, str):
2123
2124
2125
            decoder_prompt_ids = encode_tokens(
                tokenizer, decoder_prompt_raw, add_special_tokens=False
            )
2126
        else:
2127
            decoder_prompt_ids = decoder_prompt_raw
2128
2129
2130

        mm_inputs = MultiModalEncDecInputs(
            encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],
2131
2132
            **encoder_inputs,
        )
2133
        mm_inputs["prompt_token_ids"] = decoder_prompt_ids
2134
        return mm_inputs
2135
2136
2137
2138
2139
2140

    def apply(
        self,
        prompt: Union[str, list[int]],
        mm_data: MultiModalDataDict,
        hf_processor_mm_kwargs: Mapping[str, object],
2141
        tokenization_kwargs: Optional[Mapping[str, object]] = None,
2142
        *,
2143
        mm_uuids: Optional[MultiModalUUIDDict] = None,
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
    ) -> MultiModalEncDecInputs:
        """
        Process multi-modal inputs to be used in vLLM.
        The main processing steps are modified to fit encoder-decoder model:
        1. Create encoder prompt from input prompt text.
        2. Apply the HF processor on encoder prompt.
        3. Copy the input prompt text as decoder prompt inputs.
        """
        encoder_prompt = self.create_encoder_prompt(prompt, mm_data)
        encoder_inputs = super().apply(
            encoder_prompt,
            mm_data,
            hf_processor_mm_kwargs,
2157
            tokenization_kwargs,
2158
            mm_uuids=mm_uuids,
2159
2160
2161
2162
2163
2164
2165
        )

        return self._get_enc_dec_inputs(
            prompt=prompt,
            mm_data=mm_data,
            encoder_inputs=encoder_inputs,
        )