processing.py 71.3 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import time
4
from abc import ABC, abstractmethod
5
from collections import defaultdict
6
from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence
7
from dataclasses import dataclass, field, replace
8
from enum import Enum
9
from functools import lru_cache
10
11
12
13
14
15
from typing import (
    TYPE_CHECKING,
    Any,
    Generic,
    NamedTuple,
    Protocol,
16
    TypeAlias,
17
18
19
    cast,
    overload,
)
20

21
import regex as re
22
import torch
23
from typing_extensions import TypeVar, assert_never
24

25
from vllm.logger import init_logger
26
from vllm.tokenizers import TokenizerLike
27
from vllm.transformers_utils.processor import cached_processor_from_config
28
from vllm.transformers_utils.tokenizer import decode_tokens, encode_tokens
29
30
from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
31
from vllm.utils.jsontree import JSONTree, json_map_leaves
32

33
from .hasher import MultiModalHasher
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
from .inputs import (
    MultiModalDataDict,
    MultiModalEncDecInputs,
    MultiModalFieldConfig,
    MultiModalInputs,
    MultiModalKwargsItem,
    MultiModalKwargsItems,
    MultiModalKwargsOptionalItems,
    MultiModalUUIDDict,
    PlaceholderRange,
)
from .parse import (
    DictEmbeddingItems,
    EmbeddingItems,
    MultiModalDataItems,
    MultiModalDataParser,
)
51
52

if TYPE_CHECKING:
53
54
55
56
    from transformers.configuration_utils import PretrainedConfig
    from transformers.feature_extraction_utils import BatchFeature
    from transformers.processing_utils import ProcessorMixin

57
58
    from vllm.config import ModelConfig

59
    from .cache import BaseMultiModalProcessorCache
60
    from .profiling import BaseDummyInputsBuilder
61
62
63
64
65
66
67
68
else:
    PretrainedConfig = object
    BatchFeature = object
    ProcessorMixin = object

    ModelConfig = object

    BaseMultiModalProcessorCache = object
69

70
logger = init_logger(__name__)
71
72

_S = TypeVar("_S", str, list[int])
73

74
PromptSeq: TypeAlias = str | list[int]
75
"""A token sequence (list of token IDs) or text."""
76

77

78
79
@lru_cache(maxsize=2048)
def _cached_encode(
80
    tokenizer: TokenizerLike,
81
82
    text: str,
    *,
83
    add_special_tokens: bool | None = None,
84
) -> list[int]:
85
    return encode_tokens(tokenizer, text, add_special_tokens=add_special_tokens)
86
87
88
89


@lru_cache(maxsize=2048)
def _cached_decode(
90
    tokenizer: TokenizerLike,
91
92
    token_ids: tuple[int, ...],
    *,
93
    skip_special_tokens: bool | None = None,
94
) -> str:
95
96
97
    return decode_tokens(
        tokenizer, list(token_ids), skip_special_tokens=skip_special_tokens
    )
98
99


100
101
102
103
104
105
def _seq2text(
    tokenizer: TokenizerLike | None,
    seq: PromptSeq,
    *,
    use_cache: bool = True,
) -> str:
106
107
108
    if isinstance(seq, str):
        return seq

109
110
111
112
113
114
    if tokenizer is None:
        raise ValueError("You cannot decode tokens when `skip_tokenizer_init=True`")

    if not use_cache:
        return decode_tokens(tokenizer, seq)

115
116
117
    return _cached_decode(tokenizer, tuple(seq))


118
119
120
121
122
123
def _seq2tokens(
    tokenizer: TokenizerLike | None,
    seq: PromptSeq,
    *,
    use_cache: bool = True,
) -> list[int]:
124
    if isinstance(seq, str):
125
126
127
128
129
130
        if tokenizer is None:
            raise ValueError("You cannot encode text when `skip_tokenizer_init=True`")

        if not use_cache:
            return encode_tokens(tokenizer, seq, add_special_tokens=False)

131
132
133
134
135
        return _cached_encode(tokenizer, seq, add_special_tokens=False)

    return seq


136
137
138
class _GetMatchIndex(Protocol):
    def __call__(
        self,
139
        tokenizer: TokenizerLike | None,
140
141
        prompt: PromptSeq,
        start_idx: int = 0,
142
    ) -> int | None: ...
143
144


145
146
147
@dataclass
class PromptIndex:
    """Resolves to an index in the prompt."""
148

149
    get_match_index: _GetMatchIndex
150
151
152
153
154
155
156
157
158
159


class PromptIndexTargets:
    @staticmethod
    def start() -> PromptIndex:
        """
        Resolves to the start of the prompt (before the first token).

        This results in a match even if the prompt is empty.
        """
160
        return PromptIndex(lambda tokenizer, prompt, start_idx=0: 0)
161
162
163
164
165
166
167
168

    @staticmethod
    def prefix(seq: PromptSeq) -> PromptIndex:
        """
        Resolves to a location in the prompt after the given prefix.
        """

        def get_match_index(
169
            tokenizer: TokenizerLike | None,
170
            prompt: PromptSeq,
171
            start_idx: int = 0,
172
        ) -> int | None:
173
174
175
            if start_idx != 0:
                return None

176
177
178
            prefix = seq

            if isinstance(prompt, str):
179
180
                # Make both `str`
                prefix = _seq2text(tokenizer, prefix, use_cache=False)
181
            else:
182
183
                # Make both `list[int]`
                prefix = _seq2tokens(tokenizer, prefix, use_cache=False)
184
185
186
187
188
189
190
191
192
193
194
195
196

            match_idx = len(prefix)
            return match_idx if prompt[:match_idx] == prefix else None

        return PromptIndex(get_match_index)

    @staticmethod
    def end() -> PromptIndex:
        """
        Resolves to the end of the prompt (after the last token).

        This results in a match even if the prompt is empty.
        """
197
        return PromptIndex(lambda tokenizer, prompt, start_idx=0: len(prompt))
198
199


200
UpdateTarget: TypeAlias = PromptSeq | PromptIndex
201
202
203
204
"""
The token sequence or text to update.
"""

205
PromptUpdateTarget: TypeAlias = Callable[[int], UpdateTarget] | UpdateTarget
206
207
208
209
210
211
212
213
214
"""
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
output the corresponding token sequence (or text).

For convenience, you can directly pass in the token sequence (or text)
instead of a function if it does not depend on the input.
"""

215

216
@dataclass
217
class PromptUpdateDetails(Generic[_S]):
218
    """Details about the token sequence or text that are part of the update."""
219

220
    full: _S
221
    """The full content."""
222

223
    is_embed: Callable[[TokenizerLike | None, PromptSeq], torch.Tensor] | None = None
224
    """
225
226
227
    Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full],
    return a boolean mask of shape `(len(full),)` indicating which positions
    of `full` to assign embeddings to.
228
229
230
231

    `None` (default) means to assign embeddings to all positions of `full`.

    The embeddings are obtained by calling
232
    [`SupportsMultiModal.embed_multimodal`][vllm.model_executor.models.interfaces.SupportsMultiModal.embed_multimodal].
233
234
235
    """

    @staticmethod
236
    def from_seq(seq: _S) -> "PromptUpdateDetails[_S]":
237
238
239
240
241
242
243
        return PromptUpdateDetails(full=seq)

    @staticmethod
    def select_text(
        seq: _S,
        embed_text: str,
    ) -> "PromptUpdateDetails[_S]":
244
245
        def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
            embed_token_ids = _seq2tokens(tokenizer, embed_text, use_cache=False)
246
            token_ids = _seq2tokens(tokenizer, full)
247
248

            return torch.isin(
249
                torch.tensor(token_ids),
250
251
252
253
254
255
256
257
258
259
                torch.tensor(embed_token_ids),
            )

        return PromptUpdateDetails(full=seq, is_embed=is_embed)

    @staticmethod
    def select_token_id(
        seq: _S,
        embed_token_id: int,
    ) -> "PromptUpdateDetails[_S]":
260
        def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
261
262
263
264
265
            token_ids = _seq2tokens(tokenizer, full)

            return torch.tensor(token_ids) == embed_token_id

        return PromptUpdateDetails(full=seq, is_embed=is_embed)
266
267


268
PromptUpdateInfo: TypeAlias = PromptSeq | PromptUpdateDetails
269
"""
270
The token sequence or text that are part of the update.
271

272
If only part of the content corresponds to feature placeholders, you can
273
274
use [`PromptUpdateDetails`][vllm.multimodal.processing.PromptUpdateDetails] to
specify which part.
275
"""
276

277
PromptUpdateContent: TypeAlias = Callable[[int], PromptUpdateInfo] | PromptUpdateInfo
278
"""
279
280
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
281
282
283
284
285
286
287
288
289
290
291
292
293
output the corresponding token sequence (or text).

For convenience, you can directly pass in the token sequence (or text)
instead of a function if it does not depend on the input.
"""


class UpdateMode(str, Enum):
    INSERT = "insert"
    REPLACE = "replace"


@dataclass
294
class PromptUpdate(ABC):
295
296
297
298
299
300
301
    """
    Defines how to update a prompt with placeholder tokens.
    """

    modality: str
    """The modality for which the update is made."""

302
    target: PromptUpdateTarget
303
304
305
306
307
308
309
310
311
312
313
314
315
316
    """The token sequence (or text) to update."""

    @property
    @abstractmethod
    def content(self) -> PromptUpdateContent:
        """The placeholder tokens that are part of the update."""
        raise NotImplementedError

    @property
    @abstractmethod
    def mode(self) -> UpdateMode:
        """Defines how to update the prompt."""
        raise NotImplementedError

317
    def _resolve_target(self, item_idx: int) -> UpdateTarget:
318
319
320
321
        target = self.target
        if callable(target):
            target = target(item_idx)

322
        return target
323

324
    def _resolve_content(self, item_idx: int) -> PromptUpdateDetails:
325
326
327
328
329
330
331
        content = self.content
        if callable(content):
            content = content(item_idx)

        if not isinstance(content, PromptUpdateDetails):
            content = PromptUpdateDetails.from_seq(content)

332
        return content
333

334
    def resolve(self, item_idx: int) -> "ResolvedPromptUpdate":
335
336
337
338
339
340
341
342
343
        """
        Given the index of the processed item within
        [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
        output a copy of this object with its lazy attributes resolved.
        """
        return ResolvedPromptUpdate(
            modality=self.modality,
            item_idx=item_idx,
            mode=self.mode,
344
345
            target=self._resolve_target(item_idx),
            content=self._resolve_content(item_idx),
346
347
        )

348

349
@dataclass
350
351
352
353
354
355
class PromptInsertion(PromptUpdate):
    """
    Defines how to insert placeholder tokens into a prompt.

    Example:

356
357
    For each image, insert a number of `<image>` feature placeholders
    equal to the feature size of the vision encoder after the `<s>` token:
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376

    ```python
    PromptInsertion(
        modality="image",
        target="<s>",
        insertion="<image>" * image_feature_size,
    )
    ```

    Insert these tokens at the start of the prompt:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.start(),
        insertion="<image>" * image_feature_size,
    )
    ```

377
    Insert these tokens after a prefix `Images:`:
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.prefix("Images:"),
        insertion="<image>" * image_feature_size,
    )
    ```

    Insert these tokens at the end of the prompt:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.end(),
        insertion="<image>" * image_feature_size,
    )
    ```
396
397
398
399
    """

    insertion: PromptUpdateContent = field(repr=False)
    """
400
401
402
403
    Given the index of the processed item within
    [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
    output the token sequence (or text) to insert right after
    [`target`][vllm.multimodal.processing.PromptUpdate.target].
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419

    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
    """

    @property
    def content(self) -> PromptUpdateContent:
        return self.insertion

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.INSERT


@dataclass
class PromptReplacement(PromptUpdate):
420
421
    """
    Defines how to replace portions of an input prompt with placeholder tokens.
422
423
424

    Example:

425
426
    For each image, replace one `<image>` input placeholder in the prompt
    with a number of `<image>` feature placeholders
427
428
429
430
431
432
433
434
435
436
    equal to the feature size of the vision encoder:

    ```python
    PromptReplacement(
        modality="image",
        target="<image>",
        replacement="<image>" * image_feature_size,
    )
    ```

437
438
    As above, but further pad the feature placeholders with `<image_bos>`
    and `<image_eos>`, which are not supposed to be passed to the vision
439
440
441
442
443
444
445
    encoder:

    ```python
    PromptReplacement(
        modality="image",
        target="<image>",
        replacement=PromptUpdateDetails(
446
447
448
449
450
451
452
            full="".join(
                [
                    "<image_bos>",
                    "<image>" * image_feature_size,
                    "<image_eos>",
                ]
            ),
453
454
455
456
457
458
459
460
461
462
463
464
465
            features="<image>" * image_feature_size,
        ),
    )
    ```

    To avoid unnecessary tokenization during prompt replacement,
    we recommended passing token sequences instead of text:

    ```python
    PromptReplacement(
        modality="image",
        target=[image_token_id],
        replacement=PromptUpdateDetails(
466
467
468
            full=(
                [image_bos_id] + [image_token_id] * image_feature_size + [image_eos_id]
            ),
469
470
471
472
            features=[image_token_id] * image_feature_size,
        ),
    )
    ```
473
474
    """

475
    replacement: PromptUpdateContent = field(repr=False)
476
    """
477
478
479
480
    Given the index of the processed item within
    [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
    output the token sequence (or text) to replace
    [`target`][vllm.multimodal.processing.PromptUpdate.target].
481

482
483
    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
484
485
    """

486
487
488
489
490
491
492
    @property
    def content(self) -> PromptUpdateContent:
        return self.replacement

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.REPLACE
493
494


495
496
497
class _HasModalityAttr(Protocol):
    modality: str

498

499
500
class _HasModalityProp(Protocol):
    @property
501
    def modality(self) -> str: ...
502
503


504
_M = TypeVar("_M", bound=_HasModalityAttr | _HasModalityProp)
505
506
507


def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]:
508
509
    """
    Convenience function to apply
510
    [`full_groupby`][vllm.utils.collection_utils.full_groupby]
511
512
    based on modality.
    """
513
514
515
    return full_groupby(values, key=lambda x: x.modality)


516
517
518
519
520
521
522
class PromptTargetMatch(NamedTuple):
    start_idx: int
    end_idx: int


@dataclass(frozen=True)
class ResolvedPromptUpdate:
523
    """
524
525
    A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] with its
    lazy attributes resolved, apart from those related to tokenization.
526
    """
527

528
529
    modality: str
    """The modality for which the update is made."""
530

531
532
    item_idx: int
    """The index within `modality` of the item this update pertains to."""
533

534
535
    mode: UpdateMode
    """Defines how to update the prompt."""
536

537
    target: UpdateTarget
538
    """The token sequence (or text) to update."""
539

540
    content: PromptUpdateDetails = field(repr=False)
541
    """The placeholder tokens that are part of the update."""
542

543
544
545
    def iter_token_matches(
        self,
        prompt: list[int],
546
        tokenizer: TokenizerLike | None,
547
548
549
550
551
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        target = self.target
552

553
554
555
556
        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(tokenizer, prompt, start_idx)
            if match_idx is not None:
                yield PromptTargetMatch(match_idx, match_idx)
557

558
            return
559

560
561
        target_token_ids = _seq2tokens(tokenizer, target)

562
        for match in iter_token_matches(prompt, target_token_ids, start_idx=start_idx):
563
            yield PromptTargetMatch(match.start_idx, match.end_idx)
564

565
566
567
    def iter_text_matches(
        self,
        prompt: str,
568
        tokenizer: TokenizerLike | None,
569
570
571
572
573
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        target = self.target
574

575
576
577
578
        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(tokenizer, prompt, start_idx)
            if match_idx is not None:
                yield PromptTargetMatch(match_idx, match_idx)
579

580
            return
581

582
583
        target_text = _seq2text(tokenizer, target)

584
        for match in re.finditer(re.escape(target_text), prompt, pos=start_idx):
585
586
587
588
            yield PromptTargetMatch(match.start(), match.end())

    def iter_matches(
        self,
589
        prompt: list[int] | str,
590
        tokenizer: TokenizerLike | None,
591
592
593
594
595
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        if isinstance(prompt, str):
596
            return self.iter_text_matches(prompt, tokenizer, start_idx=start_idx)
597
598

        return self.iter_token_matches(prompt, tokenizer, start_idx=start_idx)
599

600
601
602
603
604
605
606
607
608
    def with_target(self, target: UpdateTarget):
        return replace(self, target=target)

    def with_content(self, content: PromptUpdateInfo):
        if not isinstance(content, PromptUpdateDetails):
            content = PromptUpdateDetails.from_seq(content)

        return replace(self, content=content)

609

610
611
612
class _TokenMatch(NamedTuple):
    start_idx: int
    end_idx: int
613
614


615
616
617
def iter_token_matches(
    token_ids: list[int],
    match_ids: list[int],
618
619
    *,
    start_idx: int = 0,
620
) -> Generator[_TokenMatch]:
621
    """
622
    Yield each occurrence of `match_ids` in `token_ids`.
623
624
625
626

    Note that empty matches are ignored.
    """
    prompt_len = len(token_ids)
627
    match_len = len(match_ids)
628

629
630
    if match_len == 0:
        return
631

632
    while start_idx < prompt_len - match_len + 1:
633
        end_idx = start_idx + match_len
634

635
636
        if token_ids[start_idx:end_idx] == match_ids:
            yield _TokenMatch(start_idx=start_idx, end_idx=end_idx)
637
638
639
640
641

            # Exclude overlapping matches
            start_idx = end_idx
        else:
            start_idx += 1
642
643


644
645
646
647
648
649
def replace_token_matches(
    token_ids: list[int],
    match_ids: list[int],
    new_ids: list[int],
) -> list[int]:
    """
650
651
    Replace each occurrence of `match_ids` in `token_ids`
    with `new_ids`.
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670

    Note that empty matches are ignored.
    """
    out_seqs = list[list[int]]()
    prev_end_idx = 0

    for match in iter_token_matches(token_ids, match_ids):
        start_idx = match.start_idx
        end_idx = match.end_idx

        out_seqs.append(token_ids[prev_end_idx:start_idx])
        out_seqs.append(new_ids)
        prev_end_idx = end_idx

    out_seqs.append(token_ids[prev_end_idx:])

    return flatten_2d_lists(out_seqs)


671
@dataclass
672
class PlaceholderFeaturesInfo:
673
    modality: str
674
    item_idx: int
675
    start_idx: int
676
    tokens: list[int]
677
    is_embed: torch.Tensor | None
678
679
680

    @property
    def length(self) -> int:
681
        return len(self.tokens)
682
683

    def to_range(self) -> PlaceholderRange:
684
685
        # TODO: Is it worth it to optimize this by stripping the
        # leading and ending positions where `is_embed=False`?
686
687
688
        return PlaceholderRange(
            offset=self.start_idx,
            length=self.length,
689
            is_embed=self.is_embed,
690
        )
691
692


693
_MatchToApply = tuple[tuple[str, int], tuple[PromptTargetMatch, int]]
694
695


696
697
698
def _find_matches(
    prompt: _S,
    mm_prompt_updates: "MultiModalPromptUpdates",
699
    tokenizer: TokenizerLike | None,
700
701
702
    *,
    prev_end_idx: int = 0,
    current_result: "MultiModalPromptUpdatesApplyResult",
703
704
) -> tuple[UpdateMode | None, list[_MatchToApply]]:
    mode: UpdateMode | None = None
705
706
707
708
709
710
711
712
713
714
715
716
    mm_matches = dict[tuple[str, int], tuple[PromptTargetMatch, int]]()

    for modality, modality_updates in mm_prompt_updates.items():
        for item_idx, item_updates in enumerate(modality_updates):
            if current_result[modality][item_idx] is not None:
                continue  # Updates have already been applied for this item

            for update_idx, update in enumerate(item_updates):
                if (modality, item_idx) in mm_matches:
                    break  # Already found a match for this item

                for match in update.iter_matches(
717
718
719
                    prompt,
                    tokenizer,
                    start_idx=prev_end_idx,
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
                ):
                    # All matches should share the same mode
                    if mode is None:
                        mode = update.mode
                    elif mode != update.mode:
                        continue

                    mm_matches[(modality, item_idx)] = match, update_idx
                    break  # Get only the first valid match per item

    # Prioritize earlier matches
    matches_to_apply = sorted(mm_matches.items(), key=lambda item: item[1][0])

    # To avoid conflicts, only replace one non-empty item at a time
    if mode == UpdateMode.REPLACE:
        matches_to_apply_ = list[_MatchToApply]()
        has_non_empty_matches = False

        for item in matches_to_apply:
            _, (match, _) = item
            if match.start_idx == match.end_idx:
                matches_to_apply_.append(item)
            elif not has_non_empty_matches:
                has_non_empty_matches = True
                matches_to_apply_.append(item)

        matches_to_apply = matches_to_apply_

    return mode, matches_to_apply
749
750


751
752
753
754
755
756
757
758
759
760
def _all_items_found(
    mm_item_counts: dict[str, int],
    mm_found_counts: dict[str, int],
) -> bool:
    return all(
        item_idx >= mm_item_counts[modality]
        for modality, item_idx in mm_found_counts.items()
    )


761
def _apply_matches(
762
    prompt: _S,
763
    mm_prompt_updates: "MultiModalPromptUpdates",
764
    tokenizer: TokenizerLike | None,
765
) -> tuple[list[_S], "MultiModalPromptUpdatesApplyResult"]:
766
    mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()}
767

768
    out_seqs = list[str | list[int]]()
769
    out_result: MultiModalPromptUpdatesApplyResult = {
770
        m: [None] * len(items) for m, items in mm_prompt_updates.items()
771
    }
772

773
    # Early exit if no items to find
774
775
776
777
778
779
    mm_found_counts = {
        m: sum(r is not None for r in res) for m, res in out_result.items()
    }
    if _all_items_found(mm_item_counts, mm_found_counts):
        return [prompt], out_result

780
781
    prev_end_idx = 0
    while True:
782
783
784
785
786
787
788
        mode, matches_to_apply = _find_matches(
            prompt,
            mm_prompt_updates,
            tokenizer,
            prev_end_idx=prev_end_idx,
            current_result=out_result,
        )
789

790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
        if mode is None:
            break  # No more matches to find

        for (modality, item_idx), (match, update_idx) in matches_to_apply:
            matched_update = mm_prompt_updates[modality][item_idx][update_idx]
            matched_content = matched_update.content.full

            if mode == UpdateMode.INSERT:
                end_idx_to_insert = match.end_idx
            elif mode == UpdateMode.REPLACE:
                end_idx_to_insert = match.start_idx
            else:
                assert_never(mode)

            out_seqs.append(prompt[prev_end_idx:end_idx_to_insert])
            out_seqs.append(
                _seq2text(tokenizer, matched_content)
                if isinstance(prompt, str)
                else _seq2tokens(tokenizer, matched_content)
            )
            out_result[modality][item_idx] = update_idx

            # Exclude overlapping matches
            prev_end_idx = match.end_idx

        # Early exit if all items found
        mm_found_counts = {
            m: sum(r is not None for r in res) for m, res in out_result.items()
        }
        if _all_items_found(mm_item_counts, mm_found_counts):
            break
821
822
823

    out_seqs.append(prompt[prev_end_idx:])

824
    return cast(list[_S], out_seqs), out_result
825
826


827
def apply_token_matches(
828
    prompt: list[int],
829
    mm_prompt_updates: "MultiModalPromptUpdates",
830
    tokenizer: TokenizerLike | None,
831
832
833
) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
    """
    Apply the updates in `mm_prompt_updates` to `prompt`.
834

835
836
837
838
    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
    appears earlier in `mm_prompt_updates` takes priority.
    """
839
    token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
840

841
    return flatten_2d_lists(token_id_seqs), result
842
843


844
def apply_text_matches(
845
    prompt: str,
846
    mm_prompt_updates: "MultiModalPromptUpdates",
847
    tokenizer: TokenizerLike | None,
848
849
850
) -> tuple[str, "MultiModalPromptUpdatesApplyResult"]:
    """
    Apply the updates in `mm_prompt_updates` to `prompt`.
851

852
853
854
855
856
    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
    appears earlier in `mm_prompt_updates` takes priority.
    """
    texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
857

858
    return "".join(texts), result
859
860


861
def _iter_placeholders(
862
    prompt: list[int],
863
    mm_prompt_updates: "MultiModalPromptUpdates",
864
    tokenizer: TokenizerLike | None,
865
) -> Iterable[PlaceholderFeaturesInfo]:
866
    """
867
    Yield each set of placeholder tokens found in `prompt`.
868
869
870

    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
871
    appears earlier in `mm_prompt_updates` takes priority.
872

873
874
    Note that empty matches are ignored.
    """
875
    mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()}
876
    item_idx_by_modality = {modality: 0 for modality in mm_prompt_updates}
877

878
879
    if _all_items_found(mm_item_counts, item_idx_by_modality):
        return
880

881
    prompt_len = len(prompt)
882
    start_idx = 0
883

884
885
886
    while start_idx < prompt_len:
        found = False

887
        for modality, modality_updates in mm_prompt_updates.items():
888
889
            item_idx = item_idx_by_modality[modality]
            if item_idx >= mm_item_counts.get(modality, 0):
890
                continue
891

892
893
            for update in modality_updates[item_idx]:
                content = update.content
894
                content_tokens_full = _seq2tokens(tokenizer, content.full)
895
896
                content_len_full = len(content_tokens_full)
                end_idx_full = start_idx + content_len_full
897

898
                if content_len_full == 0 or end_idx_full > prompt_len:
899
900
                    continue

901
                if prompt[start_idx:end_idx_full] == content_tokens_full:
902
903
                    content_is_embed = content.is_embed
                    if content_is_embed is not None:
904
                        content_is_embed = content_is_embed(tokenizer, content.full)
905
906
907
908
909
910
911
912

                    yield PlaceholderFeaturesInfo(
                        modality=modality,
                        item_idx=item_idx,
                        start_idx=start_idx,
                        tokens=content_tokens_full,
                        is_embed=content_is_embed,
                    )
913

914
                    # Exclude overlapping matches
915
                    start_idx = end_idx_full
916
917
918
                    item_idx_by_modality[modality] += 1
                    found = True
                    break
919

920
            if found:
921
922
923
                if _all_items_found(mm_item_counts, item_idx_by_modality):
                    return

924
                break  # Go back to the outer while loop
925
926
927

        if not found:
            start_idx += 1
928
929


930
931
def find_mm_placeholders(
    prompt: list[int],
932
    mm_prompt_updates: "MultiModalPromptUpdates",
933
    tokenizer: TokenizerLike | None,
934
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
935
    it = _iter_placeholders(prompt, mm_prompt_updates, tokenizer)
936
937
938
    return dict(full_groupby_modality(it))


939
_T = TypeVar("_T")
940
941
_C = TypeVar("_C", bound=PretrainedConfig, default=PretrainedConfig)
_P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin)
942
943
944
945
946
947
948
949
950


@dataclass(frozen=True)
class InputProcessingContext:
    """
    Contains information about the model which may be used to
    modify the inputs.
    """

951
    model_config: ModelConfig
952
953
    """The configuration of the model."""

954
    tokenizer: TokenizerLike | None
955
956
    """The tokenizer used to tokenize the inputs."""

957
958
959
960
961
962
963
964
    def get_tokenizer(self) -> TokenizerLike:
        if self.tokenizer is None:
            raise ValueError(
                "You cannot pass text prompts when `skip_tokenizer_init=True`"
            )

        return self.tokenizer

965
    @overload
966
    def get_hf_config(self, /) -> PretrainedConfig: ...
967
968
969
970

    @overload
    def get_hf_config(
        self,
971
        typ: type[_C] | tuple[type[_C], ...],
972
        /,
973
    ) -> _C: ...
974
975
976

    def get_hf_config(
        self,
977
        typ: type[Any] | tuple[type[Any], ...] | None = None,
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
        /,
    ) -> Any:
        """
        Get the HuggingFace configuration
        (`transformers.PretrainedConfig`) of the model,
        additionally checking its type.

        Raises:
            TypeError: If the configuration is not of the specified type.
        """
        if typ is None:
            from transformers.configuration_utils import PretrainedConfig

            typ = PretrainedConfig

        hf_config = self.model_config.hf_config
        if not isinstance(hf_config, typ):
995
996
997
998
999
            raise TypeError(
                "Invalid type of HuggingFace config. "
                f"Expected type: {typ}, but "
                f"found type: {type(hf_config)}"
            )
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022

        return hf_config

    def get_hf_image_processor_config(self) -> dict[str, Any]:
        """
        Get the HuggingFace image processor configuration of the model.
        """
        return self.model_config.hf_image_processor_config

    def get_mm_config(self):
        """
        Get the multimodal config of the model.

        Raises:
            RuntimeError: If the model is not a multimodal model.
        """
        mm_config = self.model_config.multimodal_config
        if mm_config is None:
            raise RuntimeError("Not a multimodal model")

        return mm_config

    @overload
1023
    def get_hf_processor(self, /, **kwargs: object) -> ProcessorMixin: ...
1024
1025
1026
1027

    @overload
    def get_hf_processor(
        self,
1028
        typ: type[_P] | tuple[type[_P], ...],
1029
1030
        /,
        **kwargs: object,
1031
    ) -> _P: ...
1032
1033
1034

    def get_hf_processor(
        self,
1035
        typ: type[Any] | tuple[type[Any], ...] | None = None,
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
        /,
        **kwargs: object,
    ) -> Any:
        """
        Get the HuggingFace processor
        (`transformers.ProcessorMixin`) of the model,
        additionally checking its type.

        Raises:
            TypeError: If the processor is not of the specified type.
        """
        if typ is None:
            from transformers.processing_utils import ProcessorMixin

            typ = ProcessorMixin

        return cached_processor_from_config(
            self.model_config,
            processor_cls=typ,
            tokenizer=self.tokenizer,
            **kwargs,
        )

    def init_processor(
        self,
        typ: type[_T],
        /,
        **kwargs: object,
    ) -> _T:
        """
        Initialize a HuggingFace-like processor class, merging the
        keyword arguments with those in the model's configuration.
        """
        mm_config = self.model_config.get_multimodal_config()
        base_kwargs = mm_config.mm_processor_kwargs
        if base_kwargs is None:
            base_kwargs = {}

        merged_kwargs = {**base_kwargs, **kwargs}

        return typ(**merged_kwargs)

    def _postprocess_output(
        self,
        output: JSONTree,
    ) -> JSONTree:
        def _postprocess_one(x: object):
            if isinstance(x, torch.Tensor):  # noqa: SIM102
                # This mimics the behavior of transformers.BatchFeature
                if x.is_floating_point():
                    x = x.to(dtype=self.model_config.dtype)

            return x

        return json_map_leaves(_postprocess_one, output)

    def call_hf_processor(
        self,
1094
        hf_processor: ProcessorMixin,
1095
1096
1097
1098
1099
        data: Mapping[str, object],
        kwargs: Mapping[str, object] = {},
        *,
        num_tries: int = 1,
        max_tries: int = 5,
1100
    ) -> BatchFeature | JSONTree:
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
        """
        Call `hf_processor` on the prompt `data`
        (text, image, audio...) with configurable options `kwargs`.
        """
        assert callable(hf_processor)

        mm_config = self.model_config.get_multimodal_config()
        merged_kwargs = mm_config.merge_mm_processor_kwargs(kwargs)

        allowed_kwargs = get_allowed_kwarg_only_overrides(
            hf_processor,
            merged_kwargs,
            requires_kw_only=False,
            allow_var_kwargs=True,
        )

        try:
1118
            output = hf_processor(**data, **allowed_kwargs, return_tensors="pt")
1119
1120
        except Exception as exc:
            # See https://github.com/huggingface/tokenizers/issues/537
1121
1122
1123
1124
1125
1126
            if (
                isinstance(exc, RuntimeError)
                and exc
                and exc.args[0] == "Already borrowed"
                and num_tries < max_tries
            ):
1127
1128
                logger.warning(
                    "Failed to acquire tokenizer in current thread. "
1129
1130
1131
1132
                    "Retrying (%d/%d)...",
                    num_tries,
                    max_tries,
                )
1133
1134
1135
1136
1137
1138
1139
1140
1141
                time.sleep(0.5)
                return self.call_hf_processor(
                    hf_processor,
                    data,
                    kwargs,
                    num_tries=num_tries + 1,
                    max_tries=max_tries,
                )

1142
1143
1144
1145
            msg = (
                f"Failed to apply {type(hf_processor).__name__} "
                f"on data={data} with kwargs={allowed_kwargs}"
            )
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165

            raise ValueError(msg) from exc

        # this emulates output.to(dtype=self.model_config.dtype)
        from transformers.feature_extraction_utils import BatchFeature

        if isinstance(output, BatchFeature):
            output_ = self._postprocess_output(output.data)
            return BatchFeature(output_)

        logger.warning_once(
            "%s did not return `BatchFeature`. "
            "Make sure to match the behaviour of `ProcessorMixin` when "
            "implementing custom processors.",
            type(hf_processor).__name__,
        )

        return self._postprocess_output(output)


1166
class BaseProcessingInfo:
1167
    """Base class to provide the information necessary for data processing."""
1168

1169
1170
    def __init__(self, ctx: InputProcessingContext) -> None:
        super().__init__()
1171

1172
1173
1174
1175
1176
1177
        self.ctx = ctx

    @property
    def model_id(self) -> str:
        return self.ctx.model_config.model

1178
    def get_tokenizer(self) -> TokenizerLike:
1179
        return self.ctx.get_tokenizer()
1180

1181
    def get_hf_config(self) -> PretrainedConfig:
1182
1183
        return self.ctx.get_hf_config()

1184
    def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
1185
1186
1187
1188
1189
1190
        """
        Subclasses can override this method to handle
        specific kwargs from model config or user inputs.
        """
        return self.ctx.get_hf_processor(**kwargs)

1191
    @abstractmethod
1192
    def get_supported_mm_limits(self) -> Mapping[str, int | None]:
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
        """
        Return the maximum supported number of items for each modality.

        A value of `None` means unlimited number of items.

        Omitting a modality from the returned dictionary means that
        it is not supported at all.
        """
        raise NotImplementedError

1203
1204
1205
1206
1207
1208
1209
1210
1211
    def get_allowed_mm_limits(self) -> Mapping[str, int]:
        """Return the maximum allowed number of items for each modality."""
        supported_mm_limits = self.get_supported_mm_limits()
        mm_config = self.ctx.get_mm_config()

        allowed_limits = dict[str, int]()
        for modality, supported_limit in supported_mm_limits.items():
            user_limit = mm_config.get_limit_per_prompt(modality)

1212
1213
1214
1215
1216
            allowed_limits[modality] = (
                user_limit
                if supported_limit is None
                else min(user_limit, supported_limit)
            )
1217
1218
1219

        return allowed_limits

1220
1221
1222
1223
    def get_mm_max_tokens_per_item(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
1224
    ) -> Mapping[str, int] | None:
1225
1226
        """
        Return the maximum number of tokens per item of for each modality.
1227

1228
1229
1230
1231
        When `None` (the default) is returned, vLLM will generate dummy inputs
        (images/videos) at maximum possible sizes and process them to determine
        the maximum token count per modality.

1232
1233
1234
1235
1236
        This approach works but can be very slow for certain models (e.g.,
        Qwen2.5-VL), leading to very long startup time. For better performance,
        each model can override this method to return pre-computed maximum token
        counts, avoiding the need for dummy input generation and processing.

1237
        Note:
1238
            The maximum number of tokens per item of each modality returned
1239
1240
1241
1242
            from this function should respect the model's maximum sequence
            length and the maximum number of items of each modality allowed,
            and agree with dummy inputs (images/videos) at maximum possible
            sizes.
1243
1244
1245
        """
        return None

1246
1247

_I = TypeVar("_I", bound=BaseProcessingInfo)
1248

1249
1250
MultiModalHashes = dict[str, list[str]]
"""
1251
A collection of hashes with a similar structure as
1252
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
1253
1254
"""

1255
MultiModalPromptUpdates = Mapping[str, list[Sequence[ResolvedPromptUpdate]]]
1256
1257
1258
1259
1260
"""
A collection of prompt updates with a similar structure as
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
"""

1261
MultiModalPromptUpdatesApplyResult = Mapping[str, list[int | None]]
1262
1263
1264
1265
1266
1267
1268
"""
For an item `MultiModalPromptUpdates[k][i]`,
`MultiModalPromptUpdatesApplyResult[k][i]` represents the index of the
`ResolvedPromptUpdate` instance that has been applied, or `None` if none of the
`ResolvedPromptUpdate` instances have been applied.
"""

1269
1270

class MultiModalProcessingInfo(NamedTuple):
1271
    kwargs: MultiModalKwargsOptionalItems
1272
    hashes: MultiModalHashes
1273
1274
    prompt_updates: MultiModalPromptUpdates

1275
1276

class BaseMultiModalProcessor(ABC, Generic[_I]):
1277
    """
1278
    Abstract base class to process multi-modal inputs to be used in vLLM.
1279

1280
    Not to be confused with `transformers.ProcessorMixin`.
1281
1282
    """

1283
1284
1285
1286
1287
    def __init__(
        self,
        info: _I,
        dummy_inputs: "BaseDummyInputsBuilder[_I]",
        *,
1288
        cache: BaseMultiModalProcessorCache | None = None,
1289
    ) -> None:
1290
1291
        super().__init__()

1292
1293
        self.info = info
        self.dummy_inputs = dummy_inputs
1294
        self.cache = cache
1295

1296
1297
        self.data_parser = self._get_data_parser()

1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
        # Avoid unnecessary recomputation
        self._supported_mm_limits = self.info.get_supported_mm_limits()
        self._allowed_mm_limits = self.info.get_allowed_mm_limits()

    @property
    def supported_mm_limits(self):
        return self._supported_mm_limits

    @property
    def allowed_mm_limits(self):
        return self._allowed_mm_limits

1310
    def __call__(
1311
        self,
1312
1313
        prompt: str,
        mm_data: MultiModalDataDict,
1314
        hf_processor_mm_kwargs: Mapping[str, object],
1315
        *,
1316
        mm_uuids: MultiModalUUIDDict | None = None,
1317
    ) -> MultiModalInputs:
1318
        return self.apply(prompt, mm_data, hf_processor_mm_kwargs, mm_uuids=mm_uuids)
1319

1320
1321
    def _get_data_parser(self) -> MultiModalDataParser:
        """
1322
        Construct a parser to preprocess multi-modal data items
1323
1324
        before passing them to
        [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
1325
1326

        You can support additional modalities by creating a subclass
1327
1328
        of [`MultiModalDataParser`][vllm.multimodal.parse.MultiModalDataParser]
        that has additional subparsers.
1329
1330
1331
        """
        return MultiModalDataParser()

1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
    def validate_num_items(
        self,
        modality: str,
        num_items: int,
    ) -> None:
        supported_limit = self.supported_mm_limits.get(modality, 0)
        allowed_limit = self.allowed_mm_limits.get(modality, 0)

        if supported_limit is None:
            supported_limit = allowed_limit

        limit = min(supported_limit, allowed_limit)

        if num_items > limit:
1346
            msg = f"At most {limit} {modality}(s) may be provided in one prompt."
1347
1348
1349
1350
1351
1352

            if num_items <= supported_limit:
                msg += " Set `--limit-mm-per-prompt` to increase this limit."

            raise ValueError(msg)

1353
    def _to_mm_items(
1354
1355
1356
        self,
        mm_data: MultiModalDataDict,
    ) -> MultiModalDataItems:
1357
        """
1358
1359
1360
1361
1362
        Normalize
        [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict]
        to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]
        before passing them to
        [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
1363
        """
1364
        mm_items = self.data_parser.parse_mm_data(mm_data)
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374

        mm_config = self.info.ctx.model_config.get_multimodal_config()
        if not mm_config.enable_mm_embeds:
            for modality, items in mm_items.items():
                if isinstance(items, (EmbeddingItems, DictEmbeddingItems)):
                    raise ValueError(
                        f"You must set `--enable-mm-embeds` to input "
                        f"`{modality}_embeds`"
                    )

1375
        for modality, items in mm_items.items():
1376
            self.validate_num_items(modality, len(items))
1377
1378

        return mm_items
1379

1380
1381
1382
    @abstractmethod
    def _get_mm_fields_config(
        self,
1383
        hf_inputs: BatchFeature,
1384
1385
1386
1387
1388
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, MultiModalFieldConfig]:
        """Given the HF-processed data, output the metadata of each field."""
        raise NotImplementedError

1389
    @abstractmethod
1390
    def _get_prompt_updates(
1391
        self,
1392
        mm_items: MultiModalDataItems,
1393
        hf_processor_mm_kwargs: Mapping[str, object],
1394
        out_mm_kwargs: MultiModalKwargsItems,
1395
    ) -> Sequence[PromptUpdate]:
1396
1397
        """
        Given the original multi-modal items for this modality
1398
        and HF-processed data, output the updates to perform.
1399

1400
1401
1402
1403
1404
1405
        The information returned by this method is used to update token inputs
        which bypass the HF processor. It is also used to update the output of
        HF processor if the HF process does not apply prompt updates to text
        inputs.

        Moreover, this information is critical to determine the token positions
1406
1407
        in order to construct
        [`PlaceholderRange`][vllm.multimodal.inputs.PlaceholderRange]
1408
        for each multi-modal item.
1409
1410
        """
        raise NotImplementedError
1411

1412
1413
1414
1415
1416
1417
    def _bind_and_group_updates(
        self,
        prompt_updates: Sequence[PromptUpdate],
        mm_item_counts: Mapping[str, int],
    ) -> MultiModalPromptUpdates:
        return {
1418
1419
1420
1421
            modality: [
                [update.resolve(item_idx) for update in updates]
                for item_idx in range(mm_item_counts.get(modality, 0))
            ]
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
            for modality, updates in full_groupby_modality(prompt_updates)
        }

    def _get_mm_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        out_mm_kwargs: MultiModalKwargsItems,
    ) -> MultiModalPromptUpdates:
        unbound_prompt_updates = self._get_prompt_updates(
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
            out_mm_kwargs=out_mm_kwargs,
        )

        mm_prompt_updates = self._bind_and_group_updates(
            unbound_prompt_updates,
            mm_items.get_all_counts(),
        )

        for modality, prompt_updates in mm_prompt_updates.items():
            for item_idx, item_prompt_updates in enumerate(prompt_updates):
                if len(item_prompt_updates) > 1:
                    logger.warning_once(
                        "Detected %d prompt updates for `mm_items[%r][%s]`. "
                        "Multiple prompt updates per item is now "
                        "deprecated and may be removed in v0.13. "
                        "Instead, please specify dynamic update targets "
                        "in the same prompt update definition by passing "
                        "a function to `PromptUpdate.target`.",
                        len(prompt_updates),
                        modality,
                        item_idx,
                    )

        return mm_prompt_updates

1459
    def _find_mm_placeholders(
1460
1461
        self,
        new_token_ids: list[int],
1462
        mm_prompt_updates: MultiModalPromptUpdates,
1463
    ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
1464
1465
        tokenizer = self.info.get_tokenizer()

1466
        return find_mm_placeholders(new_token_ids, mm_prompt_updates, tokenizer)
1467

1468
    def _get_hf_mm_data(
1469
        self,
1470
        mm_items: MultiModalDataItems,
1471
1472
1473
    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
        processor_data = dict[str, object]()
        passthrough_data = dict[str, object]()
1474

1475
1476
1477
        for items in mm_items.values():
            processor_data.update(items.get_processor_data())
            passthrough_data.update(items.get_passthrough_data())
1478

1479
1480
        return processor_data, passthrough_data

1481
1482
1483
    def _call_hf_processor(
        self,
        prompt: str,
1484
1485
1486
1487
        # Not to be confused with `mm_data` in `self.apply`.
        # This refers to the data to be passed to HF processor.
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
1488
        tok_kwargs: Mapping[str, object],
1489
    ) -> BatchFeature:
1490
1491
1492
1493
        """
        Call the HF processor on the prompt text and
        associated multi-modal data.
        """
1494
1495
        return self.info.ctx.call_hf_processor(
            self.info.get_hf_processor(**mm_kwargs),
1496
            dict(text=prompt, **mm_data),
1497
            dict(**mm_kwargs, **tok_kwargs),
1498
1499
        )

1500
    def _hf_processor_applies_updates(
1501
1502
1503
1504
        self,
        prompt_text: str,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1505
        tokenization_kwargs: Mapping[str, object],
1506
1507
    ) -> bool:
        """
1508
        Return whether the HF processor applies prompt updates.
1509

1510
1511
        For most HF processors, this should be `True` when multi-modal
        data items are passed, but `False` when multi-modal embeddings
1512
1513
1514
1515
        are passed.
        """
        return not any(
            isinstance(items, (EmbeddingItems, DictEmbeddingItems))
1516
1517
            for items in mm_items.values()
        )
1518

1519
    def _apply_hf_processor_text_mm(
1520
        self,
1521
        prompt_text: str,
1522
        mm_items: MultiModalDataItems,
1523
        hf_processor_mm_kwargs: Mapping[str, object],
1524
        tokenization_kwargs: Mapping[str, object],
1525
    ) -> tuple[list[int], BatchFeature, bool]:
1526
        """
1527
1528
        Apply the HF processor on the prompt text and multi-modal data
        together.
1529

1530
        In addition, return whether prompt updates have been applied.
1531
1532
1533
1534
1535
1536
1537
        """
        processor_data, passthrough_data = self._get_hf_mm_data(mm_items)

        processed_data = self._call_hf_processor(
            prompt=prompt_text,
            mm_data=processor_data,
            mm_kwargs=hf_processor_mm_kwargs,
1538
            tok_kwargs=tokenization_kwargs,
1539
1540
        )
        processed_data.update(passthrough_data)
1541

1542
        (prompt_ids,) = processed_data.pop("input_ids").tolist()
1543

1544
        is_update_applied = self._hf_processor_applies_updates(
1545
1546
1547
            prompt_text=prompt_text,
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1548
            tokenization_kwargs=tokenization_kwargs,
1549
1550
        )

1551
        return prompt_ids, processed_data, is_update_applied
1552

1553
    def _apply_hf_processor_text_only(
1554
1555
1556
1557
        self,
        prompt_text: str,
        tokenization_kwargs: Mapping[str, object],
    ) -> list[int]:
1558
        """
1559
        Apply the HF processor on the prompt text only.
1560

1561
1562
1563
        Since HF processor requires that text and multi-modal items
        correspond to each other, we create dummy multi-modal items
        to go along with the text.
1564
        """
1565
        prompt_ids, _, _ = self._apply_hf_processor_text_mm(
1566
1567
1568
            prompt_text=prompt_text,
            mm_items=MultiModalDataItems({}),
            hf_processor_mm_kwargs={},
1569
            tokenization_kwargs=tokenization_kwargs,
1570
1571
        )

1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
        return prompt_ids

    def _apply_hf_processor_tokens_only(
        self,
        prompt_tokens: list[int],
    ) -> list[int]:
        """
        Apply the HF processor on the prompt tokens only.

        Most HF processors accept prompt text but not prompt tokens.
        If the HF processor adds or removes tokens that are not related to
        multi-modal data, you should override this method so it is consistent
1584
1585
1586
        with the output of
        [`_apply_hf_processor_text_only`][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_text_only]
        on the
1587
1588
1589
1590
1591
1592
1593
1594
        corresponding text.
        """
        return prompt_tokens

    def _apply_hf_processor_mm_only(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1595
        tokenization_kwargs: Mapping[str, object],
1596
    ) -> BatchFeature:
1597
1598
1599
1600
1601
        """
        Apply the HF processor on the multi-modal data only.

        Since HF processor requires that text and multi-modal items
        correspond to each other, we generate dummy text using
1602
1603
        [`DummyInputsBuilder`][vllm.multimodal.profiling.BaseDummyInputsBuilder]
        to go along with the multi-modal data.
1604
1605
1606
        """
        mm_counts = mm_items.get_all_counts()

1607
        _, mm_processed_data, _ = self._apply_hf_processor_text_mm(
1608
            prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
1609
1610
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1611
            tokenization_kwargs=tokenization_kwargs,
1612
1613
        )

1614
        return mm_processed_data
1615
1616
1617

    def _apply_hf_processor_main(
        self,
1618
        prompt: str | list[int],
1619
1620
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1621
        tokenization_kwargs: Mapping[str, object],
1622
        *,
1623
        enable_hf_prompt_update: bool,
1624
    ) -> tuple[list[int], BatchFeature, bool]:
1625
1626
1627
        """
        Apply the HF processor on the prompt text and multi-modal data.

1628
        In addition, return whether prompt updates have been applied
1629
        (for most HF processors, this should be `True`).
1630

1631
        Note:
1632
            If `enable_hf_prompt_update=False`, we use HF processor
1633
            to perform prompt updates if available; HF processor requires
1634
            that the prompt corresponds to multi-modal items.
1635
1636
        """
        if isinstance(prompt, str):
1637
            if enable_hf_prompt_update:
1638
1639
1640
1641
                return self._apply_hf_processor_text_mm(
                    prompt_text=prompt,
                    mm_items=mm_items,
                    hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1642
                    tokenization_kwargs=tokenization_kwargs,
1643
1644
                )

1645
            prompt_ids = self._apply_hf_processor_text_only(prompt, tokenization_kwargs)
1646
1647
1648
        else:
            prompt_ids = self._apply_hf_processor_tokens_only(prompt)

1649
        mm_processed_data = self._apply_hf_processor_mm_only(
1650
            mm_items=mm_items,
1651
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1652
            tokenization_kwargs=tokenization_kwargs,
1653
1654
        )

1655
        return prompt_ids, mm_processed_data, False
1656

1657
    def _hash_mm_items(
1658
1659
1660
1661
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        tokenization_kwargs: Mapping[str, object],
1662
        *,
1663
        mm_uuids: MultiModalUUIDDict | None = None,
1664
    ) -> MultiModalHashes:
1665
        """Create MM hashes to be returned.
1666

1667

1668
1669
1670
        Note: When overrides are provided via callers of `apply`,
        `_hash_mm_items` will be bypassed and the overrides will be used.
        """
1671
1672
        model_id = self.info.model_id

1673
        hashes: MultiModalHashes = {}
1674
        mm_uuids = mm_uuids or {}
1675
1676

        for modality, items in mm_items.items():
1677
1678
1679
1680
            if modality in mm_uuids:
                mm_uuids_per_modality = mm_uuids[modality]
                if isinstance(mm_uuids_per_modality, str):
                    mm_uuids_per_modality = [mm_uuids_per_modality]
1681
1682
1683
1684

                # For None entries, compute a hash; otherwise, use provided ID.
                computed: list[str] = []
                for i, item in enumerate(items):
1685
                    item_uuid = mm_uuids_per_modality[i]
1686

1687
                    # NOTE: Even if a item_uuid is provided, we still compute a
1688
1689
1690
                    # hash if `hf_processor_mm_kwargs` or `tokenization_kwargs`
                    # are provided. This is because the processed multimodal
                    # inputs can be different depending on the processor kwargs.
1691
1692
1693
1694
1695
                    if (
                        item_uuid is None
                        or hf_processor_mm_kwargs
                        or tokenization_kwargs
                    ):
1696
1697
                        # NOTE: use provided hash string to hash with kwargs
                        # if available for better performance.
1698
                        item = item_uuid if item_uuid is not None else item
1699
1700
1701
1702
1703
                        computed.append(
                            MultiModalHasher.hash_kwargs(
                                model_id=model_id,
                                **{modality: item},
                                **hf_processor_mm_kwargs,
1704
1705
1706
                                **tokenization_kwargs,
                            )
                        )
1707
                    else:
1708
                        computed.append(item_uuid)
1709
1710
1711
                hashes[modality] = computed
            else:
                hashes[modality] = [
1712
1713
1714
1715
1716
1717
                    MultiModalHasher.hash_kwargs(
                        model_id=model_id,
                        **{modality: item},
                        **hf_processor_mm_kwargs,
                        **tokenization_kwargs,
                    )
1718
1719
1720
1721
                    for item in items
                ]

        return hashes
1722

1723
1724
    def _get_cache_missing_items(
        self,
1725
        cache: BaseMultiModalProcessorCache,
1726
1727
1728
1729
        mm_data_items: MultiModalDataItems,
        mm_hashes: MultiModalHashes,
    ) -> MultiModalDataItems:
        mm_is_cached = {
1730
            modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
1731
1732
1733
1734
        }

        mm_missing_idxs = {
            modality: [
1735
1736
                idx
                for idx, item_is_cached in enumerate(items_is_cached)
1737
1738
1739
1740
                if not item_is_cached
            ]
            for modality, items_is_cached in mm_is_cached.items()
        }
1741
1742
1743
1744
1745
1746
1747
1748
        mm_missing_data = {}
        for modality, idxs in mm_missing_idxs.items():
            missing_modality_data = []
            for idx in idxs:
                data = mm_data_items[modality][idx]
                if data is None:
                    raise ValueError(
                        f"Cache miss for {modality} at index {idx} "
1749
1750
                        f"but data is not provided."
                    )
1751
1752
1753
                else:
                    missing_modality_data.append(data)
            mm_missing_data[modality] = missing_modality_data
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767

        return self._to_mm_items(mm_missing_data)

    def _recompute_cached_prompt_update(
        self,
        cached_update: ResolvedPromptUpdate,
        new_item_idx: int,
    ) -> ResolvedPromptUpdate:
        """
        Override this if other attributes of `ResolvedPromptUpdate`
        also need to be recomputed after retrieving from the cache.
        """
        return replace(cached_update, item_idx=new_item_idx)

1768
1769
    def _merge_mm_kwargs(
        self,
1770
        cache: BaseMultiModalProcessorCache,
1771
        mm_hashes: MultiModalHashes,
1772
        mm_missing_kwargs: MultiModalKwargsItems,
1773
1774
1775
1776
1777
        mm_missing_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[MultiModalKwargsOptionalItems, MultiModalPromptUpdates]:
        # Need to calculate this at the beginning to avoid skipping cache logic
        # for subsequently repeated items in the same modality
        mm_is_cached = {
1778
            modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
1779
1780
        }

1781
        mm_missing_next_idx = defaultdict[str, int](lambda: 0)
1782

1783
        merged_kwargs = defaultdict[str, list[MultiModalKwargsItem | None]](list)
1784
1785
1786
        merged_prompt_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](
            list
        )
1787
1788
        for modality, hashes in mm_hashes.items():
            missing_kwargs = mm_missing_kwargs.get(modality, [])
1789
            missing_prompt_updates = mm_missing_prompt_updates.get(modality, [])
1790
1791

            for item_idx, item_hash in enumerate(hashes):
1792
                kwargs: MultiModalKwargsItem | None
1793
1794
1795
1796
1797
                if not mm_is_cached[modality][item_idx]:
                    missing_next_idx = mm_missing_next_idx[modality]
                    kwargs = missing_kwargs[missing_next_idx]
                    updates = missing_prompt_updates[missing_next_idx]

1798
                    mm_missing_next_idx[modality] += 1
1799
1800

                    item = kwargs, updates
1801
                else:
1802
1803
1804
1805
1806
                    item = None

                kwargs, updates = cache.get_and_update_item(item, item_hash)

                merged_kwargs[modality].append(kwargs)
1807
1808
1809
1810
1811
1812
                merged_prompt_updates[modality].append(
                    [
                        self._recompute_cached_prompt_update(update, item_idx)
                        for update in updates
                    ]
                )
1813

1814
1815
        mm_kwargs = MultiModalKwargsItems(merged_kwargs)
        mm_prompt_updates = dict(merged_prompt_updates)
1816

1817
        return mm_kwargs, mm_prompt_updates
1818
1819
1820

    def _apply_hf_processor(
        self,
1821
        prompt: str | list[int],
1822
1823
        mm_data_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1824
        tokenization_kwargs: Mapping[str, object],
1825
        *,
1826
        mm_uuids: MultiModalUUIDDict | None = None,
1827
    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1828
1829
        (
            prompt_ids,
1830
            mm_processed_data,
1831
1832
1833
1834
1835
            is_update_applied,
        ) = self._apply_hf_processor_main(
            prompt=prompt,
            mm_items=mm_data_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1836
            tokenization_kwargs=tokenization_kwargs,
1837
1838
1839
            enable_hf_prompt_update=True,
        )

1840
        mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
1841
            mm_processed_data,
1842
            self._get_mm_fields_config(mm_processed_data, hf_processor_mm_kwargs),
1843
1844
        )

1845
        # Use overrides if provided; fallback to data-dependent hashing.
1846
1847
1848
1849
1850
1851
        mm_hashes = self._hash_mm_items(
            mm_data_items,
            hf_processor_mm_kwargs,
            tokenization_kwargs,
            mm_uuids=mm_uuids,
        )
1852

1853
        mm_prompt_updates = self._get_mm_prompt_updates(
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
            mm_data_items,
            hf_processor_mm_kwargs,
            mm_kwargs,
        )

        mm_info = MultiModalProcessingInfo(
            kwargs=mm_kwargs,
            hashes=mm_hashes,
            prompt_updates=mm_prompt_updates,
        )

        return prompt_ids, mm_info, is_update_applied
1866

1867
1868
    def _cached_apply_hf_processor(
        self,
1869
        prompt: str | list[int],
1870
1871
        mm_data_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1872
        tokenization_kwargs: Mapping[str, object],
1873
        *,
1874
        mm_uuids: MultiModalUUIDDict | None = None,
1875
    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1876
1877
1878
1879
1880
1881
        """
        Apply the HF processor on the full prompt text,
        caching the results and reusing cached results.
        """
        cache = self.cache

1882
1883
        _, passthrough_data = self._get_hf_mm_data(mm_data_items)
        if cache is None or passthrough_data:
1884
            return self._apply_hf_processor(
1885
                prompt=prompt,
1886
                mm_data_items=mm_data_items,
1887
                hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1888
                tokenization_kwargs=tokenization_kwargs,
1889
                mm_uuids=mm_uuids,
1890
1891
            )

1892
1893
1894
1895
1896
1897
        mm_hashes = self._hash_mm_items(
            mm_data_items,
            hf_processor_mm_kwargs,
            tokenization_kwargs,
            mm_uuids=mm_uuids,
        )
1898
1899

        mm_missing_data_items = self._get_cache_missing_items(
1900
1901
            cache=cache,
            mm_data_items=mm_data_items,
1902
            mm_hashes=mm_hashes,
1903
        )
1904

1905
        # NOTE: `prompt` does not correspond to `mm_missing_data_items`,
1906
        # so we can't apply prompt updates until the new multimodal
1907
1908
1909
        # items are combined with the cached multimodal items
        (
            prompt_ids,
1910
            mm_missing_processed_data,
1911
            is_update_applied,
1912
        ) = self._apply_hf_processor_main(
1913
            prompt=prompt,
1914
            mm_items=mm_missing_data_items,
1915
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1916
            tokenization_kwargs=tokenization_kwargs,
1917
            enable_hf_prompt_update=False,
1918
1919
        )

1920
        mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
1921
            mm_missing_processed_data,
1922
1923
1924
            self._get_mm_fields_config(
                mm_missing_processed_data, hf_processor_mm_kwargs
            ),
1925
1926
        )

1927
1928
1929
1930
        mm_missing_prompt_updates = self._get_mm_prompt_updates(
            mm_missing_data_items,
            hf_processor_mm_kwargs,
            mm_missing_kwargs,
1931
        )
1932

1933
1934
1935
1936
1937
        mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
            cache,
            mm_hashes=mm_hashes,
            mm_missing_kwargs=mm_missing_kwargs,
            mm_missing_prompt_updates=mm_missing_prompt_updates,
1938
1939
1940
1941
        )

        mm_info = MultiModalProcessingInfo(
            kwargs=mm_kwargs,
1942
            hashes=mm_hashes,
1943
1944
            prompt_updates=mm_prompt_updates,
        )
1945

1946
        return prompt_ids, mm_info, is_update_applied
1947

1948
1949
1950
    def _apply_token_matches(
        self,
        prompt: list[int],
1951
1952
1953
1954
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
        tokenizer = self.info.get_tokenizer()
        return apply_token_matches(prompt, mm_prompt_updates, tokenizer)
1955
1956
1957
1958

    def _apply_text_matches(
        self,
        prompt: str,
1959
1960
1961
1962
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[str, MultiModalPromptUpdatesApplyResult]:
        tokenizer = self.info.get_tokenizer()
        return apply_text_matches(prompt, mm_prompt_updates, tokenizer)
1963

1964
    def _apply_prompt_updates(
1965
1966
        self,
        token_ids: list[int],
1967
        mm_prompt_updates: MultiModalPromptUpdates,
1968
    ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
1969
        tokenizer = self.info.get_tokenizer()
1970

1971
1972
1973
1974
        new_token_ids, match_result = self._apply_token_matches(
            token_ids,
            mm_prompt_updates,
        )
1975
1976
1977
1978
1979
1980
1981
1982
1983

        # If the search text does not represent a special token,
        # it may have different token IDs in the prompt, because
        # the tokens may go across the boundaries of the search text.
        # ----
        # e.g. when searching for "foo" in "food", if "food" itself makes
        # up a token, then the token ID of "foo" will not appear at all
        # ----
        # Since it is inefficient to search for all possible tokenizations
1984
1985
        # of the search text in the prompt, we instead perform string-based
        # updates on the decoded token IDs, then encode them back.
1986
        if not all(
1987
1988
1989
            all(update_idx is not None for update_idx in update_idxs)
            for update_idxs in match_result.values()
        ):
1990
            new_text, match_result = self._apply_text_matches(
1991
                _seq2text(tokenizer, token_ids, use_cache=False),
1992
                mm_prompt_updates,
1993
1994
            )

1995
            new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False)
1996

1997
        matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
1998
1999
2000
2001
        for modality, update_idxs in match_result.items():
            for item_idx, update_idx in enumerate(update_idxs):
                assert update_idx is not None, (
                    "Failed to apply prompt replacement for "
2002
2003
                    f"mm_items[{modality!r}][{item_idx}]"
                )
2004
2005

                matched_updates[modality].append(
2006
2007
                    [mm_prompt_updates[modality][item_idx][update_idx]]
                )
2008
2009

        placeholders = self._find_mm_placeholders(
2010
2011
            new_token_ids,
            dict(matched_updates),
2012
        )
2013

2014
        return new_token_ids, placeholders
2015

2016
2017
    def _validate_mm_kwargs(
        self,
2018
        mm_kwargs: MultiModalKwargsOptionalItems,
2019
2020
2021
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
2022
            items = mm_kwargs.get(modality, [])
2023
2024
2025
2026
2027
2028
2029
2030
2031

            if len(items) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} {modality} items in "
                    f"keyword arguments corresponding to {item_count} "
                    f"{modality} data items, but only found {len(items)}! "
                    "There is likely a problem with your "
                    "implementation of merged multi-modal processor for this "
                    "model (usually arising from an inconsistency between "
2032
2033
                    "`_call_hf_processor` and `_get_mm_fields_config`)."
                )
2034

2035
    def _validate_mm_updates(
2036
        self,
2037
        mm_updates: MultiModalPromptUpdates,
2038
        mm_item_counts: Mapping[str, int],
2039
    ) -> None:
2040
        for modality, item_count in mm_item_counts.items():
2041
            placeholders = mm_updates.get(modality, [])
2042

2043
            if len(placeholders) != item_count:
2044
                raise RuntimeError(
2045
                    f"Expected there to be {item_count} prompt updates "
2046
                    f"corresponding to {item_count} {modality} items, but "
2047
                    f"instead found {len(placeholders)} prompt updates! "
2048
2049
2050
                    "This is likely because you forgot to include input "
                    "placeholder tokens (e.g., `<image>`, `<|image_pad|>`) "
                    "in the prompt. If the model has a chat template, make "
2051
2052
                    "sure you have applied it before calling `LLM.generate`."
                )
2053

2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
    def _validate_mm_placeholders(
        self,
        mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
            placeholders = mm_placeholders.get(modality, [])

            if len(placeholders) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} prompt placeholders "
                    f"corresponding to {item_count} {modality} items, but "
                    f"instead found {len(placeholders)} prompt placeholders! "
                    "Make sure the implementation of `_call_hf_processor` and "
2068
2069
                    "`_get_mm_fields_config` are consistent with each other."
                )
2070

2071
2072
2073
2074
    def _maybe_apply_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        prompt_ids: list[int],
2075
        mm_kwargs: MultiModalKwargsOptionalItems,
2076
        mm_prompt_updates: MultiModalPromptUpdates,
2077
        is_update_applied: bool,
2078
    ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
2079
        mm_item_counts = mm_items.get_all_counts()
2080
        self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
2081
        self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
2082

2083
        if is_update_applied:
2084
2085
            mm_placeholders = self._find_mm_placeholders(
                prompt_ids,
2086
                mm_prompt_updates,
2087
            )
2088
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
2089
        else:
2090
            prompt_ids, mm_placeholders = self._apply_prompt_updates(
2091
                prompt_ids,
2092
                mm_prompt_updates,
2093
            )
2094
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
2095

2096
        return prompt_ids, mm_placeholders
2097
2098
2099

    def apply(
        self,
2100
        prompt: str | list[int],
2101
2102
        mm_data: MultiModalDataDict,
        hf_processor_mm_kwargs: Mapping[str, object],
2103
        tokenization_kwargs: Mapping[str, object] | None = None,
2104
        *,
2105
        mm_uuids: MultiModalUUIDDict | None = None,
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
    ) -> MultiModalInputs:
        """
        Process multi-modal inputs to be used in vLLM.

        The main steps are:

        1. Apply HF Processor on prompt text and multi-modal data together,
           outputting token IDs and processed tensors.
        2. Find and update sequences in the token IDs with placeholder tokens.
           The number of placeholder tokens equals the feature size of the
           multi-modal data outputted by the multi-modal encoder.
        3. Extract information about the placeholder tokens from the
           processed token IDs.
        """
        mm_items = self._to_mm_items(mm_data)

2122
2123
2124
        if tokenization_kwargs is None:
            tokenization_kwargs = {}

2125
2126
        (
            prompt_ids,
2127
            mm_info,
2128
2129
2130
2131
2132
            is_update_applied,
        ) = self._cached_apply_hf_processor(
            prompt,
            mm_items,
            hf_processor_mm_kwargs,
2133
            tokenization_kwargs=tokenization_kwargs,
2134
            mm_uuids=mm_uuids,
2135
2136
        )

2137
        # NOTE: tokenization_kwargs are not required to init processor
2138
        prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
2139
2140
            mm_items=mm_items,
            prompt_ids=prompt_ids,
2141
2142
            mm_kwargs=mm_info.kwargs,
            mm_prompt_updates=mm_info.prompt_updates,
2143
2144
2145
            is_update_applied=is_update_applied,
        )

2146
2147
2148
2149
        mm_placeholder_ranges = {
            modality: [item.to_range() for item in placeholders]
            for modality, placeholders in mm_placeholders.items()
        }
2150

2151
        return MultiModalInputs(
2152
            type="multimodal",
2153
            prompt_token_ids=prompt_ids,
2154
2155
            mm_kwargs=mm_info.kwargs,
            mm_hashes=mm_info.hashes,
2156
            mm_placeholders=mm_placeholder_ranges,
2157
        )
2158
2159
2160
2161
2162
2163


class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
    @abstractmethod
    def create_encoder_prompt(
        self,
2164
        prompt: str | list[int],
2165
        mm_data: MultiModalDataDict,
2166
    ) -> str | list[int]:
2167
        """
2168
        Create input prompt for the encoder. HF processor will be applied on
2169
2170
        this prompt during profiling and generation.
        """
2171
2172
        raise NotImplementedError

2173
2174
2175
2176
    @property
    def pad_dummy_encoder_prompt(self) -> bool:
        return False

2177
2178
    def create_decoder_prompt(
        self,
2179
        prompt: str | list[int],
2180
        mm_data: MultiModalDataDict,
2181
    ) -> str | list[int]:
2182
2183
2184
        """Create input prompt for the decoder."""
        return prompt

2185
    def _get_enc_dec_inputs(
2186
        self,
2187
        prompt: str | list[int],
2188
        mm_data: MultiModalDataDict,
2189
2190
        encoder_inputs: MultiModalInputs,
    ):
2191
        tokenizer = self.info.get_tokenizer()
2192
2193
        decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_data)
        if isinstance(decoder_prompt_raw, str):
2194
2195
2196
            decoder_prompt_ids = encode_tokens(
                tokenizer, decoder_prompt_raw, add_special_tokens=False
            )
2197
        else:
2198
            decoder_prompt_ids = decoder_prompt_raw
2199
2200
2201

        mm_inputs = MultiModalEncDecInputs(
            encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],
2202
2203
            **encoder_inputs,
        )
2204
        mm_inputs["prompt_token_ids"] = decoder_prompt_ids
2205
        return mm_inputs
2206
2207
2208

    def apply(
        self,
2209
        prompt: str | list[int],
2210
2211
        mm_data: MultiModalDataDict,
        hf_processor_mm_kwargs: Mapping[str, object],
2212
        tokenization_kwargs: Mapping[str, object] | None = None,
2213
        *,
2214
        mm_uuids: MultiModalUUIDDict | None = None,
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
    ) -> MultiModalEncDecInputs:
        """
        Process multi-modal inputs to be used in vLLM.
        The main processing steps are modified to fit encoder-decoder model:
        1. Create encoder prompt from input prompt text.
        2. Apply the HF processor on encoder prompt.
        3. Copy the input prompt text as decoder prompt inputs.
        """
        encoder_prompt = self.create_encoder_prompt(prompt, mm_data)
        encoder_inputs = super().apply(
            encoder_prompt,
            mm_data,
            hf_processor_mm_kwargs,
2228
            tokenization_kwargs,
2229
            mm_uuids=mm_uuids,
2230
2231
2232
2233
2234
2235
2236
        )

        return self._get_enc_dec_inputs(
            prompt=prompt,
            mm_data=mm_data,
            encoder_inputs=encoder_inputs,
        )