processing.py 63.8 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from abc import ABC, abstractmethod
4
from collections import defaultdict
5
6
from collections.abc import (Callable, Generator, ItemsView, Iterable, Mapping,
                             Sequence)
7
from dataclasses import dataclass, field, replace
8
from enum import Enum
9
from functools import lru_cache
10
from typing import (TYPE_CHECKING, Generic, NamedTuple, Optional, Protocol,
11
                    TypeVar, Union, cast)
12

13
import regex as re
14
import torch
15
from typing_extensions import assert_never
16

17
from vllm.inputs import InputProcessingContext
18
from vllm.logger import init_logger
19
20
from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
                                               encode_tokens)
21
from vllm.utils import flatten_2d_lists, full_groupby
22

23
from .hasher import MultiModalHasher
24
from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
25
26
                     MultiModalFieldConfig, MultiModalInputs,
                     MultiModalKwargsItem, MultiModalKwargsItems,
27
28
                     MultiModalKwargsOptionalItems, MultiModalUUIDDict,
                     PlaceholderRange)
29
30
from .parse import (DictEmbeddingItems, EmbeddingItems, MultiModalDataItems,
                    MultiModalDataParser)
31
32

if TYPE_CHECKING:
33
34
35
36
    from transformers.configuration_utils import PretrainedConfig
    from transformers.feature_extraction_utils import BatchFeature
    from transformers.processing_utils import ProcessorMixin

37
    from .cache import BaseMultiModalProcessorCache
38
    from .profiling import BaseDummyInputsBuilder
39

40
logger = init_logger(__name__)
41
42

_S = TypeVar("_S", str, list[int])
43
44
45

PromptSeq = Union[str, list[int]]
"""A token sequence (list of token IDs) or text."""
46

47

48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
@lru_cache(maxsize=2048)
def _cached_encode(
    tokenizer: AnyTokenizer,
    text: str,
    *,
    add_special_tokens: Optional[bool] = None,
) -> list[int]:
    return encode_tokens(tokenizer,
                         text,
                         add_special_tokens=add_special_tokens)


@lru_cache(maxsize=2048)
def _cached_decode(
    tokenizer: AnyTokenizer,
    token_ids: tuple[int, ...],
    *,
    skip_special_tokens: Optional[bool] = None,
) -> str:
    return decode_tokens(tokenizer,
                         list(token_ids),
                         skip_special_tokens=skip_special_tokens)


def _seq2text(tokenizer: AnyTokenizer, seq: PromptSeq) -> str:
    if isinstance(seq, str):
        return seq

    return _cached_decode(tokenizer, tuple(seq))


def _seq2tokens(tokenizer: AnyTokenizer, seq: PromptSeq) -> list[int]:
    if isinstance(seq, str):
        return _cached_encode(tokenizer, seq, add_special_tokens=False)

    return seq


86
87
88
89
90
91
92
93
94
95
96
class _GetMatchIndex(Protocol):

    def __call__(
        self,
        tokenizer: AnyTokenizer,
        prompt: PromptSeq,
        start_idx: int = 0,
    ) -> Optional[int]:
        ...


97
98
99
@dataclass
class PromptIndex:
    """Resolves to an index in the prompt."""
100
    get_match_index: _GetMatchIndex
101
102
103
104
105
106
107
108
109
110
111


class PromptIndexTargets:

    @staticmethod
    def start() -> PromptIndex:
        """
        Resolves to the start of the prompt (before the first token).

        This results in a match even if the prompt is empty.
        """
112
        return PromptIndex(lambda tokenizer, prompt, start_idx=0: 0)
113
114
115
116
117
118
119
120
121
122

    @staticmethod
    def prefix(seq: PromptSeq) -> PromptIndex:
        """
        Resolves to a location in the prompt after the given prefix.
        """

        def get_match_index(
            tokenizer: AnyTokenizer,
            prompt: PromptSeq,
123
            start_idx: int = 0,
124
        ) -> Optional[int]:
125
126
127
            if start_idx != 0:
                return None

128
129
130
131
132
133
134
135
136
            prefix = seq

            if isinstance(prompt, str):
                if not isinstance(prefix, str):
                    # Make both `str`
                    prefix = decode_tokens(tokenizer, prefix)
            else:
                if isinstance(prefix, str):
                    # Make both `list[int]`
137
138
139
                    prefix = encode_tokens(tokenizer,
                                           prefix,
                                           add_special_tokens=False)
140
141
142
143
144
145
146
147
148
149
150
151
152

            match_idx = len(prefix)
            return match_idx if prompt[:match_idx] == prefix else None

        return PromptIndex(get_match_index)

    @staticmethod
    def end() -> PromptIndex:
        """
        Resolves to the end of the prompt (after the last token).

        This results in a match even if the prompt is empty.
        """
153
        return PromptIndex(lambda tokenizer, prompt, start_idx=0: len(prompt))
154
155


156
UpdateTarget = Union[PromptSeq, PromptIndex]
157
158
159
160
"""
The token sequence or text to update.
"""

161
162
163
164
165
166
167
168
169
170
PromptUpdateTarget = Union[Callable[[int], UpdateTarget], UpdateTarget]
"""
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
output the corresponding token sequence (or text).

For convenience, you can directly pass in the token sequence (or text)
instead of a function if it does not depend on the input.
"""

171

172
@dataclass
173
class PromptUpdateDetails(Generic[_S]):
174
    """Details about the token sequence or text that are part of the update."""
175

176
    full: _S
177
    """The full content."""
178

179
180
    is_embed: Optional[Callable[[AnyTokenizer, PromptSeq],
                                torch.Tensor]] = None
181
    """
182
183
184
    Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full],
    return a boolean mask of shape `(len(full),)` indicating which positions
    of `full` to assign embeddings to.
185
186
187
188

    `None` (default) means to assign embeddings to all positions of `full`.

    The embeddings are obtained by calling
189
    [`SupportsMultiModal.get_multimodal_embeddings`][vllm.model_executor.models.interfaces.SupportsMultiModal.get_multimodal_embeddings].
190
191
192
    """

    @staticmethod
193
    def from_seq(seq: _S) -> "PromptUpdateDetails[_S]":
194
195
196
197
198
199
200
201
        return PromptUpdateDetails(full=seq)

    @staticmethod
    def select_text(
        seq: _S,
        embed_text: str,
    ) -> "PromptUpdateDetails[_S]":

202
203
204
        def is_embed(tokenizer: AnyTokenizer, full: PromptSeq) -> torch.Tensor:
            embed_token_ids = encode_tokens(tokenizer, embed_text)
            token_ids = _seq2tokens(tokenizer, full)
205
206

            return torch.isin(
207
                torch.tensor(token_ids),
208
209
210
211
212
213
214
215
216
217
                torch.tensor(embed_token_ids),
            )

        return PromptUpdateDetails(full=seq, is_embed=is_embed)

    @staticmethod
    def select_token_id(
        seq: _S,
        embed_token_id: int,
    ) -> "PromptUpdateDetails[_S]":
218
219
220
221
222
223
224

        def is_embed(tokenizer: AnyTokenizer, full: PromptSeq) -> torch.Tensor:
            token_ids = _seq2tokens(tokenizer, full)

            return torch.tensor(token_ids) == embed_token_id

        return PromptUpdateDetails(full=seq, is_embed=is_embed)
225
226


227
PromptUpdateInfo = Union[PromptSeq, PromptUpdateDetails]
228
"""
229
The token sequence or text that are part of the update.
230

231
If only part of the content corresponds to feature placeholders, you can
232
233
use [`PromptUpdateDetails`][vllm.multimodal.processing.PromptUpdateDetails] to
specify which part.
234
"""
235

236
237
238
PromptUpdateContent = Union[Callable[[int], PromptUpdateInfo],
                            PromptUpdateInfo]
"""
239
240
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
241
242
243
244
245
246
247
248
249
250
251
252
253
output the corresponding token sequence (or text).

For convenience, you can directly pass in the token sequence (or text)
instead of a function if it does not depend on the input.
"""


class UpdateMode(str, Enum):
    INSERT = "insert"
    REPLACE = "replace"


@dataclass
254
class PromptUpdate(ABC):
255
256
257
258
259
260
261
    """
    Defines how to update a prompt with placeholder tokens.
    """

    modality: str
    """The modality for which the update is made."""

262
    target: PromptUpdateTarget
263
264
265
266
267
268
269
270
271
272
273
274
275
276
    """The token sequence (or text) to update."""

    @property
    @abstractmethod
    def content(self) -> PromptUpdateContent:
        """The placeholder tokens that are part of the update."""
        raise NotImplementedError

    @property
    @abstractmethod
    def mode(self) -> UpdateMode:
        """Defines how to update the prompt."""
        raise NotImplementedError

277
    def _resolve_target(self, item_idx: int) -> UpdateTarget:
278
279
280
281
        target = self.target
        if callable(target):
            target = target(item_idx)

282
        return target
283

284
    def _resolve_content(self, item_idx: int) -> PromptUpdateDetails:
285
286
287
288
289
290
291
        content = self.content
        if callable(content):
            content = content(item_idx)

        if not isinstance(content, PromptUpdateDetails):
            content = PromptUpdateDetails.from_seq(content)

292
        return content
293

294
    def resolve(self, item_idx: int) -> "ResolvedPromptUpdate":
295
296
297
298
299
300
301
302
303
        """
        Given the index of the processed item within
        [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
        output a copy of this object with its lazy attributes resolved.
        """
        return ResolvedPromptUpdate(
            modality=self.modality,
            item_idx=item_idx,
            mode=self.mode,
304
305
            target=self._resolve_target(item_idx),
            content=self._resolve_content(item_idx),
306
307
        )

308

309
@dataclass
310
311
312
313
314
315
class PromptInsertion(PromptUpdate):
    """
    Defines how to insert placeholder tokens into a prompt.

    Example:

316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
    For each image, insert a number of ``<image>`` feature placeholders
    equal to the feature size of the vision encoder after the ``<s>`` token:

    ```python
    PromptInsertion(
        modality="image",
        target="<s>",
        insertion="<image>" * image_feature_size,
    )
    ```

    Insert these tokens at the start of the prompt:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.start(),
        insertion="<image>" * image_feature_size,
    )
    ```

    Insert these tokens after a prefix ``Images:``:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.prefix("Images:"),
        insertion="<image>" * image_feature_size,
    )
    ```

    Insert these tokens at the end of the prompt:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.end(),
        insertion="<image>" * image_feature_size,
    )
    ```
356
357
358
359
    """

    insertion: PromptUpdateContent = field(repr=False)
    """
360
361
362
363
    Given the index of the processed item within
    [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
    output the token sequence (or text) to insert right after
    [`target`][vllm.multimodal.processing.PromptUpdate.target].
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379

    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
    """

    @property
    def content(self) -> PromptUpdateContent:
        return self.insertion

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.INSERT


@dataclass
class PromptReplacement(PromptUpdate):
380
381
    """
    Defines how to replace portions of an input prompt with placeholder tokens.
382
383
384

    Example:

385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
    For each image, replace one ``<image>`` input placeholder in the prompt
    with a number of ``<image>`` feature placeholders
    equal to the feature size of the vision encoder:

    ```python
    PromptReplacement(
        modality="image",
        target="<image>",
        replacement="<image>" * image_feature_size,
    )
    ```

    As above, but further pad the feature placeholders with ``<image_bos>``
    and `<image_eos>``, which are not supposed to be passed to the vision
    encoder:

    ```python
    PromptReplacement(
        modality="image",
        target="<image>",
        replacement=PromptUpdateDetails(
            full="".join([
                "<image_bos>",
                "<image>" * image_feature_size,
                "<image_eos>",
            ]),
            features="<image>" * image_feature_size,
        ),
    )
    ```

    To avoid unnecessary tokenization during prompt replacement,
    we recommended passing token sequences instead of text:

    ```python
    PromptReplacement(
        modality="image",
        target=[image_token_id],
        replacement=PromptUpdateDetails(
            full=([image_bos_id] + [image_token_id] * image_feature_size
                    + [image_eos_id]),
            features=[image_token_id] * image_feature_size,
        ),
    )
    ```
430
431
    """

432
    replacement: PromptUpdateContent = field(repr=False)
433
    """
434
435
436
437
    Given the index of the processed item within
    [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
    output the token sequence (or text) to replace
    [`target`][vllm.multimodal.processing.PromptUpdate.target].
438

439
440
    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
441
442
    """

443
444
445
446
447
448
449
    @property
    def content(self) -> PromptUpdateContent:
        return self.replacement

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.REPLACE
450
451


452
453
454
class _HasModalityAttr(Protocol):
    modality: str

455

456
class _HasModalityProp(Protocol):
457

458
459
460
461
462
463
464
465
466
    @property
    def modality(self) -> str:
        ...


_M = TypeVar("_M", bound=Union[_HasModalityAttr, _HasModalityProp])


def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]:
467
468
    """Convenience function to apply [`full_groupby`][vllm.utils.full_groupby]
    based on modality."""
469
470
471
    return full_groupby(values, key=lambda x: x.modality)


472
473
474
475
476
477
478
class PromptTargetMatch(NamedTuple):
    start_idx: int
    end_idx: int


@dataclass(frozen=True)
class ResolvedPromptUpdate:
479
    """
480
481
    A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] with its
    lazy attributes resolved, apart from those related to tokenization.
482
    """
483

484
485
    modality: str
    """The modality for which the update is made."""
486

487
488
    item_idx: int
    """The index within `modality` of the item this update pertains to."""
489

490
491
    mode: UpdateMode
    """Defines how to update the prompt."""
492

493
    target: UpdateTarget
494
    """The token sequence (or text) to update."""
495

496
    content: PromptUpdateDetails = field(repr=False)
497
    """The placeholder tokens that are part of the update."""
498

499
500
501
502
503
504
505
506
507
    def iter_token_matches(
        self,
        prompt: list[int],
        tokenizer: AnyTokenizer,
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        target = self.target
508

509
510
511
512
        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(tokenizer, prompt, start_idx)
            if match_idx is not None:
                yield PromptTargetMatch(match_idx, match_idx)
513

514
            return
515

516
517
        target_token_ids = _seq2tokens(tokenizer, target)

518
        for match in iter_token_matches(prompt,
519
                                        target_token_ids,
520
521
                                        start_idx=start_idx):
            yield PromptTargetMatch(match.start_idx, match.end_idx)
522

523
524
525
526
527
528
529
530
531
    def iter_text_matches(
        self,
        prompt: str,
        tokenizer: AnyTokenizer,
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        target = self.target
532

533
534
535
536
        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(tokenizer, prompt, start_idx)
            if match_idx is not None:
                yield PromptTargetMatch(match_idx, match_idx)
537

538
            return
539

540
541
542
        target_text = _seq2text(tokenizer, target)

        for match in re.finditer(re.escape(target_text), prompt,
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
                                 pos=start_idx):
            yield PromptTargetMatch(match.start(), match.end())

    def iter_matches(
        self,
        prompt: Union[list[int], str],
        tokenizer: AnyTokenizer,
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        if isinstance(prompt, str):
            return self.iter_text_matches(prompt,
                                          tokenizer,
                                          start_idx=start_idx)

        return self.iter_token_matches(prompt, tokenizer, start_idx=start_idx)
560

561
562
563
564
565
566
567
568
569
    def with_target(self, target: UpdateTarget):
        return replace(self, target=target)

    def with_content(self, content: PromptUpdateInfo):
        if not isinstance(content, PromptUpdateDetails):
            content = PromptUpdateDetails.from_seq(content)

        return replace(self, content=content)

570

571
572
573
class _TokenMatch(NamedTuple):
    start_idx: int
    end_idx: int
574
575


576
577
578
def iter_token_matches(
    token_ids: list[int],
    match_ids: list[int],
579
580
    *,
    start_idx: int = 0,
581
) -> Generator[_TokenMatch]:
582
    """
583
    Yield each occurrence of `match_ids` in `token_ids`.
584
585
586
587

    Note that empty matches are ignored.
    """
    prompt_len = len(token_ids)
588
    match_len = len(match_ids)
589

590
591
    if match_len == 0:
        return
592

593
    while start_idx < prompt_len - match_len + 1:
594
        end_idx = start_idx + match_len
595

596
597
        if token_ids[start_idx:end_idx] == match_ids:
            yield _TokenMatch(start_idx=start_idx, end_idx=end_idx)
598
599
600
601
602

            # Exclude overlapping matches
            start_idx = end_idx
        else:
            start_idx += 1
603
604


605
606
607
608
609
610
def replace_token_matches(
    token_ids: list[int],
    match_ids: list[int],
    new_ids: list[int],
) -> list[int]:
    """
611
612
    Replace each occurrence of `match_ids` in `token_ids`
    with `new_ids`.
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631

    Note that empty matches are ignored.
    """
    out_seqs = list[list[int]]()
    prev_end_idx = 0

    for match in iter_token_matches(token_ids, match_ids):
        start_idx = match.start_idx
        end_idx = match.end_idx

        out_seqs.append(token_ids[prev_end_idx:start_idx])
        out_seqs.append(new_ids)
        prev_end_idx = end_idx

    out_seqs.append(token_ids[prev_end_idx:])

    return flatten_2d_lists(out_seqs)


632
@dataclass
633
class PlaceholderFeaturesInfo:
634
    modality: str
635
    item_idx: int
636
    start_idx: int
637
    tokens: list[int]
638
    is_embed: Optional[torch.Tensor]
639
640
641

    @property
    def length(self) -> int:
642
        return len(self.tokens)
643
644

    def to_range(self) -> PlaceholderRange:
645
646
        # TODO: Is it worth it to optimize this by stripping the
        # leading and ending positions where `is_embed=False`?
647
648
649
        return PlaceholderRange(
            offset=self.start_idx,
            length=self.length,
650
            is_embed=self.is_embed,
651
        )
652
653


654
_MatchToApply = tuple[tuple[str, int], tuple[PromptTargetMatch, int]]
655
656


657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
def _find_matches(
    prompt: _S,
    mm_prompt_updates: "MultiModalPromptUpdates",
    tokenizer: AnyTokenizer,
    *,
    prev_end_idx: int = 0,
    current_result: "MultiModalPromptUpdatesApplyResult",
) -> tuple[Optional[UpdateMode], list[_MatchToApply]]:
    mode: Optional[UpdateMode] = None
    mm_matches = dict[tuple[str, int], tuple[PromptTargetMatch, int]]()

    for modality, modality_updates in mm_prompt_updates.items():
        for item_idx, item_updates in enumerate(modality_updates):
            if current_result[modality][item_idx] is not None:
                continue  # Updates have already been applied for this item

            for update_idx, update in enumerate(item_updates):
                if (modality, item_idx) in mm_matches:
                    break  # Already found a match for this item

                for match in update.iter_matches(
                        prompt,
                        tokenizer,
                        start_idx=prev_end_idx,
                ):
                    # All matches should share the same mode
                    if mode is None:
                        mode = update.mode
                    elif mode != update.mode:
                        continue

                    mm_matches[(modality, item_idx)] = match, update_idx
                    break  # Get only the first valid match per item

    # Prioritize earlier matches
    matches_to_apply = sorted(mm_matches.items(), key=lambda item: item[1][0])

    # To avoid conflicts, only replace one non-empty item at a time
    if mode == UpdateMode.REPLACE:
        matches_to_apply_ = list[_MatchToApply]()
        has_non_empty_matches = False

        for item in matches_to_apply:
            _, (match, _) = item
            if match.start_idx == match.end_idx:
                matches_to_apply_.append(item)
            elif not has_non_empty_matches:
                has_non_empty_matches = True
                matches_to_apply_.append(item)

        matches_to_apply = matches_to_apply_

    return mode, matches_to_apply
710
711


712
def _apply_matches(
713
    prompt: _S,
714
715
716
717
718
    mm_prompt_updates: "MultiModalPromptUpdates",
    tokenizer: AnyTokenizer,
) -> tuple[list[_S], "MultiModalPromptUpdatesApplyResult"]:
    prompt_len = len(prompt)

719
    out_seqs = list[Union[str, list[int]]]()
720
721
722
723
    out_result: MultiModalPromptUpdatesApplyResult = {
        m: [None] * len(items)
        for m, items in mm_prompt_updates.items()
    }
724

725
726
727
    start_idx = prev_end_idx = 0
    while start_idx < max(prompt_len, 1):  # Allow inserts into empty prompt
        found = False
728

729
730
731
732
733
734
735
        mode, matches_to_apply = _find_matches(
            prompt,
            mm_prompt_updates,
            tokenizer,
            prev_end_idx=prev_end_idx,
            current_result=out_result,
        )
736

737
738
739
        if mode is not None:
            for (modality, item_idx), (match, update_idx) in matches_to_apply:
                found = True
740

741
742
                matched_update = mm_prompt_updates[modality][item_idx][
                    update_idx]
743
                matched_content = matched_update.content.full
744

745
746
747
748
749
750
                if mode == UpdateMode.INSERT:
                    end_idx_to_insert = match.end_idx
                elif mode == UpdateMode.REPLACE:
                    end_idx_to_insert = match.start_idx
                else:
                    assert_never(mode)
751

752
                out_seqs.append(prompt[prev_end_idx:end_idx_to_insert])
753
754
755
756
                out_seqs.append(
                    _seq2text(tokenizer, matched_content
                              ) if isinstance(prompt, str) else _seq2tokens(
                                  tokenizer, matched_content))
757
                out_result[modality][item_idx] = update_idx
758

759
760
761
762
763
                # Exclude overlapping matches
                start_idx = prev_end_idx = match.end_idx

        if not found:
            start_idx += 1
764
765
766

    out_seqs.append(prompt[prev_end_idx:])

767
    return cast(list[_S], out_seqs), out_result
768
769


770
def apply_token_matches(
771
    prompt: list[int],
772
773
774
775
776
    mm_prompt_updates: "MultiModalPromptUpdates",
    tokenizer: AnyTokenizer,
) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
    """
    Apply the updates in `mm_prompt_updates` to `prompt`.
777

778
779
780
781
782
783
    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
    appears earlier in `mm_prompt_updates` takes priority.
    """
    token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates,
                                           tokenizer)
784

785
    return flatten_2d_lists(token_id_seqs), result
786
787


788
def apply_text_matches(
789
    prompt: str,
790
791
792
793
794
    mm_prompt_updates: "MultiModalPromptUpdates",
    tokenizer: AnyTokenizer,
) -> tuple[str, "MultiModalPromptUpdatesApplyResult"]:
    """
    Apply the updates in `mm_prompt_updates` to `prompt`.
795

796
797
798
799
800
    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
    appears earlier in `mm_prompt_updates` takes priority.
    """
    texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
801

802
    return "".join(texts), result
803
804


805
def _iter_placeholders(
806
    prompt: list[int],
807
    mm_prompt_updates: "MultiModalPromptUpdates",
808
    tokenizer: AnyTokenizer,
809
) -> Iterable[PlaceholderFeaturesInfo]:
810
    """
811
    Yield each set of placeholder tokens found in `prompt`.
812
813
814

    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
815
    appears earlier in `mm_prompt_updates` takes priority.
816

817
818
    Note that empty matches are ignored.
    """
819
    prompt_len = len(prompt)
820
821
    mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()}

822
    item_idx_by_modality = defaultdict[str, int](lambda: 0)
823
824
825
826
827

    start_idx = 0
    while start_idx < prompt_len:
        found = False

828
        for modality, modality_updates in mm_prompt_updates.items():
829
830
            item_idx = item_idx_by_modality[modality]
            if item_idx >= mm_item_counts.get(modality, 0):
831
                continue
832

833
834
            for update in modality_updates[item_idx]:
                content = update.content
835
                content_tokens_full = _seq2tokens(tokenizer, content.full)
836
837
                content_len_full = len(content_tokens_full)
                end_idx_full = start_idx + content_len_full
838

839
                if content_len_full == 0 or end_idx_full > prompt_len:
840
841
                    continue

842
                if prompt[start_idx:end_idx_full] == content_tokens_full:
843
844
                    content_is_embed = content.is_embed
                    if content_is_embed is not None:
845
846
                        content_is_embed = content_is_embed(
                            tokenizer, content.full)
847
848
849
850
851
852
853
854

                    yield PlaceholderFeaturesInfo(
                        modality=modality,
                        item_idx=item_idx,
                        start_idx=start_idx,
                        tokens=content_tokens_full,
                        is_embed=content_is_embed,
                    )
855

856
                    # Exclude overlapping matches
857
                    start_idx = end_idx_full
858
859
860
                    item_idx_by_modality[modality] += 1
                    found = True
                    break
861

862
863
            if found:
                break  # Go back to the outer while loop
864
865
866

        if not found:
            start_idx += 1
867
868


869
870
def find_mm_placeholders(
    prompt: list[int],
871
    mm_prompt_updates: "MultiModalPromptUpdates",
872
    tokenizer: AnyTokenizer,
873
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
874
    it = _iter_placeholders(prompt, mm_prompt_updates, tokenizer)
875
876
877
    return dict(full_groupby_modality(it))


878
class BaseProcessingInfo:
879
    """Base class to provide the information necessary for data processing."""
880

881
882
    def __init__(self, ctx: InputProcessingContext) -> None:
        super().__init__()
883

884
885
886
887
888
889
890
        self.ctx = ctx

    @property
    def model_id(self) -> str:
        return self.ctx.model_config.model

    def get_tokenizer(self) -> AnyTokenizer:
891
892
        return self.ctx.tokenizer

893
    def get_hf_config(self) -> "PretrainedConfig":
894
895
        return self.ctx.get_hf_config()

896
    def get_hf_processor(self, **kwargs: object) -> "ProcessorMixin":
897
898
899
900
901
902
        """
        Subclasses can override this method to handle
        specific kwargs from model config or user inputs.
        """
        return self.ctx.get_hf_processor(**kwargs)

903
904
905
906
907
908
909
910
911
912
913
914
    @abstractmethod
    def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
        """
        Return the maximum supported number of items for each modality.

        A value of `None` means unlimited number of items.

        Omitting a modality from the returned dictionary means that
        it is not supported at all.
        """
        raise NotImplementedError

915
916
917
918
919
920
921
922
923
924
925
926
927
928
    def get_allowed_mm_limits(self) -> Mapping[str, int]:
        """Return the maximum allowed number of items for each modality."""
        supported_mm_limits = self.get_supported_mm_limits()
        mm_config = self.ctx.get_mm_config()

        allowed_limits = dict[str, int]()
        for modality, supported_limit in supported_mm_limits.items():
            user_limit = mm_config.get_limit_per_prompt(modality)

            allowed_limits[modality] = (user_limit if supported_limit is None
                                        else min(user_limit, supported_limit))

        return allowed_limits

929
930
931
932
933
934
935
936
937
938
939
940
    def get_mm_max_tokens_per_item(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
    ) -> Optional[Mapping[str, int]]:
        """
        Return the maximum number of tokens per item of for each modality.
        
        When `None` (the default) is returned, vLLM will generate dummy inputs
        (images/videos) at maximum possible sizes and process them to determine
        the maximum token count per modality.

941
942
943
944
945
        This approach works but can be very slow for certain models (e.g.,
        Qwen2.5-VL), leading to very long startup time. For better performance,
        each model can override this method to return pre-computed maximum token
        counts, avoiding the need for dummy input generation and processing.

946
947
948
949
950
951
        Note:
            The maximum number of tokens per item of each modality returned 
            from this function should respect the model's maximum sequence
            length and the maximum number of items of each modality allowed,
            and agree with dummy inputs (images/videos) at maximum possible
            sizes.
952
953
954
        """
        return None

955
956

_I = TypeVar("_I", bound=BaseProcessingInfo)
957

958
959
MultiModalHashes = dict[str, list[str]]
"""
960
A collection of hashes with a similar structure as
961
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
962
963
"""

964
MultiModalPromptUpdates = Mapping[str, list[Sequence[ResolvedPromptUpdate]]]
965
966
967
968
969
"""
A collection of prompt updates with a similar structure as
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
"""

970
971
972
973
974
975
976
977
MultiModalPromptUpdatesApplyResult = Mapping[str, list[Optional[int]]]
"""
For an item `MultiModalPromptUpdates[k][i]`,
`MultiModalPromptUpdatesApplyResult[k][i]` represents the index of the
`ResolvedPromptUpdate` instance that has been applied, or `None` if none of the
`ResolvedPromptUpdate` instances have been applied.
"""

978
979

class MultiModalProcessingInfo(NamedTuple):
980
    kwargs: MultiModalKwargsOptionalItems
981
    hashes: MultiModalHashes
982
983
    prompt_updates: MultiModalPromptUpdates

984
985

class BaseMultiModalProcessor(ABC, Generic[_I]):
986
    """
987
    Abstract base class to process multi-modal inputs to be used in vLLM.
988

989
    Not to be confused with `transformers.ProcessorMixin`.
990
991
    """

992
993
994
995
996
997
998
    def __init__(
        self,
        info: _I,
        dummy_inputs: "BaseDummyInputsBuilder[_I]",
        *,
        cache: Optional["BaseMultiModalProcessorCache"] = None,
    ) -> None:
999
1000
        super().__init__()

1001
1002
        self.info = info
        self.dummy_inputs = dummy_inputs
1003
        self.cache = cache
1004

1005
1006
        self.data_parser = self._get_data_parser()

1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
        # Avoid unnecessary recomputation
        self._supported_mm_limits = self.info.get_supported_mm_limits()
        self._allowed_mm_limits = self.info.get_allowed_mm_limits()

    @property
    def supported_mm_limits(self):
        return self._supported_mm_limits

    @property
    def allowed_mm_limits(self):
        return self._allowed_mm_limits

1019
    def __call__(
1020
        self,
1021
1022
        prompt: str,
        mm_data: MultiModalDataDict,
1023
        hf_processor_mm_kwargs: Mapping[str, object],
1024
        *,
1025
1026
        mm_hash_overrides: Optional[Union[dict[str, list[str]],
                                          MultiModalUUIDDict]] = None,
1027
    ) -> MultiModalInputs:
1028
1029
1030
1031
        return self.apply(prompt,
                          mm_data,
                          hf_processor_mm_kwargs,
                          mm_hash_overrides=mm_hash_overrides)
1032

1033
1034
    def _get_data_parser(self) -> MultiModalDataParser:
        """
1035
        Construct a parser to preprocess multi-modal data items
1036
1037
        before passing them to
        [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
1038
1039

        You can support additional modalities by creating a subclass
1040
1041
        of [`MultiModalDataParser`][vllm.multimodal.parse.MultiModalDataParser]
        that has additional subparsers.
1042
1043
1044
        """
        return MultiModalDataParser()

1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
    def validate_num_items(
        self,
        modality: str,
        num_items: int,
    ) -> None:
        supported_limit = self.supported_mm_limits.get(modality, 0)
        allowed_limit = self.allowed_mm_limits.get(modality, 0)

        if supported_limit is None:
            supported_limit = allowed_limit

        limit = min(supported_limit, allowed_limit)

        if num_items > limit:
            msg = (f"At most {limit} {modality}(s) may be provided in "
                   "one prompt.")

            if num_items <= supported_limit:
                msg += " Set `--limit-mm-per-prompt` to increase this limit."

            raise ValueError(msg)

1067
    def _to_mm_items(
1068
1069
1070
        self,
        mm_data: MultiModalDataDict,
    ) -> MultiModalDataItems:
1071
        """
1072
1073
1074
1075
1076
        Normalize
        [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict]
        to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]
        before passing them to
        [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
1077
        """
1078
        mm_items = self.data_parser.parse_mm_data(mm_data)
1079
1080

        for modality, items in mm_items.items():
1081
            self.validate_num_items(modality, len(items))
1082
1083

        return mm_items
1084

1085
1086
1087
    @abstractmethod
    def _get_mm_fields_config(
        self,
1088
        hf_inputs: "BatchFeature",
1089
1090
1091
1092
1093
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, MultiModalFieldConfig]:
        """Given the HF-processed data, output the metadata of each field."""
        raise NotImplementedError

1094
    @abstractmethod
1095
    def _get_prompt_updates(
1096
        self,
1097
        mm_items: MultiModalDataItems,
1098
        hf_processor_mm_kwargs: Mapping[str, object],
1099
        out_mm_kwargs: MultiModalKwargsItems,
1100
    ) -> Sequence[PromptUpdate]:
1101
1102
        """
        Given the original multi-modal items for this modality
1103
        and HF-processed data, output the updates to perform.
1104

1105
1106
1107
1108
1109
1110
        The information returned by this method is used to update token inputs
        which bypass the HF processor. It is also used to update the output of
        HF processor if the HF process does not apply prompt updates to text
        inputs.

        Moreover, this information is critical to determine the token positions
1111
1112
        in order to construct
        [`PlaceholderRange`][vllm.multimodal.inputs.PlaceholderRange]
1113
        for each multi-modal item.
1114
1115
        """
        raise NotImplementedError
1116

1117
1118
1119
1120
1121
1122
    def _bind_and_group_updates(
        self,
        prompt_updates: Sequence[PromptUpdate],
        mm_item_counts: Mapping[str, int],
    ) -> MultiModalPromptUpdates:
        return {
1123
1124
            modality: [[update.resolve(item_idx) for update in updates]
                       for item_idx in range(mm_item_counts.get(modality, 0))]
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
            for modality, updates in full_groupby_modality(prompt_updates)
        }

    def _get_mm_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        out_mm_kwargs: MultiModalKwargsItems,
    ) -> MultiModalPromptUpdates:
        unbound_prompt_updates = self._get_prompt_updates(
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
            out_mm_kwargs=out_mm_kwargs,
        )

        mm_prompt_updates = self._bind_and_group_updates(
            unbound_prompt_updates,
            mm_items.get_all_counts(),
        )

        for modality, prompt_updates in mm_prompt_updates.items():
            for item_idx, item_prompt_updates in enumerate(prompt_updates):
                if len(item_prompt_updates) > 1:
                    logger.warning_once(
                        "Detected %d prompt updates for `mm_items[%r][%s]`. "
                        "Multiple prompt updates per item is now "
                        "deprecated and may be removed in v0.13. "
                        "Instead, please specify dynamic update targets "
                        "in the same prompt update definition by passing "
                        "a function to `PromptUpdate.target`.",
                        len(prompt_updates),
                        modality,
                        item_idx,
                    )

        return mm_prompt_updates

1162
    def _find_mm_placeholders(
1163
1164
        self,
        new_token_ids: list[int],
1165
        mm_prompt_updates: MultiModalPromptUpdates,
1166
    ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
1167
1168
1169
1170
        tokenizer = self.info.get_tokenizer()

        return find_mm_placeholders(new_token_ids, mm_prompt_updates,
                                    tokenizer)
1171

1172
    def _get_hf_mm_data(
1173
        self,
1174
        mm_items: MultiModalDataItems,
1175
1176
1177
    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
        processor_data = dict[str, object]()
        passthrough_data = dict[str, object]()
1178

1179
1180
1181
        for items in mm_items.values():
            processor_data.update(items.get_processor_data())
            passthrough_data.update(items.get_passthrough_data())
1182

1183
1184
        return processor_data, passthrough_data

1185
1186
1187
    def _call_hf_processor(
        self,
        prompt: str,
1188
1189
1190
1191
        # Not to be confused with `mm_data` in `self.apply`.
        # This refers to the data to be passed to HF processor.
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
1192
        tok_kwargs: Mapping[str, object],
1193
    ) -> "BatchFeature":
1194
1195
1196
1197
        """
        Call the HF processor on the prompt text and
        associated multi-modal data.
        """
1198
1199
        return self.info.ctx.call_hf_processor(
            self.info.get_hf_processor(**mm_kwargs),
1200
            dict(text=prompt, **mm_data),
1201
            dict(**mm_kwargs, **tok_kwargs),
1202
1203
        )

1204
    def _hf_processor_applies_updates(
1205
1206
1207
1208
        self,
        prompt_text: str,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1209
        tokenization_kwargs: Mapping[str, object],
1210
1211
    ) -> bool:
        """
1212
        Return whether the HF processor applies prompt updates.
1213

1214
1215
        For most HF processors, this should be `True` when multi-modal
        data items are passed, but `False` when multi-modal embeddings
1216
1217
1218
1219
1220
1221
        are passed.
        """
        return not any(
            isinstance(items, (EmbeddingItems, DictEmbeddingItems))
            for items in mm_items.values())

1222
    def _apply_hf_processor_text_mm(
1223
        self,
1224
        prompt_text: str,
1225
        mm_items: MultiModalDataItems,
1226
        hf_processor_mm_kwargs: Mapping[str, object],
1227
        tokenization_kwargs: Mapping[str, object],
1228
    ) -> tuple[list[int], "BatchFeature", bool]:
1229
        """
1230
1231
        Apply the HF processor on the prompt text and multi-modal data
        together.
1232

1233
        In addition, return whether prompt updates have been applied.
1234
1235
1236
1237
1238
1239
1240
        """
        processor_data, passthrough_data = self._get_hf_mm_data(mm_items)

        processed_data = self._call_hf_processor(
            prompt=prompt_text,
            mm_data=processor_data,
            mm_kwargs=hf_processor_mm_kwargs,
1241
            tok_kwargs=tokenization_kwargs,
1242
1243
        )
        processed_data.update(passthrough_data)
1244

1245
        prompt_ids, = processed_data.pop("input_ids").tolist()
1246

1247
        is_update_applied = self._hf_processor_applies_updates(
1248
1249
1250
            prompt_text=prompt_text,
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1251
            tokenization_kwargs=tokenization_kwargs,
1252
1253
        )

1254
        return prompt_ids, processed_data, is_update_applied
1255

1256
    def _apply_hf_processor_text_only(
1257
1258
1259
1260
        self,
        prompt_text: str,
        tokenization_kwargs: Mapping[str, object],
    ) -> list[int]:
1261
        """
1262
        Apply the HF processor on the prompt text only.
1263

1264
1265
1266
        Since HF processor requires that text and multi-modal items
        correspond to each other, we create dummy multi-modal items
        to go along with the text.
1267
        """
1268
        prompt_ids, _, _ = self._apply_hf_processor_text_mm(
1269
1270
1271
            prompt_text=prompt_text,
            mm_items=MultiModalDataItems({}),
            hf_processor_mm_kwargs={},
1272
            tokenization_kwargs=tokenization_kwargs,
1273
1274
        )

1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
        return prompt_ids

    def _apply_hf_processor_tokens_only(
        self,
        prompt_tokens: list[int],
    ) -> list[int]:
        """
        Apply the HF processor on the prompt tokens only.

        Most HF processors accept prompt text but not prompt tokens.
        If the HF processor adds or removes tokens that are not related to
        multi-modal data, you should override this method so it is consistent
1287
1288
1289
        with the output of
        [`_apply_hf_processor_text_only`][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_text_only]
        on the
1290
1291
1292
1293
1294
1295
1296
1297
        corresponding text.
        """
        return prompt_tokens

    def _apply_hf_processor_mm_only(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1298
        tokenization_kwargs: Mapping[str, object],
1299
    ) -> "BatchFeature":
1300
1301
1302
1303
1304
        """
        Apply the HF processor on the multi-modal data only.

        Since HF processor requires that text and multi-modal items
        correspond to each other, we generate dummy text using
1305
1306
        [`DummyInputsBuilder`][vllm.multimodal.profiling.BaseDummyInputsBuilder]
        to go along with the multi-modal data.
1307
1308
1309
        """
        mm_counts = mm_items.get_all_counts()

1310
        _, mm_processed_data, _ = self._apply_hf_processor_text_mm(
1311
            prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
1312
1313
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1314
            tokenization_kwargs=tokenization_kwargs,
1315
1316
        )

1317
        return mm_processed_data
1318
1319
1320
1321
1322
1323

    def _apply_hf_processor_main(
        self,
        prompt: Union[str, list[int]],
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1324
        tokenization_kwargs: Mapping[str, object],
1325
        *,
1326
        enable_hf_prompt_update: bool,
1327
    ) -> tuple[list[int], "BatchFeature", bool]:
1328
1329
1330
        """
        Apply the HF processor on the prompt text and multi-modal data.

1331
        In addition, return whether prompt updates have been applied
1332
        (for most HF processors, this should be `True`).
1333

1334
        Note:
1335
            If `enable_hf_prompt_update=False`, we use HF processor
1336
            to perform prompt updates if available; HF processor requires
1337
            that the prompt corresponds to multi-modal items.
1338
1339
        """
        if isinstance(prompt, str):
1340
            if enable_hf_prompt_update:
1341
1342
1343
1344
                return self._apply_hf_processor_text_mm(
                    prompt_text=prompt,
                    mm_items=mm_items,
                    hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1345
                    tokenization_kwargs=tokenization_kwargs,
1346
1347
                )

1348
1349
            prompt_ids = self._apply_hf_processor_text_only(
                prompt, tokenization_kwargs)
1350
1351
1352
        else:
            prompt_ids = self._apply_hf_processor_tokens_only(prompt)

1353
        mm_processed_data = self._apply_hf_processor_mm_only(
1354
            mm_items=mm_items,
1355
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1356
            tokenization_kwargs=tokenization_kwargs,
1357
1358
        )

1359
        return prompt_ids, mm_processed_data, False
1360

1361
    def _hash_mm_items(
1362
1363
1364
1365
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        tokenization_kwargs: Mapping[str, object],
1366
1367
1368
        *,
        mm_hash_overrides: Optional[Union[dict[str, list[str]],
                                          MultiModalUUIDDict]] = None,
1369
    ) -> MultiModalHashes:
1370
1371
        """Create MM hashes to be returned (only used in V1).

1372

1373
1374
1375
        Note: When overrides are provided via callers of `apply`,
        `_hash_mm_items` will be bypassed and the overrides will be used.
        """
1376
1377
        model_id = self.info.model_id

1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
        hashes: MultiModalHashes = {}
        mm_hash_overrides = mm_hash_overrides or {}

        for modality, items in mm_items.items():
            if modality in mm_hash_overrides:
                mm_hashes = mm_hash_overrides[modality]
                if isinstance(mm_hashes, str):
                    mm_hashes = [mm_hashes]

                # For None entries, compute a hash; otherwise, use provided ID.
                computed: list[str] = []
                for i, item in enumerate(items):
                    mm_hash = mm_hashes[i]

                    # NOTE: Even if a mm_hash is provided, we still compute a
                    # hash if `hf_processor_mm_kwargs` or `tokenization_kwargs`
                    # are provided. This is because the processed multimodal
                    # inputs can be different depending on the processor kwargs.
                    if mm_hash is None or \
                        hf_processor_mm_kwargs or \
                        tokenization_kwargs:

                        # NOTE: use provided hash string to hash with kwargs
                        # if available for better performance.
                        item = mm_hash if mm_hash is not None else item
                        computed.append(
                            MultiModalHasher.hash_kwargs(
                                model_id=model_id,
                                **{modality: item},
                                **hf_processor_mm_kwargs,
                                **tokenization_kwargs))
                    else:
                        computed.append(mm_hash)
                hashes[modality] = computed
            else:
                hashes[modality] = [
                    MultiModalHasher.hash_kwargs(model_id=model_id,
                                                 **{modality: item},
                                                 **hf_processor_mm_kwargs,
                                                 **tokenization_kwargs)
                    for item in items
                ]

        return hashes
1422

1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
    def _get_cache_missing_items(
        self,
        cache: "BaseMultiModalProcessorCache",
        mm_data_items: MultiModalDataItems,
        mm_hashes: MultiModalHashes,
    ) -> MultiModalDataItems:
        mm_is_cached = {
            modality: cache.is_cached(hashes)
            for modality, hashes in mm_hashes.items()
        }

        mm_missing_idxs = {
            modality: [
                idx for idx, item_is_cached in enumerate(items_is_cached)
                if not item_is_cached
            ]
            for modality, items_is_cached in mm_is_cached.items()
        }
        mm_missing_data = {
            modality: [mm_data_items[modality][idx] for idx in idxs]
            for modality, idxs in mm_missing_idxs.items()
        }

        return self._to_mm_items(mm_missing_data)

    def _recompute_cached_prompt_update(
        self,
        cached_update: ResolvedPromptUpdate,
        new_item_idx: int,
    ) -> ResolvedPromptUpdate:
        """
        Override this if other attributes of `ResolvedPromptUpdate`
        also need to be recomputed after retrieving from the cache.
        """
        return replace(cached_update, item_idx=new_item_idx)

1459
1460
    def _merge_mm_kwargs(
        self,
1461
1462
        cache: "BaseMultiModalProcessorCache",
        mm_hashes: MultiModalHashes,
1463
        mm_missing_kwargs: MultiModalKwargsItems,
1464
1465
1466
1467
1468
1469
1470
1471
1472
        mm_missing_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[MultiModalKwargsOptionalItems, MultiModalPromptUpdates]:
        # Need to calculate this at the beginning to avoid skipping cache logic
        # for subsequently repeated items in the same modality
        mm_is_cached = {
            modality: cache.is_cached(hashes)
            for modality, hashes in mm_hashes.items()
        }

1473
        mm_missing_next_idx = defaultdict[str, int](lambda: 0)
1474

1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
        merged_kwargs = defaultdict[str,
                                    list[Optional[MultiModalKwargsItem]]](list)
        merged_prompt_updates = defaultdict[
            str, list[Sequence[ResolvedPromptUpdate]]](list)
        for modality, hashes in mm_hashes.items():
            missing_kwargs = mm_missing_kwargs.get(modality, [])
            missing_prompt_updates = mm_missing_prompt_updates.get(
                modality, [])

            for item_idx, item_hash in enumerate(hashes):
                kwargs: Optional[MultiModalKwargsItem]
                if not mm_is_cached[modality][item_idx]:
                    missing_next_idx = mm_missing_next_idx[modality]
                    kwargs = missing_kwargs[missing_next_idx]
                    updates = missing_prompt_updates[missing_next_idx]

1491
                    mm_missing_next_idx[modality] += 1
1492
1493

                    item = kwargs, updates
1494
                else:
1495
1496
1497
1498
1499
1500
1501
1502
1503
                    item = None

                kwargs, updates = cache.get_and_update_item(item, item_hash)

                merged_kwargs[modality].append(kwargs)
                merged_prompt_updates[modality].append([
                    self._recompute_cached_prompt_update(update, item_idx)
                    for update in updates
                ])
1504

1505
1506
        mm_kwargs = MultiModalKwargsItems(merged_kwargs)
        mm_prompt_updates = dict(merged_prompt_updates)
1507

1508
        return mm_kwargs, mm_prompt_updates
1509
1510
1511
1512
1513
1514

    def _apply_hf_processor(
        self,
        prompt: Union[str, list[int]],
        mm_data_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1515
        tokenization_kwargs: Mapping[str, object],
1516
        *,
1517
1518
        mm_hash_overrides: Optional[Union[dict[str, list[str]],
                                          MultiModalUUIDDict]] = None,
1519
    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1520
1521
        (
            prompt_ids,
1522
            mm_processed_data,
1523
1524
1525
1526
1527
            is_update_applied,
        ) = self._apply_hf_processor_main(
            prompt=prompt,
            mm_items=mm_data_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1528
            tokenization_kwargs=tokenization_kwargs,
1529
1530
1531
            enable_hf_prompt_update=True,
        )

1532
        mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
1533
1534
1535
1536
1537
            mm_processed_data,
            self._get_mm_fields_config(mm_processed_data,
                                       hf_processor_mm_kwargs),
        )

1538
        # Use overrides if provided; fallback to data-dependent hashing.
1539
1540
1541
1542
        mm_hashes = self._hash_mm_items(mm_data_items,
                                        hf_processor_mm_kwargs,
                                        tokenization_kwargs,
                                        mm_hash_overrides=mm_hash_overrides)
1543

1544
        mm_prompt_updates = self._get_mm_prompt_updates(
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
            mm_data_items,
            hf_processor_mm_kwargs,
            mm_kwargs,
        )

        mm_info = MultiModalProcessingInfo(
            kwargs=mm_kwargs,
            hashes=mm_hashes,
            prompt_updates=mm_prompt_updates,
        )

        return prompt_ids, mm_info, is_update_applied
1557

1558
1559
    def _cached_apply_hf_processor(
        self,
1560
        prompt: Union[str, list[int]],
1561
1562
        mm_data_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1563
        tokenization_kwargs: Mapping[str, object],
1564
        *,
1565
1566
        mm_hash_overrides: Optional[Union[dict[str, list[str]],
                                          MultiModalUUIDDict]] = None,
1567
    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1568
1569
1570
1571
1572
1573
        """
        Apply the HF processor on the full prompt text,
        caching the results and reusing cached results.
        """
        cache = self.cache

1574
1575
        _, passthrough_data = self._get_hf_mm_data(mm_data_items)
        if cache is None or passthrough_data:
1576
            return self._apply_hf_processor(
1577
                prompt=prompt,
1578
                mm_data_items=mm_data_items,
1579
                hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1580
                tokenization_kwargs=tokenization_kwargs,
1581
                mm_hash_overrides=mm_hash_overrides,
1582
1583
            )

1584
1585
1586
1587
        mm_hashes = self._hash_mm_items(mm_data_items,
                                        hf_processor_mm_kwargs,
                                        tokenization_kwargs,
                                        mm_hash_overrides=mm_hash_overrides)
1588
1589

        mm_missing_data_items = self._get_cache_missing_items(
1590
1591
            cache=cache,
            mm_data_items=mm_data_items,
1592
            mm_hashes=mm_hashes,
1593
        )
1594

1595
        # NOTE: `prompt` does not correspond to `mm_missing_data_items`,
1596
        # so we can't apply prompt updates until the new multimodal
1597
1598
1599
        # items are combined with the cached multimodal items
        (
            prompt_ids,
1600
            mm_missing_processed_data,
1601
            is_update_applied,
1602
        ) = self._apply_hf_processor_main(
1603
            prompt=prompt,
1604
            mm_items=mm_missing_data_items,
1605
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1606
            tokenization_kwargs=tokenization_kwargs,
1607
            enable_hf_prompt_update=False,
1608
1609
        )

1610
        mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
1611
1612
1613
1614
1615
            mm_missing_processed_data,
            self._get_mm_fields_config(mm_missing_processed_data,
                                       hf_processor_mm_kwargs),
        )

1616
1617
1618
1619
        mm_missing_prompt_updates = self._get_mm_prompt_updates(
            mm_missing_data_items,
            hf_processor_mm_kwargs,
            mm_missing_kwargs,
1620
        )
1621

1622
1623
1624
1625
1626
        mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
            cache,
            mm_hashes=mm_hashes,
            mm_missing_kwargs=mm_missing_kwargs,
            mm_missing_prompt_updates=mm_missing_prompt_updates,
1627
1628
1629
1630
        )

        mm_info = MultiModalProcessingInfo(
            kwargs=mm_kwargs,
1631
            hashes=mm_hashes,
1632
1633
            prompt_updates=mm_prompt_updates,
        )
1634

1635
        return prompt_ids, mm_info, is_update_applied
1636

1637
1638
1639
    def _apply_token_matches(
        self,
        prompt: list[int],
1640
1641
1642
1643
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
        tokenizer = self.info.get_tokenizer()
        return apply_token_matches(prompt, mm_prompt_updates, tokenizer)
1644
1645
1646
1647

    def _apply_text_matches(
        self,
        prompt: str,
1648
1649
1650
1651
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[str, MultiModalPromptUpdatesApplyResult]:
        tokenizer = self.info.get_tokenizer()
        return apply_text_matches(prompt, mm_prompt_updates, tokenizer)
1652

1653
    def _apply_prompt_updates(
1654
1655
        self,
        token_ids: list[int],
1656
        mm_prompt_updates: MultiModalPromptUpdates,
1657
    ) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
1658
        tokenizer = self.info.get_tokenizer()
1659

1660
1661
1662
1663
        new_token_ids, match_result = self._apply_token_matches(
            token_ids,
            mm_prompt_updates,
        )
1664
1665
1666
1667
1668
1669
1670
1671
1672

        # If the search text does not represent a special token,
        # it may have different token IDs in the prompt, because
        # the tokens may go across the boundaries of the search text.
        # ----
        # e.g. when searching for "foo" in "food", if "food" itself makes
        # up a token, then the token ID of "foo" will not appear at all
        # ----
        # Since it is inefficient to search for all possible tokenizations
1673
1674
        # of the search text in the prompt, we instead perform string-based
        # updates on the decoded token IDs, then encode them back.
1675
        if all(
1676
1677
1678
1679
1680
1681
1682
                all(update_idx is not None for update_idx in update_idxs)
                for update_idxs in match_result.values()):
            new_text = decode_tokens(tokenizer, new_token_ids)
        else:
            new_text, match_result = self._apply_text_matches(
                decode_tokens(tokenizer, token_ids),
                mm_prompt_updates,
1683
1684
            )

1685
1686
1687
1688
            new_token_ids = encode_tokens(
                tokenizer,
                new_text,
                add_special_tokens=False,
1689
1690
            )

1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
        matched_updates = defaultdict[
            str, list[Sequence[ResolvedPromptUpdate]]](list)
        for modality, update_idxs in match_result.items():
            for item_idx, update_idx in enumerate(update_idxs):
                assert update_idx is not None, (
                    "Failed to apply prompt replacement for "
                    f"mm_items[{modality!r}][{item_idx}]")

                matched_updates[modality].append(
                    [mm_prompt_updates[modality][item_idx][update_idx]])
1701
1702

        placeholders = self._find_mm_placeholders(
1703
1704
            new_token_ids,
            dict(matched_updates),
1705
        )
1706

1707
        return new_token_ids, new_text, placeholders
1708

1709
1710
    def _validate_mm_kwargs(
        self,
1711
        mm_kwargs: MultiModalKwargsOptionalItems,
1712
1713
1714
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
1715
            items = mm_kwargs.get(modality, [])
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728

            if len(items) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} {modality} items in "
                    f"keyword arguments corresponding to {item_count} "
                    f"{modality} data items, but only found {len(items)}! "
                    "There is likely a problem with your "
                    "implementation of merged multi-modal processor for this "
                    "model (usually arising from an inconsistency between "
                    "`_call_hf_processor` and `_get_mm_fields_config`).")

    def _validate_mm_placeholders(
        self,
1729
        mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
1730
        mm_item_counts: Mapping[str, int],
1731
    ) -> None:
1732
1733
1734
        for modality, item_count in mm_item_counts.items():
            placeholders = mm_placeholders.get(modality, [])

1735
            if len(placeholders) != item_count:
1736
1737
1738
                # NOTE: If you are a model developer, this can also arise from
                # an inconsistency between `_call_hf_processor` and
                # `_get_mm_fields_config` implementations
1739
                raise RuntimeError(
1740
                    f"Expected there to be {item_count} prompt updates "
1741
                    f"corresponding to {item_count} {modality} items, but "
1742
                    f"instead found {len(placeholders)} prompt updates! "
1743
1744
1745
1746
                    "This is likely because you forgot to include input "
                    "placeholder tokens (e.g., `<image>`, `<|image_pad|>`) "
                    "in the prompt. If the model has a chat template, make "
                    "sure you have applied it before calling `LLM.generate`.")
1747

1748
1749
1750
1751
    def _maybe_apply_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        prompt_ids: list[int],
1752
        mm_kwargs: MultiModalKwargsOptionalItems,
1753
        mm_prompt_updates: MultiModalPromptUpdates,
1754
1755
        is_update_applied: bool,
    ) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
1756
        mm_item_counts = mm_items.get_all_counts()
1757
1758
        self._validate_mm_kwargs(mm_kwargs, mm_item_counts)

1759
        if is_update_applied:
1760
1761
            mm_placeholders = self._find_mm_placeholders(
                prompt_ids,
1762
                mm_prompt_updates,
1763
            )
1764
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1765

1766
            tokenizer = self.info.get_tokenizer()
1767
            prompt = decode_tokens(tokenizer, prompt_ids)
1768
1769
1770
        else:
            (
                prompt_ids,
1771
                prompt,
1772
                mm_placeholders,
1773
            ) = self._apply_prompt_updates(
1774
                prompt_ids,
1775
                mm_prompt_updates,
1776
            )
1777
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1778

1779
1780
1781
1782
1783
1784
1785
        return prompt_ids, prompt, mm_placeholders

    def apply(
        self,
        prompt: Union[str, list[int]],
        mm_data: MultiModalDataDict,
        hf_processor_mm_kwargs: Mapping[str, object],
1786
        tokenization_kwargs: Optional[Mapping[str, object]] = None,
1787
        *,
1788
1789
        mm_hash_overrides: Optional[Union[dict[str, list[str]],
                                          MultiModalUUIDDict]] = None,
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
    ) -> MultiModalInputs:
        """
        Process multi-modal inputs to be used in vLLM.

        The main steps are:

        1. Apply HF Processor on prompt text and multi-modal data together,
           outputting token IDs and processed tensors.
        2. Find and update sequences in the token IDs with placeholder tokens.
           The number of placeholder tokens equals the feature size of the
           multi-modal data outputted by the multi-modal encoder.
        3. Extract information about the placeholder tokens from the
           processed token IDs.
        """
        mm_items = self._to_mm_items(mm_data)

1806
1807
1808
        if tokenization_kwargs is None:
            tokenization_kwargs = {}

1809
1810
        (
            prompt_ids,
1811
            mm_info,
1812
1813
1814
1815
1816
            is_update_applied,
        ) = self._cached_apply_hf_processor(
            prompt,
            mm_items,
            hf_processor_mm_kwargs,
1817
            tokenization_kwargs=tokenization_kwargs,
1818
            mm_hash_overrides=mm_hash_overrides,
1819
1820
        )

1821
        # NOTE: tokenization_kwargs are not required to init processor
1822
1823
1824
        prompt_ids, prompt, mm_placeholders = self._maybe_apply_prompt_updates(
            mm_items=mm_items,
            prompt_ids=prompt_ids,
1825
1826
            mm_kwargs=mm_info.kwargs,
            mm_prompt_updates=mm_info.prompt_updates,
1827
1828
1829
            is_update_applied=is_update_applied,
        )

1830
1831
1832
1833
        mm_placeholder_ranges = {
            modality: [item.to_range() for item in placeholders]
            for modality, placeholders in mm_placeholders.items()
        }
1834

1835
        return MultiModalInputs(
1836
            type="multimodal",
1837
            prompt=prompt,
1838
            prompt_token_ids=prompt_ids,
1839
1840
            mm_kwargs=mm_info.kwargs,
            mm_hashes=mm_info.hashes,
1841
            mm_placeholders=mm_placeholder_ranges,
1842
        )
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852


class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):

    @abstractmethod
    def create_encoder_prompt(
        self,
        prompt: Union[str, list[int]],
        mm_data: MultiModalDataDict,
    ) -> Union[str, list[int]]:
1853
        """
1854
        Create input prompt for the encoder. HF processor will be applied on
1855
1856
        this prompt during profiling and generation.
        """
1857
1858
        raise NotImplementedError

1859
1860
1861
1862
    @property
    def pad_dummy_encoder_prompt(self) -> bool:
        return False

1863
1864
1865
1866
1867
1868
1869
1870
    def create_decoder_prompt(
        self,
        prompt: Union[str, list[int]],
        mm_data: MultiModalDataDict,
    ) -> Union[str, list[int]]:
        """Create input prompt for the decoder."""
        return prompt

1871
    def _get_enc_dec_inputs(
1872
1873
1874
        self,
        prompt: Union[str, list[int]],
        mm_data: MultiModalDataDict,
1875
1876
        encoder_inputs: MultiModalInputs,
    ):
1877
        tokenizer = self.info.get_tokenizer()
1878
1879
        decoder_prompt = self.create_decoder_prompt(prompt, mm_data)
        if isinstance(decoder_prompt, str):
1880
            decoder_prompt_ids = encode_tokens(tokenizer,
1881
                                               decoder_prompt,
1882
1883
                                               add_special_tokens=False)
        else:
1884
1885
            decoder_prompt_ids = decoder_prompt
            decoder_prompt = decode_tokens(tokenizer, decoder_prompt)
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895

        mm_inputs = MultiModalEncDecInputs(
            encoder_prompt=encoder_inputs["prompt"],
            encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],
            **encoder_inputs)
        mm_inputs.update({
            "prompt": decoder_prompt,
            "prompt_token_ids": decoder_prompt_ids
        })
        return mm_inputs
1896
1897
1898
1899
1900
1901

    def apply(
        self,
        prompt: Union[str, list[int]],
        mm_data: MultiModalDataDict,
        hf_processor_mm_kwargs: Mapping[str, object],
1902
        tokenization_kwargs: Optional[Mapping[str, object]] = None,
1903
        *,
1904
1905
        mm_hash_overrides: Optional[Union[dict[str, list[str]],
                                          MultiModalUUIDDict]] = None,
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
    ) -> MultiModalEncDecInputs:
        """
        Process multi-modal inputs to be used in vLLM.
        The main processing steps are modified to fit encoder-decoder model:
        1. Create encoder prompt from input prompt text.
        2. Apply the HF processor on encoder prompt.
        3. Copy the input prompt text as decoder prompt inputs.
        """
        encoder_prompt = self.create_encoder_prompt(prompt, mm_data)
        encoder_inputs = super().apply(
            encoder_prompt,
            mm_data,
            hf_processor_mm_kwargs,
1919
            tokenization_kwargs,
1920
            mm_hash_overrides=mm_hash_overrides,
1921
1922
1923
1924
1925
1926
1927
        )

        return self._get_enc_dec_inputs(
            prompt=prompt,
            mm_data=mm_data,
            encoder_inputs=encoder_inputs,
        )