processor.py 62.6 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from abc import ABC, abstractmethod
4
from collections import defaultdict
5
from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence
6
from dataclasses import dataclass, field, replace
7
from enum import Enum
8
from functools import lru_cache
9
10
11
12
13
from typing import (
    TYPE_CHECKING,
    Generic,
    NamedTuple,
    Protocol,
14
    TypeAlias,
15
16
    cast,
)
17

18
import regex as re
19
import torch
20
from typing_extensions import TypeVar, assert_never
21

22
from vllm.logger import init_logger
23
from vllm.tokenizers import TokenizerLike
24
from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
25

26
27
from ..hasher import MultiModalHasher
from ..inputs import (
28
29
30
    MultiModalDataDict,
    MultiModalEncDecInputs,
    MultiModalFieldConfig,
31
    MultiModalHashes,
32
33
34
35
36
37
38
    MultiModalInputs,
    MultiModalKwargsItem,
    MultiModalKwargsItems,
    MultiModalKwargsOptionalItems,
    MultiModalUUIDDict,
    PlaceholderRange,
)
39
from ..parse import (
40
41
42
43
    DictEmbeddingItems,
    EmbeddingItems,
    MultiModalDataItems,
)
44
45
46
47
48
from .context import (
    BaseProcessingInfo,
    get_current_request_id,
    timed_preprocessor_operation,
)
49
from .dummy_inputs import BaseDummyInputsBuilder
50
51

if TYPE_CHECKING:
52
53
    from transformers.feature_extraction_utils import BatchFeature

54
    from ..cache import BaseMultiModalProcessorCache
55
56
57
58
else:
    BatchFeature = object

    BaseMultiModalProcessorCache = object
59

60
logger = init_logger(__name__)
61
62

_S = TypeVar("_S", str, list[int])
63

64

65
PromptSeq: TypeAlias = str | list[int]
66
"""A token sequence (list of token IDs) or text."""
67

68

69
70
@lru_cache(maxsize=2048)
def _cached_encode(
71
    tokenizer: TokenizerLike,
72
73
    text: str,
    *,
74
    add_special_tokens: bool = True,
75
) -> list[int]:
76
    return tokenizer.encode(text, add_special_tokens=add_special_tokens)
77
78
79
80


@lru_cache(maxsize=2048)
def _cached_decode(
81
    tokenizer: TokenizerLike,
82
83
    token_ids: tuple[int, ...],
    *,
84
    skip_special_tokens: bool = False,
85
) -> str:
86
    return tokenizer.decode(list(token_ids), skip_special_tokens=skip_special_tokens)
87
88


89
90
91
92
93
94
def _seq2text(
    tokenizer: TokenizerLike | None,
    seq: PromptSeq,
    *,
    use_cache: bool = True,
) -> str:
95
96
97
    if isinstance(seq, str):
        return seq

98
99
100
101
    if tokenizer is None:
        raise ValueError("You cannot decode tokens when `skip_tokenizer_init=True`")

    if not use_cache:
102
        return tokenizer.decode(seq)
103

104
105
106
    return _cached_decode(tokenizer, tuple(seq))


107
108
109
110
111
112
def _seq2tokens(
    tokenizer: TokenizerLike | None,
    seq: PromptSeq,
    *,
    use_cache: bool = True,
) -> list[int]:
113
    if isinstance(seq, str):
114
115
116
117
        if tokenizer is None:
            raise ValueError("You cannot encode text when `skip_tokenizer_init=True`")

        if not use_cache:
118
            return tokenizer.encode(seq, add_special_tokens=False)
119

120
121
122
123
124
        return _cached_encode(tokenizer, seq, add_special_tokens=False)

    return seq


125
126
127
class _GetMatchIndex(Protocol):
    def __call__(
        self,
128
        tokenizer: TokenizerLike | None,
129
130
        prompt: PromptSeq,
        start_idx: int = 0,
131
    ) -> int | None: ...
132
133


134
135
136
@dataclass
class PromptIndex:
    """Resolves to an index in the prompt."""
137

138
    get_match_index: _GetMatchIndex
139
140
141
142
143
144
145
146
147
148


class PromptIndexTargets:
    @staticmethod
    def start() -> PromptIndex:
        """
        Resolves to the start of the prompt (before the first token).

        This results in a match even if the prompt is empty.
        """
149
        return PromptIndex(lambda tokenizer, prompt, start_idx=0: 0)
150
151
152
153
154
155
156
157

    @staticmethod
    def prefix(seq: PromptSeq) -> PromptIndex:
        """
        Resolves to a location in the prompt after the given prefix.
        """

        def get_match_index(
158
            tokenizer: TokenizerLike | None,
159
            prompt: PromptSeq,
160
            start_idx: int = 0,
161
        ) -> int | None:
162
163
164
            if start_idx != 0:
                return None

165
166
167
            prefix = seq

            if isinstance(prompt, str):
168
169
                # Make both `str`
                prefix = _seq2text(tokenizer, prefix, use_cache=False)
170
            else:
171
172
                # Make both `list[int]`
                prefix = _seq2tokens(tokenizer, prefix, use_cache=False)
173
174
175
176
177
178
179
180
181
182
183
184
185

            match_idx = len(prefix)
            return match_idx if prompt[:match_idx] == prefix else None

        return PromptIndex(get_match_index)

    @staticmethod
    def end() -> PromptIndex:
        """
        Resolves to the end of the prompt (after the last token).

        This results in a match even if the prompt is empty.
        """
186
        return PromptIndex(lambda tokenizer, prompt, start_idx=0: len(prompt))
187
188


189
UpdateTarget: TypeAlias = PromptSeq | PromptIndex
190
191
192
193
"""
The token sequence or text to update.
"""

194
PromptUpdateTarget: TypeAlias = Callable[[int], UpdateTarget] | UpdateTarget
195
196
197
198
199
200
201
202
203
"""
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
output the corresponding token sequence (or text).

For convenience, you can directly pass in the token sequence (or text)
instead of a function if it does not depend on the input.
"""

204

205
@dataclass
206
class PromptUpdateDetails(Generic[_S]):
207
    """Details about the token sequence or text that are part of the update."""
208

209
    full: _S
210
    """The full content."""
211

212
    is_embed: Callable[[TokenizerLike | None, PromptSeq], torch.Tensor] | None = None
213
    """
214
215
216
    Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full],
    return a boolean mask of shape `(len(full),)` indicating which positions
    of `full` to assign embeddings to.
217
218
219
220

    `None` (default) means to assign embeddings to all positions of `full`.

    The embeddings are obtained by calling
221
    [`SupportsMultiModal.embed_multimodal`][vllm.model_executor.models.interfaces.SupportsMultiModal.embed_multimodal].
222
223
224
    """

    @staticmethod
225
    def from_seq(seq: _S) -> "PromptUpdateDetails[_S]":
226
227
228
229
230
231
232
        return PromptUpdateDetails(full=seq)

    @staticmethod
    def select_text(
        seq: _S,
        embed_text: str,
    ) -> "PromptUpdateDetails[_S]":
233
234
        def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
            embed_token_ids = _seq2tokens(tokenizer, embed_text, use_cache=False)
235
            token_ids = _seq2tokens(tokenizer, full)
236
237

            return torch.isin(
238
                torch.tensor(token_ids),
239
240
241
242
243
244
245
246
247
248
                torch.tensor(embed_token_ids),
            )

        return PromptUpdateDetails(full=seq, is_embed=is_embed)

    @staticmethod
    def select_token_id(
        seq: _S,
        embed_token_id: int,
    ) -> "PromptUpdateDetails[_S]":
249
        def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
250
251
252
253
254
            token_ids = _seq2tokens(tokenizer, full)

            return torch.tensor(token_ids) == embed_token_id

        return PromptUpdateDetails(full=seq, is_embed=is_embed)
255

256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
    @staticmethod
    def select_token_ids(
        seq: _S,
        embed_token_ids: list[int],
    ) -> "PromptUpdateDetails[_S]":
        def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
            token_ids = _seq2tokens(tokenizer, full)

            return torch.isin(
                torch.tensor(token_ids),
                torch.tensor(embed_token_ids),
            )

        return PromptUpdateDetails(full=seq, is_embed=is_embed)

271

272
PromptUpdateInfo: TypeAlias = PromptSeq | PromptUpdateDetails
273
"""
274
The token sequence or text that are part of the update.
275

276
If only part of the content corresponds to feature placeholders, you can
277
278
use [`PromptUpdateDetails`][vllm.multimodal.processing.PromptUpdateDetails] to
specify which part.
279
"""
280

281
PromptUpdateContent: TypeAlias = Callable[[int], PromptUpdateInfo] | PromptUpdateInfo
282
"""
283
284
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
285
286
287
288
289
290
291
292
293
294
295
296
297
output the corresponding token sequence (or text).

For convenience, you can directly pass in the token sequence (or text)
instead of a function if it does not depend on the input.
"""


class UpdateMode(str, Enum):
    INSERT = "insert"
    REPLACE = "replace"


@dataclass
298
class PromptUpdate(ABC):
299
300
301
302
303
304
305
    """
    Defines how to update a prompt with placeholder tokens.
    """

    modality: str
    """The modality for which the update is made."""

306
    target: PromptUpdateTarget
307
308
309
310
311
312
313
314
315
316
317
318
319
320
    """The token sequence (or text) to update."""

    @property
    @abstractmethod
    def content(self) -> PromptUpdateContent:
        """The placeholder tokens that are part of the update."""
        raise NotImplementedError

    @property
    @abstractmethod
    def mode(self) -> UpdateMode:
        """Defines how to update the prompt."""
        raise NotImplementedError

321
    def _resolve_target(self, item_idx: int) -> UpdateTarget:
322
323
324
325
        target = self.target
        if callable(target):
            target = target(item_idx)

326
        return target
327

328
    def _resolve_content(self, item_idx: int) -> PromptUpdateDetails:
329
330
331
332
333
334
335
        content = self.content
        if callable(content):
            content = content(item_idx)

        if not isinstance(content, PromptUpdateDetails):
            content = PromptUpdateDetails.from_seq(content)

336
        return content
337

338
    def resolve(self, item_idx: int) -> "ResolvedPromptUpdate":
339
340
341
342
343
344
345
346
347
        """
        Given the index of the processed item within
        [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
        output a copy of this object with its lazy attributes resolved.
        """
        return ResolvedPromptUpdate(
            modality=self.modality,
            item_idx=item_idx,
            mode=self.mode,
348
349
            target=self._resolve_target(item_idx),
            content=self._resolve_content(item_idx),
350
351
        )

352

353
@dataclass
354
355
356
357
358
359
class PromptInsertion(PromptUpdate):
    """
    Defines how to insert placeholder tokens into a prompt.

    Example:

360
361
    For each image, insert a number of `<image>` feature placeholders
    equal to the feature size of the vision encoder after the `<s>` token:
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380

    ```python
    PromptInsertion(
        modality="image",
        target="<s>",
        insertion="<image>" * image_feature_size,
    )
    ```

    Insert these tokens at the start of the prompt:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.start(),
        insertion="<image>" * image_feature_size,
    )
    ```

381
    Insert these tokens after a prefix `Images:`:
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.prefix("Images:"),
        insertion="<image>" * image_feature_size,
    )
    ```

    Insert these tokens at the end of the prompt:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.end(),
        insertion="<image>" * image_feature_size,
    )
    ```
400
401
402
403
    """

    insertion: PromptUpdateContent = field(repr=False)
    """
404
405
406
407
    Given the index of the processed item within
    [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
    output the token sequence (or text) to insert right after
    [`target`][vllm.multimodal.processing.PromptUpdate.target].
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423

    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
    """

    @property
    def content(self) -> PromptUpdateContent:
        return self.insertion

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.INSERT


@dataclass
class PromptReplacement(PromptUpdate):
424
425
    """
    Defines how to replace portions of an input prompt with placeholder tokens.
426
427
428

    Example:

429
430
    For each image, replace one `<image>` input placeholder in the prompt
    with a number of `<image>` feature placeholders
431
432
433
434
435
436
437
438
439
440
    equal to the feature size of the vision encoder:

    ```python
    PromptReplacement(
        modality="image",
        target="<image>",
        replacement="<image>" * image_feature_size,
    )
    ```

441
442
    As above, but further pad the feature placeholders with `<image_bos>`
    and `<image_eos>`, which are not supposed to be passed to the vision
443
444
445
446
447
448
449
    encoder:

    ```python
    PromptReplacement(
        modality="image",
        target="<image>",
        replacement=PromptUpdateDetails(
450
451
452
453
454
455
456
            full="".join(
                [
                    "<image_bos>",
                    "<image>" * image_feature_size,
                    "<image_eos>",
                ]
            ),
457
458
459
460
461
462
463
464
465
466
467
468
469
            features="<image>" * image_feature_size,
        ),
    )
    ```

    To avoid unnecessary tokenization during prompt replacement,
    we recommended passing token sequences instead of text:

    ```python
    PromptReplacement(
        modality="image",
        target=[image_token_id],
        replacement=PromptUpdateDetails(
470
471
472
            full=(
                [image_bos_id] + [image_token_id] * image_feature_size + [image_eos_id]
            ),
473
474
475
476
            features=[image_token_id] * image_feature_size,
        ),
    )
    ```
477
478
    """

479
    replacement: PromptUpdateContent = field(repr=False)
480
    """
481
482
483
484
    Given the index of the processed item within
    [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
    output the token sequence (or text) to replace
    [`target`][vllm.multimodal.processing.PromptUpdate.target].
485

486
487
    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
488
489
    """

490
491
492
493
494
495
496
    @property
    def content(self) -> PromptUpdateContent:
        return self.replacement

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.REPLACE
497
498


499
500
501
class _HasModalityAttr(Protocol):
    modality: str

502

503
504
class _HasModalityProp(Protocol):
    @property
505
    def modality(self) -> str: ...
506
507


508
_M = TypeVar("_M", bound=_HasModalityAttr | _HasModalityProp)
509
510
511


def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]:
512
513
    """
    Convenience function to apply
514
    [`full_groupby`][vllm.utils.collection_utils.full_groupby]
515
516
    based on modality.
    """
517
518
519
    return full_groupby(values, key=lambda x: x.modality)


520
521
522
523
524
525
526
class PromptTargetMatch(NamedTuple):
    start_idx: int
    end_idx: int


@dataclass(frozen=True)
class ResolvedPromptUpdate:
527
    """
528
529
    A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] with its
    lazy attributes resolved, apart from those related to tokenization.
530
    """
531

532
533
    modality: str
    """The modality for which the update is made."""
534

535
536
    item_idx: int
    """The index within `modality` of the item this update pertains to."""
537

538
539
    mode: UpdateMode
    """Defines how to update the prompt."""
540

541
    target: UpdateTarget
542
    """The token sequence (or text) to update."""
543

544
    content: PromptUpdateDetails = field(repr=False)
545
    """The placeholder tokens that are part of the update."""
546

547
548
549
    def iter_token_matches(
        self,
        prompt: list[int],
550
        tokenizer: TokenizerLike | None,
551
552
553
554
555
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        target = self.target
556

557
558
559
560
        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(tokenizer, prompt, start_idx)
            if match_idx is not None:
                yield PromptTargetMatch(match_idx, match_idx)
561

562
            return
563

564
565
        target_token_ids = _seq2tokens(tokenizer, target)

566
        for match in iter_token_matches(prompt, target_token_ids, start_idx=start_idx):
567
            yield PromptTargetMatch(match.start_idx, match.end_idx)
568

569
570
571
    def iter_text_matches(
        self,
        prompt: str,
572
        tokenizer: TokenizerLike | None,
573
574
575
576
577
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        target = self.target
578

579
580
581
582
        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(tokenizer, prompt, start_idx)
            if match_idx is not None:
                yield PromptTargetMatch(match_idx, match_idx)
583

584
            return
585

586
587
        target_text = _seq2text(tokenizer, target)

588
        for match in re.finditer(re.escape(target_text), prompt, pos=start_idx):
589
590
591
592
            yield PromptTargetMatch(match.start(), match.end())

    def iter_matches(
        self,
593
        prompt: list[int] | str,
594
        tokenizer: TokenizerLike | None,
595
596
597
598
599
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        if isinstance(prompt, str):
600
            return self.iter_text_matches(prompt, tokenizer, start_idx=start_idx)
601
602

        return self.iter_token_matches(prompt, tokenizer, start_idx=start_idx)
603

604
605
606
607
608
609
610
611
612
    def with_target(self, target: UpdateTarget):
        return replace(self, target=target)

    def with_content(self, content: PromptUpdateInfo):
        if not isinstance(content, PromptUpdateDetails):
            content = PromptUpdateDetails.from_seq(content)

        return replace(self, content=content)

613

614
615
616
class _TokenMatch(NamedTuple):
    start_idx: int
    end_idx: int
617
618


619
620
621
def iter_token_matches(
    token_ids: list[int],
    match_ids: list[int],
622
623
    *,
    start_idx: int = 0,
624
) -> Generator[_TokenMatch]:
625
    """
626
    Yield each occurrence of `match_ids` in `token_ids`.
627
628
629
630

    Note that empty matches are ignored.
    """
    prompt_len = len(token_ids)
631
    match_len = len(match_ids)
632

633
634
    if match_len == 0:
        return
635

636
    while start_idx < prompt_len - match_len + 1:
637
        end_idx = start_idx + match_len
638

639
640
        if token_ids[start_idx:end_idx] == match_ids:
            yield _TokenMatch(start_idx=start_idx, end_idx=end_idx)
641
642
643
644
645

            # Exclude overlapping matches
            start_idx = end_idx
        else:
            start_idx += 1
646
647


648
649
650
651
652
653
def replace_token_matches(
    token_ids: list[int],
    match_ids: list[int],
    new_ids: list[int],
) -> list[int]:
    """
654
655
    Replace each occurrence of `match_ids` in `token_ids`
    with `new_ids`.
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674

    Note that empty matches are ignored.
    """
    out_seqs = list[list[int]]()
    prev_end_idx = 0

    for match in iter_token_matches(token_ids, match_ids):
        start_idx = match.start_idx
        end_idx = match.end_idx

        out_seqs.append(token_ids[prev_end_idx:start_idx])
        out_seqs.append(new_ids)
        prev_end_idx = end_idx

    out_seqs.append(token_ids[prev_end_idx:])

    return flatten_2d_lists(out_seqs)


675
@dataclass
676
class PlaceholderFeaturesInfo:
677
    modality: str
678
    item_idx: int
679
    start_idx: int
680
    tokens: list[int]
681
    is_embed: torch.Tensor | None
682
683
684

    @property
    def length(self) -> int:
685
        return len(self.tokens)
686
687

    def to_range(self) -> PlaceholderRange:
688
689
        # TODO: Is it worth it to optimize this by stripping the
        # leading and ending positions where `is_embed=False`?
690
691
692
        return PlaceholderRange(
            offset=self.start_idx,
            length=self.length,
693
            is_embed=self.is_embed,
694
        )
695
696


697
_MatchToApply = tuple[tuple[str, int], tuple[PromptTargetMatch, int]]
698
699


700
701
702
def _find_matches(
    prompt: _S,
    mm_prompt_updates: "MultiModalPromptUpdates",
703
    tokenizer: TokenizerLike | None,
704
705
706
    *,
    prev_end_idx: int = 0,
    current_result: "MultiModalPromptUpdatesApplyResult",
707
708
) -> tuple[UpdateMode | None, list[_MatchToApply]]:
    mode: UpdateMode | None = None
709
710
711
712
713
714
715
716
717
718
719
720
    mm_matches = dict[tuple[str, int], tuple[PromptTargetMatch, int]]()

    for modality, modality_updates in mm_prompt_updates.items():
        for item_idx, item_updates in enumerate(modality_updates):
            if current_result[modality][item_idx] is not None:
                continue  # Updates have already been applied for this item

            for update_idx, update in enumerate(item_updates):
                if (modality, item_idx) in mm_matches:
                    break  # Already found a match for this item

                for match in update.iter_matches(
721
722
723
                    prompt,
                    tokenizer,
                    start_idx=prev_end_idx,
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
                ):
                    # All matches should share the same mode
                    if mode is None:
                        mode = update.mode
                    elif mode != update.mode:
                        continue

                    mm_matches[(modality, item_idx)] = match, update_idx
                    break  # Get only the first valid match per item

    # Prioritize earlier matches
    matches_to_apply = sorted(mm_matches.items(), key=lambda item: item[1][0])

    # To avoid conflicts, only replace one non-empty item at a time
    if mode == UpdateMode.REPLACE:
        matches_to_apply_ = list[_MatchToApply]()
        has_non_empty_matches = False

        for item in matches_to_apply:
            _, (match, _) = item
            if match.start_idx == match.end_idx:
                matches_to_apply_.append(item)
            elif not has_non_empty_matches:
                has_non_empty_matches = True
                matches_to_apply_.append(item)

        matches_to_apply = matches_to_apply_

    return mode, matches_to_apply
753
754


755
756
757
758
759
760
761
762
763
764
def _all_items_found(
    mm_item_counts: dict[str, int],
    mm_found_counts: dict[str, int],
) -> bool:
    return all(
        item_idx >= mm_item_counts[modality]
        for modality, item_idx in mm_found_counts.items()
    )


765
def _apply_matches(
766
    prompt: _S,
767
    mm_prompt_updates: "MultiModalPromptUpdates",
768
    tokenizer: TokenizerLike | None,
769
) -> tuple[list[_S], "MultiModalPromptUpdatesApplyResult"]:
770
    mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()}
771

772
    out_seqs = list[str | list[int]]()
773
    out_result: MultiModalPromptUpdatesApplyResult = {
774
        m: [None] * len(items) for m, items in mm_prompt_updates.items()
775
    }
776

777
    # Early exit if no items to find
778
779
780
781
782
783
    mm_found_counts = {
        m: sum(r is not None for r in res) for m, res in out_result.items()
    }
    if _all_items_found(mm_item_counts, mm_found_counts):
        return [prompt], out_result

784
785
    prev_end_idx = 0
    while True:
786
787
788
789
790
791
792
        mode, matches_to_apply = _find_matches(
            prompt,
            mm_prompt_updates,
            tokenizer,
            prev_end_idx=prev_end_idx,
            current_result=out_result,
        )
793

794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
        if mode is None:
            break  # No more matches to find

        for (modality, item_idx), (match, update_idx) in matches_to_apply:
            matched_update = mm_prompt_updates[modality][item_idx][update_idx]
            matched_content = matched_update.content.full

            if mode == UpdateMode.INSERT:
                end_idx_to_insert = match.end_idx
            elif mode == UpdateMode.REPLACE:
                end_idx_to_insert = match.start_idx
            else:
                assert_never(mode)

            out_seqs.append(prompt[prev_end_idx:end_idx_to_insert])
            out_seqs.append(
                _seq2text(tokenizer, matched_content)
                if isinstance(prompt, str)
                else _seq2tokens(tokenizer, matched_content)
            )
            out_result[modality][item_idx] = update_idx

            # Exclude overlapping matches
            prev_end_idx = match.end_idx

        # Early exit if all items found
        mm_found_counts = {
            m: sum(r is not None for r in res) for m, res in out_result.items()
        }
        if _all_items_found(mm_item_counts, mm_found_counts):
            break
825
826
827

    out_seqs.append(prompt[prev_end_idx:])

828
    return cast(list[_S], out_seqs), out_result
829
830


831
def apply_token_matches(
832
    prompt: list[int],
833
    mm_prompt_updates: "MultiModalPromptUpdates",
834
    tokenizer: TokenizerLike | None,
835
836
837
) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
    """
    Apply the updates in `mm_prompt_updates` to `prompt`.
838

839
840
841
842
    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
    appears earlier in `mm_prompt_updates` takes priority.
    """
843
    token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
844

845
    return flatten_2d_lists(token_id_seqs), result
846
847


848
def apply_text_matches(
849
    prompt: str,
850
    mm_prompt_updates: "MultiModalPromptUpdates",
851
    tokenizer: TokenizerLike | None,
852
853
854
) -> tuple[str, "MultiModalPromptUpdatesApplyResult"]:
    """
    Apply the updates in `mm_prompt_updates` to `prompt`.
855

856
857
858
859
860
    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
    appears earlier in `mm_prompt_updates` takes priority.
    """
    texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
861

862
    return "".join(texts), result
863
864


865
def _iter_placeholders(
866
    prompt: list[int],
867
    mm_prompt_updates: "MultiModalPromptUpdates",
868
    tokenizer: TokenizerLike | None,
869
) -> Iterable[PlaceholderFeaturesInfo]:
870
    """
871
    Yield each set of placeholder tokens found in `prompt`.
872
873
874

    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
875
    appears earlier in `mm_prompt_updates` takes priority.
876

877
878
    Note that empty matches are ignored.
    """
879
    mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()}
880
    item_idx_by_modality = {modality: 0 for modality in mm_prompt_updates}
881

882
883
    if _all_items_found(mm_item_counts, item_idx_by_modality):
        return
884

885
    prompt_len = len(prompt)
886
    start_idx = 0
887

888
889
890
    while start_idx < prompt_len:
        found = False

891
        for modality, modality_updates in mm_prompt_updates.items():
892
893
            item_idx = item_idx_by_modality[modality]
            if item_idx >= mm_item_counts.get(modality, 0):
894
                continue
895

896
897
            for update in modality_updates[item_idx]:
                content = update.content
898
                content_tokens_full = _seq2tokens(tokenizer, content.full)
899
900
                content_len_full = len(content_tokens_full)
                end_idx_full = start_idx + content_len_full
901

902
                if content_len_full == 0 or end_idx_full > prompt_len:
903
904
                    continue

905
                if prompt[start_idx:end_idx_full] == content_tokens_full:
906
907
                    content_is_embed = content.is_embed
                    if content_is_embed is not None:
908
                        content_is_embed = content_is_embed(tokenizer, content.full)
909
910
911
912
913
914
915
916

                    yield PlaceholderFeaturesInfo(
                        modality=modality,
                        item_idx=item_idx,
                        start_idx=start_idx,
                        tokens=content_tokens_full,
                        is_embed=content_is_embed,
                    )
917

918
                    # Exclude overlapping matches
919
                    start_idx = end_idx_full
920
921
922
                    item_idx_by_modality[modality] += 1
                    found = True
                    break
923

924
            if found:
925
926
927
                if _all_items_found(mm_item_counts, item_idx_by_modality):
                    return

928
                break  # Go back to the outer while loop
929
930
931

        if not found:
            start_idx += 1
932
933


934
935
def find_mm_placeholders(
    prompt: list[int],
936
    mm_prompt_updates: "MultiModalPromptUpdates",
937
    tokenizer: TokenizerLike | None,
938
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
939
    it = _iter_placeholders(prompt, mm_prompt_updates, tokenizer)
940
941
942
    return dict(full_groupby_modality(it))


943
944
945
MultiModalIsCached = dict[str, list[bool]]
"""
A collection of the `is_cached` flag for each item, with a similar structure as
946
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
947
948
"""

949
MultiModalPromptUpdates = Mapping[str, list[Sequence[ResolvedPromptUpdate]]]
950
951
952
953
954
"""
A collection of prompt updates with a similar structure as
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
"""

955
MultiModalPromptUpdatesApplyResult = Mapping[str, list[int | None]]
956
957
958
959
960
961
962
"""
For an item `MultiModalPromptUpdates[k][i]`,
`MultiModalPromptUpdatesApplyResult[k][i]` represents the index of the
`ResolvedPromptUpdate` instance that has been applied, or `None` if none of the
`ResolvedPromptUpdate` instances have been applied.
"""

963
964
_I = TypeVar("_I", bound=BaseProcessingInfo)

965
966

class MultiModalProcessingInfo(NamedTuple):
967
    kwargs: MultiModalKwargsOptionalItems
968
    hashes: MultiModalHashes
969
970
    prompt_updates: MultiModalPromptUpdates

971
972

class BaseMultiModalProcessor(ABC, Generic[_I]):
973
    """
974
    Abstract base class to process multi-modal inputs to be used in vLLM.
975

976
    Not to be confused with `transformers.ProcessorMixin`.
977
978
    """

979
980
981
982
983
    def __init__(
        self,
        info: _I,
        dummy_inputs: "BaseDummyInputsBuilder[_I]",
        *,
984
        cache: BaseMultiModalProcessorCache | None = None,
985
    ) -> None:
986
987
        super().__init__()

988
989
        self.info = info
        self.dummy_inputs = dummy_inputs
990
        self.cache = cache
991

992
993
994
995
996
997
998
999
1000
1001
        if hasattr(self, "_get_data_parser"):
            logger.warning_once(
                "BaseMultiModalProcessor._get_data_parser is deprecated "
                "and will be removed in v0.16."
                "You should override `info.build_data_parser` instead."
            )

            self.data_parser = self._get_data_parser()  # type: ignore
        else:
            self.data_parser = self.info.get_data_parser()
1002

1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
        # Avoid unnecessary recomputation
        self._supported_mm_limits = self.info.get_supported_mm_limits()
        self._allowed_mm_limits = self.info.get_allowed_mm_limits()

    @property
    def supported_mm_limits(self):
        return self._supported_mm_limits

    @property
    def allowed_mm_limits(self):
        return self._allowed_mm_limits

1015
    def __call__(
1016
        self,
1017
1018
        prompt: str,
        mm_data: MultiModalDataDict,
1019
        hf_processor_mm_kwargs: Mapping[str, object],
1020
        *,
1021
        mm_uuids: MultiModalUUIDDict | None = None,
1022
    ) -> MultiModalInputs:
1023
        return self.apply(prompt, mm_data, hf_processor_mm_kwargs, mm_uuids=mm_uuids)
1024

1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
    def validate_num_items(
        self,
        modality: str,
        num_items: int,
    ) -> None:
        supported_limit = self.supported_mm_limits.get(modality, 0)
        allowed_limit = self.allowed_mm_limits.get(modality, 0)

        if supported_limit is None:
            supported_limit = allowed_limit

        limit = min(supported_limit, allowed_limit)

        if num_items > limit:
1039
            msg = f"At most {limit} {modality}(s) may be provided in one prompt."
1040
1041
1042
1043
1044
1045

            if num_items <= supported_limit:
                msg += " Set `--limit-mm-per-prompt` to increase this limit."

            raise ValueError(msg)

1046
    def _to_mm_items(
1047
1048
1049
        self,
        mm_data: MultiModalDataDict,
    ) -> MultiModalDataItems:
1050
        """
1051
1052
1053
1054
1055
        Normalize
        [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict]
        to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]
        before passing them to
        [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
1056
        """
1057
        mm_items = self.data_parser.parse_mm_data(mm_data)
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067

        mm_config = self.info.ctx.model_config.get_multimodal_config()
        if not mm_config.enable_mm_embeds:
            for modality, items in mm_items.items():
                if isinstance(items, (EmbeddingItems, DictEmbeddingItems)):
                    raise ValueError(
                        f"You must set `--enable-mm-embeds` to input "
                        f"`{modality}_embeds`"
                    )

1068
        for modality, items in mm_items.items():
1069
            self.validate_num_items(modality, len(items))
1070
1071

        return mm_items
1072

1073
1074
1075
    @abstractmethod
    def _get_mm_fields_config(
        self,
1076
        hf_inputs: BatchFeature,
1077
1078
1079
1080
1081
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, MultiModalFieldConfig]:
        """Given the HF-processed data, output the metadata of each field."""
        raise NotImplementedError

1082
    @abstractmethod
1083
    def _get_prompt_updates(
1084
        self,
1085
        mm_items: MultiModalDataItems,
1086
        hf_processor_mm_kwargs: Mapping[str, object],
1087
        out_mm_kwargs: MultiModalKwargsItems,
1088
    ) -> Sequence[PromptUpdate]:
1089
1090
        """
        Given the original multi-modal items for this modality
1091
        and HF-processed data, output the updates to perform.
1092

1093
1094
1095
1096
1097
1098
        The information returned by this method is used to update token inputs
        which bypass the HF processor. It is also used to update the output of
        HF processor if the HF process does not apply prompt updates to text
        inputs.

        Moreover, this information is critical to determine the token positions
1099
1100
        in order to construct
        [`PlaceholderRange`][vllm.multimodal.inputs.PlaceholderRange]
1101
        for each multi-modal item.
1102
1103
        """
        raise NotImplementedError
1104

1105
1106
1107
1108
1109
1110
    def _bind_and_group_updates(
        self,
        prompt_updates: Sequence[PromptUpdate],
        mm_item_counts: Mapping[str, int],
    ) -> MultiModalPromptUpdates:
        return {
1111
1112
1113
1114
            modality: [
                [update.resolve(item_idx) for update in updates]
                for item_idx in range(mm_item_counts.get(modality, 0))
            ]
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
            for modality, updates in full_groupby_modality(prompt_updates)
        }

    def _get_mm_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        out_mm_kwargs: MultiModalKwargsItems,
    ) -> MultiModalPromptUpdates:
        unbound_prompt_updates = self._get_prompt_updates(
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
            out_mm_kwargs=out_mm_kwargs,
        )

        mm_prompt_updates = self._bind_and_group_updates(
            unbound_prompt_updates,
            mm_items.get_all_counts(),
        )

        for modality, prompt_updates in mm_prompt_updates.items():
            for item_idx, item_prompt_updates in enumerate(prompt_updates):
                if len(item_prompt_updates) > 1:
                    logger.warning_once(
                        "Detected %d prompt updates for `mm_items[%r][%s]`. "
                        "Multiple prompt updates per item is now "
                        "deprecated and may be removed in v0.13. "
                        "Instead, please specify dynamic update targets "
                        "in the same prompt update definition by passing "
                        "a function to `PromptUpdate.target`.",
                        len(prompt_updates),
                        modality,
                        item_idx,
                    )

        return mm_prompt_updates

1152
    def _find_mm_placeholders(
1153
1154
        self,
        new_token_ids: list[int],
1155
        mm_prompt_updates: MultiModalPromptUpdates,
1156
    ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
1157
1158
        tokenizer = self.info.get_tokenizer()

1159
        return find_mm_placeholders(new_token_ids, mm_prompt_updates, tokenizer)
1160

1161
    def _get_hf_mm_data(
1162
        self,
1163
        mm_items: MultiModalDataItems,
1164
1165
1166
    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
        processor_data = dict[str, object]()
        passthrough_data = dict[str, object]()
1167

1168
1169
1170
        for items in mm_items.values():
            processor_data.update(items.get_processor_data())
            passthrough_data.update(items.get_passthrough_data())
1171

1172
1173
        return processor_data, passthrough_data

1174
1175
1176
    def _call_hf_processor(
        self,
        prompt: str,
1177
1178
1179
1180
        # Not to be confused with `mm_data` in `self.apply`.
        # This refers to the data to be passed to HF processor.
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
1181
        tok_kwargs: Mapping[str, object],
1182
    ) -> BatchFeature:
1183
1184
1185
1186
        """
        Call the HF processor on the prompt text and
        associated multi-modal data.
        """
1187
        with timed_preprocessor_operation(self.info.ctx, "hf_processor"):
1188
1189
1190
1191
1192
            return self.info.ctx.call_hf_processor(
                self.info.get_hf_processor(**mm_kwargs),
                dict(text=prompt, **mm_data),
                dict(**mm_kwargs, **tok_kwargs),
            )
1193

1194
    def _hf_processor_applies_updates(
1195
1196
1197
1198
        self,
        prompt_text: str,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1199
        tokenization_kwargs: Mapping[str, object],
1200
1201
    ) -> bool:
        """
1202
        Return whether the HF processor applies prompt updates.
1203

1204
1205
        For most HF processors, this should be `True` when multi-modal
        data items are passed, but `False` when multi-modal embeddings
1206
1207
1208
1209
        are passed.
        """
        return not any(
            isinstance(items, (EmbeddingItems, DictEmbeddingItems))
1210
1211
            for items in mm_items.values()
        )
1212

1213
    def _apply_hf_processor_text_mm(
1214
        self,
1215
        prompt_text: str,
1216
        mm_items: MultiModalDataItems,
1217
        hf_processor_mm_kwargs: Mapping[str, object],
1218
        tokenization_kwargs: Mapping[str, object],
1219
    ) -> tuple[list[int], BatchFeature, bool]:
1220
        """
1221
1222
        Apply the HF processor on the prompt text and multi-modal data
        together.
1223

1224
        In addition, return whether prompt updates have been applied.
1225
1226
1227
1228
1229
1230
1231
        """
        processor_data, passthrough_data = self._get_hf_mm_data(mm_items)

        processed_data = self._call_hf_processor(
            prompt=prompt_text,
            mm_data=processor_data,
            mm_kwargs=hf_processor_mm_kwargs,
1232
            tok_kwargs=tokenization_kwargs,
1233
1234
        )
        processed_data.update(passthrough_data)
1235

1236
        (prompt_ids,) = processed_data.pop("input_ids").tolist()
1237

1238
        is_update_applied = self._hf_processor_applies_updates(
1239
1240
1241
            prompt_text=prompt_text,
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1242
            tokenization_kwargs=tokenization_kwargs,
1243
1244
        )

1245
        return prompt_ids, processed_data, is_update_applied
1246

1247
    def _apply_hf_processor_text_only(
1248
1249
1250
1251
        self,
        prompt_text: str,
        tokenization_kwargs: Mapping[str, object],
    ) -> list[int]:
1252
        """
1253
        Apply the HF processor on the prompt text only.
1254

1255
1256
1257
        Since HF processor requires that text and multi-modal items
        correspond to each other, we create dummy multi-modal items
        to go along with the text.
1258
        """
1259
        prompt_ids, _, _ = self._apply_hf_processor_text_mm(
1260
1261
1262
            prompt_text=prompt_text,
            mm_items=MultiModalDataItems({}),
            hf_processor_mm_kwargs={},
1263
            tokenization_kwargs=tokenization_kwargs,
1264
1265
        )

1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
        return prompt_ids

    def _apply_hf_processor_tokens_only(
        self,
        prompt_tokens: list[int],
    ) -> list[int]:
        """
        Apply the HF processor on the prompt tokens only.

        Most HF processors accept prompt text but not prompt tokens.
        If the HF processor adds or removes tokens that are not related to
        multi-modal data, you should override this method so it is consistent
1278
1279
1280
        with the output of
        [`_apply_hf_processor_text_only`][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_text_only]
        on the
1281
1282
1283
1284
1285
1286
1287
1288
        corresponding text.
        """
        return prompt_tokens

    def _apply_hf_processor_mm_only(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1289
        tokenization_kwargs: Mapping[str, object],
1290
    ) -> BatchFeature:
1291
1292
1293
1294
1295
        """
        Apply the HF processor on the multi-modal data only.

        Since HF processor requires that text and multi-modal items
        correspond to each other, we generate dummy text using
1296
        [`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder]
1297
        to go along with the multi-modal data.
1298
1299
1300
        """
        mm_counts = mm_items.get_all_counts()

1301
        _, mm_processed_data, _ = self._apply_hf_processor_text_mm(
1302
            prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
1303
1304
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1305
            tokenization_kwargs=tokenization_kwargs,
1306
1307
        )

1308
        return mm_processed_data
1309
1310
1311

    def _apply_hf_processor_main(
        self,
1312
        prompt: str | list[int],
1313
1314
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1315
        tokenization_kwargs: Mapping[str, object],
1316
        *,
1317
        enable_hf_prompt_update: bool,
1318
    ) -> tuple[list[int], BatchFeature, bool]:
1319
1320
1321
        """
        Apply the HF processor on the prompt text and multi-modal data.

1322
        In addition, return whether prompt updates have been applied
1323
        (for most HF processors, this should be `True`).
1324

1325
        Note:
1326
            If `enable_hf_prompt_update=False`, we use HF processor
1327
            to perform prompt updates if available; HF processor requires
1328
            that the prompt corresponds to multi-modal items.
1329
1330
        """
        if isinstance(prompt, str):
1331
            if enable_hf_prompt_update:
1332
1333
1334
1335
                return self._apply_hf_processor_text_mm(
                    prompt_text=prompt,
                    mm_items=mm_items,
                    hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1336
                    tokenization_kwargs=tokenization_kwargs,
1337
1338
                )

1339
            prompt_ids = self._apply_hf_processor_text_only(prompt, tokenization_kwargs)
1340
1341
1342
        else:
            prompt_ids = self._apply_hf_processor_tokens_only(prompt)

1343
        mm_processed_data = self._apply_hf_processor_mm_only(
1344
            mm_items=mm_items,
1345
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1346
            tokenization_kwargs=tokenization_kwargs,
1347
1348
        )

1349
        return prompt_ids, mm_processed_data, False
1350

1351
    def _hash_mm_items(
1352
1353
1354
1355
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        tokenization_kwargs: Mapping[str, object],
1356
        *,
1357
        mm_uuids: MultiModalUUIDDict | None = None,
1358
    ) -> MultiModalHashes:
1359
        """Create MM hashes to be returned.
1360

1361

1362
1363
1364
        Note: When overrides are provided via callers of `apply`,
        `_hash_mm_items` will be bypassed and the overrides will be used.
        """
1365
1366
        model_id = self.info.model_id

1367
        hashes: MultiModalHashes = {}
1368
        mm_uuids = mm_uuids or {}
1369
1370

        for modality, items in mm_items.items():
1371
1372
1373
1374
            if modality in mm_uuids:
                mm_uuids_per_modality = mm_uuids[modality]
                if isinstance(mm_uuids_per_modality, str):
                    mm_uuids_per_modality = [mm_uuids_per_modality]
1375
1376
1377

                # For None entries, compute a hash; otherwise, use provided ID.
                computed: list[str] = []
1378
                for i, item in enumerate(items.get_all_items_for_hash()):
1379
                    item_uuid = mm_uuids_per_modality[i]
1380

1381
                    # NOTE: Even if a item_uuid is provided, we still compute a
1382
1383
1384
                    # hash if `hf_processor_mm_kwargs` or `tokenization_kwargs`
                    # are provided. This is because the processed multimodal
                    # inputs can be different depending on the processor kwargs.
1385
1386
1387
1388
1389
                    if (
                        item_uuid is None
                        or hf_processor_mm_kwargs
                        or tokenization_kwargs
                    ):
1390
1391
                        # NOTE: use provided hash string to hash with kwargs
                        # if available for better performance.
1392
                        item = item_uuid if item_uuid is not None else item
1393
1394
1395
1396
1397
                        computed.append(
                            MultiModalHasher.hash_kwargs(
                                model_id=model_id,
                                **{modality: item},
                                **hf_processor_mm_kwargs,
1398
1399
1400
                                **tokenization_kwargs,
                            )
                        )
1401
                    else:
1402
                        computed.append(item_uuid)
1403
1404
1405
                hashes[modality] = computed
            else:
                hashes[modality] = [
1406
1407
1408
1409
1410
1411
                    MultiModalHasher.hash_kwargs(
                        model_id=model_id,
                        **{modality: item},
                        **hf_processor_mm_kwargs,
                        **tokenization_kwargs,
                    )
1412
1413
1414
1415
                    for item in items
                ]

        return hashes
1416

1417
1418
    def _get_cache_missing_items(
        self,
1419
        cache: BaseMultiModalProcessorCache,
1420
1421
        mm_data_items: MultiModalDataItems,
        mm_hashes: MultiModalHashes,
1422
    ) -> tuple[MultiModalIsCached, MultiModalDataItems]:
1423
        mm_is_cached = {
1424
            modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
1425
1426
1427
1428
        }

        mm_missing_idxs = {
            modality: [
1429
1430
                idx
                for idx, item_is_cached in enumerate(items_is_cached)
1431
1432
1433
1434
                if not item_is_cached
            ]
            for modality, items_is_cached in mm_is_cached.items()
        }
1435
1436
1437
1438
1439
1440
1441
1442
        mm_missing_data = {}
        for modality, idxs in mm_missing_idxs.items():
            missing_modality_data = []
            for idx in idxs:
                data = mm_data_items[modality][idx]
                if data is None:
                    raise ValueError(
                        f"Cache miss for {modality} at index {idx} "
1443
1444
                        f"but data is not provided."
                    )
1445
1446
1447
                else:
                    missing_modality_data.append(data)
            mm_missing_data[modality] = missing_modality_data
1448

1449
        return mm_is_cached, self._to_mm_items(mm_missing_data)
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461

    def _recompute_cached_prompt_update(
        self,
        cached_update: ResolvedPromptUpdate,
        new_item_idx: int,
    ) -> ResolvedPromptUpdate:
        """
        Override this if other attributes of `ResolvedPromptUpdate`
        also need to be recomputed after retrieving from the cache.
        """
        return replace(cached_update, item_idx=new_item_idx)

1462
1463
    def _merge_mm_kwargs(
        self,
1464
        cache: BaseMultiModalProcessorCache,
1465
        mm_hashes: MultiModalHashes,
1466
        mm_is_cached: MultiModalIsCached,
1467
        mm_missing_kwargs: MultiModalKwargsItems,
1468
1469
        mm_missing_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[MultiModalKwargsOptionalItems, MultiModalPromptUpdates]:
1470
1471
1472
1473
1474
        # Need to touch all mm hashes before update to avoid hash in updated
        # list evict during update
        for hashes in mm_hashes.values():
            for item_hash in hashes:
                cache.touch_sender_cache_item(item_hash)
1475

1476
        mm_missing_next_idx = defaultdict[str, int](lambda: 0)
1477

1478
        merged_kwargs = defaultdict[str, list[MultiModalKwargsItem | None]](list)
1479
1480
1481
        merged_prompt_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](
            list
        )
1482
1483
        for modality, hashes in mm_hashes.items():
            missing_kwargs = mm_missing_kwargs.get(modality, [])
1484
            missing_prompt_updates = mm_missing_prompt_updates.get(modality, [])
1485
1486
1487
1488

            for item_idx, item_hash in enumerate(hashes):
                if not mm_is_cached[modality][item_idx]:
                    missing_next_idx = mm_missing_next_idx[modality]
1489
1490
                    missing_kwargs_item = missing_kwargs[missing_next_idx]
                    missing_updates_item = missing_prompt_updates[missing_next_idx]
1491

1492
                    mm_missing_next_idx[modality] += 1
1493

1494
                    item = missing_kwargs_item, missing_updates_item
1495
                else:
1496
1497
1498
1499
1500
                    item = None

                kwargs, updates = cache.get_and_update_item(item, item_hash)

                merged_kwargs[modality].append(kwargs)
1501
1502
1503
1504
1505
1506
                merged_prompt_updates[modality].append(
                    [
                        self._recompute_cached_prompt_update(update, item_idx)
                        for update in updates
                    ]
                )
1507

1508
1509
        mm_kwargs = MultiModalKwargsItems(merged_kwargs)
        mm_prompt_updates = dict(merged_prompt_updates)
1510

1511
        return mm_kwargs, mm_prompt_updates
1512
1513
1514

    def _apply_hf_processor(
        self,
1515
        prompt: str | list[int],
1516
1517
        mm_data_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1518
        tokenization_kwargs: Mapping[str, object],
1519
        *,
1520
        mm_uuids: MultiModalUUIDDict | None = None,
1521
    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1522
1523
        (
            prompt_ids,
1524
            mm_processed_data,
1525
1526
1527
1528
1529
            is_update_applied,
        ) = self._apply_hf_processor_main(
            prompt=prompt,
            mm_items=mm_data_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1530
            tokenization_kwargs=tokenization_kwargs,
1531
1532
1533
            enable_hf_prompt_update=True,
        )

1534
        mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
1535
            mm_processed_data,
1536
            self._get_mm_fields_config(mm_processed_data, hf_processor_mm_kwargs),
1537
1538
        )

1539
        # Use overrides if provided; fallback to data-dependent hashing.
1540
        with timed_preprocessor_operation(self.info.ctx, "hashing"):
1541
1542
1543
1544
1545
1546
            mm_hashes = self._hash_mm_items(
                mm_data_items,
                hf_processor_mm_kwargs,
                tokenization_kwargs,
                mm_uuids=mm_uuids,
            )
1547

1548
        mm_prompt_updates = self._get_mm_prompt_updates(
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
            mm_data_items,
            hf_processor_mm_kwargs,
            mm_kwargs,
        )

        mm_info = MultiModalProcessingInfo(
            kwargs=mm_kwargs,
            hashes=mm_hashes,
            prompt_updates=mm_prompt_updates,
        )

        return prompt_ids, mm_info, is_update_applied
1561

1562
1563
    def _cached_apply_hf_processor(
        self,
1564
        prompt: str | list[int],
1565
1566
        mm_data_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1567
        tokenization_kwargs: Mapping[str, object],
1568
        *,
1569
        mm_uuids: MultiModalUUIDDict | None = None,
1570
    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1571
1572
1573
1574
1575
1576
        """
        Apply the HF processor on the full prompt text,
        caching the results and reusing cached results.
        """
        cache = self.cache

1577
1578
        _, passthrough_data = self._get_hf_mm_data(mm_data_items)
        if cache is None or passthrough_data:
1579
            return self._apply_hf_processor(
1580
                prompt=prompt,
1581
                mm_data_items=mm_data_items,
1582
                hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1583
                tokenization_kwargs=tokenization_kwargs,
1584
                mm_uuids=mm_uuids,
1585
1586
            )

1587
        with timed_preprocessor_operation(self.info.ctx, "hashing"):
1588
1589
1590
1591
1592
1593
            mm_hashes = self._hash_mm_items(
                mm_data_items,
                hf_processor_mm_kwargs,
                tokenization_kwargs,
                mm_uuids=mm_uuids,
            )
1594

1595
        with timed_preprocessor_operation(self.info.ctx, "cache_lookup"):
1596
1597
1598
1599
1600
            mm_is_cached, mm_missing_data_items = self._get_cache_missing_items(
                cache=cache,
                mm_data_items=mm_data_items,
                mm_hashes=mm_hashes,
            )
1601

1602
        # NOTE: `prompt` does not correspond to `mm_missing_data_items`,
1603
        # so we can't apply prompt updates until the new multimodal
1604
1605
1606
        # items are combined with the cached multimodal items
        (
            prompt_ids,
1607
            mm_missing_processed_data,
1608
            is_update_applied,
1609
        ) = self._apply_hf_processor_main(
1610
            prompt=prompt,
1611
            mm_items=mm_missing_data_items,
1612
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1613
            tokenization_kwargs=tokenization_kwargs,
1614
            enable_hf_prompt_update=False,
1615
1616
        )

1617
        mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
1618
            mm_missing_processed_data,
1619
1620
1621
            self._get_mm_fields_config(
                mm_missing_processed_data, hf_processor_mm_kwargs
            ),
1622
1623
        )

1624
1625
1626
1627
        mm_missing_prompt_updates = self._get_mm_prompt_updates(
            mm_missing_data_items,
            hf_processor_mm_kwargs,
            mm_missing_kwargs,
1628
        )
1629

1630
        with timed_preprocessor_operation(self.info.ctx, "cache_lookup"):
1631
1632
1633
1634
1635
1636
1637
            mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
                cache,
                mm_hashes=mm_hashes,
                mm_is_cached=mm_is_cached,
                mm_missing_kwargs=mm_missing_kwargs,
                mm_missing_prompt_updates=mm_missing_prompt_updates,
            )
1638
1639
1640

        mm_info = MultiModalProcessingInfo(
            kwargs=mm_kwargs,
1641
            hashes=mm_hashes,
1642
1643
            prompt_updates=mm_prompt_updates,
        )
1644

1645
        return prompt_ids, mm_info, is_update_applied
1646

1647
1648
1649
    def _apply_token_matches(
        self,
        prompt: list[int],
1650
1651
1652
1653
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
        tokenizer = self.info.get_tokenizer()
        return apply_token_matches(prompt, mm_prompt_updates, tokenizer)
1654
1655
1656
1657

    def _apply_text_matches(
        self,
        prompt: str,
1658
1659
1660
1661
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[str, MultiModalPromptUpdatesApplyResult]:
        tokenizer = self.info.get_tokenizer()
        return apply_text_matches(prompt, mm_prompt_updates, tokenizer)
1662

1663
    def _apply_prompt_updates(
1664
1665
        self,
        token_ids: list[int],
1666
        mm_prompt_updates: MultiModalPromptUpdates,
1667
    ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
1668
        tokenizer = self.info.get_tokenizer()
1669

1670
1671
1672
1673
        new_token_ids, match_result = self._apply_token_matches(
            token_ids,
            mm_prompt_updates,
        )
1674
1675
1676
1677
1678
1679
1680
1681
1682

        # If the search text does not represent a special token,
        # it may have different token IDs in the prompt, because
        # the tokens may go across the boundaries of the search text.
        # ----
        # e.g. when searching for "foo" in "food", if "food" itself makes
        # up a token, then the token ID of "foo" will not appear at all
        # ----
        # Since it is inefficient to search for all possible tokenizations
1683
1684
        # of the search text in the prompt, we instead perform string-based
        # updates on the decoded token IDs, then encode them back.
1685
        if not all(
1686
1687
1688
            all(update_idx is not None for update_idx in update_idxs)
            for update_idxs in match_result.values()
        ):
1689
            new_text, match_result = self._apply_text_matches(
1690
                _seq2text(tokenizer, token_ids, use_cache=False),
1691
                mm_prompt_updates,
1692
1693
            )

1694
            new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False)
1695

1696
        matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
1697
1698
1699
1700
        for modality, update_idxs in match_result.items():
            for item_idx, update_idx in enumerate(update_idxs):
                assert update_idx is not None, (
                    "Failed to apply prompt replacement for "
1701
1702
                    f"mm_items[{modality!r}][{item_idx}]"
                )
1703
1704

                matched_updates[modality].append(
1705
1706
                    [mm_prompt_updates[modality][item_idx][update_idx]]
                )
1707
1708

        placeholders = self._find_mm_placeholders(
1709
1710
            new_token_ids,
            dict(matched_updates),
1711
        )
1712

1713
        return new_token_ids, placeholders
1714

1715
1716
    def _validate_mm_kwargs(
        self,
1717
        mm_kwargs: MultiModalKwargsOptionalItems,
1718
1719
1720
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
1721
            items = mm_kwargs.get(modality, [])
1722
1723
1724
1725
1726
1727
1728
1729
1730

            if len(items) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} {modality} items in "
                    f"keyword arguments corresponding to {item_count} "
                    f"{modality} data items, but only found {len(items)}! "
                    "There is likely a problem with your "
                    "implementation of merged multi-modal processor for this "
                    "model (usually arising from an inconsistency between "
1731
1732
                    "`_call_hf_processor` and `_get_mm_fields_config`)."
                )
1733

1734
    def _validate_mm_updates(
1735
        self,
1736
        mm_updates: MultiModalPromptUpdates,
1737
        mm_item_counts: Mapping[str, int],
1738
    ) -> None:
1739
        for modality, item_count in mm_item_counts.items():
1740
            placeholders = mm_updates.get(modality, [])
1741

1742
            if len(placeholders) != item_count:
1743
                raise RuntimeError(
1744
                    f"Expected there to be {item_count} prompt updates "
1745
                    f"corresponding to {item_count} {modality} items, but "
1746
                    f"instead found {len(placeholders)} prompt updates! "
1747
1748
1749
                    "This is likely because you forgot to include input "
                    "placeholder tokens (e.g., `<image>`, `<|image_pad|>`) "
                    "in the prompt. If the model has a chat template, make "
1750
1751
                    "sure you have applied it before calling `LLM.generate`."
                )
1752

1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
    def _validate_mm_placeholders(
        self,
        mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
            placeholders = mm_placeholders.get(modality, [])

            if len(placeholders) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} prompt placeholders "
                    f"corresponding to {item_count} {modality} items, but "
                    f"instead found {len(placeholders)} prompt placeholders! "
                    "Make sure the implementation of `_call_hf_processor` and "
1767
1768
                    "`_get_mm_fields_config` are consistent with each other."
                )
1769

1770
1771
1772
1773
    def _maybe_apply_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        prompt_ids: list[int],
1774
        mm_kwargs: MultiModalKwargsOptionalItems,
1775
        mm_prompt_updates: MultiModalPromptUpdates,
1776
        is_update_applied: bool,
1777
    ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
1778
        mm_item_counts = mm_items.get_all_counts()
1779
        self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
1780
        self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
1781

1782
        if is_update_applied:
1783
1784
            mm_placeholders = self._find_mm_placeholders(
                prompt_ids,
1785
                mm_prompt_updates,
1786
            )
1787
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1788
        else:
1789
            prompt_ids, mm_placeholders = self._apply_prompt_updates(
1790
                prompt_ids,
1791
                mm_prompt_updates,
1792
            )
1793
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1794

1795
        return prompt_ids, mm_placeholders
1796
1797
1798

    def apply(
        self,
1799
        prompt: str | list[int],
1800
1801
        mm_data: MultiModalDataDict,
        hf_processor_mm_kwargs: Mapping[str, object],
1802
        tokenization_kwargs: Mapping[str, object] | None = None,
1803
        *,
1804
        mm_uuids: MultiModalUUIDDict | None = None,
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
    ) -> MultiModalInputs:
        """
        Process multi-modal inputs to be used in vLLM.

        The main steps are:

        1. Apply HF Processor on prompt text and multi-modal data together,
           outputting token IDs and processed tensors.
        2. Find and update sequences in the token IDs with placeholder tokens.
           The number of placeholder tokens equals the feature size of the
           multi-modal data outputted by the multi-modal encoder.
        3. Extract information about the placeholder tokens from the
           processed token IDs.
        """
1819
1820
1821
1822
        request_id = get_current_request_id()
        if request_id is not None:
            self.info.ctx.create_timing_stats(request_id)

1823
1824
        mm_items = self._to_mm_items(mm_data)

1825
1826
1827
        if tokenization_kwargs is None:
            tokenization_kwargs = {}

1828
1829
        (
            prompt_ids,
1830
            mm_info,
1831
1832
1833
1834
1835
            is_update_applied,
        ) = self._cached_apply_hf_processor(
            prompt,
            mm_items,
            hf_processor_mm_kwargs,
1836
            tokenization_kwargs=tokenization_kwargs,
1837
            mm_uuids=mm_uuids,
1838
1839
        )

1840
        # NOTE: tokenization_kwargs are not required to init processor
1841
        with timed_preprocessor_operation(self.info.ctx, "prompt_update"):
1842
1843
1844
1845
1846
1847
1848
            prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
                mm_items=mm_items,
                prompt_ids=prompt_ids,
                mm_kwargs=mm_info.kwargs,
                mm_prompt_updates=mm_info.prompt_updates,
                is_update_applied=is_update_applied,
            )
1849

1850
1851
1852
1853
        mm_placeholder_ranges = {
            modality: [item.to_range() for item in placeholders]
            for modality, placeholders in mm_placeholders.items()
        }
1854

1855
        return MultiModalInputs(
1856
            type="multimodal",
1857
            prompt_token_ids=prompt_ids,
1858
1859
            mm_kwargs=mm_info.kwargs,
            mm_hashes=mm_info.hashes,
1860
            mm_placeholders=mm_placeholder_ranges,
1861
        )
1862
1863
1864
1865
1866
1867


class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
    @abstractmethod
    def create_encoder_prompt(
        self,
1868
        prompt: str | list[int],
1869
        mm_data: MultiModalDataDict,
1870
    ) -> str | list[int]:
1871
        """
1872
        Create input prompt for the encoder. HF processor will be applied on
1873
1874
        this prompt during profiling and generation.
        """
1875
1876
        raise NotImplementedError

1877
1878
    def create_decoder_prompt(
        self,
1879
        prompt: str | list[int],
1880
        mm_data: MultiModalDataDict,
1881
    ) -> str | list[int]:
1882
1883
1884
        """Create input prompt for the decoder."""
        return prompt

1885
    def _get_enc_dec_inputs(
1886
        self,
1887
        prompt: str | list[int],
1888
        mm_data: MultiModalDataDict,
1889
1890
        encoder_inputs: MultiModalInputs,
    ):
1891
        tokenizer = self.info.get_tokenizer()
1892
1893
        decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_data)
        if isinstance(decoder_prompt_raw, str):
1894
1895
            decoder_prompt_ids = tokenizer.encode(
                decoder_prompt_raw, add_special_tokens=False
1896
            )
1897
        else:
1898
            decoder_prompt_ids = decoder_prompt_raw
1899
1900
1901

        mm_inputs = MultiModalEncDecInputs(
            encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],
1902
1903
            **encoder_inputs,
        )
1904
        mm_inputs["prompt_token_ids"] = decoder_prompt_ids
1905
        return mm_inputs
1906
1907
1908

    def apply(
        self,
1909
        prompt: str | list[int],
1910
1911
        mm_data: MultiModalDataDict,
        hf_processor_mm_kwargs: Mapping[str, object],
1912
        tokenization_kwargs: Mapping[str, object] | None = None,
1913
        *,
1914
        mm_uuids: MultiModalUUIDDict | None = None,
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
    ) -> MultiModalEncDecInputs:
        """
        Process multi-modal inputs to be used in vLLM.
        The main processing steps are modified to fit encoder-decoder model:
        1. Create encoder prompt from input prompt text.
        2. Apply the HF processor on encoder prompt.
        3. Copy the input prompt text as decoder prompt inputs.
        """
        encoder_prompt = self.create_encoder_prompt(prompt, mm_data)
        encoder_inputs = super().apply(
            encoder_prompt,
            mm_data,
            hf_processor_mm_kwargs,
1928
            tokenization_kwargs,
1929
            mm_uuids=mm_uuids,
1930
1931
1932
1933
1934
1935
1936
        )

        return self._get_enc_dec_inputs(
            prompt=prompt,
            mm_data=mm_data,
            encoder_inputs=encoder_inputs,
        )