processor.py 62 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from abc import ABC, abstractmethod
4
from collections import defaultdict
5
from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence
6
from dataclasses import dataclass, field, replace
7
from enum import Enum
8
from functools import lru_cache
9
10
11
12
13
from typing import (
    TYPE_CHECKING,
    Generic,
    NamedTuple,
    Protocol,
14
    TypeAlias,
15
16
    cast,
)
17

18
import regex as re
19
import torch
20
from typing_extensions import TypeVar, assert_never, deprecated
21

22
from vllm.logger import init_logger
23
from vllm.tokenizers import TokenizerLike
24
from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
25

26
27
from ..hasher import MultiModalHasher
from ..inputs import (
28
29
30
    MultiModalDataDict,
    MultiModalEncDecInputs,
    MultiModalFieldConfig,
31
    MultiModalHashes,
32
33
34
35
36
37
38
    MultiModalInputs,
    MultiModalKwargsItem,
    MultiModalKwargsItems,
    MultiModalKwargsOptionalItems,
    MultiModalUUIDDict,
    PlaceholderRange,
)
39
from ..parse import (
40
41
42
43
    DictEmbeddingItems,
    EmbeddingItems,
    MultiModalDataItems,
)
44
45
46
47
48
from .context import (
    BaseProcessingInfo,
    get_current_request_id,
    timed_preprocessor_operation,
)
49
from .dummy_inputs import BaseDummyInputsBuilder
50
51

if TYPE_CHECKING:
52
53
    from transformers.feature_extraction_utils import BatchFeature

54
    from ..cache import BaseMultiModalProcessorCache
55
56
57
58
else:
    BatchFeature = object

    BaseMultiModalProcessorCache = object
59

60
logger = init_logger(__name__)
61
62

_S = TypeVar("_S", str, list[int])
63

64

65
PromptSeq: TypeAlias = str | list[int]
66
"""A token sequence (list of token IDs) or text."""
67

68

69
70
@lru_cache(maxsize=2048)
def _cached_encode(
71
    tokenizer: TokenizerLike,
72
73
    text: str,
    *,
74
    add_special_tokens: bool = True,
75
) -> list[int]:
76
    return tokenizer.encode(text, add_special_tokens=add_special_tokens)
77
78
79
80


@lru_cache(maxsize=2048)
def _cached_decode(
81
    tokenizer: TokenizerLike,
82
83
    token_ids: tuple[int, ...],
    *,
84
    skip_special_tokens: bool = False,
85
) -> str:
86
    return tokenizer.decode(list(token_ids), skip_special_tokens=skip_special_tokens)
87
88


89
90
91
92
93
94
def _seq2text(
    tokenizer: TokenizerLike | None,
    seq: PromptSeq,
    *,
    use_cache: bool = True,
) -> str:
95
96
97
    if isinstance(seq, str):
        return seq

98
99
100
101
    if tokenizer is None:
        raise ValueError("You cannot decode tokens when `skip_tokenizer_init=True`")

    if not use_cache:
102
        return tokenizer.decode(seq)
103

104
105
106
    return _cached_decode(tokenizer, tuple(seq))


107
108
109
110
111
112
def _seq2tokens(
    tokenizer: TokenizerLike | None,
    seq: PromptSeq,
    *,
    use_cache: bool = True,
) -> list[int]:
113
    if isinstance(seq, str):
114
115
116
117
        if tokenizer is None:
            raise ValueError("You cannot encode text when `skip_tokenizer_init=True`")

        if not use_cache:
118
            return tokenizer.encode(seq, add_special_tokens=False)
119

120
121
122
123
124
        return _cached_encode(tokenizer, seq, add_special_tokens=False)

    return seq


125
126
127
class _GetMatchIndex(Protocol):
    def __call__(
        self,
128
        tokenizer: TokenizerLike | None,
129
130
        prompt: PromptSeq,
        start_idx: int = 0,
131
    ) -> int | None: ...
132
133


134
135
136
@dataclass
class PromptIndex:
    """Resolves to an index in the prompt."""
137

138
    get_match_index: _GetMatchIndex
139
140
141
142
143
144
145
146
147
148


class PromptIndexTargets:
    @staticmethod
    def start() -> PromptIndex:
        """
        Resolves to the start of the prompt (before the first token).

        This results in a match even if the prompt is empty.
        """
149
        return PromptIndex(lambda tokenizer, prompt, start_idx=0: 0)
150
151
152
153
154
155
156
157

    @staticmethod
    def prefix(seq: PromptSeq) -> PromptIndex:
        """
        Resolves to a location in the prompt after the given prefix.
        """

        def get_match_index(
158
            tokenizer: TokenizerLike | None,
159
            prompt: PromptSeq,
160
            start_idx: int = 0,
161
        ) -> int | None:
162
163
164
            if start_idx != 0:
                return None

165
166
167
            prefix = seq

            if isinstance(prompt, str):
168
169
                # Make both `str`
                prefix = _seq2text(tokenizer, prefix, use_cache=False)
170
            else:
171
172
                # Make both `list[int]`
                prefix = _seq2tokens(tokenizer, prefix, use_cache=False)
173
174
175
176
177
178
179
180
181
182
183
184
185

            match_idx = len(prefix)
            return match_idx if prompt[:match_idx] == prefix else None

        return PromptIndex(get_match_index)

    @staticmethod
    def end() -> PromptIndex:
        """
        Resolves to the end of the prompt (after the last token).

        This results in a match even if the prompt is empty.
        """
186
        return PromptIndex(lambda tokenizer, prompt, start_idx=0: len(prompt))
187
188


189
UpdateTarget: TypeAlias = PromptSeq | PromptIndex
190
191
192
193
"""
The token sequence or text to update.
"""

194
PromptUpdateTarget: TypeAlias = Callable[[int], UpdateTarget] | UpdateTarget
195
196
197
198
199
200
201
202
203
"""
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
output the corresponding token sequence (or text).

For convenience, you can directly pass in the token sequence (or text)
instead of a function if it does not depend on the input.
"""

204

205
@dataclass
206
class PromptUpdateDetails(Generic[_S]):
207
    """Details about the token sequence or text that are part of the update."""
208

209
    full: _S
210
    """The full content."""
211

212
    is_embed: Callable[[TokenizerLike | None, PromptSeq], torch.Tensor] | None = None
213
    """
214
215
216
    Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full],
    return a boolean mask of shape `(len(full),)` indicating which positions
    of `full` to assign embeddings to.
217
218
219
220

    `None` (default) means to assign embeddings to all positions of `full`.

    The embeddings are obtained by calling
221
    [`SupportsMultiModal.embed_multimodal`][vllm.model_executor.models.interfaces.SupportsMultiModal.embed_multimodal].
222
223
224
    """

    @staticmethod
225
    def from_seq(seq: _S) -> "PromptUpdateDetails[_S]":
226
227
228
229
230
231
232
        return PromptUpdateDetails(full=seq)

    @staticmethod
    def select_text(
        seq: _S,
        embed_text: str,
    ) -> "PromptUpdateDetails[_S]":
233
234
        def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
            embed_token_ids = _seq2tokens(tokenizer, embed_text, use_cache=False)
235
            token_ids = _seq2tokens(tokenizer, full)
236
237

            return torch.isin(
238
                torch.tensor(token_ids),
239
240
241
242
243
244
245
246
247
248
                torch.tensor(embed_token_ids),
            )

        return PromptUpdateDetails(full=seq, is_embed=is_embed)

    @staticmethod
    def select_token_id(
        seq: _S,
        embed_token_id: int,
    ) -> "PromptUpdateDetails[_S]":
249
        def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
250
251
252
253
254
            token_ids = _seq2tokens(tokenizer, full)

            return torch.tensor(token_ids) == embed_token_id

        return PromptUpdateDetails(full=seq, is_embed=is_embed)
255

256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
    @staticmethod
    def select_token_ids(
        seq: _S,
        embed_token_ids: list[int],
    ) -> "PromptUpdateDetails[_S]":
        def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
            token_ids = _seq2tokens(tokenizer, full)

            return torch.isin(
                torch.tensor(token_ids),
                torch.tensor(embed_token_ids),
            )

        return PromptUpdateDetails(full=seq, is_embed=is_embed)

271

272
PromptUpdateInfo: TypeAlias = PromptSeq | PromptUpdateDetails
273
"""
274
The token sequence or text that are part of the update.
275

276
If only part of the content corresponds to feature placeholders, you can
277
278
use [`PromptUpdateDetails`][vllm.multimodal.processing.PromptUpdateDetails] to
specify which part.
279
"""
280

281
PromptUpdateContent: TypeAlias = Callable[[int], PromptUpdateInfo] | PromptUpdateInfo
282
"""
283
284
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
285
286
287
288
289
290
291
292
293
294
295
296
297
output the corresponding token sequence (or text).

For convenience, you can directly pass in the token sequence (or text)
instead of a function if it does not depend on the input.
"""


class UpdateMode(str, Enum):
    INSERT = "insert"
    REPLACE = "replace"


@dataclass
298
class PromptUpdate(ABC):
299
300
301
302
303
304
305
    """
    Defines how to update a prompt with placeholder tokens.
    """

    modality: str
    """The modality for which the update is made."""

306
    target: PromptUpdateTarget
307
308
309
310
311
312
313
314
315
316
317
318
319
320
    """The token sequence (or text) to update."""

    @property
    @abstractmethod
    def content(self) -> PromptUpdateContent:
        """The placeholder tokens that are part of the update."""
        raise NotImplementedError

    @property
    @abstractmethod
    def mode(self) -> UpdateMode:
        """Defines how to update the prompt."""
        raise NotImplementedError

321
    def _resolve_target(self, item_idx: int) -> UpdateTarget:
322
323
324
325
        target = self.target
        if callable(target):
            target = target(item_idx)

326
        return target
327

328
    def _resolve_content(self, item_idx: int) -> PromptUpdateDetails:
329
330
331
332
333
334
335
        content = self.content
        if callable(content):
            content = content(item_idx)

        if not isinstance(content, PromptUpdateDetails):
            content = PromptUpdateDetails.from_seq(content)

336
        return content
337

338
    def resolve(self, item_idx: int) -> "ResolvedPromptUpdate":
339
340
341
342
343
344
345
346
347
        """
        Given the index of the processed item within
        [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
        output a copy of this object with its lazy attributes resolved.
        """
        return ResolvedPromptUpdate(
            modality=self.modality,
            item_idx=item_idx,
            mode=self.mode,
348
349
            target=self._resolve_target(item_idx),
            content=self._resolve_content(item_idx),
350
351
        )

352

353
@dataclass
354
355
356
357
358
359
class PromptInsertion(PromptUpdate):
    """
    Defines how to insert placeholder tokens into a prompt.

    Example:

360
361
    For each image, insert a number of `<image>` feature placeholders
    equal to the feature size of the vision encoder after the `<s>` token:
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380

    ```python
    PromptInsertion(
        modality="image",
        target="<s>",
        insertion="<image>" * image_feature_size,
    )
    ```

    Insert these tokens at the start of the prompt:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.start(),
        insertion="<image>" * image_feature_size,
    )
    ```

381
    Insert these tokens after a prefix `Images:`:
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.prefix("Images:"),
        insertion="<image>" * image_feature_size,
    )
    ```

    Insert these tokens at the end of the prompt:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.end(),
        insertion="<image>" * image_feature_size,
    )
    ```
400
401
402
403
    """

    insertion: PromptUpdateContent = field(repr=False)
    """
404
405
406
407
    Given the index of the processed item within
    [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
    output the token sequence (or text) to insert right after
    [`target`][vllm.multimodal.processing.PromptUpdate.target].
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423

    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
    """

    @property
    def content(self) -> PromptUpdateContent:
        return self.insertion

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.INSERT


@dataclass
class PromptReplacement(PromptUpdate):
424
425
    """
    Defines how to replace portions of an input prompt with placeholder tokens.
426
427
428

    Example:

429
430
    For each image, replace one `<image>` input placeholder in the prompt
    with a number of `<image>` feature placeholders
431
432
433
434
435
436
437
438
439
440
    equal to the feature size of the vision encoder:

    ```python
    PromptReplacement(
        modality="image",
        target="<image>",
        replacement="<image>" * image_feature_size,
    )
    ```

441
442
    As above, but further pad the feature placeholders with `<image_bos>`
    and `<image_eos>`, which are not supposed to be passed to the vision
443
444
445
446
447
448
449
    encoder:

    ```python
    PromptReplacement(
        modality="image",
        target="<image>",
        replacement=PromptUpdateDetails(
450
451
452
453
454
455
456
            full="".join(
                [
                    "<image_bos>",
                    "<image>" * image_feature_size,
                    "<image_eos>",
                ]
            ),
457
458
459
460
461
462
463
464
465
466
467
468
469
            features="<image>" * image_feature_size,
        ),
    )
    ```

    To avoid unnecessary tokenization during prompt replacement,
    we recommended passing token sequences instead of text:

    ```python
    PromptReplacement(
        modality="image",
        target=[image_token_id],
        replacement=PromptUpdateDetails(
470
471
472
            full=(
                [image_bos_id] + [image_token_id] * image_feature_size + [image_eos_id]
            ),
473
474
475
476
            features=[image_token_id] * image_feature_size,
        ),
    )
    ```
477
478
    """

479
    replacement: PromptUpdateContent = field(repr=False)
480
    """
481
482
483
484
    Given the index of the processed item within
    [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
    output the token sequence (or text) to replace
    [`target`][vllm.multimodal.processing.PromptUpdate.target].
485

486
487
    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
488
489
    """

490
491
492
493
494
495
496
    @property
    def content(self) -> PromptUpdateContent:
        return self.replacement

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.REPLACE
497
498


499
500
501
class _HasModalityAttr(Protocol):
    modality: str

502

503
504
class _HasModalityProp(Protocol):
    @property
505
    def modality(self) -> str: ...
506
507


508
_M = TypeVar("_M", bound=_HasModalityAttr | _HasModalityProp)
509
510
511


def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]:
512
513
    """
    Convenience function to apply
514
    [`full_groupby`][vllm.utils.collection_utils.full_groupby]
515
516
    based on modality.
    """
517
518
519
    return full_groupby(values, key=lambda x: x.modality)


520
521
522
523
524
525
526
class PromptTargetMatch(NamedTuple):
    start_idx: int
    end_idx: int


@dataclass(frozen=True)
class ResolvedPromptUpdate:
527
    """
528
529
    A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] with its
    lazy attributes resolved, apart from those related to tokenization.
530
    """
531

532
533
    modality: str
    """The modality for which the update is made."""
534

535
536
    item_idx: int
    """The index within `modality` of the item this update pertains to."""
537

538
539
    mode: UpdateMode
    """Defines how to update the prompt."""
540

541
    target: UpdateTarget
542
    """The token sequence (or text) to update."""
543

544
    content: PromptUpdateDetails = field(repr=False)
545
    """The placeholder tokens that are part of the update."""
546

547
548
549
    def iter_token_matches(
        self,
        prompt: list[int],
550
        tokenizer: TokenizerLike | None,
551
552
553
554
555
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        target = self.target
556

557
558
559
560
        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(tokenizer, prompt, start_idx)
            if match_idx is not None:
                yield PromptTargetMatch(match_idx, match_idx)
561

562
            return
563

564
565
        target_token_ids = _seq2tokens(tokenizer, target)

566
        for match in iter_token_matches(prompt, target_token_ids, start_idx=start_idx):
567
            yield PromptTargetMatch(match.start_idx, match.end_idx)
568

569
570
571
    def iter_text_matches(
        self,
        prompt: str,
572
        tokenizer: TokenizerLike | None,
573
574
575
576
577
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        target = self.target
578

579
580
581
582
        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(tokenizer, prompt, start_idx)
            if match_idx is not None:
                yield PromptTargetMatch(match_idx, match_idx)
583

584
            return
585

586
587
        target_text = _seq2text(tokenizer, target)

588
        for match in re.finditer(re.escape(target_text), prompt, pos=start_idx):
589
590
591
592
            yield PromptTargetMatch(match.start(), match.end())

    def iter_matches(
        self,
593
        prompt: list[int] | str,
594
        tokenizer: TokenizerLike | None,
595
596
597
598
599
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        if isinstance(prompt, str):
600
            return self.iter_text_matches(prompt, tokenizer, start_idx=start_idx)
601
602

        return self.iter_token_matches(prompt, tokenizer, start_idx=start_idx)
603

604
605
606
607
608
609
610
611
612
    def with_target(self, target: UpdateTarget):
        return replace(self, target=target)

    def with_content(self, content: PromptUpdateInfo):
        if not isinstance(content, PromptUpdateDetails):
            content = PromptUpdateDetails.from_seq(content)

        return replace(self, content=content)

613

614
615
616
class _TokenMatch(NamedTuple):
    start_idx: int
    end_idx: int
617
618


619
620
621
def iter_token_matches(
    token_ids: list[int],
    match_ids: list[int],
622
623
    *,
    start_idx: int = 0,
624
) -> Generator[_TokenMatch]:
625
    """
626
    Yield each occurrence of `match_ids` in `token_ids`.
627
628
629
630

    Note that empty matches are ignored.
    """
    prompt_len = len(token_ids)
631
    match_len = len(match_ids)
632

633
634
    if match_len == 0:
        return
635

636
    while start_idx < prompt_len - match_len + 1:
637
        end_idx = start_idx + match_len
638

639
640
        if token_ids[start_idx:end_idx] == match_ids:
            yield _TokenMatch(start_idx=start_idx, end_idx=end_idx)
641
642
643
644
645

            # Exclude overlapping matches
            start_idx = end_idx
        else:
            start_idx += 1
646
647


648
649
650
651
652
653
def replace_token_matches(
    token_ids: list[int],
    match_ids: list[int],
    new_ids: list[int],
) -> list[int]:
    """
654
655
    Replace each occurrence of `match_ids` in `token_ids`
    with `new_ids`.
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674

    Note that empty matches are ignored.
    """
    out_seqs = list[list[int]]()
    prev_end_idx = 0

    for match in iter_token_matches(token_ids, match_ids):
        start_idx = match.start_idx
        end_idx = match.end_idx

        out_seqs.append(token_ids[prev_end_idx:start_idx])
        out_seqs.append(new_ids)
        prev_end_idx = end_idx

    out_seqs.append(token_ids[prev_end_idx:])

    return flatten_2d_lists(out_seqs)


675
@dataclass
676
class PlaceholderFeaturesInfo:
677
    modality: str
678
    item_idx: int
679
    start_idx: int
680
    tokens: list[int]
681
    is_embed: torch.Tensor | None
682
683
684

    @property
    def length(self) -> int:
685
        return len(self.tokens)
686
687

    def to_range(self) -> PlaceholderRange:
688
689
        # TODO: Is it worth it to optimize this by stripping the
        # leading and ending positions where `is_embed=False`?
690
691
692
        return PlaceholderRange(
            offset=self.start_idx,
            length=self.length,
693
            is_embed=self.is_embed,
694
        )
695
696


697
_MatchToApply = tuple[tuple[str, int], tuple[PromptTargetMatch, int]]
698
699


700
701
702
def _find_matches(
    prompt: _S,
    mm_prompt_updates: "MultiModalPromptUpdates",
703
    tokenizer: TokenizerLike | None,
704
705
706
    *,
    prev_end_idx: int = 0,
    current_result: "MultiModalPromptUpdatesApplyResult",
707
708
) -> tuple[UpdateMode | None, list[_MatchToApply]]:
    mode: UpdateMode | None = None
709
710
711
712
713
714
715
716
717
718
719
720
    mm_matches = dict[tuple[str, int], tuple[PromptTargetMatch, int]]()

    for modality, modality_updates in mm_prompt_updates.items():
        for item_idx, item_updates in enumerate(modality_updates):
            if current_result[modality][item_idx] is not None:
                continue  # Updates have already been applied for this item

            for update_idx, update in enumerate(item_updates):
                if (modality, item_idx) in mm_matches:
                    break  # Already found a match for this item

                for match in update.iter_matches(
721
722
723
                    prompt,
                    tokenizer,
                    start_idx=prev_end_idx,
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
                ):
                    # All matches should share the same mode
                    if mode is None:
                        mode = update.mode
                    elif mode != update.mode:
                        continue

                    mm_matches[(modality, item_idx)] = match, update_idx
                    break  # Get only the first valid match per item

    # Prioritize earlier matches
    matches_to_apply = sorted(mm_matches.items(), key=lambda item: item[1][0])

    # To avoid conflicts, only replace one non-empty item at a time
    if mode == UpdateMode.REPLACE:
        matches_to_apply_ = list[_MatchToApply]()
        has_non_empty_matches = False

        for item in matches_to_apply:
            _, (match, _) = item
            if match.start_idx == match.end_idx:
                matches_to_apply_.append(item)
            elif not has_non_empty_matches:
                has_non_empty_matches = True
                matches_to_apply_.append(item)

        matches_to_apply = matches_to_apply_

    return mode, matches_to_apply
753
754


755
756
757
758
759
760
761
762
763
764
def _all_items_found(
    mm_item_counts: dict[str, int],
    mm_found_counts: dict[str, int],
) -> bool:
    return all(
        item_idx >= mm_item_counts[modality]
        for modality, item_idx in mm_found_counts.items()
    )


765
def _apply_matches(
766
    prompt: _S,
767
    mm_prompt_updates: "MultiModalPromptUpdates",
768
    tokenizer: TokenizerLike | None,
769
) -> tuple[list[_S], "MultiModalPromptUpdatesApplyResult"]:
770
    mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()}
771

772
    out_seqs = list[str | list[int]]()
773
    out_result: MultiModalPromptUpdatesApplyResult = {
774
        m: [None] * len(items) for m, items in mm_prompt_updates.items()
775
    }
776

777
    # Early exit if no items to find
778
779
780
781
782
783
    mm_found_counts = {
        m: sum(r is not None for r in res) for m, res in out_result.items()
    }
    if _all_items_found(mm_item_counts, mm_found_counts):
        return [prompt], out_result

784
785
    prev_end_idx = 0
    while True:
786
787
788
789
790
791
792
        mode, matches_to_apply = _find_matches(
            prompt,
            mm_prompt_updates,
            tokenizer,
            prev_end_idx=prev_end_idx,
            current_result=out_result,
        )
793

794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
        if mode is None:
            break  # No more matches to find

        for (modality, item_idx), (match, update_idx) in matches_to_apply:
            matched_update = mm_prompt_updates[modality][item_idx][update_idx]
            matched_content = matched_update.content.full

            if mode == UpdateMode.INSERT:
                end_idx_to_insert = match.end_idx
            elif mode == UpdateMode.REPLACE:
                end_idx_to_insert = match.start_idx
            else:
                assert_never(mode)

            out_seqs.append(prompt[prev_end_idx:end_idx_to_insert])
            out_seqs.append(
                _seq2text(tokenizer, matched_content)
                if isinstance(prompt, str)
                else _seq2tokens(tokenizer, matched_content)
            )
            out_result[modality][item_idx] = update_idx

            # Exclude overlapping matches
            prev_end_idx = match.end_idx

        # Early exit if all items found
        mm_found_counts = {
            m: sum(r is not None for r in res) for m, res in out_result.items()
        }
        if _all_items_found(mm_item_counts, mm_found_counts):
            break
825
826
827

    out_seqs.append(prompt[prev_end_idx:])

828
    return cast(list[_S], out_seqs), out_result
829
830


831
def apply_token_matches(
832
    prompt: list[int],
833
    mm_prompt_updates: "MultiModalPromptUpdates",
834
    tokenizer: TokenizerLike | None,
835
836
837
) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
    """
    Apply the updates in `mm_prompt_updates` to `prompt`.
838

839
840
841
842
    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
    appears earlier in `mm_prompt_updates` takes priority.
    """
843
    token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
844

845
    return flatten_2d_lists(token_id_seqs), result
846
847


848
def apply_text_matches(
849
    prompt: str,
850
    mm_prompt_updates: "MultiModalPromptUpdates",
851
    tokenizer: TokenizerLike | None,
852
853
854
) -> tuple[str, "MultiModalPromptUpdatesApplyResult"]:
    """
    Apply the updates in `mm_prompt_updates` to `prompt`.
855

856
857
858
859
860
    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
    appears earlier in `mm_prompt_updates` takes priority.
    """
    texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
861

862
    return "".join(texts), result
863
864


865
def _iter_placeholders(
866
    prompt: list[int],
867
    mm_prompt_updates: "MultiModalPromptUpdates",
868
    tokenizer: TokenizerLike | None,
869
) -> Iterable[PlaceholderFeaturesInfo]:
870
    """
871
    Yield each set of placeholder tokens found in `prompt`.
872
873
874

    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
875
    appears earlier in `mm_prompt_updates` takes priority.
876

877
878
    Note that empty matches are ignored.
    """
879
    mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()}
880
    item_idx_by_modality = {modality: 0 for modality in mm_prompt_updates}
881

882
883
    if _all_items_found(mm_item_counts, item_idx_by_modality):
        return
884

885
    prompt_len = len(prompt)
886
    start_idx = 0
887

888
889
890
    while start_idx < prompt_len:
        found = False

891
        for modality, modality_updates in mm_prompt_updates.items():
892
893
            item_idx = item_idx_by_modality[modality]
            if item_idx >= mm_item_counts.get(modality, 0):
894
                continue
895

896
897
            for update in modality_updates[item_idx]:
                content = update.content
898
                content_tokens_full = _seq2tokens(tokenizer, content.full)
899
900
                content_len_full = len(content_tokens_full)
                end_idx_full = start_idx + content_len_full
901

902
                if content_len_full == 0 or end_idx_full > prompt_len:
903
904
                    continue

905
                if prompt[start_idx:end_idx_full] == content_tokens_full:
906
907
                    content_is_embed = content.is_embed
                    if content_is_embed is not None:
908
                        content_is_embed = content_is_embed(tokenizer, content.full)
909
910
911
912
913
914
915
916

                    yield PlaceholderFeaturesInfo(
                        modality=modality,
                        item_idx=item_idx,
                        start_idx=start_idx,
                        tokens=content_tokens_full,
                        is_embed=content_is_embed,
                    )
917

918
                    # Exclude overlapping matches
919
                    start_idx = end_idx_full
920
921
922
                    item_idx_by_modality[modality] += 1
                    found = True
                    break
923

924
            if found:
925
926
927
                if _all_items_found(mm_item_counts, item_idx_by_modality):
                    return

928
                break  # Go back to the outer while loop
929
930
931

        if not found:
            start_idx += 1
932
933


934
935
def find_mm_placeholders(
    prompt: list[int],
936
    mm_prompt_updates: "MultiModalPromptUpdates",
937
    tokenizer: TokenizerLike | None,
938
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
939
    it = _iter_placeholders(prompt, mm_prompt_updates, tokenizer)
940
941
942
    return dict(full_groupby_modality(it))


943
944
945
MultiModalIsCached = dict[str, list[bool]]
"""
A collection of the `is_cached` flag for each item, with a similar structure as
946
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
947
948
"""

949
MultiModalPromptUpdates = Mapping[str, list[Sequence[ResolvedPromptUpdate]]]
950
951
952
953
954
"""
A collection of prompt updates with a similar structure as
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
"""

955
MultiModalPromptUpdatesApplyResult = Mapping[str, list[int | None]]
956
957
958
959
960
961
962
"""
For an item `MultiModalPromptUpdates[k][i]`,
`MultiModalPromptUpdatesApplyResult[k][i]` represents the index of the
`ResolvedPromptUpdate` instance that has been applied, or `None` if none of the
`ResolvedPromptUpdate` instances have been applied.
"""

963
964
_I = TypeVar("_I", bound=BaseProcessingInfo)

965
966

class MultiModalProcessingInfo(NamedTuple):
967
    kwargs: MultiModalKwargsOptionalItems
968
    hashes: MultiModalHashes
969
970
    prompt_updates: MultiModalPromptUpdates

971
972

class BaseMultiModalProcessor(ABC, Generic[_I]):
973
    """
974
    Abstract base class to process multi-modal inputs to be used in vLLM.
975

976
    Not to be confused with `transformers.ProcessorMixin`.
977
978
    """

979
980
981
982
983
    def __init__(
        self,
        info: _I,
        dummy_inputs: "BaseDummyInputsBuilder[_I]",
        *,
984
        cache: BaseMultiModalProcessorCache | None = None,
985
    ) -> None:
986
987
        super().__init__()

988
989
        self.info = info
        self.dummy_inputs = dummy_inputs
990
        self.cache = cache
991

992
993
994
995
996
997
998
999
1000
1001
        if hasattr(self, "_get_data_parser"):
            logger.warning_once(
                "BaseMultiModalProcessor._get_data_parser is deprecated "
                "and will be removed in v0.16."
                "You should override `info.build_data_parser` instead."
            )

            self.data_parser = self._get_data_parser()  # type: ignore
        else:
            self.data_parser = self.info.get_data_parser()
1002

1003
    @property
1004
    @deprecated("Will be removed in v0.17. Use `info.supported_mm_limits` instead.")
1005
    def supported_mm_limits(self):
1006
        return self.info.supported_mm_limits
1007
1008

    @property
1009
    @deprecated("Will be removed in v0.17. Use `info.allowed_mm_limits` instead.")
1010
    def allowed_mm_limits(self):
1011
        return self.info.allowed_mm_limits
1012

1013
    def __call__(
1014
        self,
1015
1016
        prompt: str,
        mm_data: MultiModalDataDict,
1017
        hf_processor_mm_kwargs: Mapping[str, object],
1018
        *,
1019
        mm_uuids: MultiModalUUIDDict | None = None,
1020
    ) -> MultiModalInputs:
1021
        return self.apply(prompt, mm_data, hf_processor_mm_kwargs, mm_uuids=mm_uuids)
1022

1023
    def _to_mm_items(
1024
1025
1026
        self,
        mm_data: MultiModalDataDict,
    ) -> MultiModalDataItems:
1027
        """
1028
1029
1030
1031
1032
        Normalize
        [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict]
        to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]
        before passing them to
        [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
1033
        """
1034
        mm_items = self.data_parser.parse_mm_data(mm_data)
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044

        mm_config = self.info.ctx.model_config.get_multimodal_config()
        if not mm_config.enable_mm_embeds:
            for modality, items in mm_items.items():
                if isinstance(items, (EmbeddingItems, DictEmbeddingItems)):
                    raise ValueError(
                        f"You must set `--enable-mm-embeds` to input "
                        f"`{modality}_embeds`"
                    )

1045
        for modality, items in mm_items.items():
1046
            self.info.validate_num_items(modality, len(items))
1047
1048

        return mm_items
1049

1050
1051
1052
    @abstractmethod
    def _get_mm_fields_config(
        self,
1053
        hf_inputs: BatchFeature,
1054
1055
1056
1057
1058
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, MultiModalFieldConfig]:
        """Given the HF-processed data, output the metadata of each field."""
        raise NotImplementedError

1059
    @abstractmethod
1060
    def _get_prompt_updates(
1061
        self,
1062
        mm_items: MultiModalDataItems,
1063
        hf_processor_mm_kwargs: Mapping[str, object],
1064
        out_mm_kwargs: MultiModalKwargsItems,
1065
    ) -> Sequence[PromptUpdate]:
1066
1067
        """
        Given the original multi-modal items for this modality
1068
        and HF-processed data, output the updates to perform.
1069

1070
1071
1072
1073
1074
1075
        The information returned by this method is used to update token inputs
        which bypass the HF processor. It is also used to update the output of
        HF processor if the HF process does not apply prompt updates to text
        inputs.

        Moreover, this information is critical to determine the token positions
1076
1077
        in order to construct
        [`PlaceholderRange`][vllm.multimodal.inputs.PlaceholderRange]
1078
        for each multi-modal item.
1079
1080
        """
        raise NotImplementedError
1081

1082
1083
1084
1085
1086
1087
    def _bind_and_group_updates(
        self,
        prompt_updates: Sequence[PromptUpdate],
        mm_item_counts: Mapping[str, int],
    ) -> MultiModalPromptUpdates:
        return {
1088
1089
1090
1091
            modality: [
                [update.resolve(item_idx) for update in updates]
                for item_idx in range(mm_item_counts.get(modality, 0))
            ]
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
            for modality, updates in full_groupby_modality(prompt_updates)
        }

    def _get_mm_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        out_mm_kwargs: MultiModalKwargsItems,
    ) -> MultiModalPromptUpdates:
        unbound_prompt_updates = self._get_prompt_updates(
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
            out_mm_kwargs=out_mm_kwargs,
        )

        mm_prompt_updates = self._bind_and_group_updates(
            unbound_prompt_updates,
            mm_items.get_all_counts(),
        )

        for modality, prompt_updates in mm_prompt_updates.items():
            for item_idx, item_prompt_updates in enumerate(prompt_updates):
                if len(item_prompt_updates) > 1:
                    logger.warning_once(
                        "Detected %d prompt updates for `mm_items[%r][%s]`. "
                        "Multiple prompt updates per item is now "
                        "deprecated and may be removed in v0.13. "
                        "Instead, please specify dynamic update targets "
                        "in the same prompt update definition by passing "
                        "a function to `PromptUpdate.target`.",
                        len(prompt_updates),
                        modality,
                        item_idx,
                    )

        return mm_prompt_updates

1129
    def _find_mm_placeholders(
1130
1131
        self,
        new_token_ids: list[int],
1132
        mm_prompt_updates: MultiModalPromptUpdates,
1133
    ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
1134
1135
        tokenizer = self.info.get_tokenizer()

1136
        return find_mm_placeholders(new_token_ids, mm_prompt_updates, tokenizer)
1137

1138
    def _get_hf_mm_data(
1139
        self,
1140
        mm_items: MultiModalDataItems,
1141
1142
1143
    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
        processor_data = dict[str, object]()
        passthrough_data = dict[str, object]()
1144

1145
1146
1147
        for items in mm_items.values():
            processor_data.update(items.get_processor_data())
            passthrough_data.update(items.get_passthrough_data())
1148

1149
1150
        return processor_data, passthrough_data

1151
1152
1153
    def _call_hf_processor(
        self,
        prompt: str,
1154
1155
1156
1157
        # Not to be confused with `mm_data` in `self.apply`.
        # This refers to the data to be passed to HF processor.
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
1158
        tok_kwargs: Mapping[str, object],
1159
    ) -> BatchFeature:
1160
1161
1162
1163
        """
        Call the HF processor on the prompt text and
        associated multi-modal data.
        """
1164
        with timed_preprocessor_operation(self.info.ctx, "hf_processor"):
1165
1166
1167
1168
1169
            return self.info.ctx.call_hf_processor(
                self.info.get_hf_processor(**mm_kwargs),
                dict(text=prompt, **mm_data),
                dict(**mm_kwargs, **tok_kwargs),
            )
1170

1171
    def _hf_processor_applies_updates(
1172
1173
1174
1175
        self,
        prompt_text: str,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1176
        tokenization_kwargs: Mapping[str, object],
1177
1178
    ) -> bool:
        """
1179
        Return whether the HF processor applies prompt updates.
1180

1181
1182
        For most HF processors, this should be `True` when multi-modal
        data items are passed, but `False` when multi-modal embeddings
1183
1184
1185
1186
        are passed.
        """
        return not any(
            isinstance(items, (EmbeddingItems, DictEmbeddingItems))
1187
1188
            for items in mm_items.values()
        )
1189

1190
    def _apply_hf_processor_text_mm(
1191
        self,
1192
        prompt_text: str,
1193
        mm_items: MultiModalDataItems,
1194
        hf_processor_mm_kwargs: Mapping[str, object],
1195
        tokenization_kwargs: Mapping[str, object],
1196
    ) -> tuple[list[int], BatchFeature, bool]:
1197
        """
1198
1199
        Apply the HF processor on the prompt text and multi-modal data
        together.
1200

1201
        In addition, return whether prompt updates have been applied.
1202
1203
1204
1205
1206
1207
1208
        """
        processor_data, passthrough_data = self._get_hf_mm_data(mm_items)

        processed_data = self._call_hf_processor(
            prompt=prompt_text,
            mm_data=processor_data,
            mm_kwargs=hf_processor_mm_kwargs,
1209
            tok_kwargs=tokenization_kwargs,
1210
1211
        )
        processed_data.update(passthrough_data)
1212

1213
        (prompt_ids,) = processed_data.pop("input_ids").tolist()
1214

1215
        is_update_applied = self._hf_processor_applies_updates(
1216
1217
1218
            prompt_text=prompt_text,
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1219
            tokenization_kwargs=tokenization_kwargs,
1220
1221
        )

1222
        return prompt_ids, processed_data, is_update_applied
1223

1224
    def _apply_hf_processor_text_only(
1225
1226
1227
1228
        self,
        prompt_text: str,
        tokenization_kwargs: Mapping[str, object],
    ) -> list[int]:
1229
        """
1230
        Apply the HF processor on the prompt text only.
1231

1232
1233
1234
        Since HF processor requires that text and multi-modal items
        correspond to each other, we create dummy multi-modal items
        to go along with the text.
1235
        """
1236
        prompt_ids, _, _ = self._apply_hf_processor_text_mm(
1237
1238
1239
            prompt_text=prompt_text,
            mm_items=MultiModalDataItems({}),
            hf_processor_mm_kwargs={},
1240
            tokenization_kwargs=tokenization_kwargs,
1241
1242
        )

1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
        return prompt_ids

    def _apply_hf_processor_tokens_only(
        self,
        prompt_tokens: list[int],
    ) -> list[int]:
        """
        Apply the HF processor on the prompt tokens only.

        Most HF processors accept prompt text but not prompt tokens.
        If the HF processor adds or removes tokens that are not related to
        multi-modal data, you should override this method so it is consistent
1255
1256
1257
        with the output of
        [`_apply_hf_processor_text_only`][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_text_only]
        on the
1258
1259
1260
1261
1262
1263
1264
1265
        corresponding text.
        """
        return prompt_tokens

    def _apply_hf_processor_mm_only(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1266
        tokenization_kwargs: Mapping[str, object],
1267
    ) -> BatchFeature:
1268
1269
1270
1271
1272
        """
        Apply the HF processor on the multi-modal data only.

        Since HF processor requires that text and multi-modal items
        correspond to each other, we generate dummy text using
1273
        [`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder]
1274
        to go along with the multi-modal data.
1275
1276
1277
        """
        mm_counts = mm_items.get_all_counts()

1278
        _, mm_processed_data, _ = self._apply_hf_processor_text_mm(
1279
            prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
1280
1281
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1282
            tokenization_kwargs=tokenization_kwargs,
1283
1284
        )

1285
        return mm_processed_data
1286
1287
1288

    def _apply_hf_processor_main(
        self,
1289
        prompt: str | list[int],
1290
1291
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1292
        tokenization_kwargs: Mapping[str, object],
1293
        *,
1294
        enable_hf_prompt_update: bool,
1295
    ) -> tuple[list[int], BatchFeature, bool]:
1296
1297
1298
        """
        Apply the HF processor on the prompt text and multi-modal data.

1299
        In addition, return whether prompt updates have been applied
1300
        (for most HF processors, this should be `True`).
1301

1302
        Note:
1303
            If `enable_hf_prompt_update=False`, we use HF processor
1304
            to perform prompt updates if available; HF processor requires
1305
            that the prompt corresponds to multi-modal items.
1306
1307
        """
        if isinstance(prompt, str):
1308
            if enable_hf_prompt_update:
1309
1310
1311
1312
                return self._apply_hf_processor_text_mm(
                    prompt_text=prompt,
                    mm_items=mm_items,
                    hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1313
                    tokenization_kwargs=tokenization_kwargs,
1314
1315
                )

1316
            prompt_ids = self._apply_hf_processor_text_only(prompt, tokenization_kwargs)
1317
1318
1319
        else:
            prompt_ids = self._apply_hf_processor_tokens_only(prompt)

1320
        mm_processed_data = self._apply_hf_processor_mm_only(
1321
            mm_items=mm_items,
1322
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1323
            tokenization_kwargs=tokenization_kwargs,
1324
1325
        )

1326
        return prompt_ids, mm_processed_data, False
1327

1328
    def _hash_mm_items(
1329
1330
1331
1332
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        tokenization_kwargs: Mapping[str, object],
1333
        *,
1334
        mm_uuids: MultiModalUUIDDict | None = None,
1335
    ) -> MultiModalHashes:
1336
        """Create MM hashes to be returned.
1337

1338

1339
1340
1341
        Note: When overrides are provided via callers of `apply`,
        `_hash_mm_items` will be bypassed and the overrides will be used.
        """
1342
1343
        model_id = self.info.model_id

1344
        hashes: MultiModalHashes = {}
1345
        mm_uuids = mm_uuids or {}
1346
1347

        for modality, items in mm_items.items():
1348
1349
1350
1351
            if modality in mm_uuids:
                mm_uuids_per_modality = mm_uuids[modality]
                if isinstance(mm_uuids_per_modality, str):
                    mm_uuids_per_modality = [mm_uuids_per_modality]
1352
1353
1354

                # For None entries, compute a hash; otherwise, use provided ID.
                computed: list[str] = []
1355
                for i, item in enumerate(items.get_all_items_for_hash()):
1356
                    item_uuid = mm_uuids_per_modality[i]
1357

1358
                    # NOTE: Even if a item_uuid is provided, we still compute a
1359
1360
1361
                    # hash if `hf_processor_mm_kwargs` or `tokenization_kwargs`
                    # are provided. This is because the processed multimodal
                    # inputs can be different depending on the processor kwargs.
1362
1363
1364
1365
1366
                    if (
                        item_uuid is None
                        or hf_processor_mm_kwargs
                        or tokenization_kwargs
                    ):
1367
1368
                        # NOTE: use provided hash string to hash with kwargs
                        # if available for better performance.
1369
                        item = item_uuid if item_uuid is not None else item
1370
1371
1372
1373
1374
                        computed.append(
                            MultiModalHasher.hash_kwargs(
                                model_id=model_id,
                                **{modality: item},
                                **hf_processor_mm_kwargs,
1375
1376
1377
                                **tokenization_kwargs,
                            )
                        )
1378
                    else:
1379
                        computed.append(item_uuid)
1380
1381
1382
                hashes[modality] = computed
            else:
                hashes[modality] = [
1383
1384
1385
1386
1387
1388
                    MultiModalHasher.hash_kwargs(
                        model_id=model_id,
                        **{modality: item},
                        **hf_processor_mm_kwargs,
                        **tokenization_kwargs,
                    )
1389
1390
1391
1392
                    for item in items
                ]

        return hashes
1393

1394
1395
    def _get_cache_missing_items(
        self,
1396
        cache: BaseMultiModalProcessorCache,
1397
1398
        mm_data_items: MultiModalDataItems,
        mm_hashes: MultiModalHashes,
1399
    ) -> tuple[MultiModalIsCached, MultiModalDataItems]:
1400
        mm_is_cached = {
1401
            modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
1402
1403
1404
1405
        }

        mm_missing_idxs = {
            modality: [
1406
1407
                idx
                for idx, item_is_cached in enumerate(items_is_cached)
1408
1409
1410
1411
                if not item_is_cached
            ]
            for modality, items_is_cached in mm_is_cached.items()
        }
1412
1413
1414
1415
1416
1417
1418
1419
        mm_missing_data = {}
        for modality, idxs in mm_missing_idxs.items():
            missing_modality_data = []
            for idx in idxs:
                data = mm_data_items[modality][idx]
                if data is None:
                    raise ValueError(
                        f"Cache miss for {modality} at index {idx} "
1420
1421
                        f"but data is not provided."
                    )
1422
1423
1424
                else:
                    missing_modality_data.append(data)
            mm_missing_data[modality] = missing_modality_data
1425

1426
        return mm_is_cached, self._to_mm_items(mm_missing_data)
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438

    def _recompute_cached_prompt_update(
        self,
        cached_update: ResolvedPromptUpdate,
        new_item_idx: int,
    ) -> ResolvedPromptUpdate:
        """
        Override this if other attributes of `ResolvedPromptUpdate`
        also need to be recomputed after retrieving from the cache.
        """
        return replace(cached_update, item_idx=new_item_idx)

1439
1440
    def _merge_mm_kwargs(
        self,
1441
        cache: BaseMultiModalProcessorCache,
1442
        mm_hashes: MultiModalHashes,
1443
        mm_is_cached: MultiModalIsCached,
1444
        mm_missing_kwargs: MultiModalKwargsItems,
1445
1446
        mm_missing_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[MultiModalKwargsOptionalItems, MultiModalPromptUpdates]:
1447
1448
1449
1450
1451
        # Need to touch all mm hashes before update to avoid hash in updated
        # list evict during update
        for hashes in mm_hashes.values():
            for item_hash in hashes:
                cache.touch_sender_cache_item(item_hash)
1452

1453
        mm_missing_next_idx = defaultdict[str, int](lambda: 0)
1454

1455
        merged_kwargs = defaultdict[str, list[MultiModalKwargsItem | None]](list)
1456
1457
1458
        merged_prompt_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](
            list
        )
1459
1460
        for modality, hashes in mm_hashes.items():
            missing_kwargs = mm_missing_kwargs.get(modality, [])
1461
            missing_prompt_updates = mm_missing_prompt_updates.get(modality, [])
1462
1463
1464
1465

            for item_idx, item_hash in enumerate(hashes):
                if not mm_is_cached[modality][item_idx]:
                    missing_next_idx = mm_missing_next_idx[modality]
1466
1467
                    missing_kwargs_item = missing_kwargs[missing_next_idx]
                    missing_updates_item = missing_prompt_updates[missing_next_idx]
1468

1469
                    mm_missing_next_idx[modality] += 1
1470

1471
                    item = missing_kwargs_item, missing_updates_item
1472
                else:
1473
1474
1475
1476
1477
                    item = None

                kwargs, updates = cache.get_and_update_item(item, item_hash)

                merged_kwargs[modality].append(kwargs)
1478
1479
1480
1481
1482
1483
                merged_prompt_updates[modality].append(
                    [
                        self._recompute_cached_prompt_update(update, item_idx)
                        for update in updates
                    ]
                )
1484

1485
1486
        mm_kwargs = MultiModalKwargsItems(merged_kwargs)
        mm_prompt_updates = dict(merged_prompt_updates)
1487

1488
        return mm_kwargs, mm_prompt_updates
1489
1490
1491

    def _apply_hf_processor(
        self,
1492
        prompt: str | list[int],
1493
1494
        mm_data_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1495
        tokenization_kwargs: Mapping[str, object],
1496
        *,
1497
        mm_uuids: MultiModalUUIDDict | None = None,
1498
    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1499
1500
        (
            prompt_ids,
1501
            mm_processed_data,
1502
1503
1504
1505
1506
            is_update_applied,
        ) = self._apply_hf_processor_main(
            prompt=prompt,
            mm_items=mm_data_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1507
            tokenization_kwargs=tokenization_kwargs,
1508
1509
1510
            enable_hf_prompt_update=True,
        )

1511
        mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
1512
            mm_processed_data,
1513
            self._get_mm_fields_config(mm_processed_data, hf_processor_mm_kwargs),
1514
1515
        )

1516
        # Use overrides if provided; fallback to data-dependent hashing.
1517
        with timed_preprocessor_operation(self.info.ctx, "hashing"):
1518
1519
1520
1521
1522
1523
            mm_hashes = self._hash_mm_items(
                mm_data_items,
                hf_processor_mm_kwargs,
                tokenization_kwargs,
                mm_uuids=mm_uuids,
            )
1524

1525
        mm_prompt_updates = self._get_mm_prompt_updates(
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
            mm_data_items,
            hf_processor_mm_kwargs,
            mm_kwargs,
        )

        mm_info = MultiModalProcessingInfo(
            kwargs=mm_kwargs,
            hashes=mm_hashes,
            prompt_updates=mm_prompt_updates,
        )

        return prompt_ids, mm_info, is_update_applied
1538

1539
1540
    def _cached_apply_hf_processor(
        self,
1541
        prompt: str | list[int],
1542
1543
        mm_data_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1544
        tokenization_kwargs: Mapping[str, object],
1545
        *,
1546
        mm_uuids: MultiModalUUIDDict | None = None,
1547
    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1548
1549
1550
1551
1552
1553
        """
        Apply the HF processor on the full prompt text,
        caching the results and reusing cached results.
        """
        cache = self.cache

1554
1555
        _, passthrough_data = self._get_hf_mm_data(mm_data_items)
        if cache is None or passthrough_data:
1556
            return self._apply_hf_processor(
1557
                prompt=prompt,
1558
                mm_data_items=mm_data_items,
1559
                hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1560
                tokenization_kwargs=tokenization_kwargs,
1561
                mm_uuids=mm_uuids,
1562
1563
            )

1564
        with timed_preprocessor_operation(self.info.ctx, "hashing"):
1565
1566
1567
1568
1569
1570
            mm_hashes = self._hash_mm_items(
                mm_data_items,
                hf_processor_mm_kwargs,
                tokenization_kwargs,
                mm_uuids=mm_uuids,
            )
1571

1572
        with timed_preprocessor_operation(self.info.ctx, "cache_lookup"):
1573
1574
1575
1576
1577
            mm_is_cached, mm_missing_data_items = self._get_cache_missing_items(
                cache=cache,
                mm_data_items=mm_data_items,
                mm_hashes=mm_hashes,
            )
1578

1579
        # NOTE: `prompt` does not correspond to `mm_missing_data_items`,
1580
        # so we can't apply prompt updates until the new multimodal
1581
1582
1583
        # items are combined with the cached multimodal items
        (
            prompt_ids,
1584
            mm_missing_processed_data,
1585
            is_update_applied,
1586
        ) = self._apply_hf_processor_main(
1587
            prompt=prompt,
1588
            mm_items=mm_missing_data_items,
1589
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1590
            tokenization_kwargs=tokenization_kwargs,
1591
            enable_hf_prompt_update=False,
1592
1593
        )

1594
        mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
1595
            mm_missing_processed_data,
1596
1597
1598
            self._get_mm_fields_config(
                mm_missing_processed_data, hf_processor_mm_kwargs
            ),
1599
1600
        )

1601
1602
1603
1604
        mm_missing_prompt_updates = self._get_mm_prompt_updates(
            mm_missing_data_items,
            hf_processor_mm_kwargs,
            mm_missing_kwargs,
1605
        )
1606

1607
        with timed_preprocessor_operation(self.info.ctx, "cache_lookup"):
1608
1609
1610
1611
1612
1613
1614
            mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
                cache,
                mm_hashes=mm_hashes,
                mm_is_cached=mm_is_cached,
                mm_missing_kwargs=mm_missing_kwargs,
                mm_missing_prompt_updates=mm_missing_prompt_updates,
            )
1615
1616
1617

        mm_info = MultiModalProcessingInfo(
            kwargs=mm_kwargs,
1618
            hashes=mm_hashes,
1619
1620
            prompt_updates=mm_prompt_updates,
        )
1621

1622
        return prompt_ids, mm_info, is_update_applied
1623

1624
1625
1626
    def _apply_token_matches(
        self,
        prompt: list[int],
1627
1628
1629
1630
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
        tokenizer = self.info.get_tokenizer()
        return apply_token_matches(prompt, mm_prompt_updates, tokenizer)
1631
1632
1633
1634

    def _apply_text_matches(
        self,
        prompt: str,
1635
1636
1637
1638
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[str, MultiModalPromptUpdatesApplyResult]:
        tokenizer = self.info.get_tokenizer()
        return apply_text_matches(prompt, mm_prompt_updates, tokenizer)
1639

1640
    def _apply_prompt_updates(
1641
1642
        self,
        token_ids: list[int],
1643
        mm_prompt_updates: MultiModalPromptUpdates,
1644
    ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
1645
        tokenizer = self.info.get_tokenizer()
1646

1647
1648
1649
1650
        new_token_ids, match_result = self._apply_token_matches(
            token_ids,
            mm_prompt_updates,
        )
1651
1652
1653
1654
1655
1656
1657
1658
1659

        # If the search text does not represent a special token,
        # it may have different token IDs in the prompt, because
        # the tokens may go across the boundaries of the search text.
        # ----
        # e.g. when searching for "foo" in "food", if "food" itself makes
        # up a token, then the token ID of "foo" will not appear at all
        # ----
        # Since it is inefficient to search for all possible tokenizations
1660
1661
        # of the search text in the prompt, we instead perform string-based
        # updates on the decoded token IDs, then encode them back.
1662
        if not all(
1663
1664
1665
            all(update_idx is not None for update_idx in update_idxs)
            for update_idxs in match_result.values()
        ):
1666
            new_text, match_result = self._apply_text_matches(
1667
                _seq2text(tokenizer, token_ids, use_cache=False),
1668
                mm_prompt_updates,
1669
1670
            )

1671
            new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False)
1672

1673
        matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
1674
1675
1676
1677
        for modality, update_idxs in match_result.items():
            for item_idx, update_idx in enumerate(update_idxs):
                assert update_idx is not None, (
                    "Failed to apply prompt replacement for "
1678
1679
                    f"mm_items[{modality!r}][{item_idx}]"
                )
1680
1681

                matched_updates[modality].append(
1682
1683
                    [mm_prompt_updates[modality][item_idx][update_idx]]
                )
1684
1685

        placeholders = self._find_mm_placeholders(
1686
1687
            new_token_ids,
            dict(matched_updates),
1688
        )
1689

1690
        return new_token_ids, placeholders
1691

1692
1693
    def _validate_mm_kwargs(
        self,
1694
        mm_kwargs: MultiModalKwargsOptionalItems,
1695
1696
1697
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
1698
            items = mm_kwargs.get(modality, [])
1699
1700
1701
1702
1703
1704
1705
1706
1707

            if len(items) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} {modality} items in "
                    f"keyword arguments corresponding to {item_count} "
                    f"{modality} data items, but only found {len(items)}! "
                    "There is likely a problem with your "
                    "implementation of merged multi-modal processor for this "
                    "model (usually arising from an inconsistency between "
1708
1709
                    "`_call_hf_processor` and `_get_mm_fields_config`)."
                )
1710

1711
    def _validate_mm_updates(
1712
        self,
1713
        mm_updates: MultiModalPromptUpdates,
1714
        mm_item_counts: Mapping[str, int],
1715
    ) -> None:
1716
        for modality, item_count in mm_item_counts.items():
1717
            placeholders = mm_updates.get(modality, [])
1718

1719
            if len(placeholders) != item_count:
1720
                raise RuntimeError(
1721
                    f"Expected there to be {item_count} prompt updates "
1722
                    f"corresponding to {item_count} {modality} items, but "
1723
                    f"instead found {len(placeholders)} prompt updates! "
1724
1725
1726
                    "This is likely because you forgot to include input "
                    "placeholder tokens (e.g., `<image>`, `<|image_pad|>`) "
                    "in the prompt. If the model has a chat template, make "
1727
1728
                    "sure you have applied it before calling `LLM.generate`."
                )
1729

1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
    def _validate_mm_placeholders(
        self,
        mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
            placeholders = mm_placeholders.get(modality, [])

            if len(placeholders) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} prompt placeholders "
                    f"corresponding to {item_count} {modality} items, but "
                    f"instead found {len(placeholders)} prompt placeholders! "
                    "Make sure the implementation of `_call_hf_processor` and "
1744
1745
                    "`_get_mm_fields_config` are consistent with each other."
                )
1746

1747
1748
1749
1750
    def _maybe_apply_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        prompt_ids: list[int],
1751
        mm_kwargs: MultiModalKwargsOptionalItems,
1752
        mm_prompt_updates: MultiModalPromptUpdates,
1753
        is_update_applied: bool,
1754
    ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
1755
        mm_item_counts = mm_items.get_all_counts()
1756
        self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
1757
        self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
1758

1759
        if is_update_applied:
1760
1761
            mm_placeholders = self._find_mm_placeholders(
                prompt_ids,
1762
                mm_prompt_updates,
1763
            )
1764
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1765
        else:
1766
            prompt_ids, mm_placeholders = self._apply_prompt_updates(
1767
                prompt_ids,
1768
                mm_prompt_updates,
1769
            )
1770
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1771

1772
        return prompt_ids, mm_placeholders
1773
1774
1775

    def apply(
        self,
1776
        prompt: str | list[int],
1777
1778
        mm_data: MultiModalDataDict,
        hf_processor_mm_kwargs: Mapping[str, object],
1779
        tokenization_kwargs: Mapping[str, object] | None = None,
1780
        *,
1781
        mm_uuids: MultiModalUUIDDict | None = None,
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
    ) -> MultiModalInputs:
        """
        Process multi-modal inputs to be used in vLLM.

        The main steps are:

        1. Apply HF Processor on prompt text and multi-modal data together,
           outputting token IDs and processed tensors.
        2. Find and update sequences in the token IDs with placeholder tokens.
           The number of placeholder tokens equals the feature size of the
           multi-modal data outputted by the multi-modal encoder.
        3. Extract information about the placeholder tokens from the
           processed token IDs.
        """
1796
1797
1798
1799
        request_id = get_current_request_id()
        if request_id is not None:
            self.info.ctx.create_timing_stats(request_id)

1800
1801
        mm_items = self._to_mm_items(mm_data)

1802
1803
1804
        if tokenization_kwargs is None:
            tokenization_kwargs = {}

1805
1806
        (
            prompt_ids,
1807
            mm_info,
1808
1809
1810
1811
1812
            is_update_applied,
        ) = self._cached_apply_hf_processor(
            prompt,
            mm_items,
            hf_processor_mm_kwargs,
1813
            tokenization_kwargs=tokenization_kwargs,
1814
            mm_uuids=mm_uuids,
1815
1816
        )

1817
        # NOTE: tokenization_kwargs are not required to init processor
1818
        with timed_preprocessor_operation(self.info.ctx, "prompt_update"):
1819
1820
1821
1822
1823
1824
1825
            prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
                mm_items=mm_items,
                prompt_ids=prompt_ids,
                mm_kwargs=mm_info.kwargs,
                mm_prompt_updates=mm_info.prompt_updates,
                is_update_applied=is_update_applied,
            )
1826

1827
1828
1829
1830
        mm_placeholder_ranges = {
            modality: [item.to_range() for item in placeholders]
            for modality, placeholders in mm_placeholders.items()
        }
1831

1832
        return MultiModalInputs(
1833
            type="multimodal",
1834
            prompt_token_ids=prompt_ids,
1835
1836
            mm_kwargs=mm_info.kwargs,
            mm_hashes=mm_info.hashes,
1837
            mm_placeholders=mm_placeholder_ranges,
1838
        )
1839
1840
1841
1842
1843
1844


class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
    @abstractmethod
    def create_encoder_prompt(
        self,
1845
        prompt: str | list[int],
1846
        mm_data: MultiModalDataDict,
1847
    ) -> str | list[int]:
1848
        """
1849
        Create input prompt for the encoder. HF processor will be applied on
1850
1851
        this prompt during profiling and generation.
        """
1852
1853
        raise NotImplementedError

1854
1855
    def create_decoder_prompt(
        self,
1856
        prompt: str | list[int],
1857
        mm_data: MultiModalDataDict,
1858
    ) -> str | list[int]:
1859
1860
1861
        """Create input prompt for the decoder."""
        return prompt

1862
    def _get_enc_dec_inputs(
1863
        self,
1864
        prompt: str | list[int],
1865
        mm_data: MultiModalDataDict,
1866
1867
        encoder_inputs: MultiModalInputs,
    ):
1868
        tokenizer = self.info.get_tokenizer()
1869
1870
        decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_data)
        if isinstance(decoder_prompt_raw, str):
1871
1872
            decoder_prompt_ids = tokenizer.encode(
                decoder_prompt_raw, add_special_tokens=False
1873
            )
1874
        else:
1875
            decoder_prompt_ids = decoder_prompt_raw
1876
1877
1878

        mm_inputs = MultiModalEncDecInputs(
            encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],
1879
1880
            **encoder_inputs,
        )
1881
        mm_inputs["prompt_token_ids"] = decoder_prompt_ids
1882
        return mm_inputs
1883
1884
1885

    def apply(
        self,
1886
        prompt: str | list[int],
1887
1888
        mm_data: MultiModalDataDict,
        hf_processor_mm_kwargs: Mapping[str, object],
1889
        tokenization_kwargs: Mapping[str, object] | None = None,
1890
        *,
1891
        mm_uuids: MultiModalUUIDDict | None = None,
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
    ) -> MultiModalEncDecInputs:
        """
        Process multi-modal inputs to be used in vLLM.
        The main processing steps are modified to fit encoder-decoder model:
        1. Create encoder prompt from input prompt text.
        2. Apply the HF processor on encoder prompt.
        3. Copy the input prompt text as decoder prompt inputs.
        """
        encoder_prompt = self.create_encoder_prompt(prompt, mm_data)
        encoder_inputs = super().apply(
            encoder_prompt,
            mm_data,
            hf_processor_mm_kwargs,
1905
            tokenization_kwargs,
1906
            mm_uuids=mm_uuids,
1907
1908
1909
1910
1911
1912
1913
        )

        return self._get_enc_dec_inputs(
            prompt=prompt,
            mm_data=mm_data,
            encoder_inputs=encoder_inputs,
        )