processor.py 61.1 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from abc import ABC, abstractmethod
4
from collections import defaultdict
5
from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence
6
from dataclasses import dataclass, field, replace
7
from enum import Enum
8
from functools import lru_cache
9
10
11
12
13
from typing import (
    TYPE_CHECKING,
    Generic,
    NamedTuple,
    Protocol,
14
    TypeAlias,
15
16
    cast,
)
17

18
import regex as re
19
import torch
20
from typing_extensions import TypeVar, assert_never, deprecated
21

22
from vllm.logger import init_logger
23
from vllm.tokenizers import TokenizerLike
24
from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
25

26
27
from ..hasher import MultiModalHasher
from ..inputs import (
28
29
    MultiModalEncDecInputs,
    MultiModalFieldConfig,
30
    MultiModalHashes,
31
32
33
34
35
36
37
    MultiModalInputs,
    MultiModalKwargsItem,
    MultiModalKwargsItems,
    MultiModalKwargsOptionalItems,
    MultiModalUUIDDict,
    PlaceholderRange,
)
38
from ..parse import (
39
40
41
42
    DictEmbeddingItems,
    EmbeddingItems,
    MultiModalDataItems,
)
43
44
45
46
47
from .context import (
    BaseProcessingInfo,
    get_current_request_id,
    timed_preprocessor_operation,
)
48
from .dummy_inputs import BaseDummyInputsBuilder
49
50

if TYPE_CHECKING:
51
52
    from transformers.feature_extraction_utils import BatchFeature

53
    from ..cache import BaseMultiModalProcessorCache
54
55
56
57
else:
    BatchFeature = object

    BaseMultiModalProcessorCache = object
58

59
logger = init_logger(__name__)
60
61

_S = TypeVar("_S", str, list[int])
62

63

64
PromptSeq: TypeAlias = str | list[int]
65
"""A token sequence (list of token IDs) or text."""
66

67

68
69
@lru_cache(maxsize=2048)
def _cached_encode(
70
    tokenizer: TokenizerLike,
71
72
    text: str,
    *,
73
    add_special_tokens: bool = True,
74
) -> list[int]:
75
    return tokenizer.encode(text, add_special_tokens=add_special_tokens)
76
77
78
79


@lru_cache(maxsize=2048)
def _cached_decode(
80
    tokenizer: TokenizerLike,
81
82
    token_ids: tuple[int, ...],
    *,
83
    skip_special_tokens: bool = False,
84
) -> str:
85
    return tokenizer.decode(list(token_ids), skip_special_tokens=skip_special_tokens)
86
87


88
89
90
91
92
93
def _seq2text(
    tokenizer: TokenizerLike | None,
    seq: PromptSeq,
    *,
    use_cache: bool = True,
) -> str:
94
95
96
    if isinstance(seq, str):
        return seq

97
98
99
100
    if tokenizer is None:
        raise ValueError("You cannot decode tokens when `skip_tokenizer_init=True`")

    if not use_cache:
101
        return tokenizer.decode(seq)
102

103
104
105
    return _cached_decode(tokenizer, tuple(seq))


106
107
108
109
110
111
def _seq2tokens(
    tokenizer: TokenizerLike | None,
    seq: PromptSeq,
    *,
    use_cache: bool = True,
) -> list[int]:
112
    if isinstance(seq, str):
113
114
115
116
        if tokenizer is None:
            raise ValueError("You cannot encode text when `skip_tokenizer_init=True`")

        if not use_cache:
117
            return tokenizer.encode(seq, add_special_tokens=False)
118

119
120
121
122
123
        return _cached_encode(tokenizer, seq, add_special_tokens=False)

    return seq


124
125
126
class _GetMatchIndex(Protocol):
    def __call__(
        self,
127
        tokenizer: TokenizerLike | None,
128
129
        prompt: PromptSeq,
        start_idx: int = 0,
130
    ) -> int | None: ...
131
132


133
134
135
@dataclass
class PromptIndex:
    """Resolves to an index in the prompt."""
136

137
    get_match_index: _GetMatchIndex
138
139
140
141
142
143
144
145
146
147


class PromptIndexTargets:
    @staticmethod
    def start() -> PromptIndex:
        """
        Resolves to the start of the prompt (before the first token).

        This results in a match even if the prompt is empty.
        """
148
        return PromptIndex(lambda tokenizer, prompt, start_idx=0: 0)
149
150
151
152
153
154
155
156

    @staticmethod
    def prefix(seq: PromptSeq) -> PromptIndex:
        """
        Resolves to a location in the prompt after the given prefix.
        """

        def get_match_index(
157
            tokenizer: TokenizerLike | None,
158
            prompt: PromptSeq,
159
            start_idx: int = 0,
160
        ) -> int | None:
161
162
163
            if start_idx != 0:
                return None

164
165
166
            prefix = seq

            if isinstance(prompt, str):
167
168
                # Make both `str`
                prefix = _seq2text(tokenizer, prefix, use_cache=False)
169
            else:
170
171
                # Make both `list[int]`
                prefix = _seq2tokens(tokenizer, prefix, use_cache=False)
172
173
174
175
176
177
178
179
180
181
182
183
184

            match_idx = len(prefix)
            return match_idx if prompt[:match_idx] == prefix else None

        return PromptIndex(get_match_index)

    @staticmethod
    def end() -> PromptIndex:
        """
        Resolves to the end of the prompt (after the last token).

        This results in a match even if the prompt is empty.
        """
185
        return PromptIndex(lambda tokenizer, prompt, start_idx=0: len(prompt))
186
187


188
UpdateTarget: TypeAlias = PromptSeq | PromptIndex
189
190
191
192
"""
The token sequence or text to update.
"""

193
PromptUpdateTarget: TypeAlias = Callable[[int], UpdateTarget] | UpdateTarget
194
195
196
197
198
199
200
201
202
"""
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
output the corresponding token sequence (or text).

For convenience, you can directly pass in the token sequence (or text)
instead of a function if it does not depend on the input.
"""

203

204
@dataclass
205
class PromptUpdateDetails(Generic[_S]):
206
    """Details about the token sequence or text that are part of the update."""
207

208
    full: _S
209
    """The full content."""
210

211
    is_embed: Callable[[TokenizerLike | None, PromptSeq], torch.Tensor] | None = None
212
    """
213
214
215
    Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full],
    return a boolean mask of shape `(len(full),)` indicating which positions
    of `full` to assign embeddings to.
216
217
218
219

    `None` (default) means to assign embeddings to all positions of `full`.

    The embeddings are obtained by calling
220
    [`SupportsMultiModal.embed_multimodal`][vllm.model_executor.models.interfaces.SupportsMultiModal.embed_multimodal].
221
222
223
    """

    @staticmethod
224
    def from_seq(seq: _S) -> "PromptUpdateDetails[_S]":
225
226
227
228
229
230
231
        return PromptUpdateDetails(full=seq)

    @staticmethod
    def select_text(
        seq: _S,
        embed_text: str,
    ) -> "PromptUpdateDetails[_S]":
232
233
        def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
            embed_token_ids = _seq2tokens(tokenizer, embed_text, use_cache=False)
234
            token_ids = _seq2tokens(tokenizer, full)
235
236

            return torch.isin(
237
                torch.tensor(token_ids),
238
239
240
241
242
243
244
245
246
247
                torch.tensor(embed_token_ids),
            )

        return PromptUpdateDetails(full=seq, is_embed=is_embed)

    @staticmethod
    def select_token_id(
        seq: _S,
        embed_token_id: int,
    ) -> "PromptUpdateDetails[_S]":
248
        def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
249
250
251
252
253
            token_ids = _seq2tokens(tokenizer, full)

            return torch.tensor(token_ids) == embed_token_id

        return PromptUpdateDetails(full=seq, is_embed=is_embed)
254

255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
    @staticmethod
    def select_token_ids(
        seq: _S,
        embed_token_ids: list[int],
    ) -> "PromptUpdateDetails[_S]":
        def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
            token_ids = _seq2tokens(tokenizer, full)

            return torch.isin(
                torch.tensor(token_ids),
                torch.tensor(embed_token_ids),
            )

        return PromptUpdateDetails(full=seq, is_embed=is_embed)

270

271
PromptUpdateInfo: TypeAlias = PromptSeq | PromptUpdateDetails
272
"""
273
The token sequence or text that are part of the update.
274

275
If only part of the content corresponds to feature placeholders, you can
276
277
use [`PromptUpdateDetails`][vllm.multimodal.processing.PromptUpdateDetails] to
specify which part.
278
"""
279

280
PromptUpdateContent: TypeAlias = Callable[[int], PromptUpdateInfo] | PromptUpdateInfo
281
"""
282
283
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
284
285
286
287
288
289
290
291
292
293
294
295
296
output the corresponding token sequence (or text).

For convenience, you can directly pass in the token sequence (or text)
instead of a function if it does not depend on the input.
"""


class UpdateMode(str, Enum):
    INSERT = "insert"
    REPLACE = "replace"


@dataclass
297
class PromptUpdate(ABC):
298
299
300
301
302
303
304
    """
    Defines how to update a prompt with placeholder tokens.
    """

    modality: str
    """The modality for which the update is made."""

305
    target: PromptUpdateTarget
306
307
308
309
310
311
312
313
314
315
316
317
318
319
    """The token sequence (or text) to update."""

    @property
    @abstractmethod
    def content(self) -> PromptUpdateContent:
        """The placeholder tokens that are part of the update."""
        raise NotImplementedError

    @property
    @abstractmethod
    def mode(self) -> UpdateMode:
        """Defines how to update the prompt."""
        raise NotImplementedError

320
    def _resolve_target(self, item_idx: int) -> UpdateTarget:
321
322
323
324
        target = self.target
        if callable(target):
            target = target(item_idx)

325
        return target
326

327
    def _resolve_content(self, item_idx: int) -> PromptUpdateDetails:
328
329
330
331
332
333
334
        content = self.content
        if callable(content):
            content = content(item_idx)

        if not isinstance(content, PromptUpdateDetails):
            content = PromptUpdateDetails.from_seq(content)

335
        return content
336

337
    def resolve(self, item_idx: int) -> "ResolvedPromptUpdate":
338
339
340
341
342
343
344
345
346
        """
        Given the index of the processed item within
        [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
        output a copy of this object with its lazy attributes resolved.
        """
        return ResolvedPromptUpdate(
            modality=self.modality,
            item_idx=item_idx,
            mode=self.mode,
347
348
            target=self._resolve_target(item_idx),
            content=self._resolve_content(item_idx),
349
350
        )

351

352
@dataclass
353
354
355
356
357
358
class PromptInsertion(PromptUpdate):
    """
    Defines how to insert placeholder tokens into a prompt.

    Example:

359
360
    For each image, insert a number of `<image>` feature placeholders
    equal to the feature size of the vision encoder after the `<s>` token:
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379

    ```python
    PromptInsertion(
        modality="image",
        target="<s>",
        insertion="<image>" * image_feature_size,
    )
    ```

    Insert these tokens at the start of the prompt:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.start(),
        insertion="<image>" * image_feature_size,
    )
    ```

380
    Insert these tokens after a prefix `Images:`:
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.prefix("Images:"),
        insertion="<image>" * image_feature_size,
    )
    ```

    Insert these tokens at the end of the prompt:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.end(),
        insertion="<image>" * image_feature_size,
    )
    ```
399
400
401
402
    """

    insertion: PromptUpdateContent = field(repr=False)
    """
403
404
405
406
    Given the index of the processed item within
    [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
    output the token sequence (or text) to insert right after
    [`target`][vllm.multimodal.processing.PromptUpdate.target].
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422

    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
    """

    @property
    def content(self) -> PromptUpdateContent:
        return self.insertion

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.INSERT


@dataclass
class PromptReplacement(PromptUpdate):
423
424
    """
    Defines how to replace portions of an input prompt with placeholder tokens.
425
426
427

    Example:

428
429
    For each image, replace one `<image>` input placeholder in the prompt
    with a number of `<image>` feature placeholders
430
431
432
433
434
435
436
437
438
439
    equal to the feature size of the vision encoder:

    ```python
    PromptReplacement(
        modality="image",
        target="<image>",
        replacement="<image>" * image_feature_size,
    )
    ```

440
441
    As above, but further pad the feature placeholders with `<image_bos>`
    and `<image_eos>`, which are not supposed to be passed to the vision
442
443
444
445
446
447
448
    encoder:

    ```python
    PromptReplacement(
        modality="image",
        target="<image>",
        replacement=PromptUpdateDetails(
449
450
451
452
453
454
455
            full="".join(
                [
                    "<image_bos>",
                    "<image>" * image_feature_size,
                    "<image_eos>",
                ]
            ),
456
457
458
459
460
461
462
463
464
465
466
467
468
            features="<image>" * image_feature_size,
        ),
    )
    ```

    To avoid unnecessary tokenization during prompt replacement,
    we recommended passing token sequences instead of text:

    ```python
    PromptReplacement(
        modality="image",
        target=[image_token_id],
        replacement=PromptUpdateDetails(
469
470
471
            full=(
                [image_bos_id] + [image_token_id] * image_feature_size + [image_eos_id]
            ),
472
473
474
475
            features=[image_token_id] * image_feature_size,
        ),
    )
    ```
476
477
    """

478
    replacement: PromptUpdateContent = field(repr=False)
479
    """
480
481
482
483
    Given the index of the processed item within
    [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
    output the token sequence (or text) to replace
    [`target`][vllm.multimodal.processing.PromptUpdate.target].
484

485
486
    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
487
488
    """

489
490
491
492
493
494
495
    @property
    def content(self) -> PromptUpdateContent:
        return self.replacement

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.REPLACE
496
497


498
499
500
class _HasModalityAttr(Protocol):
    modality: str

501

502
503
class _HasModalityProp(Protocol):
    @property
504
    def modality(self) -> str: ...
505
506


507
_M = TypeVar("_M", bound=_HasModalityAttr | _HasModalityProp)
508
509
510


def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]:
511
512
    """
    Convenience function to apply
513
    [`full_groupby`][vllm.utils.collection_utils.full_groupby]
514
515
    based on modality.
    """
516
517
518
    return full_groupby(values, key=lambda x: x.modality)


519
520
521
522
523
524
525
class PromptTargetMatch(NamedTuple):
    start_idx: int
    end_idx: int


@dataclass(frozen=True)
class ResolvedPromptUpdate:
526
    """
527
528
    A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] with its
    lazy attributes resolved, apart from those related to tokenization.
529
    """
530

531
532
    modality: str
    """The modality for which the update is made."""
533

534
535
    item_idx: int
    """The index within `modality` of the item this update pertains to."""
536

537
538
    mode: UpdateMode
    """Defines how to update the prompt."""
539

540
    target: UpdateTarget
541
    """The token sequence (or text) to update."""
542

543
    content: PromptUpdateDetails = field(repr=False)
544
    """The placeholder tokens that are part of the update."""
545

546
547
548
    def iter_token_matches(
        self,
        prompt: list[int],
549
        tokenizer: TokenizerLike | None,
550
551
552
553
554
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        target = self.target
555

556
557
558
559
        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(tokenizer, prompt, start_idx)
            if match_idx is not None:
                yield PromptTargetMatch(match_idx, match_idx)
560

561
            return
562

563
564
        target_token_ids = _seq2tokens(tokenizer, target)

565
        for match in iter_token_matches(prompt, target_token_ids, start_idx=start_idx):
566
            yield PromptTargetMatch(match.start_idx, match.end_idx)
567

568
569
570
    def iter_text_matches(
        self,
        prompt: str,
571
        tokenizer: TokenizerLike | None,
572
573
574
575
576
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        target = self.target
577

578
579
580
581
        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(tokenizer, prompt, start_idx)
            if match_idx is not None:
                yield PromptTargetMatch(match_idx, match_idx)
582

583
            return
584

585
586
        target_text = _seq2text(tokenizer, target)

587
        for match in re.finditer(re.escape(target_text), prompt, pos=start_idx):
588
589
590
591
            yield PromptTargetMatch(match.start(), match.end())

    def iter_matches(
        self,
592
        prompt: list[int] | str,
593
        tokenizer: TokenizerLike | None,
594
595
596
597
598
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        if isinstance(prompt, str):
599
            return self.iter_text_matches(prompt, tokenizer, start_idx=start_idx)
600
601

        return self.iter_token_matches(prompt, tokenizer, start_idx=start_idx)
602

603
604
605
606
607
608
609
610
611
    def with_target(self, target: UpdateTarget):
        return replace(self, target=target)

    def with_content(self, content: PromptUpdateInfo):
        if not isinstance(content, PromptUpdateDetails):
            content = PromptUpdateDetails.from_seq(content)

        return replace(self, content=content)

612

613
614
615
class _TokenMatch(NamedTuple):
    start_idx: int
    end_idx: int
616
617


618
619
620
def iter_token_matches(
    token_ids: list[int],
    match_ids: list[int],
621
622
    *,
    start_idx: int = 0,
623
) -> Generator[_TokenMatch]:
624
    """
625
    Yield each occurrence of `match_ids` in `token_ids`.
626
627
628
629

    Note that empty matches are ignored.
    """
    prompt_len = len(token_ids)
630
    match_len = len(match_ids)
631

632
633
    if match_len == 0:
        return
634

635
    while start_idx < prompt_len - match_len + 1:
636
        end_idx = start_idx + match_len
637

638
639
        if token_ids[start_idx:end_idx] == match_ids:
            yield _TokenMatch(start_idx=start_idx, end_idx=end_idx)
640
641
642
643
644

            # Exclude overlapping matches
            start_idx = end_idx
        else:
            start_idx += 1
645
646


647
648
649
650
651
652
def replace_token_matches(
    token_ids: list[int],
    match_ids: list[int],
    new_ids: list[int],
) -> list[int]:
    """
653
654
    Replace each occurrence of `match_ids` in `token_ids`
    with `new_ids`.
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673

    Note that empty matches are ignored.
    """
    out_seqs = list[list[int]]()
    prev_end_idx = 0

    for match in iter_token_matches(token_ids, match_ids):
        start_idx = match.start_idx
        end_idx = match.end_idx

        out_seqs.append(token_ids[prev_end_idx:start_idx])
        out_seqs.append(new_ids)
        prev_end_idx = end_idx

    out_seqs.append(token_ids[prev_end_idx:])

    return flatten_2d_lists(out_seqs)


674
@dataclass
675
class PlaceholderFeaturesInfo:
676
    modality: str
677
    item_idx: int
678
    start_idx: int
679
    tokens: list[int]
680
    is_embed: torch.Tensor | None
681
682
683

    @property
    def length(self) -> int:
684
        return len(self.tokens)
685
686

    def to_range(self) -> PlaceholderRange:
687
688
        # TODO: Is it worth it to optimize this by stripping the
        # leading and ending positions where `is_embed=False`?
689
690
691
        return PlaceholderRange(
            offset=self.start_idx,
            length=self.length,
692
            is_embed=self.is_embed,
693
        )
694
695


696
_MatchToApply = tuple[tuple[str, int], tuple[PromptTargetMatch, int]]
697
698


699
700
701
def _find_matches(
    prompt: _S,
    mm_prompt_updates: "MultiModalPromptUpdates",
702
    tokenizer: TokenizerLike | None,
703
704
705
    *,
    prev_end_idx: int = 0,
    current_result: "MultiModalPromptUpdatesApplyResult",
706
707
) -> tuple[UpdateMode | None, list[_MatchToApply]]:
    mode: UpdateMode | None = None
708
709
710
711
712
713
714
715
716
717
718
719
    mm_matches = dict[tuple[str, int], tuple[PromptTargetMatch, int]]()

    for modality, modality_updates in mm_prompt_updates.items():
        for item_idx, item_updates in enumerate(modality_updates):
            if current_result[modality][item_idx] is not None:
                continue  # Updates have already been applied for this item

            for update_idx, update in enumerate(item_updates):
                if (modality, item_idx) in mm_matches:
                    break  # Already found a match for this item

                for match in update.iter_matches(
720
721
722
                    prompt,
                    tokenizer,
                    start_idx=prev_end_idx,
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
                ):
                    # All matches should share the same mode
                    if mode is None:
                        mode = update.mode
                    elif mode != update.mode:
                        continue

                    mm_matches[(modality, item_idx)] = match, update_idx
                    break  # Get only the first valid match per item

    # Prioritize earlier matches
    matches_to_apply = sorted(mm_matches.items(), key=lambda item: item[1][0])

    # To avoid conflicts, only replace one non-empty item at a time
    if mode == UpdateMode.REPLACE:
        matches_to_apply_ = list[_MatchToApply]()
        has_non_empty_matches = False

        for item in matches_to_apply:
            _, (match, _) = item
            if match.start_idx == match.end_idx:
                matches_to_apply_.append(item)
            elif not has_non_empty_matches:
                has_non_empty_matches = True
                matches_to_apply_.append(item)

        matches_to_apply = matches_to_apply_

    return mode, matches_to_apply
752
753


754
755
756
757
758
759
760
761
762
763
def _all_items_found(
    mm_item_counts: dict[str, int],
    mm_found_counts: dict[str, int],
) -> bool:
    return all(
        item_idx >= mm_item_counts[modality]
        for modality, item_idx in mm_found_counts.items()
    )


764
def _apply_matches(
765
    prompt: _S,
766
    mm_prompt_updates: "MultiModalPromptUpdates",
767
    tokenizer: TokenizerLike | None,
768
) -> tuple[list[_S], "MultiModalPromptUpdatesApplyResult"]:
769
    mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()}
770

771
    out_seqs = list[str | list[int]]()
772
    out_result: MultiModalPromptUpdatesApplyResult = {
773
        m: [None] * len(items) for m, items in mm_prompt_updates.items()
774
    }
775

776
    # Early exit if no items to find
777
778
779
780
781
782
    mm_found_counts = {
        m: sum(r is not None for r in res) for m, res in out_result.items()
    }
    if _all_items_found(mm_item_counts, mm_found_counts):
        return [prompt], out_result

783
784
    prev_end_idx = 0
    while True:
785
786
787
788
789
790
791
        mode, matches_to_apply = _find_matches(
            prompt,
            mm_prompt_updates,
            tokenizer,
            prev_end_idx=prev_end_idx,
            current_result=out_result,
        )
792

793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
        if mode is None:
            break  # No more matches to find

        for (modality, item_idx), (match, update_idx) in matches_to_apply:
            matched_update = mm_prompt_updates[modality][item_idx][update_idx]
            matched_content = matched_update.content.full

            if mode == UpdateMode.INSERT:
                end_idx_to_insert = match.end_idx
            elif mode == UpdateMode.REPLACE:
                end_idx_to_insert = match.start_idx
            else:
                assert_never(mode)

            out_seqs.append(prompt[prev_end_idx:end_idx_to_insert])
            out_seqs.append(
                _seq2text(tokenizer, matched_content)
                if isinstance(prompt, str)
                else _seq2tokens(tokenizer, matched_content)
            )
            out_result[modality][item_idx] = update_idx

            # Exclude overlapping matches
            prev_end_idx = match.end_idx

        # Early exit if all items found
        mm_found_counts = {
            m: sum(r is not None for r in res) for m, res in out_result.items()
        }
        if _all_items_found(mm_item_counts, mm_found_counts):
            break
824
825
826

    out_seqs.append(prompt[prev_end_idx:])

827
    return cast(list[_S], out_seqs), out_result
828
829


830
def apply_token_matches(
831
    prompt: list[int],
832
    mm_prompt_updates: "MultiModalPromptUpdates",
833
    tokenizer: TokenizerLike | None,
834
835
836
) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
    """
    Apply the updates in `mm_prompt_updates` to `prompt`.
837

838
839
840
841
    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
    appears earlier in `mm_prompt_updates` takes priority.
    """
842
    token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
843

844
    return flatten_2d_lists(token_id_seqs), result
845
846


847
def apply_text_matches(
848
    prompt: str,
849
    mm_prompt_updates: "MultiModalPromptUpdates",
850
    tokenizer: TokenizerLike | None,
851
852
853
) -> tuple[str, "MultiModalPromptUpdatesApplyResult"]:
    """
    Apply the updates in `mm_prompt_updates` to `prompt`.
854

855
856
857
858
859
    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
    appears earlier in `mm_prompt_updates` takes priority.
    """
    texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
860

861
    return "".join(texts), result
862
863


864
def _iter_placeholders(
865
    prompt: list[int],
866
    mm_prompt_updates: "MultiModalPromptUpdates",
867
    tokenizer: TokenizerLike | None,
868
) -> Iterable[PlaceholderFeaturesInfo]:
869
    """
870
    Yield each set of placeholder tokens found in `prompt`.
871
872
873

    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
874
    appears earlier in `mm_prompt_updates` takes priority.
875

876
877
    Note that empty matches are ignored.
    """
878
    mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()}
879
    item_idx_by_modality = {modality: 0 for modality in mm_prompt_updates}
880

881
882
    if _all_items_found(mm_item_counts, item_idx_by_modality):
        return
883

884
    prompt_len = len(prompt)
885
    start_idx = 0
886

887
888
889
    while start_idx < prompt_len:
        found = False

890
        for modality, modality_updates in mm_prompt_updates.items():
891
892
            item_idx = item_idx_by_modality[modality]
            if item_idx >= mm_item_counts.get(modality, 0):
893
                continue
894

895
896
            for update in modality_updates[item_idx]:
                content = update.content
897
                content_tokens_full = _seq2tokens(tokenizer, content.full)
898
899
                content_len_full = len(content_tokens_full)
                end_idx_full = start_idx + content_len_full
900

901
                if content_len_full == 0 or end_idx_full > prompt_len:
902
903
                    continue

904
                if prompt[start_idx:end_idx_full] == content_tokens_full:
905
906
                    content_is_embed = content.is_embed
                    if content_is_embed is not None:
907
                        content_is_embed = content_is_embed(tokenizer, content.full)
908
909
910
911
912
913
914
915

                    yield PlaceholderFeaturesInfo(
                        modality=modality,
                        item_idx=item_idx,
                        start_idx=start_idx,
                        tokens=content_tokens_full,
                        is_embed=content_is_embed,
                    )
916

917
                    # Exclude overlapping matches
918
                    start_idx = end_idx_full
919
920
921
                    item_idx_by_modality[modality] += 1
                    found = True
                    break
922

923
            if found:
924
925
926
                if _all_items_found(mm_item_counts, item_idx_by_modality):
                    return

927
                break  # Go back to the outer while loop
928
929
930

        if not found:
            start_idx += 1
931
932


933
934
def find_mm_placeholders(
    prompt: list[int],
935
    mm_prompt_updates: "MultiModalPromptUpdates",
936
    tokenizer: TokenizerLike | None,
937
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
938
    it = _iter_placeholders(prompt, mm_prompt_updates, tokenizer)
939
940
941
    return dict(full_groupby_modality(it))


942
943
944
MultiModalIsCached = dict[str, list[bool]]
"""
A collection of the `is_cached` flag for each item, with a similar structure as
945
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
946
947
"""

948
MultiModalPromptUpdates = Mapping[str, list[Sequence[ResolvedPromptUpdate]]]
949
950
951
952
953
"""
A collection of prompt updates with a similar structure as
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
"""

954
MultiModalPromptUpdatesApplyResult = Mapping[str, list[int | None]]
955
956
957
958
959
960
961
"""
For an item `MultiModalPromptUpdates[k][i]`,
`MultiModalPromptUpdatesApplyResult[k][i]` represents the index of the
`ResolvedPromptUpdate` instance that has been applied, or `None` if none of the
`ResolvedPromptUpdate` instances have been applied.
"""

962
963
_I = TypeVar("_I", bound=BaseProcessingInfo)

964
965

class MultiModalProcessingInfo(NamedTuple):
966
    kwargs: MultiModalKwargsOptionalItems
967
    hashes: MultiModalHashes
968
969
    prompt_updates: MultiModalPromptUpdates

970
971

class BaseMultiModalProcessor(ABC, Generic[_I]):
972
    """
973
    Abstract base class to process multi-modal inputs to be used in vLLM.
974

975
    Not to be confused with `transformers.ProcessorMixin`.
976
977
    """

978
979
980
981
982
    def __init__(
        self,
        info: _I,
        dummy_inputs: "BaseDummyInputsBuilder[_I]",
        *,
983
        cache: BaseMultiModalProcessorCache | None = None,
984
    ) -> None:
985
986
        super().__init__()

987
988
        self.info = info
        self.dummy_inputs = dummy_inputs
989
        self.cache = cache
990

991
        # TODO: Remove in v0.18
992
        if hasattr(self, "_get_data_parser"):
993
994
995
996
            raise ValueError(
                "BaseMultiModalProcessor._get_data_parser has been "
                "moved to `BaseProcessingInfo.build_data_parser` in v0.16. "
                "You should override `BaseProcessingInfo.build_data_parser` instead."
997
998
            )

999
        self.data_parser = self.info.get_data_parser()
1000

1001
    @property
1002
    @deprecated("Will be removed in v0.17. Use `info.supported_mm_limits` instead.")
1003
    def supported_mm_limits(self):
1004
        return self.info.supported_mm_limits
1005
1006

    @property
1007
    @deprecated("Will be removed in v0.17. Use `info.allowed_mm_limits` instead.")
1008
    def allowed_mm_limits(self):
1009
        return self.info.allowed_mm_limits
1010

1011
    def __call__(
1012
        self,
1013
        prompt: str,
1014
        mm_items: MultiModalDataItems,
1015
        hf_processor_mm_kwargs: Mapping[str, object],
1016
        *,
1017
        mm_uuids: MultiModalUUIDDict | None = None,
1018
    ) -> MultiModalInputs:
1019
        return self.apply(prompt, mm_items, hf_processor_mm_kwargs, mm_uuids=mm_uuids)
1020

1021
1022
1023
    @abstractmethod
    def _get_mm_fields_config(
        self,
1024
        hf_inputs: BatchFeature,
1025
1026
1027
1028
1029
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, MultiModalFieldConfig]:
        """Given the HF-processed data, output the metadata of each field."""
        raise NotImplementedError

1030
    @abstractmethod
1031
    def _get_prompt_updates(
1032
        self,
1033
        mm_items: MultiModalDataItems,
1034
        hf_processor_mm_kwargs: Mapping[str, object],
1035
        out_mm_kwargs: MultiModalKwargsItems,
1036
    ) -> Sequence[PromptUpdate]:
1037
1038
        """
        Given the original multi-modal items for this modality
1039
        and HF-processed data, output the updates to perform.
1040

1041
1042
1043
1044
1045
1046
        The information returned by this method is used to update token inputs
        which bypass the HF processor. It is also used to update the output of
        HF processor if the HF process does not apply prompt updates to text
        inputs.

        Moreover, this information is critical to determine the token positions
1047
1048
        in order to construct
        [`PlaceholderRange`][vllm.multimodal.inputs.PlaceholderRange]
1049
        for each multi-modal item.
1050
1051
        """
        raise NotImplementedError
1052

1053
1054
1055
1056
1057
1058
    def _bind_and_group_updates(
        self,
        prompt_updates: Sequence[PromptUpdate],
        mm_item_counts: Mapping[str, int],
    ) -> MultiModalPromptUpdates:
        return {
1059
1060
1061
1062
            modality: [
                [update.resolve(item_idx) for update in updates]
                for item_idx in range(mm_item_counts.get(modality, 0))
            ]
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
            for modality, updates in full_groupby_modality(prompt_updates)
        }

    def _get_mm_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        out_mm_kwargs: MultiModalKwargsItems,
    ) -> MultiModalPromptUpdates:
        unbound_prompt_updates = self._get_prompt_updates(
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
            out_mm_kwargs=out_mm_kwargs,
        )

        mm_prompt_updates = self._bind_and_group_updates(
            unbound_prompt_updates,
            mm_items.get_all_counts(),
        )

        for modality, prompt_updates in mm_prompt_updates.items():
            for item_idx, item_prompt_updates in enumerate(prompt_updates):
                if len(item_prompt_updates) > 1:
                    logger.warning_once(
                        "Detected %d prompt updates for `mm_items[%r][%s]`. "
                        "Multiple prompt updates per item is now "
                        "deprecated and may be removed in v0.13. "
                        "Instead, please specify dynamic update targets "
                        "in the same prompt update definition by passing "
                        "a function to `PromptUpdate.target`.",
                        len(prompt_updates),
                        modality,
                        item_idx,
                    )

        return mm_prompt_updates

1100
    def _find_mm_placeholders(
1101
1102
        self,
        new_token_ids: list[int],
1103
        mm_prompt_updates: MultiModalPromptUpdates,
1104
    ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
1105
1106
        tokenizer = self.info.get_tokenizer()

1107
        return find_mm_placeholders(new_token_ids, mm_prompt_updates, tokenizer)
1108

1109
    def _get_hf_mm_data(
1110
        self,
1111
        mm_items: MultiModalDataItems,
1112
    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
1113
        """Extract processor and passthrough data from multi-modal items."""
1114
1115
        processor_data = dict[str, object]()
        passthrough_data = dict[str, object]()
1116

1117
1118
1119
        for items in mm_items.values():
            processor_data.update(items.get_processor_data())
            passthrough_data.update(items.get_passthrough_data())
1120

1121
1122
        return processor_data, passthrough_data

1123
1124
1125
    def _call_hf_processor(
        self,
        prompt: str,
1126
1127
1128
1129
        # Not to be confused with `mm_data` in `self.apply`.
        # This refers to the data to be passed to HF processor.
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
1130
        tok_kwargs: Mapping[str, object],
1131
    ) -> BatchFeature:
1132
1133
1134
1135
        """
        Call the HF processor on the prompt text and
        associated multi-modal data.
        """
1136
        with timed_preprocessor_operation(self.info.ctx, "hf_processor"):
1137
1138
1139
1140
1141
            return self.info.ctx.call_hf_processor(
                self.info.get_hf_processor(**mm_kwargs),
                dict(text=prompt, **mm_data),
                dict(**mm_kwargs, **tok_kwargs),
            )
1142

1143
    def _hf_processor_applies_updates(
1144
1145
1146
1147
        self,
        prompt_text: str,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1148
        tokenization_kwargs: Mapping[str, object],
1149
1150
    ) -> bool:
        """
1151
        Return whether the HF processor applies prompt updates.
1152

1153
1154
        For most HF processors, this should be `True` when multi-modal
        data items are passed, but `False` when multi-modal embeddings
1155
1156
1157
1158
        are passed.
        """
        return not any(
            isinstance(items, (EmbeddingItems, DictEmbeddingItems))
1159
1160
            for items in mm_items.values()
        )
1161

1162
    def _apply_hf_processor_text_mm(
1163
        self,
1164
        prompt_text: str,
1165
        mm_items: MultiModalDataItems,
1166
        hf_processor_mm_kwargs: Mapping[str, object],
1167
        tokenization_kwargs: Mapping[str, object],
1168
    ) -> tuple[list[int], BatchFeature, bool]:
1169
        """
1170
1171
        Apply the HF processor on the prompt text and multi-modal data
        together.
1172

1173
        In addition, return whether prompt updates have been applied.
1174
1175
1176
1177
1178
1179
1180
        """
        processor_data, passthrough_data = self._get_hf_mm_data(mm_items)

        processed_data = self._call_hf_processor(
            prompt=prompt_text,
            mm_data=processor_data,
            mm_kwargs=hf_processor_mm_kwargs,
1181
            tok_kwargs=tokenization_kwargs,
1182
1183
        )
        processed_data.update(passthrough_data)
1184

1185
        (prompt_ids,) = processed_data.pop("input_ids").tolist()
1186

1187
        is_update_applied = self._hf_processor_applies_updates(
1188
1189
1190
            prompt_text=prompt_text,
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1191
            tokenization_kwargs=tokenization_kwargs,
1192
1193
        )

1194
        return prompt_ids, processed_data, is_update_applied
1195

1196
    def _apply_hf_processor_text_only(
1197
1198
1199
1200
        self,
        prompt_text: str,
        tokenization_kwargs: Mapping[str, object],
    ) -> list[int]:
1201
        """
1202
        Apply the HF processor on the prompt text only.
1203

1204
1205
1206
        Since HF processor requires that text and multi-modal items
        correspond to each other, we create dummy multi-modal items
        to go along with the text.
1207
        """
1208
        prompt_ids, _, _ = self._apply_hf_processor_text_mm(
1209
1210
1211
            prompt_text=prompt_text,
            mm_items=MultiModalDataItems({}),
            hf_processor_mm_kwargs={},
1212
            tokenization_kwargs=tokenization_kwargs,
1213
1214
        )

1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
        return prompt_ids

    def _apply_hf_processor_tokens_only(
        self,
        prompt_tokens: list[int],
    ) -> list[int]:
        """
        Apply the HF processor on the prompt tokens only.

        Most HF processors accept prompt text but not prompt tokens.
        If the HF processor adds or removes tokens that are not related to
        multi-modal data, you should override this method so it is consistent
1227
1228
1229
        with the output of
        [`_apply_hf_processor_text_only`][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_text_only]
        on the
1230
1231
1232
1233
1234
1235
1236
1237
        corresponding text.
        """
        return prompt_tokens

    def _apply_hf_processor_mm_only(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1238
        tokenization_kwargs: Mapping[str, object],
1239
    ) -> BatchFeature:
1240
1241
1242
1243
1244
        """
        Apply the HF processor on the multi-modal data only.

        Since HF processor requires that text and multi-modal items
        correspond to each other, we generate dummy text using
1245
        [`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder]
1246
        to go along with the multi-modal data.
1247
1248
1249
        """
        mm_counts = mm_items.get_all_counts()

1250
        _, mm_processed_data, _ = self._apply_hf_processor_text_mm(
1251
            prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
1252
1253
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1254
            tokenization_kwargs=tokenization_kwargs,
1255
1256
        )

1257
        return mm_processed_data
1258
1259
1260

    def _apply_hf_processor_main(
        self,
1261
        prompt: str | list[int],
1262
1263
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1264
        tokenization_kwargs: Mapping[str, object],
1265
        *,
1266
        enable_hf_prompt_update: bool,
1267
    ) -> tuple[list[int], BatchFeature, bool]:
1268
1269
1270
        """
        Apply the HF processor on the prompt text and multi-modal data.

1271
        In addition, return whether prompt updates have been applied
1272
        (for most HF processors, this should be `True`).
1273

1274
        Note:
1275
            If `enable_hf_prompt_update=False`, we use HF processor
1276
            to perform prompt updates if available; HF processor requires
1277
            that the prompt corresponds to multi-modal items.
1278
1279
        """
        if isinstance(prompt, str):
1280
            if enable_hf_prompt_update:
1281
1282
1283
1284
                return self._apply_hf_processor_text_mm(
                    prompt_text=prompt,
                    mm_items=mm_items,
                    hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1285
                    tokenization_kwargs=tokenization_kwargs,
1286
1287
                )

1288
            prompt_ids = self._apply_hf_processor_text_only(prompt, tokenization_kwargs)
1289
1290
1291
        else:
            prompt_ids = self._apply_hf_processor_tokens_only(prompt)

1292
        mm_processed_data = self._apply_hf_processor_mm_only(
1293
            mm_items=mm_items,
1294
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1295
            tokenization_kwargs=tokenization_kwargs,
1296
1297
        )

1298
        return prompt_ids, mm_processed_data, False
1299

1300
    def _hash_mm_items(
1301
1302
1303
1304
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        tokenization_kwargs: Mapping[str, object],
1305
        *,
1306
        mm_uuids: MultiModalUUIDDict | None = None,
1307
    ) -> MultiModalHashes:
1308
        """Create MM hashes to be returned.
1309

1310

1311
1312
1313
        Note: When overrides are provided via callers of `apply`,
        `_hash_mm_items` will be bypassed and the overrides will be used.
        """
1314
1315
        model_id = self.info.model_id

1316
        hashes: MultiModalHashes = {}
1317
        mm_uuids = mm_uuids or {}
1318
1319

        for modality, items in mm_items.items():
1320
1321
1322
1323
            if modality in mm_uuids:
                mm_uuids_per_modality = mm_uuids[modality]
                if isinstance(mm_uuids_per_modality, str):
                    mm_uuids_per_modality = [mm_uuids_per_modality]
1324
1325
1326

                # For None entries, compute a hash; otherwise, use provided ID.
                computed: list[str] = []
1327
                for i, item in enumerate(items.get_all_items_for_hash()):
1328
                    item_uuid = mm_uuids_per_modality[i]
1329

1330
                    # NOTE: Even if a item_uuid is provided, we still compute a
1331
1332
1333
                    # hash if `hf_processor_mm_kwargs` or `tokenization_kwargs`
                    # are provided. This is because the processed multimodal
                    # inputs can be different depending on the processor kwargs.
1334
1335
1336
1337
1338
                    if (
                        item_uuid is None
                        or hf_processor_mm_kwargs
                        or tokenization_kwargs
                    ):
1339
1340
                        # NOTE: use provided hash string to hash with kwargs
                        # if available for better performance.
1341
                        item = item_uuid if item_uuid is not None else item
1342
1343
1344
1345
1346
                        computed.append(
                            MultiModalHasher.hash_kwargs(
                                model_id=model_id,
                                **{modality: item},
                                **hf_processor_mm_kwargs,
1347
1348
1349
                                **tokenization_kwargs,
                            )
                        )
1350
                    else:
1351
                        computed.append(item_uuid)
1352
1353
1354
                hashes[modality] = computed
            else:
                hashes[modality] = [
1355
1356
1357
1358
1359
1360
                    MultiModalHasher.hash_kwargs(
                        model_id=model_id,
                        **{modality: item},
                        **hf_processor_mm_kwargs,
                        **tokenization_kwargs,
                    )
1361
1362
1363
1364
                    for item in items
                ]

        return hashes
1365

1366
1367
    def _get_cache_missing_items(
        self,
1368
        cache: BaseMultiModalProcessorCache,
1369
1370
        mm_data_items: MultiModalDataItems,
        mm_hashes: MultiModalHashes,
1371
    ) -> tuple[MultiModalIsCached, MultiModalDataItems]:
1372
        mm_is_cached = {
1373
            modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
1374
1375
1376
1377
        }

        mm_missing_idxs = {
            modality: [
1378
1379
                idx
                for idx, item_is_cached in enumerate(items_is_cached)
1380
1381
1382
1383
                if not item_is_cached
            ]
            for modality, items_is_cached in mm_is_cached.items()
        }
1384

1385
1386
1387
1388
1389
1390
1391
1392
        mm_missing_data = {}
        for modality, idxs in mm_missing_idxs.items():
            missing_modality_data = []
            for idx in idxs:
                data = mm_data_items[modality][idx]
                if data is None:
                    raise ValueError(
                        f"Cache miss for {modality} at index {idx} "
1393
1394
                        f"but data is not provided."
                    )
1395
1396
1397
                else:
                    missing_modality_data.append(data)
            mm_missing_data[modality] = missing_modality_data
1398

1399
        mm_missing_items = self.info.parse_mm_data(mm_missing_data, validate=False)
1400
1401

        return mm_is_cached, mm_missing_items
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413

    def _recompute_cached_prompt_update(
        self,
        cached_update: ResolvedPromptUpdate,
        new_item_idx: int,
    ) -> ResolvedPromptUpdate:
        """
        Override this if other attributes of `ResolvedPromptUpdate`
        also need to be recomputed after retrieving from the cache.
        """
        return replace(cached_update, item_idx=new_item_idx)

1414
1415
    def _merge_mm_kwargs(
        self,
1416
        cache: BaseMultiModalProcessorCache,
1417
        mm_hashes: MultiModalHashes,
1418
        mm_is_cached: MultiModalIsCached,
1419
        mm_missing_kwargs: MultiModalKwargsItems,
1420
1421
        mm_missing_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[MultiModalKwargsOptionalItems, MultiModalPromptUpdates]:
1422
1423
1424
1425
1426
        # Need to touch all mm hashes before update to avoid hash in updated
        # list evict during update
        for hashes in mm_hashes.values():
            for item_hash in hashes:
                cache.touch_sender_cache_item(item_hash)
1427

1428
        mm_missing_next_idx = defaultdict[str, int](lambda: 0)
1429

1430
        merged_kwargs = defaultdict[str, list[MultiModalKwargsItem | None]](list)
1431
1432
1433
        merged_prompt_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](
            list
        )
1434
1435
        for modality, hashes in mm_hashes.items():
            missing_kwargs = mm_missing_kwargs.get(modality, [])
1436
            missing_prompt_updates = mm_missing_prompt_updates.get(modality, [])
1437
1438
1439
1440

            for item_idx, item_hash in enumerate(hashes):
                if not mm_is_cached[modality][item_idx]:
                    missing_next_idx = mm_missing_next_idx[modality]
1441
1442
                    missing_kwargs_item = missing_kwargs[missing_next_idx]
                    missing_updates_item = missing_prompt_updates[missing_next_idx]
1443

1444
                    mm_missing_next_idx[modality] += 1
1445

1446
                    item = missing_kwargs_item, missing_updates_item
1447
                else:
1448
1449
1450
1451
1452
                    item = None

                kwargs, updates = cache.get_and_update_item(item, item_hash)

                merged_kwargs[modality].append(kwargs)
1453
1454
1455
1456
1457
1458
                merged_prompt_updates[modality].append(
                    [
                        self._recompute_cached_prompt_update(update, item_idx)
                        for update in updates
                    ]
                )
1459

1460
1461
        mm_kwargs = MultiModalKwargsItems(merged_kwargs)
        mm_prompt_updates = dict(merged_prompt_updates)
1462

1463
        return mm_kwargs, mm_prompt_updates
1464
1465
1466

    def _apply_hf_processor(
        self,
1467
        prompt: str | list[int],
1468
1469
        mm_data_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1470
        tokenization_kwargs: Mapping[str, object],
1471
        *,
1472
        mm_uuids: MultiModalUUIDDict | None = None,
1473
    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1474
1475
        (
            prompt_ids,
1476
            mm_processed_data,
1477
1478
1479
1480
1481
            is_update_applied,
        ) = self._apply_hf_processor_main(
            prompt=prompt,
            mm_items=mm_data_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1482
            tokenization_kwargs=tokenization_kwargs,
1483
1484
1485
            enable_hf_prompt_update=True,
        )

1486
        mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
1487
            mm_processed_data,
1488
            self._get_mm_fields_config(mm_processed_data, hf_processor_mm_kwargs),
1489
1490
        )

1491
        # Use overrides if provided; fallback to data-dependent hashing.
1492
        with timed_preprocessor_operation(self.info.ctx, "hashing"):
1493
1494
1495
1496
1497
1498
            mm_hashes = self._hash_mm_items(
                mm_data_items,
                hf_processor_mm_kwargs,
                tokenization_kwargs,
                mm_uuids=mm_uuids,
            )
1499

1500
        mm_prompt_updates = self._get_mm_prompt_updates(
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
            mm_data_items,
            hf_processor_mm_kwargs,
            mm_kwargs,
        )

        mm_info = MultiModalProcessingInfo(
            kwargs=mm_kwargs,
            hashes=mm_hashes,
            prompt_updates=mm_prompt_updates,
        )

        return prompt_ids, mm_info, is_update_applied
1513

1514
1515
    def _cached_apply_hf_processor(
        self,
1516
        prompt: str | list[int],
1517
1518
        mm_data_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1519
        tokenization_kwargs: Mapping[str, object],
1520
        *,
1521
        mm_uuids: MultiModalUUIDDict | None = None,
1522
    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1523
1524
1525
1526
1527
1528
        """
        Apply the HF processor on the full prompt text,
        caching the results and reusing cached results.
        """
        cache = self.cache

1529
1530
        _, passthrough_data = self._get_hf_mm_data(mm_data_items)
        if cache is None or passthrough_data:
1531
            return self._apply_hf_processor(
1532
                prompt=prompt,
1533
                mm_data_items=mm_data_items,
1534
                hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1535
                tokenization_kwargs=tokenization_kwargs,
1536
                mm_uuids=mm_uuids,
1537
1538
            )

1539
        with timed_preprocessor_operation(self.info.ctx, "hashing"):
1540
1541
1542
1543
1544
1545
            mm_hashes = self._hash_mm_items(
                mm_data_items,
                hf_processor_mm_kwargs,
                tokenization_kwargs,
                mm_uuids=mm_uuids,
            )
1546

1547
        with timed_preprocessor_operation(self.info.ctx, "cache_lookup"):
1548
1549
1550
1551
1552
            mm_is_cached, mm_missing_data_items = self._get_cache_missing_items(
                cache=cache,
                mm_data_items=mm_data_items,
                mm_hashes=mm_hashes,
            )
1553

1554
        # NOTE: `prompt` does not correspond to `mm_missing_data_items`,
1555
        # so we can't apply prompt updates until the new multimodal
1556
1557
1558
        # items are combined with the cached multimodal items
        (
            prompt_ids,
1559
            mm_missing_processed_data,
1560
            is_update_applied,
1561
        ) = self._apply_hf_processor_main(
1562
            prompt=prompt,
1563
            mm_items=mm_missing_data_items,
1564
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1565
            tokenization_kwargs=tokenization_kwargs,
1566
            enable_hf_prompt_update=False,
1567
1568
        )

1569
        mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
1570
            mm_missing_processed_data,
1571
1572
1573
            self._get_mm_fields_config(
                mm_missing_processed_data, hf_processor_mm_kwargs
            ),
1574
1575
        )

1576
1577
1578
1579
        mm_missing_prompt_updates = self._get_mm_prompt_updates(
            mm_missing_data_items,
            hf_processor_mm_kwargs,
            mm_missing_kwargs,
1580
        )
1581

1582
        with timed_preprocessor_operation(self.info.ctx, "cache_lookup"):
1583
1584
1585
1586
1587
1588
1589
            mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
                cache,
                mm_hashes=mm_hashes,
                mm_is_cached=mm_is_cached,
                mm_missing_kwargs=mm_missing_kwargs,
                mm_missing_prompt_updates=mm_missing_prompt_updates,
            )
1590
1591
1592

        mm_info = MultiModalProcessingInfo(
            kwargs=mm_kwargs,
1593
            hashes=mm_hashes,
1594
1595
            prompt_updates=mm_prompt_updates,
        )
1596

1597
        return prompt_ids, mm_info, is_update_applied
1598

1599
1600
1601
    def _apply_token_matches(
        self,
        prompt: list[int],
1602
1603
1604
1605
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
        tokenizer = self.info.get_tokenizer()
        return apply_token_matches(prompt, mm_prompt_updates, tokenizer)
1606
1607
1608
1609

    def _apply_text_matches(
        self,
        prompt: str,
1610
1611
1612
1613
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[str, MultiModalPromptUpdatesApplyResult]:
        tokenizer = self.info.get_tokenizer()
        return apply_text_matches(prompt, mm_prompt_updates, tokenizer)
1614

1615
    def _apply_prompt_updates(
1616
1617
        self,
        token_ids: list[int],
1618
        mm_prompt_updates: MultiModalPromptUpdates,
1619
    ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
1620
        """Apply multi-modal prompt updates to token IDs."""
1621
        tokenizer = self.info.get_tokenizer()
1622

1623
1624
1625
1626
        new_token_ids, match_result = self._apply_token_matches(
            token_ids,
            mm_prompt_updates,
        )
1627
1628
1629
1630
1631
1632
1633
1634
1635

        # If the search text does not represent a special token,
        # it may have different token IDs in the prompt, because
        # the tokens may go across the boundaries of the search text.
        # ----
        # e.g. when searching for "foo" in "food", if "food" itself makes
        # up a token, then the token ID of "foo" will not appear at all
        # ----
        # Since it is inefficient to search for all possible tokenizations
1636
1637
        # of the search text in the prompt, we instead perform string-based
        # updates on the decoded token IDs, then encode them back.
1638
        if not all(
1639
1640
1641
            all(update_idx is not None for update_idx in update_idxs)
            for update_idxs in match_result.values()
        ):
1642
            new_text, match_result = self._apply_text_matches(
1643
                _seq2text(tokenizer, token_ids, use_cache=False),
1644
                mm_prompt_updates,
1645
1646
            )

1647
            new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False)
1648

1649
        matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
1650
1651
1652
1653
        for modality, update_idxs in match_result.items():
            for item_idx, update_idx in enumerate(update_idxs):
                assert update_idx is not None, (
                    "Failed to apply prompt replacement for "
1654
1655
                    f"mm_items[{modality!r}][{item_idx}]"
                )
1656
1657

                matched_updates[modality].append(
1658
1659
                    [mm_prompt_updates[modality][item_idx][update_idx]]
                )
1660
1661

        placeholders = self._find_mm_placeholders(
1662
1663
            new_token_ids,
            dict(matched_updates),
1664
        )
1665

1666
        return new_token_ids, placeholders
1667

1668
1669
    def _validate_mm_kwargs(
        self,
1670
        mm_kwargs: MultiModalKwargsOptionalItems,
1671
1672
1673
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
1674
            items = mm_kwargs.get(modality, [])
1675
1676
1677
1678
1679
1680
1681
1682
1683

            if len(items) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} {modality} items in "
                    f"keyword arguments corresponding to {item_count} "
                    f"{modality} data items, but only found {len(items)}! "
                    "There is likely a problem with your "
                    "implementation of merged multi-modal processor for this "
                    "model (usually arising from an inconsistency between "
1684
1685
                    "`_call_hf_processor` and `_get_mm_fields_config`)."
                )
1686

1687
    def _validate_mm_updates(
1688
        self,
1689
        mm_updates: MultiModalPromptUpdates,
1690
        mm_item_counts: Mapping[str, int],
1691
    ) -> None:
1692
        for modality, item_count in mm_item_counts.items():
1693
            placeholders = mm_updates.get(modality, [])
1694

1695
            if len(placeholders) != item_count:
1696
                raise RuntimeError(
1697
                    f"Expected there to be {item_count} prompt updates "
1698
                    f"corresponding to {item_count} {modality} items, but "
1699
                    f"instead found {len(placeholders)} prompt updates! "
1700
1701
1702
                    "This is likely because you forgot to include input "
                    "placeholder tokens (e.g., `<image>`, `<|image_pad|>`) "
                    "in the prompt. If the model has a chat template, make "
1703
1704
                    "sure you have applied it before calling `LLM.generate`."
                )
1705

1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
    def _validate_mm_placeholders(
        self,
        mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
            placeholders = mm_placeholders.get(modality, [])

            if len(placeholders) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} prompt placeholders "
                    f"corresponding to {item_count} {modality} items, but "
                    f"instead found {len(placeholders)} prompt placeholders! "
                    "Make sure the implementation of `_call_hf_processor` and "
1720
1721
                    "`_get_mm_fields_config` are consistent with each other."
                )
1722

1723
1724
1725
1726
    def _maybe_apply_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        prompt_ids: list[int],
1727
        mm_kwargs: MultiModalKwargsOptionalItems,
1728
        mm_prompt_updates: MultiModalPromptUpdates,
1729
        is_update_applied: bool,
1730
    ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
1731
        mm_item_counts = mm_items.get_all_counts()
1732
        self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
1733
        self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
1734

1735
        if is_update_applied:
1736
1737
            mm_placeholders = self._find_mm_placeholders(
                prompt_ids,
1738
                mm_prompt_updates,
1739
            )
1740
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1741
        else:
1742
            prompt_ids, mm_placeholders = self._apply_prompt_updates(
1743
                prompt_ids,
1744
                mm_prompt_updates,
1745
            )
1746
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1747

1748
        return prompt_ids, mm_placeholders
1749
1750
1751

    def apply(
        self,
1752
        prompt: str | list[int],
1753
        mm_items: MultiModalDataItems,
1754
        hf_processor_mm_kwargs: Mapping[str, object],
1755
        tokenization_kwargs: Mapping[str, object] | None = None,
1756
        *,
1757
        mm_uuids: MultiModalUUIDDict | None = None,
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
    ) -> MultiModalInputs:
        """
        Process multi-modal inputs to be used in vLLM.

        The main steps are:

        1. Apply HF Processor on prompt text and multi-modal data together,
           outputting token IDs and processed tensors.
        2. Find and update sequences in the token IDs with placeholder tokens.
           The number of placeholder tokens equals the feature size of the
           multi-modal data outputted by the multi-modal encoder.
        3. Extract information about the placeholder tokens from the
           processed token IDs.
        """
1772
1773
1774
1775
        request_id = get_current_request_id()
        if request_id is not None:
            self.info.ctx.create_timing_stats(request_id)

1776
1777
1778
        if tokenization_kwargs is None:
            tokenization_kwargs = {}

1779
1780
        (
            prompt_ids,
1781
            mm_info,
1782
1783
1784
1785
1786
            is_update_applied,
        ) = self._cached_apply_hf_processor(
            prompt,
            mm_items,
            hf_processor_mm_kwargs,
1787
            tokenization_kwargs=tokenization_kwargs,
1788
            mm_uuids=mm_uuids,
1789
1790
        )

1791
        # NOTE: tokenization_kwargs are not required to init processor
1792
        with timed_preprocessor_operation(self.info.ctx, "prompt_update"):
1793
1794
1795
1796
1797
1798
1799
            prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
                mm_items=mm_items,
                prompt_ids=prompt_ids,
                mm_kwargs=mm_info.kwargs,
                mm_prompt_updates=mm_info.prompt_updates,
                is_update_applied=is_update_applied,
            )
1800

1801
1802
1803
1804
        mm_placeholder_ranges = {
            modality: [item.to_range() for item in placeholders]
            for modality, placeholders in mm_placeholders.items()
        }
1805

1806
        return MultiModalInputs(
1807
            type="multimodal",
1808
            prompt_token_ids=prompt_ids,
1809
1810
            mm_kwargs=mm_info.kwargs,
            mm_hashes=mm_info.hashes,
1811
            mm_placeholders=mm_placeholder_ranges,
1812
        )
1813
1814
1815
1816
1817
1818


class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
    @abstractmethod
    def create_encoder_prompt(
        self,
1819
        prompt: str | list[int],
1820
        mm_items: MultiModalDataItems,
1821
    ) -> str | list[int]:
1822
        """
1823
        Create input prompt for the encoder. HF processor will be applied on
1824
1825
        this prompt during profiling and generation.
        """
1826
1827
        raise NotImplementedError

1828
1829
    def create_decoder_prompt(
        self,
1830
        prompt: str | list[int],
1831
        mm_items: MultiModalDataItems,
1832
    ) -> str | list[int]:
1833
1834
1835
        """Create input prompt for the decoder."""
        return prompt

1836
    def _get_enc_dec_inputs(
1837
        self,
1838
        prompt: str | list[int],
1839
        mm_items: MultiModalDataItems,
1840
1841
        encoder_inputs: MultiModalInputs,
    ):
1842
        tokenizer = self.info.get_tokenizer()
1843
        decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_items)
1844
        if isinstance(decoder_prompt_raw, str):
1845
1846
            decoder_prompt_ids = tokenizer.encode(
                decoder_prompt_raw, add_special_tokens=False
1847
            )
1848
        else:
1849
            decoder_prompt_ids = decoder_prompt_raw
1850
1851
1852

        mm_inputs = MultiModalEncDecInputs(
            encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],
1853
1854
            **encoder_inputs,
        )
1855
        mm_inputs["prompt_token_ids"] = decoder_prompt_ids
1856
        return mm_inputs
1857
1858
1859

    def apply(
        self,
1860
        prompt: str | list[int],
1861
        mm_items: MultiModalDataItems,
1862
        hf_processor_mm_kwargs: Mapping[str, object],
1863
        tokenization_kwargs: Mapping[str, object] | None = None,
1864
        *,
1865
        mm_uuids: MultiModalUUIDDict | None = None,
1866
1867
1868
1869
1870
1871
1872
1873
    ) -> MultiModalEncDecInputs:
        """
        Process multi-modal inputs to be used in vLLM.
        The main processing steps are modified to fit encoder-decoder model:
        1. Create encoder prompt from input prompt text.
        2. Apply the HF processor on encoder prompt.
        3. Copy the input prompt text as decoder prompt inputs.
        """
1874
        encoder_prompt = self.create_encoder_prompt(prompt, mm_items)
1875
1876
        encoder_inputs = super().apply(
            encoder_prompt,
1877
            mm_items,
1878
            hf_processor_mm_kwargs,
1879
            tokenization_kwargs,
1880
            mm_uuids=mm_uuids,
1881
1882
1883
1884
        )

        return self._get_enc_dec_inputs(
            prompt=prompt,
1885
            mm_items=mm_items,
1886
1887
            encoder_inputs=encoder_inputs,
        )