processor.py 56.1 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from abc import ABC, abstractmethod
4
from collections import defaultdict
5
from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence
6
from dataclasses import dataclass, field, replace
7
from enum import Enum
8
from functools import lru_cache
9
from typing import TYPE_CHECKING, Generic, NamedTuple, Protocol, TypeAlias, cast
10

11
import regex as re
12
import torch
13
from typing_extensions import TypeVar, assert_never
14

15
16
17
18
19
20
21
from vllm.inputs import (
    MultiModalEncDecInput,
    MultiModalHashes,
    MultiModalInput,
    mm_enc_dec_input,
    mm_input,
)
22
from vllm.logger import init_logger
23
from vllm.tokenizers import TokenizerLike
24
from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
25

26
from ..inputs import (
27
28
29
30
31
32
    MultiModalFieldConfig,
    MultiModalKwargsItem,
    MultiModalKwargsItems,
    MultiModalKwargsOptionalItems,
    PlaceholderRange,
)
33
from ..parse import (
34
35
36
    DictEmbeddingItems,
    EmbeddingItems,
    MultiModalDataItems,
37
    MultiModalUUIDItems,
38
)
39
from .context import BaseProcessingInfo, TimingContext
40
from .dummy_inputs import BaseDummyInputsBuilder
41
from .inputs import ProcessorInputs
42
43

if TYPE_CHECKING:
44
45
    from transformers.feature_extraction_utils import BatchFeature

46
    from ..cache import BaseMultiModalProcessorCache
47
48
49
50
else:
    BatchFeature = object

    BaseMultiModalProcessorCache = object
51

52
logger = init_logger(__name__)
53
54

_S = TypeVar("_S", str, list[int])
55

56

57
PromptSeq: TypeAlias = str | list[int]
58
"""A token sequence (list of token IDs) or text."""
59

60

61
62
@lru_cache(maxsize=2048)
def _cached_encode(
63
    tokenizer: TokenizerLike,
64
65
    text: str,
    *,
66
    add_special_tokens: bool = True,
67
) -> list[int]:
68
    return tokenizer.encode(text, add_special_tokens=add_special_tokens)
69
70
71
72


@lru_cache(maxsize=2048)
def _cached_decode(
73
    tokenizer: TokenizerLike,
74
75
    token_ids: tuple[int, ...],
    *,
76
    skip_special_tokens: bool = False,
77
) -> str:
78
    return tokenizer.decode(list(token_ids), skip_special_tokens=skip_special_tokens)
79
80


81
82
83
84
85
86
def _seq2text(
    tokenizer: TokenizerLike | None,
    seq: PromptSeq,
    *,
    use_cache: bool = True,
) -> str:
87
88
89
    if isinstance(seq, str):
        return seq

90
91
92
93
    if tokenizer is None:
        raise ValueError("You cannot decode tokens when `skip_tokenizer_init=True`")

    if not use_cache:
94
        return tokenizer.decode(seq)
95

96
97
98
    return _cached_decode(tokenizer, tuple(seq))


99
100
101
102
103
104
def _seq2tokens(
    tokenizer: TokenizerLike | None,
    seq: PromptSeq,
    *,
    use_cache: bool = True,
) -> list[int]:
105
    if isinstance(seq, str):
106
107
108
109
        if tokenizer is None:
            raise ValueError("You cannot encode text when `skip_tokenizer_init=True`")

        if not use_cache:
110
            return tokenizer.encode(seq, add_special_tokens=False)
111

112
113
114
115
116
        return _cached_encode(tokenizer, seq, add_special_tokens=False)

    return seq


117
118
119
class _GetMatchIndex(Protocol):
    def __call__(
        self,
120
        tokenizer: TokenizerLike | None,
121
122
        prompt: PromptSeq,
        start_idx: int = 0,
123
    ) -> int | None: ...
124
125


126
127
128
@dataclass
class PromptIndex:
    """Resolves to an index in the prompt."""
129

130
    get_match_index: _GetMatchIndex
131
132
133
134
135
136
137
138
139
140


class PromptIndexTargets:
    @staticmethod
    def start() -> PromptIndex:
        """
        Resolves to the start of the prompt (before the first token).

        This results in a match even if the prompt is empty.
        """
141
        return PromptIndex(lambda tokenizer, prompt, start_idx=0: 0)
142
143
144
145
146
147
148
149

    @staticmethod
    def prefix(seq: PromptSeq) -> PromptIndex:
        """
        Resolves to a location in the prompt after the given prefix.
        """

        def get_match_index(
150
            tokenizer: TokenizerLike | None,
151
            prompt: PromptSeq,
152
            start_idx: int = 0,
153
        ) -> int | None:
154
155
156
            if start_idx != 0:
                return None

157
158
159
            prefix = seq

            if isinstance(prompt, str):
160
161
                # Make both `str`
                prefix = _seq2text(tokenizer, prefix, use_cache=False)
162
            else:
163
164
                # Make both `list[int]`
                prefix = _seq2tokens(tokenizer, prefix, use_cache=False)
165
166
167
168
169
170
171
172
173
174
175
176
177

            match_idx = len(prefix)
            return match_idx if prompt[:match_idx] == prefix else None

        return PromptIndex(get_match_index)

    @staticmethod
    def end() -> PromptIndex:
        """
        Resolves to the end of the prompt (after the last token).

        This results in a match even if the prompt is empty.
        """
178
        return PromptIndex(lambda tokenizer, prompt, start_idx=0: len(prompt))
179
180


181
UpdateTarget: TypeAlias = PromptSeq | PromptIndex
182
183
184
185
"""
The token sequence or text to update.
"""

186
PromptUpdateTarget: TypeAlias = Callable[[int], UpdateTarget] | UpdateTarget
187
188
189
190
191
192
193
194
195
"""
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
output the corresponding token sequence (or text).

For convenience, you can directly pass in the token sequence (or text)
instead of a function if it does not depend on the input.
"""

196

197
@dataclass
198
class PromptUpdateDetails(Generic[_S]):
199
    """Details about the token sequence or text that are part of the update."""
200

201
    full: _S
202
    """The full content."""
203

204
    is_embed: Callable[[TokenizerLike | None, PromptSeq], torch.Tensor] | None = None
205
    """
206
207
208
    Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full],
    return a boolean mask of shape `(len(full),)` indicating which positions
    of `full` to assign embeddings to.
209
210
211
212

    `None` (default) means to assign embeddings to all positions of `full`.

    The embeddings are obtained by calling
213
    [`SupportsMultiModal.embed_multimodal`][vllm.model_executor.models.interfaces.SupportsMultiModal.embed_multimodal].
214
215
216
    """

    @staticmethod
217
    def from_seq(seq: _S) -> "PromptUpdateDetails[_S]":
218
219
220
221
222
223
224
        return PromptUpdateDetails(full=seq)

    @staticmethod
    def select_text(
        seq: _S,
        embed_text: str,
    ) -> "PromptUpdateDetails[_S]":
225
226
        def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
            embed_token_ids = _seq2tokens(tokenizer, embed_text, use_cache=False)
227
            token_ids = _seq2tokens(tokenizer, full)
228
229

            return torch.isin(
230
                torch.tensor(token_ids),
231
232
233
234
235
236
237
238
239
240
                torch.tensor(embed_token_ids),
            )

        return PromptUpdateDetails(full=seq, is_embed=is_embed)

    @staticmethod
    def select_token_id(
        seq: _S,
        embed_token_id: int,
    ) -> "PromptUpdateDetails[_S]":
241
        def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
242
243
244
245
246
            token_ids = _seq2tokens(tokenizer, full)

            return torch.tensor(token_ids) == embed_token_id

        return PromptUpdateDetails(full=seq, is_embed=is_embed)
247

248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
    @staticmethod
    def select_token_ids(
        seq: _S,
        embed_token_ids: list[int],
    ) -> "PromptUpdateDetails[_S]":
        def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
            token_ids = _seq2tokens(tokenizer, full)

            return torch.isin(
                torch.tensor(token_ids),
                torch.tensor(embed_token_ids),
            )

        return PromptUpdateDetails(full=seq, is_embed=is_embed)

263

264
PromptUpdateInfo: TypeAlias = PromptSeq | PromptUpdateDetails
265
"""
266
The token sequence or text that are part of the update.
267

268
If only part of the content corresponds to feature placeholders, you can
269
270
use [`PromptUpdateDetails`][vllm.multimodal.processing.PromptUpdateDetails] to
specify which part.
271
"""
272

273
PromptUpdateContent: TypeAlias = Callable[[int], PromptUpdateInfo] | PromptUpdateInfo
274
"""
275
276
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
277
278
279
280
281
282
283
284
285
286
287
288
289
output the corresponding token sequence (or text).

For convenience, you can directly pass in the token sequence (or text)
instead of a function if it does not depend on the input.
"""


class UpdateMode(str, Enum):
    INSERT = "insert"
    REPLACE = "replace"


@dataclass
290
class PromptUpdate(ABC):
291
292
293
294
295
296
297
    """
    Defines how to update a prompt with placeholder tokens.
    """

    modality: str
    """The modality for which the update is made."""

298
    target: PromptUpdateTarget
299
300
301
302
303
304
305
306
307
308
309
310
311
312
    """The token sequence (or text) to update."""

    @property
    @abstractmethod
    def content(self) -> PromptUpdateContent:
        """The placeholder tokens that are part of the update."""
        raise NotImplementedError

    @property
    @abstractmethod
    def mode(self) -> UpdateMode:
        """Defines how to update the prompt."""
        raise NotImplementedError

313
    def _resolve_target(self, item_idx: int) -> UpdateTarget:
314
315
316
317
        target = self.target
        if callable(target):
            target = target(item_idx)

318
        return target
319

320
    def _resolve_content(self, item_idx: int) -> PromptUpdateDetails:
321
322
323
324
325
326
327
        content = self.content
        if callable(content):
            content = content(item_idx)

        if not isinstance(content, PromptUpdateDetails):
            content = PromptUpdateDetails.from_seq(content)

328
        return content
329

330
    def resolve(self, item_idx: int) -> "ResolvedPromptUpdate":
331
332
333
334
335
336
337
338
339
        """
        Given the index of the processed item within
        [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
        output a copy of this object with its lazy attributes resolved.
        """
        return ResolvedPromptUpdate(
            modality=self.modality,
            item_idx=item_idx,
            mode=self.mode,
340
341
            target=self._resolve_target(item_idx),
            content=self._resolve_content(item_idx),
342
343
        )

344

345
@dataclass
346
347
348
349
350
351
class PromptInsertion(PromptUpdate):
    """
    Defines how to insert placeholder tokens into a prompt.

    Example:

352
353
    For each image, insert a number of `<image>` feature placeholders
    equal to the feature size of the vision encoder after the `<s>` token:
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372

    ```python
    PromptInsertion(
        modality="image",
        target="<s>",
        insertion="<image>" * image_feature_size,
    )
    ```

    Insert these tokens at the start of the prompt:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.start(),
        insertion="<image>" * image_feature_size,
    )
    ```

373
    Insert these tokens after a prefix `Images:`:
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.prefix("Images:"),
        insertion="<image>" * image_feature_size,
    )
    ```

    Insert these tokens at the end of the prompt:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.end(),
        insertion="<image>" * image_feature_size,
    )
    ```
392
393
394
395
    """

    insertion: PromptUpdateContent = field(repr=False)
    """
396
397
398
399
    Given the index of the processed item within
    [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
    output the token sequence (or text) to insert right after
    [`target`][vllm.multimodal.processing.PromptUpdate.target].
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415

    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
    """

    @property
    def content(self) -> PromptUpdateContent:
        return self.insertion

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.INSERT


@dataclass
class PromptReplacement(PromptUpdate):
416
417
    """
    Defines how to replace portions of an input prompt with placeholder tokens.
418
419
420

    Example:

421
422
    For each image, replace one `<image>` input placeholder in the prompt
    with a number of `<image>` feature placeholders
423
424
425
426
427
428
429
430
431
432
    equal to the feature size of the vision encoder:

    ```python
    PromptReplacement(
        modality="image",
        target="<image>",
        replacement="<image>" * image_feature_size,
    )
    ```

433
434
    As above, but further pad the feature placeholders with `<image_bos>`
    and `<image_eos>`, which are not supposed to be passed to the vision
435
436
437
438
439
440
441
    encoder:

    ```python
    PromptReplacement(
        modality="image",
        target="<image>",
        replacement=PromptUpdateDetails(
442
443
444
445
446
447
448
            full="".join(
                [
                    "<image_bos>",
                    "<image>" * image_feature_size,
                    "<image_eos>",
                ]
            ),
449
450
451
452
453
454
455
456
457
458
459
460
461
            features="<image>" * image_feature_size,
        ),
    )
    ```

    To avoid unnecessary tokenization during prompt replacement,
    we recommended passing token sequences instead of text:

    ```python
    PromptReplacement(
        modality="image",
        target=[image_token_id],
        replacement=PromptUpdateDetails(
462
463
464
            full=(
                [image_bos_id] + [image_token_id] * image_feature_size + [image_eos_id]
            ),
465
466
467
468
            features=[image_token_id] * image_feature_size,
        ),
    )
    ```
469
470
    """

471
    replacement: PromptUpdateContent = field(repr=False)
472
    """
473
474
475
476
    Given the index of the processed item within
    [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
    output the token sequence (or text) to replace
    [`target`][vllm.multimodal.processing.PromptUpdate.target].
477

478
479
    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
480
481
    """

482
483
484
485
486
487
488
    @property
    def content(self) -> PromptUpdateContent:
        return self.replacement

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.REPLACE
489
490


491
492
493
class _HasModalityAttr(Protocol):
    modality: str

494

495
496
class _HasModalityProp(Protocol):
    @property
497
    def modality(self) -> str: ...
498
499


500
_M = TypeVar("_M", bound=_HasModalityAttr | _HasModalityProp)
501
502
503


def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]:
504
505
    """
    Convenience function to apply
506
    [`full_groupby`][vllm.utils.collection_utils.full_groupby]
507
508
    based on modality.
    """
509
510
511
    return full_groupby(values, key=lambda x: x.modality)


512
513
514
515
516
517
518
class PromptTargetMatch(NamedTuple):
    start_idx: int
    end_idx: int


@dataclass(frozen=True)
class ResolvedPromptUpdate:
519
    """
520
521
    A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] with its
    lazy attributes resolved, apart from those related to tokenization.
522
    """
523

524
525
    modality: str
    """The modality for which the update is made."""
526

527
528
    item_idx: int
    """The index within `modality` of the item this update pertains to."""
529

530
531
    mode: UpdateMode
    """Defines how to update the prompt."""
532

533
    target: UpdateTarget
534
    """The token sequence (or text) to update."""
535

536
    content: PromptUpdateDetails = field(repr=False)
537
    """The placeholder tokens that are part of the update."""
538

539
540
541
    def iter_token_matches(
        self,
        prompt: list[int],
542
        tokenizer: TokenizerLike | None,
543
544
545
546
547
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        target = self.target
548

549
550
551
552
        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(tokenizer, prompt, start_idx)
            if match_idx is not None:
                yield PromptTargetMatch(match_idx, match_idx)
553

554
            return
555

556
557
        target_token_ids = _seq2tokens(tokenizer, target)

558
        for match in iter_token_matches(prompt, target_token_ids, start_idx=start_idx):
559
            yield PromptTargetMatch(match.start_idx, match.end_idx)
560

561
562
563
    def iter_text_matches(
        self,
        prompt: str,
564
        tokenizer: TokenizerLike | None,
565
566
567
568
569
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        target = self.target
570

571
572
573
574
        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(tokenizer, prompt, start_idx)
            if match_idx is not None:
                yield PromptTargetMatch(match_idx, match_idx)
575

576
            return
577

578
579
        target_text = _seq2text(tokenizer, target)

580
        for match in re.finditer(re.escape(target_text), prompt, pos=start_idx):
581
582
583
584
            yield PromptTargetMatch(match.start(), match.end())

    def iter_matches(
        self,
585
        prompt: list[int] | str,
586
        tokenizer: TokenizerLike | None,
587
588
589
590
591
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        if isinstance(prompt, str):
592
            return self.iter_text_matches(prompt, tokenizer, start_idx=start_idx)
593
594

        return self.iter_token_matches(prompt, tokenizer, start_idx=start_idx)
595

596
597
598
599
600
601
602
603
604
    def with_target(self, target: UpdateTarget):
        return replace(self, target=target)

    def with_content(self, content: PromptUpdateInfo):
        if not isinstance(content, PromptUpdateDetails):
            content = PromptUpdateDetails.from_seq(content)

        return replace(self, content=content)

605

606
607
608
class _TokenMatch(NamedTuple):
    start_idx: int
    end_idx: int
609
610


611
612
613
def iter_token_matches(
    token_ids: list[int],
    match_ids: list[int],
614
615
    *,
    start_idx: int = 0,
616
) -> Generator[_TokenMatch]:
617
    """
618
    Yield each occurrence of `match_ids` in `token_ids`.
619
620
621
622

    Note that empty matches are ignored.
    """
    prompt_len = len(token_ids)
623
    match_len = len(match_ids)
624

625
626
    if match_len == 0:
        return
627

628
    while start_idx < prompt_len - match_len + 1:
629
        end_idx = start_idx + match_len
630

631
632
        if token_ids[start_idx:end_idx] == match_ids:
            yield _TokenMatch(start_idx=start_idx, end_idx=end_idx)
633
634
635
636
637

            # Exclude overlapping matches
            start_idx = end_idx
        else:
            start_idx += 1
638
639


640
641
642
643
644
645
def replace_token_matches(
    token_ids: list[int],
    match_ids: list[int],
    new_ids: list[int],
) -> list[int]:
    """
646
647
    Replace each occurrence of `match_ids` in `token_ids`
    with `new_ids`.
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666

    Note that empty matches are ignored.
    """
    out_seqs = list[list[int]]()
    prev_end_idx = 0

    for match in iter_token_matches(token_ids, match_ids):
        start_idx = match.start_idx
        end_idx = match.end_idx

        out_seqs.append(token_ids[prev_end_idx:start_idx])
        out_seqs.append(new_ids)
        prev_end_idx = end_idx

    out_seqs.append(token_ids[prev_end_idx:])

    return flatten_2d_lists(out_seqs)


667
@dataclass
668
class PlaceholderFeaturesInfo:
669
    modality: str
670
    item_idx: int
671
    start_idx: int
672
    tokens: list[int]
673
    is_embed: torch.Tensor | None
674
675
676

    @property
    def length(self) -> int:
677
        return len(self.tokens)
678
679

    def to_range(self) -> PlaceholderRange:
680
681
        # TODO: Is it worth it to optimize this by stripping the
        # leading and ending positions where `is_embed=False`?
682
683
684
        return PlaceholderRange(
            offset=self.start_idx,
            length=self.length,
685
            is_embed=self.is_embed,
686
        )
687
688


689
_MatchToApply = tuple[tuple[str, int], tuple[PromptTargetMatch, int]]
690
691


692
693
694
def _find_matches(
    prompt: _S,
    mm_prompt_updates: "MultiModalPromptUpdates",
695
    tokenizer: TokenizerLike | None,
696
697
698
    *,
    prev_end_idx: int = 0,
    current_result: "MultiModalPromptUpdatesApplyResult",
699
700
) -> tuple[UpdateMode | None, list[_MatchToApply]]:
    mode: UpdateMode | None = None
701
702
703
704
705
706
707
708
709
710
711
712
    mm_matches = dict[tuple[str, int], tuple[PromptTargetMatch, int]]()

    for modality, modality_updates in mm_prompt_updates.items():
        for item_idx, item_updates in enumerate(modality_updates):
            if current_result[modality][item_idx] is not None:
                continue  # Updates have already been applied for this item

            for update_idx, update in enumerate(item_updates):
                if (modality, item_idx) in mm_matches:
                    break  # Already found a match for this item

                for match in update.iter_matches(
713
714
715
                    prompt,
                    tokenizer,
                    start_idx=prev_end_idx,
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
                ):
                    # All matches should share the same mode
                    if mode is None:
                        mode = update.mode
                    elif mode != update.mode:
                        continue

                    mm_matches[(modality, item_idx)] = match, update_idx
                    break  # Get only the first valid match per item

    # Prioritize earlier matches
    matches_to_apply = sorted(mm_matches.items(), key=lambda item: item[1][0])

    # To avoid conflicts, only replace one non-empty item at a time
    if mode == UpdateMode.REPLACE:
        matches_to_apply_ = list[_MatchToApply]()
        has_non_empty_matches = False

        for item in matches_to_apply:
            _, (match, _) = item
            if match.start_idx == match.end_idx:
                matches_to_apply_.append(item)
            elif not has_non_empty_matches:
                has_non_empty_matches = True
                matches_to_apply_.append(item)

        matches_to_apply = matches_to_apply_

    return mode, matches_to_apply
745
746


747
748
749
750
751
752
753
754
755
756
def _all_items_found(
    mm_item_counts: dict[str, int],
    mm_found_counts: dict[str, int],
) -> bool:
    return all(
        item_idx >= mm_item_counts[modality]
        for modality, item_idx in mm_found_counts.items()
    )


757
def _apply_matches(
758
    prompt: _S,
759
    mm_prompt_updates: "MultiModalPromptUpdates",
760
    tokenizer: TokenizerLike | None,
761
) -> tuple[list[_S], "MultiModalPromptUpdatesApplyResult"]:
762
    mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()}
763

764
    out_seqs = list[str | list[int]]()
765
    out_result: MultiModalPromptUpdatesApplyResult = {
766
        m: [None] * len(items) for m, items in mm_prompt_updates.items()
767
    }
768

769
    # Early exit if no items to find
770
771
772
773
774
775
    mm_found_counts = {
        m: sum(r is not None for r in res) for m, res in out_result.items()
    }
    if _all_items_found(mm_item_counts, mm_found_counts):
        return [prompt], out_result

776
777
    prev_end_idx = 0
    while True:
778
779
780
781
782
783
784
        mode, matches_to_apply = _find_matches(
            prompt,
            mm_prompt_updates,
            tokenizer,
            prev_end_idx=prev_end_idx,
            current_result=out_result,
        )
785

786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
        if mode is None:
            break  # No more matches to find

        for (modality, item_idx), (match, update_idx) in matches_to_apply:
            matched_update = mm_prompt_updates[modality][item_idx][update_idx]
            matched_content = matched_update.content.full

            if mode == UpdateMode.INSERT:
                end_idx_to_insert = match.end_idx
            elif mode == UpdateMode.REPLACE:
                end_idx_to_insert = match.start_idx
            else:
                assert_never(mode)

            out_seqs.append(prompt[prev_end_idx:end_idx_to_insert])
            out_seqs.append(
                _seq2text(tokenizer, matched_content)
                if isinstance(prompt, str)
                else _seq2tokens(tokenizer, matched_content)
            )
            out_result[modality][item_idx] = update_idx

            # Exclude overlapping matches
            prev_end_idx = match.end_idx

        # Early exit if all items found
        mm_found_counts = {
            m: sum(r is not None for r in res) for m, res in out_result.items()
        }
        if _all_items_found(mm_item_counts, mm_found_counts):
            break
817
818
819

    out_seqs.append(prompt[prev_end_idx:])

820
    return cast(list[_S], out_seqs), out_result
821
822


823
def apply_token_matches(
824
    prompt: list[int],
825
    mm_prompt_updates: "MultiModalPromptUpdates",
826
    tokenizer: TokenizerLike | None,
827
828
829
) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
    """
    Apply the updates in `mm_prompt_updates` to `prompt`.
830

831
832
833
834
    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
    appears earlier in `mm_prompt_updates` takes priority.
    """
835
    token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
836

837
    return flatten_2d_lists(token_id_seqs), result
838
839


840
def apply_text_matches(
841
    prompt: str,
842
    mm_prompt_updates: "MultiModalPromptUpdates",
843
    tokenizer: TokenizerLike | None,
844
845
846
) -> tuple[str, "MultiModalPromptUpdatesApplyResult"]:
    """
    Apply the updates in `mm_prompt_updates` to `prompt`.
847

848
849
850
851
852
    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
    appears earlier in `mm_prompt_updates` takes priority.
    """
    texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
853

854
    return "".join(texts), result
855
856


857
def _iter_placeholders(
858
    prompt: list[int],
859
    mm_prompt_updates: "MultiModalPromptUpdates",
860
    tokenizer: TokenizerLike | None,
861
) -> Iterable[PlaceholderFeaturesInfo]:
862
    """
863
    Yield each set of placeholder tokens found in `prompt`.
864
865
866

    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
867
    appears earlier in `mm_prompt_updates` takes priority.
868

869
870
    Note that empty matches are ignored.
    """
871
    mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()}
872
    item_idx_by_modality = {modality: 0 for modality in mm_prompt_updates}
873

874
875
    if _all_items_found(mm_item_counts, item_idx_by_modality):
        return
876

877
    prompt_len = len(prompt)
878
    start_idx = 0
879

880
881
882
    while start_idx < prompt_len:
        found = False

883
        for modality, modality_updates in mm_prompt_updates.items():
884
885
            item_idx = item_idx_by_modality[modality]
            if item_idx >= mm_item_counts.get(modality, 0):
886
                continue
887

888
889
            for update in modality_updates[item_idx]:
                content = update.content
890
                content_tokens_full = _seq2tokens(tokenizer, content.full)
891
892
                content_len_full = len(content_tokens_full)
                end_idx_full = start_idx + content_len_full
893

894
                if content_len_full == 0 or end_idx_full > prompt_len:
895
896
                    continue

897
                if prompt[start_idx:end_idx_full] == content_tokens_full:
898
899
                    content_is_embed = content.is_embed
                    if content_is_embed is not None:
900
                        content_is_embed = content_is_embed(tokenizer, content.full)
901
902
903
904
905
906
907
908

                    yield PlaceholderFeaturesInfo(
                        modality=modality,
                        item_idx=item_idx,
                        start_idx=start_idx,
                        tokens=content_tokens_full,
                        is_embed=content_is_embed,
                    )
909

910
                    # Exclude overlapping matches
911
                    start_idx = end_idx_full
912
913
914
                    item_idx_by_modality[modality] += 1
                    found = True
                    break
915

916
            if found:
917
918
919
                if _all_items_found(mm_item_counts, item_idx_by_modality):
                    return

920
                break  # Go back to the outer while loop
921
922
923

        if not found:
            start_idx += 1
924
925


926
927
def find_mm_placeholders(
    prompt: list[int],
928
    mm_prompt_updates: "MultiModalPromptUpdates",
929
    tokenizer: TokenizerLike | None,
930
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
931
    it = _iter_placeholders(prompt, mm_prompt_updates, tokenizer)
932
933
934
    return dict(full_groupby_modality(it))


935
936
937
MultiModalIsCached = dict[str, list[bool]]
"""
A collection of the `is_cached` flag for each item, with a similar structure as
938
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
939
940
"""

941
MultiModalPromptUpdates = Mapping[str, list[Sequence[ResolvedPromptUpdate]]]
942
943
944
945
946
"""
A collection of prompt updates with a similar structure as
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
"""

947
MultiModalPromptUpdatesApplyResult = Mapping[str, list[int | None]]
948
949
950
951
952
953
954
"""
For an item `MultiModalPromptUpdates[k][i]`,
`MultiModalPromptUpdatesApplyResult[k][i]` represents the index of the
`ResolvedPromptUpdate` instance that has been applied, or `None` if none of the
`ResolvedPromptUpdate` instances have been applied.
"""

955
956
_I = TypeVar("_I", bound=BaseProcessingInfo)

957
958

class MultiModalProcessingInfo(NamedTuple):
959
    kwargs: MultiModalKwargsOptionalItems
960
    hashes: MultiModalHashes
961
962
    prompt_updates: MultiModalPromptUpdates

963
964

class BaseMultiModalProcessor(ABC, Generic[_I]):
965
    """
966
    Abstract base class to process multi-modal inputs to be used in vLLM.
967

968
    Not to be confused with `transformers.ProcessorMixin`.
969
970
    """

971
972
973
974
975
    def __init__(
        self,
        info: _I,
        dummy_inputs: "BaseDummyInputsBuilder[_I]",
        *,
976
        cache: BaseMultiModalProcessorCache | None = None,
977
    ) -> None:
978
979
        super().__init__()

980
981
        self.info = info
        self.dummy_inputs = dummy_inputs
982
        self.cache = cache
983

984
        self.data_parser = self.info.get_data_parser()
985

986
    def __call__(
987
        self,
988
        prompt: str,
989
        mm_items: MultiModalDataItems,
990
991
        mm_uuid_items: MultiModalUUIDItems | None = None,
        hf_processor_mm_kwargs: Mapping[str, object] | None = None,
992
    ) -> MultiModalInput:
993
        processor_inputs = ProcessorInputs(
994
995
996
            prompt,
            mm_items,
            mm_uuid_items,
997
            hf_processor_mm_kwargs=hf_processor_mm_kwargs or {},
998
        )
999

1000
1001
        return self.apply(processor_inputs, TimingContext(enabled=False))

1002
1003
1004
    @abstractmethod
    def _get_mm_fields_config(
        self,
1005
        hf_inputs: BatchFeature,
1006
1007
1008
1009
1010
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, MultiModalFieldConfig]:
        """Given the HF-processed data, output the metadata of each field."""
        raise NotImplementedError

1011
    @abstractmethod
1012
    def _get_prompt_updates(
1013
        self,
1014
        mm_items: MultiModalDataItems,
1015
        hf_processor_mm_kwargs: Mapping[str, object],
1016
        out_mm_kwargs: MultiModalKwargsItems,
1017
    ) -> Sequence[PromptUpdate]:
1018
1019
        """
        Given the original multi-modal items for this modality
1020
        and HF-processed data, output the updates to perform.
1021

1022
1023
1024
1025
1026
1027
        The information returned by this method is used to update token inputs
        which bypass the HF processor. It is also used to update the output of
        HF processor if the HF process does not apply prompt updates to text
        inputs.

        Moreover, this information is critical to determine the token positions
1028
1029
        in order to construct
        [`PlaceholderRange`][vllm.multimodal.inputs.PlaceholderRange]
1030
        for each multi-modal item.
1031
1032
        """
        raise NotImplementedError
1033

1034
1035
1036
1037
1038
1039
    def _bind_and_group_updates(
        self,
        prompt_updates: Sequence[PromptUpdate],
        mm_item_counts: Mapping[str, int],
    ) -> MultiModalPromptUpdates:
        return {
1040
1041
1042
1043
            modality: [
                [update.resolve(item_idx) for update in updates]
                for item_idx in range(mm_item_counts.get(modality, 0))
            ]
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
            for modality, updates in full_groupby_modality(prompt_updates)
        }

    def _get_mm_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        out_mm_kwargs: MultiModalKwargsItems,
    ) -> MultiModalPromptUpdates:
        unbound_prompt_updates = self._get_prompt_updates(
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
            out_mm_kwargs=out_mm_kwargs,
        )

        mm_prompt_updates = self._bind_and_group_updates(
            unbound_prompt_updates,
            mm_items.get_all_counts(),
        )

        return mm_prompt_updates

1066
    def _find_mm_placeholders(
1067
1068
        self,
        new_token_ids: list[int],
1069
        mm_prompt_updates: MultiModalPromptUpdates,
1070
    ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
1071
1072
        tokenizer = self.info.get_tokenizer()

1073
        return find_mm_placeholders(new_token_ids, mm_prompt_updates, tokenizer)
1074

1075
    def _get_hf_mm_data(
1076
        self,
1077
        mm_items: MultiModalDataItems,
1078
    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
1079
        """Extract processor and passthrough data from multi-modal items."""
1080
1081
        processor_data = dict[str, object]()
        passthrough_data = dict[str, object]()
1082

1083
1084
1085
        for items in mm_items.values():
            processor_data.update(items.get_processor_data())
            passthrough_data.update(items.get_passthrough_data())
1086

1087
1088
        return processor_data, passthrough_data

1089
1090
1091
    def _call_hf_processor(
        self,
        prompt: str,
1092
1093
1094
1095
        # Not to be confused with `mm_data` in `self.apply`.
        # This refers to the data to be passed to HF processor.
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
1096
        tok_kwargs: Mapping[str, object],
1097
    ) -> BatchFeature:
1098
1099
1100
1101
        """
        Call the HF processor on the prompt text and
        associated multi-modal data.
        """
1102
1103
1104
1105
1106
        return self.info.ctx.call_hf_processor(
            self.info.get_hf_processor(**mm_kwargs),
            dict(text=prompt, **mm_data),
            dict(**mm_kwargs, **tok_kwargs),
        )
1107

1108
    def _hf_processor_applies_updates(
1109
1110
1111
1112
        self,
        prompt_text: str,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1113
        tokenization_kwargs: Mapping[str, object],
1114
1115
    ) -> bool:
        """
1116
        Return whether the HF processor applies prompt updates.
1117

1118
1119
        For most HF processors, this should be `True` when multi-modal
        data items are passed, but `False` when multi-modal embeddings
1120
1121
1122
1123
        are passed.
        """
        return not any(
            isinstance(items, (EmbeddingItems, DictEmbeddingItems))
1124
1125
            for items in mm_items.values()
        )
1126

1127
    def _apply_hf_processor_text_mm(
1128
        self,
1129
        prompt_text: str,
1130
        mm_items: MultiModalDataItems,
1131
        hf_processor_mm_kwargs: Mapping[str, object],
1132
        tokenization_kwargs: Mapping[str, object],
1133
    ) -> tuple[list[int], BatchFeature, bool]:
1134
        """
1135
1136
        Apply the HF processor on the prompt text and multi-modal data
        together.
1137

1138
        In addition, return whether prompt updates have been applied.
1139
        """
1140
1141
1142
1143
        valid_mm_items = mm_items.select(
            {k for k, c in mm_items.get_all_counts().items() if c > 0}
        )
        processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
1144
1145
1146
1147
1148

        processed_data = self._call_hf_processor(
            prompt=prompt_text,
            mm_data=processor_data,
            mm_kwargs=hf_processor_mm_kwargs,
1149
            tok_kwargs=tokenization_kwargs,
1150
1151
        )
        processed_data.update(passthrough_data)
1152

1153
        (prompt_ids,) = processed_data.pop("input_ids").tolist()
1154

1155
        is_update_applied = self._hf_processor_applies_updates(
1156
1157
1158
            prompt_text=prompt_text,
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1159
            tokenization_kwargs=tokenization_kwargs,
1160
1161
        )

1162
        return prompt_ids, processed_data, is_update_applied
1163

1164
    def _apply_hf_processor_text_only(
1165
1166
1167
1168
        self,
        prompt_text: str,
        tokenization_kwargs: Mapping[str, object],
    ) -> list[int]:
1169
        """
1170
        Apply the HF processor on the prompt text only.
1171

1172
1173
1174
        Since HF processor requires that text and multi-modal items
        correspond to each other, we create dummy multi-modal items
        to go along with the text.
1175
        """
1176
        prompt_ids, _, _ = self._apply_hf_processor_text_mm(
1177
1178
1179
            prompt_text=prompt_text,
            mm_items=MultiModalDataItems({}),
            hf_processor_mm_kwargs={},
1180
            tokenization_kwargs=tokenization_kwargs,
1181
1182
        )

1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
        return prompt_ids

    def _apply_hf_processor_tokens_only(
        self,
        prompt_tokens: list[int],
    ) -> list[int]:
        """
        Apply the HF processor on the prompt tokens only.

        Most HF processors accept prompt text but not prompt tokens.
        If the HF processor adds or removes tokens that are not related to
        multi-modal data, you should override this method so it is consistent
1195
1196
1197
        with the output of
        [`_apply_hf_processor_text_only`][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_text_only]
        on the
1198
1199
1200
1201
1202
1203
1204
1205
        corresponding text.
        """
        return prompt_tokens

    def _apply_hf_processor_mm_only(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1206
        tokenization_kwargs: Mapping[str, object],
1207
    ) -> BatchFeature:
1208
1209
1210
1211
1212
        """
        Apply the HF processor on the multi-modal data only.

        Since HF processor requires that text and multi-modal items
        correspond to each other, we generate dummy text using
1213
        [`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder]
1214
        to go along with the multi-modal data.
1215
1216
1217
        """
        mm_counts = mm_items.get_all_counts()

1218
        _, mm_processed_data, _ = self._apply_hf_processor_text_mm(
1219
            prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
1220
1221
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1222
            tokenization_kwargs=tokenization_kwargs,
1223
1224
        )

1225
        return mm_processed_data
1226
1227
1228

    def _apply_hf_processor_main(
        self,
1229
        prompt: str | list[int],
1230
1231
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1232
        tokenization_kwargs: Mapping[str, object],
1233
        *,
1234
        enable_hf_prompt_update: bool,
1235
    ) -> tuple[list[int], BatchFeature, bool]:
1236
1237
1238
        """
        Apply the HF processor on the prompt text and multi-modal data.

1239
        In addition, return whether prompt updates have been applied
1240
        (for most HF processors, this should be `True`).
1241

1242
        Note:
1243
            If `enable_hf_prompt_update=False`, we use HF processor
1244
            to perform prompt updates if available; HF processor requires
1245
            that the prompt corresponds to multi-modal items.
1246
1247
        """
        if isinstance(prompt, str):
1248
            if enable_hf_prompt_update:
1249
1250
1251
1252
                return self._apply_hf_processor_text_mm(
                    prompt_text=prompt,
                    mm_items=mm_items,
                    hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1253
                    tokenization_kwargs=tokenization_kwargs,
1254
1255
                )

1256
            prompt_ids = self._apply_hf_processor_text_only(prompt, tokenization_kwargs)
1257
1258
1259
        else:
            prompt_ids = self._apply_hf_processor_tokens_only(prompt)

1260
        mm_processed_data = self._apply_hf_processor_mm_only(
1261
            mm_items=mm_items,
1262
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1263
            tokenization_kwargs=tokenization_kwargs,
1264
1265
        )

1266
        return prompt_ids, mm_processed_data, False
1267

1268
1269
    def _get_cache_missing_items(
        self,
1270
        cache: BaseMultiModalProcessorCache,
1271
1272
        mm_data_items: MultiModalDataItems,
        mm_hashes: MultiModalHashes,
1273
    ) -> tuple[MultiModalIsCached, MultiModalDataItems]:
1274
        mm_is_cached = {
1275
            modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
1276
1277
1278
1279
        }

        mm_missing_idxs = {
            modality: [
1280
1281
                idx
                for idx, item_is_cached in enumerate(items_is_cached)
1282
1283
1284
1285
                if not item_is_cached
            ]
            for modality, items_is_cached in mm_is_cached.items()
        }
1286

1287
1288
1289
1290
1291
1292
1293
1294
        mm_missing_data = {}
        for modality, idxs in mm_missing_idxs.items():
            missing_modality_data = []
            for idx in idxs:
                data = mm_data_items[modality][idx]
                if data is None:
                    raise ValueError(
                        f"Cache miss for {modality} at index {idx} "
1295
1296
                        f"but data is not provided."
                    )
1297
1298
1299
                else:
                    missing_modality_data.append(data)
            mm_missing_data[modality] = missing_modality_data
1300

1301
        mm_missing_items = self.info.parse_mm_data(mm_missing_data, validate=False)
1302
1303

        return mm_is_cached, mm_missing_items
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315

    def _recompute_cached_prompt_update(
        self,
        cached_update: ResolvedPromptUpdate,
        new_item_idx: int,
    ) -> ResolvedPromptUpdate:
        """
        Override this if other attributes of `ResolvedPromptUpdate`
        also need to be recomputed after retrieving from the cache.
        """
        return replace(cached_update, item_idx=new_item_idx)

1316
1317
    def _merge_mm_kwargs(
        self,
1318
        cache: BaseMultiModalProcessorCache,
1319
        mm_hashes: MultiModalHashes,
1320
        mm_is_cached: MultiModalIsCached,
1321
        mm_missing_kwargs: MultiModalKwargsItems,
1322
1323
        mm_missing_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[MultiModalKwargsOptionalItems, MultiModalPromptUpdates]:
1324
1325
1326
1327
1328
        # Need to touch all mm hashes before update to avoid hash in updated
        # list evict during update
        for hashes in mm_hashes.values():
            for item_hash in hashes:
                cache.touch_sender_cache_item(item_hash)
1329

1330
        mm_missing_next_idx = defaultdict[str, int](lambda: 0)
1331

1332
        merged_kwargs = defaultdict[str, list[MultiModalKwargsItem | None]](list)
1333
1334
1335
        merged_prompt_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](
            list
        )
1336
1337
        for modality, hashes in mm_hashes.items():
            missing_kwargs = mm_missing_kwargs.get(modality, [])
1338
            missing_prompt_updates = mm_missing_prompt_updates.get(modality, [])
1339
1340
1341
1342

            for item_idx, item_hash in enumerate(hashes):
                if not mm_is_cached[modality][item_idx]:
                    missing_next_idx = mm_missing_next_idx[modality]
1343
1344
                    missing_kwargs_item = missing_kwargs[missing_next_idx]
                    missing_updates_item = missing_prompt_updates[missing_next_idx]
1345

1346
                    mm_missing_next_idx[modality] += 1
1347

1348
                    item = missing_kwargs_item, missing_updates_item
1349
                else:
1350
1351
1352
1353
1354
                    item = None

                kwargs, updates = cache.get_and_update_item(item, item_hash)

                merged_kwargs[modality].append(kwargs)
1355
1356
1357
1358
1359
1360
                merged_prompt_updates[modality].append(
                    [
                        self._recompute_cached_prompt_update(update, item_idx)
                        for update in updates
                    ]
                )
1361

1362
1363
        mm_kwargs = MultiModalKwargsItems(merged_kwargs)
        mm_prompt_updates = dict(merged_prompt_updates)
1364

1365
        return mm_kwargs, mm_prompt_updates
1366
1367
1368

    def _apply_hf_processor(
        self,
1369
1370
        inputs: ProcessorInputs,
        timing_ctx: TimingContext,
1371
    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
        with timing_ctx.record("apply_hf_processor"):
            (
                prompt_ids,
                mm_processed_data,
                is_update_applied,
            ) = self._apply_hf_processor_main(
                prompt=inputs.prompt,
                mm_items=inputs.mm_data_items,
                hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
                tokenization_kwargs=inputs.tokenization_kwargs,
                enable_hf_prompt_update=True,
            )
1384

1385
        mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
1386
            mm_processed_data,
1387
1388
1389
            self._get_mm_fields_config(
                mm_processed_data, inputs.hf_processor_mm_kwargs
            ),
1390
1391
        )

1392
        # Use overrides if provided; fallback to data-dependent hashing.
1393
1394
        with timing_ctx.record("get_mm_hashes"):
            mm_hashes = inputs.get_mm_hashes(self.info.model_id)
1395

1396
        mm_prompt_updates = self._get_mm_prompt_updates(
1397
1398
            inputs.mm_data_items,
            inputs.hf_processor_mm_kwargs,
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
            mm_kwargs,
        )

        mm_info = MultiModalProcessingInfo(
            kwargs=mm_kwargs,
            hashes=mm_hashes,
            prompt_updates=mm_prompt_updates,
        )

        return prompt_ids, mm_info, is_update_applied
1409

1410
1411
    def _cached_apply_hf_processor(
        self,
1412
1413
        inputs: ProcessorInputs,
        timing_ctx: TimingContext,
1414
    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1415
1416
1417
1418
1419
1420
        """
        Apply the HF processor on the full prompt text,
        caching the results and reusing cached results.
        """
        cache = self.cache

1421
        _, passthrough_data = self._get_hf_mm_data(inputs.mm_data_items)
1422
        if cache is None or passthrough_data:
1423
            return self._apply_hf_processor(inputs, timing_ctx)
1424

1425
1426
        with timing_ctx.record("get_mm_hashes"):
            mm_hashes = inputs.get_mm_hashes(self.info.model_id)
1427

1428
        with timing_ctx.record("get_cache_missing_items"):
1429
1430
            mm_is_cached, mm_missing_data_items = self._get_cache_missing_items(
                cache=cache,
1431
                mm_data_items=inputs.mm_data_items,
1432
1433
                mm_hashes=mm_hashes,
            )
1434

1435
        # NOTE: `prompt` does not correspond to `mm_missing_data_items`,
1436
        # so we can't apply prompt updates until the new multimodal
1437
        # items are combined with the cached multimodal items
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
        with timing_ctx.record("apply_hf_processor"):
            (
                prompt_ids,
                mm_missing_processed_data,
                is_update_applied,
            ) = self._apply_hf_processor_main(
                prompt=inputs.prompt,
                mm_items=mm_missing_data_items,
                hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
                tokenization_kwargs=inputs.tokenization_kwargs,
                enable_hf_prompt_update=False,
            )
1450

1451
        mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
1452
            mm_missing_processed_data,
1453
            self._get_mm_fields_config(
1454
                mm_missing_processed_data, inputs.hf_processor_mm_kwargs
1455
            ),
1456
1457
        )

1458
1459
        mm_missing_prompt_updates = self._get_mm_prompt_updates(
            mm_missing_data_items,
1460
            inputs.hf_processor_mm_kwargs,
1461
            mm_missing_kwargs,
1462
        )
1463

1464
        with timing_ctx.record("merge_mm_kwargs"):
1465
1466
1467
1468
1469
1470
1471
            mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
                cache,
                mm_hashes=mm_hashes,
                mm_is_cached=mm_is_cached,
                mm_missing_kwargs=mm_missing_kwargs,
                mm_missing_prompt_updates=mm_missing_prompt_updates,
            )
1472
1473
1474

        mm_info = MultiModalProcessingInfo(
            kwargs=mm_kwargs,
1475
            hashes=mm_hashes,
1476
1477
            prompt_updates=mm_prompt_updates,
        )
1478

1479
        return prompt_ids, mm_info, is_update_applied
1480

1481
1482
1483
    def _apply_token_matches(
        self,
        prompt: list[int],
1484
1485
1486
1487
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
        tokenizer = self.info.get_tokenizer()
        return apply_token_matches(prompt, mm_prompt_updates, tokenizer)
1488
1489
1490
1491

    def _apply_text_matches(
        self,
        prompt: str,
1492
1493
1494
1495
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[str, MultiModalPromptUpdatesApplyResult]:
        tokenizer = self.info.get_tokenizer()
        return apply_text_matches(prompt, mm_prompt_updates, tokenizer)
1496

1497
    def _apply_prompt_updates(
1498
1499
        self,
        token_ids: list[int],
1500
        mm_prompt_updates: MultiModalPromptUpdates,
1501
    ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
1502
        """Apply multi-modal prompt updates to token IDs."""
1503
        tokenizer = self.info.get_tokenizer()
1504

1505
1506
1507
1508
        new_token_ids, match_result = self._apply_token_matches(
            token_ids,
            mm_prompt_updates,
        )
1509
1510
1511
1512
1513
1514
1515
1516
1517

        # If the search text does not represent a special token,
        # it may have different token IDs in the prompt, because
        # the tokens may go across the boundaries of the search text.
        # ----
        # e.g. when searching for "foo" in "food", if "food" itself makes
        # up a token, then the token ID of "foo" will not appear at all
        # ----
        # Since it is inefficient to search for all possible tokenizations
1518
1519
        # of the search text in the prompt, we instead perform string-based
        # updates on the decoded token IDs, then encode them back.
1520
        if not all(
1521
1522
1523
            all(update_idx is not None for update_idx in update_idxs)
            for update_idxs in match_result.values()
        ):
1524
            new_text, match_result = self._apply_text_matches(
1525
                _seq2text(tokenizer, token_ids, use_cache=False),
1526
                mm_prompt_updates,
1527
1528
            )

1529
            new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False)
1530

1531
        matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
1532
1533
1534
1535
        for modality, update_idxs in match_result.items():
            for item_idx, update_idx in enumerate(update_idxs):
                assert update_idx is not None, (
                    "Failed to apply prompt replacement for "
1536
1537
                    f"mm_items[{modality!r}][{item_idx}]"
                )
1538
1539

                matched_updates[modality].append(
1540
1541
                    [mm_prompt_updates[modality][item_idx][update_idx]]
                )
1542
1543

        placeholders = self._find_mm_placeholders(
1544
1545
            new_token_ids,
            dict(matched_updates),
1546
        )
1547

1548
        return new_token_ids, placeholders
1549

1550
1551
    def _validate_mm_kwargs(
        self,
1552
        mm_kwargs: MultiModalKwargsOptionalItems,
1553
1554
1555
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
1556
            items = mm_kwargs.get(modality, [])
1557
1558
1559
1560
1561
1562
1563
1564
1565

            if len(items) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} {modality} items in "
                    f"keyword arguments corresponding to {item_count} "
                    f"{modality} data items, but only found {len(items)}! "
                    "There is likely a problem with your "
                    "implementation of merged multi-modal processor for this "
                    "model (usually arising from an inconsistency between "
1566
1567
                    "`_call_hf_processor` and `_get_mm_fields_config`)."
                )
1568

1569
    def _validate_mm_updates(
1570
        self,
1571
        mm_updates: MultiModalPromptUpdates,
1572
        mm_item_counts: Mapping[str, int],
1573
    ) -> None:
1574
        for modality, item_count in mm_item_counts.items():
1575
            placeholders = mm_updates.get(modality, [])
1576

1577
            if len(placeholders) != item_count:
1578
                raise RuntimeError(
1579
                    f"Expected there to be {item_count} prompt updates "
1580
                    f"corresponding to {item_count} {modality} items, but "
1581
                    f"instead found {len(placeholders)} prompt updates! "
1582
1583
1584
                    "This is likely because you forgot to include input "
                    "placeholder tokens (e.g., `<image>`, `<|image_pad|>`) "
                    "in the prompt. If the model has a chat template, make "
1585
1586
                    "sure you have applied it before calling `LLM.generate`."
                )
1587

1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
    def _validate_mm_placeholders(
        self,
        mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
            placeholders = mm_placeholders.get(modality, [])

            if len(placeholders) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} prompt placeholders "
                    f"corresponding to {item_count} {modality} items, but "
                    f"instead found {len(placeholders)} prompt placeholders! "
                    "Make sure the implementation of `_call_hf_processor` and "
1602
1603
                    "`_get_mm_fields_config` are consistent with each other."
                )
1604

1605
1606
1607
1608
    def _maybe_apply_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        prompt_ids: list[int],
1609
        mm_kwargs: MultiModalKwargsOptionalItems,
1610
        mm_prompt_updates: MultiModalPromptUpdates,
1611
        is_update_applied: bool,
1612
    ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
1613
        mm_item_counts = mm_items.get_all_counts()
1614
        self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
1615
        self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
1616

1617
        if is_update_applied:
1618
1619
            mm_placeholders = self._find_mm_placeholders(
                prompt_ids,
1620
                mm_prompt_updates,
1621
            )
1622
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1623
        else:
1624
            prompt_ids, mm_placeholders = self._apply_prompt_updates(
1625
                prompt_ids,
1626
                mm_prompt_updates,
1627
            )
1628
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1629

1630
        return prompt_ids, mm_placeholders
1631
1632
1633

    def apply(
        self,
1634
1635
        inputs: ProcessorInputs,
        timing_ctx: TimingContext,
1636
    ) -> MultiModalInput:
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
        """
        Process multi-modal inputs to be used in vLLM.

        The main steps are:

        1. Apply HF Processor on prompt text and multi-modal data together,
           outputting token IDs and processed tensors.
        2. Find and update sequences in the token IDs with placeholder tokens.
           The number of placeholder tokens equals the feature size of the
           multi-modal data outputted by the multi-modal encoder.
        3. Extract information about the placeholder tokens from the
           processed token IDs.
        """
        (
            prompt_ids,
1652
            mm_info,
1653
            is_update_applied,
1654
        ) = self._cached_apply_hf_processor(inputs, timing_ctx)
1655

1656
        # NOTE: tokenization_kwargs are not required to init processor
1657
        with timing_ctx.record("apply_prompt_updates"):
1658
            prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
1659
                mm_items=inputs.mm_data_items,
1660
1661
1662
1663
1664
                prompt_ids=prompt_ids,
                mm_kwargs=mm_info.kwargs,
                mm_prompt_updates=mm_info.prompt_updates,
                is_update_applied=is_update_applied,
            )
1665

1666
1667
1668
1669
        mm_placeholder_ranges = {
            modality: [item.to_range() for item in placeholders]
            for modality, placeholders in mm_placeholders.items()
        }
1670

1671
        return mm_input(
1672
            prompt_token_ids=prompt_ids,
1673
1674
            mm_kwargs=mm_info.kwargs,
            mm_hashes=mm_info.hashes,
1675
            mm_placeholders=mm_placeholder_ranges,
1676
        )
1677
1678
1679


class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
Ekagra Ranjan's avatar
Ekagra Ranjan committed
1680
1681
    skip_decoder_start_token: bool = False

1682
1683
1684
    @abstractmethod
    def create_encoder_prompt(
        self,
1685
        prompt: str | list[int],
1686
        mm_items: MultiModalDataItems,
1687
    ) -> str | list[int]:
1688
        """
1689
        Create input prompt for the encoder. HF processor will be applied on
1690
1691
        this prompt during profiling and generation.
        """
1692
1693
        raise NotImplementedError

1694
1695
    def create_decoder_prompt(
        self,
1696
        prompt: str | list[int],
1697
        mm_items: MultiModalDataItems,
1698
    ) -> str | list[int]:
1699
1700
1701
        """Create input prompt for the decoder."""
        return prompt

1702
    def _get_enc_dec_inputs(
1703
        self,
1704
        prompt: str | list[int],
1705
        mm_items: MultiModalDataItems,
1706
        encoder_inputs: MultiModalInput,
1707
    ):
1708
        tokenizer = self.info.get_tokenizer()
1709
        decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_items)
1710
        if isinstance(decoder_prompt_raw, str):
1711
            decoder_prompt_text = decoder_prompt_raw
1712
1713
            decoder_prompt_ids = tokenizer.encode(
                decoder_prompt_raw, add_special_tokens=False
1714
            )
1715
        else:
1716
            decoder_prompt_text = None
1717
            decoder_prompt_ids = decoder_prompt_raw
1718

1719
        return mm_enc_dec_input(
1720
1721
            encoder_inputs,
            decoder_prompt_ids,
1722
            decoder_prompt=decoder_prompt_text,
1723
        )
1724
1725
1726

    def apply(
        self,
1727
1728
        inputs: ProcessorInputs,
        timing_ctx: TimingContext,
1729
    ) -> MultiModalEncDecInput:
1730
1731
1732
1733
1734
1735
1736
        """
        Process multi-modal inputs to be used in vLLM.
        The main processing steps are modified to fit encoder-decoder model:
        1. Create encoder prompt from input prompt text.
        2. Apply the HF processor on encoder prompt.
        3. Copy the input prompt text as decoder prompt inputs.
        """
1737
1738
1739
1740
1741
        encoder_prompt = self.create_encoder_prompt(
            inputs.prompt,
            inputs.mm_data_items,
        )
        encoder_processor_inputs = ProcessorInputs(
1742
            encoder_prompt,
1743
1744
1745
1746
            inputs.mm_data_items,
            inputs.mm_uuid_items,
            hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
            tokenization_kwargs=inputs.tokenization_kwargs,
1747
1748
        )

1749
1750
        encoder_inputs = super().apply(encoder_processor_inputs, timing_ctx)

1751
        return self._get_enc_dec_inputs(
1752
1753
            prompt=inputs.prompt,
            mm_items=inputs.mm_data_items,
1754
1755
            encoder_inputs=encoder_inputs,
        )