processor.py 57.1 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from abc import ABC, abstractmethod
4
from collections import defaultdict
5
from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence
6
from dataclasses import dataclass, field, replace
7
from enum import Enum
8
9
10
11
12
13
14
15
16
from functools import lru_cache, partial
from typing import (
    TYPE_CHECKING,
    Generic,
    NamedTuple,
    Protocol,
    TypeAlias,
    cast,
)
17

18
import regex as re
19
import torch
20
from typing_extensions import TypeVar, assert_never
21

22
23
24
25
26
27
28
from vllm.inputs import (
    MultiModalEncDecInput,
    MultiModalHashes,
    MultiModalInput,
    mm_enc_dec_input,
    mm_input,
)
29
from vllm.logger import init_logger
30
from vllm.tokenizers import TokenizerLike
31
from vllm.transformers_utils.processor import call_hf_processor_mm_only
32
from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
33

34
from ..inputs import (
35
36
37
38
39
40
    MultiModalFieldConfig,
    MultiModalKwargsItem,
    MultiModalKwargsItems,
    MultiModalKwargsOptionalItems,
    PlaceholderRange,
)
41
from ..parse import (
42
43
44
    DictEmbeddingItems,
    EmbeddingItems,
    MultiModalDataItems,
45
    MultiModalUUIDItems,
46
)
47
from .context import BaseProcessingInfo, TimingContext
48
from .dummy_inputs import BaseDummyInputsBuilder
49
from .inputs import ProcessorInputs
50
51

if TYPE_CHECKING:
52
53
    from transformers.feature_extraction_utils import BatchFeature

54
    from ..cache import BaseMultiModalProcessorCache
55
56
57
58
else:
    BatchFeature = object

    BaseMultiModalProcessorCache = object
59

60
logger = init_logger(__name__)
61
62

_S = TypeVar("_S", str, list[int])
63

64

65
PromptSeq: TypeAlias = str | list[int]
66
"""A token sequence (list of token IDs) or text."""
67

68

69
70
@lru_cache(maxsize=2048)
def _cached_encode(
71
    tokenizer: TokenizerLike,
72
73
    text: str,
    *,
74
    add_special_tokens: bool = True,
75
) -> list[int]:
76
    return tokenizer.encode(text, add_special_tokens=add_special_tokens)
77
78
79
80


@lru_cache(maxsize=2048)
def _cached_decode(
81
    tokenizer: TokenizerLike,
82
83
    token_ids: tuple[int, ...],
    *,
84
    skip_special_tokens: bool = False,
85
) -> str:
86
    return tokenizer.decode(list(token_ids), skip_special_tokens=skip_special_tokens)
87
88


89
90
91
92
93
94
def _seq2text(
    tokenizer: TokenizerLike | None,
    seq: PromptSeq,
    *,
    use_cache: bool = True,
) -> str:
95
96
97
    if isinstance(seq, str):
        return seq

98
99
100
101
    if tokenizer is None:
        raise ValueError("You cannot decode tokens when `skip_tokenizer_init=True`")

    if not use_cache:
102
        return tokenizer.decode(seq)
103

104
105
106
    return _cached_decode(tokenizer, tuple(seq))


107
108
109
110
111
112
def _seq2tokens(
    tokenizer: TokenizerLike | None,
    seq: PromptSeq,
    *,
    use_cache: bool = True,
) -> list[int]:
113
    if isinstance(seq, str):
114
115
116
117
        if tokenizer is None:
            raise ValueError("You cannot encode text when `skip_tokenizer_init=True`")

        if not use_cache:
118
            return tokenizer.encode(seq, add_special_tokens=False)
119

120
121
122
123
124
        return _cached_encode(tokenizer, seq, add_special_tokens=False)

    return seq


125
126
127
class _GetMatchIndex(Protocol):
    def __call__(
        self,
128
        tokenizer: TokenizerLike | None,
129
130
        prompt: PromptSeq,
        start_idx: int = 0,
131
    ) -> int | None: ...
132
133


134
135
136
@dataclass
class PromptIndex:
    """Resolves to an index in the prompt."""
137

138
    get_match_index: _GetMatchIndex
139
140
141
142
143
144
145
146
147
148


class PromptIndexTargets:
    @staticmethod
    def start() -> PromptIndex:
        """
        Resolves to the start of the prompt (before the first token).

        This results in a match even if the prompt is empty.
        """
149
        return PromptIndex(lambda tokenizer, prompt, start_idx=0: 0)
150
151
152
153
154
155
156
157

    @staticmethod
    def prefix(seq: PromptSeq) -> PromptIndex:
        """
        Resolves to a location in the prompt after the given prefix.
        """

        def get_match_index(
158
            tokenizer: TokenizerLike | None,
159
            prompt: PromptSeq,
160
            start_idx: int = 0,
161
        ) -> int | None:
162
163
164
            if start_idx != 0:
                return None

165
166
167
            prefix = seq

            if isinstance(prompt, str):
168
169
                # Make both `str`
                prefix = _seq2text(tokenizer, prefix, use_cache=False)
170
            else:
171
172
                # Make both `list[int]`
                prefix = _seq2tokens(tokenizer, prefix, use_cache=False)
173
174
175
176
177
178
179
180
181
182
183
184
185

            match_idx = len(prefix)
            return match_idx if prompt[:match_idx] == prefix else None

        return PromptIndex(get_match_index)

    @staticmethod
    def end() -> PromptIndex:
        """
        Resolves to the end of the prompt (after the last token).

        This results in a match even if the prompt is empty.
        """
186
        return PromptIndex(lambda tokenizer, prompt, start_idx=0: len(prompt))
187
188


189
UpdateTarget: TypeAlias = PromptSeq | PromptIndex
190
191
192
193
"""
The token sequence or text to update.
"""

194
PromptUpdateTarget: TypeAlias = Callable[[int], UpdateTarget] | UpdateTarget
195
196
197
198
199
200
201
202
203
"""
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
output the corresponding token sequence (or text).

For convenience, you can directly pass in the token sequence (or text)
instead of a function if it does not depend on the input.
"""

204

205
@dataclass
206
class PromptUpdateDetails(Generic[_S]):
207
    """Details about the token sequence or text that are part of the update."""
208

209
    full: _S
210
    """The full content."""
211

212
    is_embed: Callable[[TokenizerLike | None, PromptSeq], torch.Tensor] | None = None
213
    """
214
215
216
    Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full],
    return a boolean mask of shape `(len(full),)` indicating which positions
    of `full` to assign embeddings to.
217
218
219
220

    `None` (default) means to assign embeddings to all positions of `full`.

    The embeddings are obtained by calling
221
    [`SupportsMultiModal.embed_multimodal`][vllm.model_executor.models.interfaces.SupportsMultiModal.embed_multimodal].
222
223
224
    """

    @staticmethod
225
    def from_seq(seq: _S) -> "PromptUpdateDetails[_S]":
226
227
228
229
230
231
232
        return PromptUpdateDetails(full=seq)

    @staticmethod
    def select_text(
        seq: _S,
        embed_text: str,
    ) -> "PromptUpdateDetails[_S]":
233
234
        def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
            embed_token_ids = _seq2tokens(tokenizer, embed_text, use_cache=False)
235
            token_ids = _seq2tokens(tokenizer, full)
236
237

            return torch.isin(
238
                torch.tensor(token_ids),
239
240
241
242
243
244
245
246
247
248
                torch.tensor(embed_token_ids),
            )

        return PromptUpdateDetails(full=seq, is_embed=is_embed)

    @staticmethod
    def select_token_id(
        seq: _S,
        embed_token_id: int,
    ) -> "PromptUpdateDetails[_S]":
249
        def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
250
251
252
253
254
            token_ids = _seq2tokens(tokenizer, full)

            return torch.tensor(token_ids) == embed_token_id

        return PromptUpdateDetails(full=seq, is_embed=is_embed)
255

256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
    @staticmethod
    def select_token_ids(
        seq: _S,
        embed_token_ids: list[int],
    ) -> "PromptUpdateDetails[_S]":
        def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
            token_ids = _seq2tokens(tokenizer, full)

            return torch.isin(
                torch.tensor(token_ids),
                torch.tensor(embed_token_ids),
            )

        return PromptUpdateDetails(full=seq, is_embed=is_embed)

271

272
PromptUpdateInfo: TypeAlias = PromptSeq | PromptUpdateDetails
273
"""
274
The token sequence or text that are part of the update.
275

276
If only part of the content corresponds to feature placeholders, you can
277
278
use [`PromptUpdateDetails`][vllm.multimodal.processing.PromptUpdateDetails] to
specify which part.
279
"""
280

281
PromptUpdateContent: TypeAlias = Callable[[int], PromptUpdateInfo] | PromptUpdateInfo
282
"""
283
284
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
285
286
287
288
289
290
291
292
293
294
295
296
297
output the corresponding token sequence (or text).

For convenience, you can directly pass in the token sequence (or text)
instead of a function if it does not depend on the input.
"""


class UpdateMode(str, Enum):
    INSERT = "insert"
    REPLACE = "replace"


@dataclass
298
class PromptUpdate(ABC):
299
300
301
302
303
304
305
    """
    Defines how to update a prompt with placeholder tokens.
    """

    modality: str
    """The modality for which the update is made."""

306
    target: PromptUpdateTarget
307
308
309
310
311
312
313
314
315
316
317
318
319
320
    """The token sequence (or text) to update."""

    @property
    @abstractmethod
    def content(self) -> PromptUpdateContent:
        """The placeholder tokens that are part of the update."""
        raise NotImplementedError

    @property
    @abstractmethod
    def mode(self) -> UpdateMode:
        """Defines how to update the prompt."""
        raise NotImplementedError

321
    def _resolve_target(self, item_idx: int) -> UpdateTarget:
322
323
324
325
        target = self.target
        if callable(target):
            target = target(item_idx)

326
        return target
327

328
    def _resolve_content(self, item_idx: int) -> PromptUpdateDetails:
329
330
331
332
333
334
335
        content = self.content
        if callable(content):
            content = content(item_idx)

        if not isinstance(content, PromptUpdateDetails):
            content = PromptUpdateDetails.from_seq(content)

336
        return content
337

338
    def resolve(self, item_idx: int) -> "ResolvedPromptUpdate":
339
340
341
342
343
344
345
346
347
        """
        Given the index of the processed item within
        [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
        output a copy of this object with its lazy attributes resolved.
        """
        return ResolvedPromptUpdate(
            modality=self.modality,
            item_idx=item_idx,
            mode=self.mode,
348
349
            target=self._resolve_target(item_idx),
            content=self._resolve_content(item_idx),
350
351
        )

352

353
@dataclass
354
355
356
357
358
359
class PromptInsertion(PromptUpdate):
    """
    Defines how to insert placeholder tokens into a prompt.

    Example:

360
361
    For each image, insert a number of `<image>` feature placeholders
    equal to the feature size of the vision encoder after the `<s>` token:
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380

    ```python
    PromptInsertion(
        modality="image",
        target="<s>",
        insertion="<image>" * image_feature_size,
    )
    ```

    Insert these tokens at the start of the prompt:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.start(),
        insertion="<image>" * image_feature_size,
    )
    ```

381
    Insert these tokens after a prefix `Images:`:
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.prefix("Images:"),
        insertion="<image>" * image_feature_size,
    )
    ```

    Insert these tokens at the end of the prompt:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.end(),
        insertion="<image>" * image_feature_size,
    )
    ```
400
401
402
403
    """

    insertion: PromptUpdateContent = field(repr=False)
    """
404
405
406
407
    Given the index of the processed item within
    [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
    output the token sequence (or text) to insert right after
    [`target`][vllm.multimodal.processing.PromptUpdate.target].
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423

    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
    """

    @property
    def content(self) -> PromptUpdateContent:
        return self.insertion

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.INSERT


@dataclass
class PromptReplacement(PromptUpdate):
424
425
    """
    Defines how to replace portions of an input prompt with placeholder tokens.
426
427
428

    Example:

429
430
    For each image, replace one `<image>` input placeholder in the prompt
    with a number of `<image>` feature placeholders
431
432
433
434
435
436
437
438
439
440
    equal to the feature size of the vision encoder:

    ```python
    PromptReplacement(
        modality="image",
        target="<image>",
        replacement="<image>" * image_feature_size,
    )
    ```

441
442
    As above, but further pad the feature placeholders with `<image_bos>`
    and `<image_eos>`, which are not supposed to be passed to the vision
443
444
445
446
447
448
449
    encoder:

    ```python
    PromptReplacement(
        modality="image",
        target="<image>",
        replacement=PromptUpdateDetails(
450
451
452
453
454
455
456
            full="".join(
                [
                    "<image_bos>",
                    "<image>" * image_feature_size,
                    "<image_eos>",
                ]
            ),
457
458
459
460
461
462
463
464
465
466
467
468
469
            features="<image>" * image_feature_size,
        ),
    )
    ```

    To avoid unnecessary tokenization during prompt replacement,
    we recommended passing token sequences instead of text:

    ```python
    PromptReplacement(
        modality="image",
        target=[image_token_id],
        replacement=PromptUpdateDetails(
470
471
472
            full=(
                [image_bos_id] + [image_token_id] * image_feature_size + [image_eos_id]
            ),
473
474
475
476
            features=[image_token_id] * image_feature_size,
        ),
    )
    ```
477
478
    """

479
    replacement: PromptUpdateContent = field(repr=False)
480
    """
481
482
483
484
    Given the index of the processed item within
    [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
    output the token sequence (or text) to replace
    [`target`][vllm.multimodal.processing.PromptUpdate.target].
485

486
487
    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
488
489
    """

490
491
492
493
494
495
496
    @property
    def content(self) -> PromptUpdateContent:
        return self.replacement

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.REPLACE
497
498


499
500
501
class _HasModalityAttr(Protocol):
    modality: str

502

503
504
class _HasModalityProp(Protocol):
    @property
505
    def modality(self) -> str: ...
506
507


508
_M = TypeVar("_M", bound=_HasModalityAttr | _HasModalityProp)
509
510
511


def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]:
512
513
    """
    Convenience function to apply
514
    [`full_groupby`][vllm.utils.collection_utils.full_groupby]
515
516
    based on modality.
    """
517
518
519
    return full_groupby(values, key=lambda x: x.modality)


520
521
522
523
524
525
526
class PromptTargetMatch(NamedTuple):
    start_idx: int
    end_idx: int


@dataclass(frozen=True)
class ResolvedPromptUpdate:
527
    """
528
529
    A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] with its
    lazy attributes resolved, apart from those related to tokenization.
530
    """
531

532
533
    modality: str
    """The modality for which the update is made."""
534

535
536
    item_idx: int
    """The index within `modality` of the item this update pertains to."""
537

538
539
    mode: UpdateMode
    """Defines how to update the prompt."""
540

541
    target: UpdateTarget
542
    """The token sequence (or text) to update."""
543

544
    content: PromptUpdateDetails = field(repr=False)
545
    """The placeholder tokens that are part of the update."""
546

547
548
549
    def iter_token_matches(
        self,
        prompt: list[int],
550
        tokenizer: TokenizerLike | None,
551
552
553
554
555
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        target = self.target
556

557
558
559
560
        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(tokenizer, prompt, start_idx)
            if match_idx is not None:
                yield PromptTargetMatch(match_idx, match_idx)
561

562
            return
563

564
565
        target_token_ids = _seq2tokens(tokenizer, target)

566
        for match in iter_token_matches(prompt, target_token_ids, start_idx=start_idx):
567
            yield PromptTargetMatch(match.start_idx, match.end_idx)
568

569
570
571
    def iter_text_matches(
        self,
        prompt: str,
572
        tokenizer: TokenizerLike | None,
573
574
575
576
577
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        target = self.target
578

579
580
581
582
        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(tokenizer, prompt, start_idx)
            if match_idx is not None:
                yield PromptTargetMatch(match_idx, match_idx)
583

584
            return
585

586
587
        target_text = _seq2text(tokenizer, target)

588
        for match in re.finditer(re.escape(target_text), prompt, pos=start_idx):
589
590
591
592
            yield PromptTargetMatch(match.start(), match.end())

    def iter_matches(
        self,
593
        prompt: list[int] | str,
594
        tokenizer: TokenizerLike | None,
595
596
597
598
599
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        if isinstance(prompt, str):
600
            return self.iter_text_matches(prompt, tokenizer, start_idx=start_idx)
601
602

        return self.iter_token_matches(prompt, tokenizer, start_idx=start_idx)
603

604
605
606
607
608
609
610
611
612
    def with_target(self, target: UpdateTarget):
        return replace(self, target=target)

    def with_content(self, content: PromptUpdateInfo):
        if not isinstance(content, PromptUpdateDetails):
            content = PromptUpdateDetails.from_seq(content)

        return replace(self, content=content)

613

614
615
616
class _TokenMatch(NamedTuple):
    start_idx: int
    end_idx: int
617
618


619
620
621
def iter_token_matches(
    token_ids: list[int],
    match_ids: list[int],
622
623
    *,
    start_idx: int = 0,
624
) -> Generator[_TokenMatch]:
625
    """
626
    Yield each occurrence of `match_ids` in `token_ids`.
627
628
629
630

    Note that empty matches are ignored.
    """
    prompt_len = len(token_ids)
631
    match_len = len(match_ids)
632

633
634
    if match_len == 0:
        return
635

636
    while start_idx < prompt_len - match_len + 1:
637
        end_idx = start_idx + match_len
638

639
640
        if token_ids[start_idx:end_idx] == match_ids:
            yield _TokenMatch(start_idx=start_idx, end_idx=end_idx)
641
642
643
644
645

            # Exclude overlapping matches
            start_idx = end_idx
        else:
            start_idx += 1
646
647


648
649
650
651
652
653
def replace_token_matches(
    token_ids: list[int],
    match_ids: list[int],
    new_ids: list[int],
) -> list[int]:
    """
654
655
    Replace each occurrence of `match_ids` in `token_ids`
    with `new_ids`.
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674

    Note that empty matches are ignored.
    """
    out_seqs = list[list[int]]()
    prev_end_idx = 0

    for match in iter_token_matches(token_ids, match_ids):
        start_idx = match.start_idx
        end_idx = match.end_idx

        out_seqs.append(token_ids[prev_end_idx:start_idx])
        out_seqs.append(new_ids)
        prev_end_idx = end_idx

    out_seqs.append(token_ids[prev_end_idx:])

    return flatten_2d_lists(out_seqs)


675
@dataclass
676
class PlaceholderFeaturesInfo:
677
    modality: str
678
    item_idx: int
679
    start_idx: int
680
    tokens: list[int]
681
    is_embed: torch.Tensor | None
682
683
684

    @property
    def length(self) -> int:
685
        return len(self.tokens)
686
687

    def to_range(self) -> PlaceholderRange:
688
689
        # TODO: Is it worth it to optimize this by stripping the
        # leading and ending positions where `is_embed=False`?
690
691
692
        return PlaceholderRange(
            offset=self.start_idx,
            length=self.length,
693
            is_embed=self.is_embed,
694
        )
695
696


697
_MatchToApply = tuple[tuple[str, int], tuple[PromptTargetMatch, int]]
698
699


700
701
702
def _find_matches(
    prompt: _S,
    mm_prompt_updates: "MultiModalPromptUpdates",
703
    tokenizer: TokenizerLike | None,
704
705
706
    *,
    prev_end_idx: int = 0,
    current_result: "MultiModalPromptUpdatesApplyResult",
707
708
) -> tuple[UpdateMode | None, list[_MatchToApply]]:
    mode: UpdateMode | None = None
709
710
711
712
713
714
715
716
717
718
719
720
    mm_matches = dict[tuple[str, int], tuple[PromptTargetMatch, int]]()

    for modality, modality_updates in mm_prompt_updates.items():
        for item_idx, item_updates in enumerate(modality_updates):
            if current_result[modality][item_idx] is not None:
                continue  # Updates have already been applied for this item

            for update_idx, update in enumerate(item_updates):
                if (modality, item_idx) in mm_matches:
                    break  # Already found a match for this item

                for match in update.iter_matches(
721
722
723
                    prompt,
                    tokenizer,
                    start_idx=prev_end_idx,
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
                ):
                    # All matches should share the same mode
                    if mode is None:
                        mode = update.mode
                    elif mode != update.mode:
                        continue

                    mm_matches[(modality, item_idx)] = match, update_idx
                    break  # Get only the first valid match per item

    # Prioritize earlier matches
    matches_to_apply = sorted(mm_matches.items(), key=lambda item: item[1][0])

    # To avoid conflicts, only replace one non-empty item at a time
    if mode == UpdateMode.REPLACE:
        matches_to_apply_ = list[_MatchToApply]()
        has_non_empty_matches = False

        for item in matches_to_apply:
            _, (match, _) = item
            if match.start_idx == match.end_idx:
                matches_to_apply_.append(item)
            elif not has_non_empty_matches:
                has_non_empty_matches = True
                matches_to_apply_.append(item)

        matches_to_apply = matches_to_apply_

    return mode, matches_to_apply
753
754


755
756
757
758
759
760
761
762
763
764
def _all_items_found(
    mm_item_counts: dict[str, int],
    mm_found_counts: dict[str, int],
) -> bool:
    return all(
        item_idx >= mm_item_counts[modality]
        for modality, item_idx in mm_found_counts.items()
    )


765
def _apply_matches(
766
    prompt: _S,
767
    mm_prompt_updates: "MultiModalPromptUpdates",
768
    tokenizer: TokenizerLike | None,
769
) -> tuple[list[_S], "MultiModalPromptUpdatesApplyResult"]:
770
    mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()}
771

772
    out_seqs = list[str | list[int]]()
773
    out_result: MultiModalPromptUpdatesApplyResult = {
774
        m: [None] * len(items) for m, items in mm_prompt_updates.items()
775
    }
776

777
    # Early exit if no items to find
778
779
780
781
782
783
    mm_found_counts = {
        m: sum(r is not None for r in res) for m, res in out_result.items()
    }
    if _all_items_found(mm_item_counts, mm_found_counts):
        return [prompt], out_result

784
785
    prev_end_idx = 0
    while True:
786
787
788
789
790
791
792
        mode, matches_to_apply = _find_matches(
            prompt,
            mm_prompt_updates,
            tokenizer,
            prev_end_idx=prev_end_idx,
            current_result=out_result,
        )
793

794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
        if mode is None:
            break  # No more matches to find

        for (modality, item_idx), (match, update_idx) in matches_to_apply:
            matched_update = mm_prompt_updates[modality][item_idx][update_idx]
            matched_content = matched_update.content.full

            if mode == UpdateMode.INSERT:
                end_idx_to_insert = match.end_idx
            elif mode == UpdateMode.REPLACE:
                end_idx_to_insert = match.start_idx
            else:
                assert_never(mode)

            out_seqs.append(prompt[prev_end_idx:end_idx_to_insert])
            out_seqs.append(
                _seq2text(tokenizer, matched_content)
                if isinstance(prompt, str)
                else _seq2tokens(tokenizer, matched_content)
            )
            out_result[modality][item_idx] = update_idx

            # Exclude overlapping matches
            prev_end_idx = match.end_idx

        # Early exit if all items found
        mm_found_counts = {
            m: sum(r is not None for r in res) for m, res in out_result.items()
        }
        if _all_items_found(mm_item_counts, mm_found_counts):
            break
825
826
827

    out_seqs.append(prompt[prev_end_idx:])

828
    return cast(list[_S], out_seqs), out_result
829
830


831
def apply_token_matches(
832
    prompt: list[int],
833
    mm_prompt_updates: "MultiModalPromptUpdates",
834
    tokenizer: TokenizerLike | None,
835
836
837
) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
    """
    Apply the updates in `mm_prompt_updates` to `prompt`.
838

839
840
841
842
    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
    appears earlier in `mm_prompt_updates` takes priority.
    """
843
    token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
844

845
    return flatten_2d_lists(token_id_seqs), result
846
847


848
def apply_text_matches(
849
    prompt: str,
850
    mm_prompt_updates: "MultiModalPromptUpdates",
851
    tokenizer: TokenizerLike | None,
852
853
854
) -> tuple[str, "MultiModalPromptUpdatesApplyResult"]:
    """
    Apply the updates in `mm_prompt_updates` to `prompt`.
855

856
857
858
859
860
    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
    appears earlier in `mm_prompt_updates` takes priority.
    """
    texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
861

862
    return "".join(texts), result
863
864


865
def _iter_placeholders(
866
    prompt: list[int],
867
    mm_prompt_updates: "MultiModalPromptUpdates",
868
    tokenizer: TokenizerLike | None,
869
) -> Iterable[PlaceholderFeaturesInfo]:
870
    """
871
    Yield each set of placeholder tokens found in `prompt`.
872
873
874

    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
875
    appears earlier in `mm_prompt_updates` takes priority.
876

877
878
    Note that empty matches are ignored.
    """
879
    mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()}
880
    item_idx_by_modality = {modality: 0 for modality in mm_prompt_updates}
881

882
883
    if _all_items_found(mm_item_counts, item_idx_by_modality):
        return
884

885
    prompt_len = len(prompt)
886
    start_idx = 0
887

888
889
890
    while start_idx < prompt_len:
        found = False

891
        for modality, modality_updates in mm_prompt_updates.items():
892
893
            item_idx = item_idx_by_modality[modality]
            if item_idx >= mm_item_counts.get(modality, 0):
894
                continue
895

896
897
            for update in modality_updates[item_idx]:
                content = update.content
898
                content_tokens_full = _seq2tokens(tokenizer, content.full)
899
900
                content_len_full = len(content_tokens_full)
                end_idx_full = start_idx + content_len_full
901

902
                if content_len_full == 0 or end_idx_full > prompt_len:
903
904
                    continue

905
                if prompt[start_idx:end_idx_full] == content_tokens_full:
906
907
                    content_is_embed = content.is_embed
                    if content_is_embed is not None:
908
                        content_is_embed = content_is_embed(tokenizer, content.full)
909
910
911
912
913
914
915
916

                    yield PlaceholderFeaturesInfo(
                        modality=modality,
                        item_idx=item_idx,
                        start_idx=start_idx,
                        tokens=content_tokens_full,
                        is_embed=content_is_embed,
                    )
917

918
                    # Exclude overlapping matches
919
                    start_idx = end_idx_full
920
921
922
                    item_idx_by_modality[modality] += 1
                    found = True
                    break
923

924
            if found:
925
926
927
                if _all_items_found(mm_item_counts, item_idx_by_modality):
                    return

928
                break  # Go back to the outer while loop
929
930
931

        if not found:
            start_idx += 1
932
933


934
935
def find_mm_placeholders(
    prompt: list[int],
936
    mm_prompt_updates: "MultiModalPromptUpdates",
937
    tokenizer: TokenizerLike | None,
938
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
939
    it = _iter_placeholders(prompt, mm_prompt_updates, tokenizer)
940
941
942
    return dict(full_groupby_modality(it))


943
944
945
MultiModalIsCached = dict[str, list[bool]]
"""
A collection of the `is_cached` flag for each item, with a similar structure as
946
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
947
948
"""

949
MultiModalPromptUpdates = Mapping[str, list[Sequence[ResolvedPromptUpdate]]]
950
951
952
953
954
"""
A collection of prompt updates with a similar structure as
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
"""

955
MultiModalPromptUpdatesApplyResult = Mapping[str, list[int | None]]
956
957
958
959
960
961
962
"""
For an item `MultiModalPromptUpdates[k][i]`,
`MultiModalPromptUpdatesApplyResult[k][i]` represents the index of the
`ResolvedPromptUpdate` instance that has been applied, or `None` if none of the
`ResolvedPromptUpdate` instances have been applied.
"""

963
964
_I = TypeVar("_I", bound=BaseProcessingInfo)

965
966

class MultiModalProcessingInfo(NamedTuple):
967
    kwargs: MultiModalKwargsOptionalItems
968
    hashes: MultiModalHashes
969
970
    prompt_updates: MultiModalPromptUpdates

971
972

class BaseMultiModalProcessor(ABC, Generic[_I]):
973
    """
974
    Abstract base class to process multi-modal inputs to be used in vLLM.
975

976
    Not to be confused with `transformers.ProcessorMixin`.
977
978
    """

979
980
981
982
983
    def __init__(
        self,
        info: _I,
        dummy_inputs: "BaseDummyInputsBuilder[_I]",
        *,
984
        cache: BaseMultiModalProcessorCache | None = None,
985
    ) -> None:
986
987
        super().__init__()

988
989
        self.info = info
        self.dummy_inputs = dummy_inputs
990
        self.cache = cache
991

992
        self.data_parser = self.info.get_data_parser()
993

994
    def __call__(
995
        self,
996
        prompt: str,
997
        mm_items: MultiModalDataItems,
998
999
        mm_uuid_items: MultiModalUUIDItems | None = None,
        hf_processor_mm_kwargs: Mapping[str, object] | None = None,
1000
    ) -> MultiModalInput:
1001
        processor_inputs = ProcessorInputs(
1002
1003
1004
            prompt,
            mm_items,
            mm_uuid_items,
1005
            hf_processor_mm_kwargs=hf_processor_mm_kwargs or {},
1006
        )
1007

1008
1009
        return self.apply(processor_inputs, TimingContext(enabled=False))

1010
1011
1012
    @abstractmethod
    def _get_mm_fields_config(
        self,
1013
        hf_inputs: BatchFeature,
1014
1015
1016
1017
1018
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, MultiModalFieldConfig]:
        """Given the HF-processed data, output the metadata of each field."""
        raise NotImplementedError

1019
    @abstractmethod
1020
    def _get_prompt_updates(
1021
        self,
1022
        mm_items: MultiModalDataItems,
1023
        hf_processor_mm_kwargs: Mapping[str, object],
1024
        out_mm_kwargs: MultiModalKwargsItems,
1025
    ) -> Sequence[PromptUpdate]:
1026
1027
        """
        Given the original multi-modal items for this modality
1028
        and HF-processed data, output the updates to perform.
1029

1030
1031
1032
1033
1034
1035
        The information returned by this method is used to update token inputs
        which bypass the HF processor. It is also used to update the output of
        HF processor if the HF process does not apply prompt updates to text
        inputs.

        Moreover, this information is critical to determine the token positions
1036
1037
        in order to construct
        [`PlaceholderRange`][vllm.multimodal.inputs.PlaceholderRange]
1038
        for each multi-modal item.
1039
1040
        """
        raise NotImplementedError
1041

1042
1043
1044
1045
1046
1047
    def _bind_and_group_updates(
        self,
        prompt_updates: Sequence[PromptUpdate],
        mm_item_counts: Mapping[str, int],
    ) -> MultiModalPromptUpdates:
        return {
1048
1049
1050
1051
            modality: [
                [update.resolve(item_idx) for update in updates]
                for item_idx in range(mm_item_counts.get(modality, 0))
            ]
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
            for modality, updates in full_groupby_modality(prompt_updates)
        }

    def _get_mm_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        out_mm_kwargs: MultiModalKwargsItems,
    ) -> MultiModalPromptUpdates:
        unbound_prompt_updates = self._get_prompt_updates(
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
            out_mm_kwargs=out_mm_kwargs,
        )

        mm_prompt_updates = self._bind_and_group_updates(
            unbound_prompt_updates,
            mm_items.get_all_counts(),
        )

        return mm_prompt_updates

1074
    def _find_mm_placeholders(
1075
1076
        self,
        new_token_ids: list[int],
1077
        mm_prompt_updates: MultiModalPromptUpdates,
1078
    ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
1079
1080
        tokenizer = self.info.get_tokenizer()

1081
        return find_mm_placeholders(new_token_ids, mm_prompt_updates, tokenizer)
1082

1083
    def _get_hf_mm_data(
1084
        self,
1085
        mm_items: MultiModalDataItems,
1086
    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
1087
        """Extract processor and passthrough data from multi-modal items."""
1088
1089
        processor_data = dict[str, object]()
        passthrough_data = dict[str, object]()
1090

1091
1092
1093
        for items in mm_items.values():
            processor_data.update(items.get_processor_data())
            passthrough_data.update(items.get_passthrough_data())
1094

1095
1096
        return processor_data, passthrough_data

1097
1098
1099
    def _call_hf_processor(
        self,
        prompt: str,
1100
1101
1102
1103
        # Not to be confused with `mm_data` in `self.apply`.
        # This refers to the data to be passed to HF processor.
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
1104
        tok_kwargs: Mapping[str, object],
1105
    ) -> BatchFeature:
1106
1107
1108
1109
        """
        Call the HF processor on the prompt text and
        associated multi-modal data.
        """
1110
1111
1112
1113
1114
        return self.info.ctx.call_hf_processor(
            self.info.get_hf_processor(**mm_kwargs),
            dict(text=prompt, **mm_data),
            dict(**mm_kwargs, **tok_kwargs),
        )
1115

1116
    def _hf_processor_applies_updates(
1117
1118
1119
1120
        self,
        prompt_text: str,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1121
        tokenization_kwargs: Mapping[str, object],
1122
1123
    ) -> bool:
        """
1124
        Return whether the HF processor applies prompt updates.
1125

1126
1127
        For most HF processors, this should be `True` when multi-modal
        data items are passed, but `False` when multi-modal embeddings
1128
1129
1130
1131
        are passed.
        """
        return not any(
            isinstance(items, (EmbeddingItems, DictEmbeddingItems))
1132
1133
            for items in mm_items.values()
        )
1134

1135
    def _apply_hf_processor_text_mm(
1136
        self,
1137
        prompt_text: str,
1138
        mm_items: MultiModalDataItems,
1139
        hf_processor_mm_kwargs: Mapping[str, object],
1140
        tokenization_kwargs: Mapping[str, object],
1141
    ) -> tuple[list[int], BatchFeature, bool]:
1142
        """
1143
1144
        Apply the HF processor on the prompt text and multi-modal data
        together.
1145

1146
        In addition, return whether prompt updates have been applied.
1147
        """
1148
1149
1150
1151
        valid_mm_items = mm_items.select(
            {k for k, c in mm_items.get_all_counts().items() if c > 0}
        )
        processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
1152
1153
1154
1155
1156

        processed_data = self._call_hf_processor(
            prompt=prompt_text,
            mm_data=processor_data,
            mm_kwargs=hf_processor_mm_kwargs,
1157
            tok_kwargs=tokenization_kwargs,
1158
1159
        )
        processed_data.update(passthrough_data)
1160

1161
1162
1163
1164
1165
        input_ids = processed_data.pop("input_ids")
        if not isinstance(input_ids, list):
            input_ids = input_ids.tolist()

        (prompt_ids,) = input_ids
1166

1167
        is_update_applied = self._hf_processor_applies_updates(
1168
1169
1170
            prompt_text=prompt_text,
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1171
            tokenization_kwargs=tokenization_kwargs,
1172
1173
        )

1174
        return prompt_ids, processed_data, is_update_applied
1175

1176
    def _apply_hf_processor_text_only(
1177
1178
1179
1180
        self,
        prompt_text: str,
        tokenization_kwargs: Mapping[str, object],
    ) -> list[int]:
1181
        """
1182
        Apply the HF processor on the prompt text only.
1183

1184
1185
1186
        Since HF processor requires that text and multi-modal items
        correspond to each other, we create dummy multi-modal items
        to go along with the text.
1187
        """
1188
        prompt_ids, _, _ = self._apply_hf_processor_text_mm(
1189
1190
1191
            prompt_text=prompt_text,
            mm_items=MultiModalDataItems({}),
            hf_processor_mm_kwargs={},
1192
            tokenization_kwargs=tokenization_kwargs,
1193
1194
        )

1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
        return prompt_ids

    def _apply_hf_processor_tokens_only(
        self,
        prompt_tokens: list[int],
    ) -> list[int]:
        """
        Apply the HF processor on the prompt tokens only.

        Most HF processors accept prompt text but not prompt tokens.
        If the HF processor adds or removes tokens that are not related to
        multi-modal data, you should override this method so it is consistent
1207
1208
1209
        with the output of
        [`_apply_hf_processor_text_only`][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_text_only]
        on the
1210
1211
1212
1213
1214
1215
1216
1217
        corresponding text.
        """
        return prompt_tokens

    def _apply_hf_processor_mm_only(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1218
        tokenization_kwargs: Mapping[str, object],
1219
    ) -> BatchFeature:
1220
1221
1222
1223
1224
        """
        Apply the HF processor on the multi-modal data only.

        Since HF processor requires that text and multi-modal items
        correspond to each other, we generate dummy text using
1225
        [`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder]
1226
        to go along with the multi-modal data.
1227
        """
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
        # Custom logic based on text inputs
        if type(self)._call_hf_processor != BaseMultiModalProcessor._call_hf_processor:
            mm_counts = mm_items.get_all_counts()

            _, mm_processed_data, _ = self._apply_hf_processor_text_mm(
                prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
                mm_items=mm_items,
                hf_processor_mm_kwargs=hf_processor_mm_kwargs,
                tokenization_kwargs=tokenization_kwargs,
            )
1238

1239
1240
1241
1242
            return mm_processed_data

        valid_mm_items = mm_items.select(
            {k for k, c in mm_items.get_all_counts().items() if c > 0}
1243
        )
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
        processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

        processed_data = self.info.ctx.call_hf_processor(
            partial(
                call_hf_processor_mm_only,
                self.info.get_hf_processor(**hf_processor_mm_kwargs),
            ),
            processor_data,
            dict(**hf_processor_mm_kwargs, **tokenization_kwargs),
        )
        processed_data.update(passthrough_data)
1255

1256
        return processed_data
1257
1258
1259

    def _apply_hf_processor_main(
        self,
1260
        prompt: str | list[int],
1261
1262
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1263
        tokenization_kwargs: Mapping[str, object],
1264
        *,
1265
        enable_hf_prompt_update: bool,
1266
    ) -> tuple[list[int], BatchFeature, bool]:
1267
1268
1269
        """
        Apply the HF processor on the prompt text and multi-modal data.

1270
        In addition, return whether prompt updates have been applied
1271
        (for most HF processors, this should be `True`).
1272

1273
        Note:
1274
            If `enable_hf_prompt_update=False`, we use HF processor
1275
            to perform prompt updates if available; HF processor requires
1276
            that the prompt corresponds to multi-modal items.
1277
1278
        """
        if isinstance(prompt, str):
1279
            if enable_hf_prompt_update:
1280
1281
1282
1283
                return self._apply_hf_processor_text_mm(
                    prompt_text=prompt,
                    mm_items=mm_items,
                    hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1284
                    tokenization_kwargs=tokenization_kwargs,
1285
1286
                )

1287
            prompt_ids = self._apply_hf_processor_text_only(prompt, tokenization_kwargs)
1288
1289
1290
        else:
            prompt_ids = self._apply_hf_processor_tokens_only(prompt)

1291
        mm_processed_data = self._apply_hf_processor_mm_only(
1292
            mm_items=mm_items,
1293
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1294
            tokenization_kwargs=tokenization_kwargs,
1295
1296
        )

1297
        return prompt_ids, mm_processed_data, False
1298

1299
1300
    def _get_cache_missing_items(
        self,
1301
        cache: BaseMultiModalProcessorCache,
1302
1303
        mm_data_items: MultiModalDataItems,
        mm_hashes: MultiModalHashes,
1304
    ) -> tuple[MultiModalIsCached, MultiModalDataItems]:
1305
        mm_is_cached = {
1306
            modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
1307
1308
1309
1310
        }

        mm_missing_idxs = {
            modality: [
1311
1312
                idx
                for idx, item_is_cached in enumerate(items_is_cached)
1313
1314
1315
1316
                if not item_is_cached
            ]
            for modality, items_is_cached in mm_is_cached.items()
        }
1317

1318
1319
1320
1321
1322
1323
1324
1325
        mm_missing_data = {}
        for modality, idxs in mm_missing_idxs.items():
            missing_modality_data = []
            for idx in idxs:
                data = mm_data_items[modality][idx]
                if data is None:
                    raise ValueError(
                        f"Cache miss for {modality} at index {idx} "
1326
1327
                        f"but data is not provided."
                    )
1328
1329
1330
                else:
                    missing_modality_data.append(data)
            mm_missing_data[modality] = missing_modality_data
1331

1332
        mm_missing_items = self.info.parse_mm_data(mm_missing_data, validate=False)
1333
1334

        return mm_is_cached, mm_missing_items
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346

    def _recompute_cached_prompt_update(
        self,
        cached_update: ResolvedPromptUpdate,
        new_item_idx: int,
    ) -> ResolvedPromptUpdate:
        """
        Override this if other attributes of `ResolvedPromptUpdate`
        also need to be recomputed after retrieving from the cache.
        """
        return replace(cached_update, item_idx=new_item_idx)

1347
1348
    def _merge_mm_kwargs(
        self,
1349
        cache: BaseMultiModalProcessorCache,
1350
        mm_hashes: MultiModalHashes,
1351
        mm_is_cached: MultiModalIsCached,
1352
        mm_missing_kwargs: MultiModalKwargsItems,
1353
1354
        mm_missing_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[MultiModalKwargsOptionalItems, MultiModalPromptUpdates]:
1355
1356
1357
1358
1359
        # Need to touch all mm hashes before update to avoid hash in updated
        # list evict during update
        for hashes in mm_hashes.values():
            for item_hash in hashes:
                cache.touch_sender_cache_item(item_hash)
1360

1361
        mm_missing_next_idx = defaultdict[str, int](lambda: 0)
1362

1363
        merged_kwargs = defaultdict[str, list[MultiModalKwargsItem | None]](list)
1364
1365
1366
        merged_prompt_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](
            list
        )
1367
1368
        for modality, hashes in mm_hashes.items():
            missing_kwargs = mm_missing_kwargs.get(modality, [])
1369
            missing_prompt_updates = mm_missing_prompt_updates.get(modality, [])
1370
1371
1372
1373

            for item_idx, item_hash in enumerate(hashes):
                if not mm_is_cached[modality][item_idx]:
                    missing_next_idx = mm_missing_next_idx[modality]
1374
1375
                    missing_kwargs_item = missing_kwargs[missing_next_idx]
                    missing_updates_item = missing_prompt_updates[missing_next_idx]
1376

1377
                    mm_missing_next_idx[modality] += 1
1378

1379
                    item = missing_kwargs_item, missing_updates_item
1380
                else:
1381
1382
1383
1384
1385
                    item = None

                kwargs, updates = cache.get_and_update_item(item, item_hash)

                merged_kwargs[modality].append(kwargs)
1386
1387
1388
1389
1390
1391
                merged_prompt_updates[modality].append(
                    [
                        self._recompute_cached_prompt_update(update, item_idx)
                        for update in updates
                    ]
                )
1392

1393
1394
        mm_kwargs = MultiModalKwargsItems(merged_kwargs)
        mm_prompt_updates = dict(merged_prompt_updates)
1395

1396
        return mm_kwargs, mm_prompt_updates
1397
1398
1399

    def _apply_hf_processor(
        self,
1400
1401
        inputs: ProcessorInputs,
        timing_ctx: TimingContext,
1402
    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
        with timing_ctx.record("apply_hf_processor"):
            (
                prompt_ids,
                mm_processed_data,
                is_update_applied,
            ) = self._apply_hf_processor_main(
                prompt=inputs.prompt,
                mm_items=inputs.mm_data_items,
                hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
                tokenization_kwargs=inputs.tokenization_kwargs,
                enable_hf_prompt_update=True,
            )
1415

1416
        mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
1417
            mm_processed_data,
1418
1419
1420
            self._get_mm_fields_config(
                mm_processed_data, inputs.hf_processor_mm_kwargs
            ),
1421
1422
        )

1423
        # Use overrides if provided; fallback to data-dependent hashing.
1424
1425
        with timing_ctx.record("get_mm_hashes"):
            mm_hashes = inputs.get_mm_hashes(self.info.model_id)
1426

1427
        mm_prompt_updates = self._get_mm_prompt_updates(
1428
1429
            inputs.mm_data_items,
            inputs.hf_processor_mm_kwargs,
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
            mm_kwargs,
        )

        mm_info = MultiModalProcessingInfo(
            kwargs=mm_kwargs,
            hashes=mm_hashes,
            prompt_updates=mm_prompt_updates,
        )

        return prompt_ids, mm_info, is_update_applied
1440

1441
1442
    def _cached_apply_hf_processor(
        self,
1443
1444
        inputs: ProcessorInputs,
        timing_ctx: TimingContext,
1445
    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1446
1447
1448
1449
1450
1451
        """
        Apply the HF processor on the full prompt text,
        caching the results and reusing cached results.
        """
        cache = self.cache

1452
        _, passthrough_data = self._get_hf_mm_data(inputs.mm_data_items)
1453
        if cache is None or passthrough_data:
1454
            return self._apply_hf_processor(inputs, timing_ctx)
1455

1456
1457
        with timing_ctx.record("get_mm_hashes"):
            mm_hashes = inputs.get_mm_hashes(self.info.model_id)
1458

1459
        with timing_ctx.record("get_cache_missing_items"):
1460
1461
            mm_is_cached, mm_missing_data_items = self._get_cache_missing_items(
                cache=cache,
1462
                mm_data_items=inputs.mm_data_items,
1463
1464
                mm_hashes=mm_hashes,
            )
1465

1466
        # NOTE: `prompt` does not correspond to `mm_missing_data_items`,
1467
        # so we can't apply prompt updates until the new multimodal
1468
        # items are combined with the cached multimodal items
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
        with timing_ctx.record("apply_hf_processor"):
            (
                prompt_ids,
                mm_missing_processed_data,
                is_update_applied,
            ) = self._apply_hf_processor_main(
                prompt=inputs.prompt,
                mm_items=mm_missing_data_items,
                hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
                tokenization_kwargs=inputs.tokenization_kwargs,
                enable_hf_prompt_update=False,
            )
1481

1482
        mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
1483
            mm_missing_processed_data,
1484
            self._get_mm_fields_config(
1485
                mm_missing_processed_data, inputs.hf_processor_mm_kwargs
1486
            ),
1487
1488
        )

1489
1490
        mm_missing_prompt_updates = self._get_mm_prompt_updates(
            mm_missing_data_items,
1491
            inputs.hf_processor_mm_kwargs,
1492
            mm_missing_kwargs,
1493
        )
1494

1495
        with timing_ctx.record("merge_mm_kwargs"):
1496
1497
1498
1499
1500
1501
1502
            mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
                cache,
                mm_hashes=mm_hashes,
                mm_is_cached=mm_is_cached,
                mm_missing_kwargs=mm_missing_kwargs,
                mm_missing_prompt_updates=mm_missing_prompt_updates,
            )
1503
1504
1505

        mm_info = MultiModalProcessingInfo(
            kwargs=mm_kwargs,
1506
            hashes=mm_hashes,
1507
1508
            prompt_updates=mm_prompt_updates,
        )
1509

1510
        return prompt_ids, mm_info, is_update_applied
1511

1512
1513
1514
    def _apply_token_matches(
        self,
        prompt: list[int],
1515
1516
1517
1518
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
        tokenizer = self.info.get_tokenizer()
        return apply_token_matches(prompt, mm_prompt_updates, tokenizer)
1519
1520
1521
1522

    def _apply_text_matches(
        self,
        prompt: str,
1523
1524
1525
1526
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[str, MultiModalPromptUpdatesApplyResult]:
        tokenizer = self.info.get_tokenizer()
        return apply_text_matches(prompt, mm_prompt_updates, tokenizer)
1527

1528
    def _apply_prompt_updates(
1529
1530
        self,
        token_ids: list[int],
1531
        mm_prompt_updates: MultiModalPromptUpdates,
1532
    ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
1533
        """Apply multi-modal prompt updates to token IDs."""
1534
        tokenizer = self.info.get_tokenizer()
1535

1536
1537
1538
1539
        new_token_ids, match_result = self._apply_token_matches(
            token_ids,
            mm_prompt_updates,
        )
1540
1541
1542
1543
1544
1545
1546
1547
1548

        # If the search text does not represent a special token,
        # it may have different token IDs in the prompt, because
        # the tokens may go across the boundaries of the search text.
        # ----
        # e.g. when searching for "foo" in "food", if "food" itself makes
        # up a token, then the token ID of "foo" will not appear at all
        # ----
        # Since it is inefficient to search for all possible tokenizations
1549
1550
        # of the search text in the prompt, we instead perform string-based
        # updates on the decoded token IDs, then encode them back.
1551
        if not all(
1552
1553
1554
            all(update_idx is not None for update_idx in update_idxs)
            for update_idxs in match_result.values()
        ):
1555
            new_text, match_result = self._apply_text_matches(
1556
                _seq2text(tokenizer, token_ids, use_cache=False),
1557
                mm_prompt_updates,
1558
1559
            )

1560
            new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False)
1561

1562
        matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
1563
1564
1565
1566
        for modality, update_idxs in match_result.items():
            for item_idx, update_idx in enumerate(update_idxs):
                assert update_idx is not None, (
                    "Failed to apply prompt replacement for "
1567
1568
                    f"mm_items[{modality!r}][{item_idx}]"
                )
1569
1570

                matched_updates[modality].append(
1571
1572
                    [mm_prompt_updates[modality][item_idx][update_idx]]
                )
1573
1574

        placeholders = self._find_mm_placeholders(
1575
1576
            new_token_ids,
            dict(matched_updates),
1577
        )
1578

1579
        return new_token_ids, placeholders
1580

1581
1582
    def _validate_mm_kwargs(
        self,
1583
        mm_kwargs: MultiModalKwargsOptionalItems,
1584
1585
1586
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
1587
            items = mm_kwargs.get(modality, [])
1588
1589
1590
1591
1592
1593
1594
1595
1596

            if len(items) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} {modality} items in "
                    f"keyword arguments corresponding to {item_count} "
                    f"{modality} data items, but only found {len(items)}! "
                    "There is likely a problem with your "
                    "implementation of merged multi-modal processor for this "
                    "model (usually arising from an inconsistency between "
1597
1598
                    "`_call_hf_processor` and `_get_mm_fields_config`)."
                )
1599

1600
    def _validate_mm_updates(
1601
        self,
1602
        mm_updates: MultiModalPromptUpdates,
1603
        mm_item_counts: Mapping[str, int],
1604
    ) -> None:
1605
        for modality, item_count in mm_item_counts.items():
1606
            placeholders = mm_updates.get(modality, [])
1607

1608
            if len(placeholders) != item_count:
1609
                raise RuntimeError(
1610
                    f"Expected there to be {item_count} prompt updates "
1611
                    f"corresponding to {item_count} {modality} items, but "
1612
                    f"instead found {len(placeholders)} prompt updates! "
1613
1614
1615
                    "This is likely because you forgot to include input "
                    "placeholder tokens (e.g., `<image>`, `<|image_pad|>`) "
                    "in the prompt. If the model has a chat template, make "
1616
1617
                    "sure you have applied it before calling `LLM.generate`."
                )
1618

1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
    def _validate_mm_placeholders(
        self,
        mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
            placeholders = mm_placeholders.get(modality, [])

            if len(placeholders) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} prompt placeholders "
                    f"corresponding to {item_count} {modality} items, but "
                    f"instead found {len(placeholders)} prompt placeholders! "
                    "Make sure the implementation of `_call_hf_processor` and "
1633
1634
                    "`_get_mm_fields_config` are consistent with each other."
                )
1635

1636
1637
1638
1639
    def _maybe_apply_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        prompt_ids: list[int],
1640
        mm_kwargs: MultiModalKwargsOptionalItems,
1641
        mm_prompt_updates: MultiModalPromptUpdates,
1642
        is_update_applied: bool,
1643
    ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
1644
        mm_item_counts = mm_items.get_all_counts()
1645
        self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
1646
        self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
1647

1648
        if is_update_applied:
1649
1650
            mm_placeholders = self._find_mm_placeholders(
                prompt_ids,
1651
                mm_prompt_updates,
1652
            )
1653
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1654
        else:
1655
            prompt_ids, mm_placeholders = self._apply_prompt_updates(
1656
                prompt_ids,
1657
                mm_prompt_updates,
1658
            )
1659
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1660

1661
        return prompt_ids, mm_placeholders
1662
1663
1664

    def apply(
        self,
1665
1666
        inputs: ProcessorInputs,
        timing_ctx: TimingContext,
1667
    ) -> MultiModalInput:
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
        """
        Process multi-modal inputs to be used in vLLM.

        The main steps are:

        1. Apply HF Processor on prompt text and multi-modal data together,
           outputting token IDs and processed tensors.
        2. Find and update sequences in the token IDs with placeholder tokens.
           The number of placeholder tokens equals the feature size of the
           multi-modal data outputted by the multi-modal encoder.
        3. Extract information about the placeholder tokens from the
           processed token IDs.
        """
        (
            prompt_ids,
1683
            mm_info,
1684
            is_update_applied,
1685
        ) = self._cached_apply_hf_processor(inputs, timing_ctx)
1686

1687
        # NOTE: tokenization_kwargs are not required to init processor
1688
        with timing_ctx.record("apply_prompt_updates"):
1689
            prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
1690
                mm_items=inputs.mm_data_items,
1691
1692
1693
1694
1695
                prompt_ids=prompt_ids,
                mm_kwargs=mm_info.kwargs,
                mm_prompt_updates=mm_info.prompt_updates,
                is_update_applied=is_update_applied,
            )
1696

1697
1698
1699
1700
        mm_placeholder_ranges = {
            modality: [item.to_range() for item in placeholders]
            for modality, placeholders in mm_placeholders.items()
        }
1701

1702
        return mm_input(
1703
            prompt_token_ids=prompt_ids,
1704
1705
            mm_kwargs=mm_info.kwargs,
            mm_hashes=mm_info.hashes,
1706
            mm_placeholders=mm_placeholder_ranges,
1707
        )
1708
1709
1710


class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
Ekagra Ranjan's avatar
Ekagra Ranjan committed
1711
1712
    skip_decoder_start_token: bool = False

1713
1714
1715
    @abstractmethod
    def create_encoder_prompt(
        self,
1716
        prompt: str | list[int],
1717
        mm_items: MultiModalDataItems,
1718
    ) -> str | list[int]:
1719
        """
1720
        Create input prompt for the encoder. HF processor will be applied on
1721
1722
        this prompt during profiling and generation.
        """
1723
1724
        raise NotImplementedError

1725
1726
    def create_decoder_prompt(
        self,
1727
        prompt: str | list[int],
1728
        mm_items: MultiModalDataItems,
1729
    ) -> str | list[int]:
1730
1731
1732
        """Create input prompt for the decoder."""
        return prompt

1733
    def _get_enc_dec_inputs(
1734
        self,
1735
        prompt: str | list[int],
1736
        mm_items: MultiModalDataItems,
1737
        encoder_inputs: MultiModalInput,
1738
    ):
1739
        tokenizer = self.info.get_tokenizer()
1740
        decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_items)
1741
        if isinstance(decoder_prompt_raw, str):
1742
            decoder_prompt_text = decoder_prompt_raw
1743
1744
            decoder_prompt_ids = tokenizer.encode(
                decoder_prompt_raw, add_special_tokens=False
1745
            )
1746
        else:
1747
            decoder_prompt_text = None
1748
            decoder_prompt_ids = decoder_prompt_raw
1749

1750
        return mm_enc_dec_input(
1751
1752
            encoder_inputs,
            decoder_prompt_ids,
1753
            decoder_prompt=decoder_prompt_text,
1754
        )
1755
1756
1757

    def apply(
        self,
1758
1759
        inputs: ProcessorInputs,
        timing_ctx: TimingContext,
1760
    ) -> MultiModalEncDecInput:
1761
1762
1763
1764
1765
1766
1767
        """
        Process multi-modal inputs to be used in vLLM.
        The main processing steps are modified to fit encoder-decoder model:
        1. Create encoder prompt from input prompt text.
        2. Apply the HF processor on encoder prompt.
        3. Copy the input prompt text as decoder prompt inputs.
        """
1768
1769
1770
1771
1772
        encoder_prompt = self.create_encoder_prompt(
            inputs.prompt,
            inputs.mm_data_items,
        )
        encoder_processor_inputs = ProcessorInputs(
1773
            encoder_prompt,
1774
1775
1776
1777
            inputs.mm_data_items,
            inputs.mm_uuid_items,
            hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
            tokenization_kwargs=inputs.tokenization_kwargs,
1778
1779
        )

1780
1781
        encoder_inputs = super().apply(encoder_processor_inputs, timing_ctx)

1782
        return self._get_enc_dec_inputs(
1783
1784
            prompt=inputs.prompt,
            mm_items=inputs.mm_data_items,
1785
1786
            encoder_inputs=encoder_inputs,
        )