processing.py 51.4 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
import re
3
import sys
4
from abc import ABC, abstractmethod
5
from collections import defaultdict
6
7
from collections.abc import (Callable, Generator, ItemsView, Iterable, Mapping,
                             Sequence)
8
from dataclasses import dataclass, field
9
from enum import Enum
10
from functools import lru_cache
11
from typing import (TYPE_CHECKING, Generic, NamedTuple, Optional, Protocol,
12
                    TypeVar, Union, cast)
13

14
15
import torch
from cachetools import LRUCache
16
from transformers import BatchFeature, PretrainedConfig, ProcessorMixin
17
from typing_extensions import assert_never
18

19
from vllm.inputs import InputProcessingContext
20
from vllm.jsontree import json_map_leaves, json_reduce_leaves
21
from vllm.logger import init_logger
22
23
from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
                                               encode_tokens)
24
from vllm.utils import GiB_bytes, flatten_2d_lists, full_groupby
25

26
from .hasher import MultiModalHasher
27
28
29
from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
                     MultiModalFieldConfig, MultiModalInputs, MultiModalKwargs,
                     MultiModalKwargsItem, PlaceholderRange)
30
31
from .parse import (DictEmbeddingItems, EmbeddingItems, MultiModalDataItems,
                    MultiModalDataParser)
32
33
34

if TYPE_CHECKING:
    from .profiling import BaseDummyInputsBuilder
35

36
logger = init_logger(__name__)
37
38

_S = TypeVar("_S", str, list[int])
39
40
41

PromptSeq = Union[str, list[int]]
"""A token sequence (list of token IDs) or text."""
42

43

44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
@dataclass
class PromptIndex:
    """Resolves to an index in the prompt."""
    get_match_index: Callable[[AnyTokenizer, PromptSeq], Optional[int]]


class PromptIndexTargets:

    @staticmethod
    def start() -> PromptIndex:
        """
        Resolves to the start of the prompt (before the first token).

        This results in a match even if the prompt is empty.
        """
        return PromptIndex(lambda tok, prompt: 0)

    @staticmethod
    def prefix(seq: PromptSeq) -> PromptIndex:
        """
        Resolves to a location in the prompt after the given prefix.
        """

        def get_match_index(
            tokenizer: AnyTokenizer,
            prompt: PromptSeq,
        ) -> Optional[int]:
            prefix = seq

            if isinstance(prompt, str):
                if not isinstance(prefix, str):
                    # Make both `str`
                    prefix = decode_tokens(tokenizer, prefix)
            else:
                if isinstance(prefix, str):
                    # Make both `list[int]`
                    prefix = encode_tokens(tokenizer, prefix)

            match_idx = len(prefix)
            return match_idx if prompt[:match_idx] == prefix else None

        return PromptIndex(get_match_index)

    @staticmethod
    def end() -> PromptIndex:
        """
        Resolves to the end of the prompt (after the last token).

        This results in a match even if the prompt is empty.
        """
        return PromptIndex(lambda tok, prompt: len(prompt))


PromptTarget = Union[PromptSeq, PromptIndex]
"""
The token sequence or text to update.
"""


103
@dataclass
104
105
class PromptUpdateDetails:
    """Details about the token sequence or text that are part of the update."""
106
107

    full: PromptSeq
108
    """The full content."""
109

110
    features: PromptSeq
111
    """
112
    The part of the content that corresponds to feature placeholders;
113
114
    this will be replaced by the output of the vision encoder during model
    inference.
115
116
117
    """

    @staticmethod
118
119
    def from_seq(seq: PromptSeq) -> "PromptUpdateDetails":
        return PromptUpdateDetails(full=seq, features=seq)
120
121


122
PromptUpdateInfo = Union[PromptSeq, PromptUpdateDetails]
123
"""
124
The token sequence or text that are part of the update.
125

126
127
If only part of the content corresponds to feature placeholders, you can
use :class:`PromptUpdateDetails` to specify which part.
128
"""
129

130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
PromptUpdateContent = Union[Callable[[int], PromptUpdateInfo],
                            PromptUpdateInfo]
"""
Given the index of the processed item within :attr:`modality`,
output the corresponding token sequence (or text).

For convenience, you can directly pass in the token sequence (or text)
instead of a function if it does not depend on the input.
"""


class UpdateMode(str, Enum):
    INSERT = "insert"
    REPLACE = "replace"


@dataclass
147
class PromptUpdate(ABC):
148
149
150
151
152
153
154
    """
    Defines how to update a prompt with placeholder tokens.
    """

    modality: str
    """The modality for which the update is made."""

155
    target: PromptTarget
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
    """The token sequence (or text) to update."""

    @property
    @abstractmethod
    def content(self) -> PromptUpdateContent:
        """The placeholder tokens that are part of the update."""
        raise NotImplementedError

    @property
    @abstractmethod
    def mode(self) -> UpdateMode:
        """Defines how to update the prompt."""
        raise NotImplementedError

    def bind(self, tokenizer: AnyTokenizer) -> "BoundPromptUpdate":
        return BoundPromptUpdate(
            _origin=self,
            tokenizer=tokenizer,
        )

176

177
@dataclass
178
179
180
181
182
183
184
class PromptInsertion(PromptUpdate):
    """
    Defines how to insert placeholder tokens into a prompt.

    Example:

        For each image, insert a number of ``<image>`` feature placeholders
185
        equal to the feature size of the vision encoder after the ``<s>`` token:
186
187
188
189
190

        .. code-block:: python

            PromptInsertion(
                modality="image",
191
                target="<s>",
192
193
194
                insertion="<image>" * image_feature_size,
            )

195
        Insert these tokens at the start of the prompt:
196
197
198
199
200

        .. code-block:: python

            PromptInsertion(
                modality="image",
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
                target=PromptIndexTargets.start(),
                insertion="<image>" * image_feature_size,
            )

        Insert these tokens after a prefix ``Images:``:

        .. code-block:: python

            PromptInsertion(
                modality="image",
                target=PromptIndexTargets.prefix("Images:"),
                insertion="<image>" * image_feature_size,
            )

        Insert these tokens at the end of the prompt:

        .. code-block:: python

            PromptInsertion(
                modality="image",
                target=PromptIndexTargets.end(),
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
                insertion="<image>" * image_feature_size,
            )
    """

    insertion: PromptUpdateContent = field(repr=False)
    """
    Given the index of the processed item within :attr:`modality`,
    output the token sequence (or text) to insert right after :attr:`target`.

    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
    """

    @property
    def content(self) -> PromptUpdateContent:
        return self.insertion

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.INSERT


@dataclass
class PromptReplacement(PromptUpdate):
246
247
    """
    Defines how to replace portions of an input prompt with placeholder tokens.
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271

    Example:

        For each image, replace one ``<image>`` input placeholder in the prompt
        with a number of ``<image>`` feature placeholders
        equal to the feature size of the vision encoder:

        .. code-block:: python

            PromptReplacement(
                modality="image",
                target="<image>",
                replacement="<image>" * image_feature_size,
            )

        As above, but further pad the feature placeholders with ``<image_bos>``
        and `<image_eos>``, which are not supposed to be passed to the vision
        encoder:

        .. code-block:: python

            PromptReplacement(
                modality="image",
                target="<image>",
272
                replacement=PromptUpdateDetails(
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
                    full="".join([
                        "<image_bos>",
                        "<image>" * image_feature_size,
                        "<image_eos>",
                    ]),
                    features="<image>" * image_feature_size,
                ),
            )

        To avoid unnecessary tokenization during prompt replacement,
        we recommended passing token sequences instead of text:

        .. code-block:: python

            PromptReplacement(
                modality="image",
                target=[image_token_id],
290
                replacement=PromptUpdateDetails(
291
292
293
294
295
                    full=([image_bos_id] + [image_token_id] * image_feature_size
                          + [image_eos_id]),
                    features=[image_token_id] * image_feature_size,
                ),
            )
296
297
    """

298
    replacement: PromptUpdateContent = field(repr=False)
299
    """
300
    Given the index of the processed item within :attr:`modality`,
301
    output the token sequence (or text) to replace :attr:`target`.
302

303
304
    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
305
306
    """

307
308
309
310
311
312
313
    @property
    def content(self) -> PromptUpdateContent:
        return self.replacement

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.REPLACE
314
315


316
317
318
319
320
321
322
@lru_cache(maxsize=2048)
def _cached_encode(
    tokenizer: AnyTokenizer,
    text: str,
    *,
    add_special_tokens: bool = False,
) -> list[int]:
323
324
325
    return encode_tokens(tokenizer,
                         text,
                         add_special_tokens=add_special_tokens)
326
327


328
329
330
331
332
333
334
@lru_cache(maxsize=2048)
def _cached_decode(
    tokenizer: AnyTokenizer,
    token_ids: tuple[int, ...],
    *,
    skip_special_tokens: bool = False,
) -> str:
335
336
337
    return decode_tokens(tokenizer,
                         list(token_ids),
                         skip_special_tokens=skip_special_tokens)
338
339
340
341
342


class _HasModalityAttr(Protocol):
    modality: str

343

344
class _HasModalityProp(Protocol):
345

346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
    @property
    def modality(self) -> str:
        ...


_M = TypeVar("_M", bound=Union[_HasModalityAttr, _HasModalityProp])


def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]:
    """Convenience function to apply :func:`full_groupby` based on modality."""
    return full_groupby(values, key=lambda x: x.modality)


@dataclass
class _BoundPromptSequence:
361
362
363
364
    """
    A :data:`_PromptSeq` bound to a tokenizer to automatically
    convert between token sequence and text representations.
    """
365
366
    tokenizer: AnyTokenizer = field(repr=False)

367
368
369
    _text: Optional[str]
    _token_ids: Optional[list[int]]

370
    @staticmethod
371
372
    def from_seq(
        tokenizer: AnyTokenizer,
373
        seq: PromptSeq,
374
    ) -> "_BoundPromptSequence":
375
376
377
378
379
380
        return _BoundPromptSequence(
            tokenizer=tokenizer,
            _text=seq if isinstance(seq, str) else None,
            _token_ids=seq if isinstance(seq, list) else None,
        )

381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
    def __post_init__(self) -> None:
        if self._text is None and self._token_ids is None:
            raise ValueError("At least one of 'text' and 'token_ids' must be "
                             "specified")

    @property
    def text(self) -> str:
        if self._text is None:
            assert self._token_ids is not None
            self._text = _cached_decode(self.tokenizer, tuple(self._token_ids))

        return self._text

    @property
    def token_ids(self) -> list[int]:
        if self._token_ids is None:
            assert self._text is not None
            self._token_ids = _cached_encode(self.tokenizer, self._text)

        return self._token_ids


403
@dataclass
404
class _BoundPromptContent:
405
406
407
408
    full: _BoundPromptSequence
    features: _BoundPromptSequence


409
@dataclass
410
class BoundPromptUpdate:
411
    """
412
413
    A :class:`PromptUpdate` bound to a tokenizer to automatically convert
    :attr:`target` and the result of :meth:`get_content` between
414
415
    token sequence and text representations.
    """
416
    _origin: PromptUpdate
417
    tokenizer: AnyTokenizer = field(repr=False)
418

419
    def __post_init__(self) -> None:
420
421
422
423
424
        self._content_cache = dict[int, _BoundPromptContent]()

    @property
    def modality(self) -> str:
        return self._origin.modality
425
426

    @property
427
    def target(self) -> Union[_BoundPromptSequence, PromptIndex]:
428
        """The token sequence (or text) to update."""
429
430
431
432
433
434
        target = self._origin.target

        if isinstance(target, PromptIndex):
            return target

        return _BoundPromptSequence.from_seq(self.tokenizer, target)
435

436
437
438
439
440
441
442
443
444
445
446
    @property
    def content(self) -> PromptUpdateContent:
        """The placeholder tokens that are part of the update."""
        return self._origin.content

    @property
    def mode(self) -> UpdateMode:
        """Defines how to update the prompt."""
        return self._origin.mode

    def get_content(self, item_idx: int) -> _BoundPromptContent:
447
448
        """
        Given the index of the processed item within :attr:`modality`,
449
        output the token sequence (or text) to update.
450
        """
451
452
        content = self.content
        if callable(content):
453
            cache_key = item_idx
454
455
            if cache_key in self._content_cache:
                return self._content_cache[cache_key]
456

457
            content = content(item_idx)
458
459
460
        else:
            cache_key = None

461
462
        if not isinstance(content, PromptUpdateDetails):
            content = PromptUpdateDetails.from_seq(content)
463
464

        bound_full = _BoundPromptSequence.from_seq(self.tokenizer,
465
                                                   content.full)
466
        bound_features = _BoundPromptSequence.from_seq(self.tokenizer,
467
468
469
                                                       content.features)
        bound_content = _BoundPromptContent(full=bound_full,
                                            features=bound_features)
470
471

        if cache_key is not None:
472
            self._content_cache[cache_key] = bound_content
473

474
        return bound_content
475
476


477
478
479
class _TokenMatch(NamedTuple):
    start_idx: int
    end_idx: int
480
481


482
483
484
def iter_token_matches(
    token_ids: list[int],
    match_ids: list[int],
485
) -> Generator[_TokenMatch]:
486
487
488
489
490
491
    """
    Yield each occurrence of :code:`match_ids` in :code:`token_ids`.

    Note that empty matches are ignored.
    """
    prompt_len = len(token_ids)
492
    match_len = len(match_ids)
493

494
495
    if match_len == 0:
        return
496

497
498
    start_idx = 0
    while start_idx < prompt_len - match_len + 1:
499
        end_idx = start_idx + match_len
500

501
502
        if token_ids[start_idx:end_idx] == match_ids:
            yield _TokenMatch(start_idx=start_idx, end_idx=end_idx)
503
504
505
506
507

            # Exclude overlapping matches
            start_idx = end_idx
        else:
            start_idx += 1
508
509


510
@dataclass(repr=False)
511
512
class _PromptTargetMatch(ABC):
    _origin: BoundPromptUpdate
513
514
515

    @property
    def modality(self) -> str:
516
        return self._origin.modality
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532

    @property
    @abstractmethod
    def start_idx(self) -> int:
        raise NotImplementedError

    @property
    @abstractmethod
    def end_idx(self) -> int:
        raise NotImplementedError

    def __repr__(self) -> str:
        return (f"{type(self).__name__}(modality={self.modality!r}, "
                f"start_idx={self.start_idx!r}, end_idx={self.end_idx!r})")


533
534
535
536
537
538
539
540
541
542
543
544
545
@dataclass(repr=False)
class _PromptTargetIndexMatch(_PromptTargetMatch):
    match_idx: int

    @property
    def start_idx(self) -> int:
        return self.match_idx

    @property
    def end_idx(self) -> int:
        return self.match_idx


546
@dataclass(repr=False)
547
class _PromptTargetTokenMatch(_PromptTargetMatch):
548
549
550
551
552
553
554
555
556
557
558
559
    match: _TokenMatch

    @property
    def start_idx(self) -> int:
        return self.match.start_idx

    @property
    def end_idx(self) -> int:
        return self.match.end_idx


@dataclass(repr=False)
560
class _PromptTargetTextMatch(_PromptTargetMatch):
561
562
563
564
565
566
567
568
569
570
    match: re.Match[str]

    @property
    def start_idx(self) -> int:
        return self.match.start()

    @property
    def end_idx(self) -> int:
        return self.match.end()

571

572
@dataclass
573
class PlaceholderFeaturesInfo:
574
    modality: str
575
    item_idx: int
576
    start_idx: int
577
    tokens: list[int]
578
579
580

    @property
    def length(self) -> int:
581
        return len(self.tokens)
582
583
584
585
586
587

    def to_range(self) -> PlaceholderRange:
        return PlaceholderRange(
            offset=self.start_idx,
            length=self.length,
        )
588
589
590
591


def find_token_matches(
    prompt: list[int],
592
593
594
    prompt_updates: Sequence[BoundPromptUpdate],
) -> Sequence[_PromptTargetMatch]:
    """Return each target of :code:`prompt_updates` found in :code:`prompt`."""
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610

    def get_matches(update: BoundPromptUpdate):
        target = update.target

        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(update.tokenizer, prompt)
            if match_idx is None:
                return []

            return [_PromptTargetIndexMatch(update, match_idx)]

        return [
            _PromptTargetTokenMatch(update, match)
            for match in iter_token_matches(prompt, target.token_ids)
        ]

611
    return [
612
        match for update in prompt_updates for match in get_matches(update)
613
614
615
616
617
    ]


def find_text_matches(
    prompt: str,
618
619
620
    prompt_updates: Sequence[BoundPromptUpdate],
) -> Sequence[_PromptTargetMatch]:
    """Return each target of :code:`prompt_updates` found in :code:`prompt`."""
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636

    def get_matches(update: BoundPromptUpdate):
        target = update.target

        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(update.tokenizer, prompt)
            if match_idx is None:
                return []

            return [_PromptTargetIndexMatch(update, match_idx)]

        return [
            _PromptTargetTextMatch(update, match)
            for match in re.finditer(re.escape(target.text), prompt)
        ]

637
    return [
638
        match for update in prompt_updates for match in get_matches(update)
639
640
641
642
    ]


def _resolve_matches(
643
    prompt: PromptSeq,
644
645
    mm_matches: Mapping[str, Sequence[_PromptTargetMatch]],
) -> list[_PromptTargetMatch]:
646
    """
647
    Resolve :code:`mm_matches` to ensure that there are no overlapping matches,
648
    and sort them such that earlier matches take priority over later ones.
649
    """
650
651
    matches = [m for matches in mm_matches.values() for m in matches]

652
    seen_matches: list[Optional[_PromptTargetMatch]] = [None] * len(prompt)
653

654
    for match in matches:
655
656
657
658
659
        for idx in range(match.start_idx, match.end_idx):
            if seen_matches[idx] is not None:
                raise ValueError("Found overlapping matches "
                                 f"({seen_matches[idx]} and {match}) "
                                 f"at index={idx} of prompt={prompt}")
660

661
            seen_matches[idx] = match
662
663
664
665

    return sorted(matches, key=lambda x: x.start_idx)


666
def _apply_matches(
667
    prompt: _S,
668
    mm_matches: Mapping[str, Sequence[_PromptTargetMatch]],
669
    mm_item_counts: Mapping[str, int],
670
) -> list[_S]:
671
672
    """Apply the updates in :code:`mm_matches` to :code:`prompt`."""
    out_seqs = list[Union[str, list[int]]]()
673
    prev_end_idx = 0
674
    next_idx_by_modality = defaultdict[str, int](lambda: 0)
675

676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
    for match in _resolve_matches(prompt, mm_matches):
        modality = match.modality

        item_start_idx = next_idx_by_modality[modality]
        max_item_count = mm_item_counts.get(modality, 0)
        if item_start_idx >= max_item_count:
            continue

        start_idx = match.start_idx
        end_idx = match.end_idx
        origin = match._origin
        mode = origin.mode

        if mode == UpdateMode.INSERT:
            out_seqs.append(prompt[prev_end_idx:end_idx])
            num_inserts = max_item_count
        elif mode == UpdateMode.REPLACE:
            out_seqs.append(prompt[prev_end_idx:start_idx])
            num_inserts = max_item_count if start_idx == end_idx else 1
        else:
            assert_never(mode)
697

698
        item_end_idx = min(item_start_idx + num_inserts, max_item_count)
699

700
        for item_idx in range(item_start_idx, item_end_idx):
701
            content = origin.get_content(item_idx)
702
703
            insert_seq = (content.full.text if isinstance(prompt, str) else
                          content.full.token_ids)
704

705
            out_seqs.append(insert_seq)
706

707
708
        prev_end_idx = end_idx
        next_idx_by_modality[modality] += item_end_idx - item_start_idx
709
710
711

    out_seqs.append(prompt[prev_end_idx:])

712
    return cast(list[_S], out_seqs)
713
714


715
def apply_token_matches(
716
    prompt: list[int],
717
    mm_matches: Mapping[str, Sequence[_PromptTargetMatch]],
718
    mm_item_counts: Mapping[str, int],
719
) -> list[int]:
720
    """Apply the updates in :code:`mm_matches` to :code:`prompt`."""
721
    if not mm_matches:
722
723
        return prompt

724
    token_id_seqs = _apply_matches(prompt, mm_matches, mm_item_counts)
725
726

    return flatten_2d_lists(token_id_seqs)
727
728


729
def apply_text_matches(
730
    prompt: str,
731
    mm_matches: Mapping[str, Sequence[_PromptTargetMatch]],
732
    mm_item_counts: Mapping[str, int],
733
) -> str:
734
    """Apply the updates in :code:`mm_matches` to :code:`prompt`."""
735
    if not mm_matches:
736
        return prompt
737

738
    texts = _apply_matches(prompt, mm_matches, mm_item_counts)
739
740

    return "".join(texts)
741
742


743
def _iter_placeholders(
744
    mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
745
    prompt: list[int],
746
    mm_item_counts: Mapping[str, int],
747
) -> Iterable[PlaceholderFeaturesInfo]:
748
749
750
751
752
    """
    Yield each set of placeholder tokens found in :code:`prompt`.

    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
753
    appears earlier in `mm_prompt_updates` takes priority.
754

755
756
    Note that empty matches are ignored.
    """
757
    prompt_len = len(prompt)
758
    item_idx_by_modality = defaultdict[str, int](lambda: 0)
759
760
761
762
763

    start_idx = 0
    while start_idx < prompt_len:
        found = False

764
        for modality, modality_updates in mm_prompt_updates.items():
765
766
            item_idx = item_idx_by_modality[modality]
            if item_idx >= mm_item_counts.get(modality, 0):
767
                continue
768

769
770
771
772
773
            for update_info in modality_updates:
                content = update_info.get_content(item_idx)
                content_tokens_full = content.full.token_ids
                content_len_full = len(content_tokens_full)
                end_idx_full = start_idx + content_len_full
774

775
                if content_len_full == 0 or end_idx_full > prompt_len:
776
777
                    continue

778
779
                if prompt[start_idx:end_idx_full] == content_tokens_full:
                    content_tokens_feat = content.features.token_ids
780
781
782

                    try:
                        match = next(
783
784
                            iter_token_matches(content_tokens_full,
                                               content_tokens_feat))
785
786
787
788
                        yield PlaceholderFeaturesInfo(
                            modality=modality,
                            item_idx=item_idx,
                            start_idx=start_idx + match.start_idx,
789
                            tokens=content_tokens_feat,
790
791
792
                        )
                    except StopIteration:
                        raise AssertionError(
793
794
                            f"{content_tokens_feat=} should be a "
                            f"subsequence of {content_tokens_full=}") from None
795

796
                    # Exclude overlapping matches
797
                    start_idx = end_idx_full
798
799
800
                    item_idx_by_modality[modality] += 1
                    found = True
                    break
801

802
803
            if found:
                break  # Go back to the outer while loop
804
805
806

        if not found:
            start_idx += 1
807
808


809
def find_mm_placeholders(
810
    mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
811
812
    prompt: list[int],
    mm_item_counts: Mapping[str, int],
813
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
814
    it = _iter_placeholders(mm_prompt_updates, prompt, mm_item_counts)
815
816
817
    return dict(full_groupby_modality(it))


818
819
820
_V = TypeVar("_V", bound="Union[MultiModalKwargs, MultiModalKwargsItem]")


821
822
class ProcessingCache:

823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
    @staticmethod
    def get_lru_cache(
        capacity_gb: int,
        value_type: type[_V],
    ) -> LRUCache[str, _V]:

        def get_size(leaf: object) -> int:
            if isinstance(leaf, torch.Tensor):
                return leaf.nbytes  # sys.getsizeof doesn't work for tensors

            return sys.getsizeof(leaf)

        return LRUCache[str, _V](
            GiB_bytes * capacity_gb,
            getsizeof=lambda x: json_reduce_leaves(
                lambda a, b: a + b,
                json_map_leaves(get_size, x),
            ),
        )

    def __init__(self, capacity_gb: int) -> None:
844
845
846
847
        super().__init__()

        # DEBUG: Set to None to disable
        self.debug_cache_hit_ratio_steps: Optional[int] = None
848
849
        self.debug_cache_hits = 0
        self.debug_cache_total = 0
850

851
        self._cache = self.get_lru_cache(capacity_gb, MultiModalKwargsItem)
852
853
854
855
856
857

    def _maybe_log_cache_stats(self) -> None:
        steps = self.debug_cache_hit_ratio_steps
        if not steps:
            return

858
859
        total = self.debug_cache_total
        if total > 0 and total % steps == 0:
860
            logger.debug("ProcessingCache: hit_ratio = %.2f",
861
                         self.debug_cache_hits / total)
862
863
864
865
866
867
868

    def get(
        self,
        model_id: str,
        modality: str,
        input_item: object,
        input_kwargs: Mapping[str, object],
869
    ) -> Optional[MultiModalKwargsItem]:
870
871
872
873
874
875
876
877
878
879
880
        """
        Get a processed multi-modal item from the cache
        according to its dependencies, including:

        - The model ID
        - The modality of the item
        - The original data item passed to the HF processor
        - The configuration options of the HF processor
        """
        self._maybe_log_cache_stats()

881
882
883
        cache_key = MultiModalHasher.hash_kwargs(model_id=model_id,
                                                 **{modality: input_item},
                                                 **input_kwargs)
884
885
886
887
888
889
890

        if self.debug_cache_hit_ratio_steps:
            if cache_key in self._cache:
                self.debug_cache_hits += 1

            self.debug_cache_total += 1

891
892
893
894
895
896
897
898
        return self._cache.get(cache_key)

    def put(
        self,
        model_id: str,
        modality: str,
        input_item: object,
        input_kwargs: Mapping[str, object],
899
        output_kwargs: MultiModalKwargsItem,
900
901
902
903
904
    ) -> None:
        """
        Put a processed multi-modal item into the cache
        according to its dependencies (see :meth:`get`).
        """
905
906
907
        cache_key = MultiModalHasher.hash_kwargs(model_id=model_id,
                                                 **{modality: input_item},
                                                 **input_kwargs)
908
        self._cache[cache_key] = output_kwargs
909
910


911
class BaseProcessingInfo:
912
    """Base class to provide the information necessary for data processing."""
913

914
915
    def __init__(self, ctx: InputProcessingContext) -> None:
        super().__init__()
916

917
918
919
920
921
922
923
        self.ctx = ctx

    @property
    def model_id(self) -> str:
        return self.ctx.model_config.model

    def get_tokenizer(self) -> AnyTokenizer:
924
925
        return self.ctx.tokenizer

926
    def get_hf_config(self) -> PretrainedConfig:
927
928
        return self.ctx.get_hf_config()

929
    def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
930
931
932
933
934
935
        """
        Subclasses can override this method to handle
        specific kwargs from model config or user inputs.
        """
        return self.ctx.get_hf_processor(**kwargs)

936
937
938
939
940
941
942
943
944
945
946
947
948
    @abstractmethod
    def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
        """
        Return the maximum supported number of items for each modality.

        A value of `None` means unlimited number of items.

        Omitting a modality from the returned dictionary means that
        it is not supported at all.
        """
        raise NotImplementedError

    @abstractmethod
949
950
951
952
953
    def get_mm_max_tokens_per_item(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
    ) -> Mapping[str, int]:
954
955
956
957
958
959
960
961
962
963
964
        """
        Get the maximum possible number of tokens per data item
        for each modality.

        The dictionary returned by this method should have the same
        keys as that returned by :meth:`get_supported_mm_limits`.
        """
        raise NotImplementedError


_I = TypeVar("_I", bound=BaseProcessingInfo)
965

966
967

class BaseMultiModalProcessor(ABC, Generic[_I]):
968
    """
969
    Abstract base class to process multi-modal inputs to be used in vLLM.
970
971

    Not to be confused with :class:`transformers.ProcessorMixin`.
972
973
    """

974
    def __init__(self,
975
976
                 info: _I,
                 dummy_inputs: "BaseDummyInputsBuilder[_I]",
977
978
979
                 *,
                 cache: Optional[ProcessingCache] = None,
                 enable_sanity_checks: bool = True) -> None:
980
981
982
983
984
985
        if get_repls := getattr(self, "_get_prompt_replacements", None):
            logger.warning_once("`_get_prompt_replacements` has been renamed "
                                "to `_get_prompt_updates`. The old name will "
                                "be removed in an upcoming release.")
            self._get_prompt_updates = get_repls  # type: ignore[method-assign]

986
987
        super().__init__()

988
989
        self.info = info
        self.dummy_inputs = dummy_inputs
990
991
        self.cache = cache
        self.enable_sanity_checks = enable_sanity_checks
992

993
994
        self.data_parser = self._get_data_parser()

995
    def __call__(
996
        self,
997
998
        prompt: str,
        mm_data: MultiModalDataDict,
999
        hf_processor_mm_kwargs: Mapping[str, object],
1000
    ) -> MultiModalInputs:
1001
        return self.apply(prompt, mm_data, hf_processor_mm_kwargs)
1002

1003
1004
    def _get_data_parser(self) -> MultiModalDataParser:
        """
1005
        Construct a parser to preprocess multi-modal data items
1006
1007
1008
1009
1010
1011
1012
1013
        before passing them to :meth:`_get_hf_mm_data`.

        You can support additional modalities by creating a subclass
        of :class:`MultiModalDataParser` that has additional subparsers.
        """
        return MultiModalDataParser()

    def _to_mm_items(
1014
1015
1016
        self,
        mm_data: MultiModalDataDict,
    ) -> MultiModalDataItems:
1017
1018
1019
1020
        """
        Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`
        before passing them to :meth:`_get_hf_mm_data`.
        """
1021
        mm_items = self.data_parser.parse_mm_data(mm_data)
1022
        mm_config = self.info.ctx.get_mm_config()
1023
1024

        for modality, items in mm_items.items():
1025
            limit = mm_config.get_limit_per_prompt(modality)
1026
1027
1028
1029
1030
1031
1032
            if len(items) > limit:
                raise ValueError(
                    f"You set {modality}={limit} (or defaulted to 1) in "
                    f"`--limit-mm-per-prompt`, but passed {len(items)} "
                    f"{modality} items in the same prompt.")

        return mm_items
1033

1034
1035
1036
1037
1038
1039
1040
1041
1042
    @abstractmethod
    def _get_mm_fields_config(
        self,
        hf_inputs: BatchFeature,
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, MultiModalFieldConfig]:
        """Given the HF-processed data, output the metadata of each field."""
        raise NotImplementedError

1043
    @abstractmethod
1044
    def _get_prompt_updates(
1045
        self,
1046
        mm_items: MultiModalDataItems,
1047
1048
        hf_processor_mm_kwargs: Mapping[str, object],
        out_mm_kwargs: MultiModalKwargs,
1049
    ) -> list[PromptUpdate]:
1050
1051
        """
        Given the original multi-modal items for this modality
1052
        and HF-processed data, output the updates to perform.
1053

1054
1055
        Notes:
            - You should not assume that HF processor always performs prompt
1056
              updates: in :meth:`_apply_hf_processor_missing`, this method
1057
1058
              is called on text-only and multimodal-only inputs separately,
              instead of passing them in the same call.
1059
1060
            - The update information returned by this method is also used to
              determine the placeholder token positions for each multi-modal
1061
              item.
1062
1063
        """
        raise NotImplementedError
1064

1065
    def _find_mm_placeholders(
1066
        self,
1067
        mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
1068
        new_token_ids: list[int],
1069
        mm_item_counts: Mapping[str, int],
1070
    ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
1071
        return find_mm_placeholders(mm_prompt_updates, new_token_ids,
1072
                                    mm_item_counts)
1073

1074
    def _get_hf_mm_data(
1075
        self,
1076
        mm_items: MultiModalDataItems,
1077
1078
1079
    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
        processor_data = dict[str, object]()
        passthrough_data = dict[str, object]()
1080

1081
1082
1083
        for items in mm_items.values():
            processor_data.update(items.get_processor_data())
            passthrough_data.update(items.get_passthrough_data())
1084

1085
1086
        return processor_data, passthrough_data

1087
1088
1089
    def _call_hf_processor(
        self,
        prompt: str,
1090
1091
1092
1093
        # Not to be confused with `mm_data` in `self.apply`.
        # This refers to the data to be passed to HF processor.
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
1094
    ) -> BatchFeature:
1095
1096
1097
1098
        """
        Call the HF processor on the prompt text and
        associated multi-modal data.
        """
1099
1100
        return self.info.ctx.call_hf_processor(
            self.info.get_hf_processor(**mm_kwargs),
1101
1102
            dict(text=prompt, **mm_data),
            mm_kwargs,
1103
1104
        )

1105
    def _hf_processor_applies_updates(
1106
1107
1108
1109
1110
1111
        self,
        prompt_text: str,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> bool:
        """
1112
        Return whether the HF processor applies prompt updates.
1113
1114
1115
1116
1117
1118
1119
1120
1121

        For most HF processors, this should be :code:`True` when multi-modal
        data items are passed, but :code:`False` when multi-modal embeddings
        are passed.
        """
        return not any(
            isinstance(items, (EmbeddingItems, DictEmbeddingItems))
            for items in mm_items.values())

1122
    def _apply_hf_processor_text_mm(
1123
        self,
1124
        prompt_text: str,
1125
        mm_items: MultiModalDataItems,
1126
        hf_processor_mm_kwargs: Mapping[str, object],
1127
    ) -> tuple[list[int], MultiModalKwargs, bool]:
1128
        """
1129
1130
        Apply the HF processor on the prompt text and multi-modal data
        together.
1131

1132
        In addition, return whether prompt updates have been applied.
1133
1134
1135
1136
1137
1138
1139
1140
1141
        """
        processor_data, passthrough_data = self._get_hf_mm_data(mm_items)

        processed_data = self._call_hf_processor(
            prompt=prompt_text,
            mm_data=processor_data,
            mm_kwargs=hf_processor_mm_kwargs,
        )
        processed_data.update(passthrough_data)
1142

1143
        prompt_ids, = processed_data.pop("input_ids").tolist()
1144

1145
1146
1147
        mm_kwargs = MultiModalKwargs.from_hf_inputs(
            processed_data,
            self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs),
1148
        )
1149

1150
        is_update_applied = self._hf_processor_applies_updates(
1151
1152
1153
1154
1155
            prompt_text=prompt_text,
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
        )

1156
        return prompt_ids, mm_kwargs, is_update_applied
1157

1158
    def _apply_hf_processor_text_only(self, prompt_text: str) -> list[int]:
1159
        """
1160
        Apply the HF processor on the prompt text only.
1161

1162
1163
1164
        Since HF processor requires that text and multi-modal items
        correspond to each other, we create dummy multi-modal items
        to go along with the text.
1165
        """
1166
        prompt_ids, _, _ = self._apply_hf_processor_text_mm(
1167
1168
1169
1170
1171
            prompt_text=prompt_text,
            mm_items=MultiModalDataItems({}),
            hf_processor_mm_kwargs={},
        )

1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
        return prompt_ids

    def _apply_hf_processor_tokens_only(
        self,
        prompt_tokens: list[int],
    ) -> list[int]:
        """
        Apply the HF processor on the prompt tokens only.

        Most HF processors accept prompt text but not prompt tokens.
        If the HF processor adds or removes tokens that are not related to
        multi-modal data, you should override this method so it is consistent
        with the output of :meth:`_apply_hf_processor_text_only` on the
        corresponding text.
        """
        return prompt_tokens

    def _apply_hf_processor_mm_only(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> MultiModalKwargs:
        """
        Apply the HF processor on the multi-modal data only.

        Since HF processor requires that text and multi-modal items
        correspond to each other, we generate dummy text using
        :class:`DummyInputsBuilder` to go along with the multi-modal data.
        """
        mm_counts = mm_items.get_all_counts()

1203
1204
        dummy_inputs = self.dummy_inputs.get_dummy_processor_inputs(
            self.info.ctx.model_config.max_model_len,
1205
            mm_counts,
1206
        )
1207

1208
        _, mm_kwargs, _ = self._apply_hf_processor_text_mm(
1209
            prompt_text=dummy_inputs.prompt_text,
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
        )

        return mm_kwargs

    def _apply_hf_processor_main(
        self,
        prompt: Union[str, list[int]],
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        *,
1222
        enable_hf_prompt_update: bool,
1223
    ) -> tuple[list[int], MultiModalKwargs, bool]:
1224
1225
1226
        """
        Apply the HF processor on the prompt text and multi-modal data.

1227
        In addition, return whether prompt updates have been applied
1228
1229
        (for most HF processors, this should be :code:`True`).

1230
        Note:
1231
1232
            If :code:`enable_hf_prompt_update=False`, we use HF processor
            to perform prompt updates if available; HF processor requires
1233
            that the prompt corresponds to multi-modal items.
1234
1235
        """
        if isinstance(prompt, str):
1236
            if enable_hf_prompt_update:
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
                return self._apply_hf_processor_text_mm(
                    prompt_text=prompt,
                    mm_items=mm_items,
                    hf_processor_mm_kwargs=hf_processor_mm_kwargs,
                )

            prompt_ids = self._apply_hf_processor_text_only(prompt)
        else:
            prompt_ids = self._apply_hf_processor_tokens_only(prompt)

1247
        mm_kwargs = self._apply_hf_processor_mm_only(
1248
            mm_items=mm_items,
1249
1250
1251
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
        )

1252
        return prompt_ids, mm_kwargs, False
1253
1254
1255

    def _cached_apply_hf_processor(
        self,
1256
        prompt: Union[str, list[int]],
1257
1258
        mm_data_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1259
    ) -> tuple[list[int], MultiModalKwargs, bool]:
1260
1261
1262
1263
1264
        """
        Apply the HF processor on the full prompt text,
        caching the results and reusing cached results.
        """
        cache = self.cache
1265
        model_id = self.info.model_id
1266

1267
1268
        _, passthrough_data = self._get_hf_mm_data(mm_data_items)
        if cache is None or passthrough_data:
1269
1270
            return self._apply_hf_processor_main(
                prompt=prompt,
1271
1272
                mm_items=mm_data_items,
                hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1273
                enable_hf_prompt_update=True,
1274
1275
            )

1276
        mm_maybe_cached_kw_items = {
1277
1278
1279
1280
1281
1282
1283
1284
            modality: [
                cache.get(model_id, modality, item, hf_processor_mm_kwargs)
                for item in items
            ]
            for modality, items in mm_data_items.items()
        }

        mm_missing_idxs = {
1285
1286
1287
            modality:
            [idx for idx, item in enumerate(kw_items) if item is None]
            for modality, kw_items in mm_maybe_cached_kw_items.items()
1288
1289
1290
1291
1292
        }
        mm_missing_data = {
            modality: [mm_data_items[modality][idx] for idx in idxs]
            for modality, idxs in mm_missing_idxs.items()
        }
1293
        mm_missing_data_items = self._to_mm_items(mm_missing_data)
1294

1295
        # NOTE: `prompt` does not correspond to `mm_missing_data_items`,
1296
        # so we can't apply prompt updates until the new multimodal
1297
1298
1299
1300
        # items are combined with the cached multimodal items
        (
            prompt_ids,
            mm_missing_kwargs,
1301
            is_update_applied,
1302
        ) = self._apply_hf_processor_main(
1303
1304
            prompt=prompt,
            mm_items=mm_missing_data_items,
1305
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1306
            enable_hf_prompt_update=False,
1307
1308
1309
1310
1311
1312
1313
        )

        mm_missing_next_idx = {
            modality: 0
            for modality in mm_missing_data_items
        }

1314
1315
1316
1317
1318
        merged_kw_items = list[MultiModalKwargsItem]()
        for modality, kw_items in mm_maybe_cached_kw_items.items():
            for idx, kw_item in enumerate(kw_items):
                if kw_item is None:
                    kw_item = mm_missing_kwargs.get_item(
1319
1320
1321
1322
1323
1324
1325
1326
1327
                        modality,
                        mm_missing_next_idx[modality],
                    )

                    cache.put(
                        model_id,
                        modality,
                        mm_data_items[modality][idx],
                        hf_processor_mm_kwargs,
1328
                        kw_item,
1329
1330
1331
1332
                    )

                    mm_missing_next_idx[modality] += 1

1333
                merged_kw_items.append(kw_item)
1334
1335

        if self.enable_sanity_checks:
1336
            mm_missing_counts = mm_missing_data_items.get_all_counts()
1337
1338
1339
1340
1341
1342
            assert all(
                item_count == mm_missing_counts[modality]
                for modality, item_count in mm_missing_next_idx.items()), dict(
                    mm_missing_next_idx=mm_missing_next_idx,
                    mm_missing_counts=mm_missing_counts)

1343
        mm_kwargs = MultiModalKwargs.from_items(merged_kw_items)
1344

1345
        return prompt_ids, mm_kwargs, is_update_applied
1346

1347
    def _bind_and_group_updates(
1348
        self,
1349
1350
        prompt_updates: Sequence[PromptUpdate],
    ) -> dict[str, Sequence[BoundPromptUpdate]]:
1351
        tokenizer = self.info.get_tokenizer()
1352

1353
        it = (update.bind(tokenizer) for update in prompt_updates)
1354
        return dict(full_groupby_modality(it))
1355

1356
    def _apply_prompt_updates(
1357
1358
        self,
        token_ids: list[int],
1359
        mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
1360
        mm_item_counts: Mapping[str, int],
1361
    ) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
1362
        tokenizer = self.info.get_tokenizer()
1363

1364
        mm_token_matches = {
1365
1366
            modality: find_token_matches(token_ids, updates)
            for modality, updates in mm_prompt_updates.items()
1367
        }
1368
1369
        mm_match_counts = {
            modality: len(matches)
1370
            for modality, matches in mm_token_matches.items()
1371
        }
1372
1373
1374
1375
1376
1377
1378
1379
1380

        # If the search text does not represent a special token,
        # it may have different token IDs in the prompt, because
        # the tokens may go across the boundaries of the search text.
        # ----
        # e.g. when searching for "foo" in "food", if "food" itself makes
        # up a token, then the token ID of "foo" will not appear at all
        # ----
        # Since it is inefficient to search for all possible tokenizations
1381
1382
        # of the search text in the prompt, we instead perform string-based
        # updates on the decoded token IDs, then encode them back.
1383
        if all(
1384
1385
            mm_match_counts.get(modality, 0) >= item_count
            for modality, item_count in mm_item_counts.items()
1386
        ):  # yapf: disable
1387
            token_ids = apply_token_matches(
1388
                token_ids,
1389
                mm_token_matches,
1390
                mm_item_counts,
1391
1392
            )

1393
            text = decode_tokens(tokenizer, token_ids)
1394
1395
            matched_updates = {
                modality: [match._origin for match in token_matches]
1396
1397
                for modality, token_matches in mm_token_matches.items()
            }
1398
        else:
1399
            text = decode_tokens(tokenizer, token_ids)
1400

1401
            mm_text_matches = {
1402
1403
                modality: find_text_matches(text, updates)
                for modality, updates in mm_prompt_updates.items()
1404
            }
1405
            text = apply_text_matches(
1406
                text,
1407
                mm_text_matches,
1408
                mm_item_counts,
1409
1410
            )

1411
1412
1413
            token_ids = encode_tokens(tokenizer,
                                      text,
                                      add_special_tokens=False)
1414
1415
            matched_updates = {
                modality: [match._origin for match in token_matches]
1416
1417
1418
1419
                for modality, token_matches in mm_text_matches.items()
            }

        placeholders = self._find_mm_placeholders(
1420
            matched_updates,
1421
1422
1423
            token_ids,
            mm_item_counts,
        )
1424
1425

        return token_ids, text, placeholders
1426

1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
    def _validate_mm_kwargs(
        self,
        mm_kwargs: MultiModalKwargs,
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
            if modality in mm_kwargs.modalities:
                items = mm_kwargs.get_items(modality)
            else:
                items = []

            if len(items) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} {modality} items in "
                    f"keyword arguments corresponding to {item_count} "
                    f"{modality} data items, but only found {len(items)}! "
                    "There is likely a problem with your "
                    "implementation of merged multi-modal processor for this "
                    "model (usually arising from an inconsistency between "
                    "`_call_hf_processor` and `_get_mm_fields_config`).")

    def _validate_mm_placeholders(
        self,
1450
        mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
1451
        mm_item_counts: Mapping[str, int],
1452
    ) -> None:
1453
1454
1455
        for modality, item_count in mm_item_counts.items():
            placeholders = mm_placeholders.get(modality, [])

1456
            if len(placeholders) != item_count:
1457
                raise RuntimeError(
1458
                    f"Expected there to be {item_count} prompt updates "
1459
                    f"corresponding to {item_count} {modality} items, but "
1460
                    f"instead found {len(placeholders)} prompt updates! "
1461
                    "Either the prompt text has missing/incorrect tokens for "
1462
1463
1464
                    "multi-modal inputs, or there is a problem with your "
                    "implementation of merged multi-modal processor for this "
                    "model (usually arising from an inconsistency between "
1465
                    "`_call_hf_processor` and `_get_prompt_updates`).")
1466

1467
1468
    def apply(
        self,
1469
        prompt: Union[str, list[int]],
1470
        mm_data: MultiModalDataDict,
1471
        hf_processor_mm_kwargs: Mapping[str, object],
1472
        return_mm_hashes: bool = False,
1473
    ) -> MultiModalInputs:
1474
1475
1476
1477
1478
1479
1480
        """
        Process multi-modal inputs to be used in vLLM.

        The main steps are:

        1. Apply HF Processor on prompt text and multi-modal data together,
           outputting token IDs and processed tensors.
1481
        2. Find and update sequences in the token IDs with placeholder tokens.
1482
1483
1484
1485
1486
           The number of placeholder tokens equals the feature size of the
           multi-modal data outputted by the multi-modal encoder.
        3. Extract information about the placeholder tokens from the
           processed token IDs.
        """
1487
        mm_items = self._to_mm_items(mm_data)
1488

1489
        # Create MM hashes to be returned (only used in V1)
1490
1491
1492
        # TODO: Use these hash keys for caching operations in apply_hf_processor
        # instead of rehashing.

1493
        if return_mm_hashes:
1494
            model_id = self.info.model_id
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
            mm_hashes = {
                modality: [
                    MultiModalHasher.hash_kwargs(model_id=model_id,
                                                 **{modality: item},
                                                 **hf_processor_mm_kwargs)
                    for item in items
                ]
                for modality, items in mm_items.items()
            }
        else:
            mm_hashes = None

1507
1508
1509
        (
            prompt_ids,
            mm_kwargs,
1510
            is_update_applied,
1511
        ) = self._cached_apply_hf_processor(
1512
            prompt,
1513
1514
1515
            mm_items,
            hf_processor_mm_kwargs,
        )
1516

1517
        unbound_prompt_updates = self._get_prompt_updates(
1518
1519
1520
1521
            mm_items,
            hf_processor_mm_kwargs,
            mm_kwargs,
        )
1522
1523
        mm_prompt_updates = self._bind_and_group_updates(
            unbound_prompt_updates)
1524

1525
        mm_item_counts = mm_items.get_all_counts()
1526
1527
        self._validate_mm_kwargs(mm_kwargs, mm_item_counts)

1528
        if is_update_applied:
1529
            mm_placeholders = self._find_mm_placeholders(
1530
                mm_prompt_updates,
1531
                prompt_ids,
1532
1533
                mm_item_counts,
            )
1534
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1535

1536
            tokenizer = self.info.get_tokenizer()
1537
            prompt = decode_tokens(tokenizer, prompt_ids)
1538
1539
1540
        else:
            (
                prompt_ids,
1541
                prompt,
1542
                mm_placeholders,
1543
            ) = self._apply_prompt_updates(
1544
                prompt_ids,
1545
                mm_prompt_updates,
1546
                mm_item_counts,
1547
            )
1548
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1549
1550
1551
1552
1553

        mm_placeholder_ranges = {
            modality: [item.to_range() for item in placeholders]
            for modality, placeholders in mm_placeholders.items()
        }
1554

1555
        return MultiModalInputs(
1556
            type="multimodal",
1557
            prompt=prompt,
1558
            prompt_token_ids=prompt_ids,
1559
            mm_kwargs=mm_kwargs,
1560
            mm_hashes=mm_hashes,
1561
            mm_placeholders=mm_placeholder_ranges,
1562
        )
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572


class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):

    @abstractmethod
    def create_encoder_prompt(
        self,
        prompt: Union[str, list[int]],
        mm_data: MultiModalDataDict,
    ) -> Union[str, list[int]]:
1573
1574
1575
1576
        """
        Create input prompt for the encoder. HF processor will be applied on 
        this prompt during profiling and generation.
        """
1577
1578
        raise NotImplementedError

1579
1580
1581
1582
1583
1584
1585
1586
    def create_decoder_prompt(
        self,
        prompt: Union[str, list[int]],
        mm_data: MultiModalDataDict,
    ) -> Union[str, list[int]]:
        """Create input prompt for the decoder."""
        return prompt

1587
1588
1589
1590
1591
    def apply(
        self,
        prompt: Union[str, list[int]],
        mm_data: MultiModalDataDict,
        hf_processor_mm_kwargs: Mapping[str, object],
1592
        return_mm_hashes: bool = False,
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
    ) -> MultiModalEncDecInputs:
        """
        Process multi-modal inputs to be used in vLLM.
        The main processing steps are modified to fit encoder-decoder model:
        1. Create encoder prompt from input prompt text.
        2. Apply the HF processor on encoder prompt.
        3. Copy the input prompt text as decoder prompt inputs.
        """
        encoder_prompt = self.create_encoder_prompt(prompt, mm_data)
        encoder_inputs = super().apply(
            encoder_prompt,
            mm_data,
            hf_processor_mm_kwargs,
1606
            return_mm_hashes,
1607
1608
1609
        )

        tokenizer = self.info.get_tokenizer()
1610
1611
        decoder_prompt = self.create_decoder_prompt(prompt, mm_data)
        if isinstance(decoder_prompt, str):
1612
            decoder_prompt_ids = encode_tokens(tokenizer,
1613
                                               decoder_prompt,
1614
1615
                                               add_special_tokens=False)
        else:
1616
1617
            decoder_prompt_ids = decoder_prompt
            decoder_prompt = decode_tokens(tokenizer, decoder_prompt)
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627

        mm_inputs = MultiModalEncDecInputs(
            encoder_prompt=encoder_inputs["prompt"],
            encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],
            **encoder_inputs)
        mm_inputs.update({
            "prompt": decoder_prompt,
            "prompt_token_ids": decoder_prompt_ids
        })
        return mm_inputs