processing.py 50.4 KB
Newer Older
1
2
# SPDX-License-Identifier: Apache-2.0

3
4
import re
from abc import ABC, abstractmethod
5
from collections import defaultdict
6
7
from collections.abc import (Callable, Generator, ItemsView, Iterable, Mapping,
                             Sequence)
8
from dataclasses import dataclass, field
9
from enum import Enum
10
from functools import lru_cache
11
from typing import (TYPE_CHECKING, Generic, NamedTuple, Optional, Protocol,
12
                    TypeVar, Union, cast)
13

14
from transformers import BatchFeature, PretrainedConfig, ProcessorMixin
15
from typing_extensions import assert_never
16

17
from vllm.inputs import InputProcessingContext
18
from vllm.logger import init_logger
19
20
from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
                                               encode_tokens)
21
from vllm.utils import LRUCache, flatten_2d_lists, full_groupby
22

23
from .hasher import MultiModalHasher
24
25
26
from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
                     MultiModalFieldConfig, MultiModalInputs, MultiModalKwargs,
                     MultiModalKwargsItem, PlaceholderRange)
27
28
from .parse import (DictEmbeddingItems, EmbeddingItems, MultiModalDataItems,
                    MultiModalDataParser)
29
30
31

if TYPE_CHECKING:
    from .profiling import BaseDummyInputsBuilder
32

33
logger = init_logger(__name__)
34
35

_S = TypeVar("_S", str, list[int])
36
37
38

PromptSeq = Union[str, list[int]]
"""A token sequence (list of token IDs) or text."""
39

40

41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
@dataclass
class PromptIndex:
    """Resolves to an index in the prompt."""
    get_match_index: Callable[[AnyTokenizer, PromptSeq], Optional[int]]


class PromptIndexTargets:

    @staticmethod
    def start() -> PromptIndex:
        """
        Resolves to the start of the prompt (before the first token).

        This results in a match even if the prompt is empty.
        """
        return PromptIndex(lambda tok, prompt: 0)

    @staticmethod
    def prefix(seq: PromptSeq) -> PromptIndex:
        """
        Resolves to a location in the prompt after the given prefix.
        """

        def get_match_index(
            tokenizer: AnyTokenizer,
            prompt: PromptSeq,
        ) -> Optional[int]:
            prefix = seq

            if isinstance(prompt, str):
                if not isinstance(prefix, str):
                    # Make both `str`
                    prefix = decode_tokens(tokenizer, prefix)
            else:
                if isinstance(prefix, str):
                    # Make both `list[int]`
                    prefix = encode_tokens(tokenizer, prefix)

            match_idx = len(prefix)
            return match_idx if prompt[:match_idx] == prefix else None

        return PromptIndex(get_match_index)

    @staticmethod
    def end() -> PromptIndex:
        """
        Resolves to the end of the prompt (after the last token).

        This results in a match even if the prompt is empty.
        """
        return PromptIndex(lambda tok, prompt: len(prompt))


PromptTarget = Union[PromptSeq, PromptIndex]
"""
The token sequence or text to update.
"""


100
@dataclass
101
102
class PromptUpdateDetails:
    """Details about the token sequence or text that are part of the update."""
103
104

    full: PromptSeq
105
    """The full content."""
106

107
    features: PromptSeq
108
    """
109
    The part of the content that corresponds to feature placeholders;
110
111
    this will be replaced by the output of the vision encoder during model
    inference.
112
113
114
    """

    @staticmethod
115
116
    def from_seq(seq: PromptSeq) -> "PromptUpdateDetails":
        return PromptUpdateDetails(full=seq, features=seq)
117
118


119
PromptUpdateInfo = Union[PromptSeq, PromptUpdateDetails]
120
"""
121
The token sequence or text that are part of the update.
122

123
124
If only part of the content corresponds to feature placeholders, you can
use :class:`PromptUpdateDetails` to specify which part.
125
"""
126

127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
PromptUpdateContent = Union[Callable[[int], PromptUpdateInfo],
                            PromptUpdateInfo]
"""
Given the index of the processed item within :attr:`modality`,
output the corresponding token sequence (or text).

For convenience, you can directly pass in the token sequence (or text)
instead of a function if it does not depend on the input.
"""


class UpdateMode(str, Enum):
    INSERT = "insert"
    REPLACE = "replace"


@dataclass
144
class PromptUpdate(ABC):
145
146
147
148
149
150
151
    """
    Defines how to update a prompt with placeholder tokens.
    """

    modality: str
    """The modality for which the update is made."""

152
    target: PromptTarget
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
    """The token sequence (or text) to update."""

    @property
    @abstractmethod
    def content(self) -> PromptUpdateContent:
        """The placeholder tokens that are part of the update."""
        raise NotImplementedError

    @property
    @abstractmethod
    def mode(self) -> UpdateMode:
        """Defines how to update the prompt."""
        raise NotImplementedError

    def bind(self, tokenizer: AnyTokenizer) -> "BoundPromptUpdate":
        return BoundPromptUpdate(
            _origin=self,
            tokenizer=tokenizer,
        )

173

174
@dataclass
175
176
177
178
179
180
181
class PromptInsertion(PromptUpdate):
    """
    Defines how to insert placeholder tokens into a prompt.

    Example:

        For each image, insert a number of ``<image>`` feature placeholders
182
        equal to the feature size of the vision encoder after the ``<s>`` token:
183
184
185
186
187

        .. code-block:: python

            PromptInsertion(
                modality="image",
188
                target="<s>",
189
190
191
                insertion="<image>" * image_feature_size,
            )

192
        Insert these tokens at the start of the prompt:
193
194
195
196
197

        .. code-block:: python

            PromptInsertion(
                modality="image",
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
                target=PromptIndexTargets.start(),
                insertion="<image>" * image_feature_size,
            )

        Insert these tokens after a prefix ``Images:``:

        .. code-block:: python

            PromptInsertion(
                modality="image",
                target=PromptIndexTargets.prefix("Images:"),
                insertion="<image>" * image_feature_size,
            )

        Insert these tokens at the end of the prompt:

        .. code-block:: python

            PromptInsertion(
                modality="image",
                target=PromptIndexTargets.end(),
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
                insertion="<image>" * image_feature_size,
            )
    """

    insertion: PromptUpdateContent = field(repr=False)
    """
    Given the index of the processed item within :attr:`modality`,
    output the token sequence (or text) to insert right after :attr:`target`.

    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
    """

    @property
    def content(self) -> PromptUpdateContent:
        return self.insertion

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.INSERT


@dataclass
class PromptReplacement(PromptUpdate):
243
244
    """
    Defines how to replace portions of an input prompt with placeholder tokens.
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268

    Example:

        For each image, replace one ``<image>`` input placeholder in the prompt
        with a number of ``<image>`` feature placeholders
        equal to the feature size of the vision encoder:

        .. code-block:: python

            PromptReplacement(
                modality="image",
                target="<image>",
                replacement="<image>" * image_feature_size,
            )

        As above, but further pad the feature placeholders with ``<image_bos>``
        and `<image_eos>``, which are not supposed to be passed to the vision
        encoder:

        .. code-block:: python

            PromptReplacement(
                modality="image",
                target="<image>",
269
                replacement=PromptUpdateDetails(
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
                    full="".join([
                        "<image_bos>",
                        "<image>" * image_feature_size,
                        "<image_eos>",
                    ]),
                    features="<image>" * image_feature_size,
                ),
            )

        To avoid unnecessary tokenization during prompt replacement,
        we recommended passing token sequences instead of text:

        .. code-block:: python

            PromptReplacement(
                modality="image",
                target=[image_token_id],
287
                replacement=PromptUpdateDetails(
288
289
290
291
292
                    full=([image_bos_id] + [image_token_id] * image_feature_size
                          + [image_eos_id]),
                    features=[image_token_id] * image_feature_size,
                ),
            )
293
294
    """

295
    replacement: PromptUpdateContent = field(repr=False)
296
    """
297
    Given the index of the processed item within :attr:`modality`,
298
    output the token sequence (or text) to replace :attr:`target`.
299

300
301
    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
302
303
    """

304
305
306
307
308
309
310
    @property
    def content(self) -> PromptUpdateContent:
        return self.replacement

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.REPLACE
311
312


313
314
315
316
317
318
319
@lru_cache(maxsize=2048)
def _cached_encode(
    tokenizer: AnyTokenizer,
    text: str,
    *,
    add_special_tokens: bool = False,
) -> list[int]:
320
321
322
    return encode_tokens(tokenizer,
                         text,
                         add_special_tokens=add_special_tokens)
323
324


325
326
327
328
329
330
331
@lru_cache(maxsize=2048)
def _cached_decode(
    tokenizer: AnyTokenizer,
    token_ids: tuple[int, ...],
    *,
    skip_special_tokens: bool = False,
) -> str:
332
333
334
    return decode_tokens(tokenizer,
                         list(token_ids),
                         skip_special_tokens=skip_special_tokens)
335
336
337
338
339


class _HasModalityAttr(Protocol):
    modality: str

340

341
class _HasModalityProp(Protocol):
342

343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
    @property
    def modality(self) -> str:
        ...


_M = TypeVar("_M", bound=Union[_HasModalityAttr, _HasModalityProp])


def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]:
    """Convenience function to apply :func:`full_groupby` based on modality."""
    return full_groupby(values, key=lambda x: x.modality)


@dataclass
class _BoundPromptSequence:
358
359
360
361
    """
    A :data:`_PromptSeq` bound to a tokenizer to automatically
    convert between token sequence and text representations.
    """
362
363
    tokenizer: AnyTokenizer = field(repr=False)

364
365
366
    _text: Optional[str]
    _token_ids: Optional[list[int]]

367
    @staticmethod
368
369
    def from_seq(
        tokenizer: AnyTokenizer,
370
        seq: PromptSeq,
371
    ) -> "_BoundPromptSequence":
372
373
374
375
376
377
        return _BoundPromptSequence(
            tokenizer=tokenizer,
            _text=seq if isinstance(seq, str) else None,
            _token_ids=seq if isinstance(seq, list) else None,
        )

378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
    def __post_init__(self) -> None:
        if self._text is None and self._token_ids is None:
            raise ValueError("At least one of 'text' and 'token_ids' must be "
                             "specified")

    @property
    def text(self) -> str:
        if self._text is None:
            assert self._token_ids is not None
            self._text = _cached_decode(self.tokenizer, tuple(self._token_ids))

        return self._text

    @property
    def token_ids(self) -> list[int]:
        if self._token_ids is None:
            assert self._text is not None
            self._token_ids = _cached_encode(self.tokenizer, self._text)

        return self._token_ids


400
@dataclass
401
class _BoundPromptContent:
402
403
404
405
    full: _BoundPromptSequence
    features: _BoundPromptSequence


406
@dataclass
407
class BoundPromptUpdate:
408
    """
409
410
    A :class:`PromptUpdate` bound to a tokenizer to automatically convert
    :attr:`target` and the result of :meth:`get_content` between
411
412
    token sequence and text representations.
    """
413
    _origin: PromptUpdate
414
    tokenizer: AnyTokenizer = field(repr=False)
415

416
    def __post_init__(self) -> None:
417
418
419
420
421
        self._content_cache = dict[int, _BoundPromptContent]()

    @property
    def modality(self) -> str:
        return self._origin.modality
422
423

    @property
424
    def target(self) -> Union[_BoundPromptSequence, PromptIndex]:
425
        """The token sequence (or text) to update."""
426
427
428
429
430
431
        target = self._origin.target

        if isinstance(target, PromptIndex):
            return target

        return _BoundPromptSequence.from_seq(self.tokenizer, target)
432

433
434
435
436
437
438
439
440
441
442
443
    @property
    def content(self) -> PromptUpdateContent:
        """The placeholder tokens that are part of the update."""
        return self._origin.content

    @property
    def mode(self) -> UpdateMode:
        """Defines how to update the prompt."""
        return self._origin.mode

    def get_content(self, item_idx: int) -> _BoundPromptContent:
444
445
        """
        Given the index of the processed item within :attr:`modality`,
446
        output the token sequence (or text) to update.
447
        """
448
449
        content = self.content
        if callable(content):
450
            cache_key = item_idx
451
452
            if cache_key in self._content_cache:
                return self._content_cache[cache_key]
453

454
            content = content(item_idx)
455
456
457
        else:
            cache_key = None

458
459
        if not isinstance(content, PromptUpdateDetails):
            content = PromptUpdateDetails.from_seq(content)
460
461

        bound_full = _BoundPromptSequence.from_seq(self.tokenizer,
462
                                                   content.full)
463
        bound_features = _BoundPromptSequence.from_seq(self.tokenizer,
464
465
466
                                                       content.features)
        bound_content = _BoundPromptContent(full=bound_full,
                                            features=bound_features)
467
468

        if cache_key is not None:
469
            self._content_cache[cache_key] = bound_content
470

471
        return bound_content
472
473


474
475
476
class _TokenMatch(NamedTuple):
    start_idx: int
    end_idx: int
477
478


479
480
481
def iter_token_matches(
    token_ids: list[int],
    match_ids: list[int],
482
) -> Generator[_TokenMatch]:
483
484
485
486
487
488
    """
    Yield each occurrence of :code:`match_ids` in :code:`token_ids`.

    Note that empty matches are ignored.
    """
    prompt_len = len(token_ids)
489
    match_len = len(match_ids)
490

491
492
    if match_len == 0:
        return
493

494
495
    start_idx = 0
    while start_idx < prompt_len - match_len + 1:
496
        end_idx = start_idx + match_len
497

498
499
        if token_ids[start_idx:end_idx] == match_ids:
            yield _TokenMatch(start_idx=start_idx, end_idx=end_idx)
500
501
502
503
504

            # Exclude overlapping matches
            start_idx = end_idx
        else:
            start_idx += 1
505
506


507
@dataclass(repr=False)
508
509
class _PromptTargetMatch(ABC):
    _origin: BoundPromptUpdate
510
511
512

    @property
    def modality(self) -> str:
513
        return self._origin.modality
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529

    @property
    @abstractmethod
    def start_idx(self) -> int:
        raise NotImplementedError

    @property
    @abstractmethod
    def end_idx(self) -> int:
        raise NotImplementedError

    def __repr__(self) -> str:
        return (f"{type(self).__name__}(modality={self.modality!r}, "
                f"start_idx={self.start_idx!r}, end_idx={self.end_idx!r})")


530
531
532
533
534
535
536
537
538
539
540
541
542
@dataclass(repr=False)
class _PromptTargetIndexMatch(_PromptTargetMatch):
    match_idx: int

    @property
    def start_idx(self) -> int:
        return self.match_idx

    @property
    def end_idx(self) -> int:
        return self.match_idx


543
@dataclass(repr=False)
544
class _PromptTargetTokenMatch(_PromptTargetMatch):
545
546
547
548
549
550
551
552
553
554
555
556
    match: _TokenMatch

    @property
    def start_idx(self) -> int:
        return self.match.start_idx

    @property
    def end_idx(self) -> int:
        return self.match.end_idx


@dataclass(repr=False)
557
class _PromptTargetTextMatch(_PromptTargetMatch):
558
559
560
561
562
563
564
565
566
567
    match: re.Match[str]

    @property
    def start_idx(self) -> int:
        return self.match.start()

    @property
    def end_idx(self) -> int:
        return self.match.end()

568

569
@dataclass
570
class PlaceholderFeaturesInfo:
571
    modality: str
572
    item_idx: int
573
    start_idx: int
574
    tokens: list[int]
575
576
577

    @property
    def length(self) -> int:
578
        return len(self.tokens)
579
580
581
582
583
584

    def to_range(self) -> PlaceholderRange:
        return PlaceholderRange(
            offset=self.start_idx,
            length=self.length,
        )
585
586
587
588


def find_token_matches(
    prompt: list[int],
589
590
591
    prompt_updates: Sequence[BoundPromptUpdate],
) -> Sequence[_PromptTargetMatch]:
    """Return each target of :code:`prompt_updates` found in :code:`prompt`."""
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607

    def get_matches(update: BoundPromptUpdate):
        target = update.target

        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(update.tokenizer, prompt)
            if match_idx is None:
                return []

            return [_PromptTargetIndexMatch(update, match_idx)]

        return [
            _PromptTargetTokenMatch(update, match)
            for match in iter_token_matches(prompt, target.token_ids)
        ]

608
    return [
609
        match for update in prompt_updates for match in get_matches(update)
610
611
612
613
614
    ]


def find_text_matches(
    prompt: str,
615
616
617
    prompt_updates: Sequence[BoundPromptUpdate],
) -> Sequence[_PromptTargetMatch]:
    """Return each target of :code:`prompt_updates` found in :code:`prompt`."""
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633

    def get_matches(update: BoundPromptUpdate):
        target = update.target

        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(update.tokenizer, prompt)
            if match_idx is None:
                return []

            return [_PromptTargetIndexMatch(update, match_idx)]

        return [
            _PromptTargetTextMatch(update, match)
            for match in re.finditer(re.escape(target.text), prompt)
        ]

634
    return [
635
        match for update in prompt_updates for match in get_matches(update)
636
637
638
639
    ]


def _resolve_matches(
640
    prompt: PromptSeq,
641
642
    mm_matches: Mapping[str, Sequence[_PromptTargetMatch]],
) -> list[_PromptTargetMatch]:
643
    """
644
    Resolve :code:`mm_matches` to ensure that there are no overlapping matches,
645
    and sort them such that earlier matches take priority over later ones.
646
    """
647
648
    matches = [m for matches in mm_matches.values() for m in matches]

649
    seen_matches: list[Optional[_PromptTargetMatch]] = [None] * len(prompt)
650

651
    for match in matches:
652
653
654
655
656
        for idx in range(match.start_idx, match.end_idx):
            if seen_matches[idx] is not None:
                raise ValueError("Found overlapping matches "
                                 f"({seen_matches[idx]} and {match}) "
                                 f"at index={idx} of prompt={prompt}")
657

658
            seen_matches[idx] = match
659
660
661
662

    return sorted(matches, key=lambda x: x.start_idx)


663
def _apply_matches(
664
    prompt: _S,
665
    mm_matches: Mapping[str, Sequence[_PromptTargetMatch]],
666
    mm_item_counts: Mapping[str, int],
667
) -> list[_S]:
668
669
    """Apply the updates in :code:`mm_matches` to :code:`prompt`."""
    out_seqs = list[Union[str, list[int]]]()
670
    prev_end_idx = 0
671
    next_idx_by_modality = defaultdict[str, int](lambda: 0)
672

673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
    for match in _resolve_matches(prompt, mm_matches):
        modality = match.modality

        item_start_idx = next_idx_by_modality[modality]
        max_item_count = mm_item_counts.get(modality, 0)
        if item_start_idx >= max_item_count:
            continue

        start_idx = match.start_idx
        end_idx = match.end_idx
        origin = match._origin
        mode = origin.mode

        if mode == UpdateMode.INSERT:
            out_seqs.append(prompt[prev_end_idx:end_idx])
            num_inserts = max_item_count
        elif mode == UpdateMode.REPLACE:
            out_seqs.append(prompt[prev_end_idx:start_idx])
            num_inserts = max_item_count if start_idx == end_idx else 1
        else:
            assert_never(mode)
694

695
        item_end_idx = min(item_start_idx + num_inserts, max_item_count)
696

697
        for item_idx in range(item_start_idx, item_end_idx):
698
            content = origin.get_content(item_idx)
699
700
            insert_seq = (content.full.text if isinstance(prompt, str) else
                          content.full.token_ids)
701

702
            out_seqs.append(insert_seq)
703

704
705
        prev_end_idx = end_idx
        next_idx_by_modality[modality] += item_end_idx - item_start_idx
706
707
708

    out_seqs.append(prompt[prev_end_idx:])

709
    return cast(list[_S], out_seqs)
710
711


712
def apply_token_matches(
713
    prompt: list[int],
714
    mm_matches: Mapping[str, Sequence[_PromptTargetMatch]],
715
    mm_item_counts: Mapping[str, int],
716
) -> list[int]:
717
    """Apply the updates in :code:`mm_matches` to :code:`prompt`."""
718
    if not mm_matches:
719
720
        return prompt

721
    token_id_seqs = _apply_matches(prompt, mm_matches, mm_item_counts)
722
723

    return flatten_2d_lists(token_id_seqs)
724
725


726
def apply_text_matches(
727
    prompt: str,
728
    mm_matches: Mapping[str, Sequence[_PromptTargetMatch]],
729
    mm_item_counts: Mapping[str, int],
730
) -> str:
731
    """Apply the updates in :code:`mm_matches` to :code:`prompt`."""
732
    if not mm_matches:
733
        return prompt
734

735
    texts = _apply_matches(prompt, mm_matches, mm_item_counts)
736
737

    return "".join(texts)
738
739


740
def _iter_placeholders(
741
    mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
742
    prompt: list[int],
743
    mm_item_counts: Mapping[str, int],
744
) -> Iterable[PlaceholderFeaturesInfo]:
745
746
747
748
749
    """
    Yield each set of placeholder tokens found in :code:`prompt`.

    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
750
    appears earlier in `mm_prompt_updates` takes priority.
751

752
753
    Note that empty matches are ignored.
    """
754
    prompt_len = len(prompt)
755
    item_idx_by_modality = defaultdict[str, int](lambda: 0)
756
757
758
759
760

    start_idx = 0
    while start_idx < prompt_len:
        found = False

761
        for modality, modality_updates in mm_prompt_updates.items():
762
763
            item_idx = item_idx_by_modality[modality]
            if item_idx >= mm_item_counts.get(modality, 0):
764
                continue
765

766
767
768
769
770
            for update_info in modality_updates:
                content = update_info.get_content(item_idx)
                content_tokens_full = content.full.token_ids
                content_len_full = len(content_tokens_full)
                end_idx_full = start_idx + content_len_full
771

772
                if content_len_full == 0 or end_idx_full > prompt_len:
773
774
                    continue

775
776
                if prompt[start_idx:end_idx_full] == content_tokens_full:
                    content_tokens_feat = content.features.token_ids
777
778
779

                    try:
                        match = next(
780
781
                            iter_token_matches(content_tokens_full,
                                               content_tokens_feat))
782
783
784
785
                        yield PlaceholderFeaturesInfo(
                            modality=modality,
                            item_idx=item_idx,
                            start_idx=start_idx + match.start_idx,
786
                            tokens=content_tokens_feat,
787
788
789
                        )
                    except StopIteration:
                        raise AssertionError(
790
791
                            f"{content_tokens_feat=} should be a "
                            f"subsequence of {content_tokens_full=}") from None
792

793
                    # Exclude overlapping matches
794
                    start_idx = end_idx_full
795
796
797
                    item_idx_by_modality[modality] += 1
                    found = True
                    break
798

799
800
            if found:
                break  # Go back to the outer while loop
801
802
803

        if not found:
            start_idx += 1
804
805


806
def find_mm_placeholders(
807
    mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
808
809
    prompt: list[int],
    mm_item_counts: Mapping[str, int],
810
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
811
    it = _iter_placeholders(mm_prompt_updates, prompt, mm_item_counts)
812
813
814
    return dict(full_groupby_modality(it))


815
816
817
818
819
820
821
822
class ProcessingCache:

    def __init__(self, capacity: int) -> None:
        super().__init__()

        # DEBUG: Set to None to disable
        self.debug_cache_hit_ratio_steps: Optional[int] = None

823
        self._cache = LRUCache[str, MultiModalKwargsItem](capacity)
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840

    def _maybe_log_cache_stats(self) -> None:
        steps = self.debug_cache_hit_ratio_steps
        if not steps:
            return

        cache_stats = self._cache.stat()
        if cache_stats.total % steps == 0:
            logger.debug("ProcessingCache: hit_ratio = %.2f",
                         cache_stats.hit_ratio)

    def get(
        self,
        model_id: str,
        modality: str,
        input_item: object,
        input_kwargs: Mapping[str, object],
841
    ) -> Optional[MultiModalKwargsItem]:
842
843
844
845
846
847
848
849
850
851
852
        """
        Get a processed multi-modal item from the cache
        according to its dependencies, including:

        - The model ID
        - The modality of the item
        - The original data item passed to the HF processor
        - The configuration options of the HF processor
        """
        self._maybe_log_cache_stats()

853
854
855
        cache_key = MultiModalHasher.hash_kwargs(model_id=model_id,
                                                 **{modality: input_item},
                                                 **input_kwargs)
856
857
858
859
860
861
862
863
        return self._cache.get(cache_key)

    def put(
        self,
        model_id: str,
        modality: str,
        input_item: object,
        input_kwargs: Mapping[str, object],
864
        output_kwargs: MultiModalKwargsItem,
865
866
867
868
869
    ) -> None:
        """
        Put a processed multi-modal item into the cache
        according to its dependencies (see :meth:`get`).
        """
870
871
872
        cache_key = MultiModalHasher.hash_kwargs(model_id=model_id,
                                                 **{modality: input_item},
                                                 **input_kwargs)
873
        self._cache.put(cache_key, output_kwargs)
874
875


876
class BaseProcessingInfo:
877
    """Base class to provide the information necessary for data processing."""
878

879
880
    def __init__(self, ctx: InputProcessingContext) -> None:
        super().__init__()
881

882
883
884
885
886
887
888
        self.ctx = ctx

    @property
    def model_id(self) -> str:
        return self.ctx.model_config.model

    def get_tokenizer(self) -> AnyTokenizer:
889
890
        return self.ctx.tokenizer

891
    def get_hf_config(self) -> PretrainedConfig:
892
893
        return self.ctx.get_hf_config()

894
    def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
895
896
897
898
899
900
        """
        Subclasses can override this method to handle
        specific kwargs from model config or user inputs.
        """
        return self.ctx.get_hf_processor(**kwargs)

901
902
903
904
905
906
907
908
909
910
911
912
913
    @abstractmethod
    def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
        """
        Return the maximum supported number of items for each modality.

        A value of `None` means unlimited number of items.

        Omitting a modality from the returned dictionary means that
        it is not supported at all.
        """
        raise NotImplementedError

    @abstractmethod
914
915
916
917
918
    def get_mm_max_tokens_per_item(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
    ) -> Mapping[str, int]:
919
920
921
922
923
924
925
926
927
928
929
        """
        Get the maximum possible number of tokens per data item
        for each modality.

        The dictionary returned by this method should have the same
        keys as that returned by :meth:`get_supported_mm_limits`.
        """
        raise NotImplementedError


_I = TypeVar("_I", bound=BaseProcessingInfo)
930

931
932

class BaseMultiModalProcessor(ABC, Generic[_I]):
933
    """
934
    Abstract base class to process multi-modal inputs to be used in vLLM.
935
936

    Not to be confused with :class:`transformers.ProcessorMixin`.
937
938
    """

939
    def __init__(self,
940
941
                 info: _I,
                 dummy_inputs: "BaseDummyInputsBuilder[_I]",
942
943
944
                 *,
                 cache: Optional[ProcessingCache] = None,
                 enable_sanity_checks: bool = True) -> None:
945
946
947
948
949
950
        if get_repls := getattr(self, "_get_prompt_replacements", None):
            logger.warning_once("`_get_prompt_replacements` has been renamed "
                                "to `_get_prompt_updates`. The old name will "
                                "be removed in an upcoming release.")
            self._get_prompt_updates = get_repls  # type: ignore[method-assign]

951
952
        super().__init__()

953
954
        self.info = info
        self.dummy_inputs = dummy_inputs
955
956
        self.cache = cache
        self.enable_sanity_checks = enable_sanity_checks
957

958
959
        self.data_parser = self._get_data_parser()

960
    def __call__(
961
        self,
962
963
        prompt: str,
        mm_data: MultiModalDataDict,
964
        hf_processor_mm_kwargs: Mapping[str, object],
965
    ) -> MultiModalInputs:
966
        return self.apply(prompt, mm_data, hf_processor_mm_kwargs)
967

968
969
    def _get_data_parser(self) -> MultiModalDataParser:
        """
970
        Construct a parser to preprocess multi-modal data items
971
972
973
974
975
976
977
978
        before passing them to :meth:`_get_hf_mm_data`.

        You can support additional modalities by creating a subclass
        of :class:`MultiModalDataParser` that has additional subparsers.
        """
        return MultiModalDataParser()

    def _to_mm_items(
979
980
981
        self,
        mm_data: MultiModalDataDict,
    ) -> MultiModalDataItems:
982
983
984
985
        """
        Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`
        before passing them to :meth:`_get_hf_mm_data`.
        """
986
        mm_items = self.data_parser.parse_mm_data(mm_data)
987
        mm_config = self.info.ctx.get_mm_config()
988
989

        for modality, items in mm_items.items():
990
            limit = mm_config.get_limit_per_prompt(modality)
991
992
993
994
995
996
997
            if len(items) > limit:
                raise ValueError(
                    f"You set {modality}={limit} (or defaulted to 1) in "
                    f"`--limit-mm-per-prompt`, but passed {len(items)} "
                    f"{modality} items in the same prompt.")

        return mm_items
998

999
1000
1001
1002
1003
1004
1005
1006
1007
    @abstractmethod
    def _get_mm_fields_config(
        self,
        hf_inputs: BatchFeature,
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, MultiModalFieldConfig]:
        """Given the HF-processed data, output the metadata of each field."""
        raise NotImplementedError

1008
    @abstractmethod
1009
    def _get_prompt_updates(
1010
        self,
1011
        mm_items: MultiModalDataItems,
1012
1013
        hf_processor_mm_kwargs: Mapping[str, object],
        out_mm_kwargs: MultiModalKwargs,
1014
    ) -> list[PromptUpdate]:
1015
1016
        """
        Given the original multi-modal items for this modality
1017
        and HF-processed data, output the updates to perform.
1018

1019
1020
        Notes:
            - You should not assume that HF processor always performs prompt
1021
              updates: in :meth:`_apply_hf_processor_missing`, this method
1022
1023
              is called on text-only and multimodal-only inputs separately,
              instead of passing them in the same call.
1024
1025
            - The update information returned by this method is also used to
              determine the placeholder token positions for each multi-modal
1026
              item.
1027
1028
        """
        raise NotImplementedError
1029

1030
    def _find_mm_placeholders(
1031
        self,
1032
        mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
1033
        new_token_ids: list[int],
1034
        mm_item_counts: Mapping[str, int],
1035
    ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
1036
        return find_mm_placeholders(mm_prompt_updates, new_token_ids,
1037
                                    mm_item_counts)
1038

1039
    def _get_hf_mm_data(
1040
        self,
1041
        mm_items: MultiModalDataItems,
1042
1043
1044
    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
        processor_data = dict[str, object]()
        passthrough_data = dict[str, object]()
1045

1046
1047
1048
        for items in mm_items.values():
            processor_data.update(items.get_processor_data())
            passthrough_data.update(items.get_passthrough_data())
1049

1050
1051
        return processor_data, passthrough_data

1052
1053
1054
    def _call_hf_processor(
        self,
        prompt: str,
1055
1056
1057
1058
        # Not to be confused with `mm_data` in `self.apply`.
        # This refers to the data to be passed to HF processor.
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
1059
    ) -> BatchFeature:
1060
1061
1062
1063
        """
        Call the HF processor on the prompt text and
        associated multi-modal data.
        """
1064
1065
        return self.info.ctx.call_hf_processor(
            self.info.get_hf_processor(**mm_kwargs),
1066
1067
            dict(text=prompt, **mm_data),
            mm_kwargs,
1068
1069
        )

1070
    def _hf_processor_applies_updates(
1071
1072
1073
1074
1075
1076
        self,
        prompt_text: str,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> bool:
        """
1077
        Return whether the HF processor applies prompt updates.
1078
1079
1080
1081
1082
1083
1084
1085
1086

        For most HF processors, this should be :code:`True` when multi-modal
        data items are passed, but :code:`False` when multi-modal embeddings
        are passed.
        """
        return not any(
            isinstance(items, (EmbeddingItems, DictEmbeddingItems))
            for items in mm_items.values())

1087
    def _apply_hf_processor_text_mm(
1088
        self,
1089
        prompt_text: str,
1090
        mm_items: MultiModalDataItems,
1091
        hf_processor_mm_kwargs: Mapping[str, object],
1092
    ) -> tuple[list[int], MultiModalKwargs, bool]:
1093
        """
1094
1095
        Apply the HF processor on the prompt text and multi-modal data
        together.
1096

1097
        In addition, return whether prompt updates have been applied.
1098
1099
1100
1101
1102
1103
1104
1105
1106
        """
        processor_data, passthrough_data = self._get_hf_mm_data(mm_items)

        processed_data = self._call_hf_processor(
            prompt=prompt_text,
            mm_data=processor_data,
            mm_kwargs=hf_processor_mm_kwargs,
        )
        processed_data.update(passthrough_data)
1107

1108
        prompt_ids, = processed_data.pop("input_ids").tolist()
1109

1110
1111
1112
        mm_kwargs = MultiModalKwargs.from_hf_inputs(
            processed_data,
            self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs),
1113
        )
1114

1115
        is_update_applied = self._hf_processor_applies_updates(
1116
1117
1118
1119
1120
            prompt_text=prompt_text,
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
        )

1121
        return prompt_ids, mm_kwargs, is_update_applied
1122

1123
    def _apply_hf_processor_text_only(self, prompt_text: str) -> list[int]:
1124
        """
1125
        Apply the HF processor on the prompt text only.
1126

1127
1128
1129
        Since HF processor requires that text and multi-modal items
        correspond to each other, we create dummy multi-modal items
        to go along with the text.
1130
        """
1131
        prompt_ids, _, _ = self._apply_hf_processor_text_mm(
1132
1133
1134
1135
1136
            prompt_text=prompt_text,
            mm_items=MultiModalDataItems({}),
            hf_processor_mm_kwargs={},
        )

1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
        return prompt_ids

    def _apply_hf_processor_tokens_only(
        self,
        prompt_tokens: list[int],
    ) -> list[int]:
        """
        Apply the HF processor on the prompt tokens only.

        Most HF processors accept prompt text but not prompt tokens.
        If the HF processor adds or removes tokens that are not related to
        multi-modal data, you should override this method so it is consistent
        with the output of :meth:`_apply_hf_processor_text_only` on the
        corresponding text.
        """
        return prompt_tokens

    def _apply_hf_processor_mm_only(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> MultiModalKwargs:
        """
        Apply the HF processor on the multi-modal data only.

        Since HF processor requires that text and multi-modal items
        correspond to each other, we generate dummy text using
        :class:`DummyInputsBuilder` to go along with the multi-modal data.
        """
        mm_counts = mm_items.get_all_counts()

1168
1169
        dummy_inputs = self.dummy_inputs.get_dummy_processor_inputs(
            self.info.ctx.model_config.max_model_len,
1170
            mm_counts,
1171
        )
1172

1173
        _, mm_kwargs, _ = self._apply_hf_processor_text_mm(
1174
            prompt_text=dummy_inputs.prompt_text,
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
        )

        return mm_kwargs

    def _apply_hf_processor_main(
        self,
        prompt: Union[str, list[int]],
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        *,
1187
        enable_hf_prompt_update: bool,
1188
    ) -> tuple[list[int], MultiModalKwargs, bool]:
1189
1190
1191
        """
        Apply the HF processor on the prompt text and multi-modal data.

1192
        In addition, return whether prompt updates have been applied
1193
1194
        (for most HF processors, this should be :code:`True`).

1195
        Note:
1196
1197
            If :code:`enable_hf_prompt_update=False`, we use HF processor
            to perform prompt updates if available; HF processor requires
1198
            that the prompt corresponds to multi-modal items.
1199
1200
        """
        if isinstance(prompt, str):
1201
            if enable_hf_prompt_update:
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
                return self._apply_hf_processor_text_mm(
                    prompt_text=prompt,
                    mm_items=mm_items,
                    hf_processor_mm_kwargs=hf_processor_mm_kwargs,
                )

            prompt_ids = self._apply_hf_processor_text_only(prompt)
        else:
            prompt_ids = self._apply_hf_processor_tokens_only(prompt)

1212
        mm_kwargs = self._apply_hf_processor_mm_only(
1213
            mm_items=mm_items,
1214
1215
1216
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
        )

1217
        return prompt_ids, mm_kwargs, False
1218
1219
1220

    def _cached_apply_hf_processor(
        self,
1221
        prompt: Union[str, list[int]],
1222
1223
        mm_data_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1224
    ) -> tuple[list[int], MultiModalKwargs, bool]:
1225
1226
1227
1228
1229
        """
        Apply the HF processor on the full prompt text,
        caching the results and reusing cached results.
        """
        cache = self.cache
1230
        model_id = self.info.model_id
1231

1232
1233
        _, passthrough_data = self._get_hf_mm_data(mm_data_items)
        if cache is None or passthrough_data:
1234
1235
            return self._apply_hf_processor_main(
                prompt=prompt,
1236
1237
                mm_items=mm_data_items,
                hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1238
                enable_hf_prompt_update=True,
1239
1240
            )

1241
        mm_maybe_cached_kw_items = {
1242
1243
1244
1245
1246
1247
1248
1249
            modality: [
                cache.get(model_id, modality, item, hf_processor_mm_kwargs)
                for item in items
            ]
            for modality, items in mm_data_items.items()
        }

        mm_missing_idxs = {
1250
1251
1252
            modality:
            [idx for idx, item in enumerate(kw_items) if item is None]
            for modality, kw_items in mm_maybe_cached_kw_items.items()
1253
1254
1255
1256
1257
        }
        mm_missing_data = {
            modality: [mm_data_items[modality][idx] for idx in idxs]
            for modality, idxs in mm_missing_idxs.items()
        }
1258
        mm_missing_data_items = self._to_mm_items(mm_missing_data)
1259

1260
        # NOTE: `prompt` does not correspond to `mm_missing_data_items`,
1261
        # so we can't apply prompt updates until the new multimodal
1262
1263
1264
1265
        # items are combined with the cached multimodal items
        (
            prompt_ids,
            mm_missing_kwargs,
1266
            is_update_applied,
1267
        ) = self._apply_hf_processor_main(
1268
1269
            prompt=prompt,
            mm_items=mm_missing_data_items,
1270
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1271
            enable_hf_prompt_update=False,
1272
1273
1274
1275
1276
1277
1278
        )

        mm_missing_next_idx = {
            modality: 0
            for modality in mm_missing_data_items
        }

1279
1280
1281
1282
1283
        merged_kw_items = list[MultiModalKwargsItem]()
        for modality, kw_items in mm_maybe_cached_kw_items.items():
            for idx, kw_item in enumerate(kw_items):
                if kw_item is None:
                    kw_item = mm_missing_kwargs.get_item(
1284
1285
1286
1287
1288
1289
1290
1291
1292
                        modality,
                        mm_missing_next_idx[modality],
                    )

                    cache.put(
                        model_id,
                        modality,
                        mm_data_items[modality][idx],
                        hf_processor_mm_kwargs,
1293
                        kw_item,
1294
1295
1296
1297
                    )

                    mm_missing_next_idx[modality] += 1

1298
                merged_kw_items.append(kw_item)
1299
1300

        if self.enable_sanity_checks:
1301
            mm_missing_counts = mm_missing_data_items.get_all_counts()
1302
1303
1304
1305
1306
1307
            assert all(
                item_count == mm_missing_counts[modality]
                for modality, item_count in mm_missing_next_idx.items()), dict(
                    mm_missing_next_idx=mm_missing_next_idx,
                    mm_missing_counts=mm_missing_counts)

1308
        mm_kwargs = MultiModalKwargs.from_items(merged_kw_items)
1309

1310
        return prompt_ids, mm_kwargs, is_update_applied
1311

1312
    def _bind_and_group_updates(
1313
        self,
1314
1315
        prompt_updates: Sequence[PromptUpdate],
    ) -> dict[str, Sequence[BoundPromptUpdate]]:
1316
        tokenizer = self.info.get_tokenizer()
1317

1318
        it = (update.bind(tokenizer) for update in prompt_updates)
1319
        return dict(full_groupby_modality(it))
1320

1321
    def _apply_prompt_updates(
1322
1323
        self,
        token_ids: list[int],
1324
        mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
1325
        mm_item_counts: Mapping[str, int],
1326
    ) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
1327
        tokenizer = self.info.get_tokenizer()
1328

1329
        mm_token_matches = {
1330
1331
            modality: find_token_matches(token_ids, updates)
            for modality, updates in mm_prompt_updates.items()
1332
        }
1333
1334
        mm_match_counts = {
            modality: len(matches)
1335
            for modality, matches in mm_token_matches.items()
1336
        }
1337
1338
1339
1340
1341
1342
1343
1344
1345

        # If the search text does not represent a special token,
        # it may have different token IDs in the prompt, because
        # the tokens may go across the boundaries of the search text.
        # ----
        # e.g. when searching for "foo" in "food", if "food" itself makes
        # up a token, then the token ID of "foo" will not appear at all
        # ----
        # Since it is inefficient to search for all possible tokenizations
1346
1347
        # of the search text in the prompt, we instead perform string-based
        # updates on the decoded token IDs, then encode them back.
1348
        if all(
1349
1350
            mm_match_counts.get(modality, 0) >= item_count
            for modality, item_count in mm_item_counts.items()
1351
        ):  # yapf: disable
1352
            token_ids = apply_token_matches(
1353
                token_ids,
1354
                mm_token_matches,
1355
                mm_item_counts,
1356
1357
            )

1358
            text = decode_tokens(tokenizer, token_ids)
1359
1360
            matched_updates = {
                modality: [match._origin for match in token_matches]
1361
1362
                for modality, token_matches in mm_token_matches.items()
            }
1363
        else:
1364
            text = decode_tokens(tokenizer, token_ids)
1365

1366
            mm_text_matches = {
1367
1368
                modality: find_text_matches(text, updates)
                for modality, updates in mm_prompt_updates.items()
1369
            }
1370
            text = apply_text_matches(
1371
                text,
1372
                mm_text_matches,
1373
                mm_item_counts,
1374
1375
            )

1376
1377
1378
            token_ids = encode_tokens(tokenizer,
                                      text,
                                      add_special_tokens=False)
1379
1380
            matched_updates = {
                modality: [match._origin for match in token_matches]
1381
1382
1383
1384
                for modality, token_matches in mm_text_matches.items()
            }

        placeholders = self._find_mm_placeholders(
1385
            matched_updates,
1386
1387
1388
            token_ids,
            mm_item_counts,
        )
1389
1390

        return token_ids, text, placeholders
1391

1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
    def _validate_mm_kwargs(
        self,
        mm_kwargs: MultiModalKwargs,
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
            if modality in mm_kwargs.modalities:
                items = mm_kwargs.get_items(modality)
            else:
                items = []

            if len(items) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} {modality} items in "
                    f"keyword arguments corresponding to {item_count} "
                    f"{modality} data items, but only found {len(items)}! "
                    "There is likely a problem with your "
                    "implementation of merged multi-modal processor for this "
                    "model (usually arising from an inconsistency between "
                    "`_call_hf_processor` and `_get_mm_fields_config`).")

    def _validate_mm_placeholders(
        self,
1415
        mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
1416
        mm_item_counts: Mapping[str, int],
1417
    ) -> None:
1418
1419
1420
        for modality, item_count in mm_item_counts.items():
            placeholders = mm_placeholders.get(modality, [])

1421
            if len(placeholders) != item_count:
1422
                raise RuntimeError(
1423
                    f"Expected there to be {item_count} prompt updates "
1424
                    f"corresponding to {item_count} {modality} items, but "
1425
                    f"instead found {len(placeholders)} prompt updates! "
1426
                    "Either the prompt text has missing/incorrect tokens for "
1427
1428
1429
                    "multi-modal inputs, or there is a problem with your "
                    "implementation of merged multi-modal processor for this "
                    "model (usually arising from an inconsistency between "
1430
                    "`_call_hf_processor` and `_get_prompt_updates`).")
1431

1432
1433
    def apply(
        self,
1434
        prompt: Union[str, list[int]],
1435
        mm_data: MultiModalDataDict,
1436
        hf_processor_mm_kwargs: Mapping[str, object],
1437
        return_mm_hashes: bool = False,
1438
    ) -> MultiModalInputs:
1439
1440
1441
1442
1443
1444
1445
        """
        Process multi-modal inputs to be used in vLLM.

        The main steps are:

        1. Apply HF Processor on prompt text and multi-modal data together,
           outputting token IDs and processed tensors.
1446
        2. Find and update sequences in the token IDs with placeholder tokens.
1447
1448
1449
1450
1451
           The number of placeholder tokens equals the feature size of the
           multi-modal data outputted by the multi-modal encoder.
        3. Extract information about the placeholder tokens from the
           processed token IDs.
        """
1452
        mm_items = self._to_mm_items(mm_data)
1453

1454
        # Create MM hashes to be returned (only used in V1)
1455
1456
1457
        # TODO: Use these hash keys for caching operations in apply_hf_processor
        # instead of rehashing.

1458
        if return_mm_hashes:
1459
            model_id = self.info.model_id
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
            mm_hashes = {
                modality: [
                    MultiModalHasher.hash_kwargs(model_id=model_id,
                                                 **{modality: item},
                                                 **hf_processor_mm_kwargs)
                    for item in items
                ]
                for modality, items in mm_items.items()
            }
        else:
            mm_hashes = None

1472
1473
1474
        (
            prompt_ids,
            mm_kwargs,
1475
            is_update_applied,
1476
        ) = self._cached_apply_hf_processor(
1477
            prompt,
1478
1479
1480
            mm_items,
            hf_processor_mm_kwargs,
        )
1481

1482
        unbound_prompt_updates = self._get_prompt_updates(
1483
1484
1485
1486
            mm_items,
            hf_processor_mm_kwargs,
            mm_kwargs,
        )
1487
1488
        mm_prompt_updates = self._bind_and_group_updates(
            unbound_prompt_updates)
1489

1490
        mm_item_counts = mm_items.get_all_counts()
1491
1492
        self._validate_mm_kwargs(mm_kwargs, mm_item_counts)

1493
        if is_update_applied:
1494
            mm_placeholders = self._find_mm_placeholders(
1495
                mm_prompt_updates,
1496
                prompt_ids,
1497
1498
                mm_item_counts,
            )
1499
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1500

1501
            tokenizer = self.info.get_tokenizer()
1502
            prompt = decode_tokens(tokenizer, prompt_ids)
1503
1504
1505
        else:
            (
                prompt_ids,
1506
                prompt,
1507
                mm_placeholders,
1508
            ) = self._apply_prompt_updates(
1509
                prompt_ids,
1510
                mm_prompt_updates,
1511
                mm_item_counts,
1512
            )
1513
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1514
1515
1516
1517
1518

        mm_placeholder_ranges = {
            modality: [item.to_range() for item in placeholders]
            for modality, placeholders in mm_placeholders.items()
        }
1519

1520
        return MultiModalInputs(
1521
            type="multimodal",
1522
            prompt=prompt,
1523
            prompt_token_ids=prompt_ids,
1524
            mm_kwargs=mm_kwargs,
1525
            mm_hashes=mm_hashes,
1526
            mm_placeholders=mm_placeholder_ranges,
1527
        )
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537


class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):

    @abstractmethod
    def create_encoder_prompt(
        self,
        prompt: Union[str, list[int]],
        mm_data: MultiModalDataDict,
    ) -> Union[str, list[int]]:
1538
1539
1540
1541
        """
        Create input prompt for the encoder. HF processor will be applied on 
        this prompt during profiling and generation.
        """
1542
1543
        raise NotImplementedError

1544
1545
1546
1547
1548
1549
1550
1551
    def create_decoder_prompt(
        self,
        prompt: Union[str, list[int]],
        mm_data: MultiModalDataDict,
    ) -> Union[str, list[int]]:
        """Create input prompt for the decoder."""
        return prompt

1552
1553
1554
1555
1556
    def apply(
        self,
        prompt: Union[str, list[int]],
        mm_data: MultiModalDataDict,
        hf_processor_mm_kwargs: Mapping[str, object],
1557
        return_mm_hashes: bool = False,
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
    ) -> MultiModalEncDecInputs:
        """
        Process multi-modal inputs to be used in vLLM.
        The main processing steps are modified to fit encoder-decoder model:
        1. Create encoder prompt from input prompt text.
        2. Apply the HF processor on encoder prompt.
        3. Copy the input prompt text as decoder prompt inputs.
        """
        encoder_prompt = self.create_encoder_prompt(prompt, mm_data)
        encoder_inputs = super().apply(
            encoder_prompt,
            mm_data,
            hf_processor_mm_kwargs,
1571
            return_mm_hashes,
1572
1573
1574
        )

        tokenizer = self.info.get_tokenizer()
1575
1576
        decoder_prompt = self.create_decoder_prompt(prompt, mm_data)
        if isinstance(decoder_prompt, str):
1577
            decoder_prompt_ids = encode_tokens(tokenizer,
1578
                                               decoder_prompt,
1579
1580
                                               add_special_tokens=False)
        else:
1581
1582
            decoder_prompt_ids = decoder_prompt
            decoder_prompt = decode_tokens(tokenizer, decoder_prompt)
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592

        mm_inputs = MultiModalEncDecInputs(
            encoder_prompt=encoder_inputs["prompt"],
            encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],
            **encoder_inputs)
        mm_inputs.update({
            "prompt": decoder_prompt,
            "prompt_token_ids": decoder_prompt_ids
        })
        return mm_inputs