processing.py 69.4 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import time
4
from abc import ABC, abstractmethod
5
from collections import defaultdict
6
from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence
7
from dataclasses import dataclass, field, replace
8
from enum import Enum
9
from functools import lru_cache
10
11
12
13
14
15
from typing import (
    TYPE_CHECKING,
    Any,
    Generic,
    NamedTuple,
    Protocol,
16
    TypeAlias,
17
18
19
    cast,
    overload,
)
20

21
import regex as re
22
import torch
23
from typing_extensions import TypeVar, assert_never
24

25
from vllm.logger import init_logger
26
from vllm.transformers_utils.processor import cached_processor_from_config
27
28
from vllm.transformers_utils.tokenizer import AnyTokenizer, decode_tokens, encode_tokens
from vllm.utils import flatten_2d_lists, full_groupby, get_allowed_kwarg_only_overrides
29
from vllm.utils.jsontree import JSONTree, json_map_leaves
30

31
from .hasher import MultiModalHasher
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
from .inputs import (
    MultiModalDataDict,
    MultiModalEncDecInputs,
    MultiModalFieldConfig,
    MultiModalInputs,
    MultiModalKwargsItem,
    MultiModalKwargsItems,
    MultiModalKwargsOptionalItems,
    MultiModalUUIDDict,
    PlaceholderRange,
)
from .parse import (
    DictEmbeddingItems,
    EmbeddingItems,
    MultiModalDataItems,
    MultiModalDataParser,
)
49
50

if TYPE_CHECKING:
51
52
53
54
    from transformers.configuration_utils import PretrainedConfig
    from transformers.feature_extraction_utils import BatchFeature
    from transformers.processing_utils import ProcessorMixin

55
56
    from vllm.config import ModelConfig

57
    from .cache import BaseMultiModalProcessorCache
58
    from .profiling import BaseDummyInputsBuilder
59
60
61
62
63
64
65
66
else:
    PretrainedConfig = object
    BatchFeature = object
    ProcessorMixin = object

    ModelConfig = object

    BaseMultiModalProcessorCache = object
67

68
logger = init_logger(__name__)
69
70

_S = TypeVar("_S", str, list[int])
71

72
PromptSeq: TypeAlias = str | list[int]
73
"""A token sequence (list of token IDs) or text."""
74

75

76
77
78
79
80
@lru_cache(maxsize=2048)
def _cached_encode(
    tokenizer: AnyTokenizer,
    text: str,
    *,
81
    add_special_tokens: bool | None = None,
82
) -> list[int]:
83
    return encode_tokens(tokenizer, text, add_special_tokens=add_special_tokens)
84
85
86
87
88
89
90


@lru_cache(maxsize=2048)
def _cached_decode(
    tokenizer: AnyTokenizer,
    token_ids: tuple[int, ...],
    *,
91
    skip_special_tokens: bool | None = None,
92
) -> str:
93
94
95
    return decode_tokens(
        tokenizer, list(token_ids), skip_special_tokens=skip_special_tokens
    )
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111


def _seq2text(tokenizer: AnyTokenizer, seq: PromptSeq) -> str:
    if isinstance(seq, str):
        return seq

    return _cached_decode(tokenizer, tuple(seq))


def _seq2tokens(tokenizer: AnyTokenizer, seq: PromptSeq) -> list[int]:
    if isinstance(seq, str):
        return _cached_encode(tokenizer, seq, add_special_tokens=False)

    return seq


112
113
114
115
116
117
class _GetMatchIndex(Protocol):
    def __call__(
        self,
        tokenizer: AnyTokenizer,
        prompt: PromptSeq,
        start_idx: int = 0,
118
    ) -> int | None: ...
119
120


121
122
123
@dataclass
class PromptIndex:
    """Resolves to an index in the prompt."""
124

125
    get_match_index: _GetMatchIndex
126
127
128
129
130
131
132
133
134
135


class PromptIndexTargets:
    @staticmethod
    def start() -> PromptIndex:
        """
        Resolves to the start of the prompt (before the first token).

        This results in a match even if the prompt is empty.
        """
136
        return PromptIndex(lambda tokenizer, prompt, start_idx=0: 0)
137
138
139
140
141
142
143
144
145
146

    @staticmethod
    def prefix(seq: PromptSeq) -> PromptIndex:
        """
        Resolves to a location in the prompt after the given prefix.
        """

        def get_match_index(
            tokenizer: AnyTokenizer,
            prompt: PromptSeq,
147
            start_idx: int = 0,
148
        ) -> int | None:
149
150
151
            if start_idx != 0:
                return None

152
153
154
155
156
157
158
159
160
            prefix = seq

            if isinstance(prompt, str):
                if not isinstance(prefix, str):
                    # Make both `str`
                    prefix = decode_tokens(tokenizer, prefix)
            else:
                if isinstance(prefix, str):
                    # Make both `list[int]`
161
                    prefix = encode_tokens(tokenizer, prefix, add_special_tokens=False)
162
163
164
165
166
167
168
169
170
171
172
173
174

            match_idx = len(prefix)
            return match_idx if prompt[:match_idx] == prefix else None

        return PromptIndex(get_match_index)

    @staticmethod
    def end() -> PromptIndex:
        """
        Resolves to the end of the prompt (after the last token).

        This results in a match even if the prompt is empty.
        """
175
        return PromptIndex(lambda tokenizer, prompt, start_idx=0: len(prompt))
176
177


178
UpdateTarget: TypeAlias = PromptSeq | PromptIndex
179
180
181
182
"""
The token sequence or text to update.
"""

183
PromptUpdateTarget: TypeAlias = Callable[[int], UpdateTarget] | UpdateTarget
184
185
186
187
188
189
190
191
192
"""
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
output the corresponding token sequence (or text).

For convenience, you can directly pass in the token sequence (or text)
instead of a function if it does not depend on the input.
"""

193

194
@dataclass
195
class PromptUpdateDetails(Generic[_S]):
196
    """Details about the token sequence or text that are part of the update."""
197

198
    full: _S
199
    """The full content."""
200

201
    is_embed: Callable[[AnyTokenizer, PromptSeq], torch.Tensor] | None = None
202
    """
203
204
205
    Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full],
    return a boolean mask of shape `(len(full),)` indicating which positions
    of `full` to assign embeddings to.
206
207
208
209

    `None` (default) means to assign embeddings to all positions of `full`.

    The embeddings are obtained by calling
210
    [`SupportsMultiModal.get_multimodal_embeddings`][vllm.model_executor.models.interfaces.SupportsMultiModal.get_multimodal_embeddings].
211
212
213
    """

    @staticmethod
214
    def from_seq(seq: _S) -> "PromptUpdateDetails[_S]":
215
216
217
218
219
220
221
        return PromptUpdateDetails(full=seq)

    @staticmethod
    def select_text(
        seq: _S,
        embed_text: str,
    ) -> "PromptUpdateDetails[_S]":
222
223
224
        def is_embed(tokenizer: AnyTokenizer, full: PromptSeq) -> torch.Tensor:
            embed_token_ids = encode_tokens(tokenizer, embed_text)
            token_ids = _seq2tokens(tokenizer, full)
225
226

            return torch.isin(
227
                torch.tensor(token_ids),
228
229
230
231
232
233
234
235
236
237
                torch.tensor(embed_token_ids),
            )

        return PromptUpdateDetails(full=seq, is_embed=is_embed)

    @staticmethod
    def select_token_id(
        seq: _S,
        embed_token_id: int,
    ) -> "PromptUpdateDetails[_S]":
238
239
240
241
242
243
        def is_embed(tokenizer: AnyTokenizer, full: PromptSeq) -> torch.Tensor:
            token_ids = _seq2tokens(tokenizer, full)

            return torch.tensor(token_ids) == embed_token_id

        return PromptUpdateDetails(full=seq, is_embed=is_embed)
244
245


246
PromptUpdateInfo: TypeAlias = PromptSeq | PromptUpdateDetails
247
"""
248
The token sequence or text that are part of the update.
249

250
If only part of the content corresponds to feature placeholders, you can
251
252
use [`PromptUpdateDetails`][vllm.multimodal.processing.PromptUpdateDetails] to
specify which part.
253
"""
254

255
PromptUpdateContent: TypeAlias = Callable[[int], PromptUpdateInfo] | PromptUpdateInfo
256
"""
257
258
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
259
260
261
262
263
264
265
266
267
268
269
270
271
output the corresponding token sequence (or text).

For convenience, you can directly pass in the token sequence (or text)
instead of a function if it does not depend on the input.
"""


class UpdateMode(str, Enum):
    INSERT = "insert"
    REPLACE = "replace"


@dataclass
272
class PromptUpdate(ABC):
273
274
275
276
277
278
279
    """
    Defines how to update a prompt with placeholder tokens.
    """

    modality: str
    """The modality for which the update is made."""

280
    target: PromptUpdateTarget
281
282
283
284
285
286
287
288
289
290
291
292
293
294
    """The token sequence (or text) to update."""

    @property
    @abstractmethod
    def content(self) -> PromptUpdateContent:
        """The placeholder tokens that are part of the update."""
        raise NotImplementedError

    @property
    @abstractmethod
    def mode(self) -> UpdateMode:
        """Defines how to update the prompt."""
        raise NotImplementedError

295
    def _resolve_target(self, item_idx: int) -> UpdateTarget:
296
297
298
299
        target = self.target
        if callable(target):
            target = target(item_idx)

300
        return target
301

302
    def _resolve_content(self, item_idx: int) -> PromptUpdateDetails:
303
304
305
306
307
308
309
        content = self.content
        if callable(content):
            content = content(item_idx)

        if not isinstance(content, PromptUpdateDetails):
            content = PromptUpdateDetails.from_seq(content)

310
        return content
311

312
    def resolve(self, item_idx: int) -> "ResolvedPromptUpdate":
313
314
315
316
317
318
319
320
321
        """
        Given the index of the processed item within
        [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
        output a copy of this object with its lazy attributes resolved.
        """
        return ResolvedPromptUpdate(
            modality=self.modality,
            item_idx=item_idx,
            mode=self.mode,
322
323
            target=self._resolve_target(item_idx),
            content=self._resolve_content(item_idx),
324
325
        )

326

327
@dataclass
328
329
330
331
332
333
class PromptInsertion(PromptUpdate):
    """
    Defines how to insert placeholder tokens into a prompt.

    Example:

334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
    For each image, insert a number of ``<image>`` feature placeholders
    equal to the feature size of the vision encoder after the ``<s>`` token:

    ```python
    PromptInsertion(
        modality="image",
        target="<s>",
        insertion="<image>" * image_feature_size,
    )
    ```

    Insert these tokens at the start of the prompt:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.start(),
        insertion="<image>" * image_feature_size,
    )
    ```

    Insert these tokens after a prefix ``Images:``:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.prefix("Images:"),
        insertion="<image>" * image_feature_size,
    )
    ```

    Insert these tokens at the end of the prompt:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.end(),
        insertion="<image>" * image_feature_size,
    )
    ```
374
375
376
377
    """

    insertion: PromptUpdateContent = field(repr=False)
    """
378
379
380
381
    Given the index of the processed item within
    [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
    output the token sequence (or text) to insert right after
    [`target`][vllm.multimodal.processing.PromptUpdate.target].
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397

    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
    """

    @property
    def content(self) -> PromptUpdateContent:
        return self.insertion

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.INSERT


@dataclass
class PromptReplacement(PromptUpdate):
398
399
    """
    Defines how to replace portions of an input prompt with placeholder tokens.
400
401
402

    Example:

403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
    For each image, replace one ``<image>`` input placeholder in the prompt
    with a number of ``<image>`` feature placeholders
    equal to the feature size of the vision encoder:

    ```python
    PromptReplacement(
        modality="image",
        target="<image>",
        replacement="<image>" * image_feature_size,
    )
    ```

    As above, but further pad the feature placeholders with ``<image_bos>``
    and `<image_eos>``, which are not supposed to be passed to the vision
    encoder:

    ```python
    PromptReplacement(
        modality="image",
        target="<image>",
        replacement=PromptUpdateDetails(
424
425
426
427
428
429
430
            full="".join(
                [
                    "<image_bos>",
                    "<image>" * image_feature_size,
                    "<image_eos>",
                ]
            ),
431
432
433
434
435
436
437
438
439
440
441
442
443
            features="<image>" * image_feature_size,
        ),
    )
    ```

    To avoid unnecessary tokenization during prompt replacement,
    we recommended passing token sequences instead of text:

    ```python
    PromptReplacement(
        modality="image",
        target=[image_token_id],
        replacement=PromptUpdateDetails(
444
445
446
            full=(
                [image_bos_id] + [image_token_id] * image_feature_size + [image_eos_id]
            ),
447
448
449
450
            features=[image_token_id] * image_feature_size,
        ),
    )
    ```
451
452
    """

453
    replacement: PromptUpdateContent = field(repr=False)
454
    """
455
456
457
458
    Given the index of the processed item within
    [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
    output the token sequence (or text) to replace
    [`target`][vllm.multimodal.processing.PromptUpdate.target].
459

460
461
    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
462
463
    """

464
465
466
467
468
469
470
    @property
    def content(self) -> PromptUpdateContent:
        return self.replacement

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.REPLACE
471
472


473
474
475
class _HasModalityAttr(Protocol):
    modality: str

476

477
478
class _HasModalityProp(Protocol):
    @property
479
    def modality(self) -> str: ...
480
481


482
_M = TypeVar("_M", bound=_HasModalityAttr | _HasModalityProp)
483
484
485


def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]:
486
487
    """Convenience function to apply [`full_groupby`][vllm.utils.full_groupby]
    based on modality."""
488
489
490
    return full_groupby(values, key=lambda x: x.modality)


491
492
493
494
495
496
497
class PromptTargetMatch(NamedTuple):
    start_idx: int
    end_idx: int


@dataclass(frozen=True)
class ResolvedPromptUpdate:
498
    """
499
500
    A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] with its
    lazy attributes resolved, apart from those related to tokenization.
501
    """
502

503
504
    modality: str
    """The modality for which the update is made."""
505

506
507
    item_idx: int
    """The index within `modality` of the item this update pertains to."""
508

509
510
    mode: UpdateMode
    """Defines how to update the prompt."""
511

512
    target: UpdateTarget
513
    """The token sequence (or text) to update."""
514

515
    content: PromptUpdateDetails = field(repr=False)
516
    """The placeholder tokens that are part of the update."""
517

518
519
520
521
522
523
524
525
526
    def iter_token_matches(
        self,
        prompt: list[int],
        tokenizer: AnyTokenizer,
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        target = self.target
527

528
529
530
531
        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(tokenizer, prompt, start_idx)
            if match_idx is not None:
                yield PromptTargetMatch(match_idx, match_idx)
532

533
            return
534

535
536
        target_token_ids = _seq2tokens(tokenizer, target)

537
        for match in iter_token_matches(prompt, target_token_ids, start_idx=start_idx):
538
            yield PromptTargetMatch(match.start_idx, match.end_idx)
539

540
541
542
543
544
545
546
547
548
    def iter_text_matches(
        self,
        prompt: str,
        tokenizer: AnyTokenizer,
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        target = self.target
549

550
551
552
553
        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(tokenizer, prompt, start_idx)
            if match_idx is not None:
                yield PromptTargetMatch(match_idx, match_idx)
554

555
            return
556

557
558
        target_text = _seq2text(tokenizer, target)

559
        for match in re.finditer(re.escape(target_text), prompt, pos=start_idx):
560
561
562
563
            yield PromptTargetMatch(match.start(), match.end())

    def iter_matches(
        self,
564
        prompt: list[int] | str,
565
566
567
568
569
570
        tokenizer: AnyTokenizer,
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        if isinstance(prompt, str):
571
            return self.iter_text_matches(prompt, tokenizer, start_idx=start_idx)
572
573

        return self.iter_token_matches(prompt, tokenizer, start_idx=start_idx)
574

575
576
577
578
579
580
581
582
583
    def with_target(self, target: UpdateTarget):
        return replace(self, target=target)

    def with_content(self, content: PromptUpdateInfo):
        if not isinstance(content, PromptUpdateDetails):
            content = PromptUpdateDetails.from_seq(content)

        return replace(self, content=content)

584

585
586
587
class _TokenMatch(NamedTuple):
    start_idx: int
    end_idx: int
588
589


590
591
592
def iter_token_matches(
    token_ids: list[int],
    match_ids: list[int],
593
594
    *,
    start_idx: int = 0,
595
) -> Generator[_TokenMatch]:
596
    """
597
    Yield each occurrence of `match_ids` in `token_ids`.
598
599
600
601

    Note that empty matches are ignored.
    """
    prompt_len = len(token_ids)
602
    match_len = len(match_ids)
603

604
605
    if match_len == 0:
        return
606

607
    while start_idx < prompt_len - match_len + 1:
608
        end_idx = start_idx + match_len
609

610
611
        if token_ids[start_idx:end_idx] == match_ids:
            yield _TokenMatch(start_idx=start_idx, end_idx=end_idx)
612
613
614
615
616

            # Exclude overlapping matches
            start_idx = end_idx
        else:
            start_idx += 1
617
618


619
620
621
622
623
624
def replace_token_matches(
    token_ids: list[int],
    match_ids: list[int],
    new_ids: list[int],
) -> list[int]:
    """
625
626
    Replace each occurrence of `match_ids` in `token_ids`
    with `new_ids`.
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645

    Note that empty matches are ignored.
    """
    out_seqs = list[list[int]]()
    prev_end_idx = 0

    for match in iter_token_matches(token_ids, match_ids):
        start_idx = match.start_idx
        end_idx = match.end_idx

        out_seqs.append(token_ids[prev_end_idx:start_idx])
        out_seqs.append(new_ids)
        prev_end_idx = end_idx

    out_seqs.append(token_ids[prev_end_idx:])

    return flatten_2d_lists(out_seqs)


646
@dataclass
647
class PlaceholderFeaturesInfo:
648
    modality: str
649
    item_idx: int
650
    start_idx: int
651
    tokens: list[int]
652
    is_embed: torch.Tensor | None
653
654
655

    @property
    def length(self) -> int:
656
        return len(self.tokens)
657
658

    def to_range(self) -> PlaceholderRange:
659
660
        # TODO: Is it worth it to optimize this by stripping the
        # leading and ending positions where `is_embed=False`?
661
662
663
        return PlaceholderRange(
            offset=self.start_idx,
            length=self.length,
664
            is_embed=self.is_embed,
665
        )
666
667


668
_MatchToApply = tuple[tuple[str, int], tuple[PromptTargetMatch, int]]
669
670


671
672
673
674
675
676
677
def _find_matches(
    prompt: _S,
    mm_prompt_updates: "MultiModalPromptUpdates",
    tokenizer: AnyTokenizer,
    *,
    prev_end_idx: int = 0,
    current_result: "MultiModalPromptUpdatesApplyResult",
678
679
) -> tuple[UpdateMode | None, list[_MatchToApply]]:
    mode: UpdateMode | None = None
680
681
682
683
684
685
686
687
688
689
690
691
    mm_matches = dict[tuple[str, int], tuple[PromptTargetMatch, int]]()

    for modality, modality_updates in mm_prompt_updates.items():
        for item_idx, item_updates in enumerate(modality_updates):
            if current_result[modality][item_idx] is not None:
                continue  # Updates have already been applied for this item

            for update_idx, update in enumerate(item_updates):
                if (modality, item_idx) in mm_matches:
                    break  # Already found a match for this item

                for match in update.iter_matches(
692
693
694
                    prompt,
                    tokenizer,
                    start_idx=prev_end_idx,
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
                ):
                    # All matches should share the same mode
                    if mode is None:
                        mode = update.mode
                    elif mode != update.mode:
                        continue

                    mm_matches[(modality, item_idx)] = match, update_idx
                    break  # Get only the first valid match per item

    # Prioritize earlier matches
    matches_to_apply = sorted(mm_matches.items(), key=lambda item: item[1][0])

    # To avoid conflicts, only replace one non-empty item at a time
    if mode == UpdateMode.REPLACE:
        matches_to_apply_ = list[_MatchToApply]()
        has_non_empty_matches = False

        for item in matches_to_apply:
            _, (match, _) = item
            if match.start_idx == match.end_idx:
                matches_to_apply_.append(item)
            elif not has_non_empty_matches:
                has_non_empty_matches = True
                matches_to_apply_.append(item)

        matches_to_apply = matches_to_apply_

    return mode, matches_to_apply
724
725


726
def _apply_matches(
727
    prompt: _S,
728
729
730
731
732
    mm_prompt_updates: "MultiModalPromptUpdates",
    tokenizer: AnyTokenizer,
) -> tuple[list[_S], "MultiModalPromptUpdatesApplyResult"]:
    prompt_len = len(prompt)

733
    out_seqs = list[str | list[int]]()
734
    out_result: MultiModalPromptUpdatesApplyResult = {
735
        m: [None] * len(items) for m, items in mm_prompt_updates.items()
736
    }
737

738
739
740
    start_idx = prev_end_idx = 0
    while start_idx < max(prompt_len, 1):  # Allow inserts into empty prompt
        found = False
741

742
743
744
745
746
747
748
        mode, matches_to_apply = _find_matches(
            prompt,
            mm_prompt_updates,
            tokenizer,
            prev_end_idx=prev_end_idx,
            current_result=out_result,
        )
749

750
751
752
        if mode is not None:
            for (modality, item_idx), (match, update_idx) in matches_to_apply:
                found = True
753

754
                matched_update = mm_prompt_updates[modality][item_idx][update_idx]
755
                matched_content = matched_update.content.full
756

757
758
759
760
761
762
                if mode == UpdateMode.INSERT:
                    end_idx_to_insert = match.end_idx
                elif mode == UpdateMode.REPLACE:
                    end_idx_to_insert = match.start_idx
                else:
                    assert_never(mode)
763

764
                out_seqs.append(prompt[prev_end_idx:end_idx_to_insert])
765
                out_seqs.append(
766
767
768
769
                    _seq2text(tokenizer, matched_content)
                    if isinstance(prompt, str)
                    else _seq2tokens(tokenizer, matched_content)
                )
770
                out_result[modality][item_idx] = update_idx
771

772
773
774
775
776
                # Exclude overlapping matches
                start_idx = prev_end_idx = match.end_idx

        if not found:
            start_idx += 1
777
778
779

    out_seqs.append(prompt[prev_end_idx:])

780
    return cast(list[_S], out_seqs), out_result
781
782


783
def apply_token_matches(
784
    prompt: list[int],
785
786
787
788
789
    mm_prompt_updates: "MultiModalPromptUpdates",
    tokenizer: AnyTokenizer,
) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
    """
    Apply the updates in `mm_prompt_updates` to `prompt`.
790

791
792
793
794
    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
    appears earlier in `mm_prompt_updates` takes priority.
    """
795
    token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
796

797
    return flatten_2d_lists(token_id_seqs), result
798
799


800
def apply_text_matches(
801
    prompt: str,
802
803
804
805
806
    mm_prompt_updates: "MultiModalPromptUpdates",
    tokenizer: AnyTokenizer,
) -> tuple[str, "MultiModalPromptUpdatesApplyResult"]:
    """
    Apply the updates in `mm_prompt_updates` to `prompt`.
807

808
809
810
811
812
    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
    appears earlier in `mm_prompt_updates` takes priority.
    """
    texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
813

814
    return "".join(texts), result
815
816


817
def _iter_placeholders(
818
    prompt: list[int],
819
    mm_prompt_updates: "MultiModalPromptUpdates",
820
    tokenizer: AnyTokenizer,
821
) -> Iterable[PlaceholderFeaturesInfo]:
822
    """
823
    Yield each set of placeholder tokens found in `prompt`.
824
825
826

    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
827
    appears earlier in `mm_prompt_updates` takes priority.
828

829
830
    Note that empty matches are ignored.
    """
831
    prompt_len = len(prompt)
832
833
    mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()}

834
    item_idx_by_modality = defaultdict[str, int](lambda: 0)
835
836
837
838
839

    start_idx = 0
    while start_idx < prompt_len:
        found = False

840
        for modality, modality_updates in mm_prompt_updates.items():
841
842
            item_idx = item_idx_by_modality[modality]
            if item_idx >= mm_item_counts.get(modality, 0):
843
                continue
844

845
846
            for update in modality_updates[item_idx]:
                content = update.content
847
                content_tokens_full = _seq2tokens(tokenizer, content.full)
848
849
                content_len_full = len(content_tokens_full)
                end_idx_full = start_idx + content_len_full
850

851
                if content_len_full == 0 or end_idx_full > prompt_len:
852
853
                    continue

854
                if prompt[start_idx:end_idx_full] == content_tokens_full:
855
856
                    content_is_embed = content.is_embed
                    if content_is_embed is not None:
857
                        content_is_embed = content_is_embed(tokenizer, content.full)
858
859
860
861
862
863
864
865

                    yield PlaceholderFeaturesInfo(
                        modality=modality,
                        item_idx=item_idx,
                        start_idx=start_idx,
                        tokens=content_tokens_full,
                        is_embed=content_is_embed,
                    )
866

867
                    # Exclude overlapping matches
868
                    start_idx = end_idx_full
869
870
871
                    item_idx_by_modality[modality] += 1
                    found = True
                    break
872

873
874
            if found:
                break  # Go back to the outer while loop
875
876
877

        if not found:
            start_idx += 1
878
879


880
881
def find_mm_placeholders(
    prompt: list[int],
882
    mm_prompt_updates: "MultiModalPromptUpdates",
883
    tokenizer: AnyTokenizer,
884
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
885
    it = _iter_placeholders(prompt, mm_prompt_updates, tokenizer)
886
887
888
    return dict(full_groupby_modality(it))


889
_T = TypeVar("_T")
890
891
_C = TypeVar("_C", bound=PretrainedConfig, default=PretrainedConfig)
_P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin)
892
893
894
895
896
897
898
899
900


@dataclass(frozen=True)
class InputProcessingContext:
    """
    Contains information about the model which may be used to
    modify the inputs.
    """

901
    model_config: ModelConfig
902
903
904
905
906
907
    """The configuration of the model."""

    tokenizer: AnyTokenizer
    """The tokenizer used to tokenize the inputs."""

    @overload
908
    def get_hf_config(self, /) -> PretrainedConfig: ...
909
910
911
912

    @overload
    def get_hf_config(
        self,
913
        typ: type[_C] | tuple[type[_C], ...],
914
        /,
915
    ) -> _C: ...
916
917
918

    def get_hf_config(
        self,
919
        typ: type[Any] | tuple[type[Any], ...] | None = None,
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
        /,
    ) -> Any:
        """
        Get the HuggingFace configuration
        (`transformers.PretrainedConfig`) of the model,
        additionally checking its type.

        Raises:
            TypeError: If the configuration is not of the specified type.
        """
        if typ is None:
            from transformers.configuration_utils import PretrainedConfig

            typ = PretrainedConfig

        hf_config = self.model_config.hf_config
        if not isinstance(hf_config, typ):
937
938
939
940
941
            raise TypeError(
                "Invalid type of HuggingFace config. "
                f"Expected type: {typ}, but "
                f"found type: {type(hf_config)}"
            )
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964

        return hf_config

    def get_hf_image_processor_config(self) -> dict[str, Any]:
        """
        Get the HuggingFace image processor configuration of the model.
        """
        return self.model_config.hf_image_processor_config

    def get_mm_config(self):
        """
        Get the multimodal config of the model.

        Raises:
            RuntimeError: If the model is not a multimodal model.
        """
        mm_config = self.model_config.multimodal_config
        if mm_config is None:
            raise RuntimeError("Not a multimodal model")

        return mm_config

    @overload
965
    def get_hf_processor(self, /, **kwargs: object) -> ProcessorMixin: ...
966
967
968
969

    @overload
    def get_hf_processor(
        self,
970
        typ: type[_P] | tuple[type[_P], ...],
971
972
        /,
        **kwargs: object,
973
    ) -> _P: ...
974
975
976

    def get_hf_processor(
        self,
977
        typ: type[Any] | tuple[type[Any], ...] | None = None,
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
        /,
        **kwargs: object,
    ) -> Any:
        """
        Get the HuggingFace processor
        (`transformers.ProcessorMixin`) of the model,
        additionally checking its type.

        Raises:
            TypeError: If the processor is not of the specified type.
        """
        if typ is None:
            from transformers.processing_utils import ProcessorMixin

            typ = ProcessorMixin

        return cached_processor_from_config(
            self.model_config,
            processor_cls=typ,
            tokenizer=self.tokenizer,
            **kwargs,
        )

    def init_processor(
        self,
        typ: type[_T],
        /,
        **kwargs: object,
    ) -> _T:
        """
        Initialize a HuggingFace-like processor class, merging the
        keyword arguments with those in the model's configuration.
        """
        mm_config = self.model_config.get_multimodal_config()
        base_kwargs = mm_config.mm_processor_kwargs
        if base_kwargs is None:
            base_kwargs = {}

        merged_kwargs = {**base_kwargs, **kwargs}

        return typ(**merged_kwargs)

    def _postprocess_output(
        self,
        output: JSONTree,
    ) -> JSONTree:
        def _postprocess_one(x: object):
            if isinstance(x, torch.Tensor):  # noqa: SIM102
                # This mimics the behavior of transformers.BatchFeature
                if x.is_floating_point():
                    x = x.to(dtype=self.model_config.dtype)

            return x

        return json_map_leaves(_postprocess_one, output)

    def call_hf_processor(
        self,
1036
        hf_processor: ProcessorMixin,
1037
1038
1039
1040
1041
        data: Mapping[str, object],
        kwargs: Mapping[str, object] = {},
        *,
        num_tries: int = 1,
        max_tries: int = 5,
1042
    ) -> BatchFeature | JSONTree:
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
        """
        Call `hf_processor` on the prompt `data`
        (text, image, audio...) with configurable options `kwargs`.
        """
        assert callable(hf_processor)

        mm_config = self.model_config.get_multimodal_config()
        merged_kwargs = mm_config.merge_mm_processor_kwargs(kwargs)

        allowed_kwargs = get_allowed_kwarg_only_overrides(
            hf_processor,
            merged_kwargs,
            requires_kw_only=False,
            allow_var_kwargs=True,
        )

        try:
1060
            output = hf_processor(**data, **allowed_kwargs, return_tensors="pt")
1061
1062
        except Exception as exc:
            # See https://github.com/huggingface/tokenizers/issues/537
1063
1064
1065
1066
1067
1068
            if (
                isinstance(exc, RuntimeError)
                and exc
                and exc.args[0] == "Already borrowed"
                and num_tries < max_tries
            ):
1069
1070
                logger.warning(
                    "Failed to acquire tokenizer in current thread. "
1071
1072
1073
1074
                    "Retrying (%d/%d)...",
                    num_tries,
                    max_tries,
                )
1075
1076
1077
1078
1079
1080
1081
1082
1083
                time.sleep(0.5)
                return self.call_hf_processor(
                    hf_processor,
                    data,
                    kwargs,
                    num_tries=num_tries + 1,
                    max_tries=max_tries,
                )

1084
1085
1086
1087
            msg = (
                f"Failed to apply {type(hf_processor).__name__} "
                f"on data={data} with kwargs={allowed_kwargs}"
            )
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107

            raise ValueError(msg) from exc

        # this emulates output.to(dtype=self.model_config.dtype)
        from transformers.feature_extraction_utils import BatchFeature

        if isinstance(output, BatchFeature):
            output_ = self._postprocess_output(output.data)
            return BatchFeature(output_)

        logger.warning_once(
            "%s did not return `BatchFeature`. "
            "Make sure to match the behaviour of `ProcessorMixin` when "
            "implementing custom processors.",
            type(hf_processor).__name__,
        )

        return self._postprocess_output(output)


1108
class BaseProcessingInfo:
1109
    """Base class to provide the information necessary for data processing."""
1110

1111
1112
    def __init__(self, ctx: InputProcessingContext) -> None:
        super().__init__()
1113

1114
1115
1116
1117
1118
1119
1120
        self.ctx = ctx

    @property
    def model_id(self) -> str:
        return self.ctx.model_config.model

    def get_tokenizer(self) -> AnyTokenizer:
1121
1122
        return self.ctx.tokenizer

1123
    def get_hf_config(self) -> PretrainedConfig:
1124
1125
        return self.ctx.get_hf_config()

1126
    def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
1127
1128
1129
1130
1131
1132
        """
        Subclasses can override this method to handle
        specific kwargs from model config or user inputs.
        """
        return self.ctx.get_hf_processor(**kwargs)

1133
    @abstractmethod
1134
    def get_supported_mm_limits(self) -> Mapping[str, int | None]:
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
        """
        Return the maximum supported number of items for each modality.

        A value of `None` means unlimited number of items.

        Omitting a modality from the returned dictionary means that
        it is not supported at all.
        """
        raise NotImplementedError

1145
1146
1147
1148
1149
1150
1151
1152
1153
    def get_allowed_mm_limits(self) -> Mapping[str, int]:
        """Return the maximum allowed number of items for each modality."""
        supported_mm_limits = self.get_supported_mm_limits()
        mm_config = self.ctx.get_mm_config()

        allowed_limits = dict[str, int]()
        for modality, supported_limit in supported_mm_limits.items():
            user_limit = mm_config.get_limit_per_prompt(modality)

1154
1155
1156
1157
1158
            allowed_limits[modality] = (
                user_limit
                if supported_limit is None
                else min(user_limit, supported_limit)
            )
1159
1160
1161

        return allowed_limits

1162
1163
1164
1165
    def get_mm_max_tokens_per_item(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
1166
    ) -> Mapping[str, int] | None:
1167
1168
        """
        Return the maximum number of tokens per item of for each modality.
1169

1170
1171
1172
1173
        When `None` (the default) is returned, vLLM will generate dummy inputs
        (images/videos) at maximum possible sizes and process them to determine
        the maximum token count per modality.

1174
1175
1176
1177
1178
        This approach works but can be very slow for certain models (e.g.,
        Qwen2.5-VL), leading to very long startup time. For better performance,
        each model can override this method to return pre-computed maximum token
        counts, avoiding the need for dummy input generation and processing.

1179
        Note:
1180
            The maximum number of tokens per item of each modality returned
1181
1182
1183
1184
            from this function should respect the model's maximum sequence
            length and the maximum number of items of each modality allowed,
            and agree with dummy inputs (images/videos) at maximum possible
            sizes.
1185
1186
1187
        """
        return None

1188
1189

_I = TypeVar("_I", bound=BaseProcessingInfo)
1190

1191
1192
MultiModalHashes = dict[str, list[str]]
"""
1193
A collection of hashes with a similar structure as
1194
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
1195
1196
"""

1197
MultiModalPromptUpdates = Mapping[str, list[Sequence[ResolvedPromptUpdate]]]
1198
1199
1200
1201
1202
"""
A collection of prompt updates with a similar structure as
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
"""

1203
MultiModalPromptUpdatesApplyResult = Mapping[str, list[int | None]]
1204
1205
1206
1207
1208
1209
1210
"""
For an item `MultiModalPromptUpdates[k][i]`,
`MultiModalPromptUpdatesApplyResult[k][i]` represents the index of the
`ResolvedPromptUpdate` instance that has been applied, or `None` if none of the
`ResolvedPromptUpdate` instances have been applied.
"""

1211
1212

class MultiModalProcessingInfo(NamedTuple):
1213
    kwargs: MultiModalKwargsOptionalItems
1214
    hashes: MultiModalHashes
1215
1216
    prompt_updates: MultiModalPromptUpdates

1217
1218

class BaseMultiModalProcessor(ABC, Generic[_I]):
1219
    """
1220
    Abstract base class to process multi-modal inputs to be used in vLLM.
1221

1222
    Not to be confused with `transformers.ProcessorMixin`.
1223
1224
    """

1225
1226
1227
1228
1229
    def __init__(
        self,
        info: _I,
        dummy_inputs: "BaseDummyInputsBuilder[_I]",
        *,
1230
        cache: BaseMultiModalProcessorCache | None = None,
1231
    ) -> None:
1232
1233
        super().__init__()

1234
1235
        self.info = info
        self.dummy_inputs = dummy_inputs
1236
        self.cache = cache
1237

1238
1239
        self.data_parser = self._get_data_parser()

1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
        # Avoid unnecessary recomputation
        self._supported_mm_limits = self.info.get_supported_mm_limits()
        self._allowed_mm_limits = self.info.get_allowed_mm_limits()

    @property
    def supported_mm_limits(self):
        return self._supported_mm_limits

    @property
    def allowed_mm_limits(self):
        return self._allowed_mm_limits

1252
    def __call__(
1253
        self,
1254
1255
        prompt: str,
        mm_data: MultiModalDataDict,
1256
        hf_processor_mm_kwargs: Mapping[str, object],
1257
        *,
1258
        mm_uuids: MultiModalUUIDDict | None = None,
1259
    ) -> MultiModalInputs:
1260
        return self.apply(prompt, mm_data, hf_processor_mm_kwargs, mm_uuids=mm_uuids)
1261

1262
1263
    def _get_data_parser(self) -> MultiModalDataParser:
        """
1264
        Construct a parser to preprocess multi-modal data items
1265
1266
        before passing them to
        [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
1267
1268

        You can support additional modalities by creating a subclass
1269
1270
        of [`MultiModalDataParser`][vllm.multimodal.parse.MultiModalDataParser]
        that has additional subparsers.
1271
1272
1273
        """
        return MultiModalDataParser()

1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
    def validate_num_items(
        self,
        modality: str,
        num_items: int,
    ) -> None:
        supported_limit = self.supported_mm_limits.get(modality, 0)
        allowed_limit = self.allowed_mm_limits.get(modality, 0)

        if supported_limit is None:
            supported_limit = allowed_limit

        limit = min(supported_limit, allowed_limit)

        if num_items > limit:
1288
            msg = f"At most {limit} {modality}(s) may be provided in one prompt."
1289
1290
1291
1292
1293
1294

            if num_items <= supported_limit:
                msg += " Set `--limit-mm-per-prompt` to increase this limit."

            raise ValueError(msg)

1295
    def _to_mm_items(
1296
1297
1298
        self,
        mm_data: MultiModalDataDict,
    ) -> MultiModalDataItems:
1299
        """
1300
1301
1302
1303
1304
        Normalize
        [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict]
        to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]
        before passing them to
        [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
1305
        """
1306
        mm_items = self.data_parser.parse_mm_data(mm_data)
1307
        for modality, items in mm_items.items():
1308
            self.validate_num_items(modality, len(items))
1309
1310

        return mm_items
1311

1312
1313
1314
    @abstractmethod
    def _get_mm_fields_config(
        self,
1315
        hf_inputs: BatchFeature,
1316
1317
1318
1319
1320
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, MultiModalFieldConfig]:
        """Given the HF-processed data, output the metadata of each field."""
        raise NotImplementedError

1321
    @abstractmethod
1322
    def _get_prompt_updates(
1323
        self,
1324
        mm_items: MultiModalDataItems,
1325
        hf_processor_mm_kwargs: Mapping[str, object],
1326
        out_mm_kwargs: MultiModalKwargsItems,
1327
    ) -> Sequence[PromptUpdate]:
1328
1329
        """
        Given the original multi-modal items for this modality
1330
        and HF-processed data, output the updates to perform.
1331

1332
1333
1334
1335
1336
1337
        The information returned by this method is used to update token inputs
        which bypass the HF processor. It is also used to update the output of
        HF processor if the HF process does not apply prompt updates to text
        inputs.

        Moreover, this information is critical to determine the token positions
1338
1339
        in order to construct
        [`PlaceholderRange`][vllm.multimodal.inputs.PlaceholderRange]
1340
        for each multi-modal item.
1341
1342
        """
        raise NotImplementedError
1343

1344
1345
1346
1347
1348
1349
    def _bind_and_group_updates(
        self,
        prompt_updates: Sequence[PromptUpdate],
        mm_item_counts: Mapping[str, int],
    ) -> MultiModalPromptUpdates:
        return {
1350
1351
1352
1353
            modality: [
                [update.resolve(item_idx) for update in updates]
                for item_idx in range(mm_item_counts.get(modality, 0))
            ]
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
            for modality, updates in full_groupby_modality(prompt_updates)
        }

    def _get_mm_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        out_mm_kwargs: MultiModalKwargsItems,
    ) -> MultiModalPromptUpdates:
        unbound_prompt_updates = self._get_prompt_updates(
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
            out_mm_kwargs=out_mm_kwargs,
        )

        mm_prompt_updates = self._bind_and_group_updates(
            unbound_prompt_updates,
            mm_items.get_all_counts(),
        )

        for modality, prompt_updates in mm_prompt_updates.items():
            for item_idx, item_prompt_updates in enumerate(prompt_updates):
                if len(item_prompt_updates) > 1:
                    logger.warning_once(
                        "Detected %d prompt updates for `mm_items[%r][%s]`. "
                        "Multiple prompt updates per item is now "
                        "deprecated and may be removed in v0.13. "
                        "Instead, please specify dynamic update targets "
                        "in the same prompt update definition by passing "
                        "a function to `PromptUpdate.target`.",
                        len(prompt_updates),
                        modality,
                        item_idx,
                    )

        return mm_prompt_updates

1391
    def _find_mm_placeholders(
1392
1393
        self,
        new_token_ids: list[int],
1394
        mm_prompt_updates: MultiModalPromptUpdates,
1395
    ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
1396
1397
        tokenizer = self.info.get_tokenizer()

1398
        return find_mm_placeholders(new_token_ids, mm_prompt_updates, tokenizer)
1399

1400
    def _get_hf_mm_data(
1401
        self,
1402
        mm_items: MultiModalDataItems,
1403
1404
1405
    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
        processor_data = dict[str, object]()
        passthrough_data = dict[str, object]()
1406

1407
1408
1409
        for items in mm_items.values():
            processor_data.update(items.get_processor_data())
            passthrough_data.update(items.get_passthrough_data())
1410

1411
1412
        return processor_data, passthrough_data

1413
1414
1415
    def _call_hf_processor(
        self,
        prompt: str,
1416
1417
1418
1419
        # Not to be confused with `mm_data` in `self.apply`.
        # This refers to the data to be passed to HF processor.
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
1420
        tok_kwargs: Mapping[str, object],
1421
    ) -> BatchFeature:
1422
1423
1424
1425
        """
        Call the HF processor on the prompt text and
        associated multi-modal data.
        """
1426
1427
        return self.info.ctx.call_hf_processor(
            self.info.get_hf_processor(**mm_kwargs),
1428
            dict(text=prompt, **mm_data),
1429
            dict(**mm_kwargs, **tok_kwargs),
1430
1431
        )

1432
    def _hf_processor_applies_updates(
1433
1434
1435
1436
        self,
        prompt_text: str,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1437
        tokenization_kwargs: Mapping[str, object],
1438
1439
    ) -> bool:
        """
1440
        Return whether the HF processor applies prompt updates.
1441

1442
1443
        For most HF processors, this should be `True` when multi-modal
        data items are passed, but `False` when multi-modal embeddings
1444
1445
1446
1447
        are passed.
        """
        return not any(
            isinstance(items, (EmbeddingItems, DictEmbeddingItems))
1448
1449
            for items in mm_items.values()
        )
1450

1451
    def _apply_hf_processor_text_mm(
1452
        self,
1453
        prompt_text: str,
1454
        mm_items: MultiModalDataItems,
1455
        hf_processor_mm_kwargs: Mapping[str, object],
1456
        tokenization_kwargs: Mapping[str, object],
1457
    ) -> tuple[list[int], BatchFeature, bool]:
1458
        """
1459
1460
        Apply the HF processor on the prompt text and multi-modal data
        together.
1461

1462
        In addition, return whether prompt updates have been applied.
1463
1464
1465
1466
1467
1468
1469
        """
        processor_data, passthrough_data = self._get_hf_mm_data(mm_items)

        processed_data = self._call_hf_processor(
            prompt=prompt_text,
            mm_data=processor_data,
            mm_kwargs=hf_processor_mm_kwargs,
1470
            tok_kwargs=tokenization_kwargs,
1471
1472
        )
        processed_data.update(passthrough_data)
1473

1474
        (prompt_ids,) = processed_data.pop("input_ids").tolist()
1475

1476
        is_update_applied = self._hf_processor_applies_updates(
1477
1478
1479
            prompt_text=prompt_text,
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1480
            tokenization_kwargs=tokenization_kwargs,
1481
1482
        )

1483
        return prompt_ids, processed_data, is_update_applied
1484

1485
    def _apply_hf_processor_text_only(
1486
1487
1488
1489
        self,
        prompt_text: str,
        tokenization_kwargs: Mapping[str, object],
    ) -> list[int]:
1490
        """
1491
        Apply the HF processor on the prompt text only.
1492

1493
1494
1495
        Since HF processor requires that text and multi-modal items
        correspond to each other, we create dummy multi-modal items
        to go along with the text.
1496
        """
1497
        prompt_ids, _, _ = self._apply_hf_processor_text_mm(
1498
1499
1500
            prompt_text=prompt_text,
            mm_items=MultiModalDataItems({}),
            hf_processor_mm_kwargs={},
1501
            tokenization_kwargs=tokenization_kwargs,
1502
1503
        )

1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
        return prompt_ids

    def _apply_hf_processor_tokens_only(
        self,
        prompt_tokens: list[int],
    ) -> list[int]:
        """
        Apply the HF processor on the prompt tokens only.

        Most HF processors accept prompt text but not prompt tokens.
        If the HF processor adds or removes tokens that are not related to
        multi-modal data, you should override this method so it is consistent
1516
1517
1518
        with the output of
        [`_apply_hf_processor_text_only`][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_text_only]
        on the
1519
1520
1521
1522
1523
1524
1525
1526
        corresponding text.
        """
        return prompt_tokens

    def _apply_hf_processor_mm_only(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1527
        tokenization_kwargs: Mapping[str, object],
1528
    ) -> BatchFeature:
1529
1530
1531
1532
1533
        """
        Apply the HF processor on the multi-modal data only.

        Since HF processor requires that text and multi-modal items
        correspond to each other, we generate dummy text using
1534
1535
        [`DummyInputsBuilder`][vllm.multimodal.profiling.BaseDummyInputsBuilder]
        to go along with the multi-modal data.
1536
1537
1538
        """
        mm_counts = mm_items.get_all_counts()

1539
        _, mm_processed_data, _ = self._apply_hf_processor_text_mm(
1540
            prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
1541
1542
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1543
            tokenization_kwargs=tokenization_kwargs,
1544
1545
        )

1546
        return mm_processed_data
1547
1548
1549

    def _apply_hf_processor_main(
        self,
1550
        prompt: str | list[int],
1551
1552
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1553
        tokenization_kwargs: Mapping[str, object],
1554
        *,
1555
        enable_hf_prompt_update: bool,
1556
    ) -> tuple[list[int], BatchFeature, bool]:
1557
1558
1559
        """
        Apply the HF processor on the prompt text and multi-modal data.

1560
        In addition, return whether prompt updates have been applied
1561
        (for most HF processors, this should be `True`).
1562

1563
        Note:
1564
            If `enable_hf_prompt_update=False`, we use HF processor
1565
            to perform prompt updates if available; HF processor requires
1566
            that the prompt corresponds to multi-modal items.
1567
1568
        """
        if isinstance(prompt, str):
1569
            if enable_hf_prompt_update:
1570
1571
1572
1573
                return self._apply_hf_processor_text_mm(
                    prompt_text=prompt,
                    mm_items=mm_items,
                    hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1574
                    tokenization_kwargs=tokenization_kwargs,
1575
1576
                )

1577
            prompt_ids = self._apply_hf_processor_text_only(prompt, tokenization_kwargs)
1578
1579
1580
        else:
            prompt_ids = self._apply_hf_processor_tokens_only(prompt)

1581
        mm_processed_data = self._apply_hf_processor_mm_only(
1582
            mm_items=mm_items,
1583
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1584
            tokenization_kwargs=tokenization_kwargs,
1585
1586
        )

1587
        return prompt_ids, mm_processed_data, False
1588

1589
    def _hash_mm_items(
1590
1591
1592
1593
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        tokenization_kwargs: Mapping[str, object],
1594
        *,
1595
        mm_uuids: MultiModalUUIDDict | None = None,
1596
    ) -> MultiModalHashes:
1597
        """Create MM hashes to be returned.
1598

1599

1600
1601
1602
        Note: When overrides are provided via callers of `apply`,
        `_hash_mm_items` will be bypassed and the overrides will be used.
        """
1603
1604
        model_id = self.info.model_id

1605
        hashes: MultiModalHashes = {}
1606
        mm_uuids = mm_uuids or {}
1607
1608

        for modality, items in mm_items.items():
1609
1610
1611
1612
            if modality in mm_uuids:
                mm_uuids_per_modality = mm_uuids[modality]
                if isinstance(mm_uuids_per_modality, str):
                    mm_uuids_per_modality = [mm_uuids_per_modality]
1613
1614
1615
1616

                # For None entries, compute a hash; otherwise, use provided ID.
                computed: list[str] = []
                for i, item in enumerate(items):
1617
                    item_uuid = mm_uuids_per_modality[i]
1618

1619
                    # NOTE: Even if a item_uuid is provided, we still compute a
1620
1621
1622
                    # hash if `hf_processor_mm_kwargs` or `tokenization_kwargs`
                    # are provided. This is because the processed multimodal
                    # inputs can be different depending on the processor kwargs.
1623
1624
1625
1626
1627
                    if (
                        item_uuid is None
                        or hf_processor_mm_kwargs
                        or tokenization_kwargs
                    ):
1628
1629
                        # NOTE: use provided hash string to hash with kwargs
                        # if available for better performance.
1630
                        item = item_uuid if item_uuid is not None else item
1631
1632
1633
1634
1635
                        computed.append(
                            MultiModalHasher.hash_kwargs(
                                model_id=model_id,
                                **{modality: item},
                                **hf_processor_mm_kwargs,
1636
1637
1638
                                **tokenization_kwargs,
                            )
                        )
1639
                    else:
1640
                        computed.append(item_uuid)
1641
1642
1643
                hashes[modality] = computed
            else:
                hashes[modality] = [
1644
1645
1646
1647
1648
1649
                    MultiModalHasher.hash_kwargs(
                        model_id=model_id,
                        **{modality: item},
                        **hf_processor_mm_kwargs,
                        **tokenization_kwargs,
                    )
1650
1651
1652
1653
                    for item in items
                ]

        return hashes
1654

1655
1656
    def _get_cache_missing_items(
        self,
1657
        cache: BaseMultiModalProcessorCache,
1658
1659
1660
1661
        mm_data_items: MultiModalDataItems,
        mm_hashes: MultiModalHashes,
    ) -> MultiModalDataItems:
        mm_is_cached = {
1662
            modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
1663
1664
1665
1666
        }

        mm_missing_idxs = {
            modality: [
1667
1668
                idx
                for idx, item_is_cached in enumerate(items_is_cached)
1669
1670
1671
1672
                if not item_is_cached
            ]
            for modality, items_is_cached in mm_is_cached.items()
        }
1673
1674
1675
1676
1677
1678
1679
1680
        mm_missing_data = {}
        for modality, idxs in mm_missing_idxs.items():
            missing_modality_data = []
            for idx in idxs:
                data = mm_data_items[modality][idx]
                if data is None:
                    raise ValueError(
                        f"Cache miss for {modality} at index {idx} "
1681
1682
                        f"but data is not provided."
                    )
1683
1684
1685
                else:
                    missing_modality_data.append(data)
            mm_missing_data[modality] = missing_modality_data
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699

        return self._to_mm_items(mm_missing_data)

    def _recompute_cached_prompt_update(
        self,
        cached_update: ResolvedPromptUpdate,
        new_item_idx: int,
    ) -> ResolvedPromptUpdate:
        """
        Override this if other attributes of `ResolvedPromptUpdate`
        also need to be recomputed after retrieving from the cache.
        """
        return replace(cached_update, item_idx=new_item_idx)

1700
1701
    def _merge_mm_kwargs(
        self,
1702
        cache: BaseMultiModalProcessorCache,
1703
        mm_hashes: MultiModalHashes,
1704
        mm_missing_kwargs: MultiModalKwargsItems,
1705
1706
1707
1708
1709
        mm_missing_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[MultiModalKwargsOptionalItems, MultiModalPromptUpdates]:
        # Need to calculate this at the beginning to avoid skipping cache logic
        # for subsequently repeated items in the same modality
        mm_is_cached = {
1710
            modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
1711
1712
        }

1713
        mm_missing_next_idx = defaultdict[str, int](lambda: 0)
1714

1715
        merged_kwargs = defaultdict[str, list[MultiModalKwargsItem | None]](list)
1716
1717
1718
        merged_prompt_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](
            list
        )
1719
1720
        for modality, hashes in mm_hashes.items():
            missing_kwargs = mm_missing_kwargs.get(modality, [])
1721
            missing_prompt_updates = mm_missing_prompt_updates.get(modality, [])
1722
1723

            for item_idx, item_hash in enumerate(hashes):
1724
                kwargs: MultiModalKwargsItem | None
1725
1726
1727
1728
1729
                if not mm_is_cached[modality][item_idx]:
                    missing_next_idx = mm_missing_next_idx[modality]
                    kwargs = missing_kwargs[missing_next_idx]
                    updates = missing_prompt_updates[missing_next_idx]

1730
                    mm_missing_next_idx[modality] += 1
1731
1732

                    item = kwargs, updates
1733
                else:
1734
1735
1736
1737
1738
                    item = None

                kwargs, updates = cache.get_and_update_item(item, item_hash)

                merged_kwargs[modality].append(kwargs)
1739
1740
1741
1742
1743
1744
                merged_prompt_updates[modality].append(
                    [
                        self._recompute_cached_prompt_update(update, item_idx)
                        for update in updates
                    ]
                )
1745

1746
1747
        mm_kwargs = MultiModalKwargsItems(merged_kwargs)
        mm_prompt_updates = dict(merged_prompt_updates)
1748

1749
        return mm_kwargs, mm_prompt_updates
1750
1751
1752

    def _apply_hf_processor(
        self,
1753
        prompt: str | list[int],
1754
1755
        mm_data_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1756
        tokenization_kwargs: Mapping[str, object],
1757
        *,
1758
        mm_uuids: MultiModalUUIDDict | None = None,
1759
    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1760
1761
        (
            prompt_ids,
1762
            mm_processed_data,
1763
1764
1765
1766
1767
            is_update_applied,
        ) = self._apply_hf_processor_main(
            prompt=prompt,
            mm_items=mm_data_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1768
            tokenization_kwargs=tokenization_kwargs,
1769
1770
1771
            enable_hf_prompt_update=True,
        )

1772
        mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
1773
            mm_processed_data,
1774
            self._get_mm_fields_config(mm_processed_data, hf_processor_mm_kwargs),
1775
1776
        )

1777
        # Use overrides if provided; fallback to data-dependent hashing.
1778
1779
1780
1781
1782
1783
        mm_hashes = self._hash_mm_items(
            mm_data_items,
            hf_processor_mm_kwargs,
            tokenization_kwargs,
            mm_uuids=mm_uuids,
        )
1784

1785
        mm_prompt_updates = self._get_mm_prompt_updates(
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
            mm_data_items,
            hf_processor_mm_kwargs,
            mm_kwargs,
        )

        mm_info = MultiModalProcessingInfo(
            kwargs=mm_kwargs,
            hashes=mm_hashes,
            prompt_updates=mm_prompt_updates,
        )

        return prompt_ids, mm_info, is_update_applied
1798

1799
1800
    def _cached_apply_hf_processor(
        self,
1801
        prompt: str | list[int],
1802
1803
        mm_data_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1804
        tokenization_kwargs: Mapping[str, object],
1805
        *,
1806
        mm_uuids: MultiModalUUIDDict | None = None,
1807
    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1808
1809
1810
1811
1812
1813
        """
        Apply the HF processor on the full prompt text,
        caching the results and reusing cached results.
        """
        cache = self.cache

1814
1815
        _, passthrough_data = self._get_hf_mm_data(mm_data_items)
        if cache is None or passthrough_data:
1816
            return self._apply_hf_processor(
1817
                prompt=prompt,
1818
                mm_data_items=mm_data_items,
1819
                hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1820
                tokenization_kwargs=tokenization_kwargs,
1821
                mm_uuids=mm_uuids,
1822
1823
            )

1824
1825
1826
1827
1828
1829
        mm_hashes = self._hash_mm_items(
            mm_data_items,
            hf_processor_mm_kwargs,
            tokenization_kwargs,
            mm_uuids=mm_uuids,
        )
1830
1831

        mm_missing_data_items = self._get_cache_missing_items(
1832
1833
            cache=cache,
            mm_data_items=mm_data_items,
1834
            mm_hashes=mm_hashes,
1835
        )
1836

1837
        # NOTE: `prompt` does not correspond to `mm_missing_data_items`,
1838
        # so we can't apply prompt updates until the new multimodal
1839
1840
1841
        # items are combined with the cached multimodal items
        (
            prompt_ids,
1842
            mm_missing_processed_data,
1843
            is_update_applied,
1844
        ) = self._apply_hf_processor_main(
1845
            prompt=prompt,
1846
            mm_items=mm_missing_data_items,
1847
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1848
            tokenization_kwargs=tokenization_kwargs,
1849
            enable_hf_prompt_update=False,
1850
1851
        )

1852
        mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
1853
            mm_missing_processed_data,
1854
1855
1856
            self._get_mm_fields_config(
                mm_missing_processed_data, hf_processor_mm_kwargs
            ),
1857
1858
        )

1859
1860
1861
1862
        mm_missing_prompt_updates = self._get_mm_prompt_updates(
            mm_missing_data_items,
            hf_processor_mm_kwargs,
            mm_missing_kwargs,
1863
        )
1864

1865
1866
1867
1868
1869
        mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
            cache,
            mm_hashes=mm_hashes,
            mm_missing_kwargs=mm_missing_kwargs,
            mm_missing_prompt_updates=mm_missing_prompt_updates,
1870
1871
1872
1873
        )

        mm_info = MultiModalProcessingInfo(
            kwargs=mm_kwargs,
1874
            hashes=mm_hashes,
1875
1876
            prompt_updates=mm_prompt_updates,
        )
1877

1878
        return prompt_ids, mm_info, is_update_applied
1879

1880
1881
1882
    def _apply_token_matches(
        self,
        prompt: list[int],
1883
1884
1885
1886
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
        tokenizer = self.info.get_tokenizer()
        return apply_token_matches(prompt, mm_prompt_updates, tokenizer)
1887
1888
1889
1890

    def _apply_text_matches(
        self,
        prompt: str,
1891
1892
1893
1894
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[str, MultiModalPromptUpdatesApplyResult]:
        tokenizer = self.info.get_tokenizer()
        return apply_text_matches(prompt, mm_prompt_updates, tokenizer)
1895

1896
    def _apply_prompt_updates(
1897
1898
        self,
        token_ids: list[int],
1899
        mm_prompt_updates: MultiModalPromptUpdates,
1900
    ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
1901
        tokenizer = self.info.get_tokenizer()
1902

1903
1904
1905
1906
        new_token_ids, match_result = self._apply_token_matches(
            token_ids,
            mm_prompt_updates,
        )
1907
1908
1909
1910
1911
1912
1913
1914
1915

        # If the search text does not represent a special token,
        # it may have different token IDs in the prompt, because
        # the tokens may go across the boundaries of the search text.
        # ----
        # e.g. when searching for "foo" in "food", if "food" itself makes
        # up a token, then the token ID of "foo" will not appear at all
        # ----
        # Since it is inefficient to search for all possible tokenizations
1916
1917
        # of the search text in the prompt, we instead perform string-based
        # updates on the decoded token IDs, then encode them back.
1918
        if not all(
1919
1920
1921
            all(update_idx is not None for update_idx in update_idxs)
            for update_idxs in match_result.values()
        ):
1922
1923
1924
            new_text, match_result = self._apply_text_matches(
                decode_tokens(tokenizer, token_ids),
                mm_prompt_updates,
1925
1926
            )

1927
1928
1929
1930
            new_token_ids = encode_tokens(
                tokenizer,
                new_text,
                add_special_tokens=False,
1931
1932
            )

1933
        matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
1934
1935
1936
1937
        for modality, update_idxs in match_result.items():
            for item_idx, update_idx in enumerate(update_idxs):
                assert update_idx is not None, (
                    "Failed to apply prompt replacement for "
1938
1939
                    f"mm_items[{modality!r}][{item_idx}]"
                )
1940
1941

                matched_updates[modality].append(
1942
1943
                    [mm_prompt_updates[modality][item_idx][update_idx]]
                )
1944
1945

        placeholders = self._find_mm_placeholders(
1946
1947
            new_token_ids,
            dict(matched_updates),
1948
        )
1949

1950
        return new_token_ids, placeholders
1951

1952
1953
    def _validate_mm_kwargs(
        self,
1954
        mm_kwargs: MultiModalKwargsOptionalItems,
1955
1956
1957
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
1958
            items = mm_kwargs.get(modality, [])
1959
1960
1961
1962
1963
1964
1965
1966
1967

            if len(items) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} {modality} items in "
                    f"keyword arguments corresponding to {item_count} "
                    f"{modality} data items, but only found {len(items)}! "
                    "There is likely a problem with your "
                    "implementation of merged multi-modal processor for this "
                    "model (usually arising from an inconsistency between "
1968
1969
                    "`_call_hf_processor` and `_get_mm_fields_config`)."
                )
1970

1971
    def _validate_mm_updates(
1972
        self,
1973
        mm_updates: MultiModalPromptUpdates,
1974
        mm_item_counts: Mapping[str, int],
1975
    ) -> None:
1976
        for modality, item_count in mm_item_counts.items():
1977
            placeholders = mm_updates.get(modality, [])
1978

1979
            if len(placeholders) != item_count:
1980
                raise RuntimeError(
1981
                    f"Expected there to be {item_count} prompt updates "
1982
                    f"corresponding to {item_count} {modality} items, but "
1983
                    f"instead found {len(placeholders)} prompt updates! "
1984
1985
1986
                    "This is likely because you forgot to include input "
                    "placeholder tokens (e.g., `<image>`, `<|image_pad|>`) "
                    "in the prompt. If the model has a chat template, make "
1987
1988
                    "sure you have applied it before calling `LLM.generate`."
                )
1989

1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
    def _validate_mm_placeholders(
        self,
        mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
            placeholders = mm_placeholders.get(modality, [])

            if len(placeholders) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} prompt placeholders "
                    f"corresponding to {item_count} {modality} items, but "
                    f"instead found {len(placeholders)} prompt placeholders! "
                    "Make sure the implementation of `_call_hf_processor` and "
2004
2005
                    "`_get_mm_fields_config` are consistent with each other."
                )
2006

2007
2008
2009
2010
    def _maybe_apply_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        prompt_ids: list[int],
2011
        mm_kwargs: MultiModalKwargsOptionalItems,
2012
        mm_prompt_updates: MultiModalPromptUpdates,
2013
        is_update_applied: bool,
2014
    ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
2015
        mm_item_counts = mm_items.get_all_counts()
2016
        self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
2017
        self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
2018

2019
        if is_update_applied:
2020
2021
            mm_placeholders = self._find_mm_placeholders(
                prompt_ids,
2022
                mm_prompt_updates,
2023
            )
2024
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
2025
        else:
2026
            prompt_ids, mm_placeholders = self._apply_prompt_updates(
2027
                prompt_ids,
2028
                mm_prompt_updates,
2029
            )
2030
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
2031

2032
        return prompt_ids, mm_placeholders
2033
2034
2035

    def apply(
        self,
2036
        prompt: str | list[int],
2037
2038
        mm_data: MultiModalDataDict,
        hf_processor_mm_kwargs: Mapping[str, object],
2039
        tokenization_kwargs: Mapping[str, object] | None = None,
2040
        *,
2041
        mm_uuids: MultiModalUUIDDict | None = None,
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
    ) -> MultiModalInputs:
        """
        Process multi-modal inputs to be used in vLLM.

        The main steps are:

        1. Apply HF Processor on prompt text and multi-modal data together,
           outputting token IDs and processed tensors.
        2. Find and update sequences in the token IDs with placeholder tokens.
           The number of placeholder tokens equals the feature size of the
           multi-modal data outputted by the multi-modal encoder.
        3. Extract information about the placeholder tokens from the
           processed token IDs.
        """
        mm_items = self._to_mm_items(mm_data)

2058
2059
2060
        if tokenization_kwargs is None:
            tokenization_kwargs = {}

2061
2062
        (
            prompt_ids,
2063
            mm_info,
2064
2065
2066
2067
2068
            is_update_applied,
        ) = self._cached_apply_hf_processor(
            prompt,
            mm_items,
            hf_processor_mm_kwargs,
2069
            tokenization_kwargs=tokenization_kwargs,
2070
            mm_uuids=mm_uuids,
2071
2072
        )

2073
        # NOTE: tokenization_kwargs are not required to init processor
2074
        prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
2075
2076
            mm_items=mm_items,
            prompt_ids=prompt_ids,
2077
2078
            mm_kwargs=mm_info.kwargs,
            mm_prompt_updates=mm_info.prompt_updates,
2079
2080
2081
            is_update_applied=is_update_applied,
        )

2082
2083
2084
2085
        mm_placeholder_ranges = {
            modality: [item.to_range() for item in placeholders]
            for modality, placeholders in mm_placeholders.items()
        }
2086

2087
        return MultiModalInputs(
2088
            type="multimodal",
2089
            prompt_token_ids=prompt_ids,
2090
2091
            mm_kwargs=mm_info.kwargs,
            mm_hashes=mm_info.hashes,
2092
            mm_placeholders=mm_placeholder_ranges,
2093
        )
2094
2095
2096
2097
2098
2099


class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
    @abstractmethod
    def create_encoder_prompt(
        self,
2100
        prompt: str | list[int],
2101
        mm_data: MultiModalDataDict,
2102
    ) -> str | list[int]:
2103
        """
2104
        Create input prompt for the encoder. HF processor will be applied on
2105
2106
        this prompt during profiling and generation.
        """
2107
2108
        raise NotImplementedError

2109
2110
2111
2112
    @property
    def pad_dummy_encoder_prompt(self) -> bool:
        return False

2113
2114
    def create_decoder_prompt(
        self,
2115
        prompt: str | list[int],
2116
        mm_data: MultiModalDataDict,
2117
    ) -> str | list[int]:
2118
2119
2120
        """Create input prompt for the decoder."""
        return prompt

2121
    def _get_enc_dec_inputs(
2122
        self,
2123
        prompt: str | list[int],
2124
        mm_data: MultiModalDataDict,
2125
2126
        encoder_inputs: MultiModalInputs,
    ):
2127
        tokenizer = self.info.get_tokenizer()
2128
2129
        decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_data)
        if isinstance(decoder_prompt_raw, str):
2130
2131
2132
            decoder_prompt_ids = encode_tokens(
                tokenizer, decoder_prompt_raw, add_special_tokens=False
            )
2133
        else:
2134
            decoder_prompt_ids = decoder_prompt_raw
2135
2136
2137

        mm_inputs = MultiModalEncDecInputs(
            encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],
2138
2139
            **encoder_inputs,
        )
2140
        mm_inputs["prompt_token_ids"] = decoder_prompt_ids
2141
        return mm_inputs
2142
2143
2144

    def apply(
        self,
2145
        prompt: str | list[int],
2146
2147
        mm_data: MultiModalDataDict,
        hf_processor_mm_kwargs: Mapping[str, object],
2148
        tokenization_kwargs: Mapping[str, object] | None = None,
2149
        *,
2150
        mm_uuids: MultiModalUUIDDict | None = None,
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
    ) -> MultiModalEncDecInputs:
        """
        Process multi-modal inputs to be used in vLLM.
        The main processing steps are modified to fit encoder-decoder model:
        1. Create encoder prompt from input prompt text.
        2. Apply the HF processor on encoder prompt.
        3. Copy the input prompt text as decoder prompt inputs.
        """
        encoder_prompt = self.create_encoder_prompt(prompt, mm_data)
        encoder_inputs = super().apply(
            encoder_prompt,
            mm_data,
            hf_processor_mm_kwargs,
2164
            tokenization_kwargs,
2165
            mm_uuids=mm_uuids,
2166
2167
2168
2169
2170
2171
2172
        )

        return self._get_enc_dec_inputs(
            prompt=prompt,
            mm_data=mm_data,
            encoder_inputs=encoder_inputs,
        )