processing.py 79.3 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import contextvars
import threading
5
import time
6
from abc import ABC, abstractmethod
7
from collections import defaultdict
8
from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence
9
from contextlib import contextmanager
10
from dataclasses import dataclass, field, replace
11
from enum import Enum
12
from functools import lru_cache
13
14
15
16
17
18
from typing import (
    TYPE_CHECKING,
    Any,
    Generic,
    NamedTuple,
    Protocol,
19
    TypeAlias,
20
21
22
    cast,
    overload,
)
23

24
import regex as re
25
import torch
26
from typing_extensions import TypeVar, assert_never
27

28
from vllm.logger import init_logger
29
from vllm.tokenizers import TokenizerLike
30
from vllm.transformers_utils.processor import cached_processor_from_config
31
32
from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
33
from vllm.utils.jsontree import JSONTree, json_map_leaves
34

35
from .hasher import MultiModalHasher
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
from .inputs import (
    MultiModalDataDict,
    MultiModalEncDecInputs,
    MultiModalFieldConfig,
    MultiModalInputs,
    MultiModalKwargsItem,
    MultiModalKwargsItems,
    MultiModalKwargsOptionalItems,
    MultiModalUUIDDict,
    PlaceholderRange,
)
from .parse import (
    DictEmbeddingItems,
    EmbeddingItems,
    MultiModalDataItems,
    MultiModalDataParser,
)
53
54

if TYPE_CHECKING:
55
56
57
58
    from transformers.configuration_utils import PretrainedConfig
    from transformers.feature_extraction_utils import BatchFeature
    from transformers.processing_utils import ProcessorMixin

59
    from vllm.config import ModelConfig, ObservabilityConfig
60

61
    from .cache import BaseMultiModalProcessorCache
62
    from .profiling import BaseDummyInputsBuilder
63
64
65
66
67
68
else:
    PretrainedConfig = object
    BatchFeature = object
    ProcessorMixin = object

    ModelConfig = object
69
    ObservabilityConfig = object
70
71

    BaseMultiModalProcessorCache = object
72

73
logger = init_logger(__name__)
74
75

_S = TypeVar("_S", str, list[int])
76

77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
_request_id_context: contextvars.ContextVar[str | None] = contextvars.ContextVar(
    "_request_id_context", default=None
)


def get_current_request_id() -> str | None:
    """Get the current request_id from the context, if available."""
    return _request_id_context.get()


@contextmanager
def set_request_id(request_id: str) -> Generator[None, None, None]:
    """Context manager to set the request_id for the current context."""
    token = _request_id_context.set(request_id)
    try:
        yield
    finally:
        _request_id_context.reset(token)


@dataclass
class MultiModalProcessorTimingStats:
    """Per-request timing statistics for multimodal processor stages."""

    hf_processor_time: float = 0.0
    """Time spent in HuggingFace processor calls (seconds)."""

    hashing_time: float = 0.0
    """Time spent computing multimodal item hashes (seconds)."""

    cache_lookup_time: float = 0.0
    """Time spent in cache lookups and merges (seconds)."""

    prompt_update_time: float = 0.0
    """Time spent applying prompt updates and finding placeholders (seconds)."""

    total_time: float = 0.0
    """Total processing time (seconds)."""

    def to_dict(self) -> dict[str, float]:
        """Convert stats to a dictionary for JSON serialization."""
        return {
            "hf_processor_time": self.hf_processor_time,
            "hashing_time": self.hashing_time,
            "cache_lookup_time": self.cache_lookup_time,
            "prompt_update_time": self.prompt_update_time,
            "total_time": self.total_time,
        }


def get_timing_stats_from_engine_client(
    engine_client: Any,
) -> dict[str, dict[str, float]]:
    """
    Get all timing stats from the context associated with the engine client.

    Args:
        engine_client: The engine client that has input_processor.

    Returns:
        A dictionary mapping request_id to stats dict.
    """
    try:
        if not engine_client.vllm_config.observability_config.enable_mm_processor_stats:
            return {}
    except (AttributeError, RuntimeError):
        return {}

    try:
        input_processor = engine_client.input_processor
        input_preprocessor = input_processor.input_preprocessor

        if hasattr(input_preprocessor, "_get_mm_processor"):
            mm_processor = input_preprocessor._get_mm_processor()
            if mm_processor is not None and hasattr(mm_processor, "info"):
                ctx = mm_processor.info.ctx
                return ctx.get_all_timing_stats()
    except (AttributeError, RuntimeError):
        pass

    return {}


@contextmanager
def _timed_operation(ctx: "InputProcessingContext", stage_name: str):
    """
    Context manager to time an operation using the context's timing stats.

    The request_id is automatically retrieved from the context variable,
    so it doesn't need to be passed as a parameter.

    Args:
        ctx: The InputProcessingContext containing the timing stats registry.
        stage_name: Name of the stage being timed.
    """
    request_id = get_current_request_id()
    if ctx is None or request_id is None:
        yield
        return

    stats = ctx.get_timing_stats(request_id)
    if stats is None:
        yield
        return

    start_time = time.perf_counter()
    try:
        yield
    finally:
        elapsed = time.perf_counter() - start_time
        if stage_name == "hf_processor":
            stats.hf_processor_time += elapsed
        elif stage_name == "hashing":
            stats.hashing_time += elapsed
        elif stage_name == "cache_lookup":
            stats.cache_lookup_time += elapsed
        elif stage_name == "prompt_update":
            stats.prompt_update_time += elapsed
        stats.total_time += elapsed


198
PromptSeq: TypeAlias = str | list[int]
199
"""A token sequence (list of token IDs) or text."""
200

201

202
203
@lru_cache(maxsize=2048)
def _cached_encode(
204
    tokenizer: TokenizerLike,
205
206
    text: str,
    *,
207
    add_special_tokens: bool = True,
208
) -> list[int]:
209
    return tokenizer.encode(text, add_special_tokens=add_special_tokens)
210
211
212
213


@lru_cache(maxsize=2048)
def _cached_decode(
214
    tokenizer: TokenizerLike,
215
216
    token_ids: tuple[int, ...],
    *,
217
    skip_special_tokens: bool = False,
218
) -> str:
219
    return tokenizer.decode(list(token_ids), skip_special_tokens=skip_special_tokens)
220
221


222
223
224
225
226
227
def _seq2text(
    tokenizer: TokenizerLike | None,
    seq: PromptSeq,
    *,
    use_cache: bool = True,
) -> str:
228
229
230
    if isinstance(seq, str):
        return seq

231
232
233
234
    if tokenizer is None:
        raise ValueError("You cannot decode tokens when `skip_tokenizer_init=True`")

    if not use_cache:
235
        return tokenizer.decode(seq)
236

237
238
239
    return _cached_decode(tokenizer, tuple(seq))


240
241
242
243
244
245
def _seq2tokens(
    tokenizer: TokenizerLike | None,
    seq: PromptSeq,
    *,
    use_cache: bool = True,
) -> list[int]:
246
    if isinstance(seq, str):
247
248
249
250
        if tokenizer is None:
            raise ValueError("You cannot encode text when `skip_tokenizer_init=True`")

        if not use_cache:
251
            return tokenizer.encode(seq, add_special_tokens=False)
252

253
254
255
256
257
        return _cached_encode(tokenizer, seq, add_special_tokens=False)

    return seq


258
259
260
class _GetMatchIndex(Protocol):
    def __call__(
        self,
261
        tokenizer: TokenizerLike | None,
262
263
        prompt: PromptSeq,
        start_idx: int = 0,
264
    ) -> int | None: ...
265
266


267
268
269
@dataclass
class PromptIndex:
    """Resolves to an index in the prompt."""
270

271
    get_match_index: _GetMatchIndex
272
273
274
275
276
277
278
279
280
281


class PromptIndexTargets:
    @staticmethod
    def start() -> PromptIndex:
        """
        Resolves to the start of the prompt (before the first token).

        This results in a match even if the prompt is empty.
        """
282
        return PromptIndex(lambda tokenizer, prompt, start_idx=0: 0)
283
284
285
286
287
288
289
290

    @staticmethod
    def prefix(seq: PromptSeq) -> PromptIndex:
        """
        Resolves to a location in the prompt after the given prefix.
        """

        def get_match_index(
291
            tokenizer: TokenizerLike | None,
292
            prompt: PromptSeq,
293
            start_idx: int = 0,
294
        ) -> int | None:
295
296
297
            if start_idx != 0:
                return None

298
299
300
            prefix = seq

            if isinstance(prompt, str):
301
302
                # Make both `str`
                prefix = _seq2text(tokenizer, prefix, use_cache=False)
303
            else:
304
305
                # Make both `list[int]`
                prefix = _seq2tokens(tokenizer, prefix, use_cache=False)
306
307
308
309
310
311
312
313
314
315
316
317
318

            match_idx = len(prefix)
            return match_idx if prompt[:match_idx] == prefix else None

        return PromptIndex(get_match_index)

    @staticmethod
    def end() -> PromptIndex:
        """
        Resolves to the end of the prompt (after the last token).

        This results in a match even if the prompt is empty.
        """
319
        return PromptIndex(lambda tokenizer, prompt, start_idx=0: len(prompt))
320
321


322
UpdateTarget: TypeAlias = PromptSeq | PromptIndex
323
324
325
326
"""
The token sequence or text to update.
"""

327
PromptUpdateTarget: TypeAlias = Callable[[int], UpdateTarget] | UpdateTarget
328
329
330
331
332
333
334
335
336
"""
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
output the corresponding token sequence (or text).

For convenience, you can directly pass in the token sequence (or text)
instead of a function if it does not depend on the input.
"""

337

338
@dataclass
339
class PromptUpdateDetails(Generic[_S]):
340
    """Details about the token sequence or text that are part of the update."""
341

342
    full: _S
343
    """The full content."""
344

345
    is_embed: Callable[[TokenizerLike | None, PromptSeq], torch.Tensor] | None = None
346
    """
347
348
349
    Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full],
    return a boolean mask of shape `(len(full),)` indicating which positions
    of `full` to assign embeddings to.
350
351
352
353

    `None` (default) means to assign embeddings to all positions of `full`.

    The embeddings are obtained by calling
354
    [`SupportsMultiModal.embed_multimodal`][vllm.model_executor.models.interfaces.SupportsMultiModal.embed_multimodal].
355
356
357
    """

    @staticmethod
358
    def from_seq(seq: _S) -> "PromptUpdateDetails[_S]":
359
360
361
362
363
364
365
        return PromptUpdateDetails(full=seq)

    @staticmethod
    def select_text(
        seq: _S,
        embed_text: str,
    ) -> "PromptUpdateDetails[_S]":
366
367
        def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
            embed_token_ids = _seq2tokens(tokenizer, embed_text, use_cache=False)
368
            token_ids = _seq2tokens(tokenizer, full)
369
370

            return torch.isin(
371
                torch.tensor(token_ids),
372
373
374
375
376
377
378
379
380
381
                torch.tensor(embed_token_ids),
            )

        return PromptUpdateDetails(full=seq, is_embed=is_embed)

    @staticmethod
    def select_token_id(
        seq: _S,
        embed_token_id: int,
    ) -> "PromptUpdateDetails[_S]":
382
        def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
383
384
385
386
387
            token_ids = _seq2tokens(tokenizer, full)

            return torch.tensor(token_ids) == embed_token_id

        return PromptUpdateDetails(full=seq, is_embed=is_embed)
388
389


390
PromptUpdateInfo: TypeAlias = PromptSeq | PromptUpdateDetails
391
"""
392
The token sequence or text that are part of the update.
393

394
If only part of the content corresponds to feature placeholders, you can
395
396
use [`PromptUpdateDetails`][vllm.multimodal.processing.PromptUpdateDetails] to
specify which part.
397
"""
398

399
PromptUpdateContent: TypeAlias = Callable[[int], PromptUpdateInfo] | PromptUpdateInfo
400
"""
401
402
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
403
404
405
406
407
408
409
410
411
412
413
414
415
output the corresponding token sequence (or text).

For convenience, you can directly pass in the token sequence (or text)
instead of a function if it does not depend on the input.
"""


class UpdateMode(str, Enum):
    INSERT = "insert"
    REPLACE = "replace"


@dataclass
416
class PromptUpdate(ABC):
417
418
419
420
421
422
423
    """
    Defines how to update a prompt with placeholder tokens.
    """

    modality: str
    """The modality for which the update is made."""

424
    target: PromptUpdateTarget
425
426
427
428
429
430
431
432
433
434
435
436
437
438
    """The token sequence (or text) to update."""

    @property
    @abstractmethod
    def content(self) -> PromptUpdateContent:
        """The placeholder tokens that are part of the update."""
        raise NotImplementedError

    @property
    @abstractmethod
    def mode(self) -> UpdateMode:
        """Defines how to update the prompt."""
        raise NotImplementedError

439
    def _resolve_target(self, item_idx: int) -> UpdateTarget:
440
441
442
443
        target = self.target
        if callable(target):
            target = target(item_idx)

444
        return target
445

446
    def _resolve_content(self, item_idx: int) -> PromptUpdateDetails:
447
448
449
450
451
452
453
        content = self.content
        if callable(content):
            content = content(item_idx)

        if not isinstance(content, PromptUpdateDetails):
            content = PromptUpdateDetails.from_seq(content)

454
        return content
455

456
    def resolve(self, item_idx: int) -> "ResolvedPromptUpdate":
457
458
459
460
461
462
463
464
465
        """
        Given the index of the processed item within
        [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
        output a copy of this object with its lazy attributes resolved.
        """
        return ResolvedPromptUpdate(
            modality=self.modality,
            item_idx=item_idx,
            mode=self.mode,
466
467
            target=self._resolve_target(item_idx),
            content=self._resolve_content(item_idx),
468
469
        )

470

471
@dataclass
472
473
474
475
476
477
class PromptInsertion(PromptUpdate):
    """
    Defines how to insert placeholder tokens into a prompt.

    Example:

478
479
    For each image, insert a number of `<image>` feature placeholders
    equal to the feature size of the vision encoder after the `<s>` token:
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498

    ```python
    PromptInsertion(
        modality="image",
        target="<s>",
        insertion="<image>" * image_feature_size,
    )
    ```

    Insert these tokens at the start of the prompt:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.start(),
        insertion="<image>" * image_feature_size,
    )
    ```

499
    Insert these tokens after a prefix `Images:`:
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.prefix("Images:"),
        insertion="<image>" * image_feature_size,
    )
    ```

    Insert these tokens at the end of the prompt:

    ```python
    PromptInsertion(
        modality="image",
        target=PromptIndexTargets.end(),
        insertion="<image>" * image_feature_size,
    )
    ```
518
519
520
521
    """

    insertion: PromptUpdateContent = field(repr=False)
    """
522
523
524
525
    Given the index of the processed item within
    [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
    output the token sequence (or text) to insert right after
    [`target`][vllm.multimodal.processing.PromptUpdate.target].
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541

    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
    """

    @property
    def content(self) -> PromptUpdateContent:
        return self.insertion

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.INSERT


@dataclass
class PromptReplacement(PromptUpdate):
542
543
    """
    Defines how to replace portions of an input prompt with placeholder tokens.
544
545
546

    Example:

547
548
    For each image, replace one `<image>` input placeholder in the prompt
    with a number of `<image>` feature placeholders
549
550
551
552
553
554
555
556
557
558
    equal to the feature size of the vision encoder:

    ```python
    PromptReplacement(
        modality="image",
        target="<image>",
        replacement="<image>" * image_feature_size,
    )
    ```

559
560
    As above, but further pad the feature placeholders with `<image_bos>`
    and `<image_eos>`, which are not supposed to be passed to the vision
561
562
563
564
565
566
567
    encoder:

    ```python
    PromptReplacement(
        modality="image",
        target="<image>",
        replacement=PromptUpdateDetails(
568
569
570
571
572
573
574
            full="".join(
                [
                    "<image_bos>",
                    "<image>" * image_feature_size,
                    "<image_eos>",
                ]
            ),
575
576
577
578
579
580
581
582
583
584
585
586
587
            features="<image>" * image_feature_size,
        ),
    )
    ```

    To avoid unnecessary tokenization during prompt replacement,
    we recommended passing token sequences instead of text:

    ```python
    PromptReplacement(
        modality="image",
        target=[image_token_id],
        replacement=PromptUpdateDetails(
588
589
590
            full=(
                [image_bos_id] + [image_token_id] * image_feature_size + [image_eos_id]
            ),
591
592
593
594
            features=[image_token_id] * image_feature_size,
        ),
    )
    ```
595
596
    """

597
    replacement: PromptUpdateContent = field(repr=False)
598
    """
599
600
601
602
    Given the index of the processed item within
    [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
    output the token sequence (or text) to replace
    [`target`][vllm.multimodal.processing.PromptUpdate.target].
603

604
605
    For convenience, you can directly pass in the token sequence (or text)
    instead of a function if it does not depend on the input.
606
607
    """

608
609
610
611
612
613
614
    @property
    def content(self) -> PromptUpdateContent:
        return self.replacement

    @property
    def mode(self) -> UpdateMode:
        return UpdateMode.REPLACE
615
616


617
618
619
class _HasModalityAttr(Protocol):
    modality: str

620

621
622
class _HasModalityProp(Protocol):
    @property
623
    def modality(self) -> str: ...
624
625


626
_M = TypeVar("_M", bound=_HasModalityAttr | _HasModalityProp)
627
628
629


def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]:
630
631
    """
    Convenience function to apply
632
    [`full_groupby`][vllm.utils.collection_utils.full_groupby]
633
634
    based on modality.
    """
635
636
637
    return full_groupby(values, key=lambda x: x.modality)


638
639
640
641
642
643
644
class PromptTargetMatch(NamedTuple):
    start_idx: int
    end_idx: int


@dataclass(frozen=True)
class ResolvedPromptUpdate:
645
    """
646
647
    A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] with its
    lazy attributes resolved, apart from those related to tokenization.
648
    """
649

650
651
    modality: str
    """The modality for which the update is made."""
652

653
654
    item_idx: int
    """The index within `modality` of the item this update pertains to."""
655

656
657
    mode: UpdateMode
    """Defines how to update the prompt."""
658

659
    target: UpdateTarget
660
    """The token sequence (or text) to update."""
661

662
    content: PromptUpdateDetails = field(repr=False)
663
    """The placeholder tokens that are part of the update."""
664

665
666
667
    def iter_token_matches(
        self,
        prompt: list[int],
668
        tokenizer: TokenizerLike | None,
669
670
671
672
673
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        target = self.target
674

675
676
677
678
        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(tokenizer, prompt, start_idx)
            if match_idx is not None:
                yield PromptTargetMatch(match_idx, match_idx)
679

680
            return
681

682
683
        target_token_ids = _seq2tokens(tokenizer, target)

684
        for match in iter_token_matches(prompt, target_token_ids, start_idx=start_idx):
685
            yield PromptTargetMatch(match.start_idx, match.end_idx)
686

687
688
689
    def iter_text_matches(
        self,
        prompt: str,
690
        tokenizer: TokenizerLike | None,
691
692
693
694
695
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        target = self.target
696

697
698
699
700
        if isinstance(target, PromptIndex):
            match_idx = target.get_match_index(tokenizer, prompt, start_idx)
            if match_idx is not None:
                yield PromptTargetMatch(match_idx, match_idx)
701

702
            return
703

704
705
        target_text = _seq2text(tokenizer, target)

706
        for match in re.finditer(re.escape(target_text), prompt, pos=start_idx):
707
708
709
710
            yield PromptTargetMatch(match.start(), match.end())

    def iter_matches(
        self,
711
        prompt: list[int] | str,
712
        tokenizer: TokenizerLike | None,
713
714
715
716
717
        *,
        start_idx: int = 0,
    ) -> Generator[PromptTargetMatch]:
        """Yield each instance of `self.target` found in `prompt`."""
        if isinstance(prompt, str):
718
            return self.iter_text_matches(prompt, tokenizer, start_idx=start_idx)
719
720

        return self.iter_token_matches(prompt, tokenizer, start_idx=start_idx)
721

722
723
724
725
726
727
728
729
730
    def with_target(self, target: UpdateTarget):
        return replace(self, target=target)

    def with_content(self, content: PromptUpdateInfo):
        if not isinstance(content, PromptUpdateDetails):
            content = PromptUpdateDetails.from_seq(content)

        return replace(self, content=content)

731

732
733
734
class _TokenMatch(NamedTuple):
    start_idx: int
    end_idx: int
735
736


737
738
739
def iter_token_matches(
    token_ids: list[int],
    match_ids: list[int],
740
741
    *,
    start_idx: int = 0,
742
) -> Generator[_TokenMatch]:
743
    """
744
    Yield each occurrence of `match_ids` in `token_ids`.
745
746
747
748

    Note that empty matches are ignored.
    """
    prompt_len = len(token_ids)
749
    match_len = len(match_ids)
750

751
752
    if match_len == 0:
        return
753

754
    while start_idx < prompt_len - match_len + 1:
755
        end_idx = start_idx + match_len
756

757
758
        if token_ids[start_idx:end_idx] == match_ids:
            yield _TokenMatch(start_idx=start_idx, end_idx=end_idx)
759
760
761
762
763

            # Exclude overlapping matches
            start_idx = end_idx
        else:
            start_idx += 1
764
765


766
767
768
769
770
771
def replace_token_matches(
    token_ids: list[int],
    match_ids: list[int],
    new_ids: list[int],
) -> list[int]:
    """
772
773
    Replace each occurrence of `match_ids` in `token_ids`
    with `new_ids`.
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792

    Note that empty matches are ignored.
    """
    out_seqs = list[list[int]]()
    prev_end_idx = 0

    for match in iter_token_matches(token_ids, match_ids):
        start_idx = match.start_idx
        end_idx = match.end_idx

        out_seqs.append(token_ids[prev_end_idx:start_idx])
        out_seqs.append(new_ids)
        prev_end_idx = end_idx

    out_seqs.append(token_ids[prev_end_idx:])

    return flatten_2d_lists(out_seqs)


793
@dataclass
794
class PlaceholderFeaturesInfo:
795
    modality: str
796
    item_idx: int
797
    start_idx: int
798
    tokens: list[int]
799
    is_embed: torch.Tensor | None
800
801
802

    @property
    def length(self) -> int:
803
        return len(self.tokens)
804
805

    def to_range(self) -> PlaceholderRange:
806
807
        # TODO: Is it worth it to optimize this by stripping the
        # leading and ending positions where `is_embed=False`?
808
809
810
        return PlaceholderRange(
            offset=self.start_idx,
            length=self.length,
811
            is_embed=self.is_embed,
812
        )
813
814


815
_MatchToApply = tuple[tuple[str, int], tuple[PromptTargetMatch, int]]
816
817


818
819
820
def _find_matches(
    prompt: _S,
    mm_prompt_updates: "MultiModalPromptUpdates",
821
    tokenizer: TokenizerLike | None,
822
823
824
    *,
    prev_end_idx: int = 0,
    current_result: "MultiModalPromptUpdatesApplyResult",
825
826
) -> tuple[UpdateMode | None, list[_MatchToApply]]:
    mode: UpdateMode | None = None
827
828
829
830
831
832
833
834
835
836
837
838
    mm_matches = dict[tuple[str, int], tuple[PromptTargetMatch, int]]()

    for modality, modality_updates in mm_prompt_updates.items():
        for item_idx, item_updates in enumerate(modality_updates):
            if current_result[modality][item_idx] is not None:
                continue  # Updates have already been applied for this item

            for update_idx, update in enumerate(item_updates):
                if (modality, item_idx) in mm_matches:
                    break  # Already found a match for this item

                for match in update.iter_matches(
839
840
841
                    prompt,
                    tokenizer,
                    start_idx=prev_end_idx,
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
                ):
                    # All matches should share the same mode
                    if mode is None:
                        mode = update.mode
                    elif mode != update.mode:
                        continue

                    mm_matches[(modality, item_idx)] = match, update_idx
                    break  # Get only the first valid match per item

    # Prioritize earlier matches
    matches_to_apply = sorted(mm_matches.items(), key=lambda item: item[1][0])

    # To avoid conflicts, only replace one non-empty item at a time
    if mode == UpdateMode.REPLACE:
        matches_to_apply_ = list[_MatchToApply]()
        has_non_empty_matches = False

        for item in matches_to_apply:
            _, (match, _) = item
            if match.start_idx == match.end_idx:
                matches_to_apply_.append(item)
            elif not has_non_empty_matches:
                has_non_empty_matches = True
                matches_to_apply_.append(item)

        matches_to_apply = matches_to_apply_

    return mode, matches_to_apply
871
872


873
874
875
876
877
878
879
880
881
882
def _all_items_found(
    mm_item_counts: dict[str, int],
    mm_found_counts: dict[str, int],
) -> bool:
    return all(
        item_idx >= mm_item_counts[modality]
        for modality, item_idx in mm_found_counts.items()
    )


883
def _apply_matches(
884
    prompt: _S,
885
    mm_prompt_updates: "MultiModalPromptUpdates",
886
    tokenizer: TokenizerLike | None,
887
) -> tuple[list[_S], "MultiModalPromptUpdatesApplyResult"]:
888
    mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()}
889

890
    out_seqs = list[str | list[int]]()
891
    out_result: MultiModalPromptUpdatesApplyResult = {
892
        m: [None] * len(items) for m, items in mm_prompt_updates.items()
893
    }
894

895
    # Early exit if no items to find
896
897
898
899
900
901
    mm_found_counts = {
        m: sum(r is not None for r in res) for m, res in out_result.items()
    }
    if _all_items_found(mm_item_counts, mm_found_counts):
        return [prompt], out_result

902
903
    prev_end_idx = 0
    while True:
904
905
906
907
908
909
910
        mode, matches_to_apply = _find_matches(
            prompt,
            mm_prompt_updates,
            tokenizer,
            prev_end_idx=prev_end_idx,
            current_result=out_result,
        )
911

912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
        if mode is None:
            break  # No more matches to find

        for (modality, item_idx), (match, update_idx) in matches_to_apply:
            matched_update = mm_prompt_updates[modality][item_idx][update_idx]
            matched_content = matched_update.content.full

            if mode == UpdateMode.INSERT:
                end_idx_to_insert = match.end_idx
            elif mode == UpdateMode.REPLACE:
                end_idx_to_insert = match.start_idx
            else:
                assert_never(mode)

            out_seqs.append(prompt[prev_end_idx:end_idx_to_insert])
            out_seqs.append(
                _seq2text(tokenizer, matched_content)
                if isinstance(prompt, str)
                else _seq2tokens(tokenizer, matched_content)
            )
            out_result[modality][item_idx] = update_idx

            # Exclude overlapping matches
            prev_end_idx = match.end_idx

        # Early exit if all items found
        mm_found_counts = {
            m: sum(r is not None for r in res) for m, res in out_result.items()
        }
        if _all_items_found(mm_item_counts, mm_found_counts):
            break
943
944
945

    out_seqs.append(prompt[prev_end_idx:])

946
    return cast(list[_S], out_seqs), out_result
947
948


949
def apply_token_matches(
950
    prompt: list[int],
951
    mm_prompt_updates: "MultiModalPromptUpdates",
952
    tokenizer: TokenizerLike | None,
953
954
955
) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
    """
    Apply the updates in `mm_prompt_updates` to `prompt`.
956

957
958
959
960
    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
    appears earlier in `mm_prompt_updates` takes priority.
    """
961
    token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
962

963
    return flatten_2d_lists(token_id_seqs), result
964
965


966
def apply_text_matches(
967
    prompt: str,
968
    mm_prompt_updates: "MultiModalPromptUpdates",
969
    tokenizer: TokenizerLike | None,
970
971
972
) -> tuple[str, "MultiModalPromptUpdatesApplyResult"]:
    """
    Apply the updates in `mm_prompt_updates` to `prompt`.
973

974
975
976
977
978
    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
    appears earlier in `mm_prompt_updates` takes priority.
    """
    texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
979

980
    return "".join(texts), result
981
982


983
def _iter_placeholders(
984
    prompt: list[int],
985
    mm_prompt_updates: "MultiModalPromptUpdates",
986
    tokenizer: TokenizerLike | None,
987
) -> Iterable[PlaceholderFeaturesInfo]:
988
    """
989
    Yield each set of placeholder tokens found in `prompt`.
990
991
992

    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
993
    appears earlier in `mm_prompt_updates` takes priority.
994

995
996
    Note that empty matches are ignored.
    """
997
    mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()}
998
    item_idx_by_modality = {modality: 0 for modality in mm_prompt_updates}
999

1000
1001
    if _all_items_found(mm_item_counts, item_idx_by_modality):
        return
1002

1003
    prompt_len = len(prompt)
1004
    start_idx = 0
1005

1006
1007
1008
    while start_idx < prompt_len:
        found = False

1009
        for modality, modality_updates in mm_prompt_updates.items():
1010
1011
            item_idx = item_idx_by_modality[modality]
            if item_idx >= mm_item_counts.get(modality, 0):
1012
                continue
1013

1014
1015
            for update in modality_updates[item_idx]:
                content = update.content
1016
                content_tokens_full = _seq2tokens(tokenizer, content.full)
1017
1018
                content_len_full = len(content_tokens_full)
                end_idx_full = start_idx + content_len_full
1019

1020
                if content_len_full == 0 or end_idx_full > prompt_len:
1021
1022
                    continue

1023
                if prompt[start_idx:end_idx_full] == content_tokens_full:
1024
1025
                    content_is_embed = content.is_embed
                    if content_is_embed is not None:
1026
                        content_is_embed = content_is_embed(tokenizer, content.full)
1027
1028
1029
1030
1031
1032
1033
1034

                    yield PlaceholderFeaturesInfo(
                        modality=modality,
                        item_idx=item_idx,
                        start_idx=start_idx,
                        tokens=content_tokens_full,
                        is_embed=content_is_embed,
                    )
1035

1036
                    # Exclude overlapping matches
1037
                    start_idx = end_idx_full
1038
1039
1040
                    item_idx_by_modality[modality] += 1
                    found = True
                    break
1041

1042
            if found:
1043
1044
1045
                if _all_items_found(mm_item_counts, item_idx_by_modality):
                    return

1046
                break  # Go back to the outer while loop
1047
1048
1049

        if not found:
            start_idx += 1
1050
1051


1052
1053
def find_mm_placeholders(
    prompt: list[int],
1054
    mm_prompt_updates: "MultiModalPromptUpdates",
1055
    tokenizer: TokenizerLike | None,
1056
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
1057
    it = _iter_placeholders(prompt, mm_prompt_updates, tokenizer)
1058
1059
1060
    return dict(full_groupby_modality(it))


1061
_T = TypeVar("_T")
1062
1063
_C = TypeVar("_C", bound=PretrainedConfig, default=PretrainedConfig)
_P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin)
1064
1065
1066
1067
1068
1069
1070
1071
1072


@dataclass(frozen=True)
class InputProcessingContext:
    """
    Contains information about the model which may be used to
    modify the inputs.
    """

1073
1074
    model_config: ModelConfig
    """The configuration of the model."""
1075

1076
    tokenizer: TokenizerLike | None
1077
1078
    """The tokenizer used to tokenize the inputs."""

1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
    observability_config: "ObservabilityConfig | None" = field(
        default=None, compare=False, repr=False
    )
    """Configuration for observability features."""

    timing_stats_registry: dict[str, MultiModalProcessorTimingStats] = field(
        default_factory=dict, compare=False, repr=False
    )
    """Registry for storing timing stats keyed by request_id."""

    _timing_stats_registry_lock: threading.Lock = field(
        default_factory=threading.Lock, compare=False, repr=False
    )
    """Lock for thread-safe access to timing_stats_registry."""

1094
1095
1096
1097
1098
1099
1100
1101
    def get_tokenizer(self) -> TokenizerLike:
        if self.tokenizer is None:
            raise ValueError(
                "You cannot pass text prompts when `skip_tokenizer_init=True`"
            )

        return self.tokenizer

1102
    @overload
1103
    def get_hf_config(self, /) -> PretrainedConfig: ...
1104
1105
1106
1107

    @overload
    def get_hf_config(
        self,
1108
        typ: type[_C] | tuple[type[_C], ...],
1109
        /,
1110
    ) -> _C: ...
1111
1112
1113

    def get_hf_config(
        self,
1114
        typ: type[Any] | tuple[type[Any], ...] | None = None,
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
        /,
    ) -> Any:
        """
        Get the HuggingFace configuration
        (`transformers.PretrainedConfig`) of the model,
        additionally checking its type.

        Raises:
            TypeError: If the configuration is not of the specified type.
        """
        if typ is None:
            from transformers.configuration_utils import PretrainedConfig

            typ = PretrainedConfig

        hf_config = self.model_config.hf_config
        if not isinstance(hf_config, typ):
1132
1133
1134
1135
1136
            raise TypeError(
                "Invalid type of HuggingFace config. "
                f"Expected type: {typ}, but "
                f"found type: {type(hf_config)}"
            )
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159

        return hf_config

    def get_hf_image_processor_config(self) -> dict[str, Any]:
        """
        Get the HuggingFace image processor configuration of the model.
        """
        return self.model_config.hf_image_processor_config

    def get_mm_config(self):
        """
        Get the multimodal config of the model.

        Raises:
            RuntimeError: If the model is not a multimodal model.
        """
        mm_config = self.model_config.multimodal_config
        if mm_config is None:
            raise RuntimeError("Not a multimodal model")

        return mm_config

    @overload
1160
    def get_hf_processor(self, /, **kwargs: object) -> ProcessorMixin: ...
1161
1162
1163
1164

    @overload
    def get_hf_processor(
        self,
1165
        typ: type[_P] | tuple[type[_P], ...],
1166
1167
        /,
        **kwargs: object,
1168
    ) -> _P: ...
1169
1170
1171

    def get_hf_processor(
        self,
1172
        typ: type[Any] | tuple[type[Any], ...] | None = None,
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
        /,
        **kwargs: object,
    ) -> Any:
        """
        Get the HuggingFace processor
        (`transformers.ProcessorMixin`) of the model,
        additionally checking its type.

        Raises:
            TypeError: If the processor is not of the specified type.
        """
        if typ is None:
            from transformers.processing_utils import ProcessorMixin

            typ = ProcessorMixin

        return cached_processor_from_config(
1190
            self.model_config,
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
            processor_cls=typ,
            tokenizer=self.tokenizer,
            **kwargs,
        )

    def init_processor(
        self,
        typ: type[_T],
        /,
        **kwargs: object,
    ) -> _T:
        """
        Initialize a HuggingFace-like processor class, merging the
        keyword arguments with those in the model's configuration.
        """
        mm_config = self.model_config.get_multimodal_config()
        base_kwargs = mm_config.mm_processor_kwargs
        if base_kwargs is None:
            base_kwargs = {}

        merged_kwargs = {**base_kwargs, **kwargs}

        return typ(**merged_kwargs)

    def _postprocess_output(
        self,
        output: JSONTree,
    ) -> JSONTree:
        def _postprocess_one(x: object):
            if isinstance(x, torch.Tensor):  # noqa: SIM102
                # This mimics the behavior of transformers.BatchFeature
                if x.is_floating_point():
                    x = x.to(dtype=self.model_config.dtype)

            return x

        return json_map_leaves(_postprocess_one, output)

    def call_hf_processor(
        self,
1231
        hf_processor: ProcessorMixin,
1232
1233
1234
1235
1236
        data: Mapping[str, object],
        kwargs: Mapping[str, object] = {},
        *,
        num_tries: int = 1,
        max_tries: int = 5,
1237
    ) -> BatchFeature | JSONTree:
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
        """
        Call `hf_processor` on the prompt `data`
        (text, image, audio...) with configurable options `kwargs`.
        """
        assert callable(hf_processor)

        mm_config = self.model_config.get_multimodal_config()
        merged_kwargs = mm_config.merge_mm_processor_kwargs(kwargs)

        allowed_kwargs = get_allowed_kwarg_only_overrides(
            hf_processor,
            merged_kwargs,
            requires_kw_only=False,
            allow_var_kwargs=True,
        )

        try:
1255
            output = hf_processor(**data, **allowed_kwargs, return_tensors="pt")
1256
1257
        except Exception as exc:
            # See https://github.com/huggingface/tokenizers/issues/537
1258
1259
1260
1261
1262
1263
            if (
                isinstance(exc, RuntimeError)
                and exc
                and exc.args[0] == "Already borrowed"
                and num_tries < max_tries
            ):
1264
1265
                logger.warning(
                    "Failed to acquire tokenizer in current thread. "
1266
1267
1268
1269
                    "Retrying (%d/%d)...",
                    num_tries,
                    max_tries,
                )
1270
1271
1272
1273
1274
1275
1276
1277
1278
                time.sleep(0.5)
                return self.call_hf_processor(
                    hf_processor,
                    data,
                    kwargs,
                    num_tries=num_tries + 1,
                    max_tries=max_tries,
                )

1279
1280
1281
1282
            msg = (
                f"Failed to apply {type(hf_processor).__name__} "
                f"on data={data} with kwargs={allowed_kwargs}"
            )
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301

            raise ValueError(msg) from exc

        # this emulates output.to(dtype=self.model_config.dtype)
        from transformers.feature_extraction_utils import BatchFeature

        if isinstance(output, BatchFeature):
            output_ = self._postprocess_output(output.data)
            return BatchFeature(output_)

        logger.warning_once(
            "%s did not return `BatchFeature`. "
            "Make sure to match the behaviour of `ProcessorMixin` when "
            "implementing custom processors.",
            type(hf_processor).__name__,
        )

        return self._postprocess_output(output)

1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
    def get_timing_stats(
        self, request_id: str
    ) -> MultiModalProcessorTimingStats | None:
        """
        Get timing stats for a request.
        """
        if (
            self.observability_config is None
            or not self.observability_config.enable_mm_processor_stats
        ):
            return None
        with self._timing_stats_registry_lock:
            return self.timing_stats_registry.get(request_id)

    def create_timing_stats(self, request_id: str) -> MultiModalProcessorTimingStats:
        """
        Create and store timing stats in the registry for a request.

        This should be called at the start of processing for a request.
        The stats object is created immediately and stored in the registry.
        """
        if (
            self.observability_config is None
            or not self.observability_config.enable_mm_processor_stats
        ):
            return MultiModalProcessorTimingStats()

        with self._timing_stats_registry_lock:
            if request_id in self.timing_stats_registry:
                raise ValueError(
                    f"Timing stats already exist for request_id: {request_id}"
                )
            stats = MultiModalProcessorTimingStats()
            self.timing_stats_registry[request_id] = stats
            return stats

    def clear_timing_stats_registry(self) -> int:
        """
        Clear all stats from the registry. Returns the number of stats cleared.
        """
        if (
            self.observability_config is None
            or not self.observability_config.enable_mm_processor_stats
        ):
            return 0
        with self._timing_stats_registry_lock:
            count = len(self.timing_stats_registry)
            self.timing_stats_registry.clear()
            return count

    def get_all_timing_stats(self) -> dict[str, dict[str, float]]:
        """
        Get all timing stats as a dictionary for API endpoints.
        """
        if (
            self.observability_config is None
            or not self.observability_config.enable_mm_processor_stats
        ):
            return {}
        with self._timing_stats_registry_lock:
            return {
                rid: stats.to_dict()
                for rid, stats in self.timing_stats_registry.items()
            }

1367

1368
class BaseProcessingInfo:
1369
    """Base class to provide the information necessary for data processing."""
1370

1371
1372
    def __init__(self, ctx: InputProcessingContext) -> None:
        super().__init__()
1373

1374
1375
1376
1377
1378
1379
        self.ctx = ctx

    @property
    def model_id(self) -> str:
        return self.ctx.model_config.model

1380
    def get_tokenizer(self) -> TokenizerLike:
1381
        return self.ctx.get_tokenizer()
1382

1383
    def get_hf_config(self) -> PretrainedConfig:
1384
1385
        return self.ctx.get_hf_config()

1386
    def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
1387
1388
1389
1390
1391
1392
        """
        Subclasses can override this method to handle
        specific kwargs from model config or user inputs.
        """
        return self.ctx.get_hf_processor(**kwargs)

1393
    @abstractmethod
1394
    def get_supported_mm_limits(self) -> Mapping[str, int | None]:
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
        """
        Return the maximum supported number of items for each modality.

        A value of `None` means unlimited number of items.

        Omitting a modality from the returned dictionary means that
        it is not supported at all.
        """
        raise NotImplementedError

1405
1406
1407
1408
1409
1410
1411
1412
1413
    def get_allowed_mm_limits(self) -> Mapping[str, int]:
        """Return the maximum allowed number of items for each modality."""
        supported_mm_limits = self.get_supported_mm_limits()
        mm_config = self.ctx.get_mm_config()

        allowed_limits = dict[str, int]()
        for modality, supported_limit in supported_mm_limits.items():
            user_limit = mm_config.get_limit_per_prompt(modality)

1414
1415
1416
1417
1418
            allowed_limits[modality] = (
                user_limit
                if supported_limit is None
                else min(user_limit, supported_limit)
            )
1419
1420
1421

        return allowed_limits

1422
1423
1424
1425
    def get_mm_max_tokens_per_item(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
1426
    ) -> Mapping[str, int] | None:
1427
1428
        """
        Return the maximum number of tokens per item of for each modality.
1429

1430
1431
1432
1433
        When `None` (the default) is returned, vLLM will generate dummy inputs
        (images/videos) at maximum possible sizes and process them to determine
        the maximum token count per modality.

1434
1435
1436
1437
1438
        This approach works but can be very slow for certain models (e.g.,
        Qwen2.5-VL), leading to very long startup time. For better performance,
        each model can override this method to return pre-computed maximum token
        counts, avoiding the need for dummy input generation and processing.

1439
        Note:
1440
            The maximum number of tokens per item of each modality returned
1441
1442
1443
1444
            from this function should respect the model's maximum sequence
            length and the maximum number of items of each modality allowed,
            and agree with dummy inputs (images/videos) at maximum possible
            sizes.
1445
1446
1447
        """
        return None

1448
1449

_I = TypeVar("_I", bound=BaseProcessingInfo)
1450

1451
1452
MultiModalHashes = dict[str, list[str]]
"""
1453
1454
1455
1456
1457
1458
1459
A collection of the multi-modal hash for each item, with a similar structure as
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
"""

MultiModalIsCached = dict[str, list[bool]]
"""
A collection of the `is_cached` flag for each item, with a similar structure as
1460
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
1461
1462
"""

1463
MultiModalPromptUpdates = Mapping[str, list[Sequence[ResolvedPromptUpdate]]]
1464
1465
1466
1467
1468
"""
A collection of prompt updates with a similar structure as
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
"""

1469
MultiModalPromptUpdatesApplyResult = Mapping[str, list[int | None]]
1470
1471
1472
1473
1474
1475
1476
"""
For an item `MultiModalPromptUpdates[k][i]`,
`MultiModalPromptUpdatesApplyResult[k][i]` represents the index of the
`ResolvedPromptUpdate` instance that has been applied, or `None` if none of the
`ResolvedPromptUpdate` instances have been applied.
"""

1477
1478

class MultiModalProcessingInfo(NamedTuple):
1479
    kwargs: MultiModalKwargsOptionalItems
1480
    hashes: MultiModalHashes
1481
1482
    prompt_updates: MultiModalPromptUpdates

1483
1484

class BaseMultiModalProcessor(ABC, Generic[_I]):
1485
    """
1486
    Abstract base class to process multi-modal inputs to be used in vLLM.
1487

1488
    Not to be confused with `transformers.ProcessorMixin`.
1489
1490
    """

1491
1492
1493
1494
1495
    def __init__(
        self,
        info: _I,
        dummy_inputs: "BaseDummyInputsBuilder[_I]",
        *,
1496
        cache: BaseMultiModalProcessorCache | None = None,
1497
    ) -> None:
1498
1499
        super().__init__()

1500
1501
        self.info = info
        self.dummy_inputs = dummy_inputs
1502
        self.cache = cache
1503

1504
1505
        self.data_parser = self._get_data_parser()

1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
        # Avoid unnecessary recomputation
        self._supported_mm_limits = self.info.get_supported_mm_limits()
        self._allowed_mm_limits = self.info.get_allowed_mm_limits()

    @property
    def supported_mm_limits(self):
        return self._supported_mm_limits

    @property
    def allowed_mm_limits(self):
        return self._allowed_mm_limits

1518
    def __call__(
1519
        self,
1520
1521
        prompt: str,
        mm_data: MultiModalDataDict,
1522
        hf_processor_mm_kwargs: Mapping[str, object],
1523
        *,
1524
        mm_uuids: MultiModalUUIDDict | None = None,
1525
    ) -> MultiModalInputs:
1526
        return self.apply(prompt, mm_data, hf_processor_mm_kwargs, mm_uuids=mm_uuids)
1527

1528
1529
    def _get_data_parser(self) -> MultiModalDataParser:
        """
1530
        Construct a parser to preprocess multi-modal data items
1531
1532
        before passing them to
        [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
1533
1534

        You can support additional modalities by creating a subclass
1535
1536
        of [`MultiModalDataParser`][vllm.multimodal.parse.MultiModalDataParser]
        that has additional subparsers.
1537
        """
1538
1539
1540
1541
1542
1543
1544
1545
1546
        # Get expected hidden size for embedding validation if mm_embeds enabled
        # This validates hidden dimensions to prevent vulnerabilities: embeddings
        # with correct ndim but wrong shape could cause crashes at inference time
        mm_config = self.info.ctx.model_config.get_multimodal_config()
        expected_hidden_size = None
        if mm_config.enable_mm_embeds:
            expected_hidden_size = self.info.ctx.model_config.get_inputs_embeds_size()

        return MultiModalDataParser(expected_hidden_size=expected_hidden_size)
1547

1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
    def validate_num_items(
        self,
        modality: str,
        num_items: int,
    ) -> None:
        supported_limit = self.supported_mm_limits.get(modality, 0)
        allowed_limit = self.allowed_mm_limits.get(modality, 0)

        if supported_limit is None:
            supported_limit = allowed_limit

        limit = min(supported_limit, allowed_limit)

        if num_items > limit:
1562
            msg = f"At most {limit} {modality}(s) may be provided in one prompt."
1563
1564
1565
1566
1567
1568

            if num_items <= supported_limit:
                msg += " Set `--limit-mm-per-prompt` to increase this limit."

            raise ValueError(msg)

1569
    def _to_mm_items(
1570
1571
1572
        self,
        mm_data: MultiModalDataDict,
    ) -> MultiModalDataItems:
1573
        """
1574
1575
1576
1577
1578
        Normalize
        [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict]
        to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]
        before passing them to
        [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
1579
        """
1580
        mm_items = self.data_parser.parse_mm_data(mm_data)
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590

        mm_config = self.info.ctx.model_config.get_multimodal_config()
        if not mm_config.enable_mm_embeds:
            for modality, items in mm_items.items():
                if isinstance(items, (EmbeddingItems, DictEmbeddingItems)):
                    raise ValueError(
                        f"You must set `--enable-mm-embeds` to input "
                        f"`{modality}_embeds`"
                    )

1591
        for modality, items in mm_items.items():
1592
            self.validate_num_items(modality, len(items))
1593
1594

        return mm_items
1595

1596
1597
1598
    @abstractmethod
    def _get_mm_fields_config(
        self,
1599
        hf_inputs: BatchFeature,
1600
1601
1602
1603
1604
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, MultiModalFieldConfig]:
        """Given the HF-processed data, output the metadata of each field."""
        raise NotImplementedError

1605
    @abstractmethod
1606
    def _get_prompt_updates(
1607
        self,
1608
        mm_items: MultiModalDataItems,
1609
        hf_processor_mm_kwargs: Mapping[str, object],
1610
        out_mm_kwargs: MultiModalKwargsItems,
1611
    ) -> Sequence[PromptUpdate]:
1612
1613
        """
        Given the original multi-modal items for this modality
1614
        and HF-processed data, output the updates to perform.
1615

1616
1617
1618
1619
1620
1621
        The information returned by this method is used to update token inputs
        which bypass the HF processor. It is also used to update the output of
        HF processor if the HF process does not apply prompt updates to text
        inputs.

        Moreover, this information is critical to determine the token positions
1622
1623
        in order to construct
        [`PlaceholderRange`][vllm.multimodal.inputs.PlaceholderRange]
1624
        for each multi-modal item.
1625
1626
        """
        raise NotImplementedError
1627

1628
1629
1630
1631
1632
1633
    def _bind_and_group_updates(
        self,
        prompt_updates: Sequence[PromptUpdate],
        mm_item_counts: Mapping[str, int],
    ) -> MultiModalPromptUpdates:
        return {
1634
1635
1636
1637
            modality: [
                [update.resolve(item_idx) for update in updates]
                for item_idx in range(mm_item_counts.get(modality, 0))
            ]
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
            for modality, updates in full_groupby_modality(prompt_updates)
        }

    def _get_mm_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        out_mm_kwargs: MultiModalKwargsItems,
    ) -> MultiModalPromptUpdates:
        unbound_prompt_updates = self._get_prompt_updates(
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
            out_mm_kwargs=out_mm_kwargs,
        )

        mm_prompt_updates = self._bind_and_group_updates(
            unbound_prompt_updates,
            mm_items.get_all_counts(),
        )

        for modality, prompt_updates in mm_prompt_updates.items():
            for item_idx, item_prompt_updates in enumerate(prompt_updates):
                if len(item_prompt_updates) > 1:
                    logger.warning_once(
                        "Detected %d prompt updates for `mm_items[%r][%s]`. "
                        "Multiple prompt updates per item is now "
                        "deprecated and may be removed in v0.13. "
                        "Instead, please specify dynamic update targets "
                        "in the same prompt update definition by passing "
                        "a function to `PromptUpdate.target`.",
                        len(prompt_updates),
                        modality,
                        item_idx,
                    )

        return mm_prompt_updates

1675
    def _find_mm_placeholders(
1676
1677
        self,
        new_token_ids: list[int],
1678
        mm_prompt_updates: MultiModalPromptUpdates,
1679
    ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
1680
1681
        tokenizer = self.info.get_tokenizer()

1682
        return find_mm_placeholders(new_token_ids, mm_prompt_updates, tokenizer)
1683

1684
    def _get_hf_mm_data(
1685
        self,
1686
        mm_items: MultiModalDataItems,
1687
1688
1689
    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
        processor_data = dict[str, object]()
        passthrough_data = dict[str, object]()
1690

1691
1692
1693
        for items in mm_items.values():
            processor_data.update(items.get_processor_data())
            passthrough_data.update(items.get_passthrough_data())
1694

1695
1696
        return processor_data, passthrough_data

1697
1698
1699
    def _call_hf_processor(
        self,
        prompt: str,
1700
1701
1702
1703
        # Not to be confused with `mm_data` in `self.apply`.
        # This refers to the data to be passed to HF processor.
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
1704
        tok_kwargs: Mapping[str, object],
1705
    ) -> BatchFeature:
1706
1707
1708
1709
        """
        Call the HF processor on the prompt text and
        associated multi-modal data.
        """
1710
1711
1712
1713
1714
1715
        with _timed_operation(self.info.ctx, "hf_processor"):
            return self.info.ctx.call_hf_processor(
                self.info.get_hf_processor(**mm_kwargs),
                dict(text=prompt, **mm_data),
                dict(**mm_kwargs, **tok_kwargs),
            )
1716

1717
    def _hf_processor_applies_updates(
1718
1719
1720
1721
        self,
        prompt_text: str,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1722
        tokenization_kwargs: Mapping[str, object],
1723
1724
    ) -> bool:
        """
1725
        Return whether the HF processor applies prompt updates.
1726

1727
1728
        For most HF processors, this should be `True` when multi-modal
        data items are passed, but `False` when multi-modal embeddings
1729
1730
1731
1732
        are passed.
        """
        return not any(
            isinstance(items, (EmbeddingItems, DictEmbeddingItems))
1733
1734
            for items in mm_items.values()
        )
1735

1736
    def _apply_hf_processor_text_mm(
1737
        self,
1738
        prompt_text: str,
1739
        mm_items: MultiModalDataItems,
1740
        hf_processor_mm_kwargs: Mapping[str, object],
1741
        tokenization_kwargs: Mapping[str, object],
1742
    ) -> tuple[list[int], BatchFeature, bool]:
1743
        """
1744
1745
        Apply the HF processor on the prompt text and multi-modal data
        together.
1746

1747
        In addition, return whether prompt updates have been applied.
1748
1749
1750
1751
1752
1753
1754
        """
        processor_data, passthrough_data = self._get_hf_mm_data(mm_items)

        processed_data = self._call_hf_processor(
            prompt=prompt_text,
            mm_data=processor_data,
            mm_kwargs=hf_processor_mm_kwargs,
1755
            tok_kwargs=tokenization_kwargs,
1756
1757
        )
        processed_data.update(passthrough_data)
1758

1759
        (prompt_ids,) = processed_data.pop("input_ids").tolist()
1760

1761
        is_update_applied = self._hf_processor_applies_updates(
1762
1763
1764
            prompt_text=prompt_text,
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1765
            tokenization_kwargs=tokenization_kwargs,
1766
1767
        )

1768
        return prompt_ids, processed_data, is_update_applied
1769

1770
    def _apply_hf_processor_text_only(
1771
1772
1773
1774
        self,
        prompt_text: str,
        tokenization_kwargs: Mapping[str, object],
    ) -> list[int]:
1775
        """
1776
        Apply the HF processor on the prompt text only.
1777

1778
1779
1780
        Since HF processor requires that text and multi-modal items
        correspond to each other, we create dummy multi-modal items
        to go along with the text.
1781
        """
1782
        prompt_ids, _, _ = self._apply_hf_processor_text_mm(
1783
1784
1785
            prompt_text=prompt_text,
            mm_items=MultiModalDataItems({}),
            hf_processor_mm_kwargs={},
1786
            tokenization_kwargs=tokenization_kwargs,
1787
1788
        )

1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
        return prompt_ids

    def _apply_hf_processor_tokens_only(
        self,
        prompt_tokens: list[int],
    ) -> list[int]:
        """
        Apply the HF processor on the prompt tokens only.

        Most HF processors accept prompt text but not prompt tokens.
        If the HF processor adds or removes tokens that are not related to
        multi-modal data, you should override this method so it is consistent
1801
1802
1803
        with the output of
        [`_apply_hf_processor_text_only`][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_text_only]
        on the
1804
1805
1806
1807
1808
1809
1810
1811
        corresponding text.
        """
        return prompt_tokens

    def _apply_hf_processor_mm_only(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1812
        tokenization_kwargs: Mapping[str, object],
1813
    ) -> BatchFeature:
1814
1815
1816
1817
1818
        """
        Apply the HF processor on the multi-modal data only.

        Since HF processor requires that text and multi-modal items
        correspond to each other, we generate dummy text using
1819
1820
        [`DummyInputsBuilder`][vllm.multimodal.profiling.BaseDummyInputsBuilder]
        to go along with the multi-modal data.
1821
1822
1823
        """
        mm_counts = mm_items.get_all_counts()

1824
        _, mm_processed_data, _ = self._apply_hf_processor_text_mm(
1825
            prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
1826
1827
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1828
            tokenization_kwargs=tokenization_kwargs,
1829
1830
        )

1831
        return mm_processed_data
1832
1833
1834

    def _apply_hf_processor_main(
        self,
1835
        prompt: str | list[int],
1836
1837
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
1838
        tokenization_kwargs: Mapping[str, object],
1839
        *,
1840
        enable_hf_prompt_update: bool,
1841
    ) -> tuple[list[int], BatchFeature, bool]:
1842
1843
1844
        """
        Apply the HF processor on the prompt text and multi-modal data.

1845
        In addition, return whether prompt updates have been applied
1846
        (for most HF processors, this should be `True`).
1847

1848
        Note:
1849
            If `enable_hf_prompt_update=False`, we use HF processor
1850
            to perform prompt updates if available; HF processor requires
1851
            that the prompt corresponds to multi-modal items.
1852
1853
        """
        if isinstance(prompt, str):
1854
            if enable_hf_prompt_update:
1855
1856
1857
1858
                return self._apply_hf_processor_text_mm(
                    prompt_text=prompt,
                    mm_items=mm_items,
                    hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1859
                    tokenization_kwargs=tokenization_kwargs,
1860
1861
                )

1862
            prompt_ids = self._apply_hf_processor_text_only(prompt, tokenization_kwargs)
1863
1864
1865
        else:
            prompt_ids = self._apply_hf_processor_tokens_only(prompt)

1866
        mm_processed_data = self._apply_hf_processor_mm_only(
1867
            mm_items=mm_items,
1868
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1869
            tokenization_kwargs=tokenization_kwargs,
1870
1871
        )

1872
        return prompt_ids, mm_processed_data, False
1873

1874
    def _hash_mm_items(
1875
1876
1877
1878
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        tokenization_kwargs: Mapping[str, object],
1879
        *,
1880
        mm_uuids: MultiModalUUIDDict | None = None,
1881
    ) -> MultiModalHashes:
1882
        """Create MM hashes to be returned.
1883

1884

1885
1886
1887
        Note: When overrides are provided via callers of `apply`,
        `_hash_mm_items` will be bypassed and the overrides will be used.
        """
1888
1889
        model_id = self.info.model_id

1890
        hashes: MultiModalHashes = {}
1891
        mm_uuids = mm_uuids or {}
1892
1893

        for modality, items in mm_items.items():
1894
1895
1896
1897
            if modality in mm_uuids:
                mm_uuids_per_modality = mm_uuids[modality]
                if isinstance(mm_uuids_per_modality, str):
                    mm_uuids_per_modality = [mm_uuids_per_modality]
1898
1899
1900

                # For None entries, compute a hash; otherwise, use provided ID.
                computed: list[str] = []
1901
                for i, item in enumerate(items.get_all_items_for_hash()):
1902
                    item_uuid = mm_uuids_per_modality[i]
1903

1904
                    # NOTE: Even if a item_uuid is provided, we still compute a
1905
1906
1907
                    # hash if `hf_processor_mm_kwargs` or `tokenization_kwargs`
                    # are provided. This is because the processed multimodal
                    # inputs can be different depending on the processor kwargs.
1908
1909
1910
1911
1912
                    if (
                        item_uuid is None
                        or hf_processor_mm_kwargs
                        or tokenization_kwargs
                    ):
1913
1914
                        # NOTE: use provided hash string to hash with kwargs
                        # if available for better performance.
1915
                        item = item_uuid if item_uuid is not None else item
1916
1917
1918
1919
1920
                        computed.append(
                            MultiModalHasher.hash_kwargs(
                                model_id=model_id,
                                **{modality: item},
                                **hf_processor_mm_kwargs,
1921
1922
1923
                                **tokenization_kwargs,
                            )
                        )
1924
                    else:
1925
                        computed.append(item_uuid)
1926
1927
1928
                hashes[modality] = computed
            else:
                hashes[modality] = [
1929
1930
1931
1932
1933
1934
                    MultiModalHasher.hash_kwargs(
                        model_id=model_id,
                        **{modality: item},
                        **hf_processor_mm_kwargs,
                        **tokenization_kwargs,
                    )
1935
1936
1937
1938
                    for item in items
                ]

        return hashes
1939

1940
1941
    def _get_cache_missing_items(
        self,
1942
        cache: BaseMultiModalProcessorCache,
1943
1944
        mm_data_items: MultiModalDataItems,
        mm_hashes: MultiModalHashes,
1945
    ) -> tuple[MultiModalIsCached, MultiModalDataItems]:
1946
        mm_is_cached = {
1947
            modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
1948
1949
1950
1951
        }

        mm_missing_idxs = {
            modality: [
1952
1953
                idx
                for idx, item_is_cached in enumerate(items_is_cached)
1954
1955
1956
1957
                if not item_is_cached
            ]
            for modality, items_is_cached in mm_is_cached.items()
        }
1958
1959
1960
1961
1962
1963
1964
1965
        mm_missing_data = {}
        for modality, idxs in mm_missing_idxs.items():
            missing_modality_data = []
            for idx in idxs:
                data = mm_data_items[modality][idx]
                if data is None:
                    raise ValueError(
                        f"Cache miss for {modality} at index {idx} "
1966
1967
                        f"but data is not provided."
                    )
1968
1969
1970
                else:
                    missing_modality_data.append(data)
            mm_missing_data[modality] = missing_modality_data
1971

1972
        return mm_is_cached, self._to_mm_items(mm_missing_data)
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984

    def _recompute_cached_prompt_update(
        self,
        cached_update: ResolvedPromptUpdate,
        new_item_idx: int,
    ) -> ResolvedPromptUpdate:
        """
        Override this if other attributes of `ResolvedPromptUpdate`
        also need to be recomputed after retrieving from the cache.
        """
        return replace(cached_update, item_idx=new_item_idx)

1985
1986
    def _merge_mm_kwargs(
        self,
1987
        cache: BaseMultiModalProcessorCache,
1988
        mm_hashes: MultiModalHashes,
1989
        mm_is_cached: MultiModalIsCached,
1990
        mm_missing_kwargs: MultiModalKwargsItems,
1991
1992
        mm_missing_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[MultiModalKwargsOptionalItems, MultiModalPromptUpdates]:
1993
1994
1995
1996
1997
        # Need to touch all mm hashes before update to avoid hash in updated
        # list evict during update
        for hashes in mm_hashes.values():
            for item_hash in hashes:
                cache.touch_sender_cache_item(item_hash)
1998

1999
        mm_missing_next_idx = defaultdict[str, int](lambda: 0)
2000

2001
        merged_kwargs = defaultdict[str, list[MultiModalKwargsItem | None]](list)
2002
2003
2004
        merged_prompt_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](
            list
        )
2005
2006
        for modality, hashes in mm_hashes.items():
            missing_kwargs = mm_missing_kwargs.get(modality, [])
2007
            missing_prompt_updates = mm_missing_prompt_updates.get(modality, [])
2008
2009
2010
2011

            for item_idx, item_hash in enumerate(hashes):
                if not mm_is_cached[modality][item_idx]:
                    missing_next_idx = mm_missing_next_idx[modality]
2012
2013
                    missing_kwargs_item = missing_kwargs[missing_next_idx]
                    missing_updates_item = missing_prompt_updates[missing_next_idx]
2014

2015
                    mm_missing_next_idx[modality] += 1
2016

2017
                    item = missing_kwargs_item, missing_updates_item
2018
                else:
2019
2020
2021
2022
2023
                    item = None

                kwargs, updates = cache.get_and_update_item(item, item_hash)

                merged_kwargs[modality].append(kwargs)
2024
2025
2026
2027
2028
2029
                merged_prompt_updates[modality].append(
                    [
                        self._recompute_cached_prompt_update(update, item_idx)
                        for update in updates
                    ]
                )
2030

2031
2032
        mm_kwargs = MultiModalKwargsItems(merged_kwargs)
        mm_prompt_updates = dict(merged_prompt_updates)
2033

2034
        return mm_kwargs, mm_prompt_updates
2035
2036
2037

    def _apply_hf_processor(
        self,
2038
        prompt: str | list[int],
2039
2040
        mm_data_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
2041
        tokenization_kwargs: Mapping[str, object],
2042
        *,
2043
        mm_uuids: MultiModalUUIDDict | None = None,
2044
    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
2045
2046
        (
            prompt_ids,
2047
            mm_processed_data,
2048
2049
2050
2051
2052
            is_update_applied,
        ) = self._apply_hf_processor_main(
            prompt=prompt,
            mm_items=mm_data_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
2053
            tokenization_kwargs=tokenization_kwargs,
2054
2055
2056
            enable_hf_prompt_update=True,
        )

2057
        mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
2058
            mm_processed_data,
2059
            self._get_mm_fields_config(mm_processed_data, hf_processor_mm_kwargs),
2060
2061
        )

2062
        # Use overrides if provided; fallback to data-dependent hashing.
2063
2064
2065
2066
2067
2068
2069
        with _timed_operation(self.info.ctx, "hashing"):
            mm_hashes = self._hash_mm_items(
                mm_data_items,
                hf_processor_mm_kwargs,
                tokenization_kwargs,
                mm_uuids=mm_uuids,
            )
2070

2071
        mm_prompt_updates = self._get_mm_prompt_updates(
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
            mm_data_items,
            hf_processor_mm_kwargs,
            mm_kwargs,
        )

        mm_info = MultiModalProcessingInfo(
            kwargs=mm_kwargs,
            hashes=mm_hashes,
            prompt_updates=mm_prompt_updates,
        )

        return prompt_ids, mm_info, is_update_applied
2084

2085
2086
    def _cached_apply_hf_processor(
        self,
2087
        prompt: str | list[int],
2088
2089
        mm_data_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
2090
        tokenization_kwargs: Mapping[str, object],
2091
        *,
2092
        mm_uuids: MultiModalUUIDDict | None = None,
2093
    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
2094
2095
2096
2097
2098
2099
        """
        Apply the HF processor on the full prompt text,
        caching the results and reusing cached results.
        """
        cache = self.cache

2100
2101
        _, passthrough_data = self._get_hf_mm_data(mm_data_items)
        if cache is None or passthrough_data:
2102
            return self._apply_hf_processor(
2103
                prompt=prompt,
2104
                mm_data_items=mm_data_items,
2105
                hf_processor_mm_kwargs=hf_processor_mm_kwargs,
2106
                tokenization_kwargs=tokenization_kwargs,
2107
                mm_uuids=mm_uuids,
2108
2109
            )

2110
2111
2112
2113
2114
2115
2116
        with _timed_operation(self.info.ctx, "hashing"):
            mm_hashes = self._hash_mm_items(
                mm_data_items,
                hf_processor_mm_kwargs,
                tokenization_kwargs,
                mm_uuids=mm_uuids,
            )
2117

2118
2119
2120
2121
2122
2123
        with _timed_operation(self.info.ctx, "cache_lookup"):
            mm_is_cached, mm_missing_data_items = self._get_cache_missing_items(
                cache=cache,
                mm_data_items=mm_data_items,
                mm_hashes=mm_hashes,
            )
2124

2125
        # NOTE: `prompt` does not correspond to `mm_missing_data_items`,
2126
        # so we can't apply prompt updates until the new multimodal
2127
2128
2129
        # items are combined with the cached multimodal items
        (
            prompt_ids,
2130
            mm_missing_processed_data,
2131
            is_update_applied,
2132
        ) = self._apply_hf_processor_main(
2133
            prompt=prompt,
2134
            mm_items=mm_missing_data_items,
2135
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
2136
            tokenization_kwargs=tokenization_kwargs,
2137
            enable_hf_prompt_update=False,
2138
2139
        )

2140
        mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
2141
            mm_missing_processed_data,
2142
2143
2144
            self._get_mm_fields_config(
                mm_missing_processed_data, hf_processor_mm_kwargs
            ),
2145
2146
        )

2147
2148
2149
2150
        mm_missing_prompt_updates = self._get_mm_prompt_updates(
            mm_missing_data_items,
            hf_processor_mm_kwargs,
            mm_missing_kwargs,
2151
        )
2152

2153
2154
2155
2156
2157
2158
2159
2160
        with _timed_operation(self.info.ctx, "cache_lookup"):
            mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
                cache,
                mm_hashes=mm_hashes,
                mm_is_cached=mm_is_cached,
                mm_missing_kwargs=mm_missing_kwargs,
                mm_missing_prompt_updates=mm_missing_prompt_updates,
            )
2161
2162
2163

        mm_info = MultiModalProcessingInfo(
            kwargs=mm_kwargs,
2164
            hashes=mm_hashes,
2165
2166
            prompt_updates=mm_prompt_updates,
        )
2167

2168
        return prompt_ids, mm_info, is_update_applied
2169

2170
2171
2172
    def _apply_token_matches(
        self,
        prompt: list[int],
2173
2174
2175
2176
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
        tokenizer = self.info.get_tokenizer()
        return apply_token_matches(prompt, mm_prompt_updates, tokenizer)
2177
2178
2179
2180

    def _apply_text_matches(
        self,
        prompt: str,
2181
2182
2183
2184
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> tuple[str, MultiModalPromptUpdatesApplyResult]:
        tokenizer = self.info.get_tokenizer()
        return apply_text_matches(prompt, mm_prompt_updates, tokenizer)
2185

2186
    def _apply_prompt_updates(
2187
2188
        self,
        token_ids: list[int],
2189
        mm_prompt_updates: MultiModalPromptUpdates,
2190
    ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
2191
        tokenizer = self.info.get_tokenizer()
2192

2193
2194
2195
2196
        new_token_ids, match_result = self._apply_token_matches(
            token_ids,
            mm_prompt_updates,
        )
2197
2198
2199
2200
2201
2202
2203
2204
2205

        # If the search text does not represent a special token,
        # it may have different token IDs in the prompt, because
        # the tokens may go across the boundaries of the search text.
        # ----
        # e.g. when searching for "foo" in "food", if "food" itself makes
        # up a token, then the token ID of "foo" will not appear at all
        # ----
        # Since it is inefficient to search for all possible tokenizations
2206
2207
        # of the search text in the prompt, we instead perform string-based
        # updates on the decoded token IDs, then encode them back.
2208
        if not all(
2209
2210
2211
            all(update_idx is not None for update_idx in update_idxs)
            for update_idxs in match_result.values()
        ):
2212
            new_text, match_result = self._apply_text_matches(
2213
                _seq2text(tokenizer, token_ids, use_cache=False),
2214
                mm_prompt_updates,
2215
2216
            )

2217
            new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False)
2218

2219
        matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
2220
2221
2222
2223
        for modality, update_idxs in match_result.items():
            for item_idx, update_idx in enumerate(update_idxs):
                assert update_idx is not None, (
                    "Failed to apply prompt replacement for "
2224
2225
                    f"mm_items[{modality!r}][{item_idx}]"
                )
2226
2227

                matched_updates[modality].append(
2228
2229
                    [mm_prompt_updates[modality][item_idx][update_idx]]
                )
2230
2231

        placeholders = self._find_mm_placeholders(
2232
2233
            new_token_ids,
            dict(matched_updates),
2234
        )
2235

2236
        return new_token_ids, placeholders
2237

2238
2239
    def _validate_mm_kwargs(
        self,
2240
        mm_kwargs: MultiModalKwargsOptionalItems,
2241
2242
2243
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
2244
            items = mm_kwargs.get(modality, [])
2245
2246
2247
2248
2249
2250
2251
2252
2253

            if len(items) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} {modality} items in "
                    f"keyword arguments corresponding to {item_count} "
                    f"{modality} data items, but only found {len(items)}! "
                    "There is likely a problem with your "
                    "implementation of merged multi-modal processor for this "
                    "model (usually arising from an inconsistency between "
2254
2255
                    "`_call_hf_processor` and `_get_mm_fields_config`)."
                )
2256

2257
    def _validate_mm_updates(
2258
        self,
2259
        mm_updates: MultiModalPromptUpdates,
2260
        mm_item_counts: Mapping[str, int],
2261
    ) -> None:
2262
        for modality, item_count in mm_item_counts.items():
2263
            placeholders = mm_updates.get(modality, [])
2264

2265
            if len(placeholders) != item_count:
2266
                raise RuntimeError(
2267
                    f"Expected there to be {item_count} prompt updates "
2268
                    f"corresponding to {item_count} {modality} items, but "
2269
                    f"instead found {len(placeholders)} prompt updates! "
2270
2271
2272
                    "This is likely because you forgot to include input "
                    "placeholder tokens (e.g., `<image>`, `<|image_pad|>`) "
                    "in the prompt. If the model has a chat template, make "
2273
2274
                    "sure you have applied it before calling `LLM.generate`."
                )
2275

2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
    def _validate_mm_placeholders(
        self,
        mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
            placeholders = mm_placeholders.get(modality, [])

            if len(placeholders) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} prompt placeholders "
                    f"corresponding to {item_count} {modality} items, but "
                    f"instead found {len(placeholders)} prompt placeholders! "
                    "Make sure the implementation of `_call_hf_processor` and "
2290
2291
                    "`_get_mm_fields_config` are consistent with each other."
                )
2292

2293
2294
2295
2296
    def _maybe_apply_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        prompt_ids: list[int],
2297
        mm_kwargs: MultiModalKwargsOptionalItems,
2298
        mm_prompt_updates: MultiModalPromptUpdates,
2299
        is_update_applied: bool,
2300
    ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
2301
        mm_item_counts = mm_items.get_all_counts()
2302
        self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
2303
        self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
2304

2305
        if is_update_applied:
2306
2307
            mm_placeholders = self._find_mm_placeholders(
                prompt_ids,
2308
                mm_prompt_updates,
2309
            )
2310
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
2311
        else:
2312
            prompt_ids, mm_placeholders = self._apply_prompt_updates(
2313
                prompt_ids,
2314
                mm_prompt_updates,
2315
            )
2316
            self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
2317

2318
        return prompt_ids, mm_placeholders
2319
2320
2321

    def apply(
        self,
2322
        prompt: str | list[int],
2323
2324
        mm_data: MultiModalDataDict,
        hf_processor_mm_kwargs: Mapping[str, object],
2325
        tokenization_kwargs: Mapping[str, object] | None = None,
2326
        *,
2327
        mm_uuids: MultiModalUUIDDict | None = None,
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
    ) -> MultiModalInputs:
        """
        Process multi-modal inputs to be used in vLLM.

        The main steps are:

        1. Apply HF Processor on prompt text and multi-modal data together,
           outputting token IDs and processed tensors.
        2. Find and update sequences in the token IDs with placeholder tokens.
           The number of placeholder tokens equals the feature size of the
           multi-modal data outputted by the multi-modal encoder.
        3. Extract information about the placeholder tokens from the
           processed token IDs.
        """
2342
2343
2344
2345
        request_id = get_current_request_id()
        if request_id is not None:
            self.info.ctx.create_timing_stats(request_id)

2346
2347
        mm_items = self._to_mm_items(mm_data)

2348
2349
2350
        if tokenization_kwargs is None:
            tokenization_kwargs = {}

2351
2352
        (
            prompt_ids,
2353
            mm_info,
2354
2355
2356
2357
2358
            is_update_applied,
        ) = self._cached_apply_hf_processor(
            prompt,
            mm_items,
            hf_processor_mm_kwargs,
2359
            tokenization_kwargs=tokenization_kwargs,
2360
            mm_uuids=mm_uuids,
2361
2362
        )

2363
        # NOTE: tokenization_kwargs are not required to init processor
2364
2365
2366
2367
2368
2369
2370
2371
        with _timed_operation(self.info.ctx, "prompt_update"):
            prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
                mm_items=mm_items,
                prompt_ids=prompt_ids,
                mm_kwargs=mm_info.kwargs,
                mm_prompt_updates=mm_info.prompt_updates,
                is_update_applied=is_update_applied,
            )
2372

2373
2374
2375
2376
        mm_placeholder_ranges = {
            modality: [item.to_range() for item in placeholders]
            for modality, placeholders in mm_placeholders.items()
        }
2377

2378
        return MultiModalInputs(
2379
            type="multimodal",
2380
            prompt_token_ids=prompt_ids,
2381
2382
            mm_kwargs=mm_info.kwargs,
            mm_hashes=mm_info.hashes,
2383
            mm_placeholders=mm_placeholder_ranges,
2384
        )
2385
2386
2387
2388
2389
2390


class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
    @abstractmethod
    def create_encoder_prompt(
        self,
2391
        prompt: str | list[int],
2392
        mm_data: MultiModalDataDict,
2393
    ) -> str | list[int]:
2394
        """
2395
        Create input prompt for the encoder. HF processor will be applied on
2396
2397
        this prompt during profiling and generation.
        """
2398
2399
        raise NotImplementedError

2400
2401
2402
2403
    @property
    def pad_dummy_encoder_prompt(self) -> bool:
        return False

2404
2405
    def create_decoder_prompt(
        self,
2406
        prompt: str | list[int],
2407
        mm_data: MultiModalDataDict,
2408
    ) -> str | list[int]:
2409
2410
2411
        """Create input prompt for the decoder."""
        return prompt

2412
    def _get_enc_dec_inputs(
2413
        self,
2414
        prompt: str | list[int],
2415
        mm_data: MultiModalDataDict,
2416
2417
        encoder_inputs: MultiModalInputs,
    ):
2418
        tokenizer = self.info.get_tokenizer()
2419
2420
        decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_data)
        if isinstance(decoder_prompt_raw, str):
2421
2422
            decoder_prompt_ids = tokenizer.encode(
                decoder_prompt_raw, add_special_tokens=False
2423
            )
2424
        else:
2425
            decoder_prompt_ids = decoder_prompt_raw
2426
2427
2428

        mm_inputs = MultiModalEncDecInputs(
            encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],
2429
2430
            **encoder_inputs,
        )
2431
        mm_inputs["prompt_token_ids"] = decoder_prompt_ids
2432
        return mm_inputs
2433
2434
2435

    def apply(
        self,
2436
        prompt: str | list[int],
2437
2438
        mm_data: MultiModalDataDict,
        hf_processor_mm_kwargs: Mapping[str, object],
2439
        tokenization_kwargs: Mapping[str, object] | None = None,
2440
        *,
2441
        mm_uuids: MultiModalUUIDDict | None = None,
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
    ) -> MultiModalEncDecInputs:
        """
        Process multi-modal inputs to be used in vLLM.
        The main processing steps are modified to fit encoder-decoder model:
        1. Create encoder prompt from input prompt text.
        2. Apply the HF processor on encoder prompt.
        3. Copy the input prompt text as decoder prompt inputs.
        """
        encoder_prompt = self.create_encoder_prompt(prompt, mm_data)
        encoder_inputs = super().apply(
            encoder_prompt,
            mm_data,
            hf_processor_mm_kwargs,
2455
            tokenization_kwargs,
2456
            mm_uuids=mm_uuids,
2457
2458
2459
2460
2461
2462
2463
        )

        return self._get_enc_dec_inputs(
            prompt=prompt,
            mm_data=mm_data,
            encoder_inputs=encoder_inputs,
        )