processing.py 42.2 KB
Newer Older
1
2
import re
from abc import ABC, abstractmethod
3
from collections import defaultdict
4
5
from collections.abc import (Callable, Generator, ItemsView, Iterable, Mapping,
                             Sequence)
6
from dataclasses import dataclass, field
7
from functools import lru_cache
8
9
from typing import (TYPE_CHECKING, Generic, NamedTuple, Optional, Protocol,
                    TypeVar, Union)
10

11
from transformers import BatchFeature, PretrainedConfig, ProcessorMixin
12

13
14
import vllm.envs as envs
from vllm.inputs import InputProcessingContext
15
from vllm.logger import init_logger
16
17
from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
                                               encode_tokens)
18
from vllm.utils import LRUCache, flatten_2d_lists, full_groupby
19

20
from .hasher import MultiModalHasher
21
from .inputs import (MultiModalDataDict, MultiModalFieldConfig,
22
23
                     MultiModalInputs, MultiModalKwargs, MultiModalKwargsItem,
                     PlaceholderRange)
24
from .parse import MultiModalDataItems, MultiModalDataParser
25
26
27

if TYPE_CHECKING:
    from .profiling import BaseDummyInputsBuilder
28

29
logger = init_logger(__name__)
30
31

_S = TypeVar("_S", str, list[int])
32
33
34

PromptSeq = Union[str, list[int]]
"""A token sequence (list of token IDs) or text."""
35

36

37
38
@dataclass
class PromptReplacementDetails:
39
40
41
    """Details about the replacement token sequence or text."""

    full: PromptSeq
42
43
    """The full replacement."""

44
    features: PromptSeq
45
    """
46
47
48
    The part of the replacement that corresponds to feature placeholders;
    this will be replaced by the output of the vision encoder during model
    inference.
49
50
51
    """

    @staticmethod
52
    def from_seq(seq: PromptSeq) -> "PromptReplacementDetails":
53
54
55
        return PromptReplacementDetails(full=seq, features=seq)


56
57
58
59
60
61
62
PromptRepl = Union[PromptSeq, PromptReplacementDetails]
"""
The replacement token sequence or text.

If only part of the replacement corresponds to feature placeholders, you can
use :class:`PromptReplacementDetails` to specify which part.
"""
63
64


65
@dataclass
66
class PromptReplacement:
67
68
    """
    Defines how to replace portions of an input prompt with placeholder tokens.
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

    Example:

        For each image, replace one ``<image>`` input placeholder in the prompt
        with a number of ``<image>`` feature placeholders
        equal to the feature size of the vision encoder:

        .. code-block:: python

            PromptReplacement(
                modality="image",
                target="<image>",
                replacement="<image>" * image_feature_size,
            )

        As above, but further pad the feature placeholders with ``<image_bos>``
        and `<image_eos>``, which are not supposed to be passed to the vision
        encoder:

        .. code-block:: python

            PromptReplacement(
                modality="image",
                target="<image>",
                replacement=PromptReplacementDetails(
                    full="".join([
                        "<image_bos>",
                        "<image>" * image_feature_size,
                        "<image_eos>",
                    ]),
                    features="<image>" * image_feature_size,
                ),
            )

        To avoid unnecessary tokenization during prompt replacement,
        we recommended passing token sequences instead of text:

        .. code-block:: python

            PromptReplacement(
                modality="image",
                target=[image_token_id],
                replacement=PromptReplacementDetails(
                    full=([image_bos_id] + [image_token_id] * image_feature_size
                          + [image_eos_id]),
                    features=[image_token_id] * image_feature_size,
                ),
            )
117
118
    """

119
    modality: str
120
    """The modality for which the replacement is made."""
121

122
    target: PromptSeq
123
    """The token sequence (or text) to find and replace."""
124

125
126
    replacement: Union[Callable[[int], PromptRepl],
                       PromptRepl] = field(repr=False)
127
    """
128
129
    Given the index of the processed item within :attr:`modality`,
    output the replacement token sequence (or text).
130

131
132
    For convenience, you can directly pass in the replacement token sequence
    (or text) instead of a function if it does not depend on the input.
133
134
    """

135
136
    def bind(self, tokenizer: AnyTokenizer) -> "BoundPromptReplacement":
        return BoundPromptReplacement(
137
138
139
140
            tokenizer=tokenizer,
            modality=self.modality,
            _target=self.target,
            _replacement=self.replacement,
141
        )
142
143


144
145
146
147
148
149
150
@lru_cache(maxsize=2048)
def _cached_encode(
    tokenizer: AnyTokenizer,
    text: str,
    *,
    add_special_tokens: bool = False,
) -> list[int]:
151
152
153
    return encode_tokens(tokenizer,
                         text,
                         add_special_tokens=add_special_tokens)
154
155


156
157
158
159
160
161
162
@lru_cache(maxsize=2048)
def _cached_decode(
    tokenizer: AnyTokenizer,
    token_ids: tuple[int, ...],
    *,
    skip_special_tokens: bool = False,
) -> str:
163
164
165
    return decode_tokens(tokenizer,
                         list(token_ids),
                         skip_special_tokens=skip_special_tokens)
166
167
168
169
170


class _HasModalityAttr(Protocol):
    modality: str

171

172
class _HasModalityProp(Protocol):
173

174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
    @property
    def modality(self) -> str:
        ...


_M = TypeVar("_M", bound=Union[_HasModalityAttr, _HasModalityProp])


def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]:
    """Convenience function to apply :func:`full_groupby` based on modality."""
    return full_groupby(values, key=lambda x: x.modality)


@dataclass
class _BoundPromptSequence:
189
190
191
192
    """
    A :data:`_PromptSeq` bound to a tokenizer to automatically
    convert between token sequence and text representations.
    """
193
194
    tokenizer: AnyTokenizer = field(repr=False)

195
196
197
    _text: Optional[str]
    _token_ids: Optional[list[int]]

198
    @staticmethod
199
200
    def from_seq(
        tokenizer: AnyTokenizer,
201
        seq: PromptSeq,
202
    ) -> "_BoundPromptSequence":
203
204
205
206
207
208
        return _BoundPromptSequence(
            tokenizer=tokenizer,
            _text=seq if isinstance(seq, str) else None,
            _token_ids=seq if isinstance(seq, list) else None,
        )

209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
    def __post_init__(self) -> None:
        if self._text is None and self._token_ids is None:
            raise ValueError("At least one of 'text' and 'token_ids' must be "
                             "specified")

    @property
    def text(self) -> str:
        if self._text is None:
            assert self._token_ids is not None
            self._text = _cached_decode(self.tokenizer, tuple(self._token_ids))

        return self._text

    @property
    def token_ids(self) -> list[int]:
        if self._token_ids is None:
            assert self._text is not None
            self._token_ids = _cached_encode(self.tokenizer, self._text)

        return self._token_ids


231
232
233
234
235
236
@dataclass
class _BoundPromptReplacementGroup:
    full: _BoundPromptSequence
    features: _BoundPromptSequence


237
@dataclass
238
class BoundPromptReplacement:
239
240
241
242
243
    """
    A :class:`PromptReplacement` bound to a tokenizer to automatically
    convert :attr:`target` and the result of :meth:`get_replacement` between
    token sequence and text representations.
    """
244
    tokenizer: AnyTokenizer = field(repr=False)
245
246
    modality: str

247
248
249
    _target: PromptSeq
    _replacement: Union[Callable[[int], PromptRepl],
                        PromptRepl] = field(repr=False)
250

251
    def __post_init__(self) -> None:
252
        self._replacement_cache = dict[int, _BoundPromptReplacementGroup]()
253
254
255

    @property
    def target(self) -> _BoundPromptSequence:
256
        """The token sequence (or text) to find and replace."""
257
        return _BoundPromptSequence.from_seq(self.tokenizer, self._target)
258

259
    def get_replacement(self, item_idx: int) -> _BoundPromptReplacementGroup:
260
261
262
263
        """
        Given the index of the processed item within :attr:`modality`,
        output the replacement token sequence (or text).
        """
264
265
266
267
268
269
270
271
272
273
        replacement = self._replacement
        if callable(replacement):
            cache_key = item_idx
            if cache_key in self._replacement_cache:
                return self._replacement_cache[cache_key]

            replacement = replacement(item_idx)
        else:
            cache_key = None

274
275
276
277
278
279
280
281
282
283
        if not isinstance(replacement, PromptReplacementDetails):
            replacement = PromptReplacementDetails.from_seq(replacement)

        bound_full = _BoundPromptSequence.from_seq(self.tokenizer,
                                                   replacement.full)
        bound_features = _BoundPromptSequence.from_seq(self.tokenizer,
                                                       replacement.features)
        bound_replacement = _BoundPromptReplacementGroup(
            full=bound_full,
            features=bound_features,
284
285
286
287
288
289
290
291
        )

        if cache_key is not None:
            self._replacement_cache[cache_key] = bound_replacement

        return bound_replacement


292
293
294
class _TokenMatch(NamedTuple):
    start_idx: int
    end_idx: int
295
296


297
298
299
def iter_token_matches(
    token_ids: list[int],
    match_ids: list[int],
300
) -> Generator[_TokenMatch]:
301
302
303
304
305
306
    """
    Yield each occurrence of :code:`match_ids` in :code:`token_ids`.

    Note that empty matches are ignored.
    """
    prompt_len = len(token_ids)
307
    match_len = len(match_ids)
308

309
310
    if match_len == 0:
        return
311

312
313
    start_idx = 0
    while start_idx < prompt_len - match_len + 1:
314
        end_idx = start_idx + match_len
315

316
317
        if token_ids[start_idx:end_idx] == match_ids:
            yield _TokenMatch(start_idx=start_idx, end_idx=end_idx)
318
319
320
321
322

            # Exclude overlapping matches
            start_idx = end_idx
        else:
            start_idx += 1
323
324


325
326
@dataclass(repr=False)
class _PromptReplacementMatch(ABC):
327
    prompt_repl: BoundPromptReplacement
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348

    @property
    def modality(self) -> str:
        return self.prompt_repl.modality

    @property
    @abstractmethod
    def start_idx(self) -> int:
        raise NotImplementedError

    @property
    @abstractmethod
    def end_idx(self) -> int:
        raise NotImplementedError

    def __repr__(self) -> str:
        return (f"{type(self).__name__}(modality={self.modality!r}, "
                f"start_idx={self.start_idx!r}, end_idx={self.end_idx!r})")


@dataclass(repr=False)
349
class _PromptReplacementTokenMatch(_PromptReplacementMatch):
350
351
352
353
354
355
356
357
358
359
360
361
    match: _TokenMatch

    @property
    def start_idx(self) -> int:
        return self.match.start_idx

    @property
    def end_idx(self) -> int:
        return self.match.end_idx


@dataclass(repr=False)
362
class _PromptReplacementTextMatch(_PromptReplacementMatch):
363
364
365
366
367
368
369
370
371
372
    match: re.Match[str]

    @property
    def start_idx(self) -> int:
        return self.match.start()

    @property
    def end_idx(self) -> int:
        return self.match.end()

373

374
@dataclass
375
class PlaceholderFeaturesInfo:
376
    modality: str
377
    item_idx: int
378
    start_idx: int
379
    tokens: list[int]
380
381
382

    @property
    def length(self) -> int:
383
        return len(self.tokens)
384
385
386
387
388
389

    def to_range(self) -> PlaceholderRange:
        return PlaceholderRange(
            offset=self.start_idx,
            length=self.length,
        )
390
391
392
393


def find_token_matches(
    prompt: list[int],
394
    prompt_repls: Sequence[BoundPromptReplacement],
395
) -> list[_PromptReplacementTokenMatch]:
396
397
398
399
400
401
402
403
404
405
    """Return each target of :code:`prompt_repls` found in :code:`prompt`."""
    return [
        _PromptReplacementTokenMatch(prompt_repl, match)
        for prompt_repl in prompt_repls
        for match in iter_token_matches(prompt, prompt_repl.target.token_ids)
    ]


def find_text_matches(
    prompt: str,
406
    prompt_repls: Sequence[BoundPromptReplacement],
407
) -> list[_PromptReplacementTextMatch]:
408
409
410
411
412
413
414
415
416
    """Return each target of :code:`prompt_repls` found in :code:`prompt`."""
    return [
        _PromptReplacementTextMatch(prompt_repl, match)
        for prompt_repl in prompt_repls
        for match in re.finditer(re.escape(prompt_repl.target.text), prompt)
    ]


def _resolve_matches(
417
    prompt: PromptSeq,
418
    mm_matches: Mapping[str, Sequence[_PromptReplacementMatch]],
419
) -> list[_PromptReplacementMatch]:
420
    """
421
    Resolve :code:`mm_matches` to ensure that there are no overlapping matches,
422
    and sort them such that earlier matches take priority over later ones.
423
    """
424
425
    matches = [m for matches in mm_matches.values() for m in matches]

426
427
    seen_matches: list[Optional[_PromptReplacementMatch]] = [None
                                                             ] * len(prompt)
428

429
    for match in matches:
430
431
432
433
434
        for idx in range(match.start_idx, match.end_idx):
            if seen_matches[idx] is not None:
                raise ValueError("Found overlapping matches "
                                 f"({seen_matches[idx]} and {match}) "
                                 f"at index={idx} of prompt={prompt}")
435

436
            seen_matches[idx] = match
437
438
439
440
441
442

    return sorted(matches, key=lambda x: x.start_idx)


def _replace_matches(
    prompt: _S,
443
    mm_matches: Mapping[str, Sequence[_PromptReplacementMatch]],
444
    mm_item_counts: Mapping[str, int],
445
) -> list[_S]:
446
    """Apply the replacements in :code:`mm_matches` to :code:`prompt`."""
447
448
    out_seqs = list[_S]()
    prev_end_idx = 0
449
    next_idx_by_modality = defaultdict[str, int](lambda: 0)
450

451
    for match in _resolve_matches(prompt, mm_matches):
452
453
454
        modality = match.modality

        item_idx = next_idx_by_modality[modality]
455
        if item_idx >= mm_item_counts.get(modality, 0):
456
457
458
459
            continue

        start_idx = match.start_idx
        end_idx = match.end_idx
460

461
        repl_info = match.prompt_repl
462
463
464
        replacement = repl_info.get_replacement(item_idx)

        if isinstance(prompt, str):
465
            repl_seq = replacement.full.text
466
467
            out_seqs.append(prompt[prev_end_idx:start_idx] + repl_seq)
        else:
468
            repl_seq = replacement.full.token_ids
469
            out_seqs.append(prompt[prev_end_idx:start_idx] + repl_seq)
470
471
472
473
474
475
476
477
478
479
480

        prev_end_idx = end_idx
        next_idx_by_modality[modality] += 1

    out_seqs.append(prompt[prev_end_idx:])

    return out_seqs


def replace_token_matches(
    prompt: list[int],
481
    mm_matches: Mapping[str, Sequence[_PromptReplacementTokenMatch]],
482
    mm_item_counts: Mapping[str, int],
483
) -> list[int]:
484
485
    """Apply the replacements in :code:`mm_matches` to :code:`prompt`."""
    if not mm_matches:
486
487
        return prompt

488
    token_id_seqs = _replace_matches(prompt, mm_matches, mm_item_counts)
489
490

    return flatten_2d_lists(token_id_seqs)
491
492


493
494
def replace_text_matches(
    prompt: str,
495
    mm_matches: Mapping[str, Sequence[_PromptReplacementTextMatch]],
496
    mm_item_counts: Mapping[str, int],
497
) -> str:
498
499
    """Apply the replacements in :code:`mm_matches` to :code:`prompt`."""
    if not mm_matches:
500
        return prompt
501

502
    texts = _replace_matches(prompt, mm_matches, mm_item_counts)
503
504

    return "".join(texts)
505
506


507
508
def _iter_placeholders(
    mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
509
    prompt: list[int],
510
    mm_item_counts: Mapping[str, int],
511
) -> Iterable[PlaceholderFeaturesInfo]:
512
513
514
515
516
517
    """
    Yield each set of placeholder tokens found in :code:`prompt`.

    Matches are exclusive even when multiple modalities share
    the same placeholder tokens. In that case, the modality that
    appears earlier in `mm_prompt_repls` takes priority.
518

519
520
    Note that empty matches are ignored.
    """
521
    prompt_len = len(prompt)
522
    item_idx_by_modality = defaultdict[str, int](lambda: 0)
523
524
525
526
527

    start_idx = 0
    while start_idx < prompt_len:
        found = False

528
529
530
        for modality, modality_repls in mm_prompt_repls.items():
            item_idx = item_idx_by_modality[modality]
            if item_idx >= mm_item_counts.get(modality, 0):
531
                continue
532

533
534
            for repl_info in modality_repls:
                replacement = repl_info.get_replacement(item_idx)
535
536
537
                repl_tokens_full = replacement.full.token_ids
                repl_len_full = len(repl_tokens_full)
                end_idx_full = start_idx + repl_len_full
538

539
                if repl_len_full == 0 or end_idx_full > prompt_len:
540
541
                    continue

542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
                if prompt[start_idx:end_idx_full] == repl_tokens_full:
                    repl_tokens_feat = replacement.features.token_ids

                    try:
                        match = next(
                            iter_token_matches(repl_tokens_full,
                                               repl_tokens_feat))
                        yield PlaceholderFeaturesInfo(
                            modality=modality,
                            item_idx=item_idx,
                            start_idx=start_idx + match.start_idx,
                            tokens=repl_tokens_feat,
                        )
                    except StopIteration:
                        raise AssertionError(
                            f"{repl_tokens_feat=} should be a "
                            f"subsequence of {repl_tokens_full=}") from None
559

560
                    # Exclude overlapping matches
561
                    start_idx = end_idx_full
562
563
564
                    item_idx_by_modality[modality] += 1
                    found = True
                    break
565

566
567
            if found:
                break  # Go back to the outer while loop
568
569
570

        if not found:
            start_idx += 1
571
572


573
def find_mm_placeholders(
574
    mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
575
576
    prompt: list[int],
    mm_item_counts: Mapping[str, int],
577
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
578
579
580
581
    it = _iter_placeholders(mm_prompt_repls, prompt, mm_item_counts)
    return dict(full_groupby_modality(it))


582
583
584
585
586
587
588
589
class ProcessingCache:

    def __init__(self, capacity: int) -> None:
        super().__init__()

        # DEBUG: Set to None to disable
        self.debug_cache_hit_ratio_steps: Optional[int] = None

590
        self._cache = LRUCache[str, MultiModalKwargsItem](capacity)
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607

    def _maybe_log_cache_stats(self) -> None:
        steps = self.debug_cache_hit_ratio_steps
        if not steps:
            return

        cache_stats = self._cache.stat()
        if cache_stats.total % steps == 0:
            logger.debug("ProcessingCache: hit_ratio = %.2f",
                         cache_stats.hit_ratio)

    def get(
        self,
        model_id: str,
        modality: str,
        input_item: object,
        input_kwargs: Mapping[str, object],
608
    ) -> Optional[MultiModalKwargsItem]:
609
610
611
612
613
614
615
616
617
618
619
        """
        Get a processed multi-modal item from the cache
        according to its dependencies, including:

        - The model ID
        - The modality of the item
        - The original data item passed to the HF processor
        - The configuration options of the HF processor
        """
        self._maybe_log_cache_stats()

620
621
622
        cache_key = MultiModalHasher.hash_kwargs(model_id=model_id,
                                                 **{modality: input_item},
                                                 **input_kwargs)
623
624
625
626
627
628
629
630
        return self._cache.get(cache_key)

    def put(
        self,
        model_id: str,
        modality: str,
        input_item: object,
        input_kwargs: Mapping[str, object],
631
        output_kwargs: MultiModalKwargsItem,
632
633
634
635
636
    ) -> None:
        """
        Put a processed multi-modal item into the cache
        according to its dependencies (see :meth:`get`).
        """
637
638
639
        cache_key = MultiModalHasher.hash_kwargs(model_id=model_id,
                                                 **{modality: input_item},
                                                 **input_kwargs)
640
        self._cache.put(cache_key, output_kwargs)
641
642


643
class BaseProcessingInfo:
644
    """Base class to provide the information necessary for data processing."""
645

646
647
    def __init__(self, ctx: InputProcessingContext) -> None:
        super().__init__()
648

649
650
651
652
653
654
655
        self.ctx = ctx

    @property
    def model_id(self) -> str:
        return self.ctx.model_config.model

    def get_tokenizer(self) -> AnyTokenizer:
656
657
        return self.ctx.tokenizer

658
    def get_hf_config(self) -> PretrainedConfig:
659
660
        return self.ctx.get_hf_config()

661
    def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
662
663
664
665
666
667
        """
        Subclasses can override this method to handle
        specific kwargs from model config or user inputs.
        """
        return self.ctx.get_hf_processor(**kwargs)

668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
    @abstractmethod
    def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
        """
        Return the maximum supported number of items for each modality.

        A value of `None` means unlimited number of items.

        Omitting a modality from the returned dictionary means that
        it is not supported at all.
        """
        raise NotImplementedError

    @abstractmethod
    def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
        """
        Get the maximum possible number of tokens per data item
        for each modality.

        The dictionary returned by this method should have the same
        keys as that returned by :meth:`get_supported_mm_limits`.
        """
        raise NotImplementedError


_I = TypeVar("_I", bound=BaseProcessingInfo)
693

694
695

class BaseMultiModalProcessor(ABC, Generic[_I]):
696
    """
697
    Abstract base class to process multi-modal inputs to be used in vLLM.
698
699

    Not to be confused with :class:`transformers.ProcessorMixin`.
700
701
    """

702
    def __init__(self,
703
704
                 info: _I,
                 dummy_inputs: "BaseDummyInputsBuilder[_I]",
705
706
707
                 *,
                 cache: Optional[ProcessingCache] = None,
                 enable_sanity_checks: bool = True) -> None:
708
709
        super().__init__()

710
711
        self.info = info
        self.dummy_inputs = dummy_inputs
712
713
        self.cache = cache
        self.enable_sanity_checks = enable_sanity_checks
714

715
716
        self.data_parser = self._get_data_parser()

717
    def __call__(
718
        self,
719
720
        prompt: str,
        mm_data: MultiModalDataDict,
721
        hf_processor_mm_kwargs: Mapping[str, object],
722
    ) -> MultiModalInputs:
723
        return self.apply(prompt, mm_data, hf_processor_mm_kwargs)
724

725
726
    def _get_data_parser(self) -> MultiModalDataParser:
        """
727
        Construct a parser to preprocess multi-modal data items
728
729
730
731
732
733
734
735
        before passing them to :meth:`_get_hf_mm_data`.

        You can support additional modalities by creating a subclass
        of :class:`MultiModalDataParser` that has additional subparsers.
        """
        return MultiModalDataParser()

    def _to_mm_items(
736
737
738
        self,
        mm_data: MultiModalDataDict,
    ) -> MultiModalDataItems:
739
740
741
742
        """
        Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`
        before passing them to :meth:`_get_hf_mm_data`.
        """
743
        mm_items = self.data_parser.parse_mm_data(mm_data)
744

745
        mm_limits = self.info.ctx.get_mm_config().limit_per_prompt
746
747
748
749
750
751
752
753
754
        for modality, items in mm_items.items():
            limit = mm_limits.get(modality, 1)
            if len(items) > limit:
                raise ValueError(
                    f"You set {modality}={limit} (or defaulted to 1) in "
                    f"`--limit-mm-per-prompt`, but passed {len(items)} "
                    f"{modality} items in the same prompt.")

        return mm_items
755

756
757
758
759
760
761
762
763
764
    @abstractmethod
    def _get_mm_fields_config(
        self,
        hf_inputs: BatchFeature,
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, MultiModalFieldConfig]:
        """Given the HF-processed data, output the metadata of each field."""
        raise NotImplementedError

765
766
    @abstractmethod
    def _get_prompt_replacements(
767
        self,
768
        mm_items: MultiModalDataItems,
769
770
        hf_processor_mm_kwargs: Mapping[str, object],
        out_mm_kwargs: MultiModalKwargs,
771
772
773
774
775
    ) -> list[PromptReplacement]:
        """
        Given the original multi-modal items for this modality
        and HF-processed data, output the replacements to perform.

776
777
778
779
780
781
782
783
        Notes:
            - You should not assume that HF processor always performs prompt
              replacement: in :meth:`_apply_hf_processor_missing`, this method
              is called on text-only and multimodal-only inputs separately,
              instead of passing them in the same call.
            - The replacement information returned by this method is also used
              to determine the placeholder token positions for each multi-modal
              item.
784
785
        """
        raise NotImplementedError
786

787
    def _find_mm_placeholders(
788
        self,
789
        mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
790
        new_token_ids: list[int],
791
        mm_item_counts: Mapping[str, int],
792
    ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
793
794
        return find_mm_placeholders(mm_prompt_repls, new_token_ids,
                                    mm_item_counts)
795

796
    def _get_hf_mm_data(
797
        self,
798
        mm_items: MultiModalDataItems,
799
800
801
    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
        processor_data = dict[str, object]()
        passthrough_data = dict[str, object]()
802

803
804
805
        for items in mm_items.values():
            processor_data.update(items.get_processor_data())
            passthrough_data.update(items.get_passthrough_data())
806

807
808
        return processor_data, passthrough_data

809
810
811
    def _call_hf_processor(
        self,
        prompt: str,
812
813
814
815
        # Not to be confused with `mm_data` in `self.apply`.
        # This refers to the data to be passed to HF processor.
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
816
    ) -> BatchFeature:
817
818
819
820
        """
        Call the HF processor on the prompt text and
        associated multi-modal data.
        """
821
822
        return self.info.ctx.call_hf_processor(
            self.info.get_hf_processor(**mm_kwargs),
823
824
            dict(text=prompt, **mm_data),
            mm_kwargs,
825
826
        )

827
    def _apply_hf_processor_text_mm(
828
        self,
829
        prompt_text: str,
830
        mm_items: MultiModalDataItems,
831
832
833
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> tuple[list[int], MultiModalKwargs]:
        """
834
835
        Apply the HF processor on the prompt text and multi-modal data
        together.
836
837
838
839
840
841
842
843
844
        """
        processor_data, passthrough_data = self._get_hf_mm_data(mm_items)

        processed_data = self._call_hf_processor(
            prompt=prompt_text,
            mm_data=processor_data,
            mm_kwargs=hf_processor_mm_kwargs,
        )
        processed_data.update(passthrough_data)
845

846
        prompt_ids, = processed_data.pop("input_ids").tolist()
847

848
849
850
        mm_kwargs = MultiModalKwargs.from_hf_inputs(
            processed_data,
            self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs),
851
        )
852

853
854
        return prompt_ids, mm_kwargs

855
    def _apply_hf_processor_text_only(self, prompt_text: str) -> list[int]:
856
        """
857
        Apply the HF processor on the prompt text only.
858

859
860
861
        Since HF processor requires that text and multi-modal items
        correspond to each other, we create dummy multi-modal items
        to go along with the text.
862
        """
863
        prompt_ids, _ = self._apply_hf_processor_text_mm(
864
865
866
867
868
            prompt_text=prompt_text,
            mm_items=MultiModalDataItems({}),
            hf_processor_mm_kwargs={},
        )

869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
        return prompt_ids

    def _apply_hf_processor_tokens_only(
        self,
        prompt_tokens: list[int],
    ) -> list[int]:
        """
        Apply the HF processor on the prompt tokens only.

        Most HF processors accept prompt text but not prompt tokens.
        If the HF processor adds or removes tokens that are not related to
        multi-modal data, you should override this method so it is consistent
        with the output of :meth:`_apply_hf_processor_text_only` on the
        corresponding text.
        """
        return prompt_tokens

    def _apply_hf_processor_mm_only(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> MultiModalKwargs:
        """
        Apply the HF processor on the multi-modal data only.

        Since HF processor requires that text and multi-modal items
        correspond to each other, we generate dummy text using
        :class:`DummyInputsBuilder` to go along with the multi-modal data.
        """
        mm_counts = mm_items.get_all_counts()

900
901
        dummy_inputs = self.dummy_inputs.get_dummy_processor_inputs(
            self.info.ctx.model_config.max_model_len,
902
            mm_counts,
903
        )
904

905
        _, mm_kwargs = self._apply_hf_processor_text_mm(
906
            prompt_text=dummy_inputs.prompt_text,
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
            mm_items=mm_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
        )

        return mm_kwargs

    def _apply_hf_processor_main(
        self,
        prompt: Union[str, list[int]],
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        *,
        enable_hf_prompt_replacement: bool,
    ) -> tuple[list[int], MultiModalKwargs]:
        """
        Apply the HF processor on the prompt text and multi-modal data.

        Note:
            If :code:`enable_hf_prompt_replacement=False`, the prompt should
            correspond to the multi-modal items.
        """
        if isinstance(prompt, str):
            if enable_hf_prompt_replacement:
                return self._apply_hf_processor_text_mm(
                    prompt_text=prompt,
                    mm_items=mm_items,
                    hf_processor_mm_kwargs=hf_processor_mm_kwargs,
                )

            prompt_ids = self._apply_hf_processor_text_only(prompt)
        else:
            prompt_ids = self._apply_hf_processor_tokens_only(prompt)

        mm_missing_kwargs = self._apply_hf_processor_mm_only(
            mm_items=mm_items,
942
943
944
945
946
947
948
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
        )

        return prompt_ids, mm_missing_kwargs

    def _cached_apply_hf_processor(
        self,
949
        prompt: Union[str, list[int]],
950
951
952
953
954
955
956
957
        mm_data_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> tuple[list[int], MultiModalKwargs]:
        """
        Apply the HF processor on the full prompt text,
        caching the results and reusing cached results.
        """
        cache = self.cache
958
        model_id = self.info.model_id
959

960
961
        _, passthrough_data = self._get_hf_mm_data(mm_data_items)
        if cache is None or passthrough_data:
962
963
            return self._apply_hf_processor_main(
                prompt=prompt,
964
965
                mm_items=mm_data_items,
                hf_processor_mm_kwargs=hf_processor_mm_kwargs,
966
                enable_hf_prompt_replacement=True,
967
968
            )

969
        mm_maybe_cached_kw_items = {
970
971
972
973
974
975
976
977
            modality: [
                cache.get(model_id, modality, item, hf_processor_mm_kwargs)
                for item in items
            ]
            for modality, items in mm_data_items.items()
        }

        mm_missing_idxs = {
978
979
980
            modality:
            [idx for idx, item in enumerate(kw_items) if item is None]
            for modality, kw_items in mm_maybe_cached_kw_items.items()
981
982
983
984
985
        }
        mm_missing_data = {
            modality: [mm_data_items[modality][idx] for idx in idxs]
            for modality, idxs in mm_missing_idxs.items()
        }
986
        mm_missing_data_items = self._to_mm_items(mm_missing_data)
987

988
989
990
991
992
        # NOTE: `prompt` does not correspond to `mm_missing_data_items`,
        # so we need to pass `enable_hf_prompt_replacement=False`
        prompt_ids, mm_missing_kwargs = self._apply_hf_processor_main(
            prompt=prompt,
            mm_items=mm_missing_data_items,
993
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
994
            enable_hf_prompt_replacement=False,
995
996
997
998
999
1000
1001
        )

        mm_missing_next_idx = {
            modality: 0
            for modality in mm_missing_data_items
        }

1002
1003
1004
1005
1006
        merged_kw_items = list[MultiModalKwargsItem]()
        for modality, kw_items in mm_maybe_cached_kw_items.items():
            for idx, kw_item in enumerate(kw_items):
                if kw_item is None:
                    kw_item = mm_missing_kwargs.get_item(
1007
1008
1009
1010
1011
1012
1013
1014
1015
                        modality,
                        mm_missing_next_idx[modality],
                    )

                    cache.put(
                        model_id,
                        modality,
                        mm_data_items[modality][idx],
                        hf_processor_mm_kwargs,
1016
                        kw_item,
1017
1018
1019
1020
                    )

                    mm_missing_next_idx[modality] += 1

1021
                merged_kw_items.append(kw_item)
1022
1023

        if self.enable_sanity_checks:
1024
            mm_missing_counts = mm_missing_data_items.get_all_counts()
1025
1026
1027
1028
1029
1030
            assert all(
                item_count == mm_missing_counts[modality]
                for modality, item_count in mm_missing_next_idx.items()), dict(
                    mm_missing_next_idx=mm_missing_next_idx,
                    mm_missing_counts=mm_missing_counts)

1031
        mm_kwargs = MultiModalKwargs.from_items(merged_kw_items)
1032
1033

        return prompt_ids, mm_kwargs
1034

1035
    def _bind_and_group_repls(
1036
        self,
1037
        prompt_repls: list[PromptReplacement],
1038
1039
    ) -> dict[str, list[BoundPromptReplacement]]:
        tokenizer = self.info.get_tokenizer()
1040

1041
1042
        it = (prompt_repl.bind(tokenizer) for prompt_repl in prompt_repls)
        return dict(full_groupby_modality(it))
1043

1044
1045
1046
1047
    def _always_apply_prompt_replacements(self) -> bool:
        """
        A flag which can be overridden so that
        :meth:`_apply_prompt_replacements` is always called even if we
1048
1049
        detect that HF has performed processing via
        :meth:`_find_placeholders_by_modality`.
1050

1051
1052
        This is useful in cases where :meth:`_find_placeholders_by_modality`
        cannot be reliably used to detect whether HF has performed processing.
1053
1054
1055
        """
        return False

1056
1057
1058
    def _apply_prompt_replacements(
        self,
        token_ids: list[int],
1059
        mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
1060
        mm_item_counts: Mapping[str, int],
1061
    ) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
1062
        tokenizer = self.info.get_tokenizer()
1063

1064
1065
1066
1067
        mm_token_matches = {
            modality: find_token_matches(token_ids, prompt_repls)
            for modality, prompt_repls in mm_prompt_repls.items()
        }
1068
1069
        mm_match_counts = {
            modality: len(matches)
1070
            for modality, matches in mm_token_matches.items()
1071
        }
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083

        # If the search text does not represent a special token,
        # it may have different token IDs in the prompt, because
        # the tokens may go across the boundaries of the search text.
        # ----
        # e.g. when searching for "foo" in "food", if "food" itself makes
        # up a token, then the token ID of "foo" will not appear at all
        # ----
        # Since it is inefficient to search for all possible tokenizations
        # of the search text in the prompt, we instead perform string
        # replacement on the decoded token IDs, then encode them back.
        if all(
1084
1085
            mm_match_counts.get(modality, 0) >= item_count
            for modality, item_count in mm_item_counts.items()
1086
1087
1088
        ):  # yapf: disable
            token_ids = replace_token_matches(
                token_ids,
1089
                mm_token_matches,
1090
                mm_item_counts,
1091
1092
            )

1093
1094
1095
1096
1097
            text = decode_tokens(tokenizer, token_ids)
            matched_repls = {
                modality: [match.prompt_repl for match in token_matches]
                for modality, token_matches in mm_token_matches.items()
            }
1098
        else:
1099
            text = decode_tokens(tokenizer, token_ids)
1100

1101
1102
1103
1104
            mm_text_matches = {
                modality: find_text_matches(text, prompt_repls)
                for modality, prompt_repls in mm_prompt_repls.items()
            }
1105
1106
            text = replace_text_matches(
                text,
1107
                mm_text_matches,
1108
                mm_item_counts,
1109
1110
            )

1111
1112
1113
            token_ids = encode_tokens(tokenizer,
                                      text,
                                      add_special_tokens=False)
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
            matched_repls = {
                modality: [match.prompt_repl for match in token_matches]
                for modality, token_matches in mm_text_matches.items()
            }

        placeholders = self._find_mm_placeholders(
            matched_repls,
            token_ids,
            mm_item_counts,
        )
1124
1125

        return token_ids, text, placeholders
1126

1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
    def _validate_mm_kwargs(
        self,
        mm_kwargs: MultiModalKwargs,
        mm_item_counts: Mapping[str, int],
    ) -> None:
        for modality, item_count in mm_item_counts.items():
            if modality in mm_kwargs.modalities:
                items = mm_kwargs.get_items(modality)
            else:
                items = []

            if len(items) != item_count:
                raise RuntimeError(
                    f"Expected there to be {item_count} {modality} items in "
                    f"keyword arguments corresponding to {item_count} "
                    f"{modality} data items, but only found {len(items)}! "
                    "There is likely a problem with your "
                    "implementation of merged multi-modal processor for this "
                    "model (usually arising from an inconsistency between "
                    "`_call_hf_processor` and `_get_mm_fields_config`).")

    def _validate_mm_placeholders(
        self,
1150
        mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
        mm_item_counts: Mapping[str, int],
        *,
        allow_missing: bool = False,
    ) -> Mapping[str, int]:
        missing_repl_counts = dict[str, int]()

        for modality, item_count in mm_item_counts.items():
            placeholders = mm_placeholders.get(modality, [])

            if len(placeholders) != item_count and not allow_missing:
                raise RuntimeError(
                    f"Expected there to be {item_count} prompt replacements "
                    f"corresponding to {item_count} {modality} items, but only "
                    f"found {len(placeholders)} prompt replacements! Either "
                    "the prompt text has missing/incorrect tokens for "
                    "multi-modal inputs, or there is a problem with your "
                    "implementation of merged multi-modal processor for this "
                    "model (usually arising from an inconsistency between "
                    "`_call_hf_processor` and `_get_prompt_replacements`).")

            missing_repl_counts[modality] = item_count - len(placeholders)

        return missing_repl_counts

1175
1176
    def apply(
        self,
1177
        prompt: Union[str, list[int]],
1178
        mm_data: MultiModalDataDict,
1179
        hf_processor_mm_kwargs: Mapping[str, object],
1180
    ) -> MultiModalInputs:
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
        """
        Process multi-modal inputs to be used in vLLM.

        The main steps are:

        1. Apply HF Processor on prompt text and multi-modal data together,
           outputting token IDs and processed tensors.
        2. Find and replace sequences in the token IDs with placeholder tokens.
           The number of placeholder tokens equals the feature size of the
           multi-modal data outputted by the multi-modal encoder.
        3. Extract information about the placeholder tokens from the
           processed token IDs.
        """
1194
        mm_items = self._to_mm_items(mm_data)
1195

1196
1197
1198
1199
1200
        # Create MM hashes (only used in V1)
        # TODO: Use these hash keys for caching operations in apply_hf_processor
        # instead of rehashing.

        if envs.VLLM_USE_V1:
1201
            model_id = self.info.model_id
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
            mm_hashes = {
                modality: [
                    MultiModalHasher.hash_kwargs(model_id=model_id,
                                                 **{modality: item},
                                                 **hf_processor_mm_kwargs)
                    for item in items
                ]
                for modality, items in mm_items.items()
            }
        else:
            mm_hashes = None

1214
        prompt_ids, mm_kwargs = self._cached_apply_hf_processor(
1215
            prompt,
1216
1217
1218
            mm_items,
            hf_processor_mm_kwargs,
        )
1219

1220
1221
1222
1223
1224
        unbound_prompt_repls = self._get_prompt_replacements(
            mm_items,
            hf_processor_mm_kwargs,
            mm_kwargs,
        )
1225
        mm_prompt_repls = self._bind_and_group_repls(unbound_prompt_repls)
1226

1227
        mm_item_counts = mm_items.get_all_counts()
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
        self._validate_mm_kwargs(mm_kwargs, mm_item_counts)

        hf_mm_placeholders = self._find_mm_placeholders(
            mm_prompt_repls,
            prompt_ids,
            mm_item_counts,
        )

        if self._always_apply_prompt_replacements():
            mm_missing_repl_counts = mm_item_counts
            mm_missing_repls = dict(mm_prompt_repls)
        else:
            mm_missing_repl_counts = self._validate_mm_placeholders(
                hf_mm_placeholders,
                mm_item_counts,
                allow_missing=True,
            )

1246
            mm_missing_repls = dict[str, list[BoundPromptReplacement]]()
1247
1248
1249
1250
1251
1252
1253
1254
            for modality, missing_repl_count in mm_missing_repl_counts.items():
                if missing_repl_count == 0:
                    mm_missing_repls[modality] = []
                elif missing_repl_count == mm_item_counts.get(modality, 0):
                    mm_missing_repls[modality] = mm_prompt_repls[modality]
                else:
                    raise ValueError("Partial prompt replacement within "
                                     f"{modality=} is not supported")
1255

1256
1257
        # If HF processor already inserts placeholder tokens,
        # there is no need for us to insert them
1258
        if all(len(repls) == 0 for repls in mm_missing_repls.values()):
1259
            tokenizer = self.info.get_tokenizer()
1260
            prompt = decode_tokens(tokenizer, prompt_ids)
1261
            mm_placeholders = hf_mm_placeholders
1262
1263
1264
        else:
            (
                prompt_ids,
1265
                prompt,
1266
                missing_mm_placeholders,
1267
1268
            ) = self._apply_prompt_replacements(
                prompt_ids,
1269
1270
                mm_missing_repls,
                mm_missing_repl_counts,
1271
1272
            )

1273
1274
1275
1276
1277
1278
1279
1280
            mm_placeholders = {**hf_mm_placeholders, **missing_mm_placeholders}

        self._validate_mm_placeholders(mm_placeholders, mm_item_counts)

        mm_placeholder_ranges = {
            modality: [item.to_range() for item in placeholders]
            for modality, placeholders in mm_placeholders.items()
        }
1281

1282
        return MultiModalInputs(
1283
            type="multimodal",
1284
            prompt=prompt,
1285
            prompt_token_ids=prompt_ids,
1286
            mm_kwargs=mm_kwargs,
1287
            mm_hashes=mm_hashes,
1288
            mm_placeholders=mm_placeholder_ranges,
1289
        )