profiling.py 8.39 KB
Newer Older
1
2
# SPDX-License-Identifier: Apache-2.0

3
4
5
from abc import ABC, abstractmethod
from collections.abc import Mapping
from dataclasses import dataclass, field
6
from typing import Generic, NamedTuple, Optional, TypeVar, cast
7
8
9
10
11

import numpy as np
import numpy.typing as npt
from PIL import Image

12
import vllm.envs as envs
13
14
from vllm.logger import init_logger

15
from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
16
17
                     MultiModalInputs, MultiModalKwargs,
                     MultiModalPlaceholderDict)
18
19
from .processing import (BaseMultiModalProcessor, BaseProcessingInfo,
                         EncDecMultiModalProcessor)
20
21
22
23
24
25

logger = init_logger(__name__)


@dataclass
class ProcessorInputs:
26
27
28
29
    """
    Represents the keyword arguments to
    :meth:`vllm.multimodal.processing.BaseMultiModalProcessor.apply`.
    """
30
31
32
33
34
    prompt_text: str
    mm_data: MultiModalDataDict
    hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)


35
36
37
38
39
40
41
42
43
44
45
46
47
48
class DummyEncoderData(NamedTuple):
    """Dummy data used for profiling."""

    prompt_token_ids: list[int]


class DummyDecoderData(NamedTuple):
    """Dummy data used for profiling."""

    prompt_token_ids: list[int]
    multi_modal_data: MultiModalKwargs
    multi_modal_placeholders: MultiModalPlaceholderDict


49
50
51
52
_I = TypeVar("_I", bound=BaseProcessingInfo)


class BaseDummyInputsBuilder(ABC, Generic[_I]):
53
    """
54
    Abstract base class that constructs the dummy data to profile
55
56
57
    multi-modal models.
    """

58
    def __init__(self, info: _I) -> None:
59
60
        super().__init__()

61
        self.info = info
62
63
64
65
66
67
68
69

    @abstractmethod
    def get_dummy_processor_inputs(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
    ) -> ProcessorInputs:
        """
70
        Build the input which, after processing, results in
71
        the maximum possible number of placeholder tokens.
72
73
74
75
76
77
78
79
80
        """
        raise NotImplementedError

    def _get_dummy_audios(
        self,
        *,
        length: int,
        num_audios: int,
    ) -> list[npt.NDArray]:
81
82
        if num_audios == 0:
            return []
83
84
85
86
87
88
89
90
91
92
        audio = np.zeros((length, ))
        return [audio] * num_audios

    def _get_dummy_images(
        self,
        *,
        width: int,
        height: int,
        num_images: int,
    ) -> list[Image.Image]:
93
94
        if num_images == 0:
            return []
95
        image = Image.new("RGB", (width, height), color=255)
96
97
98
99
100
101
102
103
104
105
        return [image] * num_images

    def _get_dummy_videos(
        self,
        *,
        width: int,
        height: int,
        num_frames: int,
        num_videos: int,
    ) -> list[npt.NDArray]:
106
107
        if num_videos == 0:
            return []
108
        video = np.full((num_frames, width, height, 3), 255)
109
110
        return [video] * num_videos

111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132

class MultiModalProfiler(Generic[_I]):
    """
    Contains code for running memory profiling for multi-modal models.
    """

    def __init__(
        self,
        processor: BaseMultiModalProcessor[_I],
    ) -> None:
        super().__init__()

        self.processor = processor

    @property
    def processing_info(self) -> BaseProcessingInfo:
        return self.processor.info

    @property
    def dummy_inputs(self) -> BaseDummyInputsBuilder[_I]:
        return self.processor.dummy_inputs

133
    def get_mm_limits(self) -> Mapping[str, int]:
134
135
        mm_config = self.processing_info.ctx.get_mm_config()
        supported_mm_limits = self.processing_info.get_supported_mm_limits()
136
137

        mm_limits = {
138
            modality: mm_config.get_limit_per_prompt(modality)
139
140
141
142
143
144
145
146
147
148
149
150
            for modality in supported_mm_limits
        }

        for modality, supported_limit in supported_mm_limits.items():
            limit = mm_limits[modality]
            if supported_limit is not None and supported_limit < limit:
                raise ValueError(
                    f"You set {modality}={limit} (or defaulted to 1) in "
                    f"`--limit-mm-per-prompt`, but this model only supports "
                    f"at most {supported_limit} {modality} items.")

        return mm_limits
151
152
153
154

    def _get_dummy_mm_inputs(
        self,
        seq_len: int,
155
        mm_counts: Optional[Mapping[str, int]] = None,
156
    ) -> MultiModalInputs:
157
158
159
        if mm_counts is None:
            mm_counts = self.get_mm_limits()

160
161
162
163
164
        factory = self.dummy_inputs
        processor_inputs = factory.get_dummy_processor_inputs(
            seq_len, mm_counts)

        return self.processor.apply(
165
            prompt=processor_inputs.prompt_text,
166
167
168
169
            mm_data=processor_inputs.mm_data,
            hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
        )

170
    def _get_mm_num_tokens(
171
        self,
172
173
        mm_inputs: MultiModalInputs,
    ) -> Mapping[str, int]:
174
175
        placeholders_by_modality = mm_inputs["mm_placeholders"]

176
        return {
177
            modality: sum(item.get_num_embeds() for item in placeholders)
178
179
            for modality, placeholders in placeholders_by_modality.items()
        }
180

181
182
183
184
185
    def get_encoder_dummy_data(
        self,
        seq_len: int,
        mm_counts: Optional[Mapping[str, int]] = None,
    ) -> DummyEncoderData:
186
        mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)
187
188
189
190
191
192
193
        mm_inputs = cast(MultiModalEncDecInputs, mm_inputs)

        # For encoder-decoder models, use encoder prompt token ids instead of
        # decoder prompt to construct dummy seq_data for encoder profiling.
        encoder_prompt_token_ids = mm_inputs["encoder_prompt_token_ids"]

        total_len = len(encoder_prompt_token_ids)
194

195
196
197
198
199
200
        processor = cast(EncDecMultiModalProcessor, self.processor)
        if processor.pad_dummy_encoder_prompt:
            num_tokens_to_pad = max(total_len, seq_len) - total_len
            encoder_prompt_token_ids.extend([0] * num_tokens_to_pad)
        # NOTE: Whisper allows total_len > seq_len.
        elif total_len > seq_len and not envs.VLLM_USE_V1:
201
            # `max_num_batched_tokens` is defined by `SchedulerConfig`
202
            logger.warning_once(
203
                "The encoder sequence length used for profiling ("
204
205
                f"max_num_batched_tokens / max_num_seqs = {seq_len}) "
                " is too short "
206
                "to hold the multi-modal embeddings in the worst case "
207
                f"({total_len} tokens in total, out of which "
208
                f"{self._get_mm_num_tokens(mm_inputs)} are reserved for "
209
210
211
212
                "multi-modal embeddings). This may cause certain "
                "multi-modal inputs to fail during inference, even when "
                "the input text is short. To avoid this, you should "
                "increase `max_model_len`, reduce `max_num_seqs`, "
213
                "and/or reduce `mm_counts`.")
214

215
        return DummyEncoderData(encoder_prompt_token_ids)
216

217
218
219
220
221
    def get_decoder_dummy_data(
        self,
        seq_len: int,
        mm_counts: Optional[Mapping[str, int]] = None,
    ) -> DummyDecoderData:
222
        mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)
223

224
        prompt_token_ids = mm_inputs["prompt_token_ids"]
225
226
227
        total_len = len(prompt_token_ids)

        # V0 does not support chunked prefill.
228
        if total_len > seq_len and not envs.VLLM_USE_V1:
229
            # `max_num_batched_tokens` is defined by `SchedulerConfig`
230
            logger.warning_once(
231
                "The sequence length used for profiling ("
232
233
                f"max_num_batched_tokens / max_num_seqs = {seq_len}) "
                "is too short "
234
                "to hold the multi-modal embeddings in the worst case "
235
                f"({total_len} tokens in total, out of which "
236
                f"{self._get_mm_num_tokens(mm_inputs)} are reserved for "
237
238
239
240
                "multi-modal embeddings). This may cause certain "
                "multi-modal inputs to fail during inference, even when "
                "the input text is short. To avoid this, you should "
                "increase `max_model_len`, reduce `max_num_seqs`, "
241
                "and/or reduce `mm_counts`.")
242

243
244
        if total_len < seq_len:
            prompt_token_ids.extend([0] * (seq_len - total_len))
245

246
247
        return DummyDecoderData(
            prompt_token_ids=prompt_token_ids,
248
            multi_modal_data=mm_inputs["mm_kwargs"],
249
            multi_modal_placeholders=mm_inputs["mm_placeholders"],
250
        )
251
252
253
254
255
256
257
258
259

    def get_mm_max_tokens(
        self,
        seq_len: int,
        mm_counts: Optional[Mapping[str, int]] = None,
    ) -> Mapping[str, int]:
        mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)

        return self._get_mm_num_tokens(mm_inputs)