minicpmv.py 49 KB
Newer Older
1
2
# SPDX-License-Identifier: Apache-2.0

3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# Adapted from
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
# Copyright 2023 The vLLM team.
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
#
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
# and OPT implementations in this library. It has been modified from its
# original forms to accommodate minor architectural differences compared
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Alphi's avatar
Alphi committed
24
"""Inference-only MiniCPM-V model compatible with HuggingFace weights."""
25
import math
26
from collections import defaultdict
27
from collections.abc import Iterable, Mapping, Sequence
28
from functools import cached_property, partial
29
30
from typing import (Any, Callable, Literal, Optional, Set, Tuple, TypedDict,
                    Union)
31

32
import numpy as np
33
import torch
Alphi's avatar
Alphi committed
34
import torch.types
35
from torch import nn
36
from transformers import BatchFeature, PretrainedConfig
37
from typing_extensions import TypeVar
38

39
from vllm.config import VllmConfig
40
from vllm.model_executor.layers.quantization import QuantizationConfig
41
from vllm.model_executor.layers.resampler import (BaseResampler, Resampler2,
42
                                                  get_2d_sincos_pos_embed)
Joe Runde's avatar
Joe Runde committed
43
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
Jee Jee Li's avatar
Jee Jee Li committed
44
from vllm.model_executor.model_loader.utils import set_default_torch_dtype
45
46
from vllm.model_executor.models.llama import LlamaForCausalLM
from vllm.model_executor.models.minicpm import MiniCPMForCausalLM
47
from vllm.model_executor.models.module_mapping import MultiModelKeys
48
from vllm.model_executor.models.qwen2 import Qwen2ForCausalLM
49
from vllm.model_executor.sampling_metadata import SamplingMetadata
50
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
51
from vllm.multimodal.inputs import MultiModalFieldConfig, NestedTensors
52
53
from vllm.multimodal.parse import (DictEmbeddingItems, ImageItem,
                                   ImageProcessorItems, ImageSize,
54
55
                                   ModalityData, ModalityDataItems,
                                   MultiModalDataItems, MultiModalDataParser,
56
                                   VideoItem, VideoProcessorItems)
57
from vllm.multimodal.processing import (BaseMultiModalProcessor,
58
59
                                        BaseProcessingInfo, PromptReplacement,
                                        PromptUpdate)
60
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
61
from vllm.platforms import current_platform
62
from vllm.sequence import IntermediateTensors
63
from vllm.utils import flatten_2d_lists
64

Jee Jee Li's avatar
Jee Jee Li committed
65
from .idefics2_vision_model import Idefics2VisionTransformer
66
67
68
69
70
from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
                         SupportsMultiModal, SupportsPP)
from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,
                    merge_multimodal_embeddings)
from .vision import scatter_patch_features, select_patch_features
71

72
73
74
# For profile run
_MAX_FRAMES_PER_VIDEO = 16

75

Jee Jee Li's avatar
Jee Jee Li committed
76
class MiniCPMVImagePixelInputs(TypedDict):
77
    type: Literal["pixel_values"]
78
    pixel_values: list[torch.Tensor]
Jee Jee Li's avatar
Jee Jee Li committed
79
    """
80
    Shape: `(batch_size * num_images * num_slices, num_channels, height, width)`
Jee Jee Li's avatar
Jee Jee Li committed
81
82
83
84
85

    Note that the image size may vary, so we pass it as a list
    instead of a batched tensor.
    """

86
    tgt_sizes: torch.Tensor
Jee Jee Li's avatar
Jee Jee Li committed
87
    """
88
    Shape: `(batch_size * num_images * num_slices, 2)`
Jee Jee Li's avatar
Jee Jee Li committed
89

90
    This should be in `(height, width)` format.
Jee Jee Li's avatar
Jee Jee Li committed
91
92
    """

93
    embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
Jee Jee Li's avatar
Jee Jee Li committed
94
    """
95
96
    A boolean mask indicating which image embeddings correspond
    to patch tokens.
Jee Jee Li's avatar
Jee Jee Li committed
97

98
    Shape: `(batch_size * num_images, num_embeds)`
Jee Jee Li's avatar
Jee Jee Li committed
99
100
    """

101
102
103
    num_slices: torch.Tensor
    """Shape: `(batch_size * num_images)`"""

Jee Jee Li's avatar
Jee Jee Li committed
104

105
106
class MiniCPMVImageEmbeddingInputs(TypedDict):
    type: Literal["image_embeds"]
107
    image_embeds: Union[torch.Tensor, list[torch.Tensor]]
108
    """
109
    Shape: `(batch_size * num_images, num_slices, hidden_size)`
110
111
112
113
114

    `hidden_size` must match the hidden size of language model backbone.
    instead of a batched tensor.
    """

115
    embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
116
    """
117
118
    A boolean mask indicating which image embeddings correspond
    to patch tokens.
119

120
    Shape: `(batch_size * num_images, num_embeds)`
121
122
123
124
125
126
    """


MiniCPMVImageInputs = Union[MiniCPMVImagePixelInputs,
                            MiniCPMVImageEmbeddingInputs]

Jee Jee Li's avatar
Jee Jee Li committed
127
128
129
130
131
DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)


class Resampler2_5(BaseResampler):

132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
    def __init__(self,
                 num_queries: int,
                 embed_dim: int,
                 num_heads: int,
                 kv_dim: Optional[int] = None,
                 norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
                 max_size: Tuple[int, int] = (70, 70),
                 quant_config: Optional[QuantizationConfig] = None,
                 prefix: str = "") -> None:
        super().__init__(num_queries,
                         embed_dim,
                         num_heads,
                         kv_dim,
                         norm_layer,
                         quant_config=quant_config,
                         prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
148
149
150

        self.max_size = max_size
        self._set_2d_pos_cache(self.max_size)
151

Alphi's avatar
Alphi committed
152
153
    def _set_2d_pos_cache(self,
                          max_size: Tuple[int, int],
Jee Jee Li's avatar
Jee Jee Li committed
154
155
156
157
158
                          device: torch.types.Device = "cpu") -> None:
        pos_embed_arr = get_2d_sincos_pos_embed(self.embed_dim,
                                                max_size,
                                                version=(2, 5))
        pos_embed = torch.from_numpy(pos_embed_arr).float().to(device)
159
160
        self.register_buffer("pos_embed", pos_embed, persistent=False)

Alphi's avatar
Alphi committed
161
    def _adjust_pos_cache(self, tgt_sizes: torch.Tensor,
Jee Jee Li's avatar
Jee Jee Li committed
162
163
164
165
166
                          device: torch.types.Device) -> None:
        max_h = tgt_sizes[:, 0].max().item()
        max_w = tgt_sizes[:, 1].max().item()
        assert isinstance(max_h, int) and isinstance(max_w, int)

167
        if max_h > self.max_size[0] or max_w > self.max_size[1]:
Jee Jee Li's avatar
Jee Jee Li committed
168
            self.max_size = (
169
                max(max_h, self.max_size[0]),
Jee Jee Li's avatar
Jee Jee Li committed
170
171
                max(max_w, self.max_size[1]),
            )
172
173
            self._set_2d_pos_cache(self.max_size, device)

Jee Jee Li's avatar
Jee Jee Li committed
174
175
    def forward(self, x: torch.Tensor,
                tgt_sizes: torch.Tensor) -> torch.Tensor:
176
177
178
179
180
181
182
183
184
185
        assert x.shape[0] == tgt_sizes.shape[0]
        bs = x.shape[0]

        device = x.device
        dtype = x.dtype

        patch_len = tgt_sizes[:, 0] * tgt_sizes[:, 1]

        self._adjust_pos_cache(tgt_sizes, device=device)

Jee Jee Li's avatar
Jee Jee Li committed
186
187
188
        max_patch_len = patch_len.max().item()
        assert isinstance(max_patch_len, int)

189
190
191
192
193
194
        key_padding_mask = torch.zeros((bs, max_patch_len),
                                       dtype=torch.bool,
                                       device=device)

        pos_embed = []
        for i in range(bs):
Jee Jee Li's avatar
Jee Jee Li committed
195
            tgt_h, tgt_w = tgt_sizes[i].tolist()
196
197
198
199
200
201
202
203
            pos_embed.append(self.pos_embed[:tgt_h, :tgt_w, :].reshape(
                (tgt_h * tgt_w, -1)).to(dtype))  # patches * D
            key_padding_mask[i, patch_len[i]:] = True
        pos_embed = torch.nn.utils.rnn.pad_sequence(pos_embed,
                                                    batch_first=True,
                                                    padding_value=0.0).permute(
                                                        1, 0,
                                                        2)  # BLD => L * B * D
Jee Jee Li's avatar
Jee Jee Li committed
204
        x, _ = self.kv_proj(x)  # B * L * D
205
206
207
208
209
210
211
212
        x = self.ln_kv(x).permute(1, 0, 2)  # L * B * D

        q = self.ln_q(self.query)  # Q * D

        out = self.attn(
            self._repeat(q, bs),  # Q * B * D
            x + pos_embed,  # L * B * D +  L * B * D
            x,
Jee Jee Li's avatar
Jee Jee Li committed
213
214
            key_padding_mask=key_padding_mask,
        )[0]
215
216
217
218
219
220
221
222
        #  out: Q * B * D
        x = out.permute(1, 0, 2)  # B * Q * D

        x = self.ln_post(x)
        x = x @ self.proj
        return x


223
224
225
226
227
228
229
230
231
232
233
234
235
def get_version_by_config(config: PretrainedConfig) -> Tuple[int, ...]:
    version_float = getattr(config, "version", None)

    # The old configs do not include version number
    # TODO: Remove this after the HF repos are updated
    if version_float is None:
        if config.hidden_size == 2304 and config.query_num == 64:
            return (2, 0)
        return (2, 5)
    version_str = str(version_float)
    return tuple(int(x) for x in version_str.split("."))


236
def _minicpmv_field_config(hf_inputs: Mapping[str, torch.Tensor]):
237
238
239
240
241
242
    pixel_values = hf_inputs.get("pixel_values", torch.empty(0))
    num_images = len(pixel_values)

    video_pixel_values = hf_inputs.get("video_pixel_values", torch.empty(0))
    num_videos = len(video_pixel_values)

243
    return dict(
244
        pixel_values=MultiModalFieldConfig.batched("image"),
245
        image_sizes=MultiModalFieldConfig.batched("image"),
246
247
        tgt_sizes=MultiModalFieldConfig.batched("image"),
        image_embeds=MultiModalFieldConfig.batched("image"),
248
        embed_is_patch=MultiModalFieldConfig.batched("image"),
249
        video_pixel_values=MultiModalFieldConfig.batched("video"),
250
        video_image_sizes=MultiModalFieldConfig.batched("video"),
251
252
        video_tgt_sizes=MultiModalFieldConfig.batched("video"),
        video_embeds=MultiModalFieldConfig.batched("video"),
253
254
255
        video_embed_is_patch=MultiModalFieldConfig.batched("video"),
        image_token_id=MultiModalFieldConfig.shared("image", num_images),
        video_token_id=MultiModalFieldConfig.shared("video", num_videos),
256
257
258
259
260
261
262
263
    )


class MiniCPMVImageEmbeddingItems(DictEmbeddingItems):

    def __init__(
        self,
        data: Mapping[str, torch.Tensor],
264
265
266
267
        fields_factory: Callable[
            [Mapping[str, torch.Tensor]],
            Mapping[str, MultiModalFieldConfig],
        ],
268
269
270
271
272
    ) -> None:
        super().__init__(
            data,
            modality="image",
            required_fields={"image_embeds", "image_sizes"},
273
            fields_factory=fields_factory,
274
275
276
277
278
279
280
281
282
283
284
285
        )

    def get_image_size(self, index: int) -> ImageSize:
        image_size = self.get(index)["image_sizes"].tolist()
        return ImageSize(width=image_size[0], height=image_size[1])


class MiniCPMVVideoEmbeddingItems(DictEmbeddingItems):

    def __init__(
        self,
        data: Mapping[str, torch.Tensor],
286
287
288
289
        fields_factory: Callable[
            [Mapping[str, torch.Tensor]],
            Mapping[str, MultiModalFieldConfig],
        ],
290
291
292
293
294
    ) -> None:
        super().__init__(
            data,
            modality="video",
            required_fields={"video_embeds", "video_image_sizes"},
295
            fields_factory=fields_factory,
296
297
298
299
300
301
302
303
304
305
        )

    def get_frame_size(self, index: int) -> ImageSize:
        frame_size = self.get(index)["video_image_sizes"].tolist()
        return ImageSize(width=frame_size[0], height=frame_size[1])

    def get_num_frames(self, index: int) -> int:
        return len(self.get(index)["video_image_sizes"])


306
307
308
309
310
311
312
class MiniCPMVMultiModalDataParser(MultiModalDataParser):

    def _parse_image_data(
        self,
        data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],
    ) -> ModalityDataItems[Any, Any]:
        if isinstance(data, dict):
313
314
            return MiniCPMVImageEmbeddingItems(
                data,
315
                fields_factory=_minicpmv_field_config,
316
317
            )

318
319
320
321
322
323
324
        return super()._parse_image_data(data)

    def _parse_video_data(
        self,
        data: Union[dict[str, torch.Tensor], ModalityData[VideoItem]],
    ) -> ModalityDataItems[Any, Any]:
        if isinstance(data, dict):
325
326
            return MiniCPMVVideoEmbeddingItems(
                data,
327
                fields_factory=_minicpmv_field_config,
328
329
            )

330
331
332
333
334
335
336
337
338
339
        return super()._parse_video_data(data)


class MiniCPMVProcessingInfo(BaseProcessingInfo):
    image_pattern = "(<image>./</image>)"
    video_pattern = "(<video>./</video>)"

    def get_hf_config(self):
        return self.ctx.get_hf_config()

340
341
    def get_hf_processor(self, **kwargs: object):
        hf_processor = self.ctx.get_hf_processor(**kwargs)
342
343
344
345
346
347
348
349
350

        # NumPy arrays are considered as Iterable but not Sequence in
        # https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428
        image_processor = hf_processor.image_processor  # type: ignore
        for attr in ("mean", "std"):
            val = getattr(image_processor, attr)
            if isinstance(val, np.ndarray):
                setattr(image_processor, attr, val.tolist())

351
352
353
354
355
356
357
358
359
360
361
        return hf_processor

    def get_image_processor(self):
        hf_processor = self.get_hf_processor()
        image_processor = hf_processor.image_processor  # type: ignore
        return image_processor

    def get_model_version(self):
        return get_version_by_config(self.get_hf_config())

    def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
362
        mm_limits = {"image": None}
363
        if self.get_model_version() == (2, 6):
364
365
366
            mm_limits["video"] = None

        return mm_limits
367

368
369
370
371
372
    def get_mm_max_tokens_per_item(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
    ) -> Mapping[str, int]:
373
374
        mm_max_tokens = {"image": self.get_max_image_tokens()}
        if self.get_model_version() == (2, 6):
375
376
            mm_max_tokens["video"] = self.get_max_video_tokens(
                seq_len, mm_counts)
377

378
379
        return mm_max_tokens

380
381
382
383
384
385
386
387
388
389
    def get_slice_image_placeholder(
        self,
        image_size: ImageSize,
        # For MiniCPM V/O 2.6
        image_idx: int = 0,
        max_slice_nums: Optional[int] = None,
        use_image_id: bool = True,
    ) -> str:
        image_processor = self.get_image_processor()
        version = self.get_model_version()
390

391
392
        if version == (2, 0) or version == (2, 5):
            return image_processor.get_slice_image_placeholder(image_size)
393

394
395
396
397
398
399
        return image_processor.get_slice_image_placeholder(
            image_size,
            image_idx=image_idx,
            max_slice_nums=max_slice_nums,
            use_image_id=use_image_id,
        )
400

401
402
403
404
405
406
407
408
409
410
411
412
413
414
    def get_num_image_tokens(
        self,
        image_size: ImageSize,
        max_slice_nums: Optional[int] = None,
        use_image_id: bool = True,
    ) -> int:
        tokenizer = self.get_tokenizer()
        image_placeholders = self.get_slice_image_placeholder(
            image_size,
            max_slice_nums=max_slice_nums,
            use_image_id=use_image_id,
        )
        image_token_ids = tokenizer.encode(image_placeholders,
                                           add_special_tokens=False)
415

416
        return len(image_token_ids)
417
418
419

    def get_max_image_tokens(self) -> int:
        image_size = self.get_image_size_with_most_features()
420
421
422
423
        return self.get_num_image_tokens(image_size)

    def get_image_max_slice_num(self) -> int:
        return getattr(self.get_hf_config(), "max_slice_num", 9)
424
425

    def get_image_size_with_most_features(self) -> ImageSize:
426
427
428
429
430
431
432
433
434
435
436
437
438
        image_size = getattr(self.get_hf_config(), "image_size", 448)
        max_slice_num = self.get_image_max_slice_num()
        return ImageSize(width=image_size, height=image_size * max_slice_num)

    def get_max_video_frame_tokens(self) -> int:
        frame_size = self.get_video_frame_size_with_most_features()

        return self.get_num_image_tokens(
            frame_size,
            max_slice_nums=self.get_video_max_slice_num(),
            use_image_id=False,
        )

439
440
441
442
443
444
445
446
    def get_max_video_tokens(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
    ) -> int:
        num_frames = self.get_num_frames_with_most_features(seq_len, mm_counts)
        num_video_tokens_total = self.get_max_video_frame_tokens() * num_frames
        return num_video_tokens_total
447
448
449

    def get_video_max_slice_num(self) -> int:
        return 1
450

451
    def get_video_frame_size_with_most_features(self) -> ImageSize:
452
453
454
        image_size = getattr(self.get_hf_config(), "image_size", 448)
        max_slice_num = self.get_video_max_slice_num()
        return ImageSize(width=image_size, height=image_size * max_slice_num)
455

456
457
458
459
    def get_max_video_frames(self, max_tokens: int) -> int:
        num_frame_tokens = self.get_max_video_frame_tokens()
        num_frames = max_tokens // num_frame_tokens
        return num_frames
460

461
462
463
464
465
466
467
    def get_num_frames_with_most_features(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
    ) -> int:
        max_images = mm_counts.get("image", 0)
        max_videos = mm_counts.get("video", 0)
468

469
        max_image_tokens = self.get_max_image_tokens() * max_images
470
471
        max_total_frames = self.get_max_video_frames(seq_len -
                                                     max_image_tokens)
472
473
        max_frames_per_video = min(max_total_frames // max(max_videos, 1),
                                   _MAX_FRAMES_PER_VIDEO)
474

475
        return max(max_frames_per_video, 1)
476
477


478
479
480
481
482
483
_I = TypeVar("_I",
             bound=MiniCPMVProcessingInfo,
             default=MiniCPMVProcessingInfo)


class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
484

485
486
487
488
489
490
491
492
493
494
495
496
497
    def get_dummy_processor_inputs(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
    ) -> ProcessorInputs:
        num_images = mm_counts.get("image", 0)
        num_videos = mm_counts.get("video", 0)

        image_width, image_height = \
            self.info.get_image_size_with_most_features()
        video_width, video_height = \
            self.info.get_video_frame_size_with_most_features()
        num_video_frames = \
498
            self.info.get_num_frames_with_most_features(seq_len, mm_counts)
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517

        mm_data = {
            "image":
            self._get_dummy_images(width=image_width,
                                   height=image_height,
                                   num_images=num_images),
            "video": [
                self._get_dummy_images(width=video_width,
                                       height=video_height,
                                       num_images=num_video_frames)
            ] * num_videos,
        }

        image_prompt_texts = self.info.image_pattern * num_images
        video_prompt_texts = self.info.video_pattern * num_videos

        return ProcessorInputs(prompt_text=image_prompt_texts +
                               video_prompt_texts,
                               mm_data=mm_data)
518

519

520
class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
521
522
523
524
525
526
527

    def _get_data_parser(self) -> MultiModalDataParser:
        return MiniCPMVMultiModalDataParser()

    def get_image_prompt_texts(self,
                               image_size: ImageSize,
                               image_idx: int = 0) -> str:
528
529
530
531
        return self.info.get_slice_image_placeholder(
            image_size,
            image_idx=image_idx,
        )
532
533
534

    def get_video_prompt_texts(self, image_size: ImageSize,
                               num_frames: int) -> str:
535
        return self.info.get_slice_image_placeholder(
536
537
538
539
540
            image_size=image_size,
            image_idx=0,
            max_slice_nums=self.info.get_video_max_slice_num(),
            use_image_id=False,
        ) * num_frames
541

542
543
544
545
    def get_embed_is_patch(
        self,
        input_ids: list[int],
    ) -> torch.Tensor:
546
        tokenizer = self.info.get_tokenizer()
547
548
        unk_token_id = tokenizer.get_vocab()["<unk>"]
        return torch.tensor(input_ids) == unk_token_id
549

550
551
552
553
554
    def process_images(
        self,
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, NestedTensors]:
555
556
557
558
559
        if (images := mm_data.get("images")) is None:
            return {}

        parsed_images = (self._get_data_parser().parse_mm_data({
            "image": images
560
561
        }).get_items("image",
                     (MiniCPMVImageEmbeddingItems, ImageProcessorItems)))
562

563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
        if isinstance(parsed_images, MiniCPMVImageEmbeddingItems):
            image_inputs = {}
        else:
            image_inputs = self._base_call_hf_processor(
                prompts=[self.info.image_pattern] * len(parsed_images),
                mm_data={"images": [[image] for image in parsed_images]},
                mm_kwargs=mm_kwargs,
                out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
            )

        image_sizes = [
            parsed_images.get_image_size(i) for i in range(len(parsed_images))
        ]
        image_repl_features = [
            self.get_image_prompt_texts(size, idx)
            for idx, size in enumerate(image_sizes)
        ]

        tokenizer = self.info.get_tokenizer()
        image_repls_feature_tokens = [
            tokenizer.encode(image_repl, add_special_tokens=False)
            for image_repl in image_repl_features
        ]

        embed_is_patch = [
            self.get_embed_is_patch(image_repl_tokens)
            for image_repl_tokens in image_repls_feature_tokens
        ]
        image_inputs["embed_is_patch"] = embed_is_patch

        unk_token_id = tokenizer.get_vocab()["<unk>"]
        image_inputs["image_token_id"] = torch.tensor(unk_token_id)

        return image_inputs
597

598
599
600
601
602
    def process_videos(
        self,
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, NestedTensors]:
603
604
605
606
607
        if (videos := mm_data.get("videos")) is None:
            return {}

        parsed_videos = (self._get_data_parser().parse_mm_data({
            "video": videos
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
        }).get_items("video",
                     (MiniCPMVVideoEmbeddingItems, VideoProcessorItems)))

        if isinstance(parsed_videos, MiniCPMVVideoEmbeddingItems):
            video_inputs = {}
        else:
            video_inputs = self._base_call_hf_processor(
                prompts=[
                    self.info.image_pattern * len(video)
                    for video in parsed_videos
                ],
                mm_data={"images": list(parsed_videos)},
                mm_kwargs={
                    **mm_kwargs,
                    "max_slice_nums":
                    self.info.get_video_max_slice_num(),
                },
                out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
            )

        frame_sizes = [
            parsed_videos.get_frame_size(i) for i in range(len(parsed_videos))
        ]
        num_frames = [
            parsed_videos.get_num_frames(i) for i in range(len(parsed_videos))
        ]
        video_repl_features = [
            self.get_video_prompt_texts(size, nframes)
            for size, nframes in zip(frame_sizes, num_frames)
        ]

        tokenizer = self.info.get_tokenizer()
        video_repls_feature_tokens = [
            tokenizer.encode(video_repl, add_special_tokens=False)
            for video_repl in video_repl_features
        ]

        embed_is_patch = [
            self.get_embed_is_patch(video_repl_tokens)
            for video_repl_tokens in video_repls_feature_tokens
        ]
        video_inputs["embed_is_patch"] = embed_is_patch

        video_inputs = {f"video_{k}": v for k, v in video_inputs.items()}
652

653
654
        unk_token_id = tokenizer.get_vocab()["<unk>"]
        video_inputs["video_token_id"] = torch.tensor(unk_token_id)
655

656
        return video_inputs
657

658
659
660
661
    def process_mm_inputs(
        self,
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
662
    ) -> Mapping[str, NestedTensors]:
663
        return {
664
665
            **self.process_images(mm_data, mm_kwargs),
            **self.process_videos(mm_data, mm_kwargs),
666
        }
667

668
    def _base_call_hf_processor(
669
        self,
670
671
        prompts: list[str],
        mm_data: Mapping[str, Sequence[object]],
672
        mm_kwargs: Mapping[str, object],
673
674
        *,
        out_keys: set[str],
675
    ) -> dict[str, NestedTensors]:
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
        # This processor supports zipping prompt and mm_data together
        if self.info.get_model_version() == (2, 6):
            inputs = super()._call_hf_processor(
                prompt=prompts,  # type: ignore
                mm_data=mm_data,
                mm_kwargs=mm_kwargs,
            )
        else:
            inputs = defaultdict[str, list[torch.Tensor]](list)

            for i, prompt in enumerate(prompts):
                inputs_one = super()._call_hf_processor(
                    prompt=prompt,
                    mm_data={
                        k: v[i]
                        for k, v in mm_data.items()
                    },
                    mm_kwargs=mm_kwargs,
                )

                for k, v in inputs_one.items():
                    assert len(v) == 1, (k, len(v))
                    inputs[k].append(v[0])

        return {k: inputs[k] for k in out_keys}
701
702
703
704
705
706
707
708

    def _call_hf_processor(
        self,
        prompt: str,
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
    ) -> BatchFeature:
        tokenizer = self.info.get_tokenizer()
709
710

        input_ids = torch.tensor([tokenizer.encode(prompt)])
711
        mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs)
712
713

        return BatchFeature({
714
            "input_ids": input_ids,
715
            **mm_inputs,
716
        })
717

718
    def _hf_processor_applies_updates(
719
720
721
722
723
724
725
        self,
        prompt_text: str,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> bool:
        return False

726
    def _get_prompt_updates(
727
728
729
730
731
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        out_mm_kwargs: MultiModalKwargs,
    ) -> Sequence[PromptUpdate]:
732
733
734
        placeholder = {
            "image": self.info.image_pattern,
            "video": self.info.video_pattern,
735
        }
736

737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
        def get_image_replacement(item_idx: int):
            images = mm_items.get_items(
                "image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems))

            image_size = images.get_image_size(item_idx)

            return self.get_image_prompt_texts(image_size, item_idx)

        def get_video_replacement(item_idx: int):
            videos = mm_items.get_items(
                "video", (MiniCPMVVideoEmbeddingItems, VideoProcessorItems))

            frame_size = videos.get_frame_size(item_idx)
            num_frames = videos.get_num_frames(item_idx)

            return self.get_video_prompt_texts(frame_size, num_frames)

        get_replacement = {
            "image": get_image_replacement,
            "video": get_video_replacement,
        }
758
759
760
761

        return [
            PromptReplacement(modality=modality,
                              target=placeholder[modality],
762
                              replacement=get_replacement[modality])
763
764
            for modality in ("image", "video")
        ]
765

766
767
    def _get_mm_fields_config(
        self,
768
        hf_inputs: BatchFeature,
769
770
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, MultiModalFieldConfig]:
771
        return _minicpmv_field_config(hf_inputs)
772

773
774

class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
Jee Jee Li's avatar
Jee Jee Li committed
775
776
777
778
    """
    The abstract class of MiniCPMV can only be inherited, but cannot be
    instantiated.
    """
779

780
    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
781
782
783
        config = vllm_config.model_config.hf_config
        multimodal_config = vllm_config.model_config.multimodal_config
        quant_config = vllm_config.quant_config
784
        super().__init__()
785
786
787
788
        # All MiniCPM-V models disable `tie_word_embeddings` but
        # `PretrainedConfig.tie_word_embeddings` defaults to True; we cannot
        # check `tie_word_embeddings` until vLLM integrate MiniCPM-V model
        # and config class
789
790
791
        self.config = config
        self.multimodal_config = multimodal_config

792
        self.version = get_version_by_config(self.config)
793
794
795
796
797
        self.llm = self.init_llm(vllm_config=vllm_config,
                                 prefix=maybe_prefix(prefix, "llm"))
        self.vpm = self.init_vision_module(config,
                                           quant_config,
                                           prefix=maybe_prefix(prefix, "vpm"))
Jee Jee Li's avatar
Jee Jee Li committed
798
799
        self.vision_dim = (self.vpm.embed_dim if self.version == (2, 0) else
                           self.vpm.embeddings.embed_dim)
Alphi's avatar
Alphi committed
800
        self.embed_dim = self.config.hidden_size
801

802
803
804
        self.resampler = self.init_resampler(self.embed_dim,
                                             self.vision_dim,
                                             quant_config=quant_config,
805
806
                                             prefix=maybe_prefix(
                                                 prefix, "resampler"))
807

808
        self.mm_token_ids = set[int]()
809
810
811
        self.make_empty_intermediate_tensors = (
            self.llm.make_empty_intermediate_tensors)

812
813
814
815
816
817
818
    @cached_property
    def sampler(self):
        if hasattr(self.llm, "sampler"):
            return self.llm.sampler

        return get_sampler()

819
    def _parse_and_validate_vision_input(
Jee Jee Li's avatar
Jee Jee Li committed
820
        self,
821
        modality: str,
Jee Jee Li's avatar
Jee Jee Li committed
822
        **kwargs: object,
823
    ) -> Optional[MiniCPMVImageInputs]:
824
825
        pixel_values = kwargs.pop("pixel_values", None)
        image_embeds = kwargs.pop("image_embeds", None)
826

827
        if pixel_values is None and image_embeds is None:
828
829
            return None

830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
        image_token_id = kwargs.pop("image_token_id")
        if image_token_id is not None:
            assert isinstance(image_token_id, torch.Tensor)
            self.mm_token_ids.add(image_token_id.flatten().unique().item())

        embed_is_patch = kwargs.pop("embed_is_patch")
        if not isinstance(embed_is_patch, (torch.Tensor, list)):
            raise ValueError(
                f"Incorrect type of embed_is_patch for {modality=}. "
                f"Got type: {type(embed_is_patch)}")

        embed_is_patch = flatten_bn(embed_is_patch)

        if image_embeds is not None:
            if not isinstance(image_embeds, (torch.Tensor, list)):
                raise ValueError(
                    f"Incorrect type of image_embeds for {modality=}. "
                    f"Got type: {type(image_embeds)}")

            image_embeds_flat = flatten_bn(image_embeds)
850

851
            return MiniCPMVImageEmbeddingInputs(
852
                type="image_embeds",
853
854
                image_embeds=image_embeds_flat,
                embed_is_patch=embed_is_patch,
855
            )
856

857
858
859
860
        if not isinstance(pixel_values, (torch.Tensor, list)):
            raise ValueError(
                f"Incorrect type of pixel_values for {modality=}. "
                f"Got type: {type(pixel_values)}")
861

862
863
864
865
866
867
868
869
870
871
        tgt_sizes = kwargs.pop("tgt_sizes")
        if not isinstance(tgt_sizes, (torch.Tensor, list)):
            raise ValueError(f"Incorrect type of tgt_sizes for {modality=}. "
                             f"Got type: {type(tgt_sizes)}")

        num_slices = [[len(p) for p in ps] for ps in pixel_values]
        num_slices_flat = flatten_bn(torch.tensor(num_slices))

        pixel_values_flat = flatten_bn(flatten_2d_lists(pixel_values))
        tgt_sizes_flat = flatten_bn(flatten_2d_lists(tgt_sizes), concat=True)
872

Jee Jee Li's avatar
Jee Jee Li committed
873
874
875
876
877
        if len(pixel_values_flat) != len(tgt_sizes_flat):
            raise ValueError("Inconsistent flattened lengths, found: "
                             f"{len(pixel_values_flat)} vs. "
                             f"{len(tgt_sizes_flat)}")

878
        return MiniCPMVImagePixelInputs(
879
880
            type="pixel_values",
            pixel_values=pixel_values_flat,
881
882
883
            tgt_sizes=tgt_sizes_flat,
            embed_is_patch=embed_is_patch,
            num_slices=num_slices_flat,
Jee Jee Li's avatar
Jee Jee Li committed
884
        )
885

886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
    def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
        modalities = {}

        # Preserve the order of modalities if there are multiple of them
        # from the order of kwargs.
        for input_key in kwargs:
            if input_key in ("pixel_values",
                             "image_embeds") and "images" not in modalities:
                modalities["images"] = self._parse_and_validate_vision_input(
                    "images", **kwargs)
            if input_key in ("video_pixel_values",
                             "video_embeds") and "videos" not in modalities:

                def _image_key(video_key: str):
                    if video_key == "video_token_id":
                        return "image_token_id"

                    return video_key.removeprefix("video_")

                modalities["videos"] = self._parse_and_validate_vision_input(
                    "videos", **{
                        _image_key(k): v
                        for k, v in kwargs.items()
                    })

        return modalities

    def _process_vision_input(
        self,
        image_input: MiniCPMVImageInputs,
    ) -> Union[torch.Tensor, list[torch.Tensor], tuple[torch.Tensor, ...]]:
        if image_input["type"] == "image_embeds":
            return image_input["image_embeds"]

        image_features_flat = self.get_vision_hidden_states(image_input)

        # Reconstruct the batch dimension
        return image_features_flat.split(image_input["num_slices"].tolist())

    def _process_multimodal_inputs(self, modalities: dict):
        # The result multimodal_embeddings is tuple of tensors, with each
        # tensor correspoending to a multimodal data item (image or video).
        multimodal_embeddings: tuple[torch.Tensor, ...] = ()

        # NOTE: It is important to iterate over the keys in this dictionary
        # to preserve the order of the modalities.
        for modality in modalities:
            if modality == "images":
                image_input = modalities["images"]
                image_features = self._process_vision_input(image_input)
                multimodal_embeddings += tuple(
                    scatter_patch_features(
                        image_features,
                        image_input["embed_is_patch"],
                    ))
            if modality == "videos":
                video_input = modalities["videos"]
                video_features = self._process_vision_input(video_input)
                multimodal_embeddings += tuple(
                    scatter_patch_features(
                        video_features,
                        video_input["embed_is_patch"],
                    ))

        return multimodal_embeddings

    def get_multimodal_embeddings(
            self, **kwargs: object) -> Optional[MultiModalEmbeddings]:
        modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
        if not modalities:
            return None

        return self._process_multimodal_inputs(modalities)

    def get_input_embeddings(
        self,
        input_ids: torch.Tensor,
        multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
    ) -> torch.Tensor:
        inputs_embeds = self.llm.get_input_embeddings(input_ids)
        if multimodal_embeddings is not None:
            assert len(self.mm_token_ids) > 0
            inputs_embeds = merge_multimodal_embeddings(
                input_ids,
                inputs_embeds,
                select_patch_features(multimodal_embeddings),
                list(self.mm_token_ids),
            )
        return inputs_embeds
975

976
977
978
979
980
    def forward(
        self,
        input_ids: torch.Tensor,
        positions: torch.Tensor,
        intermediate_tensors: Optional[IntermediateTensors] = None,
981
        inputs_embeds: Optional[torch.Tensor] = None,
Jee Jee Li's avatar
Jee Jee Li committed
982
983
        **kwargs: Any,
    ) -> torch.Tensor:
984
        if intermediate_tensors is not None:
985
986
987
988
989
990
991
            inputs_embeds = None

        # NOTE: In v1, inputs_embeds is always generated at model runner from
        # `get_multimodal_embeddings` and `get_input_embeddings`, this
        # condition is only for v0 compatibility.
        elif inputs_embeds is None:
            vision_embeddings = self.get_multimodal_embeddings(**kwargs)
Jee Jee Li's avatar
Jee Jee Li committed
992

993
994
995
            inputs_embeds = self.get_input_embeddings(input_ids,
                                                      vision_embeddings)
            input_ids = None
996

997
        hidden_states = self.llm.model(
998
            input_ids=input_ids,
Jee Jee Li's avatar
Jee Jee Li committed
999
1000
            positions=positions,
            intermediate_tensors=intermediate_tensors,
1001
            inputs_embeds=inputs_embeds,
Jee Jee Li's avatar
Jee Jee Li committed
1002
        )
1003
        return hidden_states
1004

1005
1006
1007
1008
1009
    def compute_logits(
        self,
        hidden_states: torch.Tensor,
        sampling_metadata: SamplingMetadata,
    ) -> Optional[torch.Tensor]:
1010
        return self.llm.compute_logits(hidden_states, sampling_metadata)
1011
1012
1013
1014
1015
1016

    def sample(
        self,
        logits: torch.Tensor,
        sampling_metadata: SamplingMetadata,
    ) -> Optional[SamplerOutput]:
Alphi's avatar
Alphi committed
1017
        next_tokens = self.sampler(logits, sampling_metadata)
1018
1019
        return next_tokens

1020
1021
    def load_weights(self, weights: Iterable[Tuple[str,
                                                   torch.Tensor]]) -> Set[str]:
1022
1023
        loader = AutoWeightsLoader(self)
        return loader.load_weights(weights)
Jee Jee Li's avatar
Jee Jee Li committed
1024

1025
1026
1027
1028
1029
1030
1031
1032
    def get_mm_mapping(self) -> MultiModelKeys:
        """
        Get the module prefix in multimodal models
        """
        return MultiModelKeys.from_string_field(language_model="llm",
                                                connector="resampler",
                                                tower_model="vpm")

Jee Jee Li's avatar
Jee Jee Li committed
1033
1034
    def init_llm(
        self,
1035
        vllm_config: VllmConfig,
1036
        prefix: str = "",
Jee Jee Li's avatar
Jee Jee Li committed
1037
1038
1039
    ) -> nn.Module:
        raise NotImplementedError

1040
1041
1042
1043
    def init_vision_module(
        self,
        config: PretrainedConfig,
        quant_config: Optional[QuantizationConfig],
1044
        prefix: str = "",
1045
    ) -> nn.Module:
Jee Jee Li's avatar
Jee Jee Li committed
1046
1047
        raise NotImplementedError

1048
1049
1050
1051
1052
    def init_resampler(self,
                       embed_dim: int,
                       vision_dim: int,
                       quant_config: Optional[QuantizationConfig] = None,
                       prefix: str = "") -> nn.Module:
Jee Jee Li's avatar
Jee Jee Li committed
1053
1054
        raise NotImplementedError

1055
1056
    def get_vision_hidden_states(
            self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
Jee Jee Li's avatar
Jee Jee Li committed
1057
1058
1059
        raise NotImplementedError


1060
class MiniCPMV2_0(MiniCPMVBaseModel):
Jee Jee Li's avatar
Jee Jee Li committed
1061

1062
1063
    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
        super().__init__(vllm_config=vllm_config, prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
1064
1065
1066
1067
        assert self.version == (2, 0)

    def init_llm(
        self,
1068
        vllm_config: VllmConfig,
1069
        prefix: str = "",
Jee Jee Li's avatar
Jee Jee Li committed
1070
    ) -> nn.Module:
1071
        return MiniCPMForCausalLM(vllm_config=vllm_config, prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
1072

1073
1074
1075
1076
    def init_vision_module(
        self,
        config: PretrainedConfig,
        quant_config: Optional[QuantizationConfig],
1077
        prefix: str = "",
1078
    ) -> nn.Module:
1079
        # TODO: refactor vision model through timm wrapper from transformers
Jee Jee Li's avatar
Jee Jee Li committed
1080
1081
1082
1083
        try:
            import timm
        except ImportError:
            raise ImportError("Please install timm==0.9.10") from ImportError
1084

Jee Jee Li's avatar
Jee Jee Li committed
1085
1086
1087
1088
1089
1090
1091
1092
1093
        with set_default_torch_dtype(torch.float16):
            model = timm.create_model(
                "vit_so400m_patch14_siglip_384.webli",
                pretrained=False,
                num_classes=0,
                dynamic_img_size=True,
                dynamic_img_pad=True,
            )

1094
1095
        model = model.to(dtype=torch.get_default_dtype())

Jee Jee Li's avatar
Jee Jee Li committed
1096
1097
1098
1099
1100
1101
1102
1103
1104
        if (isinstance(model, timm.models.VisionTransformer)
                and model.attn_pool is not None):
            model.attn_pool = torch.nn.Identity()

        if self.config.drop_vision_last_layer:
            model.blocks = model.blocks[:-1]

        return model

1105
1106
1107
1108
1109
    def init_resampler(self,
                       embed_dim: int,
                       vision_dim: int,
                       quant_config: Optional[QuantizationConfig] = None,
                       prefix: str = "") -> nn.Module:
Jee Jee Li's avatar
Jee Jee Li committed
1110
        with set_default_torch_dtype(torch.float16):
1111
1112
1113
1114
1115
1116
1117
1118
1119
            resampler = Resampler2(embed_dim=embed_dim,
                                   num_heads=embed_dim // 128,
                                   grid_size=int(
                                       math.sqrt(self.config.query_num)),
                                   kv_dim=vision_dim,
                                   adaptive=False,
                                   do_post_projection=True,
                                   quant_config=quant_config,
                                   prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
1120

1121
1122
        return resampler.to(device=current_platform.device_type,
                            dtype=torch.get_default_dtype())
Jee Jee Li's avatar
Jee Jee Li committed
1123

1124
1125
1126
1127
1128
1129
1130
1131
1132
    def get_vision_hidden_states(
            self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
        pixel_values = data["pixel_values"]

        P_h, P_w = self.vpm.patch_embed.patch_size
        dtype: torch.dtype = self.vpm.pos_embed.data.dtype
        num_prefix_tokens = getattr(self.vpm, "num_prefix_tokens", 0)

        res = list[torch.Tensor]()
Jee Jee Li's avatar
Jee Jee Li committed
1133
1134
        for pixel_value in pixel_values:
            H, W = pixel_value[0].shape[-2:]
1135
            tgt_size = (math.ceil(H / P_h), math.ceil(W / P_w))
Jee Jee Li's avatar
Jee Jee Li committed
1136
1137
1138
            vision_embedding = self.vpm.forward_features(
                pixel_value.unsqueeze(0).type(dtype))

1139
1140
1141
            if num_prefix_tokens > 0:
                vision_embedding = vision_embedding[:, num_prefix_tokens:]
            res.append(self.resampler(vision_embedding, tgt_size))
Jee Jee Li's avatar
Jee Jee Li committed
1142

1143
        return torch.vstack(res)
Jee Jee Li's avatar
Jee Jee Li committed
1144
1145


1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
    packed_modules_mapping = {
        "qkv_proj": [
            "q_proj",
            "k_proj",
            "v_proj",
        ],
        "gate_up_proj": [
            "gate_proj",
            "up_proj",
        ],
    }
Jee Jee Li's avatar
Jee Jee Li committed
1158

1159
1160
    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
        super().__init__(vllm_config=vllm_config, prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
1161
1162
1163
1164
        assert self.version == (2, 5)

    def init_llm(
        self,
1165
        vllm_config: VllmConfig,
1166
        prefix: str = "",
Jee Jee Li's avatar
Jee Jee Li committed
1167
    ) -> nn.Module:
1168
        return LlamaForCausalLM(vllm_config=vllm_config, prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
1169

1170
1171
1172
1173
    def init_vision_module(
        self,
        config: PretrainedConfig,
        quant_config: Optional[QuantizationConfig],
1174
        prefix: str = "",
1175
1176
    ) -> nn.Module:
        model = Idefics2VisionTransformer(config.vision_config,
1177
1178
                                          quant_config=quant_config,
                                          prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
1179
1180
1181
1182
        if self.config.drop_vision_last_layer:
            model.encoder.layers = model.encoder.layers[:-1]
        return model

1183
1184
1185
1186
1187
    def init_resampler(self,
                       embed_dim: int,
                       vision_dim: int,
                       quant_config: Optional[QuantizationConfig] = None,
                       prefix: str = "") -> nn.Module:
Jee Jee Li's avatar
Jee Jee Li committed
1188
        with set_default_torch_dtype(torch.float16):
1189
1190
1191
1192
1193
1194
            resampler = Resampler2_5(num_queries=self.config.query_num,
                                     embed_dim=embed_dim,
                                     num_heads=embed_dim // 128,
                                     kv_dim=vision_dim,
                                     quant_config=quant_config,
                                     prefix=prefix)
1195

1196
1197
        return resampler.to(device=current_platform.device_type,
                            dtype=torch.get_default_dtype())
Jee Jee Li's avatar
Jee Jee Li committed
1198

1199
1200
1201
    def get_vision_hidden_states(
            self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
        pixel_values = data["pixel_values"]
Jee Jee Li's avatar
Jee Jee Li committed
1202
1203
        tgt_sizes = data["tgt_sizes"]

1204
1205
1206
1207
1208
        B = len(pixel_values)
        P = pixel_values[0].shape[-2]
        L = max(item.shape[-1] for item in pixel_values)
        device = pixel_values[0].device
        dtype = pixel_values[0].dtype
Jee Jee Li's avatar
Jee Jee Li committed
1209

1210
1211
1212
1213
1214
1215
        all_pixel_values = torch.zeros((B, 3, P, L),
                                       dtype=dtype,
                                       device=device)
        for i, pixel_values_item in enumerate(pixel_values):
            L_item = pixel_values_item.shape[-1]
            all_pixel_values[i, ..., :L_item] = pixel_values_item
Jee Jee Li's avatar
Jee Jee Li committed
1216

1217
1218
1219
        num_patches = tgt_sizes.prod(-1)
        max_patches = num_patches.max().item()
        assert isinstance(max_patches, int)
Jee Jee Li's avatar
Jee Jee Li committed
1220

1221
        patch_attn_mask = torch.zeros((B, max_patches),
Jee Jee Li's avatar
Jee Jee Li committed
1222
1223
                                      dtype=torch.bool,
                                      device=device)
1224
1225
        for i, num_patches_item in enumerate(num_patches):
            patch_attn_mask[i, :num_patches_item] = True
Jee Jee Li's avatar
Jee Jee Li committed
1226

1227
1228
1229
1230
1231
1232
1233
        vision_embedding = self.vpm(
            all_pixel_values,
            patch_attention_mask=patch_attn_mask.unsqueeze(1),
            tgt_sizes=None,
        )

        return self.resampler(vision_embedding, tgt_sizes)
Jee Jee Li's avatar
Jee Jee Li committed
1234
1235


1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
    packed_modules_mapping = {
        "qkv_proj": [
            "q_proj",
            "k_proj",
            "v_proj",
        ],
        "gate_up_proj": [
            "gate_proj",
            "up_proj",
        ],
    }
Jee Jee Li's avatar
Jee Jee Li committed
1248

1249
1250
    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
        super().__init__(vllm_config=vllm_config, prefix=prefix)
1251
        assert self.version == (2, 6)
Jee Jee Li's avatar
Jee Jee Li committed
1252
1253
1254

    def init_llm(
        self,
1255
        vllm_config: VllmConfig,
1256
        prefix: str = "",
Jee Jee Li's avatar
Jee Jee Li committed
1257
    ) -> nn.Module:
1258
        return Qwen2ForCausalLM(vllm_config=vllm_config, prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
1259

1260
1261
1262
1263
    def init_vision_module(
        self,
        config: PretrainedConfig,
        quant_config: Optional[QuantizationConfig],
1264
        prefix: str = "",
1265
1266
    ) -> nn.Module:
        model = Idefics2VisionTransformer(config.vision_config,
1267
1268
                                          quant_config=quant_config,
                                          prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
1269
1270
1271
1272
        if self.config.drop_vision_last_layer:
            model.encoder.layers = model.encoder.layers[:-1]
        return model

1273
1274
1275
1276
1277
    def init_resampler(self,
                       embed_dim: int,
                       vision_dim: int,
                       quant_config: Optional[QuantizationConfig] = None,
                       prefix: str = "") -> nn.Module:
Jee Jee Li's avatar
Jee Jee Li committed
1278
        with set_default_torch_dtype(torch.float16):
1279
            # The resampler in 2.6 remains consistent with the one in 2.5.
1280
1281
1282
1283
1284
1285
            resampler = Resampler2_5(num_queries=self.config.query_num,
                                     embed_dim=embed_dim,
                                     num_heads=embed_dim // 128,
                                     kv_dim=vision_dim,
                                     quant_config=quant_config,
                                     prefix=prefix)
1286

1287
1288
        return resampler.to(device=current_platform.device_type,
                            dtype=torch.get_default_dtype())
Jee Jee Li's avatar
Jee Jee Li committed
1289

1290
1291
1292
    def get_vision_hidden_states(
            self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
        pixel_values = data["pixel_values"]
Jee Jee Li's avatar
Jee Jee Li committed
1293
1294
        tgt_sizes = data["tgt_sizes"]

1295
1296
1297
1298
1299
        B = len(pixel_values)
        P = pixel_values[0].shape[-2]
        L = max(item.shape[-1] for item in pixel_values)
        device = pixel_values[0].device
        dtype = pixel_values[0].dtype
Jee Jee Li's avatar
Jee Jee Li committed
1300

1301
1302
1303
1304
1305
1306
        all_pixel_values = torch.zeros((B, 3, P, L),
                                       dtype=dtype,
                                       device=device)
        for i, pixel_values_item in enumerate(pixel_values):
            L_item = pixel_values_item.shape[-1]
            all_pixel_values[i, ..., :L_item] = pixel_values_item
Jee Jee Li's avatar
Jee Jee Li committed
1307

1308
1309
1310
        num_patches = tgt_sizes.prod(-1)
        max_patches = num_patches.max().item()
        assert isinstance(max_patches, int)
Jee Jee Li's avatar
Jee Jee Li committed
1311

1312
        patch_attn_mask = torch.zeros((B, max_patches),
Jee Jee Li's avatar
Jee Jee Li committed
1313
1314
                                      dtype=torch.bool,
                                      device=device)
1315
1316
1317
        for i, num_patches_item in enumerate(num_patches):
            patch_attn_mask[i, :num_patches_item] = True

Jee Jee Li's avatar
Jee Jee Li committed
1318
        vision_embedding = self.vpm(
1319
1320
            all_pixel_values,
            patch_attention_mask=patch_attn_mask.unsqueeze(1),
Jee Jee Li's avatar
Jee Jee Li committed
1321
            tgt_sizes=tgt_sizes,
1322
        )
Jee Jee Li's avatar
Jee Jee Li committed
1323
1324
1325
1326

        return self.resampler(vision_embedding, tgt_sizes)


1327
1328
1329
_SUPPORT_VERSION = {
    (2, 0): MiniCPMV2_0,
    (2, 5): MiniCPMV2_5,
1330
    (2, 6): MiniCPMV2_6,
1331
1332
1333
}


1334
1335
1336
1337
1338
@MULTIMODAL_REGISTRY.register_processor(
    MiniCPMVMultiModalProcessor,
    info=MiniCPMVProcessingInfo,
    dummy_inputs=MiniCPMVDummyInputsBuilder)
class MiniCPMV(MiniCPMVBaseModel, SupportsMultiModal, SupportsLoRA):
Jee Jee Li's avatar
Jee Jee Li committed
1339
1340
1341
1342
1343
    """
    Different versions of MiniCPMV use different visual encoders and LLMs,
    which is not conducive to the current integration logic of LoRA and
    bitsandbytes in vLLM. Therefore, it is necessary to separate them.
    """
1344

1345
    def __new__(cls, *, vllm_config: VllmConfig, prefix: str = ""):
1346
        config = vllm_config.model_config.hf_config
Jee Jee Li's avatar
Jee Jee Li committed
1347
1348
1349
1350
1351
1352
1353
1354
1355
        if not hasattr(config, "version"):
            if config.hidden_size == 2304 and config.query_num == 64:
                version = (2, 0)
            else:
                version = (2, 5)
        else:
            version = str(config.version).split(".")
            version = tuple([int(x) for x in version])
        # Dispatch class based on version
1356
1357
        instance_cls = _SUPPORT_VERSION.get(version)
        if instance_cls is None:
1358
1359
            raise ValueError(
                "Currently, MiniCPMV only supports versions 2.0, 2.5, and 2.6")
1360
1361
1362
1363
1364
1365
1366

        # quant_config references base class members,
        # so update values before init is called
        cls.packed_modules_mapping.update(instance_cls.packed_modules_mapping)
        cls.embedding_modules.update(instance_cls.embedding_modules)
        cls.embedding_padding_modules += instance_cls.embedding_padding_modules
        return instance_cls(vllm_config=vllm_config, prefix=prefix)