minicpmv.py 57.7 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# Adapted from
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
# Copyright 2023 The vLLM team.
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
#
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
# and OPT implementations in this library. It has been modified from its
# original forms to accommodate minor architectural differences compared
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Alphi's avatar
Alphi committed
25
"""Inference-only MiniCPM-V model compatible with HuggingFace weights."""
26

27
import math
28
from collections import defaultdict
29
from collections.abc import Callable, Iterable, Mapping, Sequence
30
from functools import partial
tc-mb's avatar
tc-mb committed
31
from itertools import chain
32
from typing import Annotated, Any, Literal, TypeAlias
33

34
import numpy as np
35
import torch
Alphi's avatar
Alphi committed
36
import torch.types
37
from torch import nn
tc-mb's avatar
tc-mb committed
38
from torch.nn.init import trunc_normal_
39
from transformers import BatchFeature, PretrainedConfig
40
from typing_extensions import TypeVar
41

42
from vllm.config import VllmConfig
43
from vllm.config.multimodal import BaseDummyOptions
44
from vllm.model_executor.layers.quantization import QuantizationConfig
45
46
47
48
49
from vllm.model_executor.layers.resampler import (
    BaseResampler,
    Resampler2,
    get_2d_sincos_pos_embed,
)
50
51
from vllm.model_executor.models.llama import LlamaForCausalLM
from vllm.model_executor.models.minicpm import MiniCPMForCausalLM
52
from vllm.model_executor.models.module_mapping import MultiModelKeys
53
from vllm.model_executor.models.qwen2 import Qwen2ForCausalLM
tc-mb's avatar
tc-mb committed
54
55
from vllm.model_executor.models.qwen3 import Qwen3ForCausalLM
from vllm.multimodal import MULTIMODAL_REGISTRY
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
from vllm.multimodal.inputs import (
    MultiModalDataDict,
    MultiModalFieldConfig,
    MultiModalKwargsItems,
    NestedTensors,
)
from vllm.multimodal.parse import (
    DictEmbeddingItems,
    ImageItem,
    ImageProcessorItems,
    ImageSize,
    ModalityData,
    ModalityDataItems,
    MultiModalDataItems,
    MultiModalDataParser,
    VideoItem,
    VideoProcessorItems,
)
74
75
from vllm.multimodal.processing import BaseDummyInputsBuilder
from vllm.multimodal.processing.processor import (
76
77
78
79
80
81
82
83
    BaseMultiModalProcessor,
    BaseProcessingInfo,
    PromptReplacement,
    PromptUpdate,
    PromptUpdateDetails,
    ResolvedPromptUpdate,
    _seq2text,
)
84
from vllm.platforms import current_platform
85
from vllm.sequence import IntermediateTensors
86
from vllm.utils.collection_utils import flatten_2d_lists
87
from vllm.utils.tensor_schema import TensorSchema, TensorShape
88
from vllm.utils.torch_utils import set_default_torch_dtype
89

Jee Jee Li's avatar
Jee Jee Li committed
90
from .idefics2_vision_model import Idefics2VisionTransformer
91
92
93
94
95
96
from .interfaces import (
    MultiModalEmbeddings,
    SupportsLoRA,
    SupportsMultiModal,
    SupportsPP,
)
97
from .utils import AutoWeightsLoader, flatten_bn, maybe_prefix
98

99
100
101
# For profile run
_MAX_FRAMES_PER_VIDEO = 16

102

103
class MiniCPMVImagePixelInputs(TensorSchema):
Jee Jee Li's avatar
Jee Jee Li committed
104
    """
105
106
107
108
109
110
    Dimensions:
        - bns: Batch size * number of images * number of slices
        - bn: Batch size * number of images
        - c: Number of channels
        - h: Height
        - w: Width
Jee Jee Li's avatar
Jee Jee Li committed
111
112
    """

113
114
    type: Literal["pixel_values"] = "pixel_values"

115
    # Note that the patch size may vary, so we pass it as a list instead of a
116
117
118
    # batched tensor.
    pixel_values: Annotated[
        list[torch.Tensor],
119
        TensorShape("bns", "c", "h", "w", dynamic_dims={"h", "w"}),
120
121
122
123
124
125
126
127
128
129
130
131
    ]
    tgt_sizes: Annotated[
        torch.Tensor,
        TensorShape("bns", 2),  # This should be in `(height, width)` format.
    ]
    num_slices: Annotated[
        torch.Tensor,
        TensorShape("bn"),
    ]


class MiniCPMVImageEmbeddingInputs(TensorSchema):
Jee Jee Li's avatar
Jee Jee Li committed
132
    """
133
134
135
136
    Dimensions:
        - bn: Batch size * number of images
        - ns: Number of slices
        - hs: Hidden size (must match language model backbone)
Jee Jee Li's avatar
Jee Jee Li committed
137
138
    """

139
    type: Literal["image_embeds"]
140
    image_embeds: Annotated[
141
        torch.Tensor | list[torch.Tensor],
142
        TensorShape("bn", "ns", "hs", dynamic_dims={"ns"}),
143
    ]
144
145


146
MiniCPMVImageInputs: TypeAlias = MiniCPMVImagePixelInputs | MiniCPMVImageEmbeddingInputs
147

Jee Jee Li's avatar
Jee Jee Li committed
148
149
150
151
DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)


class Resampler2_5(BaseResampler):
152
153
154
155
156
    def __init__(
        self,
        num_queries: int,
        embed_dim: int,
        num_heads: int,
157
        kv_dim: int | None = None,
158
159
        norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
        max_size: tuple[int, int] = (70, 70),
160
        quant_config: QuantizationConfig | None = None,
161
162
163
164
165
166
167
168
169
170
171
        prefix: str = "",
    ) -> None:
        super().__init__(
            num_queries,
            embed_dim,
            num_heads,
            kv_dim,
            norm_layer,
            quant_config=quant_config,
            prefix=prefix,
        )
Jee Jee Li's avatar
Jee Jee Li committed
172
173
174

        self.max_size = max_size
        self._set_2d_pos_cache(self.max_size)
175

176
177
178
179
180
181
    def _set_2d_pos_cache(
        self, max_size: tuple[int, int], device: torch.types.Device = "cpu"
    ) -> None:
        pos_embed_arr = get_2d_sincos_pos_embed(
            self.embed_dim, max_size, version=(2, 5)
        )
Jee Jee Li's avatar
Jee Jee Li committed
182
        pos_embed = torch.from_numpy(pos_embed_arr).float().to(device)
183
184
        self.register_buffer("pos_embed", pos_embed, persistent=False)

185
186
187
    def _adjust_pos_cache(
        self, tgt_sizes: torch.Tensor, device: torch.types.Device
    ) -> None:
Jee Jee Li's avatar
Jee Jee Li committed
188
189
190
191
        max_h = tgt_sizes[:, 0].max().item()
        max_w = tgt_sizes[:, 1].max().item()
        assert isinstance(max_h, int) and isinstance(max_w, int)

192
        if max_h > self.max_size[0] or max_w > self.max_size[1]:
Jee Jee Li's avatar
Jee Jee Li committed
193
            self.max_size = (
194
                max(max_h, self.max_size[0]),
Jee Jee Li's avatar
Jee Jee Li committed
195
196
                max(max_w, self.max_size[1]),
            )
197
198
            self._set_2d_pos_cache(self.max_size, device)

199
    def forward(self, x: torch.Tensor, tgt_sizes: torch.Tensor) -> torch.Tensor:
200
201
202
203
204
205
206
207
208
209
        assert x.shape[0] == tgt_sizes.shape[0]
        bs = x.shape[0]

        device = x.device
        dtype = x.dtype

        patch_len = tgt_sizes[:, 0] * tgt_sizes[:, 1]

        self._adjust_pos_cache(tgt_sizes, device=device)

Jee Jee Li's avatar
Jee Jee Li committed
210
211
212
        max_patch_len = patch_len.max().item()
        assert isinstance(max_patch_len, int)

213
214
215
        key_padding_mask = torch.zeros(
            (bs, max_patch_len), dtype=torch.bool, device=device
        )
216
217
218

        pos_embed = []
        for i in range(bs):
Jee Jee Li's avatar
Jee Jee Li committed
219
            tgt_h, tgt_w = tgt_sizes[i].tolist()
220
221
222
223
224
225
226
            pos_embed.append(
                self.pos_embed[:tgt_h, :tgt_w, :].reshape((tgt_h * tgt_w, -1)).to(dtype)
            )  # patches * D
            key_padding_mask[i, patch_len[i] :] = True
        pos_embed = torch.nn.utils.rnn.pad_sequence(
            pos_embed, batch_first=True, padding_value=0.0
        ).permute(1, 0, 2)  # BLD => L * B * D
Jee Jee Li's avatar
Jee Jee Li committed
227
        x, _ = self.kv_proj(x)  # B * L * D
228
229
230
231
232
233
234
235
        x = self.ln_kv(x).permute(1, 0, 2)  # L * B * D

        q = self.ln_q(self.query)  # Q * D

        out = self.attn(
            self._repeat(q, bs),  # Q * B * D
            x + pos_embed,  # L * B * D +  L * B * D
            x,
Jee Jee Li's avatar
Jee Jee Li committed
236
237
            key_padding_mask=key_padding_mask,
        )[0]
238
239
240
241
242
243
244
245
        #  out: Q * B * D
        x = out.permute(1, 0, 2)  # B * Q * D

        x = self.ln_post(x)
        x = x @ self.proj
        return x


tc-mb's avatar
tc-mb committed
246
class Resampler4_5(Resampler2_5):
247
248
249
250
251
    def __init__(
        self,
        num_queries: int,
        embed_dim: int,
        num_heads: int,
252
        kv_dim: int | None = None,
253
254
255
        norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
        max_size: tuple[int, int] = (70, 70),
        max_temporal_size: int = 36000,
256
        quant_config: QuantizationConfig | None = None,
257
258
259
260
261
262
263
264
265
266
267
268
        prefix: str = "",
    ) -> None:
        super().__init__(
            num_queries,
            embed_dim,
            num_heads,
            kv_dim,
            norm_layer,
            max_size,
            quant_config=quant_config,
            prefix=prefix,
        )
tc-mb's avatar
tc-mb committed
269

270
        trunc_normal_(self.query, std=0.02)
tc-mb's avatar
tc-mb committed
271
272
273
274
        self.max_temporal_size = max_temporal_size
        self._set_temporal_pos_cache(self.max_temporal_size)
        self.apply(self._init_weights)

275
276
277
    def get_1d_sincos_pos_embed_from_temporal_size(
        self, embed_dim: int, pos: np.ndarray
    ):
tc-mb's avatar
tc-mb committed
278
279
280
281
282
283
284
        """
        embed_dim: output dimension for each position
        pos: a list of positions to be encoded: size (M,)
        out: (M, D)
        """
        assert embed_dim % 2 == 0
        omega = np.arange(embed_dim // 2, dtype=np.float32)
285
286
        omega /= embed_dim / 2.0
        omega = 1.0 / 10000**omega  # (D/2,)
tc-mb's avatar
tc-mb committed
287
288

        pos = pos.reshape(-1)  # (M,)
289
        out = np.einsum("m,d->md", pos, omega)  # (M, D/2), outer product
tc-mb's avatar
tc-mb committed
290
291
292
293
294
295
296

        emb_sin = np.sin(out)  # (M, D/2)
        emb_cos = np.cos(out)  # (M, D/2)

        emb = np.concatenate([emb_sin, emb_cos], axis=1)  # (M, D)
        return emb

297
298
299
    def _set_temporal_pos_cache(
        self, max_temporal_size: int, device: torch.types.Device = "cpu"
    ) -> None:
tc-mb's avatar
tc-mb committed
300
        temporal_size = np.arange(max_temporal_size, dtype=np.float32)
301
302
303
304
305
306
307
308
309
        pos_embed = (
            torch.from_numpy(
                self.get_1d_sincos_pos_embed_from_temporal_size(
                    self.embed_dim, temporal_size
                )
            )
            .float()
            .to(device)
        )
tc-mb's avatar
tc-mb committed
310
311
        self.register_buffer("temporal_pos_embed", pos_embed, persistent=False)

312
313
314
    def _adjust_temporal_pos_cache(
        self, max_temporal_size: int, device: torch.types.Device = "cpu"
    ):
tc-mb's avatar
tc-mb committed
315
316
317
318
        if max_temporal_size > self.max_temporal_size:
            self.max_temporal_size = max_temporal_size
            self._set_temporal_pos_cache(self.max_temporal_size, device)

319
    def _init_weights(self, m: nn.Linear | nn.LayerNorm):
tc-mb's avatar
tc-mb committed
320
        if isinstance(m, nn.Linear):
321
            trunc_normal_(m.weight, std=0.02)
tc-mb's avatar
tc-mb committed
322
323
324
325
326
327
328
329
330
331
332
            if isinstance(m, nn.Linear) and m.bias is not None:
                nn.init.constant_(m.bias, 0)
        elif isinstance(m, nn.LayerNorm):
            nn.init.constant_(m.bias, 0)
            nn.init.constant_(m.weight, 1.0)

    def forward(
        self,
        x: torch.Tensor,
        tgt_sizes: torch.Tensor,
        # temporal_ids for high refresh rate videos
333
        temporal_ids=None,
tc-mb's avatar
tc-mb committed
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
    ) -> torch.Tensor:
        assert x.shape[0] == tgt_sizes.shape[0]
        bs = x.shape[0]

        device = x.device
        dtype = x.dtype

        patch_len = tgt_sizes[:, 0] * tgt_sizes[:, 1]

        self._adjust_pos_cache(tgt_sizes, device=device)

        temporal_pos_emb = False
        temporal_ids_flatten = None
        if temporal_ids is not None:
            # example: [[-1], [-1], [2, 6, 9]]
            temporal_ids_flatten = list(chain.from_iterable(temporal_ids))
            max_temporal_size = max(temporal_ids_flatten, default=0)
            if max_temporal_size > -1:
                temporal_pos_emb = True
            if max_temporal_size > self.max_temporal_size:
                self._adjust_temporal_pos_cache(max_temporal_size, device)

        max_patch_len = patch_len.max().item()
        assert isinstance(max_patch_len, int)

359
360
361
        key_padding_mask = torch.zeros(
            (bs, max_patch_len), dtype=torch.bool, device=device
        )
tc-mb's avatar
tc-mb committed
362
363
364
365
366
367
368
369
370
371
372
373

        x, _ = self.kv_proj(x)  # B * L * D
        x = self.ln_kv(x).permute(1, 0, 2)  # L * B * D
        q = self.ln_q(self.query)  # Q * D

        pos_embed_2d = []
        pos_embed_temporal = []
        for i in range(bs):
            tgt_h, tgt_w = tgt_sizes[i]
            if temporal_pos_emb:
                if temporal_ids_flatten[i] == -1:
                    pos_embed_temporal.append(
374
375
                        torch.zeros(self.embed_dim, dtype=dtype, device=device)
                    )
tc-mb's avatar
tc-mb committed
376
                else:
377
378
379
                    pos_embed_temporal.append(
                        self.temporal_pos_embed[temporal_ids_flatten[i]].to(dtype)
                    )  # D
tc-mb's avatar
tc-mb committed
380

381
382
383
384
            pos_embed_2d.append(
                self.pos_embed[:tgt_h, :tgt_w, :].reshape((tgt_h * tgt_w, -1)).to(dtype)
            )  # patches * D
            key_padding_mask[i, patch_len[i] :] = True
tc-mb's avatar
tc-mb committed
385
386

        pos_embed_2d = torch.nn.utils.rnn.pad_sequence(
387
388
            pos_embed_2d, batch_first=True, padding_value=0.0
        ).permute(1, 0, 2)  # BLD => L * B * D
tc-mb's avatar
tc-mb committed
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403

        k = x
        v = x + pos_embed_2d
        if pos_embed_temporal:
            k += torch.stack(pos_embed_temporal, dim=0)
            bs = len(temporal_ids)
            merge_k = []
            merge_v = []
            merge_key_padding_mask = []

            start = 0
            for tp in temporal_ids:
                end = start + len(tp)
                # L * (end-start) * D -> (end-start) * L * D
                # -> 1 * L*(end-start) * D
404
405
406
407
408
409
                merge_k.append(
                    k[:, start:end, :].permute(1, 0, 2).reshape(-1, self.embed_dim)
                )
                merge_v.append(
                    v[:, start:end, :].permute(1, 0, 2).reshape(-1, self.embed_dim)
                )
tc-mb's avatar
tc-mb committed
410
                merge_key_padding_mask.append(
411
412
                    key_padding_mask[start:end, :].reshape(-1, 1)
                )
tc-mb's avatar
tc-mb committed
413
414
415

                start = end

416
417
418
419
420
421
            k = torch.nn.utils.rnn.pad_sequence(
                merge_k, batch_first=True, padding_value=0.0
            ).permute(1, 0, 2)  # L*(end-start)
            v = torch.nn.utils.rnn.pad_sequence(
                merge_v, batch_first=True, padding_value=0.0
            ).permute(1, 0, 2)  # L*(end-start)
tc-mb's avatar
tc-mb committed
422
            key_padding_mask = torch.nn.utils.rnn.pad_sequence(
423
424
                merge_key_padding_mask, batch_first=True, padding_value=True
            ).squeeze(-1)
tc-mb's avatar
tc-mb committed
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439

        out = self.attn(
            self._repeat(q, bs),  # Q * B * D
            k,  # L * B * D +  L * B * D
            v,
            key_padding_mask=key_padding_mask,
        )[0]
        #  out: Q * B * D
        x = out.permute(1, 0, 2)  # B * Q * D

        x = self.ln_post(x)
        x = x @ self.proj
        return x


440
def get_version_by_config(config: PretrainedConfig) -> tuple[int, ...]:
441
442
443
444
445
446
447
448
449
450
451
452
    version_float = getattr(config, "version", None)

    # The old configs do not include version number
    # TODO: Remove this after the HF repos are updated
    if version_float is None:
        if config.hidden_size == 2304 and config.query_num == 64:
            return (2, 0)
        return (2, 5)
    version_str = str(version_float)
    return tuple(int(x) for x in version_str.split("."))


453
454
def _minicpmv_field_config(hf_inputs: Mapping[str, torch.Tensor]):
    return dict(
455
        pixel_values=MultiModalFieldConfig.batched("image"),
456
        image_sizes=MultiModalFieldConfig.batched("image"),
457
458
459
        tgt_sizes=MultiModalFieldConfig.batched("image"),
        image_embeds=MultiModalFieldConfig.batched("image"),
        video_pixel_values=MultiModalFieldConfig.batched("video"),
460
        video_image_sizes=MultiModalFieldConfig.batched("video"),
461
462
        video_tgt_sizes=MultiModalFieldConfig.batched("video"),
        video_embeds=MultiModalFieldConfig.batched("video"),
463
464
465
466
467
468
469
    )


class MiniCPMVImageEmbeddingItems(DictEmbeddingItems):
    def __init__(
        self,
        data: Mapping[str, torch.Tensor],
470
471
472
473
        fields_factory: Callable[
            [Mapping[str, torch.Tensor]],
            Mapping[str, MultiModalFieldConfig],
        ],
474
475
476
477
478
    ) -> None:
        super().__init__(
            data,
            modality="image",
            required_fields={"image_embeds", "image_sizes"},
479
            fields_factory=fields_factory,
480
481
482
483
484
485
486
487
488
489
490
        )

    def get_image_size(self, index: int) -> ImageSize:
        image_size = self.get(index)["image_sizes"].tolist()
        return ImageSize(width=image_size[0], height=image_size[1])


class MiniCPMVVideoEmbeddingItems(DictEmbeddingItems):
    def __init__(
        self,
        data: Mapping[str, torch.Tensor],
491
492
493
494
        fields_factory: Callable[
            [Mapping[str, torch.Tensor]],
            Mapping[str, MultiModalFieldConfig],
        ],
495
496
497
498
499
    ) -> None:
        super().__init__(
            data,
            modality="video",
            required_fields={"video_embeds", "video_image_sizes"},
500
            fields_factory=fields_factory,
501
502
503
504
505
506
507
508
509
510
        )

    def get_frame_size(self, index: int) -> ImageSize:
        frame_size = self.get(index)["video_image_sizes"].tolist()
        return ImageSize(width=frame_size[0], height=frame_size[1])

    def get_num_frames(self, index: int) -> int:
        return len(self.get(index)["video_image_sizes"])


511
512
513
class MiniCPMVMultiModalDataParser(MultiModalDataParser):
    def _parse_image_data(
        self,
514
515
        data: dict[str, torch.Tensor] | ModalityData[ImageItem],
    ) -> ModalityDataItems[Any, Any] | None:
516
        if isinstance(data, dict):
517
518
            return MiniCPMVImageEmbeddingItems(
                data,
519
                fields_factory=_minicpmv_field_config,
520
521
            )

522
523
524
525
        return super()._parse_image_data(data)

    def _parse_video_data(
        self,
526
527
        data: dict[str, torch.Tensor] | ModalityData[VideoItem],
    ) -> ModalityDataItems[Any, Any] | None:
528
        if isinstance(data, dict):
529
530
            return MiniCPMVVideoEmbeddingItems(
                data,
531
                fields_factory=_minicpmv_field_config,
532
533
            )

534
535
536
537
538
539
540
541
542
543
        return super()._parse_video_data(data)


class MiniCPMVProcessingInfo(BaseProcessingInfo):
    image_pattern = "(<image>./</image>)"
    video_pattern = "(<video>./</video>)"

    def get_hf_config(self):
        return self.ctx.get_hf_config()

544
545
    def get_hf_processor(self, **kwargs: object):
        hf_processor = self.ctx.get_hf_processor(**kwargs)
546
547
548
549
550
551
552
553
554

        # NumPy arrays are considered as Iterable but not Sequence in
        # https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428
        image_processor = hf_processor.image_processor  # type: ignore
        for attr in ("mean", "std"):
            val = getattr(image_processor, attr)
            if isinstance(val, np.ndarray):
                setattr(image_processor, attr, val.tolist())

555
556
        return hf_processor

557
558
    def get_image_processor(self, **kwargs: object):
        return self.get_hf_processor(**kwargs).image_processor
559

560
561
562
563
564
    def get_data_parser(self):
        return MiniCPMVMultiModalDataParser(
            expected_hidden_size=self._get_expected_hidden_size(),
        )

565
566
567
    def get_model_version(self):
        return get_version_by_config(self.get_hf_config())

568
    def get_supported_mm_limits(self) -> Mapping[str, int | None]:
569
        mm_limits = {"image": None}
tc-mb's avatar
tc-mb committed
570
        if self.get_model_version() in {(2, 6), (4, 0), (4, 5)}:
571
572
573
            mm_limits["video"] = None

        return mm_limits
574

575
576
577
578
579
    def get_slice_image_placeholder(
        self,
        image_size: ImageSize,
        # For MiniCPM V/O 2.6
        image_idx: int = 0,
580
        max_slice_nums: int | None = None,
581
582
583
584
        use_image_id: bool = True,
    ) -> str:
        image_processor = self.get_image_processor()
        version = self.get_model_version()
585

586
587
        if version == (2, 0) or version == (2, 5):
            return image_processor.get_slice_image_placeholder(image_size)
588

589
590
591
592
593
594
        return image_processor.get_slice_image_placeholder(
            image_size,
            image_idx=image_idx,
            max_slice_nums=max_slice_nums,
            use_image_id=use_image_id,
        )
595

596
597
598
599
    def get_sliced_grid(
        self,
        image_size: ImageSize,
        # For MiniCPM V/O 2.6
600
601
        max_slice_nums: int | None = None,
    ) -> tuple[int, int] | None:
602
603
604
605
606
607
608
609
610
611
612
613
614
615
        image_processor = self.get_image_processor()
        version = self.get_model_version()

        if version == (2, 0) or version == (2, 5):
            return image_processor.get_sliced_grid(image_size)

        if max_slice_nums is None:
            max_slice_nums = image_processor.max_slice_nums

        return image_processor.get_sliced_grid(
            image_size,
            max_slice_nums=max_slice_nums,
        )

616
617
618
    def get_num_image_tokens(
        self,
        image_size: ImageSize,
619
        max_slice_nums: int | None = None,
620
    ) -> int:
621
622
623
        image_processor = self.get_image_processor()

        grid = self.get_sliced_grid(
624
625
626
            image_size,
            max_slice_nums=max_slice_nums,
        )
627
628
629
630
        if grid is None:
            ncols = nrows = 0
        else:
            ncols, nrows = grid
631

632
        return (ncols * nrows + 1) * image_processor.image_feature_size
633
634
635

    def get_max_image_tokens(self) -> int:
        image_size = self.get_image_size_with_most_features()
636
637
638
639
        return self.get_num_image_tokens(image_size)

    def get_image_max_slice_num(self) -> int:
        return getattr(self.get_hf_config(), "max_slice_num", 9)
640
641

    def get_image_size_with_most_features(self) -> ImageSize:
642
643
644
645
646
647
648
649
650
651
652
653
        image_size = getattr(self.get_hf_config(), "image_size", 448)
        max_slice_num = self.get_image_max_slice_num()
        return ImageSize(width=image_size, height=image_size * max_slice_num)

    def get_max_video_frame_tokens(self) -> int:
        frame_size = self.get_video_frame_size_with_most_features()

        return self.get_num_image_tokens(
            frame_size,
            max_slice_nums=self.get_video_max_slice_num(),
        )

654
655
656
657
658
659
660
661
    def get_max_video_tokens(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
    ) -> int:
        num_frames = self.get_num_frames_with_most_features(seq_len, mm_counts)
        num_video_tokens_total = self.get_max_video_frame_tokens() * num_frames
        return num_video_tokens_total
662
663
664

    def get_video_max_slice_num(self) -> int:
        return 1
665

666
    def get_video_frame_size_with_most_features(self) -> ImageSize:
667
668
669
        image_size = getattr(self.get_hf_config(), "image_size", 448)
        max_slice_num = self.get_video_max_slice_num()
        return ImageSize(width=image_size, height=image_size * max_slice_num)
670

671
672
673
674
    def get_max_video_frames(self, max_tokens: int) -> int:
        num_frame_tokens = self.get_max_video_frame_tokens()
        num_frames = max_tokens // num_frame_tokens
        return num_frames
675

676
677
678
679
680
681
682
    def get_num_frames_with_most_features(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
    ) -> int:
        max_images = mm_counts.get("image", 0)
        max_videos = mm_counts.get("video", 0)
683

684
        max_image_tokens = self.get_max_image_tokens() * max_images
685
686
687
688
        max_total_frames = self.get_max_video_frames(seq_len - max_image_tokens)
        max_frames_per_video = min(
            max_total_frames // max(max_videos, 1), _MAX_FRAMES_PER_VIDEO
        )
689

690
        return max(max_frames_per_video, 1)
691
692


693
_I = TypeVar("_I", bound=MiniCPMVProcessingInfo, default=MiniCPMVProcessingInfo)
694
695
696


class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
697
698
699
700
701
702
703
704
705
706
    def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
        num_images = mm_counts.get("image", 0)
        num_videos = mm_counts.get("video", 0)

        image_prompt_texts = self.info.image_pattern * num_images
        video_prompt_texts = self.info.video_pattern * num_videos

        return image_prompt_texts + video_prompt_texts

    def get_dummy_mm_data(
707
708
709
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
710
        mm_options: Mapping[str, BaseDummyOptions] | None = None,
711
        mm_processor_kwargs: Mapping[str, object] | None = None,
712
    ) -> MultiModalDataDict:
713
714
715
        num_images = mm_counts.get("image", 0)
        num_videos = mm_counts.get("video", 0)

716
717
718
719
720
        image_width, image_height = self.info.get_image_size_with_most_features()
        video_width, video_height = self.info.get_video_frame_size_with_most_features()
        num_video_frames = self.info.get_num_frames_with_most_features(
            seq_len, mm_counts
        )
721

722
723
724
        image_overrides = mm_options.get("image") if mm_options else None
        video_overrides = mm_options.get("video") if mm_options else None

725
        return {
726
727
728
729
730
731
            "image": self._get_dummy_images(
                width=image_width,
                height=image_height,
                num_images=num_images,
                overrides=image_overrides,
            ),
732
            "video": [
733
734
735
736
737
738
739
740
                self._get_dummy_images(
                    width=video_width,
                    height=video_height,
                    num_images=num_video_frames,
                    overrides=video_overrides,
                )
            ]
            * num_videos,
741
742
743
        }


744
class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
745
    def get_image_prompt_texts(self, image_size: ImageSize, image_idx: int = 0) -> str:
746
747
748
749
        return self.info.get_slice_image_placeholder(
            image_size,
            image_idx=image_idx,
        )
750

751
752
753
754
755
756
757
758
759
760
    def get_video_prompt_texts(self, image_size: ImageSize, num_frames: int) -> str:
        return (
            self.info.get_slice_image_placeholder(
                image_size=image_size,
                image_idx=0,
                max_slice_nums=self.info.get_video_max_slice_num(),
                use_image_id=False,
            )
            * num_frames
        )
761

762
763
764
765
    def process_images(
        self,
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
766
        tok_kwargs: Mapping[str, object],
767
    ) -> Mapping[str, NestedTensors]:
768
769
770
        if (images := mm_data.get("images")) is None:
            return {}

771
772
        mm_items = self.info.parse_mm_data({"image": images}, validate=False)
        parsed_images = mm_items.get_items(
773
            "image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems)
774
        )
775

776
777
778
779
780
781
782
        if isinstance(parsed_images, MiniCPMVImageEmbeddingItems):
            image_inputs = {}
        else:
            image_inputs = self._base_call_hf_processor(
                prompts=[self.info.image_pattern] * len(parsed_images),
                mm_data={"images": [[image] for image in parsed_images]},
                mm_kwargs=mm_kwargs,
783
                tok_kwargs=tok_kwargs,
784
785
786
787
                out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
            )

        return image_inputs
788

789
790
791
792
    def process_videos(
        self,
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
793
        tok_kwargs: Mapping[str, object],
794
    ) -> Mapping[str, NestedTensors]:
795
796
797
        if (videos := mm_data.get("videos")) is None:
            return {}

798
799
        mm_items = self.info.parse_mm_data({"video": videos}, validate=False)
        parsed_videos = mm_items.get_items(
800
            "video", (MiniCPMVVideoEmbeddingItems, VideoProcessorItems)
801
        )
802
803
804
805
806
807

        if isinstance(parsed_videos, MiniCPMVVideoEmbeddingItems):
            video_inputs = {}
        else:
            video_inputs = self._base_call_hf_processor(
                prompts=[
808
                    self.info.image_pattern * len(video) for video in parsed_videos
809
810
811
812
                ],
                mm_data={"images": list(parsed_videos)},
                mm_kwargs={
                    **mm_kwargs,
813
                    "max_slice_nums": self.info.get_video_max_slice_num(),
814
                },
815
                tok_kwargs=tok_kwargs,
816
817
818
                out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
            )

819
820
        video_inputs = {f"video_{k}": v for k, v in video_inputs.items()}

821
        return video_inputs
822

823
824
825
826
    def process_mm_inputs(
        self,
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
827
        tok_kwargs: Mapping[str, object],
828
    ) -> Mapping[str, NestedTensors]:
829
        return {
830
831
            **self.process_images(mm_data, mm_kwargs, tok_kwargs),
            **self.process_videos(mm_data, mm_kwargs, tok_kwargs),
832
        }
833

834
    def _base_call_hf_processor(
835
        self,
836
837
        prompts: list[str],
        mm_data: Mapping[str, Sequence[object]],
838
        mm_kwargs: Mapping[str, object],
839
        tok_kwargs: Mapping[str, object],
840
841
        *,
        out_keys: set[str],
842
    ) -> dict[str, NestedTensors]:
843
        # This processor supports zipping prompt and mm_data together
tc-mb's avatar
tc-mb committed
844
        if self.info.get_model_version() in {(2, 6), (4, 0), (4, 5)}:
845
846
847
848
            inputs = super()._call_hf_processor(
                prompt=prompts,  # type: ignore
                mm_data=mm_data,
                mm_kwargs=mm_kwargs,
849
                tok_kwargs=tok_kwargs,
850
851
852
853
854
855
856
            )
        else:
            inputs = defaultdict[str, list[torch.Tensor]](list)

            for i, prompt in enumerate(prompts):
                inputs_one = super()._call_hf_processor(
                    prompt=prompt,
857
                    mm_data={k: v[i] for k, v in mm_data.items()},
858
                    mm_kwargs=mm_kwargs,
859
                    tok_kwargs=tok_kwargs,
860
861
862
863
864
865
866
                )

                for k, v in inputs_one.items():
                    assert len(v) == 1, (k, len(v))
                    inputs[k].append(v[0])

        return {k: inputs[k] for k in out_keys}
867
868
869
870
871
872

    def _call_hf_processor(
        self,
        prompt: str,
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
873
        tok_kwargs: Mapping[str, object],
874
875
    ) -> BatchFeature:
        tokenizer = self.info.get_tokenizer()
876

877
878
        input_ids = torch.tensor([tokenizer.encode(prompt, **tok_kwargs)])
        mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs, tok_kwargs)
879

880
881
882
883
884
885
        return BatchFeature(
            {
                "input_ids": input_ids,
                **mm_inputs,
            }
        )
886

887
    def _hf_processor_applies_updates(
888
889
890
891
        self,
        prompt_text: str,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
892
        tokenization_kwargs: Mapping[str, object],
893
894
895
    ) -> bool:
        return False

896
    def _get_prompt_updates(
897
898
899
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
900
        out_mm_kwargs: MultiModalKwargsItems,
901
    ) -> Sequence[PromptUpdate]:
902
903
904
905
        placeholders = [
            ("image", self.info.image_pattern),
            ("video", self.info.video_pattern),
        ]
tc-mb's avatar
tc-mb committed
906
907
908
909
910
911

        # hard code for inconsistency of encode-decode image_pattern
        additional_placeholders = []
        tokenizer = self.info.get_tokenizer()
        for modality, pattern in placeholders:
            sub_pattern = tokenizer.decode(
912
913
                tokenizer.encode(pattern, add_special_tokens=False)
            )
tc-mb's avatar
tc-mb committed
914
915
916
            if sub_pattern != pattern:
                additional_placeholders.append((modality, sub_pattern))
        placeholders += additional_placeholders
917

918
919
        def get_image_replacement(item_idx: int):
            images = mm_items.get_items(
920
921
                "image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems)
            )
922
923
924

            image_size = images.get_image_size(item_idx)

925
926
927
928
            return PromptUpdateDetails.select_text(
                self.get_image_prompt_texts(image_size, item_idx),
                "<unk>",
            )
929
930
931

        def get_video_replacement(item_idx: int):
            videos = mm_items.get_items(
932
933
                "video", (MiniCPMVVideoEmbeddingItems, VideoProcessorItems)
            )
934
935
936
937

            frame_size = videos.get_frame_size(item_idx)
            num_frames = videos.get_num_frames(item_idx)

938
939
940
941
            return PromptUpdateDetails.select_text(
                self.get_video_prompt_texts(frame_size, num_frames),
                "<unk>",
            )
942
943
944
945
946

        get_replacement = {
            "image": get_image_replacement,
            "video": get_video_replacement,
        }
947
948

        return [
949
950
951
            PromptReplacement(
                modality=modality, target=pattern, replacement=get_replacement[modality]
            )
tc-mb's avatar
tc-mb committed
952
            for modality, pattern in placeholders
953
        ]
954

955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
    def _recompute_cached_prompt_update(
        self,
        cached_update: ResolvedPromptUpdate,
        new_item_idx: int,
    ) -> ResolvedPromptUpdate:
        new_update = super()._recompute_cached_prompt_update(
            cached_update,
            new_item_idx,
        )

        if cached_update.modality == "image":
            tokenizer = self.info.get_tokenizer()
            image_processor = self.info.get_image_processor()
            version = self.info.get_model_version()

            text = _seq2text(tokenizer, cached_update.content.full)
            prev_item_idx = cached_update.item_idx

            if version == (2, 0) or version == (2, 5):
                im_start = image_processor.im_start_token
                im_end = image_processor.im_end_token
            else:
                im_start = image_processor.im_id_start
                im_end = image_processor.im_id_end

            new_update = new_update.with_content(
                PromptUpdateDetails.select_text(
                    text.replace(
                        f"{im_start}{prev_item_idx}{im_end}",
                        f"{im_start}{new_item_idx}{im_end}",
                        1,
                    ),
                    "<unk>",
988
989
                )
            )
990
991
992

        return new_update

993
994
    def _get_mm_fields_config(
        self,
995
        hf_inputs: BatchFeature,
996
997
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, MultiModalFieldConfig]:
998
        return _minicpmv_field_config(hf_inputs)
999

1000
1001

class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
Jee Jee Li's avatar
Jee Jee Li committed
1002
1003
1004
1005
    """
    The abstract class of MiniCPMV can only be inherited, but cannot be
    instantiated.
    """
1006

1007
1008
    supports_encoder_tp_data = True

1009
    @classmethod
1010
    def get_placeholder_str(cls, modality: str, i: int) -> str | None:
1011
1012
1013
1014
1015
1016
1017
        if modality.startswith("image"):
            return "(<image>./</image>)"
        if modality.startswith("video"):
            return "(<video>./</video>)"

        raise ValueError("Only image or video modality is supported")

1018
    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1019
1020
1021
        config = vllm_config.model_config.hf_config
        multimodal_config = vllm_config.model_config.multimodal_config
        quant_config = vllm_config.quant_config
1022
        self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
1023
        super().__init__()
1024
1025
1026
1027
        # All MiniCPM-V models disable `tie_word_embeddings` but
        # `PretrainedConfig.tie_word_embeddings` defaults to True; we cannot
        # check `tie_word_embeddings` until vLLM integrate MiniCPM-V model
        # and config class
1028
1029
1030
        self.config = config
        self.multimodal_config = multimodal_config

1031
        self.version = get_version_by_config(self.config)
1032

1033
1034
1035
1036
1037
1038
        with self._mark_language_model(vllm_config):
            self.llm = self.init_llm(
                vllm_config=vllm_config, prefix=maybe_prefix(prefix, "llm")
            )

        with self._mark_tower_model(vllm_config, {"image", "video"}):
1039
            self.vpm = self.init_vision_module(
1040
1041
1042
                config, quant_config, prefix=maybe_prefix(prefix, "vpm")
            )
            self.vision_dim = (
1043
1044
1045
                self.vpm.embed_dim
                if self.version == (2, 0)
                else self.vpm.embeddings.embed_dim
1046
1047
1048
1049
1050
1051
1052
1053
1054
            )
            self.embed_dim = self.config.hidden_size

            self.resampler = self.init_resampler(
                self.embed_dim,
                self.vision_dim,
                quant_config=quant_config,
                prefix=maybe_prefix(prefix, "resampler"),
            )
1055

1056
        self.make_empty_intermediate_tensors = self.llm.make_empty_intermediate_tensors
1057

1058
    def _parse_and_validate_vision_input(
Jee Jee Li's avatar
Jee Jee Li committed
1059
        self,
1060
        modality: str,
Jee Jee Li's avatar
Jee Jee Li committed
1061
        **kwargs: object,
1062
    ) -> MiniCPMVImageInputs | None:
1063
1064
        pixel_values = kwargs.pop("pixel_values", None)
        image_embeds = kwargs.pop("image_embeds", None)
1065

1066
        if pixel_values is None and image_embeds is None:
1067
1068
            return None

1069
        if image_embeds is not None:
1070
            return MiniCPMVImageEmbeddingInputs(
1071
                type="image_embeds",
1072
                image_embeds=image_embeds,
1073
            )
1074

1075
1076
        tgt_sizes = kwargs.pop("tgt_sizes")

1077
1078
1079
        num_slices_flat = torch.tensor([len(ps) for ps in pixel_values])
        pixel_values_flat = flatten_bn(pixel_values)
        tgt_sizes_flat = flatten_bn(tgt_sizes, concat=True)
1080

1081
        return MiniCPMVImagePixelInputs(
1082
1083
            type="pixel_values",
            pixel_values=pixel_values_flat,
1084
1085
            tgt_sizes=tgt_sizes_flat,
            num_slices=num_slices_flat,
Jee Jee Li's avatar
Jee Jee Li committed
1086
        )
1087

1088
1089
1090
1091
1092
1093
    def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
        modalities = {}

        # Preserve the order of modalities if there are multiple of them
        # from the order of kwargs.
        for input_key in kwargs:
1094
1095
1096
1097
            if (
                input_key in ("pixel_values", "image_embeds")
                and "images" not in modalities
            ):
1098
                modalities["images"] = self._parse_and_validate_vision_input(
1099
1100
1101
1102
1103
1104
                    "images", **kwargs
                )
            if (
                input_key in ("video_pixel_values", "video_embeds")
                and "videos" not in modalities
            ):
1105
                modalities["videos"] = self._parse_and_validate_vision_input(
1106
                    "videos", **{k.removeprefix("video_"): v for k, v in kwargs.items()}
1107
                )
1108
1109
1110
1111
1112
1113

        return modalities

    def _process_vision_input(
        self,
        image_input: MiniCPMVImageInputs,
1114
    ) -> torch.Tensor | list[torch.Tensor] | tuple[torch.Tensor, ...]:
1115
1116
1117
1118
1119
        if image_input["type"] == "image_embeds":
            return image_input["image_embeds"]

        image_features_flat = self.get_vision_hidden_states(image_input)

1120
        num_slices = image_input["num_slices"]
1121
        return [e.flatten(0, 1) for e in image_features_flat.split(num_slices.tolist())]
1122
1123
1124

    def _process_multimodal_inputs(self, modalities: dict):
        # The result multimodal_embeddings is tuple of tensors, with each
1125
        # tensor corresponding to a multimodal data item (image or video).
1126
1127
1128
1129
1130
1131
1132
        multimodal_embeddings: tuple[torch.Tensor, ...] = ()

        # NOTE: It is important to iterate over the keys in this dictionary
        # to preserve the order of the modalities.
        for modality in modalities:
            if modality == "images":
                image_input = modalities["images"]
1133
1134
                image_embeddings = self._process_vision_input(image_input)
                multimodal_embeddings += tuple(image_embeddings)
1135
1136
            if modality == "videos":
                video_input = modalities["videos"]
1137
1138
                video_embeddings = self._process_vision_input(video_input)
                multimodal_embeddings += tuple(video_embeddings)
1139
1140
1141

        return multimodal_embeddings

1142
    def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
1143
1144
        modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
        if not modalities:
1145
            return []
1146
1147
1148

        return self._process_multimodal_inputs(modalities)

1149
1150
    def forward(
        self,
1151
        input_ids: torch.Tensor | None,
1152
        positions: torch.Tensor,
1153
1154
        intermediate_tensors: IntermediateTensors | None = None,
        inputs_embeds: torch.Tensor | None = None,
Jee Jee Li's avatar
Jee Jee Li committed
1155
1156
        **kwargs: Any,
    ) -> torch.Tensor:
1157
        if intermediate_tensors is not None:
1158
1159
1160
            inputs_embeds = None

        hidden_states = self.llm.model(
1161
            input_ids=input_ids,
Jee Jee Li's avatar
Jee Jee Li committed
1162
1163
            positions=positions,
            intermediate_tensors=intermediate_tensors,
1164
            inputs_embeds=inputs_embeds,
Jee Jee Li's avatar
Jee Jee Li committed
1165
        )
1166
        return hidden_states
1167

1168
1169
1170
    def compute_logits(
        self,
        hidden_states: torch.Tensor,
1171
    ) -> torch.Tensor | None:
1172
        return self.llm.compute_logits(hidden_states)
1173

1174
    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
1175
1176
        loader = AutoWeightsLoader(self)
        return loader.load_weights(weights)
Jee Jee Li's avatar
Jee Jee Li committed
1177

1178
1179
1180
1181
    def get_mm_mapping(self) -> MultiModelKeys:
        """
        Get the module prefix in multimodal models
        """
1182
1183
1184
        return MultiModelKeys.from_string_field(
            language_model="llm", connector="resampler", tower_model="vpm"
        )
1185

Jee Jee Li's avatar
Jee Jee Li committed
1186
1187
    def init_llm(
        self,
1188
        vllm_config: VllmConfig,
1189
        prefix: str = "",
Jee Jee Li's avatar
Jee Jee Li committed
1190
1191
1192
    ) -> nn.Module:
        raise NotImplementedError

1193
1194
1195
    def init_vision_module(
        self,
        config: PretrainedConfig,
1196
        quant_config: QuantizationConfig | None,
1197
        prefix: str = "",
1198
    ) -> nn.Module:
Jee Jee Li's avatar
Jee Jee Li committed
1199
1200
        raise NotImplementedError

1201
1202
1203
1204
    def init_resampler(
        self,
        embed_dim: int,
        vision_dim: int,
1205
        quant_config: QuantizationConfig | None = None,
1206
1207
        prefix: str = "",
    ) -> nn.Module:
Jee Jee Li's avatar
Jee Jee Li committed
1208
1209
        raise NotImplementedError

1210
    def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
Jee Jee Li's avatar
Jee Jee Li committed
1211
1212
1213
        raise NotImplementedError


1214
class MiniCPMV2_0(MiniCPMVBaseModel):
1215
1216
    supports_encoder_tp_data = False

1217
1218
    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
        super().__init__(vllm_config=vllm_config, prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
1219
1220
1221
1222
        assert self.version == (2, 0)

    def init_llm(
        self,
1223
        vllm_config: VllmConfig,
1224
        prefix: str = "",
Jee Jee Li's avatar
Jee Jee Li committed
1225
    ) -> nn.Module:
1226
        return MiniCPMForCausalLM(vllm_config=vllm_config, prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
1227

1228
1229
1230
    def init_vision_module(
        self,
        config: PretrainedConfig,
1231
        quant_config: QuantizationConfig | None,
1232
        prefix: str = "",
1233
    ) -> nn.Module:
1234
        # TODO: refactor vision model through timm wrapper from transformers
Jee Jee Li's avatar
Jee Jee Li committed
1235
1236
1237
1238
        try:
            import timm
        except ImportError:
            raise ImportError("Please install timm==0.9.10") from ImportError
1239

Jee Jee Li's avatar
Jee Jee Li committed
1240
1241
1242
1243
1244
1245
1246
1247
1248
        with set_default_torch_dtype(torch.float16):
            model = timm.create_model(
                "vit_so400m_patch14_siglip_384.webli",
                pretrained=False,
                num_classes=0,
                dynamic_img_size=True,
                dynamic_img_pad=True,
            )

1249
1250
        model = model.to(dtype=torch.get_default_dtype())

1251
1252
1253
1254
        if (
            isinstance(model, timm.models.VisionTransformer)
            and model.attn_pool is not None
        ):
Jee Jee Li's avatar
Jee Jee Li committed
1255
1256
1257
1258
1259
1260
1261
            model.attn_pool = torch.nn.Identity()

        if self.config.drop_vision_last_layer:
            model.blocks = model.blocks[:-1]

        return model

1262
1263
1264
1265
    def init_resampler(
        self,
        embed_dim: int,
        vision_dim: int,
1266
        quant_config: QuantizationConfig | None = None,
1267
1268
        prefix: str = "",
    ) -> nn.Module:
Jee Jee Li's avatar
Jee Jee Li committed
1269
        with set_default_torch_dtype(torch.float16):
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
            resampler = Resampler2(
                embed_dim=embed_dim,
                num_heads=embed_dim // 128,
                grid_size=int(math.sqrt(self.config.query_num)),
                kv_dim=vision_dim,
                adaptive=False,
                do_post_projection=True,
                quant_config=quant_config,
                prefix=prefix,
            )

        return resampler.to(
            device=current_platform.device_type, dtype=torch.get_default_dtype()
        )

    def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
1286
1287
1288
1289
1290
1291
1292
        pixel_values = data["pixel_values"]

        P_h, P_w = self.vpm.patch_embed.patch_size
        dtype: torch.dtype = self.vpm.pos_embed.data.dtype
        num_prefix_tokens = getattr(self.vpm, "num_prefix_tokens", 0)

        res = list[torch.Tensor]()
Jee Jee Li's avatar
Jee Jee Li committed
1293
1294
        for pixel_value in pixel_values:
            H, W = pixel_value[0].shape[-2:]
1295
            tgt_size = (math.ceil(H / P_h), math.ceil(W / P_w))
Jee Jee Li's avatar
Jee Jee Li committed
1296
            vision_embedding = self.vpm.forward_features(
1297
1298
                pixel_value.unsqueeze(0).type(dtype)
            )
Jee Jee Li's avatar
Jee Jee Li committed
1299

1300
1301
1302
            if num_prefix_tokens > 0:
                vision_embedding = vision_embedding[:, num_prefix_tokens:]
            res.append(self.resampler(vision_embedding, tgt_size))
Jee Jee Li's avatar
Jee Jee Li committed
1303

1304
        return torch.vstack(res)
Jee Jee Li's avatar
Jee Jee Li committed
1305
1306


1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
    packed_modules_mapping = {
        "qkv_proj": [
            "q_proj",
            "k_proj",
            "v_proj",
        ],
        "gate_up_proj": [
            "gate_proj",
            "up_proj",
        ],
    }
Jee Jee Li's avatar
Jee Jee Li committed
1319

1320
1321
    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
        super().__init__(vllm_config=vllm_config, prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
1322
1323
1324
1325
        assert self.version == (2, 5)

    def init_llm(
        self,
1326
        vllm_config: VllmConfig,
1327
        prefix: str = "",
Jee Jee Li's avatar
Jee Jee Li committed
1328
    ) -> nn.Module:
1329
        return LlamaForCausalLM(vllm_config=vllm_config, prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
1330

1331
1332
1333
    def init_vision_module(
        self,
        config: PretrainedConfig,
1334
        quant_config: QuantizationConfig | None,
1335
        prefix: str = "",
1336
    ) -> nn.Module:
1337
1338
1339
1340
1341
        model = Idefics2VisionTransformer(
            config.vision_config,
            quant_config=quant_config,
            prefix=prefix,
        )
Jee Jee Li's avatar
Jee Jee Li committed
1342
1343
1344
1345
        if self.config.drop_vision_last_layer:
            model.encoder.layers = model.encoder.layers[:-1]
        return model

1346
1347
1348
1349
    def init_resampler(
        self,
        embed_dim: int,
        vision_dim: int,
1350
        quant_config: QuantizationConfig | None = None,
1351
1352
        prefix: str = "",
    ) -> nn.Module:
Jee Jee Li's avatar
Jee Jee Li committed
1353
        with set_default_torch_dtype(torch.float16):
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
            resampler = Resampler2_5(
                num_queries=self.config.query_num,
                embed_dim=embed_dim,
                num_heads=embed_dim // 128,
                kv_dim=vision_dim,
                quant_config=quant_config,
                prefix=prefix,
            )

        return resampler.to(
            device=current_platform.device_type, dtype=torch.get_default_dtype()
        )

    def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
1368
        pixel_values = data["pixel_values"]
Jee Jee Li's avatar
Jee Jee Li committed
1369
1370
        tgt_sizes = data["tgt_sizes"]

1371
1372
1373
1374
1375
        B = len(pixel_values)
        P = pixel_values[0].shape[-2]
        L = max(item.shape[-1] for item in pixel_values)
        device = pixel_values[0].device
        dtype = pixel_values[0].dtype
Jee Jee Li's avatar
Jee Jee Li committed
1376

1377
        all_pixel_values = torch.zeros((B, 3, P, L), dtype=dtype, device=device)
1378
1379
1380
        for i, pixel_values_item in enumerate(pixel_values):
            L_item = pixel_values_item.shape[-1]
            all_pixel_values[i, ..., :L_item] = pixel_values_item
Jee Jee Li's avatar
Jee Jee Li committed
1381

1382
1383
1384
        num_patches = tgt_sizes.prod(-1)
        max_patches = num_patches.max().item()
        assert isinstance(max_patches, int)
Jee Jee Li's avatar
Jee Jee Li committed
1385

1386
        patch_attn_mask = torch.zeros((B, max_patches), dtype=torch.bool, device=device)
1387
1388
        for i, num_patches_item in enumerate(num_patches):
            patch_attn_mask[i, :num_patches_item] = True
Jee Jee Li's avatar
Jee Jee Li committed
1389

1390
1391
1392
1393
1394
1395
1396
        vision_embedding = self.vpm(
            all_pixel_values,
            patch_attention_mask=patch_attn_mask.unsqueeze(1),
            tgt_sizes=None,
        )

        return self.resampler(vision_embedding, tgt_sizes)
Jee Jee Li's avatar
Jee Jee Li committed
1397
1398


1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
    packed_modules_mapping = {
        "qkv_proj": [
            "q_proj",
            "k_proj",
            "v_proj",
        ],
        "gate_up_proj": [
            "gate_proj",
            "up_proj",
        ],
    }
Jee Jee Li's avatar
Jee Jee Li committed
1411

1412
1413
    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
        super().__init__(vllm_config=vllm_config, prefix=prefix)
1414
        assert self.version == (2, 6)
Jee Jee Li's avatar
Jee Jee Li committed
1415
1416
1417

    def init_llm(
        self,
1418
        vllm_config: VllmConfig,
1419
        prefix: str = "",
Jee Jee Li's avatar
Jee Jee Li committed
1420
    ) -> nn.Module:
1421
        return Qwen2ForCausalLM(vllm_config=vllm_config, prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
1422

1423
1424
1425
    def init_vision_module(
        self,
        config: PretrainedConfig,
1426
        quant_config: QuantizationConfig | None = None,
1427
        prefix: str = "",
1428
    ) -> nn.Module:
1429
1430
1431
1432
1433
        model = Idefics2VisionTransformer(
            config.vision_config,
            quant_config=quant_config,
            prefix=prefix,
        )
Jee Jee Li's avatar
Jee Jee Li committed
1434
1435
1436
1437
        if self.config.drop_vision_last_layer:
            model.encoder.layers = model.encoder.layers[:-1]
        return model

1438
1439
1440
1441
    def init_resampler(
        self,
        embed_dim: int,
        vision_dim: int,
1442
        quant_config: QuantizationConfig | None = None,
1443
1444
        prefix: str = "",
    ) -> nn.Module:
Jee Jee Li's avatar
Jee Jee Li committed
1445
        with set_default_torch_dtype(torch.float16):
1446
            # The resampler in 2.6 remains consistent with the one in 2.5.
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
            resampler = Resampler2_5(
                num_queries=self.config.query_num,
                embed_dim=embed_dim,
                num_heads=embed_dim // 128,
                kv_dim=vision_dim,
                quant_config=quant_config,
                prefix=prefix,
            )

        return resampler.to(
            device=current_platform.device_type, dtype=torch.get_default_dtype()
        )

    def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
1461
        pixel_values = data["pixel_values"]
Jee Jee Li's avatar
Jee Jee Li committed
1462
1463
        tgt_sizes = data["tgt_sizes"]

1464
1465
1466
1467
1468
        B = len(pixel_values)
        P = pixel_values[0].shape[-2]
        L = max(item.shape[-1] for item in pixel_values)
        device = pixel_values[0].device
        dtype = pixel_values[0].dtype
Jee Jee Li's avatar
Jee Jee Li committed
1469

1470
        all_pixel_values = torch.zeros((B, 3, P, L), dtype=dtype, device=device)
1471
1472
1473
        for i, pixel_values_item in enumerate(pixel_values):
            L_item = pixel_values_item.shape[-1]
            all_pixel_values[i, ..., :L_item] = pixel_values_item
Jee Jee Li's avatar
Jee Jee Li committed
1474

1475
1476
1477
        num_patches = tgt_sizes.prod(-1)
        max_patches = num_patches.max().item()
        assert isinstance(max_patches, int)
Jee Jee Li's avatar
Jee Jee Li committed
1478

1479
        patch_attn_mask = torch.zeros((B, max_patches), dtype=torch.bool, device=device)
1480
1481
1482
        for i, num_patches_item in enumerate(num_patches):
            patch_attn_mask[i, :num_patches_item] = True

Jee Jee Li's avatar
Jee Jee Li committed
1483
        vision_embedding = self.vpm(
1484
1485
            all_pixel_values,
            patch_attention_mask=patch_attn_mask.unsqueeze(1),
Jee Jee Li's avatar
Jee Jee Li committed
1486
            tgt_sizes=tgt_sizes,
1487
        )
Jee Jee Li's avatar
Jee Jee Li committed
1488
1489
1490

        return self.resampler(vision_embedding, tgt_sizes)

1491
1492
    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
        loader = AutoWeightsLoader(self, skip_prefixes=["apm.", "audio", "tts"])
tc-mb's avatar
tc-mb committed
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
        return loader.load_weights(weights)


class MiniCPMV4_0(MiniCPMVBaseModel, SupportsLoRA):
    packed_modules_mapping = {
        "qkv_proj": [
            "q_proj",
            "k_proj",
            "v_proj",
        ],
        "gate_up_proj": [
            "gate_proj",
            "up_proj",
        ],
    }

    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
        super().__init__(vllm_config=vllm_config, prefix=prefix)
        assert self.version == (4, 0)

    def init_llm(
        self,
        vllm_config: VllmConfig,
        prefix: str = "",
    ) -> nn.Module:
        return LlamaForCausalLM(vllm_config=vllm_config, prefix=prefix)

    def init_vision_module(
        self,
        config: PretrainedConfig,
1523
        quant_config: QuantizationConfig | None = None,
tc-mb's avatar
tc-mb committed
1524
1525
        prefix: str = "",
    ) -> nn.Module:
1526
1527
1528
1529
1530
        model = Idefics2VisionTransformer(
            config.vision_config,
            quant_config=quant_config,
            prefix=prefix,
        )
tc-mb's avatar
tc-mb committed
1531
1532
1533
1534
1535
1536
1537
1538
        if self.config.drop_vision_last_layer:
            model.encoder.layers = model.encoder.layers[:-1]
        return model

    def init_resampler(
        self,
        embed_dim: int,
        vision_dim: int,
1539
        quant_config: QuantizationConfig | None = None,
tc-mb's avatar
tc-mb committed
1540
1541
1542
1543
        prefix: str = "",
    ) -> nn.Module:
        with set_default_torch_dtype(torch.float16):
            # The resampler in 4.0 remains consistent with the one in 2.5/2.6.
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
            resampler = Resampler2_5(
                num_queries=self.config.query_num,
                embed_dim=embed_dim,
                num_heads=embed_dim // 128,
                kv_dim=vision_dim,
                quant_config=quant_config,
                prefix=prefix,
            )

        return resampler.to(
            device=current_platform.device_type, dtype=torch.get_default_dtype()
        )

    def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
tc-mb's avatar
tc-mb committed
1558
1559
1560
1561
1562
1563
1564
1565
1566
        pixel_values = data["pixel_values"]
        tgt_sizes = data["tgt_sizes"]

        B = len(pixel_values)
        P = pixel_values[0].shape[-2]
        L = max(item.shape[-1] for item in pixel_values)
        device = pixel_values[0].device
        dtype = pixel_values[0].dtype

1567
        all_pixel_values = torch.zeros((B, 3, P, L), dtype=dtype, device=device)
tc-mb's avatar
tc-mb committed
1568
1569
1570
1571
1572
1573
1574
1575
        for i, pixel_values_item in enumerate(pixel_values):
            L_item = pixel_values_item.shape[-1]
            all_pixel_values[i, ..., :L_item] = pixel_values_item

        num_patches = tgt_sizes.prod(-1)
        max_patches = num_patches.max().item()
        assert isinstance(max_patches, int)

1576
        patch_attn_mask = torch.zeros((B, max_patches), dtype=torch.bool, device=device)
tc-mb's avatar
tc-mb committed
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
        for i, num_patches_item in enumerate(num_patches):
            patch_attn_mask[i, :num_patches_item] = True

        vision_embedding = self.vpm(
            all_pixel_values,
            patch_attention_mask=patch_attn_mask.unsqueeze(1),
            tgt_sizes=tgt_sizes,
        )

        return self.resampler(vision_embedding, tgt_sizes)

1588
1589
    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
        loader = AutoWeightsLoader(self, skip_prefixes=["apm.", "audio", "tts"])
tc-mb's avatar
tc-mb committed
1590
1591
        return loader.load_weights(weights)

Jee Jee Li's avatar
Jee Jee Li committed
1592

tc-mb's avatar
tc-mb committed
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
class MiniCPMV4_5(MiniCPMVBaseModel, SupportsLoRA):
    packed_modules_mapping = {
        "qkv_proj": [
            "q_proj",
            "k_proj",
            "v_proj",
        ],
        "gate_up_proj": [
            "gate_proj",
            "up_proj",
        ],
    }

    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
        super().__init__(vllm_config=vllm_config, prefix=prefix)
        assert self.version == (4, 5)

    def init_llm(
        self,
        vllm_config: VllmConfig,
        prefix: str = "",
    ) -> nn.Module:
        return Qwen3ForCausalLM(vllm_config=vllm_config, prefix=prefix)

    def init_vision_module(
        self,
        config: PretrainedConfig,
1620
        quant_config: QuantizationConfig | None = None,
tc-mb's avatar
tc-mb committed
1621
1622
        prefix: str = "",
    ) -> nn.Module:
1623
1624
1625
1626
1627
        model = Idefics2VisionTransformer(
            config.vision_config,
            quant_config=quant_config,
            prefix=prefix,
        )
tc-mb's avatar
tc-mb committed
1628
1629
1630
1631
1632
1633
1634
1635
        if self.config.drop_vision_last_layer:
            model.encoder.layers = model.encoder.layers[:-1]
        return model

    def init_resampler(
        self,
        embed_dim: int,
        vision_dim: int,
1636
        quant_config: QuantizationConfig | None = None,
tc-mb's avatar
tc-mb committed
1637
1638
1639
1640
        prefix: str = "",
    ) -> nn.Module:
        with set_default_torch_dtype(torch.float16):
            # The resampler in 4.0 remains consistent with the one in 2.5/2.6.
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
            resampler = Resampler4_5(
                num_queries=self.config.query_num,
                embed_dim=embed_dim,
                num_heads=embed_dim // 128,
                kv_dim=vision_dim,
                quant_config=quant_config,
                prefix=prefix,
            )

        return resampler.to(
            device=current_platform.device_type, dtype=torch.get_default_dtype()
        )

    def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
tc-mb's avatar
tc-mb committed
1655
1656
        pixel_values = data["pixel_values"]
        tgt_sizes = data["tgt_sizes"]
1657
        temporal_ids = data.get("temporal_ids", None)
tc-mb's avatar
tc-mb committed
1658
1659
1660
1661
1662
1663
1664

        B = len(pixel_values)
        P = pixel_values[0].shape[-2]
        L = max(item.shape[-1] for item in pixel_values)
        device = pixel_values[0].device
        dtype = pixel_values[0].dtype

1665
1666
1667
1668
        all_pixel_values = torch.zeros((B, 3, P, L), dtype=dtype, device=device)
        all_temporal_ids = (
            None if temporal_ids is None else flatten_2d_lists(temporal_ids)
        )
tc-mb's avatar
tc-mb committed
1669
1670
1671
1672
1673
1674
1675
1676
        for i, pixel_values_item in enumerate(pixel_values):
            L_item = pixel_values_item.shape[-1]
            all_pixel_values[i, ..., :L_item] = pixel_values_item

        num_patches = tgt_sizes.prod(-1)
        max_patches = num_patches.max().item()
        assert isinstance(max_patches, int)

1677
        patch_attn_mask = torch.zeros((B, max_patches), dtype=torch.bool, device=device)
tc-mb's avatar
tc-mb committed
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
        for i, num_patches_item in enumerate(num_patches):
            patch_attn_mask[i, :num_patches_item] = True

        vision_embedding = self.vpm(
            all_pixel_values,
            patch_attention_mask=patch_attn_mask.unsqueeze(1),
            tgt_sizes=tgt_sizes,
        )

        return self.resampler(vision_embedding, tgt_sizes, all_temporal_ids)

1689
1690
    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
        loader = AutoWeightsLoader(self, skip_prefixes=["apm.", "audio", "tts"])
tc-mb's avatar
tc-mb committed
1691
1692
1693
        return loader.load_weights(weights)


1694
1695
1696
_SUPPORT_VERSION = {
    (2, 0): MiniCPMV2_0,
    (2, 5): MiniCPMV2_5,
1697
    (2, 6): MiniCPMV2_6,
tc-mb's avatar
tc-mb committed
1698
    (4, 0): MiniCPMV4_0,
tc-mb's avatar
tc-mb committed
1699
    (4, 5): MiniCPMV4_5,
1700
1701
1702
}


1703
1704
1705
@MULTIMODAL_REGISTRY.register_processor(
    MiniCPMVMultiModalProcessor,
    info=MiniCPMVProcessingInfo,
1706
1707
    dummy_inputs=MiniCPMVDummyInputsBuilder,
)
1708
class MiniCPMV(MiniCPMVBaseModel, SupportsMultiModal, SupportsLoRA):
Jee Jee Li's avatar
Jee Jee Li committed
1709
1710
1711
1712
1713
    """
    Different versions of MiniCPMV use different visual encoders and LLMs,
    which is not conducive to the current integration logic of LoRA and
    bitsandbytes in vLLM. Therefore, it is necessary to separate them.
    """
1714

1715
    def __new__(cls, *, vllm_config: VllmConfig, prefix: str = ""):
1716
        config = vllm_config.model_config.hf_config
Jee Jee Li's avatar
Jee Jee Li committed
1717
1718
1719
1720
1721
1722
1723
1724
1725
        if not hasattr(config, "version"):
            if config.hidden_size == 2304 and config.query_num == 64:
                version = (2, 0)
            else:
                version = (2, 5)
        else:
            version = str(config.version).split(".")
            version = tuple([int(x) for x in version])
        # Dispatch class based on version
1726
1727
        instance_cls = _SUPPORT_VERSION.get(version)
        if instance_cls is None:
tc-mb's avatar
tc-mb committed
1728
            supported_versions = ", ".join(
1729
1730
1731
1732
1733
1734
                [f"{v[0]}.{v[1]}" for v in sorted(_SUPPORT_VERSION.keys())]
            )
            raise ValueError(
                f"Currently, MiniCPMV only supports versions "
                f"{supported_versions}. Got version: {version}"
            )
1735
1736
1737
1738
1739
1740

        # quant_config references base class members,
        # so update values before init is called
        cls.packed_modules_mapping.update(instance_cls.packed_modules_mapping)
        cls.embedding_modules.update(instance_cls.embedding_modules)
        return instance_cls(vllm_config=vllm_config, prefix=prefix)