minicpmv.py 58.1 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# Adapted from
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
# Copyright 2023 The vLLM team.
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
#
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
# and OPT implementations in this library. It has been modified from its
# original forms to accommodate minor architectural differences compared
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Alphi's avatar
Alphi committed
25
"""Inference-only MiniCPM-V model compatible with HuggingFace weights."""
26

27
import math
28
from collections import defaultdict
29
from collections.abc import Callable, Iterable, Mapping, Sequence
30
from functools import partial
tc-mb's avatar
tc-mb committed
31
from itertools import chain
32
from typing import Annotated, Any, Literal, TypeAlias
33

34
import numpy as np
35
import torch
Alphi's avatar
Alphi committed
36
import torch.types
37
from torch import nn
tc-mb's avatar
tc-mb committed
38
from torch.nn.init import trunc_normal_
39
from transformers import BatchFeature, PretrainedConfig
40
from typing_extensions import TypeVar
41

42
from vllm.config import VllmConfig
43
from vllm.config.multimodal import BaseDummyOptions
44
from vllm.inputs import ModalityData, MultiModalDataDict
45
from vllm.model_executor.layers.quantization import QuantizationConfig
46
47
48
49
50
from vllm.model_executor.layers.resampler import (
    BaseResampler,
    Resampler2,
    get_2d_sincos_pos_embed,
)
51
52
from vllm.model_executor.models.llama import LlamaForCausalLM
from vllm.model_executor.models.minicpm import MiniCPMForCausalLM
53
from vllm.model_executor.models.module_mapping import MultiModelKeys
54
from vllm.model_executor.models.qwen2 import Qwen2ForCausalLM
tc-mb's avatar
tc-mb committed
55
56
from vllm.model_executor.models.qwen3 import Qwen3ForCausalLM
from vllm.multimodal import MULTIMODAL_REGISTRY
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
from vllm.multimodal.inputs import (
    MultiModalFieldConfig,
    MultiModalKwargsItems,
    NestedTensors,
)
from vllm.multimodal.parse import (
    DictEmbeddingItems,
    ImageItem,
    ImageProcessorItems,
    ImageSize,
    ModalityDataItems,
    MultiModalDataItems,
    MultiModalDataParser,
    VideoItem,
    VideoProcessorItems,
)
73
74
from vllm.multimodal.processing import BaseDummyInputsBuilder
from vllm.multimodal.processing.processor import (
75
76
77
78
79
80
81
82
    BaseMultiModalProcessor,
    BaseProcessingInfo,
    PromptReplacement,
    PromptUpdate,
    PromptUpdateDetails,
    ResolvedPromptUpdate,
    _seq2text,
)
83
from vllm.platforms import current_platform
84
from vllm.sequence import IntermediateTensors
85
from vllm.utils.collection_utils import flatten_2d_lists
86
from vllm.utils.tensor_schema import TensorSchema, TensorShape
87
from vllm.utils.torch_utils import set_default_torch_dtype
88

Jee Jee Li's avatar
Jee Jee Li committed
89
from .idefics2_vision_model import Idefics2VisionTransformer
90
91
92
93
94
95
from .interfaces import (
    MultiModalEmbeddings,
    SupportsLoRA,
    SupportsMultiModal,
    SupportsPP,
)
96
from .utils import AutoWeightsLoader, flatten_bn, maybe_prefix
97

98
99
100
# For profile run
_MAX_FRAMES_PER_VIDEO = 16

101

102
class MiniCPMVImagePixelInputs(TensorSchema):
Jee Jee Li's avatar
Jee Jee Li committed
103
    """
104
105
106
107
108
109
    Dimensions:
        - bns: Batch size * number of images * number of slices
        - bn: Batch size * number of images
        - c: Number of channels
        - h: Height
        - w: Width
Jee Jee Li's avatar
Jee Jee Li committed
110
111
    """

112
113
    type: Literal["pixel_values"] = "pixel_values"

114
    # Note that the patch size may vary, so we pass it as a list instead of a
115
116
117
    # batched tensor.
    pixel_values: Annotated[
        list[torch.Tensor],
118
        TensorShape("bns", "c", "h", "w", dynamic_dims={"h", "w"}),
119
120
121
122
123
124
125
126
127
128
129
130
    ]
    tgt_sizes: Annotated[
        torch.Tensor,
        TensorShape("bns", 2),  # This should be in `(height, width)` format.
    ]
    num_slices: Annotated[
        torch.Tensor,
        TensorShape("bn"),
    ]


class MiniCPMVImageEmbeddingInputs(TensorSchema):
Jee Jee Li's avatar
Jee Jee Li committed
131
    """
132
133
134
135
    Dimensions:
        - bn: Batch size * number of images
        - ns: Number of slices
        - hs: Hidden size (must match language model backbone)
Jee Jee Li's avatar
Jee Jee Li committed
136
137
    """

138
    type: Literal["image_embeds"]
139
    image_embeds: Annotated[
140
        torch.Tensor | list[torch.Tensor],
141
        TensorShape("bn", "ns", "hs", dynamic_dims={"ns"}),
142
    ]
143
144


145
MiniCPMVImageInputs: TypeAlias = MiniCPMVImagePixelInputs | MiniCPMVImageEmbeddingInputs
146

Jee Jee Li's avatar
Jee Jee Li committed
147
148
149
150
DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)


class Resampler2_5(BaseResampler):
151
152
153
154
155
    def __init__(
        self,
        num_queries: int,
        embed_dim: int,
        num_heads: int,
156
        kv_dim: int | None = None,
157
158
        norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
        max_size: tuple[int, int] = (70, 70),
159
        quant_config: QuantizationConfig | None = None,
160
161
162
163
164
165
166
167
168
169
170
        prefix: str = "",
    ) -> None:
        super().__init__(
            num_queries,
            embed_dim,
            num_heads,
            kv_dim,
            norm_layer,
            quant_config=quant_config,
            prefix=prefix,
        )
Jee Jee Li's avatar
Jee Jee Li committed
171
172
173

        self.max_size = max_size
        self._set_2d_pos_cache(self.max_size)
174

175
176
177
178
179
180
    def _set_2d_pos_cache(
        self, max_size: tuple[int, int], device: torch.types.Device = "cpu"
    ) -> None:
        pos_embed_arr = get_2d_sincos_pos_embed(
            self.embed_dim, max_size, version=(2, 5)
        )
Jee Jee Li's avatar
Jee Jee Li committed
181
        pos_embed = torch.from_numpy(pos_embed_arr).float().to(device)
182
183
        self.register_buffer("pos_embed", pos_embed, persistent=False)

184
185
186
    def _adjust_pos_cache(
        self, tgt_sizes: torch.Tensor, device: torch.types.Device
    ) -> None:
Jee Jee Li's avatar
Jee Jee Li committed
187
188
189
190
        max_h = tgt_sizes[:, 0].max().item()
        max_w = tgt_sizes[:, 1].max().item()
        assert isinstance(max_h, int) and isinstance(max_w, int)

191
        if max_h > self.max_size[0] or max_w > self.max_size[1]:
Jee Jee Li's avatar
Jee Jee Li committed
192
            self.max_size = (
193
                max(max_h, self.max_size[0]),
Jee Jee Li's avatar
Jee Jee Li committed
194
195
                max(max_w, self.max_size[1]),
            )
196
197
            self._set_2d_pos_cache(self.max_size, device)

198
    def forward(self, x: torch.Tensor, tgt_sizes: torch.Tensor) -> torch.Tensor:
199
200
201
202
203
204
205
206
207
208
        assert x.shape[0] == tgt_sizes.shape[0]
        bs = x.shape[0]

        device = x.device
        dtype = x.dtype

        patch_len = tgt_sizes[:, 0] * tgt_sizes[:, 1]

        self._adjust_pos_cache(tgt_sizes, device=device)

Jee Jee Li's avatar
Jee Jee Li committed
209
210
211
        max_patch_len = patch_len.max().item()
        assert isinstance(max_patch_len, int)

212
213
214
        key_padding_mask = torch.zeros(
            (bs, max_patch_len), dtype=torch.bool, device=device
        )
215
216
217

        pos_embed = []
        for i in range(bs):
Jee Jee Li's avatar
Jee Jee Li committed
218
            tgt_h, tgt_w = tgt_sizes[i].tolist()
219
220
221
222
223
224
225
            pos_embed.append(
                self.pos_embed[:tgt_h, :tgt_w, :].reshape((tgt_h * tgt_w, -1)).to(dtype)
            )  # patches * D
            key_padding_mask[i, patch_len[i] :] = True
        pos_embed = torch.nn.utils.rnn.pad_sequence(
            pos_embed, batch_first=True, padding_value=0.0
        ).permute(1, 0, 2)  # BLD => L * B * D
Jee Jee Li's avatar
Jee Jee Li committed
226
        x, _ = self.kv_proj(x)  # B * L * D
227
228
229
230
231
232
233
234
        x = self.ln_kv(x).permute(1, 0, 2)  # L * B * D

        q = self.ln_q(self.query)  # Q * D

        out = self.attn(
            self._repeat(q, bs),  # Q * B * D
            x + pos_embed,  # L * B * D +  L * B * D
            x,
Jee Jee Li's avatar
Jee Jee Li committed
235
236
            key_padding_mask=key_padding_mask,
        )[0]
237
238
239
240
241
242
243
244
        #  out: Q * B * D
        x = out.permute(1, 0, 2)  # B * Q * D

        x = self.ln_post(x)
        x = x @ self.proj
        return x


tc-mb's avatar
tc-mb committed
245
class Resampler4_5(Resampler2_5):
246
247
248
249
250
    def __init__(
        self,
        num_queries: int,
        embed_dim: int,
        num_heads: int,
251
        kv_dim: int | None = None,
252
253
254
        norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
        max_size: tuple[int, int] = (70, 70),
        max_temporal_size: int = 36000,
255
        quant_config: QuantizationConfig | None = None,
256
257
258
259
260
261
262
263
264
265
266
267
        prefix: str = "",
    ) -> None:
        super().__init__(
            num_queries,
            embed_dim,
            num_heads,
            kv_dim,
            norm_layer,
            max_size,
            quant_config=quant_config,
            prefix=prefix,
        )
tc-mb's avatar
tc-mb committed
268

269
        trunc_normal_(self.query, std=0.02)
tc-mb's avatar
tc-mb committed
270
271
272
273
        self.max_temporal_size = max_temporal_size
        self._set_temporal_pos_cache(self.max_temporal_size)
        self.apply(self._init_weights)

274
275
276
    def get_1d_sincos_pos_embed_from_temporal_size(
        self, embed_dim: int, pos: np.ndarray
    ):
tc-mb's avatar
tc-mb committed
277
278
279
280
281
282
283
        """
        embed_dim: output dimension for each position
        pos: a list of positions to be encoded: size (M,)
        out: (M, D)
        """
        assert embed_dim % 2 == 0
        omega = np.arange(embed_dim // 2, dtype=np.float32)
284
285
        omega /= embed_dim / 2.0
        omega = 1.0 / 10000**omega  # (D/2,)
tc-mb's avatar
tc-mb committed
286
287

        pos = pos.reshape(-1)  # (M,)
288
        out = np.einsum("m,d->md", pos, omega)  # (M, D/2), outer product
tc-mb's avatar
tc-mb committed
289
290
291
292
293
294
295

        emb_sin = np.sin(out)  # (M, D/2)
        emb_cos = np.cos(out)  # (M, D/2)

        emb = np.concatenate([emb_sin, emb_cos], axis=1)  # (M, D)
        return emb

296
297
298
    def _set_temporal_pos_cache(
        self, max_temporal_size: int, device: torch.types.Device = "cpu"
    ) -> None:
tc-mb's avatar
tc-mb committed
299
        temporal_size = np.arange(max_temporal_size, dtype=np.float32)
300
301
302
303
304
305
306
307
308
        pos_embed = (
            torch.from_numpy(
                self.get_1d_sincos_pos_embed_from_temporal_size(
                    self.embed_dim, temporal_size
                )
            )
            .float()
            .to(device)
        )
tc-mb's avatar
tc-mb committed
309
310
        self.register_buffer("temporal_pos_embed", pos_embed, persistent=False)

311
312
313
    def _adjust_temporal_pos_cache(
        self, max_temporal_size: int, device: torch.types.Device = "cpu"
    ):
tc-mb's avatar
tc-mb committed
314
315
316
317
        if max_temporal_size > self.max_temporal_size:
            self.max_temporal_size = max_temporal_size
            self._set_temporal_pos_cache(self.max_temporal_size, device)

318
    def _init_weights(self, m: nn.Linear | nn.LayerNorm):
tc-mb's avatar
tc-mb committed
319
        if isinstance(m, nn.Linear):
320
            trunc_normal_(m.weight, std=0.02)
tc-mb's avatar
tc-mb committed
321
322
323
324
325
326
327
328
329
330
331
            if isinstance(m, nn.Linear) and m.bias is not None:
                nn.init.constant_(m.bias, 0)
        elif isinstance(m, nn.LayerNorm):
            nn.init.constant_(m.bias, 0)
            nn.init.constant_(m.weight, 1.0)

    def forward(
        self,
        x: torch.Tensor,
        tgt_sizes: torch.Tensor,
        # temporal_ids for high refresh rate videos
332
        temporal_ids=None,
tc-mb's avatar
tc-mb committed
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
    ) -> torch.Tensor:
        assert x.shape[0] == tgt_sizes.shape[0]
        bs = x.shape[0]

        device = x.device
        dtype = x.dtype

        patch_len = tgt_sizes[:, 0] * tgt_sizes[:, 1]

        self._adjust_pos_cache(tgt_sizes, device=device)

        temporal_pos_emb = False
        temporal_ids_flatten = None
        if temporal_ids is not None:
            # example: [[-1], [-1], [2, 6, 9]]
            temporal_ids_flatten = list(chain.from_iterable(temporal_ids))
            max_temporal_size = max(temporal_ids_flatten, default=0)
            if max_temporal_size > -1:
                temporal_pos_emb = True
            if max_temporal_size > self.max_temporal_size:
                self._adjust_temporal_pos_cache(max_temporal_size, device)

        max_patch_len = patch_len.max().item()
        assert isinstance(max_patch_len, int)

358
359
360
        key_padding_mask = torch.zeros(
            (bs, max_patch_len), dtype=torch.bool, device=device
        )
tc-mb's avatar
tc-mb committed
361
362
363
364
365
366
367
368
369
370
371
372

        x, _ = self.kv_proj(x)  # B * L * D
        x = self.ln_kv(x).permute(1, 0, 2)  # L * B * D
        q = self.ln_q(self.query)  # Q * D

        pos_embed_2d = []
        pos_embed_temporal = []
        for i in range(bs):
            tgt_h, tgt_w = tgt_sizes[i]
            if temporal_pos_emb:
                if temporal_ids_flatten[i] == -1:
                    pos_embed_temporal.append(
373
374
                        torch.zeros(self.embed_dim, dtype=dtype, device=device)
                    )
tc-mb's avatar
tc-mb committed
375
                else:
376
377
378
                    pos_embed_temporal.append(
                        self.temporal_pos_embed[temporal_ids_flatten[i]].to(dtype)
                    )  # D
tc-mb's avatar
tc-mb committed
379

380
381
382
383
            pos_embed_2d.append(
                self.pos_embed[:tgt_h, :tgt_w, :].reshape((tgt_h * tgt_w, -1)).to(dtype)
            )  # patches * D
            key_padding_mask[i, patch_len[i] :] = True
tc-mb's avatar
tc-mb committed
384
385

        pos_embed_2d = torch.nn.utils.rnn.pad_sequence(
386
387
            pos_embed_2d, batch_first=True, padding_value=0.0
        ).permute(1, 0, 2)  # BLD => L * B * D
tc-mb's avatar
tc-mb committed
388

389
390
        k = x + pos_embed_2d
        v = x
tc-mb's avatar
tc-mb committed
391
392
393
394
395
396
397
398
399
400
401
402
        if pos_embed_temporal:
            k += torch.stack(pos_embed_temporal, dim=0)
            bs = len(temporal_ids)
            merge_k = []
            merge_v = []
            merge_key_padding_mask = []

            start = 0
            for tp in temporal_ids:
                end = start + len(tp)
                # L * (end-start) * D -> (end-start) * L * D
                # -> 1 * L*(end-start) * D
403
404
405
406
407
408
                merge_k.append(
                    k[:, start:end, :].permute(1, 0, 2).reshape(-1, self.embed_dim)
                )
                merge_v.append(
                    v[:, start:end, :].permute(1, 0, 2).reshape(-1, self.embed_dim)
                )
tc-mb's avatar
tc-mb committed
409
                merge_key_padding_mask.append(
410
411
                    key_padding_mask[start:end, :].reshape(-1, 1)
                )
tc-mb's avatar
tc-mb committed
412
413
414

                start = end

415
416
417
418
419
420
            k = torch.nn.utils.rnn.pad_sequence(
                merge_k, batch_first=True, padding_value=0.0
            ).permute(1, 0, 2)  # L*(end-start)
            v = torch.nn.utils.rnn.pad_sequence(
                merge_v, batch_first=True, padding_value=0.0
            ).permute(1, 0, 2)  # L*(end-start)
tc-mb's avatar
tc-mb committed
421
            key_padding_mask = torch.nn.utils.rnn.pad_sequence(
422
423
                merge_key_padding_mask, batch_first=True, padding_value=True
            ).squeeze(-1)
tc-mb's avatar
tc-mb committed
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438

        out = self.attn(
            self._repeat(q, bs),  # Q * B * D
            k,  # L * B * D +  L * B * D
            v,
            key_padding_mask=key_padding_mask,
        )[0]
        #  out: Q * B * D
        x = out.permute(1, 0, 2)  # B * Q * D

        x = self.ln_post(x)
        x = x @ self.proj
        return x


439
def get_version_by_config(config: PretrainedConfig) -> tuple[int, ...]:
440
441
442
443
444
445
446
447
448
449
450
451
    version_float = getattr(config, "version", None)

    # The old configs do not include version number
    # TODO: Remove this after the HF repos are updated
    if version_float is None:
        if config.hidden_size == 2304 and config.query_num == 64:
            return (2, 0)
        return (2, 5)
    version_str = str(version_float)
    return tuple(int(x) for x in version_str.split("."))


452
453
def _minicpmv_field_config(hf_inputs: Mapping[str, torch.Tensor]):
    return dict(
454
        pixel_values=MultiModalFieldConfig.batched("image"),
455
        image_sizes=MultiModalFieldConfig.batched("image"),
456
457
458
        tgt_sizes=MultiModalFieldConfig.batched("image"),
        image_embeds=MultiModalFieldConfig.batched("image"),
        video_pixel_values=MultiModalFieldConfig.batched("video"),
459
        video_image_sizes=MultiModalFieldConfig.batched("video"),
460
461
        video_tgt_sizes=MultiModalFieldConfig.batched("video"),
        video_embeds=MultiModalFieldConfig.batched("video"),
462
463
464
465
466
467
468
    )


class MiniCPMVImageEmbeddingItems(DictEmbeddingItems):
    def __init__(
        self,
        data: Mapping[str, torch.Tensor],
469
470
471
472
        fields_factory: Callable[
            [Mapping[str, torch.Tensor]],
            Mapping[str, MultiModalFieldConfig],
        ],
473
474
475
476
477
    ) -> None:
        super().__init__(
            data,
            modality="image",
            required_fields={"image_embeds", "image_sizes"},
478
            fields_factory=fields_factory,
479
480
481
482
483
484
485
486
487
488
489
        )

    def get_image_size(self, index: int) -> ImageSize:
        image_size = self.get(index)["image_sizes"].tolist()
        return ImageSize(width=image_size[0], height=image_size[1])


class MiniCPMVVideoEmbeddingItems(DictEmbeddingItems):
    def __init__(
        self,
        data: Mapping[str, torch.Tensor],
490
491
492
493
        fields_factory: Callable[
            [Mapping[str, torch.Tensor]],
            Mapping[str, MultiModalFieldConfig],
        ],
494
495
496
497
498
    ) -> None:
        super().__init__(
            data,
            modality="video",
            required_fields={"video_embeds", "video_image_sizes"},
499
            fields_factory=fields_factory,
500
501
502
503
504
505
506
507
508
509
        )

    def get_frame_size(self, index: int) -> ImageSize:
        frame_size = self.get(index)["video_image_sizes"].tolist()
        return ImageSize(width=frame_size[0], height=frame_size[1])

    def get_num_frames(self, index: int) -> int:
        return len(self.get(index)["video_image_sizes"])


510
511
512
class MiniCPMVMultiModalDataParser(MultiModalDataParser):
    def _parse_image_data(
        self,
513
514
        data: dict[str, torch.Tensor] | ModalityData[ImageItem],
    ) -> ModalityDataItems[Any, Any] | None:
515
        if isinstance(data, dict):
516
517
            return MiniCPMVImageEmbeddingItems(
                data,
518
                fields_factory=_minicpmv_field_config,
519
520
            )

521
522
523
524
        return super()._parse_image_data(data)

    def _parse_video_data(
        self,
525
526
        data: dict[str, torch.Tensor] | ModalityData[VideoItem],
    ) -> ModalityDataItems[Any, Any] | None:
527
        if isinstance(data, dict):
528
529
            return MiniCPMVVideoEmbeddingItems(
                data,
530
                fields_factory=_minicpmv_field_config,
531
532
            )

533
534
535
536
537
538
539
540
541
542
        return super()._parse_video_data(data)


class MiniCPMVProcessingInfo(BaseProcessingInfo):
    image_pattern = "(<image>./</image>)"
    video_pattern = "(<video>./</video>)"

    def get_hf_config(self):
        return self.ctx.get_hf_config()

543
544
    def get_hf_processor(self, **kwargs: object):
        hf_processor = self.ctx.get_hf_processor(**kwargs)
545
546
547
548
549
550
551
552
553

        # NumPy arrays are considered as Iterable but not Sequence in
        # https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428
        image_processor = hf_processor.image_processor  # type: ignore
        for attr in ("mean", "std"):
            val = getattr(image_processor, attr)
            if isinstance(val, np.ndarray):
                setattr(image_processor, attr, val.tolist())

554
555
        return hf_processor

556
557
    def get_image_processor(self, **kwargs: object):
        return self.get_hf_processor(**kwargs).image_processor
558

559
560
561
562
563
    def get_data_parser(self):
        return MiniCPMVMultiModalDataParser(
            expected_hidden_size=self._get_expected_hidden_size(),
        )

564
565
566
    def get_model_version(self):
        return get_version_by_config(self.get_hf_config())

567
    def get_supported_mm_limits(self) -> Mapping[str, int | None]:
568
        mm_limits = {"image": None}
tc-mb's avatar
tc-mb committed
569
        if self.get_model_version() in {(2, 6), (4, 0), (4, 5)}:
570
571
572
            mm_limits["video"] = None

        return mm_limits
573

574
575
576
577
578
    def get_slice_image_placeholder(
        self,
        image_size: ImageSize,
        # For MiniCPM V/O 2.6
        image_idx: int = 0,
579
        max_slice_nums: int | None = None,
580
581
582
583
        use_image_id: bool = True,
    ) -> str:
        image_processor = self.get_image_processor()
        version = self.get_model_version()
584

585
586
        if version == (2, 0) or version == (2, 5):
            return image_processor.get_slice_image_placeholder(image_size)
587

588
589
590
591
592
593
        return image_processor.get_slice_image_placeholder(
            image_size,
            image_idx=image_idx,
            max_slice_nums=max_slice_nums,
            use_image_id=use_image_id,
        )
594

595
596
597
598
    def get_sliced_grid(
        self,
        image_size: ImageSize,
        # For MiniCPM V/O 2.6
599
600
        max_slice_nums: int | None = None,
    ) -> tuple[int, int] | None:
601
602
603
604
605
606
607
608
609
610
611
612
613
614
        image_processor = self.get_image_processor()
        version = self.get_model_version()

        if version == (2, 0) or version == (2, 5):
            return image_processor.get_sliced_grid(image_size)

        if max_slice_nums is None:
            max_slice_nums = image_processor.max_slice_nums

        return image_processor.get_sliced_grid(
            image_size,
            max_slice_nums=max_slice_nums,
        )

615
616
617
    def get_num_image_tokens(
        self,
        image_size: ImageSize,
618
        max_slice_nums: int | None = None,
619
    ) -> int:
620
621
622
        image_processor = self.get_image_processor()

        grid = self.get_sliced_grid(
623
624
625
            image_size,
            max_slice_nums=max_slice_nums,
        )
626
627
628
629
        if grid is None:
            ncols = nrows = 0
        else:
            ncols, nrows = grid
630

631
        return (ncols * nrows + 1) * image_processor.image_feature_size
632
633
634

    def get_max_image_tokens(self) -> int:
        image_size = self.get_image_size_with_most_features()
635
636
637
638
        return self.get_num_image_tokens(image_size)

    def get_image_max_slice_num(self) -> int:
        return getattr(self.get_hf_config(), "max_slice_num", 9)
639
640

    def get_image_size_with_most_features(self) -> ImageSize:
641
642
643
644
645
646
647
648
649
650
651
652
        image_size = getattr(self.get_hf_config(), "image_size", 448)
        max_slice_num = self.get_image_max_slice_num()
        return ImageSize(width=image_size, height=image_size * max_slice_num)

    def get_max_video_frame_tokens(self) -> int:
        frame_size = self.get_video_frame_size_with_most_features()

        return self.get_num_image_tokens(
            frame_size,
            max_slice_nums=self.get_video_max_slice_num(),
        )

653
654
655
656
657
658
659
660
    def get_max_video_tokens(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
    ) -> int:
        num_frames = self.get_num_frames_with_most_features(seq_len, mm_counts)
        num_video_tokens_total = self.get_max_video_frame_tokens() * num_frames
        return num_video_tokens_total
661
662
663

    def get_video_max_slice_num(self) -> int:
        return 1
664

665
    def get_video_frame_size_with_most_features(self) -> ImageSize:
666
667
668
        image_size = getattr(self.get_hf_config(), "image_size", 448)
        max_slice_num = self.get_video_max_slice_num()
        return ImageSize(width=image_size, height=image_size * max_slice_num)
669

670
671
672
673
    def get_max_video_frames(self, max_tokens: int) -> int:
        num_frame_tokens = self.get_max_video_frame_tokens()
        num_frames = max_tokens // num_frame_tokens
        return num_frames
674

675
676
677
678
679
680
681
    def get_num_frames_with_most_features(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
    ) -> int:
        max_images = mm_counts.get("image", 0)
        max_videos = mm_counts.get("video", 0)
682

683
        max_image_tokens = self.get_max_image_tokens() * max_images
684
685
686
687
        max_total_frames = self.get_max_video_frames(seq_len - max_image_tokens)
        max_frames_per_video = min(
            max_total_frames // max(max_videos, 1), _MAX_FRAMES_PER_VIDEO
        )
688

689
        return max(max_frames_per_video, 1)
690
691


692
_I = TypeVar("_I", bound=MiniCPMVProcessingInfo, default=MiniCPMVProcessingInfo)
693
694
695


class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
696
697
698
699
700
701
702
703
704
705
    def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
        num_images = mm_counts.get("image", 0)
        num_videos = mm_counts.get("video", 0)

        image_prompt_texts = self.info.image_pattern * num_images
        video_prompt_texts = self.info.video_pattern * num_videos

        return image_prompt_texts + video_prompt_texts

    def get_dummy_mm_data(
706
707
708
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
709
        mm_options: Mapping[str, BaseDummyOptions],
710
    ) -> MultiModalDataDict:
711
712
713
        num_images = mm_counts.get("image", 0)
        num_videos = mm_counts.get("video", 0)

714
715
716
717
718
        image_width, image_height = self.info.get_image_size_with_most_features()
        video_width, video_height = self.info.get_video_frame_size_with_most_features()
        num_video_frames = self.info.get_num_frames_with_most_features(
            seq_len, mm_counts
        )
719

720
721
        image_overrides = mm_options.get("image")
        video_overrides = mm_options.get("video")
722

723
        return {
724
725
726
727
728
729
            "image": self._get_dummy_images(
                width=image_width,
                height=image_height,
                num_images=num_images,
                overrides=image_overrides,
            ),
730
            "video": [
731
732
733
734
735
736
737
738
                self._get_dummy_images(
                    width=video_width,
                    height=video_height,
                    num_images=num_video_frames,
                    overrides=video_overrides,
                )
            ]
            * num_videos,
739
740
741
        }


742
class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
743
    def get_image_prompt_texts(self, image_size: ImageSize, image_idx: int = 0) -> str:
744
745
746
747
        return self.info.get_slice_image_placeholder(
            image_size,
            image_idx=image_idx,
        )
748

749
750
751
752
753
754
755
756
757
758
    def get_video_prompt_texts(self, image_size: ImageSize, num_frames: int) -> str:
        return (
            self.info.get_slice_image_placeholder(
                image_size=image_size,
                image_idx=0,
                max_slice_nums=self.info.get_video_max_slice_num(),
                use_image_id=False,
            )
            * num_frames
        )
759

760
761
762
763
    def process_images(
        self,
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
764
        tok_kwargs: Mapping[str, object],
765
    ) -> Mapping[str, NestedTensors]:
766
767
768
        if (images := mm_data.get("images")) is None:
            return {}

769
770
        mm_items = self.info.parse_mm_data({"image": images}, validate=False)
        parsed_images = mm_items.get_items(
771
            "image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems)
772
        )
773

774
775
776
777
778
779
780
        if isinstance(parsed_images, MiniCPMVImageEmbeddingItems):
            image_inputs = {}
        else:
            image_inputs = self._base_call_hf_processor(
                prompts=[self.info.image_pattern] * len(parsed_images),
                mm_data={"images": [[image] for image in parsed_images]},
                mm_kwargs=mm_kwargs,
781
                tok_kwargs=tok_kwargs,
782
783
784
785
                out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
            )

        return image_inputs
786

787
788
789
790
    def process_videos(
        self,
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
791
        tok_kwargs: Mapping[str, object],
792
    ) -> Mapping[str, NestedTensors]:
793
794
795
        if (videos := mm_data.get("videos")) is None:
            return {}

796
797
        mm_items = self.info.parse_mm_data({"video": videos}, validate=False)
        parsed_videos = mm_items.get_items(
798
            "video", (MiniCPMVVideoEmbeddingItems, VideoProcessorItems)
799
        )
800
801
802
803
804
805

        if isinstance(parsed_videos, MiniCPMVVideoEmbeddingItems):
            video_inputs = {}
        else:
            video_inputs = self._base_call_hf_processor(
                prompts=[
806
                    self.info.image_pattern * len(video) for video in parsed_videos
807
808
809
810
                ],
                mm_data={"images": list(parsed_videos)},
                mm_kwargs={
                    **mm_kwargs,
811
                    "max_slice_nums": self.info.get_video_max_slice_num(),
812
                },
813
                tok_kwargs=tok_kwargs,
814
815
816
                out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
            )

817
818
        video_inputs = {f"video_{k}": v for k, v in video_inputs.items()}

819
        return video_inputs
820

821
822
823
824
    def process_mm_inputs(
        self,
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
825
        tok_kwargs: Mapping[str, object],
826
    ) -> Mapping[str, NestedTensors]:
827
        return {
828
829
            **self.process_images(mm_data, mm_kwargs, tok_kwargs),
            **self.process_videos(mm_data, mm_kwargs, tok_kwargs),
830
        }
831

832
    def _base_call_hf_processor(
833
        self,
834
835
        prompts: list[str],
        mm_data: Mapping[str, Sequence[object]],
836
        mm_kwargs: Mapping[str, object],
837
        tok_kwargs: Mapping[str, object],
838
839
        *,
        out_keys: set[str],
840
    ) -> dict[str, NestedTensors]:
841
        # This processor supports zipping prompt and mm_data together
tc-mb's avatar
tc-mb committed
842
        if self.info.get_model_version() in {(2, 6), (4, 0), (4, 5)}:
843
844
845
846
            inputs = super()._call_hf_processor(
                prompt=prompts,  # type: ignore
                mm_data=mm_data,
                mm_kwargs=mm_kwargs,
847
                tok_kwargs=tok_kwargs,
848
849
850
851
852
853
854
            )
        else:
            inputs = defaultdict[str, list[torch.Tensor]](list)

            for i, prompt in enumerate(prompts):
                inputs_one = super()._call_hf_processor(
                    prompt=prompt,
855
                    mm_data={k: v[i] for k, v in mm_data.items()},
856
                    mm_kwargs=mm_kwargs,
857
                    tok_kwargs=tok_kwargs,
858
859
860
861
862
863
864
                )

                for k, v in inputs_one.items():
                    assert len(v) == 1, (k, len(v))
                    inputs[k].append(v[0])

        return {k: inputs[k] for k in out_keys}
865
866
867
868
869
870

    def _call_hf_processor(
        self,
        prompt: str,
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
871
        tok_kwargs: Mapping[str, object],
872
873
    ) -> BatchFeature:
        tokenizer = self.info.get_tokenizer()
874

875
876
        input_ids = torch.tensor([tokenizer.encode(prompt, **tok_kwargs)])
        mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs, tok_kwargs)
877

878
879
880
881
882
883
        return BatchFeature(
            {
                "input_ids": input_ids,
                **mm_inputs,
            }
        )
884

885
    def _hf_processor_applies_updates(
886
887
888
889
        self,
        prompt_text: str,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
890
        tokenization_kwargs: Mapping[str, object],
891
892
893
    ) -> bool:
        return False

894
    def _get_prompt_updates(
895
896
897
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
898
        out_mm_kwargs: MultiModalKwargsItems,
899
    ) -> Sequence[PromptUpdate]:
900
901
902
903
        placeholders = [
            ("image", self.info.image_pattern),
            ("video", self.info.video_pattern),
        ]
tc-mb's avatar
tc-mb committed
904
905
906
907
908
909

        # hard code for inconsistency of encode-decode image_pattern
        additional_placeholders = []
        tokenizer = self.info.get_tokenizer()
        for modality, pattern in placeholders:
            sub_pattern = tokenizer.decode(
910
911
                tokenizer.encode(pattern, add_special_tokens=False)
            )
tc-mb's avatar
tc-mb committed
912
913
914
            if sub_pattern != pattern:
                additional_placeholders.append((modality, sub_pattern))
        placeholders += additional_placeholders
915

916
917
        def get_image_replacement(item_idx: int):
            images = mm_items.get_items(
918
919
                "image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems)
            )
920
921
922

            image_size = images.get_image_size(item_idx)

923
924
925
926
            return PromptUpdateDetails.select_text(
                self.get_image_prompt_texts(image_size, item_idx),
                "<unk>",
            )
927
928
929

        def get_video_replacement(item_idx: int):
            videos = mm_items.get_items(
930
931
                "video", (MiniCPMVVideoEmbeddingItems, VideoProcessorItems)
            )
932
933
934
935

            frame_size = videos.get_frame_size(item_idx)
            num_frames = videos.get_num_frames(item_idx)

936
937
938
939
            return PromptUpdateDetails.select_text(
                self.get_video_prompt_texts(frame_size, num_frames),
                "<unk>",
            )
940
941
942
943
944

        get_replacement = {
            "image": get_image_replacement,
            "video": get_video_replacement,
        }
945
946

        return [
947
948
949
            PromptReplacement(
                modality=modality, target=pattern, replacement=get_replacement[modality]
            )
tc-mb's avatar
tc-mb committed
950
            for modality, pattern in placeholders
951
        ]
952

953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
    def _recompute_cached_prompt_update(
        self,
        cached_update: ResolvedPromptUpdate,
        new_item_idx: int,
    ) -> ResolvedPromptUpdate:
        new_update = super()._recompute_cached_prompt_update(
            cached_update,
            new_item_idx,
        )

        if cached_update.modality == "image":
            tokenizer = self.info.get_tokenizer()
            image_processor = self.info.get_image_processor()
            version = self.info.get_model_version()

            text = _seq2text(tokenizer, cached_update.content.full)
            prev_item_idx = cached_update.item_idx

            if version == (2, 0) or version == (2, 5):
                im_start = image_processor.im_start_token
                im_end = image_processor.im_end_token
            else:
                im_start = image_processor.im_id_start
                im_end = image_processor.im_id_end

            new_update = new_update.with_content(
                PromptUpdateDetails.select_text(
                    text.replace(
                        f"{im_start}{prev_item_idx}{im_end}",
                        f"{im_start}{new_item_idx}{im_end}",
                        1,
                    ),
                    "<unk>",
986
987
                )
            )
988
989
990

        return new_update

991
992
    def _get_mm_fields_config(
        self,
993
        hf_inputs: BatchFeature,
994
995
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, MultiModalFieldConfig]:
996
        return _minicpmv_field_config(hf_inputs)
997

998
999

class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
Jee Jee Li's avatar
Jee Jee Li committed
1000
1001
1002
1003
    """
    The abstract class of MiniCPMV can only be inherited, but cannot be
    instantiated.
    """
1004

1005
1006
    supports_encoder_tp_data = True

1007
    @classmethod
1008
    def get_placeholder_str(cls, modality: str, i: int) -> str | None:
1009
1010
1011
1012
1013
1014
1015
        if modality.startswith("image"):
            return "(<image>./</image>)"
        if modality.startswith("video"):
            return "(<video>./</video>)"

        raise ValueError("Only image or video modality is supported")

1016
    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1017
1018
1019
        config = vllm_config.model_config.hf_config
        multimodal_config = vllm_config.model_config.multimodal_config
        quant_config = vllm_config.quant_config
1020
        self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
1021
        super().__init__()
1022
1023
1024
1025
        # All MiniCPM-V models disable `tie_word_embeddings` but
        # `PretrainedConfig.tie_word_embeddings` defaults to True; we cannot
        # check `tie_word_embeddings` until vLLM integrate MiniCPM-V model
        # and config class
1026
1027
1028
        self.config = config
        self.multimodal_config = multimodal_config

1029
        self.version = get_version_by_config(self.config)
1030

1031
1032
1033
1034
1035
1036
        with self._mark_language_model(vllm_config):
            self.llm = self.init_llm(
                vllm_config=vllm_config, prefix=maybe_prefix(prefix, "llm")
            )

        with self._mark_tower_model(vllm_config, {"image", "video"}):
1037
            self.vpm = self.init_vision_module(
1038
1039
1040
                config, quant_config, prefix=maybe_prefix(prefix, "vpm")
            )
            self.vision_dim = (
1041
1042
1043
                self.vpm.embed_dim
                if self.version == (2, 0)
                else self.vpm.embeddings.embed_dim
1044
1045
1046
1047
1048
1049
1050
1051
1052
            )
            self.embed_dim = self.config.hidden_size

            self.resampler = self.init_resampler(
                self.embed_dim,
                self.vision_dim,
                quant_config=quant_config,
                prefix=maybe_prefix(prefix, "resampler"),
            )
1053
            self._resampler_moved = False
1054

1055
        self.make_empty_intermediate_tensors = self.llm.make_empty_intermediate_tensors
1056

1057
1058
1059
1060
1061
1062
1063
    def _ensure_resampler_device(self) -> None:
        if self._resampler_moved:
            return
        # Only move device, DO NOT touch dtype (fp8 quant needs its own dtype)
        self.resampler.to(current_platform.device_type)
        self._resampler_moved = True

1064
    def _parse_and_validate_vision_input(
Jee Jee Li's avatar
Jee Jee Li committed
1065
        self,
1066
        modality: str,
Jee Jee Li's avatar
Jee Jee Li committed
1067
        **kwargs: object,
1068
    ) -> MiniCPMVImageInputs | None:
1069
1070
        pixel_values = kwargs.pop("pixel_values", None)
        image_embeds = kwargs.pop("image_embeds", None)
1071

1072
        if pixel_values is None and image_embeds is None:
1073
1074
            return None

1075
        if image_embeds is not None:
1076
            return MiniCPMVImageEmbeddingInputs(
1077
                type="image_embeds",
1078
                image_embeds=image_embeds,
1079
            )
1080

1081
1082
        tgt_sizes = kwargs.pop("tgt_sizes")

1083
1084
1085
        num_slices_flat = torch.tensor([len(ps) for ps in pixel_values])
        pixel_values_flat = flatten_bn(pixel_values)
        tgt_sizes_flat = flatten_bn(tgt_sizes, concat=True)
1086

1087
        return MiniCPMVImagePixelInputs(
1088
1089
            type="pixel_values",
            pixel_values=pixel_values_flat,
1090
1091
            tgt_sizes=tgt_sizes_flat,
            num_slices=num_slices_flat,
Jee Jee Li's avatar
Jee Jee Li committed
1092
        )
1093

1094
1095
1096
1097
1098
1099
    def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
        modalities = {}

        # Preserve the order of modalities if there are multiple of them
        # from the order of kwargs.
        for input_key in kwargs:
1100
1101
1102
1103
            if (
                input_key in ("pixel_values", "image_embeds")
                and "images" not in modalities
            ):
1104
                modalities["images"] = self._parse_and_validate_vision_input(
1105
1106
1107
1108
1109
1110
                    "images", **kwargs
                )
            if (
                input_key in ("video_pixel_values", "video_embeds")
                and "videos" not in modalities
            ):
1111
                modalities["videos"] = self._parse_and_validate_vision_input(
1112
                    "videos", **{k.removeprefix("video_"): v for k, v in kwargs.items()}
1113
                )
1114
1115
1116
1117
1118
1119

        return modalities

    def _process_vision_input(
        self,
        image_input: MiniCPMVImageInputs,
1120
    ) -> torch.Tensor | list[torch.Tensor] | tuple[torch.Tensor, ...]:
1121
1122
1123
1124
1125
        if image_input["type"] == "image_embeds":
            return image_input["image_embeds"]

        image_features_flat = self.get_vision_hidden_states(image_input)

1126
        num_slices = image_input["num_slices"]
1127
        return [e.flatten(0, 1) for e in image_features_flat.split(num_slices.tolist())]
1128
1129
1130

    def _process_multimodal_inputs(self, modalities: dict):
        # The result multimodal_embeddings is tuple of tensors, with each
1131
        # tensor corresponding to a multimodal data item (image or video).
1132
1133
1134
1135
1136
1137
1138
        multimodal_embeddings: tuple[torch.Tensor, ...] = ()

        # NOTE: It is important to iterate over the keys in this dictionary
        # to preserve the order of the modalities.
        for modality in modalities:
            if modality == "images":
                image_input = modalities["images"]
1139
1140
                image_embeddings = self._process_vision_input(image_input)
                multimodal_embeddings += tuple(image_embeddings)
1141
1142
            if modality == "videos":
                video_input = modalities["videos"]
1143
1144
                video_embeddings = self._process_vision_input(video_input)
                multimodal_embeddings += tuple(video_embeddings)
1145
1146
1147

        return multimodal_embeddings

1148
    def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
1149
1150
        modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
        if not modalities:
1151
            return []
1152
1153
1154

        return self._process_multimodal_inputs(modalities)

1155
1156
    def forward(
        self,
1157
        input_ids: torch.Tensor | None,
1158
        positions: torch.Tensor,
1159
1160
        intermediate_tensors: IntermediateTensors | None = None,
        inputs_embeds: torch.Tensor | None = None,
Jee Jee Li's avatar
Jee Jee Li committed
1161
1162
        **kwargs: Any,
    ) -> torch.Tensor:
1163
        if intermediate_tensors is not None:
1164
1165
1166
            inputs_embeds = None

        hidden_states = self.llm.model(
1167
            input_ids=input_ids,
Jee Jee Li's avatar
Jee Jee Li committed
1168
1169
            positions=positions,
            intermediate_tensors=intermediate_tensors,
1170
            inputs_embeds=inputs_embeds,
Jee Jee Li's avatar
Jee Jee Li committed
1171
        )
1172
        return hidden_states
1173

1174
1175
1176
    def compute_logits(
        self,
        hidden_states: torch.Tensor,
1177
    ) -> torch.Tensor | None:
1178
        return self.llm.compute_logits(hidden_states)
1179

1180
    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
1181
        loader = AutoWeightsLoader(self)
1182
1183
1184
        loaded = loader.load_weights(weights)
        self._ensure_resampler_device()
        return loaded
Jee Jee Li's avatar
Jee Jee Li committed
1185

1186
1187
1188
1189
    def get_mm_mapping(self) -> MultiModelKeys:
        """
        Get the module prefix in multimodal models
        """
1190
1191
1192
        return MultiModelKeys.from_string_field(
            language_model="llm", connector="resampler", tower_model="vpm"
        )
1193

Jee Jee Li's avatar
Jee Jee Li committed
1194
1195
    def init_llm(
        self,
1196
        vllm_config: VllmConfig,
1197
        prefix: str = "",
Jee Jee Li's avatar
Jee Jee Li committed
1198
1199
1200
    ) -> nn.Module:
        raise NotImplementedError

1201
1202
1203
    def init_vision_module(
        self,
        config: PretrainedConfig,
1204
        quant_config: QuantizationConfig | None,
1205
        prefix: str = "",
1206
    ) -> nn.Module:
Jee Jee Li's avatar
Jee Jee Li committed
1207
1208
        raise NotImplementedError

1209
1210
1211
1212
    def init_resampler(
        self,
        embed_dim: int,
        vision_dim: int,
1213
        quant_config: QuantizationConfig | None = None,
1214
1215
        prefix: str = "",
    ) -> nn.Module:
Jee Jee Li's avatar
Jee Jee Li committed
1216
1217
        raise NotImplementedError

1218
    def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
Jee Jee Li's avatar
Jee Jee Li committed
1219
1220
1221
        raise NotImplementedError


1222
class MiniCPMV2_0(MiniCPMVBaseModel):
1223
1224
    supports_encoder_tp_data = False

1225
1226
    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
        super().__init__(vllm_config=vllm_config, prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
1227
1228
1229
1230
        assert self.version == (2, 0)

    def init_llm(
        self,
1231
        vllm_config: VllmConfig,
1232
        prefix: str = "",
Jee Jee Li's avatar
Jee Jee Li committed
1233
    ) -> nn.Module:
1234
        return MiniCPMForCausalLM(vllm_config=vllm_config, prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
1235

1236
1237
1238
    def init_vision_module(
        self,
        config: PretrainedConfig,
1239
        quant_config: QuantizationConfig | None,
1240
        prefix: str = "",
1241
    ) -> nn.Module:
1242
        # TODO: refactor vision model through timm wrapper from transformers
Jee Jee Li's avatar
Jee Jee Li committed
1243
1244
1245
1246
        try:
            import timm
        except ImportError:
            raise ImportError("Please install timm==0.9.10") from ImportError
1247

Jee Jee Li's avatar
Jee Jee Li committed
1248
1249
1250
1251
1252
1253
1254
1255
1256
        with set_default_torch_dtype(torch.float16):
            model = timm.create_model(
                "vit_so400m_patch14_siglip_384.webli",
                pretrained=False,
                num_classes=0,
                dynamic_img_size=True,
                dynamic_img_pad=True,
            )

1257
1258
        model = model.to(dtype=torch.get_default_dtype())

1259
1260
1261
1262
        if (
            isinstance(model, timm.models.VisionTransformer)
            and model.attn_pool is not None
        ):
Jee Jee Li's avatar
Jee Jee Li committed
1263
1264
1265
1266
1267
1268
1269
            model.attn_pool = torch.nn.Identity()

        if self.config.drop_vision_last_layer:
            model.blocks = model.blocks[:-1]

        return model

1270
1271
1272
1273
    def init_resampler(
        self,
        embed_dim: int,
        vision_dim: int,
1274
        quant_config: QuantizationConfig | None = None,
1275
1276
        prefix: str = "",
    ) -> nn.Module:
Jee Jee Li's avatar
Jee Jee Li committed
1277
        with set_default_torch_dtype(torch.float16):
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
            resampler = Resampler2(
                embed_dim=embed_dim,
                num_heads=embed_dim // 128,
                grid_size=int(math.sqrt(self.config.query_num)),
                kv_dim=vision_dim,
                adaptive=False,
                do_post_projection=True,
                quant_config=quant_config,
                prefix=prefix,
            )

1289
        return resampler.to(dtype=torch.get_default_dtype())
1290
1291

    def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
1292
1293
1294
1295
1296
1297
1298
        pixel_values = data["pixel_values"]

        P_h, P_w = self.vpm.patch_embed.patch_size
        dtype: torch.dtype = self.vpm.pos_embed.data.dtype
        num_prefix_tokens = getattr(self.vpm, "num_prefix_tokens", 0)

        res = list[torch.Tensor]()
Jee Jee Li's avatar
Jee Jee Li committed
1299
1300
        for pixel_value in pixel_values:
            H, W = pixel_value[0].shape[-2:]
1301
            tgt_size = (math.ceil(H / P_h), math.ceil(W / P_w))
Jee Jee Li's avatar
Jee Jee Li committed
1302
            vision_embedding = self.vpm.forward_features(
1303
1304
                pixel_value.unsqueeze(0).type(dtype)
            )
Jee Jee Li's avatar
Jee Jee Li committed
1305

1306
1307
1308
            if num_prefix_tokens > 0:
                vision_embedding = vision_embedding[:, num_prefix_tokens:]
            res.append(self.resampler(vision_embedding, tgt_size))
Jee Jee Li's avatar
Jee Jee Li committed
1309

1310
        return torch.vstack(res)
Jee Jee Li's avatar
Jee Jee Li committed
1311
1312


1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
    packed_modules_mapping = {
        "qkv_proj": [
            "q_proj",
            "k_proj",
            "v_proj",
        ],
        "gate_up_proj": [
            "gate_proj",
            "up_proj",
        ],
    }
Jee Jee Li's avatar
Jee Jee Li committed
1325

1326
1327
    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
        super().__init__(vllm_config=vllm_config, prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
1328
1329
1330
1331
        assert self.version == (2, 5)

    def init_llm(
        self,
1332
        vllm_config: VllmConfig,
1333
        prefix: str = "",
Jee Jee Li's avatar
Jee Jee Li committed
1334
    ) -> nn.Module:
1335
        return LlamaForCausalLM(vllm_config=vllm_config, prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
1336

1337
1338
1339
    def init_vision_module(
        self,
        config: PretrainedConfig,
1340
        quant_config: QuantizationConfig | None,
1341
        prefix: str = "",
1342
    ) -> nn.Module:
1343
1344
1345
        model = Idefics2VisionTransformer(
            config.vision_config,
            quant_config=quant_config,
1346
            apply_encoder_attention_mask=True,
1347
1348
            prefix=prefix,
        )
Jee Jee Li's avatar
Jee Jee Li committed
1349
1350
1351
1352
        if self.config.drop_vision_last_layer:
            model.encoder.layers = model.encoder.layers[:-1]
        return model

1353
1354
1355
1356
    def init_resampler(
        self,
        embed_dim: int,
        vision_dim: int,
1357
        quant_config: QuantizationConfig | None = None,
1358
1359
        prefix: str = "",
    ) -> nn.Module:
Jee Jee Li's avatar
Jee Jee Li committed
1360
        with set_default_torch_dtype(torch.float16):
1361
1362
1363
1364
1365
1366
1367
1368
1369
            resampler = Resampler2_5(
                num_queries=self.config.query_num,
                embed_dim=embed_dim,
                num_heads=embed_dim // 128,
                kv_dim=vision_dim,
                quant_config=quant_config,
                prefix=prefix,
            )

1370
        return resampler.to(dtype=torch.get_default_dtype())
1371
1372

    def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
1373
        pixel_values = data["pixel_values"]
Jee Jee Li's avatar
Jee Jee Li committed
1374
1375
        tgt_sizes = data["tgt_sizes"]

1376
1377
1378
1379
1380
        B = len(pixel_values)
        P = pixel_values[0].shape[-2]
        L = max(item.shape[-1] for item in pixel_values)
        device = pixel_values[0].device
        dtype = pixel_values[0].dtype
Jee Jee Li's avatar
Jee Jee Li committed
1381

1382
        all_pixel_values = torch.zeros((B, 3, P, L), dtype=dtype, device=device)
1383
1384
1385
        for i, pixel_values_item in enumerate(pixel_values):
            L_item = pixel_values_item.shape[-1]
            all_pixel_values[i, ..., :L_item] = pixel_values_item
Jee Jee Li's avatar
Jee Jee Li committed
1386

1387
1388
1389
        num_patches = tgt_sizes.prod(-1)
        max_patches = num_patches.max().item()
        assert isinstance(max_patches, int)
Jee Jee Li's avatar
Jee Jee Li committed
1390

1391
        patch_attn_mask = torch.zeros((B, max_patches), dtype=torch.bool, device=device)
1392
1393
        for i, num_patches_item in enumerate(num_patches):
            patch_attn_mask[i, :num_patches_item] = True
Jee Jee Li's avatar
Jee Jee Li committed
1394

1395
1396
1397
1398
1399
1400
1401
        vision_embedding = self.vpm(
            all_pixel_values,
            patch_attention_mask=patch_attn_mask.unsqueeze(1),
            tgt_sizes=None,
        )

        return self.resampler(vision_embedding, tgt_sizes)
Jee Jee Li's avatar
Jee Jee Li committed
1402
1403


1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
    packed_modules_mapping = {
        "qkv_proj": [
            "q_proj",
            "k_proj",
            "v_proj",
        ],
        "gate_up_proj": [
            "gate_proj",
            "up_proj",
        ],
    }
Jee Jee Li's avatar
Jee Jee Li committed
1416

1417
1418
    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
        super().__init__(vllm_config=vllm_config, prefix=prefix)
1419
        assert self.version == (2, 6)
Jee Jee Li's avatar
Jee Jee Li committed
1420
1421
1422

    def init_llm(
        self,
1423
        vllm_config: VllmConfig,
1424
        prefix: str = "",
Jee Jee Li's avatar
Jee Jee Li committed
1425
    ) -> nn.Module:
1426
        return Qwen2ForCausalLM(vllm_config=vllm_config, prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
1427

1428
1429
1430
    def init_vision_module(
        self,
        config: PretrainedConfig,
1431
        quant_config: QuantizationConfig | None = None,
1432
        prefix: str = "",
1433
    ) -> nn.Module:
1434
1435
1436
        model = Idefics2VisionTransformer(
            config.vision_config,
            quant_config=quant_config,
1437
            apply_encoder_attention_mask=True,
1438
1439
            prefix=prefix,
        )
Jee Jee Li's avatar
Jee Jee Li committed
1440
1441
1442
1443
        if self.config.drop_vision_last_layer:
            model.encoder.layers = model.encoder.layers[:-1]
        return model

1444
1445
1446
1447
    def init_resampler(
        self,
        embed_dim: int,
        vision_dim: int,
1448
        quant_config: QuantizationConfig | None = None,
1449
1450
        prefix: str = "",
    ) -> nn.Module:
Jee Jee Li's avatar
Jee Jee Li committed
1451
        with set_default_torch_dtype(torch.float16):
1452
            # The resampler in 2.6 remains consistent with the one in 2.5.
1453
1454
1455
1456
1457
1458
1459
1460
            resampler = Resampler2_5(
                num_queries=self.config.query_num,
                embed_dim=embed_dim,
                num_heads=embed_dim // 128,
                kv_dim=vision_dim,
                quant_config=quant_config,
                prefix=prefix,
            )
1461
1462

        return resampler.to(dtype=torch.get_default_dtype())
1463
1464

    def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
1465
        pixel_values = data["pixel_values"]
Jee Jee Li's avatar
Jee Jee Li committed
1466
1467
        tgt_sizes = data["tgt_sizes"]

1468
1469
1470
1471
1472
        B = len(pixel_values)
        P = pixel_values[0].shape[-2]
        L = max(item.shape[-1] for item in pixel_values)
        device = pixel_values[0].device
        dtype = pixel_values[0].dtype
Jee Jee Li's avatar
Jee Jee Li committed
1473

1474
        all_pixel_values = torch.zeros((B, 3, P, L), dtype=dtype, device=device)
1475
1476
1477
        for i, pixel_values_item in enumerate(pixel_values):
            L_item = pixel_values_item.shape[-1]
            all_pixel_values[i, ..., :L_item] = pixel_values_item
Jee Jee Li's avatar
Jee Jee Li committed
1478

1479
1480
1481
        num_patches = tgt_sizes.prod(-1)
        max_patches = num_patches.max().item()
        assert isinstance(max_patches, int)
Jee Jee Li's avatar
Jee Jee Li committed
1482

1483
        patch_attn_mask = torch.zeros((B, max_patches), dtype=torch.bool, device=device)
1484
1485
1486
        for i, num_patches_item in enumerate(num_patches):
            patch_attn_mask[i, :num_patches_item] = True

Jee Jee Li's avatar
Jee Jee Li committed
1487
        vision_embedding = self.vpm(
1488
1489
            all_pixel_values,
            patch_attention_mask=patch_attn_mask.unsqueeze(1),
Jee Jee Li's avatar
Jee Jee Li committed
1490
            tgt_sizes=tgt_sizes,
1491
        )
Jee Jee Li's avatar
Jee Jee Li committed
1492
1493
1494

        return self.resampler(vision_embedding, tgt_sizes)

1495
1496
    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
        loader = AutoWeightsLoader(self, skip_prefixes=["apm.", "audio", "tts"])
1497
1498
1499
        loaded = loader.load_weights(weights)
        self._ensure_resampler_device()
        return loaded
tc-mb's avatar
tc-mb committed
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528


class MiniCPMV4_0(MiniCPMVBaseModel, SupportsLoRA):
    packed_modules_mapping = {
        "qkv_proj": [
            "q_proj",
            "k_proj",
            "v_proj",
        ],
        "gate_up_proj": [
            "gate_proj",
            "up_proj",
        ],
    }

    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
        super().__init__(vllm_config=vllm_config, prefix=prefix)
        assert self.version == (4, 0)

    def init_llm(
        self,
        vllm_config: VllmConfig,
        prefix: str = "",
    ) -> nn.Module:
        return LlamaForCausalLM(vllm_config=vllm_config, prefix=prefix)

    def init_vision_module(
        self,
        config: PretrainedConfig,
1529
        quant_config: QuantizationConfig | None = None,
tc-mb's avatar
tc-mb committed
1530
1531
        prefix: str = "",
    ) -> nn.Module:
1532
1533
1534
        model = Idefics2VisionTransformer(
            config.vision_config,
            quant_config=quant_config,
1535
            apply_encoder_attention_mask=True,
1536
1537
            prefix=prefix,
        )
tc-mb's avatar
tc-mb committed
1538
1539
1540
1541
1542
1543
1544
1545
        if self.config.drop_vision_last_layer:
            model.encoder.layers = model.encoder.layers[:-1]
        return model

    def init_resampler(
        self,
        embed_dim: int,
        vision_dim: int,
1546
        quant_config: QuantizationConfig | None = None,
tc-mb's avatar
tc-mb committed
1547
1548
1549
1550
        prefix: str = "",
    ) -> nn.Module:
        with set_default_torch_dtype(torch.float16):
            # The resampler in 4.0 remains consistent with the one in 2.5/2.6.
1551
1552
1553
1554
1555
1556
1557
1558
            resampler = Resampler2_5(
                num_queries=self.config.query_num,
                embed_dim=embed_dim,
                num_heads=embed_dim // 128,
                kv_dim=vision_dim,
                quant_config=quant_config,
                prefix=prefix,
            )
1559
        return resampler.to(dtype=torch.get_default_dtype())
1560
1561

    def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
tc-mb's avatar
tc-mb committed
1562
1563
1564
1565
1566
1567
1568
1569
1570
        pixel_values = data["pixel_values"]
        tgt_sizes = data["tgt_sizes"]

        B = len(pixel_values)
        P = pixel_values[0].shape[-2]
        L = max(item.shape[-1] for item in pixel_values)
        device = pixel_values[0].device
        dtype = pixel_values[0].dtype

1571
        all_pixel_values = torch.zeros((B, 3, P, L), dtype=dtype, device=device)
tc-mb's avatar
tc-mb committed
1572
1573
1574
1575
1576
1577
1578
1579
        for i, pixel_values_item in enumerate(pixel_values):
            L_item = pixel_values_item.shape[-1]
            all_pixel_values[i, ..., :L_item] = pixel_values_item

        num_patches = tgt_sizes.prod(-1)
        max_patches = num_patches.max().item()
        assert isinstance(max_patches, int)

1580
        patch_attn_mask = torch.zeros((B, max_patches), dtype=torch.bool, device=device)
tc-mb's avatar
tc-mb committed
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
        for i, num_patches_item in enumerate(num_patches):
            patch_attn_mask[i, :num_patches_item] = True

        vision_embedding = self.vpm(
            all_pixel_values,
            patch_attention_mask=patch_attn_mask.unsqueeze(1),
            tgt_sizes=tgt_sizes,
        )

        return self.resampler(vision_embedding, tgt_sizes)

1592
1593
    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
        loader = AutoWeightsLoader(self, skip_prefixes=["apm.", "audio", "tts"])
1594
1595
1596
        loaded = loader.load_weights(weights)
        self._ensure_resampler_device()
        return loaded
tc-mb's avatar
tc-mb committed
1597

Jee Jee Li's avatar
Jee Jee Li committed
1598

tc-mb's avatar
tc-mb committed
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
class MiniCPMV4_5(MiniCPMVBaseModel, SupportsLoRA):
    packed_modules_mapping = {
        "qkv_proj": [
            "q_proj",
            "k_proj",
            "v_proj",
        ],
        "gate_up_proj": [
            "gate_proj",
            "up_proj",
        ],
    }

    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
        super().__init__(vllm_config=vllm_config, prefix=prefix)
        assert self.version == (4, 5)

    def init_llm(
        self,
        vllm_config: VllmConfig,
        prefix: str = "",
    ) -> nn.Module:
        return Qwen3ForCausalLM(vllm_config=vllm_config, prefix=prefix)

    def init_vision_module(
        self,
        config: PretrainedConfig,
1626
        quant_config: QuantizationConfig | None = None,
tc-mb's avatar
tc-mb committed
1627
1628
        prefix: str = "",
    ) -> nn.Module:
1629
1630
1631
        model = Idefics2VisionTransformer(
            config.vision_config,
            quant_config=quant_config,
1632
            apply_encoder_attention_mask=True,
1633
1634
            prefix=prefix,
        )
tc-mb's avatar
tc-mb committed
1635
1636
1637
1638
1639
1640
1641
1642
        if self.config.drop_vision_last_layer:
            model.encoder.layers = model.encoder.layers[:-1]
        return model

    def init_resampler(
        self,
        embed_dim: int,
        vision_dim: int,
1643
        quant_config: QuantizationConfig | None = None,
tc-mb's avatar
tc-mb committed
1644
1645
1646
1647
        prefix: str = "",
    ) -> nn.Module:
        with set_default_torch_dtype(torch.float16):
            # The resampler in 4.0 remains consistent with the one in 2.5/2.6.
1648
1649
1650
1651
1652
1653
1654
1655
            resampler = Resampler4_5(
                num_queries=self.config.query_num,
                embed_dim=embed_dim,
                num_heads=embed_dim // 128,
                kv_dim=vision_dim,
                quant_config=quant_config,
                prefix=prefix,
            )
1656
1657

        return resampler.to(dtype=torch.get_default_dtype())
1658
1659

    def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
tc-mb's avatar
tc-mb committed
1660
1661
        pixel_values = data["pixel_values"]
        tgt_sizes = data["tgt_sizes"]
1662
        temporal_ids = data.get("temporal_ids", None)
tc-mb's avatar
tc-mb committed
1663
1664
1665
1666
1667
1668
1669

        B = len(pixel_values)
        P = pixel_values[0].shape[-2]
        L = max(item.shape[-1] for item in pixel_values)
        device = pixel_values[0].device
        dtype = pixel_values[0].dtype

1670
1671
1672
1673
        all_pixel_values = torch.zeros((B, 3, P, L), dtype=dtype, device=device)
        all_temporal_ids = (
            None if temporal_ids is None else flatten_2d_lists(temporal_ids)
        )
tc-mb's avatar
tc-mb committed
1674
1675
1676
1677
1678
1679
1680
1681
        for i, pixel_values_item in enumerate(pixel_values):
            L_item = pixel_values_item.shape[-1]
            all_pixel_values[i, ..., :L_item] = pixel_values_item

        num_patches = tgt_sizes.prod(-1)
        max_patches = num_patches.max().item()
        assert isinstance(max_patches, int)

1682
        patch_attn_mask = torch.zeros((B, max_patches), dtype=torch.bool, device=device)
tc-mb's avatar
tc-mb committed
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
        for i, num_patches_item in enumerate(num_patches):
            patch_attn_mask[i, :num_patches_item] = True

        vision_embedding = self.vpm(
            all_pixel_values,
            patch_attention_mask=patch_attn_mask.unsqueeze(1),
            tgt_sizes=tgt_sizes,
        )

        return self.resampler(vision_embedding, tgt_sizes, all_temporal_ids)

1694
1695
    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
        loader = AutoWeightsLoader(self, skip_prefixes=["apm.", "audio", "tts"])
1696
1697
1698
        loaded = loader.load_weights(weights)
        self._ensure_resampler_device()
        return loaded
tc-mb's avatar
tc-mb committed
1699
1700


1701
1702
1703
_SUPPORT_VERSION = {
    (2, 0): MiniCPMV2_0,
    (2, 5): MiniCPMV2_5,
1704
    (2, 6): MiniCPMV2_6,
tc-mb's avatar
tc-mb committed
1705
    (4, 0): MiniCPMV4_0,
tc-mb's avatar
tc-mb committed
1706
    (4, 5): MiniCPMV4_5,
1707
1708
1709
}


1710
1711
1712
@MULTIMODAL_REGISTRY.register_processor(
    MiniCPMVMultiModalProcessor,
    info=MiniCPMVProcessingInfo,
1713
1714
    dummy_inputs=MiniCPMVDummyInputsBuilder,
)
1715
class MiniCPMV(MiniCPMVBaseModel, SupportsMultiModal, SupportsLoRA):
Jee Jee Li's avatar
Jee Jee Li committed
1716
1717
1718
1719
1720
    """
    Different versions of MiniCPMV use different visual encoders and LLMs,
    which is not conducive to the current integration logic of LoRA and
    bitsandbytes in vLLM. Therefore, it is necessary to separate them.
    """
1721

1722
    def __new__(cls, *, vllm_config: VllmConfig, prefix: str = ""):
1723
        config = vllm_config.model_config.hf_config
Jee Jee Li's avatar
Jee Jee Li committed
1724
1725
1726
1727
1728
1729
1730
1731
1732
        if not hasattr(config, "version"):
            if config.hidden_size == 2304 and config.query_num == 64:
                version = (2, 0)
            else:
                version = (2, 5)
        else:
            version = str(config.version).split(".")
            version = tuple([int(x) for x in version])
        # Dispatch class based on version
1733
1734
        instance_cls = _SUPPORT_VERSION.get(version)
        if instance_cls is None:
tc-mb's avatar
tc-mb committed
1735
            supported_versions = ", ".join(
1736
1737
1738
1739
1740
1741
                [f"{v[0]}.{v[1]}" for v in sorted(_SUPPORT_VERSION.keys())]
            )
            raise ValueError(
                f"Currently, MiniCPMV only supports versions "
                f"{supported_versions}. Got version: {version}"
            )
1742
1743
1744
1745
1746
1747

        # quant_config references base class members,
        # so update values before init is called
        cls.packed_modules_mapping.update(instance_cls.packed_modules_mapping)
        cls.embedding_modules.update(instance_cls.embedding_modules)
        return instance_cls(vllm_config=vllm_config, prefix=prefix)