minicpmv.py 42.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# Adapted from
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
# Copyright 2023 The vLLM team.
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
#
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
# and OPT implementations in this library. It has been modified from its
# original forms to accommodate minor architectural differences compared
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Alphi's avatar
Alphi committed
22
"""Inference-only MiniCPM-V model compatible with HuggingFace weights."""
23
24
25
import math
import re
from functools import partial
26
27
from typing import (Any, Callable, Iterable, List, Literal, Mapping, Optional,
                    Tuple, TypedDict, Union)
28
29

import torch
Alphi's avatar
Alphi committed
30
import torch.types
31
32
from PIL import Image
from torch import nn
33
from transformers import PretrainedConfig
34
from typing_extensions import NotRequired
35
36

from vllm.attention import AttentionMetadata
37
from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
38
39
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
                         InputContext, token_inputs)
Alphi's avatar
Alphi committed
40
from vllm.model_executor.layers.logits_processor import LogitsProcessor
41
from vllm.model_executor.layers.quantization import QuantizationConfig
42
from vllm.model_executor.layers.resampler import (BaseResampler, Resampler2,
43
                                                  get_2d_sincos_pos_embed)
Joe Runde's avatar
Joe Runde committed
44
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
Alphi's avatar
Alphi committed
45
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
Jee Jee Li's avatar
Jee Jee Li committed
46
from vllm.model_executor.model_loader.utils import set_default_torch_dtype
47
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
Alphi's avatar
Alphi committed
48
49
from vllm.model_executor.models.llama import LlamaModel
from vllm.model_executor.models.minicpm import MiniCPMModel
50
from vllm.model_executor.models.module_mapping import MultiModelKeys
Alphi's avatar
Alphi committed
51
from vllm.model_executor.models.qwen2 import Qwen2Model
52
from vllm.model_executor.models.utils import LLMWrapper
53
54
from vllm.model_executor.sampling_metadata import SamplingMetadata
from vllm.multimodal import MULTIMODAL_REGISTRY
55
from vllm.multimodal.base import MultiModalKwargs
56
57
from vllm.multimodal.image import cached_get_image_processor
from vllm.multimodal.utils import cached_get_tokenizer
58
from vllm.sequence import IntermediateTensors, SequenceData
59

Jee Jee Li's avatar
Jee Jee Li committed
60
from .idefics2_vision_model import Idefics2VisionTransformer
61
62
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
from .utils import is_pp_missing_parameter
Jee Jee Li's avatar
Jee Jee Li committed
63

64
_KEYS_TO_MODIFY_MAPPING = {
Alphi's avatar
Alphi committed
65
    "llm.lm_head": "lm_head",
66
67
}

68
RawImageType = Union[Image.Image, torch.Tensor]
69

70
71

class MiniCPMVRawImageInput(TypedDict):
72
    """Input mapper input with auxiliary data for computing image bounds."""
73
    image: RawImageType
74
75
76
77
78
79
80
81

    # Image bounds token ids in 0-dim scaler tensor.
    im_start_id: torch.Tensor
    im_end_id: torch.Tensor
    slice_start_id: NotRequired[torch.Tensor]
    slice_end_id: NotRequired[torch.Tensor]


Jee Jee Li's avatar
Jee Jee Li committed
82
class MiniCPMVImagePixelInputs(TypedDict):
83
84
    type: Literal["pixel_values"]
    data: List[torch.Tensor]
Jee Jee Li's avatar
Jee Jee Li committed
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
    """
    Shape: `(batch_size * num_images, num_channels, height, width)`

    Note that the image size may vary, so we pass it as a list
    instead of a batched tensor.
    """

    image_bounds: torch.Tensor
    """
    Shape: `(batch_size * num_images, 2)`

    This should be in `(start, stop)` format.
    """

    tgt_sizes: torch.Tensor
    """
    Shape: `(batch_size * num_images, 2)`

    This should be in `(height, width)` format.
    """


107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
class MiniCPMVImageEmbeddingInputs(TypedDict):
    type: Literal["image_embeds"]
    data: torch.Tensor
    """
    Shape: `(batch_size * num_images, image_feature_size, hidden_size)`

    `hidden_size` must match the hidden size of language model backbone.
    instead of a batched tensor.
    """

    image_bounds: torch.Tensor
    """
    Shape: `(batch_size * num_images, 2)`

    This should be in `(start, stop)` format.
    """


MiniCPMVImageInputs = Union[MiniCPMVImagePixelInputs,
                            MiniCPMVImageEmbeddingInputs]

Jee Jee Li's avatar
Jee Jee Li committed
128
129
130
131
132
DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)


class Resampler2_5(BaseResampler):

133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
    def __init__(self,
                 num_queries: int,
                 embed_dim: int,
                 num_heads: int,
                 kv_dim: Optional[int] = None,
                 norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
                 max_size: Tuple[int, int] = (70, 70),
                 quant_config: Optional[QuantizationConfig] = None,
                 prefix: str = "") -> None:
        super().__init__(num_queries,
                         embed_dim,
                         num_heads,
                         kv_dim,
                         norm_layer,
                         quant_config=quant_config,
                         prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
149
150
151

        self.max_size = max_size
        self._set_2d_pos_cache(self.max_size)
152
153
154

        self.apply(self._init_weights)

Alphi's avatar
Alphi committed
155
156
    def _set_2d_pos_cache(self,
                          max_size: Tuple[int, int],
Jee Jee Li's avatar
Jee Jee Li committed
157
158
159
160
161
                          device: torch.types.Device = "cpu") -> None:
        pos_embed_arr = get_2d_sincos_pos_embed(self.embed_dim,
                                                max_size,
                                                version=(2, 5))
        pos_embed = torch.from_numpy(pos_embed_arr).float().to(device)
162
163
        self.register_buffer("pos_embed", pos_embed, persistent=False)

Alphi's avatar
Alphi committed
164
    def _adjust_pos_cache(self, tgt_sizes: torch.Tensor,
Jee Jee Li's avatar
Jee Jee Li committed
165
166
167
168
169
                          device: torch.types.Device) -> None:
        max_h = tgt_sizes[:, 0].max().item()
        max_w = tgt_sizes[:, 1].max().item()
        assert isinstance(max_h, int) and isinstance(max_w, int)

170
        if max_h > self.max_size[0] or max_w > self.max_size[1]:
Jee Jee Li's avatar
Jee Jee Li committed
171
            self.max_size = (
172
                max(max_h, self.max_size[0]),
Jee Jee Li's avatar
Jee Jee Li committed
173
174
                max(max_w, self.max_size[1]),
            )
175
176
            self._set_2d_pos_cache(self.max_size, device)

Jee Jee Li's avatar
Jee Jee Li committed
177
178
    def forward(self, x: torch.Tensor,
                tgt_sizes: torch.Tensor) -> torch.Tensor:
179
180
181
182
183
184
185
186
187
188
        assert x.shape[0] == tgt_sizes.shape[0]
        bs = x.shape[0]

        device = x.device
        dtype = x.dtype

        patch_len = tgt_sizes[:, 0] * tgt_sizes[:, 1]

        self._adjust_pos_cache(tgt_sizes, device=device)

Jee Jee Li's avatar
Jee Jee Li committed
189
190
191
        max_patch_len = patch_len.max().item()
        assert isinstance(max_patch_len, int)

192
193
194
195
196
197
        key_padding_mask = torch.zeros((bs, max_patch_len),
                                       dtype=torch.bool,
                                       device=device)

        pos_embed = []
        for i in range(bs):
Jee Jee Li's avatar
Jee Jee Li committed
198
            tgt_h, tgt_w = tgt_sizes[i].tolist()
199
200
201
202
203
204
205
206
            pos_embed.append(self.pos_embed[:tgt_h, :tgt_w, :].reshape(
                (tgt_h * tgt_w, -1)).to(dtype))  # patches * D
            key_padding_mask[i, patch_len[i]:] = True
        pos_embed = torch.nn.utils.rnn.pad_sequence(pos_embed,
                                                    batch_first=True,
                                                    padding_value=0.0).permute(
                                                        1, 0,
                                                        2)  # BLD => L * B * D
Jee Jee Li's avatar
Jee Jee Li committed
207
        x, _ = self.kv_proj(x)  # B * L * D
208
209
210
211
212
213
214
215
        x = self.ln_kv(x).permute(1, 0, 2)  # L * B * D

        q = self.ln_q(self.query)  # Q * D

        out = self.attn(
            self._repeat(q, bs),  # Q * B * D
            x + pos_embed,  # L * B * D +  L * B * D
            x,
Jee Jee Li's avatar
Jee Jee Li committed
216
217
            key_padding_mask=key_padding_mask,
        )[0]
218
219
220
221
222
223
224
225
        #  out: Q * B * D
        x = out.permute(1, 0, 2)  # B * Q * D

        x = self.ln_post(x)
        x = x @ self.proj
        return x


226
def _build_image_input(ctx: InputContext,
227
                       image: RawImageType) -> MiniCPMVRawImageInput:
228
229
230
231
    tokenizer = cached_get_tokenizer(
        ctx.model_config.tokenizer,
        trust_remote_code=ctx.model_config.trust_remote_code)
    if hasattr(tokenizer, "slice_start_id"):
232
        return MiniCPMVRawImageInput(
233
234
235
236
237
238
            image=image,
            im_start_id=torch.tensor(tokenizer.im_start_id),
            im_end_id=torch.tensor(tokenizer.im_end_id),
            slice_start_id=torch.tensor(tokenizer.slice_start_id),
            slice_end_id=torch.tensor(tokenizer.slice_end_id))
    else:
239
240
241
242
        return MiniCPMVRawImageInput(
            image=image,
            im_start_id=torch.tensor(tokenizer.im_start_id),
            im_end_id=torch.tensor(tokenizer.im_end_id))
243
244


245
246
247
248
249
250
251
252
253
254
255
256
257
258
def get_version_by_config(config: PretrainedConfig) -> Tuple[int, ...]:
    version_float = getattr(config, "version", None)

    # The old configs do not include version number
    # TODO: Remove this after the HF repos are updated
    if version_float is None:
        if config.hidden_size == 2304 and config.query_num == 64:
            return (2, 0)
        return (2, 5)

    version_str = str(version_float)
    return tuple(int(x) for x in version_str.split("."))


259
def get_max_minicpmv_image_tokens(ctx: InputContext):
260
    hf_config = ctx.get_hf_config()
261
262
263
    return getattr(hf_config, "query_num", 64)


264
def dummy_seq_data_for_minicpmv(seq_len: int, num_images: int):
265
    return SequenceData.from_prompt_token_counts((0, seq_len))
266
267


268
269
def dummy_image_for_minicpmv(ctx: InputContext, hf_config: PretrainedConfig,
                             num_images: int):
270
    width = height = hf_config.image_size
271
272
273
274
    image = _build_image_input(ctx,
                               image=Image.new("RGB", (width, height),
                                               color=0))
    return {"image": [image] if num_images == 1 else [image] * num_images}
275
276


277
278
def dummy_data_for_minicpmv(ctx: InputContext, seq_len: int,
                            mm_counts: Mapping[str, int]):
279
    hf_config = ctx.get_hf_config()
280
    num_images = mm_counts["image"]
281

282
    seq_data = dummy_seq_data_for_minicpmv(seq_len, num_images)
283
    mm_data = dummy_image_for_minicpmv(ctx, hf_config, num_images)
284

285
    return DummyData(seq_data, mm_data)
286
287


288
289
def input_processor_for_minicpmv(ctx: InputContext, inputs: DecoderOnlyInputs):
    multi_modal_data = inputs.get("multi_modal_data")
290
    if multi_modal_data is None or "image" not in multi_modal_data:
291
        return inputs
292
    model_config = ctx.model_config
293
    version = get_version_by_config(model_config.hf_config)
294
295
296
    tokenizer = cached_get_tokenizer(
        model_config.tokenizer,
        trust_remote_code=model_config.trust_remote_code)
297
298
299
300
301
302
303
304
    image_processor = cached_get_image_processor(model_config.tokenizer)

    def get_placeholder(image_size: Tuple[int, int], num_image: int):
        if version == (2, 0) or version == (2, 5):
            return image_processor. \
                get_slice_image_placeholder(image_size)
        return image_processor. \
            get_slice_image_placeholder(image_size, num_image)
305

306
307
    prompt = inputs.get("prompt")
    token_ids = inputs.get("prompt_token_ids")
308
309
310
311
    if prompt is None:
        prompt = tokenizer.decode(token_ids)

    pattern = "(<image>./</image>)"
312
    images = multi_modal_data["image"]
313
    image_tags = re.findall(pattern, prompt)
Jee Jee Li's avatar
Jee Jee Li committed
314
315
316
317
    if len(image_tags) == 0:
        new_token_ids = token_ids
        new_prompt = prompt
    else:
318
319
320
321
322
323
324
325
        if isinstance(images, dict):
            image_size_list = images.get("image_size_list")
            images = [images.get("image_embeds")]
        else:
            if isinstance(images, Image.Image):
                images = [images]
            image_size_list = [image.size for image in images]

Jee Jee Li's avatar
Jee Jee Li committed
326
        text_chunks = prompt.split(pattern)
327
        new_prompt_chunks: List[str] = []
328
        for i in range(len(image_size_list)):
329
330
            new_prompt_chunks += [
                text_chunks[i],
331
                get_placeholder(image_size_list[i], i)
332
333
334
            ]
        new_prompt_chunks.append(text_chunks[-1])
        new_prompt = "".join(new_prompt_chunks)
Jee Jee Li's avatar
Jee Jee Li committed
335
336
        new_token_ids = tokenizer.encode(new_prompt)

337
338
339
340
    multi_modal_data["image"] = [
        _build_image_input(ctx, image) for image in images
    ]

341
    return token_inputs(
Jee Jee Li's avatar
Jee Jee Li committed
342
343
344
345
        prompt_token_ids=new_token_ids,
        prompt=new_prompt,
        multi_modal_data=multi_modal_data,
    )
346
347


348
349
350
351
352
353
354
355
356
357
358
359
def input_mapper_for_minicpmv(ctx: InputContext, data: object):
    model_config = ctx.model_config

    image_processor = cached_get_image_processor(
        model_config.model, trust_remote_code=model_config.trust_remote_code)
    if image_processor is None:
        raise RuntimeError("No HuggingFace processor is available "
                           "to process the image object")

    if not isinstance(data, list):
        raise ValueError(
            "Image input must be list of MiniCPMVImageInput, got (%s)", data)
360
361
362
363
364
365
366
367
368

    if len(data) > 0 and isinstance(data[0]['image'], torch.Tensor):
        batch_data = {
            "image_embeds": data[0]['image'],
        }
    else:
        batch_data = image_processor \
            .preprocess([img["image"] for img in data], return_tensors="pt") \
            .data
369
370
371
372
373
374
375
376

    if len(data) > 0:
        batch_data["im_start_id"] = data[0]["im_start_id"]
        batch_data["im_end_id"] = data[0]["im_end_id"]
        if "slice_start_id" in data[0]:
            batch_data["slice_start_id"] = data[0]["slice_start_id"]
            batch_data["slice_end_id"] = data[0]["slice_end_id"]

377
    return MultiModalKwargs(batch_data)
378
379


380
class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
Jee Jee Li's avatar
Jee Jee Li committed
381
382
383
384
    """
    The abstract class of MiniCPMV can only be inherited, but cannot be
    instantiated.
    """
385
386
387

    def __init__(
        self,
Alphi's avatar
Alphi committed
388
        config: PretrainedConfig,
389
390
391
392
393
        multimodal_config: MultiModalConfig,
        cache_config: Optional[CacheConfig] = None,
        quant_config: Optional[QuantizationConfig] = None,
    ):
        super().__init__()
394
395
396
397
        # All MiniCPM-V models disable `tie_word_embeddings` but
        # `PretrainedConfig.tie_word_embeddings` defaults to True; we cannot
        # check `tie_word_embeddings` until vLLM integrate MiniCPM-V model
        # and config class
398
399
400
        self.config = config
        self.multimodal_config = multimodal_config

401
        self.version = get_version_by_config(self.config)
402
403
404
405
406
        self.llm = self.init_llm(config,
                                 cache_config,
                                 quant_config,
                                 prefix="llm")
        self.vpm = self.init_vision_module(config, quant_config, prefix="vpm")
407
408
        param_dtype = torch.get_default_dtype()
        self.vpm.to(dtype=param_dtype)
Jee Jee Li's avatar
Jee Jee Li committed
409
410
        self.vision_dim = (self.vpm.embed_dim if self.version == (2, 0) else
                           self.vpm.embeddings.embed_dim)
Alphi's avatar
Alphi committed
411
        self.embed_dim = self.config.hidden_size
412
413
414
415
        self.resampler = self.init_resampler(self.embed_dim,
                                             self.vision_dim,
                                             quant_config=quant_config,
                                             prefix="resampler")
416
        self.resampler.to(device="cuda", dtype=param_dtype)
417
        # TODO: why is there _KEYS_TO_MODIFY_MAPPING? lm_head should be in llm
Alphi's avatar
Alphi committed
418
419
        self.lm_head = ParallelLMHead(config.vocab_size,
                                      config.hidden_size,
420
421
                                      quant_config=quant_config,
                                      prefix="llm.lm_head")
Alphi's avatar
Alphi committed
422
        self.logits_processor = LogitsProcessor(config.vocab_size)
Joe Runde's avatar
Joe Runde committed
423
        self.sampler = get_sampler()
424

425
426
427
        self.make_empty_intermediate_tensors = (
            self.llm.make_empty_intermediate_tensors)

Jee Jee Li's avatar
Jee Jee Li committed
428
429
430
    def get_embedding(
        self,
        input_ids: torch.Tensor,
431
        image_inputs: Optional[MiniCPMVImageInputs],
Jee Jee Li's avatar
Jee Jee Li committed
432
433
434
435
436
437
438
    ) -> Tuple[torch.Tensor, torch.Tensor]:
        vlm_embedding: torch.Tensor = self.llm.embed_tokens(input_ids)
        if hasattr(self.config, "scale_emb"):
            vlm_embedding *= self.config.scale_emb

        if image_inputs is None:  # No image
            vision_hidden_states = torch.tensor([], device=input_ids.device)
439
        else:
440
441
442
443
444
445
            if image_inputs["type"] == "image_embeds":
                vision_hidden_states = (image_inputs["data"].type(
                    vlm_embedding.dtype).to(vlm_embedding.device))
            else:
                vision_hidden_states = self.get_vision_hidden_states(
                    image_inputs)
Jee Jee Li's avatar
Jee Jee Li committed
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460

            # See NOTE in _parse_and_validate_inputs
            image_bounds = image_inputs["image_bounds"]
            if len(image_bounds) > 0:
                image_indices = torch.stack([
                    torch.arange(start, end, dtype=torch.long)
                    for start, end in image_bounds.tolist()
                ]).to(vlm_embedding.device)
                vlm_embedding.scatter_(
                    0,
                    image_indices.view(-1, 1).repeat(1,
                                                     vlm_embedding.shape[-1]),
                    vision_hidden_states.view(-1,
                                              vision_hidden_states.shape[-1]),
                )
461

Jee Jee Li's avatar
Jee Jee Li committed
462
        return vlm_embedding, vision_hidden_states
463

464
465
466
467
468
469
470
471
472
473
474
475
476
477
    def _get_image_bounds(
            self,
            input_ids: torch.Tensor,
            im_start_id: torch.Tensor,
            im_end_id: torch.Tensor,
            slice_start_id: Optional[torch.Tensor] = None,
            slice_end_id: Optional[torch.Tensor] = None) -> torch.Tensor:
        # All the images in the batch should share the same special image
        # bound token ids.
        start_cond = input_ids == im_start_id[0]
        end_cond = input_ids == im_end_id[0]
        if slice_start_id is not None:
            start_cond |= (input_ids == slice_start_id[0])
            end_cond |= (input_ids == slice_end_id[0])
Alphi's avatar
Alphi committed
478

Jee Jee Li's avatar
Jee Jee Li committed
479
        image_start_tokens, = torch.where(start_cond)
480
        image_start_tokens += 1
Jee Jee Li's avatar
Jee Jee Li committed
481
        image_end_tokens, = torch.where(end_cond)
Alphi's avatar
Alphi committed
482
        valid_image_nums = max(len(image_start_tokens), len(image_end_tokens))
Jee Jee Li's avatar
Jee Jee Li committed
483

484
        if valid_image_nums == 0:
Jee Jee Li's avatar
Jee Jee Li committed
485
486
487
            return torch.zeros((0, 2), device=input_ids.device)

        return torch.hstack([
488
489
490
491
            image_start_tokens[:valid_image_nums].unsqueeze(-1),
            image_end_tokens[:valid_image_nums].unsqueeze(-1),
        ])

Jee Jee Li's avatar
Jee Jee Li committed
492
493
494
495
    def _parse_and_validate_inputs(
        self,
        input_ids: torch.Tensor,
        **kwargs: object,
496
    ) -> Optional[MiniCPMVImageInputs]:
Jee Jee Li's avatar
Jee Jee Li committed
497
498
        pixel_values = kwargs.pop("pixel_values", [])
        tgt_sizes = kwargs.pop("tgt_sizes", [])
499
500
501
502
503
504
505
506
507
508
509
510
511
512
        im_start_id = kwargs.pop("im_start_id", None)
        im_end_id = kwargs.pop("im_end_id", None)
        slice_start_id = kwargs.pop("slice_start_id", None)
        slice_end_id = kwargs.pop("slice_end_id", None)
        image_embeds = kwargs.pop("image_embeds", None)

        if image_embeds is not None:
            return MiniCPMVImageEmbeddingInputs(
                image_bounds=self._get_image_bounds(input_ids, im_start_id,
                                                    im_end_id, slice_start_id,
                                                    slice_end_id),
                data=image_embeds,
                type="image_embeds",
            )
Jee Jee Li's avatar
Jee Jee Li committed
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527

        if not isinstance(pixel_values, (torch.Tensor, list)):
            raise ValueError("Incorrect type of pixel values. "
                             f"Got type: {type(pixel_values)}")

        if not isinstance(tgt_sizes, (torch.Tensor, list)):
            raise ValueError("Incorrect type of target sizes. "
                             f"Got type: {type(tgt_sizes)}")

        if len(pixel_values) != len(tgt_sizes):
            raise ValueError("Inconsistent batch lengths, found: "
                             f"{len(pixel_values)} vs. {len(tgt_sizes)}")

        pixel_values_flat: List[torch.Tensor] = []
        tgt_sizes_flat: List[torch.Tensor] = []
528
529
530
531
532
533
534
535
        for pixel_b, tgt_b in zip(pixel_values, tgt_sizes):
            if len(pixel_b) != len(tgt_b):
                raise ValueError("Inconsistent N lengths, found: "
                                 f"{len(pixel_b)} vs {len(tgt_b)}")

            for pixel_n, tgt_n in zip(pixel_b, tgt_b):
                pixel_values_flat += pixel_n
                tgt_sizes_flat += tgt_n
Jee Jee Li's avatar
Jee Jee Li committed
536
537
538
539
540
541
542
543
544
545
546

        # NOTE: Input IDs does not contain image tokens during memory profiling,
        # so we allow it to be empty
        if len(pixel_values_flat) != len(tgt_sizes_flat):
            raise ValueError("Inconsistent flattened lengths, found: "
                             f"{len(pixel_values_flat)} vs. "
                             f"{len(tgt_sizes_flat)}")

        if len(pixel_values_flat) == 0:
            return None

547
548
549
550
551
552
553
        if im_start_id is None:
            return None

        return MiniCPMVImagePixelInputs(
            image_bounds=self._get_image_bounds(input_ids, im_start_id,
                                                im_end_id, slice_start_id,
                                                slice_end_id),
554
            data=pixel_values_flat,
Jee Jee Li's avatar
Jee Jee Li committed
555
            tgt_sizes=torch.stack(tgt_sizes_flat),
556
            type="pixel_values",
Jee Jee Li's avatar
Jee Jee Li committed
557
        )
558
559
560
561
562
563
564
565

    def forward(
        self,
        input_ids: torch.Tensor,
        positions: torch.Tensor,
        kv_caches: List[torch.Tensor],
        attn_metadata: AttentionMetadata,
        intermediate_tensors: Optional[IntermediateTensors] = None,
Jee Jee Li's avatar
Jee Jee Li committed
566
567
        **kwargs: Any,
    ) -> torch.Tensor:
568
569
570
571
        if intermediate_tensors is not None:
            vlm_embeddings = None
        else:
            image_inputs = self._parse_and_validate_inputs(input_ids, **kwargs)
Jee Jee Li's avatar
Jee Jee Li committed
572

573
            vlm_embeddings, _ = self.get_embedding(input_ids, image_inputs)
Jee Jee Li's avatar
Jee Jee Li committed
574

575
576
577
578
579
        # always pass the input via `inputs_embeds`
        # to make sure the computation graph is consistent
        # for `torch.compile` integration
        input_ids = None

Jee Jee Li's avatar
Jee Jee Li committed
580
        output = self.llm(
581
            input_ids=input_ids,
Jee Jee Li's avatar
Jee Jee Li committed
582
583
584
585
586
587
            positions=positions,
            kv_caches=kv_caches,
            attn_metadata=attn_metadata,
            intermediate_tensors=intermediate_tensors,
            inputs_embeds=vlm_embeddings,
        )
588
589
        return output

590
591
592
593
594
    def compute_logits(
        self,
        hidden_states: torch.Tensor,
        sampling_metadata: SamplingMetadata,
    ) -> Optional[torch.Tensor]:
Alphi's avatar
Alphi committed
595
596
597
        logits = self.logits_processor(self.lm_head, hidden_states,
                                       sampling_metadata)
        return logits
598
599
600
601
602
603

    def sample(
        self,
        logits: torch.Tensor,
        sampling_metadata: SamplingMetadata,
    ) -> Optional[SamplerOutput]:
Alphi's avatar
Alphi committed
604
        next_tokens = self.sampler(logits, sampling_metadata)
605
606
607
608
609
610
611
612
613
614
615
616
617
        return next_tokens

    def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
        stacked_params_mapping = [
            # (param_name, shard_name, shard_id)
            ("qkv_proj", "q_proj", "q"),
            ("qkv_proj", "k_proj", "k"),
            ("qkv_proj", "v_proj", "v"),
            ("gate_up_proj", "gate_proj", 0),
            ("gate_up_proj", "up_proj", 1),
        ]
        params_dict = dict(self.named_parameters())
        for name, loaded_weight in weights:
Alphi's avatar
Alphi committed
618
619
620
            for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
                if key_to_modify in name:
                    name = name.replace(key_to_modify, new_key)
621
622
623
624
625
626
627
628
            if "rotary_emb.inv_freq" in name:
                continue
            if ("rotary_emb.cos_cached" in name
                    or "rotary_emb.sin_cached" in name):
                # Models trained using ColossalAI may include these tensors in
                # the checkpoint. Skip them.
                continue
            use_default_weight_loading = False
Jee Jee Li's avatar
Jee Jee Li committed
629
            if self.is_default_weight_loading(name):
630
631
                use_default_weight_loading = True
            else:
Jee Jee Li's avatar
Jee Jee Li committed
632
                for param_name, weight_name, shard_id in stacked_params_mapping:
633
634
                    if weight_name not in name:
                        continue
635
636
637
                    if is_pp_missing_parameter(
                            name.replace(weight_name, param_name), self):
                        continue
638
639
640
641
642
643
644
                    param = params_dict[name.replace(weight_name, param_name)]
                    weight_loader = param.weight_loader
                    weight_loader(param, loaded_weight, shard_id)
                    break
                else:
                    use_default_weight_loading = True
            if use_default_weight_loading:
645
646
                if is_pp_missing_parameter(name, self):
                    continue
647
648
649
650
                param = params_dict[name]
                weight_loader = getattr(param, "weight_loader",
                                        default_weight_loader)
                weight_loader(param, loaded_weight)
Jee Jee Li's avatar
Jee Jee Li committed
651

652
653
654
655
656
657
658
659
    def get_mm_mapping(self) -> MultiModelKeys:
        """
        Get the module prefix in multimodal models
        """
        return MultiModelKeys.from_string_field(language_model="llm",
                                                connector="resampler",
                                                tower_model="vpm")

Jee Jee Li's avatar
Jee Jee Li committed
660
661
662
663
664
    def init_llm(
        self,
        config: PretrainedConfig,
        cache_config: Optional[CacheConfig] = None,
        quant_config: Optional[QuantizationConfig] = None,
665
        prefix: str = "",
Jee Jee Li's avatar
Jee Jee Li committed
666
667
668
    ) -> nn.Module:
        raise NotImplementedError

669
670
671
672
    def init_vision_module(
        self,
        config: PretrainedConfig,
        quant_config: Optional[QuantizationConfig],
673
        prefix: str = "",
674
    ) -> nn.Module:
Jee Jee Li's avatar
Jee Jee Li committed
675
676
        raise NotImplementedError

677
678
679
680
681
    def init_resampler(self,
                       embed_dim: int,
                       vision_dim: int,
                       quant_config: Optional[QuantizationConfig] = None,
                       prefix: str = "") -> nn.Module:
Jee Jee Li's avatar
Jee Jee Li committed
682
683
684
685
686
687
688
689
690
691
        raise NotImplementedError

    def get_vision_embedding(
        self,
        pixel_values: List[torch.Tensor],
        patch_attn_mask: Optional[torch.Tensor] = None,
        tgt_sizes: Optional[torch.Tensor] = None,
    ) -> torch.Tensor:
        raise NotImplementedError

692
693
    def get_vision_hidden_states(self,
                                 data: MiniCPMVImageInputs) -> torch.Tensor:
Jee Jee Li's avatar
Jee Jee Li committed
694
695
696
697
698
699
        raise NotImplementedError

    def is_default_weight_loading(self, name: str) -> bool:
        raise NotImplementedError


700
class MiniCPMV2_0(MiniCPMVBaseModel):
Jee Jee Li's avatar
Jee Jee Li committed
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716

    def __init__(
        self,
        config: PretrainedConfig,
        multimodal_config: MultiModalConfig,
        cache_config: Optional[CacheConfig] = None,
        quant_config: Optional[QuantizationConfig] = None,
    ):
        super().__init__(config, multimodal_config, cache_config, quant_config)
        assert self.version == (2, 0)

    def init_llm(
        self,
        config: PretrainedConfig,
        cache_config: Optional[CacheConfig] = None,
        quant_config: Optional[QuantizationConfig] = None,
717
        prefix: str = "",
Jee Jee Li's avatar
Jee Jee Li committed
718
    ) -> nn.Module:
719
720
721

        return LLMWrapper(MiniCPMModel(config,
                                       cache_config=cache_config,
722
723
                                       quant_config=quant_config,
                                       prefix=prefix),
724
                          name="model")
Jee Jee Li's avatar
Jee Jee Li committed
725

726
727
728
729
    def init_vision_module(
        self,
        config: PretrainedConfig,
        quant_config: Optional[QuantizationConfig],
730
        prefix: str = "",
731
    ) -> nn.Module:
Jee Jee Li's avatar
Jee Jee Li committed
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
        # TODO :refactor this vision model
        try:
            import timm
        except ImportError:
            raise ImportError("Please install timm==0.9.10") from ImportError
        with set_default_torch_dtype(torch.float16):
            model = timm.create_model(
                "vit_so400m_patch14_siglip_384.webli",
                pretrained=False,
                num_classes=0,
                dynamic_img_size=True,
                dynamic_img_pad=True,
            )

        if (isinstance(model, timm.models.VisionTransformer)
                and model.attn_pool is not None):
            model.attn_pool = torch.nn.Identity()

        if self.config.drop_vision_last_layer:
            model.blocks = model.blocks[:-1]

        return model

755
756
757
    def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
        return self.model.embed_tokens(input_ids)

758
759
760
761
762
    def init_resampler(self,
                       embed_dim: int,
                       vision_dim: int,
                       quant_config: Optional[QuantizationConfig] = None,
                       prefix: str = "") -> nn.Module:
Jee Jee Li's avatar
Jee Jee Li committed
763
        with set_default_torch_dtype(torch.float16):
764
765
766
767
768
769
770
771
772
            resampler = Resampler2(embed_dim=embed_dim,
                                   num_heads=embed_dim // 128,
                                   grid_size=int(
                                       math.sqrt(self.config.query_num)),
                                   kv_dim=vision_dim,
                                   adaptive=False,
                                   do_post_projection=True,
                                   quant_config=quant_config,
                                   prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798

        return resampler

    def get_vision_embedding(
        self,
        pixel_values: List[torch.Tensor],
        patch_attn_mask: Optional[torch.Tensor] = None,
        tgt_sizes: Optional[torch.Tensor] = None,
    ) -> torch.Tensor:
        res = []
        dtype = self.vpm.pos_embed.data.dtype
        for pixel_value in pixel_values:
            H, W = pixel_value[0].shape[-2:]
            tgt_size = (
                math.ceil(H / self.vpm.patch_embed.patch_size[0]),
                math.ceil(W / self.vpm.patch_embed.patch_size[0]),
            )
            vision_embedding = self.vpm.forward_features(
                pixel_value.unsqueeze(0).type(dtype))
            if (hasattr(self.vpm, "num_prefix_tokens")
                    and self.vpm.num_prefix_tokens > 0):
                vision_embedding = vision_embedding[:, self.vpm.
                                                    num_prefix_tokens:]
            res.append(self.resampler(vision_embedding, tgt_size))
        return torch.vstack(res)

799
800
801
    def get_vision_hidden_states(self,
                                 data: MiniCPMVImageInputs) -> torch.Tensor:
        pixel_values = data["data"]
Jee Jee Li's avatar
Jee Jee Li committed
802
803
804
805
806
807
808

        return self.get_vision_embedding(pixel_values)

    def is_default_weight_loading(self, name: str) -> bool:
        return "resampler" in name or "vpm" in name


809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
    packed_modules_mapping = {
        "qkv_proj": [
            "q_proj",
            "k_proj",
            "v_proj",
        ],
        "gate_up_proj": [
            "gate_proj",
            "up_proj",
        ],
    }
    # LoRA specific attributes
    supported_lora_modules = [
        # vision encoder
        "fc1",
        "fc2",
        "out_proj",
        # language model
        "qkv_proj",  # same name with vision encoder
        "o_proj",
        "gate_up_proj",
        "down_proj",
        # resampler
        "kv_proj",
    ]
835
836
837
838
839
840
841
842
843
844

    # BitandBytes specific attributes
    default_bitsandbytes_target_modules = [
        ".gate_proj.",
        ".down_proj.",
        ".up_proj.",
        ".q_proj.",
        ".k_proj.",
        ".v_proj.",
        ".o_proj.",
845
846
847
848
849
850
851
852
853
854
        # vision encoder
        ".fc1.",
        ".fc2.",
        # Currently, vllm does not support BNB quantization for the `out_proj`
        # of the resampler, so it's necessary to distinguish between the
        # vision encoder and the resampler's out_proj. The same applies to
        # MiniCPMV2_6.
        ".self_attn.out_proj.",  #  vision encoder out_proj
        # resampler
        ".kv_proj.",
855
856
857
858
859
860
861
862
863
864
    ]
    bitsandbytes_stacked_params_mapping = {
        # shard_name, weight_name, index
        "q_proj": ("qkv_proj", 0),
        "k_proj": ("qkv_proj", 1),
        "v_proj": ("qkv_proj", 2),
        "gate_proj": ("gate_up_proj", 0),
        "up_proj": ("gate_up_proj", 1),
    }

865
866
    embedding_modules = {}
    embedding_padding_modules = []
Jee Jee Li's avatar
Jee Jee Li committed
867
868
869
870
871
872
873

    def __init__(
        self,
        config: PretrainedConfig,
        multimodal_config: MultiModalConfig,
        cache_config: Optional[CacheConfig] = None,
        quant_config: Optional[QuantizationConfig] = None,
874
        lora_config: Optional[LoRAConfig] = None,
Jee Jee Li's avatar
Jee Jee Li committed
875
876
877
878
879
880
881
882
883
    ):
        super().__init__(config, multimodal_config, cache_config, quant_config)
        assert self.version == (2, 5)

    def init_llm(
        self,
        config: PretrainedConfig,
        cache_config: Optional[CacheConfig] = None,
        quant_config: Optional[QuantizationConfig] = None,
884
        prefix: str = "",
Jee Jee Li's avatar
Jee Jee Li committed
885
    ) -> nn.Module:
886
887
        return LLMWrapper(LlamaModel(config,
                                     cache_config=cache_config,
888
889
                                     quant_config=quant_config,
                                     prefix=prefix),
890
                          name="model")
Jee Jee Li's avatar
Jee Jee Li committed
891

892
893
894
895
    def init_vision_module(
        self,
        config: PretrainedConfig,
        quant_config: Optional[QuantizationConfig],
896
        prefix: str = "",
897
898
    ) -> nn.Module:
        model = Idefics2VisionTransformer(config.vision_config,
899
900
                                          quant_config=quant_config,
                                          prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
901
902
903
904
        if self.config.drop_vision_last_layer:
            model.encoder.layers = model.encoder.layers[:-1]
        return model

905
906
907
908
909
    def init_resampler(self,
                       embed_dim: int,
                       vision_dim: int,
                       quant_config: Optional[QuantizationConfig] = None,
                       prefix: str = "") -> nn.Module:
Jee Jee Li's avatar
Jee Jee Li committed
910
        with set_default_torch_dtype(torch.float16):
911
912
913
914
915
916
            resampler = Resampler2_5(num_queries=self.config.query_num,
                                     embed_dim=embed_dim,
                                     num_heads=embed_dim // 128,
                                     kv_dim=vision_dim,
                                     quant_config=quant_config,
                                     prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
917
918
919
920
921
922
923
924
925
926
927
928
929
        return resampler

    def get_vision_embedding(
        self,
        pixel_values: List[torch.Tensor],
        patch_attn_mask: Optional[torch.Tensor] = None,
        tgt_sizes: Optional[torch.Tensor] = None,
    ) -> torch.Tensor:
        vision_embedding = self.vpm(pixel_values,
                                    patch_attention_mask=patch_attn_mask)
        vision_embedding = self.resampler(vision_embedding, tgt_sizes)
        return vision_embedding

930
931
932
    def get_vision_hidden_states(self,
                                 data: MiniCPMVImageInputs) -> torch.Tensor:
        pixel_values = data["data"]
Jee Jee Li's avatar
Jee Jee Li committed
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
        tgt_sizes = data["tgt_sizes"]

        device = self.vpm.embeddings.position_embedding.weight.device
        dtype = self.vpm.embeddings.position_embedding.weight.dtype
        all_pixel_values_lst = [
            i.flatten(end_dim=1).permute(1, 0) for i in pixel_values
        ]

        max_patches = (tgt_sizes[:, 0] * tgt_sizes[:, 1]).max().item()
        assert isinstance(max_patches, int)

        all_pixel_values = torch.nn.utils.rnn.pad_sequence(
            all_pixel_values_lst, batch_first=True, padding_value=0.0)
        B, L, _ = all_pixel_values.shape
        all_pixel_values = all_pixel_values.permute(0, 2,
                                                    1).reshape(B, 3, -1, L)

        patch_attn_mask = torch.zeros((B, 1, max_patches),
                                      dtype=torch.bool,
                                      device=device)
        for i in range(B):
            patch_attn_mask[i, :tgt_sizes[i][0] * tgt_sizes[i][1]] = True

        return self.get_vision_embedding(all_pixel_values.type(dtype),
                                         patch_attn_mask, tgt_sizes)

    def is_default_weight_loading(self, name: str) -> bool:
        return "resampler" in name


963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
    packed_modules_mapping = {
        "qkv_proj": [
            "q_proj",
            "k_proj",
            "v_proj",
        ],
        "gate_up_proj": [
            "gate_proj",
            "up_proj",
        ],
    }
    # LoRA specific attributes
    supported_lora_modules = [
        # vision encoder
        "fc1",
        "fc2",
        "out_proj",
        # language model
        "qkv_proj",  # same name with vision encoder
        "o_proj",
        "gate_up_proj",
        "down_proj",
        # resampler
        "kv_proj",
    ]

990
991
992
993
994
995
996
997
998
    # BitandBytes specific attributes
    default_bitsandbytes_target_modules = [
        ".gate_proj.",
        ".down_proj.",
        ".up_proj.",
        ".q_proj.",
        ".k_proj.",
        ".v_proj.",
        ".o_proj.",
999
1000
1001
1002
1003
1004
        # vision encoder
        ".fc1.",
        ".fc2.",
        ".self_attn.out_proj.",
        # resampler
        ".kv_proj.",
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
    ]
    bitsandbytes_stacked_params_mapping = {
        # shard_name, weight_name, index
        "q_proj": ("qkv_proj", 0),
        "k_proj": ("qkv_proj", 1),
        "v_proj": ("qkv_proj", 2),
        "gate_proj": ("gate_up_proj", 0),
        "up_proj": ("gate_up_proj", 1),
    }

1015
1016
    embedding_modules = {}
    embedding_padding_modules = []
Jee Jee Li's avatar
Jee Jee Li committed
1017
1018
1019
1020
1021
1022
1023
1024
1025

    def __init__(
        self,
        config: PretrainedConfig,
        multimodal_config: MultiModalConfig,
        cache_config: Optional[CacheConfig] = None,
        quant_config: Optional[QuantizationConfig] = None,
    ):
        super().__init__(config, multimodal_config, cache_config, quant_config)
1026
        assert self.version == (2, 6)
Jee Jee Li's avatar
Jee Jee Li committed
1027
1028
1029
1030
1031
1032

    def init_llm(
        self,
        config: PretrainedConfig,
        cache_config: Optional[CacheConfig] = None,
        quant_config: Optional[QuantizationConfig] = None,
1033
        prefix: str = "",
Jee Jee Li's avatar
Jee Jee Li committed
1034
    ) -> nn.Module:
1035
1036
1037

        return LLMWrapper(Qwen2Model(config,
                                     cache_config=cache_config,
1038
1039
                                     quant_config=quant_config,
                                     prefix=prefix),
1040
                          name="model")
Jee Jee Li's avatar
Jee Jee Li committed
1041

1042
1043
1044
1045
    def init_vision_module(
        self,
        config: PretrainedConfig,
        quant_config: Optional[QuantizationConfig],
1046
        prefix: str = "",
1047
1048
    ) -> nn.Module:
        model = Idefics2VisionTransformer(config.vision_config,
1049
1050
                                          quant_config=quant_config,
                                          prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
1051
1052
1053
1054
        if self.config.drop_vision_last_layer:
            model.encoder.layers = model.encoder.layers[:-1]
        return model

1055
1056
1057
1058
1059
    def init_resampler(self,
                       embed_dim: int,
                       vision_dim: int,
                       quant_config: Optional[QuantizationConfig] = None,
                       prefix: str = "") -> nn.Module:
Jee Jee Li's avatar
Jee Jee Li committed
1060
        with set_default_torch_dtype(torch.float16):
1061
            # The resampler in 2.6 remains consistent with the one in 2.5.
1062
1063
1064
1065
1066
1067
            resampler = Resampler2_5(num_queries=self.config.query_num,
                                     embed_dim=embed_dim,
                                     num_heads=embed_dim // 128,
                                     kv_dim=vision_dim,
                                     quant_config=quant_config,
                                     prefix=prefix)
Jee Jee Li's avatar
Jee Jee Li committed
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
        return resampler

    def get_vision_embedding(
        self,
        pixel_values: List[torch.Tensor],
        patch_attn_mask: Optional[torch.Tensor] = None,
        tgt_sizes: Optional[torch.Tensor] = None,
    ) -> torch.Tensor:
        vision_embedding = self.vpm(
            pixel_values,
            patch_attention_mask=patch_attn_mask,
            tgt_sizes=tgt_sizes,
1080
        )
Jee Jee Li's avatar
Jee Jee Li committed
1081
1082
        return vision_embedding

1083
1084
1085
    def get_vision_hidden_states(self,
                                 data: MiniCPMVImageInputs) -> torch.Tensor:
        pixel_values = data["data"]
Jee Jee Li's avatar
Jee Jee Li committed
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
        tgt_sizes = data["tgt_sizes"]

        device = self.vpm.embeddings.position_embedding.weight.device
        dtype = self.vpm.embeddings.position_embedding.weight.dtype
        all_pixel_values_lst = [
            i.flatten(end_dim=1).permute(1, 0) for i in pixel_values
        ]

        max_patches = (tgt_sizes[:, 0] * tgt_sizes[:, 1]).max().item()
        assert isinstance(max_patches, int)

        all_pixel_values = torch.nn.utils.rnn.pad_sequence(
            all_pixel_values_lst, batch_first=True, padding_value=0.0)
        B, L, _ = all_pixel_values.shape
        all_pixel_values = all_pixel_values.permute(0, 2,
                                                    1).reshape(B, 3, -1, L)

        patch_attn_mask = torch.zeros((B, 1, max_patches),
                                      dtype=torch.bool,
                                      device=device)
        for i in range(B):
            patch_attn_mask[i, 0, :tgt_sizes[i][0] * tgt_sizes[i][1]] = True
        vision_embedding = self.vpm(
            all_pixel_values.type(dtype),
            patch_attention_mask=patch_attn_mask,
            tgt_sizes=tgt_sizes,
1112
        )
Jee Jee Li's avatar
Jee Jee Li committed
1113
1114
1115
1116

        return self.resampler(vision_embedding, tgt_sizes)

    def is_default_weight_loading(self, name: str) -> bool:
1117
        return "resampler" in name
Jee Jee Li's avatar
Jee Jee Li committed
1118
1119


1120
1121
1122
1123
1124
1125
1126
_SUPPORT_VERSION = {
    (2, 0): MiniCPMV2_0,
    (2, 5): MiniCPMV2_5,
    (2, 6): MiniCPMV2_6
}


1127
@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_minicpmv)
Jee Jee Li's avatar
Jee Jee Li committed
1128
1129
1130
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_minicpmv_image_tokens)
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_minicpmv)
@INPUT_REGISTRY.register_input_processor(input_processor_for_minicpmv)
1131
class MiniCPMV(MiniCPMVBaseModel, SupportsLoRA):
Jee Jee Li's avatar
Jee Jee Li committed
1132
1133
1134
1135
1136
    """
    Different versions of MiniCPMV use different visual encoders and LLMs,
    which is not conducive to the current integration logic of LoRA and
    bitsandbytes in vLLM. Therefore, it is necessary to separate them.
    """
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
    # Ensure that the LoRA support check passes when the class is not
    # initialized, but set all these attributes to empty.
    packed_modules_mapping = {}
    supported_lora_modules = []
    embedding_modules = {}
    embedding_padding_modules = []

    def __new__(cls,
                config: PretrainedConfig,
                multimodal_config: MultiModalConfig,
                cache_config: Optional[CacheConfig] = None,
                quant_config: Optional[QuantizationConfig] = None,
                lora_config: Optional[LoRAConfig] = None):
Jee Jee Li's avatar
Jee Jee Li committed
1150
1151
1152
1153
1154
1155
1156
1157
1158
        if not hasattr(config, "version"):
            if config.hidden_size == 2304 and config.query_num == 64:
                version = (2, 0)
            else:
                version = (2, 5)
        else:
            version = str(config.version).split(".")
            version = tuple([int(x) for x in version])
        # Dispatch class based on version
1159
        instance_class = _SUPPORT_VERSION.get(version)
1160
1161
1162
        if instance_class is None:
            raise ValueError(
                "Currently, MiniCPMV only supports versions 2.0, 2.5, and 2.6")
Jee Jee Li's avatar
Jee Jee Li committed
1163
1164
        return instance_class(config, multimodal_config, cache_config,
                              quant_config)