qwen2_vl.py 53.9 KB
Newer Older
1
2
# SPDX-License-Identifier: Apache-2.0

3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
# Adapted from
# https://github.com/huggingface/transformers/blob/19e6e80e10118f855137b90740936c0b11ac397f/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py
# Copyright 2024 The Qwen team.
# Copyright 2023 The vLLM team.
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
#
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
# and OPT implementations in this library. It has been modified from its
# original forms to accommodate minor architectural differences compared
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Inference-only Qwen2-VL model compatible with HuggingFace weights."""
26
from functools import cached_property, partial
27
28
from typing import (Any, Callable, Iterable, List, Literal, Mapping, Optional,
                    Set, Tuple, Type, TypedDict, Union)
29
30
31
32
33

import torch
import torch.nn as nn
import torch.nn.functional as F
from einops import rearrange, repeat
34
35
36
from transformers import BatchFeature
from transformers.models.qwen2_vl import (Qwen2VLImageProcessor,
                                          Qwen2VLProcessor)
37
38
from transformers.models.qwen2_vl.configuration_qwen2_vl import (
    Qwen2VLConfig, Qwen2VLVisionConfig)
39
from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
40
41

from vllm.attention import AttentionMetadata
42
from vllm.config import VllmConfig
43
from vllm.distributed import parallel_state, tensor_model_parallel_all_gather
44
45
46
47
48
49
50
from vllm.distributed import utils as dist_utils
from vllm.logger import init_logger
from vllm.model_executor import SamplingMetadata
from vllm.model_executor.layers.activation import QuickGELU
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
                                               RowParallelLinear)
from vllm.model_executor.layers.quantization import QuantizationConfig
51
52
53
from vllm.model_executor.layers.quantization.gptq import GPTQConfig
from vllm.model_executor.layers.quantization.gptq_marlin import (
    GPTQMarlinConfig)
Joe Runde's avatar
Joe Runde committed
54
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
55
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
56
from vllm.model_executor.models.module_mapping import MultiModelKeys
57
from vllm.multimodal import MULTIMODAL_REGISTRY
58
from vllm.multimodal.inputs import (ImageItem, ModalityData,
59
                                    MultiModalFieldConfig, MultiModalKwargs,
60
                                    VideoItem)
61
from vllm.multimodal.parse import (ImageSize, ModalityDataItems,
62
                                   MultiModalDataItems, MultiModalDataParser)
63
from vllm.multimodal.processing import (BaseMultiModalProcessor,
64
65
                                        BaseProcessingInfo, PromptReplacement)
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
66
from vllm.platforms import _Backend
67
from vllm.sequence import IntermediateTensors
68
from vllm.transformers_utils.config import uses_mrope
69

70
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
71
from .utils import (AutoWeightsLoader, WeightsMapper,
72
73
                    init_vllm_registered_model, maybe_prefix,
                    merge_multimodal_embeddings)
74
from .vision import get_vit_attn_backend
75

zhuwenwen's avatar
zhuwenwen committed
76
77
78
79
80
import os
import re
from vllm import _custom_ops as ops
from vllm.model_executor.utils import pad_weight, gemm_bank_conf

81
82
logger = init_logger(__name__)

83
84
85
# For profile run
_MAX_FRAMES_PER_VIDEO = 16

86
87
88
# === Vision Inputs === #


89
90
class Qwen2VLImagePixelInputs(TypedDict):
    type: Literal["pixel_values"]
91
    pixel_values: torch.Tensor
92
    """Shape:
93
94
95
96
97
98
99
100
101
    `(num_patches, num_channels * patch_size * patch_size)`
    """

    image_grid_thw: torch.Tensor
    """Shape: `(num_images, 3)`
    This should be in `(grid_t, grid_h, grid_w)` format.
    """


102
103
class Qwen2VLImageEmbeddingInputs(TypedDict):
    type: Literal["image_embeds"]
104
105
106
107
108
109
    image_embeds: torch.Tensor
    """Supported types:
    - List[`torch.Tensor`]: A list of tensors holding all images' features.
        Each tensor holds an image's features.
    - `torch.Tensor`: A tensor holding all images' features
        (concatenation of all images' feature tensors).
110
    
111
112
113
114
115
116
117
118
    Tensor shape: `(num_image_features, hidden_size)`
    - `num_image_features` varies based on
        the number and resolution of the images.
    - `hidden_size` must match the hidden size of language model backbone.
    """

    image_grid_thw: torch.Tensor
    """Shape: `(num_images, 3)`
119
120
121
122
    This should be in `(grid_t, grid_h, grid_w)` format.
    """


123
124
125
126
Qwen2VLImageInputs = Union[Qwen2VLImagePixelInputs,
                           Qwen2VLImageEmbeddingInputs]


127
128
class Qwen2VLVideoPixelInputs(TypedDict):
    type: Literal["pixel_values_videos"]
129
    pixel_values_videos: torch.Tensor
130
131
    """Shape:
    `(num_patches,
132
133
134
135
136
      num_channels * temporal_patch_size * patch_size * patch_size)`
    """

    video_grid_thw: torch.Tensor
    """Shape: `(num_videos, 3)`
137

138
139
140
141
    This should be in `(grid_t, grid_h, grid_w)` format.
    """


142
143
144
145
146
147
148
class Qwen2VLVideoEmbeddingInputs(TypedDict):
    type: Literal["video_embeds"]
    video_embeds: torch.Tensor
    """Supported types:
    - List[`torch.Tensor`]: A list of tensors holding all videos' features.
        Each tensor holds an video's features.
    - `torch.Tensor`: A tensor holding all videos' features
149
        (concatenation of all videos' feature tensors).
150
    
151
152
153
154
155
156
157
158
    Tensor shape: `(num_image_features, hidden_size)`
    - `num_image_features` varies based on 
        the number and resolution of the videos.
    - `hidden_size` must match the hidden size of language model backbone.
    """

    video_grid_thw: torch.Tensor
    """Shape: `(num_videos, 3)`
159
160
161
162
    This should be in `(grid_t, grid_h, grid_w)` format.
    """


163
164
165
Qwen2VLVideoInputs = Union[Qwen2VLVideoPixelInputs,
                           Qwen2VLVideoEmbeddingInputs]

166
167
168
169
170
171
172
173
# === Vision Encoder === #


class Qwen2VisionMLP(nn.Module):

    def __init__(
        self,
        in_features: int,
174
        hidden_features: int,
175
176
        act_layer: Type[nn.Module] = QuickGELU,
        quant_config: Optional[QuantizationConfig] = None,
177
        prefix: str = "",
178
179
180
181
    ):
        super().__init__()
        self.fc1 = ColumnParallelLinear(in_features,
                                        hidden_features,
182
183
                                        quant_config=quant_config,
                                        prefix=f"{prefix}.fc1")
184
185
186
        self.act = act_layer()
        self.fc2 = RowParallelLinear(hidden_features,
                                     in_features,
187
188
                                     quant_config=quant_config,
                                     prefix=f"{prefix}.fc2")
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x_parallel, _ = self.fc1(x)
        x_parallel = self.act(x_parallel)
        x, _ = self.fc2(x_parallel)
        return x


def rotate_half(x: torch.Tensor, interleaved: bool = False) -> torch.Tensor:
    if not interleaved:
        x1, x2 = x.chunk(2, dim=-1)
        return torch.cat((-x2, x1), dim=-1)
    else:
        x1, x2 = x[..., ::2], x[..., 1::2]
        return rearrange(torch.stack((-x2, x1), dim=-1),
                         "... d two -> ... (d two)",
                         two=2)


def apply_rotary_emb_torch(x: torch.Tensor,
                           cos: torch.Tensor,
                           sin: torch.Tensor,
                           interleaved: bool = False) -> torch.Tensor:
    """
    x: (batch_size, seqlen, nheads, headdim)
    cos, sin: (seqlen, rotary_dim / 2) or (batch_size, seqlen, rotary_dim / 2)
    """
    ro_dim = cos.shape[-1] * 2
    assert ro_dim <= x.shape[-1]
    cos = repeat(
        cos,
        "... d -> ... 1 (2 d)" if not interleaved else "... d -> ... 1 (d 2)")
    sin = repeat(
        sin,
        "... d -> ... 1 (2 d)" if not interleaved else "... d -> ... 1 (d 2)")
    return torch.cat(
        [
            x[..., :ro_dim] * cos +
            rotate_half(x[..., :ro_dim], interleaved) * sin, x[..., ro_dim:]
        ],
        dim=-1,
    )


def apply_rotary_pos_emb_vision(t: torch.Tensor,
                                freqs: torch.Tensor) -> torch.Tensor:
    t_ = t.float()
    cos = freqs.cos()
    sin = freqs.sin()
    output = apply_rotary_emb_torch(t_, cos, sin).type_as(t)
    return output


class Qwen2VisionAttention(nn.Module):

    def __init__(
        self,
246
247
248
        embed_dim: int,
        num_heads: int,
        projection_size: int,
249
        quant_config: Optional[QuantizationConfig] = None,
250
        prefix: str = "",
251
252
253
254
    ) -> None:
        super().__init__()
        # Per attention head and per partition values.
        world_size = parallel_state.get_tensor_model_parallel_world_size()
255
256
        self.tp_size = world_size
        self.tp_rank = parallel_state.get_tensor_model_parallel_rank()
257
258
259
260
261
262
263
        self.hidden_size_per_attention_head = dist_utils.divide(
            projection_size, num_heads)
        self.num_attention_heads_per_partition = dist_utils.divide(
            num_heads, world_size)

        self.qkv = ColumnParallelLinear(input_size=embed_dim,
                                        output_size=3 * projection_size,
264
265
                                        quant_config=quant_config,
                                        prefix=f"{prefix}.qkv")
266
267
        self.proj = RowParallelLinear(input_size=projection_size,
                                      output_size=embed_dim,
268
269
                                      quant_config=quant_config,
                                      prefix=f"{prefix}.proj")
270
271

        # Detect attention implementation.
272
        self.attn_backend: _Backend = get_vit_attn_backend(support_fa=True)
273
274
275
276
277
        if self.attn_backend not in {
                _Backend.FLASH_ATTN, _Backend.TORCH_SDPA, _Backend.XFORMERS
        }:
            raise RuntimeError(
                f"Qwen2-VL does not support {self.attn_backend} backend now.")
278

279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
    def split_qkv(self, qkv: torch.Tensor) -> tuple[torch.Tensor, ...]:
        # [s, b, 3 * head * head_dim]
        seq_len, bs, _ = qkv.shape
        if self.tp_size > 1:
            qkv = tensor_model_parallel_all_gather(qkv)

        # [s, b, 3 * head * head_dim] -> 3 * [s, b, head * head_dim]
        q, k, v = qkv.chunk(3, dim=2)

        # 3 * [s, b, head * head_dim]
        if self.tp_size > 1:
            splitter = partial(dist_utils.split_tensor_along_last_dim,
                               num_partitions=self.tp_size)
            q = splitter(q)[self.tp_rank]
            k = splitter(k)[self.tp_rank]
            v = splitter(v)[self.tp_rank]

        # 3 * [s, b, head * head_dim] -> 3 * [s, b, head, head_dim]
        new_shape = (seq_len, bs, self.num_attention_heads_per_partition,
                     self.hidden_size_per_attention_head)
        q, k, v = (x.view(*new_shape) for x in (q, k, v))
        return q, k, v

302
303
304
305
    def forward(
        self,
        x: torch.Tensor,
        cu_seqlens: torch.Tensor,
306
        rotary_pos_emb: torch.Tensor,
307
308
    ) -> torch.Tensor:

309
310
        # [s, b, c] --> [s, b, 3 * head * head_dim]
        x, _ = self.qkv(x)
311

312
313
        # [s, b, 3 * head * head_dim] -> 3 * [s, b, head, head_dim]
        q, k, v = self.split_qkv(x)
314
315
        batch_size = q.shape[1]

316
317
        q, k, v = (rearrange(x, "s b ... -> b s ...").contiguous()
                   for x in (q, k, v))
318
319
320
321
        if rotary_pos_emb is not None:
            q = apply_rotary_pos_emb_vision(q, rotary_pos_emb)
            k = apply_rotary_pos_emb_vision(k, rotary_pos_emb)

322
        if self.attn_backend == _Backend.FLASH_ATTN:
323
324
325
326
            # from vllm_flash_attn.flash_attn_interface import (
            #   flash_attn_varlen_func)
            from flash_attn import flash_attn_varlen_func

327
            q, k, v = (rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v])
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342

            max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item()
            output = flash_attn_varlen_func(q,
                                            k,
                                            v,
                                            cu_seqlens_q=cu_seqlens,
                                            cu_seqlens_k=cu_seqlens,
                                            max_seqlen_q=max_seqlen,
                                            max_seqlen_k=max_seqlen,
                                            dropout_p=0,
                                            causal=False)

            context_layer = rearrange(output,
                                      "(b s) ... -> b s ...",
                                      b=batch_size)
343
        elif self.attn_backend == _Backend.TORCH_SDPA:
344
            seq_length = q.size(1)
345
            q, k, v = (rearrange(x, "b s h d -> b h s d") for x in [q, k, v])
346
347
348
349
350
351
352
353
354
355
356
357
            attention_mask = torch.zeros([1, seq_length, seq_length],
                                         device=q.device,
                                         dtype=torch.bool)
            for i in range(1, len(cu_seqlens)):
                attention_mask[..., cu_seqlens[i - 1]:cu_seqlens[i],
                               cu_seqlens[i - 1]:cu_seqlens[i]] = True
            output = F.scaled_dot_product_attention(q,
                                                    k,
                                                    v,
                                                    attention_mask,
                                                    dropout_p=0.0)
            context_layer = rearrange(output, "b h s d -> b s h d ")
358
        elif self.attn_backend == _Backend.XFORMERS:
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
            from xformers import ops as xops
            from xformers.ops.fmha.attn_bias import BlockDiagonalMask

            seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
            attn_bias = BlockDiagonalMask.from_seqlens(q_seqlen=seqlens,
                                                       kv_seqlen=None)

            context_layer = xops.memory_efficient_attention_forward(
                q, k, v, attn_bias=attn_bias, p=0, scale=None)
        context_layer = rearrange(context_layer,
                                  "b s h d -> s b (h d)").contiguous()

        output, _ = self.proj(context_layer)
        return output


class Qwen2VisionBlock(nn.Module):

    def __init__(
        self,
        dim: int,
        num_heads: int,
        mlp_ratio: float,
        act_layer: Type[nn.Module] = QuickGELU,
383
        norm_layer: Optional[Callable[[int], nn.Module]] = None,
384
        quant_config: Optional[QuantizationConfig] = None,
385
        prefix: str = "",
386
387
388
389
390
391
392
393
394
395
396
    ) -> None:
        super().__init__()
        if norm_layer is None:
            norm_layer = partial(nn.LayerNorm, eps=1e-6)
        self.norm1 = norm_layer(dim)
        self.norm2 = norm_layer(dim)
        mlp_hidden_dim = int(dim * mlp_ratio)

        self.attn = Qwen2VisionAttention(embed_dim=dim,
                                         num_heads=num_heads,
                                         projection_size=dim,
397
398
                                         quant_config=quant_config,
                                         prefix=f"{prefix}.attn")
399
400
401
        self.mlp = Qwen2VisionMLP(dim,
                                  mlp_hidden_dim,
                                  act_layer=act_layer,
402
403
                                  quant_config=quant_config,
                                  prefix=f"{prefix}.mlp")
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419

    def forward(self, x: torch.Tensor, cu_seqlens: torch.Tensor,
                rotary_pos_emb: torch.Tensor) -> torch.Tensor:
        x = x + self.attn(self.norm1(x),
                          cu_seqlens=cu_seqlens,
                          rotary_pos_emb=rotary_pos_emb)
        x = x + self.mlp(self.norm2(x))
        return x


class Qwen2VisionPatchEmbed(nn.Module):

    def __init__(
        self,
        patch_size: int = 14,
        temporal_patch_size: int = 2,
420
        in_channels: int = 3,
421
422
423
424
425
426
427
        embed_dim: int = 1152,
    ) -> None:
        super().__init__()
        self.patch_size = patch_size
        self.temporal_patch_size = temporal_patch_size
        self.embed_dim = embed_dim

428
429
        kernel_size = (temporal_patch_size, patch_size, patch_size)
        self.proj = nn.Conv3d(in_channels,
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
                              embed_dim,
                              kernel_size=kernel_size,
                              stride=kernel_size,
                              bias=False)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        L, C = x.shape
        x = x.view(L, -1, self.temporal_patch_size, self.patch_size,
                   self.patch_size)
        x = self.proj(x).view(L, self.embed_dim)
        return x


class Qwen2VisionPatchMerger(nn.Module):

    def __init__(
        self,
        d_model: int,
        context_dim: int,
449
        norm_layer: Optional[Callable[[int], nn.Module]] = None,
450
451
        spatial_merge_size: int = 2,
        quant_config: Optional[QuantizationConfig] = None,
452
        prefix: str = "",
453
454
455
456
457
458
459
460
461
462
    ) -> None:
        super().__init__()
        self.hidden_size = context_dim * (spatial_merge_size**2)
        if norm_layer is None:
            norm_layer = partial(nn.LayerNorm, eps=1e-6)
        self.ln_q = norm_layer(context_dim)
        self.mlp = nn.ModuleList([
            ColumnParallelLinear(self.hidden_size,
                                 self.hidden_size,
                                 bias=True,
463
464
                                 quant_config=quant_config,
                                 prefix=f"{prefix}.mlp.0"),
465
466
467
468
            nn.GELU(),
            RowParallelLinear(self.hidden_size,
                              d_model,
                              bias=True,
469
470
                              quant_config=quant_config,
                              prefix=f"{prefix}.mlp.2"),
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
        ])

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = self.ln_q(x)
        x = x.view(-1, self.hidden_size)

        mlp_fc1, mlp_act, mlp_fc2 = self.mlp
        x_parallel, _ = mlp_fc1(x)
        x_parallel = mlp_act(x_parallel)
        out, _ = mlp_fc2(x_parallel)
        return out


class Qwen2VisionRotaryEmbedding(nn.Module):

    def __init__(self, dim: int, theta: float = 10000.0) -> None:
        super().__init__()
        self.dim = dim
        self.theta = theta
        inv_freq = 1.0 / (theta
                          **(torch.arange(0, dim, 2, dtype=torch.float) / dim))
        self.register_buffer("inv_freq", inv_freq, persistent=False)
        self._seq_len_cached = 0
        self._freqs_cached = None

    def update_freqs_cache(self, seqlen: int) -> None:
        if seqlen > self._seq_len_cached:
            seqlen *= 2
            self._seq_len_cached = seqlen
            self.inv_freq = 1.0 / (self.theta**(torch.arange(
                0, self.dim, 2, dtype=torch.float, device=self.inv_freq.device)
                                                / self.dim))
            seq = torch.arange(seqlen,
                               device=self.inv_freq.device,
                               dtype=self.inv_freq.dtype)
            freqs = torch.outer(seq, self.inv_freq)
            self._freqs_cached = freqs

    def forward(self, seqlen: int) -> torch.Tensor:
        self.update_freqs_cache(seqlen)
        return self._freqs_cached[:seqlen]


class Qwen2VisionTransformer(nn.Module):

    def __init__(
        self,
        vision_config: Qwen2VLVisionConfig,
        norm_eps: float = 1e-6,
        quant_config: Optional[QuantizationConfig] = None,
521
        prefix: str = "",
522
523
524
    ) -> None:
        super().__init__()

525
526
527
528
529
530
531
532
533
        patch_size = vision_config.patch_size
        temporal_patch_size = vision_config.temporal_patch_size
        spatial_merge_size = vision_config.spatial_merge_size
        in_channels = vision_config.in_channels
        hidden_size = vision_config.hidden_size
        embed_dim = vision_config.embed_dim
        depth = vision_config.depth
        num_heads = vision_config.num_heads
        mlp_ratio = vision_config.mlp_ratio
534
535

        self.spatial_merge_size = spatial_merge_size
536
537
        self.num_heads = num_heads
        self.embed_dim = embed_dim
538
539
540
541

        self.patch_embed = Qwen2VisionPatchEmbed(
            patch_size=patch_size,
            temporal_patch_size=temporal_patch_size,
542
            in_channels=in_channels,
543
544
545
546
547
548
549
550
            embed_dim=embed_dim,
        )

        norm_layer = partial(nn.LayerNorm, eps=norm_eps)
        head_dim = embed_dim // num_heads
        self.rotary_pos_emb = Qwen2VisionRotaryEmbedding(head_dim // 2)

        self.blocks = nn.ModuleList([
551
552
553
554
555
556
557
            Qwen2VisionBlock(dim=embed_dim,
                             num_heads=num_heads,
                             mlp_ratio=mlp_ratio,
                             norm_layer=norm_layer,
                             quant_config=quant_config,
                             prefix=f"{prefix}.blocks.{layer_idx}")
            for layer_idx in range(depth)
558
559
560
561
562
563
        ])
        self.merger = Qwen2VisionPatchMerger(
            d_model=hidden_size,
            context_dim=embed_dim,
            norm_layer=norm_layer,
            quant_config=quant_config,
564
            prefix=f"{prefix}.merger",
565
        )
zhuwenwen's avatar
zhuwenwen committed
566
567
568
569
570
571
572
573
574
575
        
        self.quant_method = None
        if quant_config is not None:
            self.quant_method=quant_config.get_name()
            self.quant_config=quant_config
            
        self.use_llama_nn = os.environ.get('LLAMA_NN') == '1'
        self.use_gemm_pad = os.environ.get('GEMM_PAD') == '1'
        self.use_fa_pad = os.environ.get('FA_PAD') == '1'
        self.use_awq_pad = os.environ.get('AWQ_PAD') == '1'
576
577
578

    @property
    def dtype(self) -> torch.dtype:
579
        return self.patch_embed.proj.weight.dtype
580
581
582

    @property
    def device(self) -> torch.device:
583
        return self.patch_embed.proj.weight.device
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634

    def rot_pos_emb(self, grid_thw: torch.Tensor) -> torch.Tensor:
        pos_ids = []
        for t, h, w in grid_thw:
            hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
            wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
            hpos_ids = hpos_ids.reshape(
                h // self.spatial_merge_size,
                self.spatial_merge_size,
                w // self.spatial_merge_size,
                self.spatial_merge_size,
            ).permute(0, 2, 1, 3).flatten()
            wpos_ids = wpos_ids.reshape(
                h // self.spatial_merge_size,
                self.spatial_merge_size,
                w // self.spatial_merge_size,
                self.spatial_merge_size,
            ).permute(0, 2, 1, 3).flatten()
            pos_ids.append(
                torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))
        pos_ids = torch.cat(pos_ids, dim=0)
        max_grid_size = grid_thw[:, 1:].max()
        rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size)
        rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1)
        return rotary_pos_emb

    def forward(
        self,
        x: torch.Tensor,
        grid_thw: torch.Tensor,
    ) -> torch.Tensor:
        # patchify
        x = x.to(device=self.device, dtype=self.dtype)
        x = self.patch_embed(x)

        # compute position embedding
        rotary_pos_emb = self.rot_pos_emb(grid_thw)

        # compute cu_seqlens
        cu_seqlens = torch.repeat_interleave(grid_thw[:, 1] * grid_thw[:, 2],
                                             grid_thw[:, 0]).cumsum(
                                                 dim=0, dtype=torch.int32)
        cu_seqlens = F.pad(cu_seqlens, (1, 0), "constant", 0)

        # transformers
        x = x.unsqueeze(1)
        for blk in self.blocks:
            x = blk(x, cu_seqlens=cu_seqlens, rotary_pos_emb=rotary_pos_emb)

        # adapter
        x = self.merger(x)
635

636
637
        return x

638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
    def load_weights(self, weights: Iterable[Tuple[str,
                                                   torch.Tensor]]) -> Set[str]:
        stacked_params_mapping = [
            # (param_name, shard_name, shard_id)
            ("qkv_proj", "q_proj", "q"),
            ("qkv_proj", "k_proj", "k"),
            ("qkv_proj", "v_proj", "v"),
        ]
        params_dict = dict(self.named_parameters(remove_duplicate=False))
        loaded_params: Set[str] = set()

        for name, loaded_weight in weights:
            for (param_name, weight_name, shard_id) in stacked_params_mapping:
                if weight_name not in name:
                    continue
                name = name.replace(weight_name, param_name)

                param = params_dict[name]
                weight_loader = param.weight_loader
                weight_loader(param, loaded_weight, shard_id)
                break
            else:
                param = params_dict[name]
                weight_loader = getattr(param, "weight_loader",
                                        default_weight_loader)
                weight_loader(param, loaded_weight)
            loaded_params.add(name)
zhuwenwen's avatar
zhuwenwen committed
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
            
        if self.use_llama_nn and self.quant_method is None:
            lay_key_words = [
                "attn.qkv.weight",
                "attn.proj.weight",
                "mlp.fc1.weight",
                "mlp.fc2.weight",
                "mlp.0.weight",
                "mlp.2.weight",
                "self_attn.qkv_proj.weight",
                "self_attn.o_proj.weight",
                "mlp.gate_up_proj.weight",
                "mlp.down_proj.weight",
                "lm_head.weight",
            ]
            combined_words = "|".join(lay_key_words)
            
zhuwenwen's avatar
zhuwenwen committed
682
683
            # lay_qkv_words = ["attn.qkv.weight"]   
            # qkv_words = "|".join(lay_qkv_words)  
zhuwenwen's avatar
zhuwenwen committed
684
            
zhuwenwen's avatar
zhuwenwen committed
685
686
            # lay_qkv_bias_words = ["attn.qkv.bias"]   
            # qkv_bias_words = "|".join(lay_qkv_bias_words) 
zhuwenwen's avatar
zhuwenwen committed
687
            
zhuwenwen's avatar
zhuwenwen committed
688
689
            for layername in loaded_params:
                weight = params_dict[layername]
zhuwenwen's avatar
zhuwenwen committed
690
691
692
693
694
                # if self.use_fa_pad and (re.findall(qkv_bias_words, layername)):
                #     weight.data = pad_weight(weight.data, 32)
                    
                matches = re.findall(combined_words, layername)
                if matches:   
zhuwenwen's avatar
zhuwenwen committed
695
696
                    # if self.use_gemm_pad and gemm_bank_conf(weight.data.shape[0]):
                    #     weight.data = pad_weight(weight.data, 32)  
zhuwenwen's avatar
zhuwenwen committed
697
698
699
700
701
702
703
704
705
706
707
708
709
                    
                    # if self.use_fa_pad and (re.findall(qkv_words, layername)):
                    #     if not gemm_bank_conf(weight.data.shape[0]):
                    #         weight.data = pad_weight(weight.data, 32)
                        
                    _weight = torch.zeros_like(weight.data)
                    ori_shape =_weight.shape
                    
                    ops.trans_w16_gemm(_weight, weight.data, _weight.shape[0], _weight.shape[1])
                    weight.data.copy_(_weight)
                    
                    weight.data=weight.data.reshape(ori_shape[1],-1)
                    
710
711
        return loaded_params

712

Roger Wang's avatar
Roger Wang committed
713
714
class Qwen2VLEmbeddingItems(ModalityDataItems[dict[str, torch.Tensor],
                                              dict[str, torch.Tensor]]):
715

716
    def __init__(self, data: dict, modality: str) -> None:
717
        super().__init__(data, modality)
718

719
720
721
722
723
724
        grid_thw = data[f"{modality}_grid_thw"]
        slice_idxs = [0] + grid_thw.prod(-1).cumsum_(0).tolist()
        self._slices = [
            slice(slice_idxs[i], slice_idxs[i + 1])
            for i in range(len(grid_thw))
        ]
725

726
727
    def get_count(self) -> int:
        return len(self.data[f"{self.modality}_grid_thw"])
728

729
730
731
732
733
    def get(self, index: int) -> dict[str, torch.Tensor]:
        out = {}
        for k, v in self.data.items():
            if v != f"{self.modality}_grid_thw":
                v = v[self._slices[index]]
734

735
            out[k] = v
736

737
        return out
738

739
740
    def get_processor_data(self) -> Mapping[str, object]:
        return {}
741

742
743
    def get_passthrough_data(self) -> Mapping[str, object]:
        return self.data
744

745

Roger Wang's avatar
Roger Wang committed
746
class Qwen2VLImageEmbeddingItems(Qwen2VLEmbeddingItems):
747

748
749
    def __init__(self, data: dict) -> None:
        super().__init__(data, "image")
750
751


Roger Wang's avatar
Roger Wang committed
752
class Qwen2VLVideoEmbeddingItems(Qwen2VLEmbeddingItems):
753

754
755
    def __init__(self, data: dict) -> None:
        super().__init__(data, "video")
756
757


Roger Wang's avatar
Roger Wang committed
758
class Qwen2VLMultiModalDataParser(MultiModalDataParser):
759

760
761
762
763
764
    def _parse_image_data(
        self,
        data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],
    ) -> ModalityDataItems[Any, Any]:
        if isinstance(data, dict):
Roger Wang's avatar
Roger Wang committed
765
            return Qwen2VLEmbeddingItems(data, modality="image")
766

767
        return super()._parse_image_data(data)
768

769
    def _parse_video_data(
770
        self,
771
772
773
        data: Union[dict[str, torch.Tensor], ModalityData[VideoItem]],
    ) -> ModalityDataItems[Any, Any]:
        if isinstance(data, dict):
Roger Wang's avatar
Roger Wang committed
774
            return Qwen2VLEmbeddingItems(data, modality="video")
775
776
777

        return super()._parse_video_data(data)

778

779
class Qwen2VLProcessingInfo(BaseProcessingInfo):
780

781
    def get_hf_config(self):
782
783
        return self.ctx.get_hf_config(Qwen2VLConfig)

784
    def get_hf_processor(
785
786
787
788
789
790
        self,
        *,
        min_pixels: Optional[int] = None,
        max_pixels: Optional[int] = None,
    ) -> Qwen2VLProcessor:
        hf_processor = self.ctx.get_hf_processor(Qwen2VLProcessor)
791
792
        image_processor = hf_processor.image_processor  # type: ignore
        assert isinstance(image_processor, Qwen2VLImageProcessor)
793
794
795
796
797
798
799
800
801
802
803
804
805

        if min_pixels:
            image_processor.min_pixels = min_pixels
        if max_pixels:
            image_processor.max_pixels = max_pixels
        if max_pixels or min_pixels:
            image_processor.size = {
                "min_pixels": image_processor.min_pixels,
                "max_pixels": image_processor.max_pixels,
            }

        return hf_processor

806
    def get_image_processor(
807
        self,
808
809
810
811
        *,
        min_pixels: Optional[int] = None,
        max_pixels: Optional[int] = None,
    ):
812
813
        hf_processor = self.get_hf_processor(min_pixels=min_pixels,
                                             max_pixels=max_pixels)
814
815
816
        image_processor = hf_processor.image_processor  # type: ignore
        assert isinstance(image_processor, Qwen2VLImageProcessor)
        return image_processor
817

818
819
820
    def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
        return {"image": None, "video": None}

821
822
823
824
825
    def get_mm_max_tokens_per_item(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
    ) -> Mapping[str, int]:
826
827
828
829
830
        return {
            "image": self.get_max_image_tokens(),
            "video": self.get_max_video_tokens(seq_len),
        }

831
832
833
834
835
836
837
    def _get_vision_info(
        self,
        *,
        image_width: int,
        image_height: int,
        num_frames: int = 1,
        do_resize: bool = True,
838
        image_processor: Optional[Qwen2VLImageProcessor],
839
    ) -> tuple[ImageSize, int]:
840
841
842
843
        if image_processor is None:
            image_processor = self.get_image_processor()

        hf_config = self.get_hf_config()
844
        vision_config = hf_config.vision_config
845
846
847
        patch_size = vision_config.patch_size
        merge_size = vision_config.spatial_merge_size
        temporal_patch_size = vision_config.temporal_patch_size
848

849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
        if do_resize:
            resized_height, resized_width = smart_resize(
                height=image_height,
                width=image_width,
                factor=patch_size * merge_size,
                min_pixels=image_processor.min_pixels,
                max_pixels=image_processor.max_pixels,
            )
            preprocessed_size = ImageSize(width=resized_width,
                                          height=resized_height)
        else:
            preprocessed_size = ImageSize(width=image_width,
                                          height=image_height)

        grid_t = max(num_frames // temporal_patch_size, 1)
        grid_h = preprocessed_size.height // patch_size
        grid_w = preprocessed_size.width // patch_size

        num_patches = grid_t * grid_h * grid_w
        num_vision_tokens = num_patches // (merge_size**2)

        return preprocessed_size, num_vision_tokens

872
    def get_num_image_tokens(
873
874
875
876
        self,
        *,
        image_width: int,
        image_height: int,
877
        image_processor: Optional[Qwen2VLImageProcessor],
878
879
880
881
    ) -> int:
        _, num_image_tokens = self._get_vision_info(
            image_width=image_width,
            image_height=image_height,
882
            image_processor=image_processor,
883
884
885
        )
        return num_image_tokens

886
    def get_num_video_tokens(
887
888
889
890
891
        self,
        *,
        image_width: int,
        image_height: int,
        num_frames: int,
892
        image_processor: Optional[Qwen2VLImageProcessor],
893
894
895
896
897
    ) -> int:
        _, num_video_tokens = self._get_vision_info(
            image_width=image_width,
            image_height=image_height,
            num_frames=num_frames,
898
            image_processor=image_processor,
899
900
901
        )
        return num_video_tokens

902
    def get_image_size_with_most_features(self) -> ImageSize:
903
904
905
        max_image_size, _ = self._get_vision_info(
            image_width=9999999,
            image_height=9999999,
906
            image_processor=None,
907
908
909
        )
        return max_image_size

910
911
    def get_max_image_tokens(self) -> int:
        target_width, target_height = self.get_image_size_with_most_features()
912

913
        return self.get_num_image_tokens(
914
915
            image_width=target_width,
            image_height=target_height,
916
            image_processor=None,
917
        )
918
919

    def _get_max_video_frames(self, max_tokens: int) -> int:
920
        target_width, target_height = self.get_image_size_with_most_features()
921

922
923
924
925
        num_frames = 0

        while True:
            next_num_frames = num_frames + 1
926
            next_max_tokens = self.get_num_video_tokens(
927
928
929
                image_width=target_width,
                image_height=target_height,
                num_frames=next_num_frames,
930
                image_processor=None,
931
            )
932

933
            if next_max_tokens > max_tokens:
934
935
936
937
938
939
                break

            num_frames = next_num_frames

        return num_frames

940
    def get_num_frames_with_most_features(self, seq_len: int) -> int:
941
942
943
944
        mm_config = self.ctx.get_mm_config()
        max_images = mm_config.limit_per_prompt.get("image", 1)
        max_videos = mm_config.limit_per_prompt.get("video", 1)

945
        max_image_tokens = self.get_max_image_tokens() * max_images
946
947
        max_total_frames = self._get_max_video_frames(seq_len -
                                                      max_image_tokens)
948
949
        num_frames = min(max(max_total_frames // max(max_videos, 1), 1),
                         _MAX_FRAMES_PER_VIDEO)
950

951
952
953
        # Temporary workaround for https://github.com/huggingface/transformers/issues/35412
        if num_frames > 1 and num_frames % 2 == 1:
            num_frames += 1
954

955
        return num_frames
956

957
958
    def get_max_video_tokens(self, seq_len: int) -> int:
        target_width, target_height = self.get_image_size_with_most_features()
959

960
        return self.get_num_video_tokens(
961
962
            image_width=target_width,
            image_height=target_height,
963
964
            num_frames=self.get_num_frames_with_most_features(seq_len),
            image_processor=None,
965
966
        )

967
968
969

class Qwen2VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2VLProcessingInfo]):

970
971
972
973
974
975
976
977
    def get_dummy_processor_inputs(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
    ) -> ProcessorInputs:
        num_images = mm_counts.get("image", 0)
        num_videos = mm_counts.get("video", 0)

978
        hf_processor = self.info.get_hf_processor()
979
980
        image_token: str = hf_processor.image_token
        video_token: str = hf_processor.video_token
981
982
983
984
985

        target_width, target_height = \
            self.info.get_image_size_with_most_features()
        target_num_frames = \
            self.info.get_num_frames_with_most_features(seq_len)
986
987
988
989
990
991
992
993
994
995

        mm_data = {
            "image":
            self._get_dummy_images(width=target_width,
                                   height=target_height,
                                   num_images=num_images),
            "video":
            self._get_dummy_videos(
                width=target_width,
                height=target_height,
996
                num_frames=target_num_frames,
997
998
                num_videos=num_videos,
            )
999
1000
        }

1001
1002
1003
1004
        return ProcessorInputs(
            prompt_text=image_token * num_images + video_token * num_videos,
            mm_data=mm_data,
        )
1005

1006

1007
1008
class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
                                 ):
1009

1010
    def _get_data_parser(self) -> MultiModalDataParser:
Roger Wang's avatar
Roger Wang committed
1011
        return Qwen2VLMultiModalDataParser()
1012

1013
1014
1015
    def _get_prompt_replacements(
        self,
        mm_items: MultiModalDataItems,
1016
        hf_processor_mm_kwargs: Mapping[str, Any],
1017
        out_mm_kwargs: MultiModalKwargs,
1018
    ) -> list[PromptReplacement]:
1019
1020
1021
        hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
        image_processor = self.info.get_image_processor(
            **hf_processor_mm_kwargs)
1022
1023
        tokenizer = self.info.get_tokenizer()
        vocab = tokenizer.get_vocab()
1024
1025
1026
1027

        # NOTE: Only Qwen2VLProcessor in transformers 4.47.0 has
        # image_token and video_token registered
        placeholder = {
1028
1029
            "image": vocab[hf_processor.image_token],
            "video": vocab[hf_processor.video_token],
1030
        }
1031

1032
1033
1034
        merge_length = image_processor.merge_size**2

        def get_replacement_qwen2vl(item_idx: int, modality: str):
1035
1036
1037
            grid_thw = out_mm_kwargs[f"{modality}_grid_thw"][item_idx]
            assert isinstance(grid_thw, torch.Tensor)

1038
1039
            num_tokens = int(grid_thw.prod()) // merge_length
            return [placeholder[modality]] * num_tokens
1040
1041
1042
1043

        return [
            PromptReplacement(
                modality=modality,
1044
                target=[placeholder[modality]],
1045
1046
1047
1048
                replacement=partial(get_replacement_qwen2vl,
                                    modality=modality),
            ) for modality in ("image", "video")
        ]
1049

1050
    def _get_mm_fields_config(
1051
        self,
1052
1053
1054
1055
        hf_inputs: BatchFeature,
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, MultiModalFieldConfig]:
        image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
1056
        image_grid_sizes = image_grid_thw.prod(-1)
1057

1058
        video_grid_thw = hf_inputs.get("video_grid_thw", torch.empty((0, 3)))
1059
        video_grid_sizes = video_grid_thw.prod(-1)
1060

1061
        return dict(
1062
1063
1064
1065
            pixel_values=MultiModalFieldConfig.flat_from_sizes(
                "image", image_grid_sizes),
            image_embeds=MultiModalFieldConfig.flat_from_sizes(
                "image", image_grid_sizes),
1066
            image_grid_thw=MultiModalFieldConfig.batched("image"),
1067
1068
1069
1070
            pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
                "video", video_grid_sizes),
            video_embeds=MultiModalFieldConfig.flat_from_sizes(
                "video", video_grid_sizes),
1071
            video_grid_thw=MultiModalFieldConfig.batched("video"),
1072
        )
1073
1074


1075
1076
1077
@MULTIMODAL_REGISTRY.register_processor(Qwen2VLMultiModalProcessor,
                                        info=Qwen2VLProcessingInfo,
                                        dummy_inputs=Qwen2VLDummyInputsBuilder)
1078
class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
                                      SupportsLoRA, SupportsPP):
    packed_modules_mapping = {
        "qkv_proj": [
            "q_proj",
            "k_proj",
            "v_proj",
        ],
        "gate_up_proj": [
            "gate_proj",
            "up_proj",
        ],
    }
1091

1092
1093
1094
1095
1096
1097
    # LoRA specific attributes
    supported_lora_modules = [
        "qkv_proj",
        "o_proj",
        "gate_up_proj",
        "down_proj",
1098
1099
1100
1101
1102
1103
1104
1105
        # vision tower
        "qkv",
        "attn.proj",  # Distinguish patch_embed.proj
        "fc1",
        "fc2",
        # projector
        "mlp.0",
        "mlp.2"
1106
1107
1108
    ]
    embedding_modules = {}
    embedding_padding_modules = []
1109

1110
1111
1112
1113
1114
    # To ensure correct weight loading and mapping.
    hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={
        "lm_head.": "language_model.lm_head.",
        "model.": "language_model.model.",
    })
1115

1116
    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1117
        super().__init__()
1118
        config: Qwen2VLConfig = vllm_config.model_config.hf_config
1119
1120
        quant_config = vllm_config.quant_config
        multimodal_config = vllm_config.model_config.multimodal_config
1121
1122
1123
1124
1125
1126
1127

        self.config = config
        self.multimodal_config = multimodal_config

        self.visual = Qwen2VisionTransformer(
            config.vision_config,
            norm_eps=getattr(config, "rms_norm_eps", 1e-6),
1128
            quant_config=self._maybe_ignore_quant_config(quant_config),
1129
            prefix=maybe_prefix(prefix, "visual"),
1130
1131
        )

1132
1133
1134
1135
1136
        self.language_model = init_vllm_registered_model(
            vllm_config=vllm_config,
            prefix=maybe_prefix(prefix, "language_model"),
            architectures=["Qwen2ForCausalLM"],
        )
1137

1138
        self.make_empty_intermediate_tensors = (
1139
            self.language_model.make_empty_intermediate_tensors)
1140

1141
1142
1143
1144
    @cached_property
    def sampler(self):
        if hasattr(self.language_model, "sampler"):
            return self.language_model.sampler
zhuwenwen's avatar
zhuwenwen committed
1145
        
1146
        return get_sampler()
1147

1148
1149
1150
1151
1152
1153
1154
    def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
        # GPTQ configs do not have a list of ignored modules, however AutoGPTQ
        # seems to avoid vision encoder sections for some models.
        # See: https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int4
        if isinstance(quant_config, (GPTQConfig, GPTQMarlinConfig)):
            return None
        return quant_config
zhuwenwen's avatar
zhuwenwen committed
1155
    
1156

1157
    def _validate_and_reshape_mm_tensor(self, mm_input: object,
1158
1159
1160
1161
1162
1163
1164
1165
1166
                                        name: str) -> torch.Tensor:
        if not isinstance(mm_input, (torch.Tensor, list)):
            raise ValueError(f"Incorrect type of {name}. "
                             f"Got type: {type(mm_input)}")
        if isinstance(mm_input, torch.Tensor):
            if mm_input.ndim == 2:
                return mm_input
            if mm_input.ndim != 3:
                raise ValueError(f"{name} should be 2D or batched 3D tensor. "
1167
1168
                                 f"Got ndim: {mm_input.ndim} "
                                 f"(shape={mm_input.shape})")
1169
1170
1171
1172
1173
1174
1175
            return torch.concat(list(mm_input))
        else:
            return torch.concat(mm_input)

    def _parse_and_validate_image_input(
            self, **kwargs: object) -> Optional[Qwen2VLImageInputs]:
        pixel_values = kwargs.pop("pixel_values", None)
1176
        image_embeds = kwargs.pop("image_embeds", None)
1177
1178
        image_grid_thw = kwargs.pop("image_grid_thw", None)

1179
        if pixel_values is None and image_embeds is None:
1180
1181
            return None

1182
1183
1184
1185
1186
        if pixel_values is not None:
            pixel_values = self._validate_and_reshape_mm_tensor(
                pixel_values, "image pixel values")
            image_grid_thw = self._validate_and_reshape_mm_tensor(
                image_grid_thw, "image grid_thw")
1187

1188
1189
1190
            if not isinstance(pixel_values, (torch.Tensor, list)):
                raise ValueError("Incorrect type of image pixel values. "
                                 f"Got type: {type(pixel_values)}")
1191

1192
            return Qwen2VLImagePixelInputs(type="pixel_values",
1193
                                           pixel_values=pixel_values,
1194
1195
1196
                                           image_grid_thw=image_grid_thw)

        if image_embeds is not None:
1197
1198
            image_embeds = self._validate_and_reshape_mm_tensor(
                image_embeds, "image embeds")
1199
1200
            image_grid_thw = self._validate_and_reshape_mm_tensor(
                image_grid_thw, "image grid_thw")
1201

1202
1203
1204
1205
            if not isinstance(image_embeds, torch.Tensor):
                raise ValueError("Incorrect type of image embeddings. "
                                 f"Got type: {type(image_embeds)}")
            return Qwen2VLImageEmbeddingInputs(type="image_embeds",
1206
1207
                                               image_embeds=image_embeds,
                                               image_grid_thw=image_grid_thw)
1208
1209
1210
1211

    def _parse_and_validate_video_input(
            self, **kwargs: object) -> Optional[Qwen2VLVideoInputs]:
        pixel_values_videos = kwargs.pop("pixel_values_videos", None)
1212
        video_embeds = kwargs.pop("video_embeds", None)
1213
1214
        video_grid_thw = kwargs.pop("video_grid_thw", None)

1215
        if pixel_values_videos is None and video_embeds is None:
1216
1217
            return None

1218
1219
1220
1221
1222
        if pixel_values_videos is not None:
            pixel_values_videos = self._validate_and_reshape_mm_tensor(
                pixel_values_videos, "video pixel values")
            video_grid_thw = self._validate_and_reshape_mm_tensor(
                video_grid_thw, "video grid_thw")
1223

1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
            return Qwen2VLVideoPixelInputs(
                type="pixel_values_videos",
                pixel_values_videos=pixel_values_videos,
                video_grid_thw=video_grid_thw,
            )

        if video_embeds is not None:
            video_embeds = self._validate_and_reshape_mm_tensor(
                video_embeds, "video embeds")
            video_grid_thw = self._validate_and_reshape_mm_tensor(
                video_grid_thw, "video grid_thw")

            if not isinstance(video_embeds, torch.Tensor):
                raise ValueError("Incorrect type of video embeddings. "
                                 f"Got type: {type(video_embeds)}")
            return Qwen2VLVideoEmbeddingInputs(type="video_embeds",
                                               video_embeds=video_embeds,
                                               video_grid_thw=video_grid_thw)
1242

1243
1244
1245
1246
1247
1248
    def _process_image_input(
            self, image_input: Qwen2VLImageInputs) -> tuple[torch.Tensor, ...]:

        grid_thw = image_input["image_grid_thw"]
        assert grid_thw.ndim == 2

1249
        if image_input["type"] == "image_embeds":
1250
1251
1252
1253
1254
1255
1256
1257
            image_embeds = image_input["image_embeds"].type(self.visual.dtype)
        else:
            pixel_values = image_input["pixel_values"].type(self.visual.dtype)
            image_embeds = self.visual(pixel_values, grid_thw=grid_thw)

        # Split concatenated embeddings for each image item.
        merge_size = self.visual.spatial_merge_size
        sizes = grid_thw.prod(-1) // merge_size // merge_size
1258

1259
1260
1261
1262
        return image_embeds.split(sizes.tolist())

    def _process_video_input(
            self, video_input: Qwen2VLVideoInputs) -> tuple[torch.Tensor, ...]:
1263

1264
1265
        grid_thw = video_input["video_grid_thw"]
        assert grid_thw.ndim == 2
1266

1267
        if video_input["type"] == "video_embeds":
1268
1269
1270
1271
1272
            video_embeds = video_input["video_embeds"].type(self.visual.dtype)
        else:
            pixel_values_videos = video_input["pixel_values_videos"].type(
                self.visual.dtype)
            video_embeds = self.visual(pixel_values_videos, grid_thw=grid_thw)
1273

1274
1275
1276
        # Split concatenated embeddings for each video item.
        merge_size = self.visual.spatial_merge_size
        sizes = grid_thw.prod(-1) // merge_size // merge_size
1277

1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
        return video_embeds.split(sizes.tolist())

    def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
        modalities = {}

        # Preserve the order of modalities if there are multiple of them
        # from the order of kwargs.
        for input_key in kwargs:
            if input_key in ("pixel_values",
                             "image_embeds") and "images" not in modalities:
                modalities["images"] = self._parse_and_validate_image_input(
                    **kwargs)
            if input_key in ("pixel_values_videos",
                             "video_embeds") and "videos" not in modalities:
                modalities["videos"] = self._parse_and_validate_video_input(
                    **kwargs)

        return modalities
1296

1297
    def get_multimodal_embeddings(
1298
            self, **kwargs) -> Optional[tuple[torch.Tensor, ...]]:
1299

1300
1301
        modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
        if not modalities:
1302
1303
            return None

1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
        # The result multimodal_embeddings is tuple of tensors, with each
        # tensor correspoending to a multimodal data item (image or video).
        multimodal_embeddings: tuple[torch.Tensor, ...] = ()

        # NOTE: It is important to iterate over the keys in this dictionary
        # to preserve the order of the modalities.
        for modality in modalities:
            if modality == "images":
                image_input = modalities["images"]
                vision_embeddings = self._process_image_input(image_input)
                multimodal_embeddings += vision_embeddings
            if modality == "videos":
                video_input = modalities["videos"]
                video_embeddings = self._process_video_input(video_input)
                multimodal_embeddings += video_embeddings
1319
1320
1321
1322
1323
1324

        return multimodal_embeddings

    def get_input_embeddings(
        self,
        input_ids: torch.Tensor,
1325
        multimodal_embeddings: Optional[tuple[torch.Tensor, ...]] = None,
1326
    ) -> torch.Tensor:
1327
        inputs_embeds = self.language_model.get_input_embeddings(input_ids)
1328
        if multimodal_embeddings is not None:
1329
1330
1331
            inputs_embeds = merge_multimodal_embeddings(
                input_ids, inputs_embeds, multimodal_embeddings,
                [self.config.image_token_id, self.config.video_token_id])
1332
1333
        return inputs_embeds

1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
    def get_input_embeddings_v0(
        self,
        input_ids: torch.Tensor,
        image_input: Optional[tuple[torch.Tensor, ...]] = None,
        video_input: Optional[tuple[torch.Tensor, ...]] = None,
    ) -> torch.Tensor:

        inputs_embeds = self.get_input_embeddings(input_ids)
        if image_input is not None:
            image_embeds = self._process_image_input(image_input)
            inputs_embeds = merge_multimodal_embeddings(
                input_ids,
                inputs_embeds,
                image_embeds,
                placeholder_token_id=self.config.image_token_id,
            )

        if video_input is not None:
            video_embeds = self._process_video_input(video_input)
            inputs_embeds = merge_multimodal_embeddings(
                input_ids,
                inputs_embeds,
                video_embeds,
                placeholder_token_id=self.config.video_token_id,
            )
1359
1360
        return inputs_embeds

1361
1362
1363
1364
1365
1366
1367
    def forward(
        self,
        input_ids: torch.Tensor,
        positions: torch.Tensor,
        kv_caches: List[torch.Tensor],
        attn_metadata: AttentionMetadata,
        intermediate_tensors: Optional[IntermediateTensors] = None,
1368
        inputs_embeds: Optional[torch.Tensor] = None,
1369
        **kwargs: object,
1370
    ) -> Union[torch.Tensor, IntermediateTensors]:
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
        """Run forward pass for Qwen2-VL.

        Args:
            input_ids: Flattened (concatenated) input_ids corresponding to a
                batch.
            positions: Flattened (concatenated) position ids corresponding to a
                batch.
                **NOTE**: If mrope is enabled (default setting for Qwen2-VL
                opensource models), the shape will be `(3, seq_len)`,
                otherwise it will be `(seq_len,).
            pixel_values: Pixel values to be fed to a model.
                `None` if no images are passed.
            image_grid_thw: Tensor `(n_images, 3)` of image 3D grid in LLM.
                `None` if no images are passed.
            pixel_values_videos: Pixel values of videos to be fed to a model.
                `None` if no videos are passed.
            video_grid_thw: Tensor `(n_videos, 3)` of video 3D grid in LLM.
                `None` if no videos are passed.
        """

1391
        if intermediate_tensors is not None:
1392
            inputs_embeds = None
1393

1394
1395
1396
        # NOTE: In v1, inputs_embeds is always generated at model runner from
        # `get_multimodal_embeddings` and `get_input_embeddings`, this
        # condition is only for v0 compatibility.
1397
        elif inputs_embeds is None:
1398
1399
            image_input = self._parse_and_validate_image_input(**kwargs)
            video_input = self._parse_and_validate_video_input(**kwargs)
1400

1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
            if image_input is None and video_input is None:
                inputs_embeds = None
            else:
                if uses_mrope(self.config):
                    assert positions.ndim == 2 and positions.size(0) == 3, (
                        "multimodal section rotary embedding requires "
                        f"(3, seq_len) positions, but got {positions.size()}")
                inputs_embeds = self.get_input_embeddings_v0(
                    input_ids,
                    image_input=image_input,
                    video_input=video_input)
                input_ids = None
1413

1414
        hidden_states = self.language_model.model(
1415
1416
1417
1418
            input_ids=input_ids,
            positions=positions,
            kv_caches=kv_caches,
            attn_metadata=attn_metadata,
1419
            intermediate_tensors=intermediate_tensors,
1420
1421
1422
1423
            inputs_embeds=inputs_embeds,
        )
        return hidden_states

1424
1425
1426
1427
1428
1429
1430
    def compute_logits(
        self,
        hidden_states: torch.Tensor,
        sampling_metadata: SamplingMetadata,
    ) -> Optional[torch.Tensor]:
        return self.language_model.compute_logits(hidden_states,
                                                  sampling_metadata)
1431
1432
1433
1434
1435
1436

    def sample(
        self,
        logits: torch.Tensor,
        sampling_metadata: SamplingMetadata,
    ) -> Optional[SamplerOutput]:
1437
        return self.language_model.sample(logits, sampling_metadata)
1438

1439
1440
    def load_weights(self, weights: Iterable[Tuple[str,
                                                   torch.Tensor]]) -> Set[str]:
1441
1442

        loader = AutoWeightsLoader(self)
1443
        return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1444
1445
1446
1447
1448
1449
1450
1451
1452

    def get_mm_mapping(self) -> MultiModelKeys:
        """
        Get the module prefix in multimodal models
        """
        return MultiModelKeys.from_string_field(
            language_model="language_model",
            connector="visual.",
            tower_model="visual.merger.")