qwen.py 21.7 KB
Newer Older
1
2
# SPDX-License-Identifier: Apache-2.0

Qing's avatar
Qing committed
3
4
5
6
# Adapted from
# https://huggingface.co/Qwen/Qwen-7B/blob/main/modeling_qwen.py
# Copyright (c) Alibaba Cloud.
# LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE
Woosuk Kwon's avatar
Woosuk Kwon committed
7
"""Inference-only QWen model compatible with HuggingFace weights."""
Qing's avatar
Qing committed
8

9
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
10

11
12
import torch
from torch import nn
13
from transformers import PretrainedConfig
Qing's avatar
Qing committed
14

gaoqiong's avatar
gaoqiong committed
15
16
17
import os
import re

18
from vllm.attention import Attention, AttentionMetadata
19
from vllm.compilation.decorators import support_torch_compile
20
from vllm.config import CacheConfig, VllmConfig
21
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
22
from vllm.model_executor.layers.activation import SiluAndMul
23
from vllm.model_executor.layers.layernorm import RMSNorm
24
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
25
26
                                               QKVParallelLinear,
                                               RowParallelLinear)
27
from vllm.model_executor.layers.logits_processor import LogitsProcessor
28
from vllm.model_executor.layers.quantization import QuantizationConfig
29
from vllm.model_executor.layers.rotary_embedding import get_rope
Joe Runde's avatar
Joe Runde committed
30
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
31
from vllm.model_executor.layers.vocab_parallel_embedding import (
32
    ParallelLMHead, VocabParallelEmbedding)
33
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
34
from vllm.model_executor.sampling_metadata import SamplingMetadata
35
from vllm.sequence import IntermediateTensors
Qing's avatar
Qing committed
36

37
38
from .interfaces import SupportsLoRA, SupportsPP
from .utils import (is_pp_missing_parameter,
39
                    make_empty_intermediate_tensors_factory, make_layers,
40
                    maybe_prefix)
gaoqiong's avatar
gaoqiong committed
41
from vllm import _custom_ops as ops
42
from vllm.model_executor.utils import pad_weight, gemm_bank_conf
zhuwenwen's avatar
zhuwenwen committed
43
from vllm.utils import W8a8GetCacheJSON
44
45


46
class QWenMLP(nn.Module):
47
48
    """MLP for the language component of the Qwen model, which contains a
    MergedColumnParallelLinear merging 2 outputs via silu activation."""
49
50
51
52
53
54

    def __init__(
        self,
        hidden_size: int,
        intermediate_size: int,
        hidden_act: str = "silu",
55
        quant_config: Optional[QuantizationConfig] = None,
56
57
58
59
60
    ):
        super().__init__()
        self.gate_up_proj = MergedColumnParallelLinear(
            hidden_size, [intermediate_size] * 2,
            bias=False,
61
            quant_config=quant_config)
62
63
64
        self.c_proj = RowParallelLinear(intermediate_size,
                                        hidden_size,
                                        bias=False,
65
                                        quant_config=quant_config)
66
67
68
69
70
        if hidden_act != "silu":
            raise ValueError(f"Unsupported activation: {hidden_act}. "
                             "Only silu is supported for now.")
        self.act_fn = SiluAndMul()

71
    def forward(self, x: torch.Tensor) -> torch.Tensor:
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
        gate_up, _ = self.gate_up_proj(x)
        x = self.act_fn(gate_up)
        x, _ = self.c_proj(x)
        return x


class QWenAttention(nn.Module):

    def __init__(
        self,
        hidden_size: int,
        num_heads: int,
        max_position_embeddings: int,
        rope_theta: float = 10000,
        rope_scaling: Optional[Dict[str, Any]] = None,
87
        cache_config: Optional[CacheConfig] = None,
88
        quant_config: Optional[QuantizationConfig] = None,
89
        prefix: str = "",
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
    ):
        super().__init__()
        self.hidden_size = hidden_size
        tensor_model_parallel_world_size = get_tensor_model_parallel_world_size(
        )
        self.total_num_heads = num_heads
        assert self.total_num_heads % tensor_model_parallel_world_size == 0
        self.num_heads = (self.total_num_heads //
                          tensor_model_parallel_world_size)
        self.head_dim = hidden_size // self.total_num_heads
        self.c_attn = QKVParallelLinear(
            hidden_size,
            self.head_dim,
            self.total_num_heads,
            bias=True,
105
            quant_config=quant_config,
106
107
108
109
110
        )
        self.c_proj = RowParallelLinear(
            self.total_num_heads * self.head_dim,
            hidden_size,
            bias=False,
111
            quant_config=quant_config,
112
113
114
115
116
117
118
119
120
121
        )
        self.scaling = self.head_dim**-0.5

        self.rotary_emb = get_rope(
            self.head_dim,
            rotary_dim=self.head_dim,
            max_position=max_position_embeddings,
            base=rope_theta,
            rope_scaling=rope_scaling,
        )
122
123
124
        self.attn = Attention(self.num_heads,
                              self.head_dim,
                              self.scaling,
125
                              cache_config=cache_config,
126
127
                              quant_config=quant_config,
                              prefix=f"{prefix}.attn")
128
129
130
131
        self.quant_method = None
        if quant_config is not None:
            self.quant_method=quant_config.get_name()
            self.quant_config=quant_config
132
133
134
135
136

    def forward(
        self,
        positions: torch.Tensor,
        hidden_states: torch.Tensor,
137
138
        kv_cache: torch.Tensor,
        attn_metadata: AttentionMetadata,
139
140
    ) -> torch.Tensor:
        qkv, _ = self.c_attn(hidden_states)
zhuwenwen's avatar
zhuwenwen committed
141
142
        # if os.environ.get('FA_PAD') == '1' and self.quant_method is None:
        #     qkv = qkv[...,:-32]
143
144
        q, k, v = qkv.chunk(chunks=3, dim=-1)
        q, k = self.rotary_emb(positions, q, k)
145
        attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
146
147
148
149
150
151
152
153
        output, _ = self.c_proj(attn_output)
        return output


class QWenBlock(nn.Module):

    def __init__(
        self,
154
        config: PretrainedConfig,
155
        cache_config: Optional[CacheConfig] = None,
156
        quant_config: Optional[QuantizationConfig] = None,
157
        prefix: str = "",
158
159
160
161
162
163
164
165
166
167
168
    ):
        super().__init__()
        self.ln_1 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)

        rope_theta = getattr(config, "rope_theta", 10000)
        rope_scaling = getattr(config, "rope_scaling", None)
        self.attn = QWenAttention(config.hidden_size,
                                  config.num_attention_heads,
                                  config.max_position_embeddings,
                                  rope_theta=rope_theta,
                                  rope_scaling=rope_scaling,
169
                                  cache_config=cache_config,
170
171
                                  quant_config=quant_config,
                                  prefix=f"{prefix}.attn")
172
173
174
175
176

        self.ln_2 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)

        self.mlp = QWenMLP(config.hidden_size,
                           config.intermediate_size // 2,
177
                           quant_config=quant_config)
178
179
180
181
182

    def forward(
        self,
        positions: torch.Tensor,
        hidden_states: torch.Tensor,
183
184
        kv_cache: torch.Tensor,
        attn_metadata: AttentionMetadata,
185
186
187
188
189
190
191
192
193
194
195
196
        residual: Optional[torch.Tensor],
    ) -> Tuple[torch.Tensor, torch.Tensor]:
        # Self Attention
        if residual is None:
            residual = hidden_states
            hidden_states = self.ln_1(hidden_states)
        else:
            hidden_states, residual = self.ln_1(hidden_states, residual)
        hidden_states = self.attn(
            positions=positions,
            hidden_states=hidden_states,
            kv_cache=kv_cache,
197
            attn_metadata=attn_metadata,
198
199
200
201
202
203
204
205
        )

        # Fully Connected
        hidden_states, residual = self.ln_2(hidden_states, residual)
        hidden_states = self.mlp(hidden_states)
        return hidden_states, residual


206
@support_torch_compile
207
class QWenModel(nn.Module):
Qing's avatar
Qing committed
208

209
    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
210
        super().__init__()
211
212
213
214
215

        config = vllm_config.model_config.hf_config
        cache_config = vllm_config.cache_config
        quant_config = vllm_config.quant_config

216
217
218
219
220
221
222
        self.config = config
        self.vocab_size = config.vocab_size

        self.wte = VocabParallelEmbedding(
            config.vocab_size,
            config.hidden_size,
        )
223
224
        self.start_layer, self.end_layer, self.h = make_layers(
            config.num_hidden_layers,
225
226
            lambda prefix: QWenBlock(
                config, cache_config, quant_config, prefix=prefix),
227
            prefix=f"{prefix}.h")
228
        self.ln_f = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
229
230
231
        self.make_empty_intermediate_tensors = (
            make_empty_intermediate_tensors_factory(
                ["hidden_states", "residual"], config.hidden_size))
232

233
234
235
    def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
        return self.wte(input_ids)

236
237
238
239
    def forward(
        self,
        input_ids: torch.Tensor,
        positions: torch.Tensor,
240
241
        kv_caches: List[torch.Tensor],
        attn_metadata: AttentionMetadata,
242
        intermediate_tensors: Optional[IntermediateTensors],
243
        inputs_embeds: Optional[torch.Tensor] = None,
244
    ) -> Union[torch.Tensor, IntermediateTensors]:
245
        if get_pp_group().is_first_rank:
246
247
248
249
            if inputs_embeds is not None:
                hidden_states = inputs_embeds
            else:
                hidden_states = self.get_input_embeddings(input_ids)
250
251
252
253
254
            residual = None
        else:
            assert intermediate_tensors is not None
            hidden_states = intermediate_tensors["hidden_states"]
            residual = intermediate_tensors["residual"]
255

256
        for i in range(self.start_layer, self.end_layer):
257
258
259
260
            layer = self.h[i]
            hidden_states, residual = layer(
                positions,
                hidden_states,
261
                kv_caches[i - self.start_layer],
262
                attn_metadata,
263
264
                residual,
            )
265
266
267
268
269
        if not get_pp_group().is_last_rank:
            return IntermediateTensors({
                "hidden_states": hidden_states,
                "residual": residual
            })
270
271
272
273
        hidden_states, _ = self.ln_f(hidden_states, residual)
        return hidden_states


274
class QWenBaseModel(nn.Module):
275
276
277

    def __init__(
        self,
278
279
280
281
        *,
        vllm_config: VllmConfig,
        prefix: str = "",
        transformer_type: type[QWenModel] = QWenModel,
282
283
    ) -> None:
        super().__init__()
284
285
286
        config = vllm_config.model_config.hf_config
        quant_config = vllm_config.quant_config
        multimodal_config = vllm_config.model_config.multimodal_config
287
        self.config = config
288
        self.multimodal_config = multimodal_config
289
        self.quant_config = quant_config
290
291
292
        self.transformer = transformer_type(vllm_config=vllm_config,
                                            prefix=maybe_prefix(
                                                prefix, "transformer"))
293
294
295
        self.lm_head = ParallelLMHead(config.vocab_size,
                                      config.hidden_size,
                                      quant_config=quant_config)
296
297
        if self.config.tie_word_embeddings:
            self.lm_head.weight = self.transformer.wte.weight
298
        self.logits_processor = LogitsProcessor(config.vocab_size)
Joe Runde's avatar
Joe Runde committed
299
        self.sampler = get_sampler()
300
301
        self.make_empty_intermediate_tensors = (
            self.transformer.make_empty_intermediate_tensors)
gaoqiong's avatar
gaoqiong committed
302
        
303
        self.quant_method = None
gaoqiong's avatar
gaoqiong committed
304
305
306
        if quant_config is not None:
            self.quant_method=quant_config.get_name()
            self.quant_config=quant_config
zhuwenwen's avatar
zhuwenwen committed
307
        self.tritonsingleton= W8a8GetCacheJSON()
gaoqiong's avatar
gaoqiong committed
308
              
gaoqiong's avatar
gaoqiong committed
309
        self.use_llama_nn = os.environ.get('LLAMA_NN') == '1'
310
311
        self.use_gemm_pad = os.environ.get('GEMM_PAD') == '1'
        self.use_fa_pad = os.environ.get('FA_PAD') == '1'
zhuwenwen's avatar
zhuwenwen committed
312
        self.use_awq_pad = os.environ.get('AWQ_PAD') == '1'
zhuwenwen's avatar
zhuwenwen committed
313
        self.w8a8_strategy=int(os.getenv('W8A8_SUPPORT_METHODS', '1'))
314

315
316
317
318
319
    def compute_logits(
        self,
        hidden_states: torch.Tensor,
        sampling_metadata: SamplingMetadata,
    ) -> Optional[torch.Tensor]:
320
        logits = self.logits_processor(self.lm_head, hidden_states,
321
322
323
                                       sampling_metadata)
        return logits

324
325
    def sample(
        self,
326
        logits: torch.Tensor,
327
328
        sampling_metadata: SamplingMetadata,
    ) -> Optional[SamplerOutput]:
329
        next_tokens = self.sampler(logits, sampling_metadata)
330
        return next_tokens
Qing's avatar
Qing committed
331

332
333
    def load_weights(self, weights: Iterable[Tuple[str,
                                                   torch.Tensor]]) -> Set[str]:
334
335
336
337
338
339
        stacked_params_mapping = [
            # (param_name, shard_name, shard_id)
            ("gate_up_proj", "w2", 0),
            ("gate_up_proj", "w1", 1),
        ]
        params_dict = dict(self.named_parameters())
340
        loaded_params: Set[str] = set()
341
        for name, loaded_weight in weights:
Qing's avatar
Qing committed
342
343
            if "rotary_emb.inv_freq" in name:
                continue
344
            for (param_name, weight_name, shard_id) in stacked_params_mapping:
Qing's avatar
Qing committed
345
346
                if weight_name not in name:
                    continue
CHU Tianxiang's avatar
CHU Tianxiang committed
347
348
349
350
                name = name.replace(weight_name, param_name)
                # Skip loading extra bias for GPTQ models.
                if name.endswith(".bias") and name not in params_dict:
                    continue
351
352
353
                # Skip layers on other devices.
                if is_pp_missing_parameter(name, self):
                    continue
CHU Tianxiang's avatar
CHU Tianxiang committed
354
                param = params_dict[name]
355
356
                weight_loader = param.weight_loader
                weight_loader(param, loaded_weight, shard_id)
Qing's avatar
Qing committed
357
                break
358
            else:
CHU Tianxiang's avatar
CHU Tianxiang committed
359
360
361
                # Skip loading extra bias for GPTQ models.
                if name.endswith(".bias") and name not in params_dict:
                    continue
362
363
364
                # Skip layers on other devices.
                if is_pp_missing_parameter(name, self):
                    continue
365
366
367
368
                param = params_dict[name]
                weight_loader = getattr(param, "weight_loader",
                                        default_weight_loader)
                weight_loader(param, loaded_weight)
369
            loaded_params.add(name)
zhuwenwen's avatar
zhuwenwen committed
370

371
        if self.use_llama_nn and self.quant_method is None :
gaoqiong's avatar
gaoqiong committed
372
            lay_key_words = [
zhuwenwen's avatar
zhuwenwen committed
373
374
                "attn.c_attn.weight",
                "attn.c_proj.weight",
gaoqiong's avatar
gaoqiong committed
375
                "mlp.gate_up_proj.weight",
376
377
                "mlp.c_proj.weight",
                "lm_head.weight"
gaoqiong's avatar
gaoqiong committed
378
379
380
            ]
            combined_words = "|".join(lay_key_words)
            
zhuwenwen's avatar
zhuwenwen committed
381
382
            # lay_qkv_words = ["attn.c_attn.weight"]   
            # qkv_words = "|".join(lay_qkv_words)  
zhuwenwen's avatar
zhuwenwen committed
383
            
zhuwenwen's avatar
zhuwenwen committed
384
385
            # lay_qkv_bias_words = ["attn.c_attn.bias"]   
            # qkv_bias_words = "|".join(lay_qkv_bias_words) 
zhuwenwen's avatar
zhuwenwen committed
386
                      
zhuwenwen's avatar
zhuwenwen committed
387
388
            for layername in loaded_params:
                weight = params_dict[layername]
zhuwenwen's avatar
zhuwenwen committed
389
390
                # if self.use_fa_pad and (re.findall(qkv_bias_words, layername)):
                #     weight.data = pad_weight(weight.data, 32)
zhuwenwen's avatar
zhuwenwen committed
391
                
gaoqiong's avatar
gaoqiong committed
392
                matches = re.findall(combined_words, layername)
zhuwenwen's avatar
zhuwenwen committed
393
                if matches:         
zhuwenwen's avatar
zhuwenwen committed
394
395
                    # if self.use_gemm_pad and gemm_bank_conf(weight.data.shape[0]):
                    #     weight.data = pad_weight(weight.data, 32)  
396
                        
zhuwenwen's avatar
zhuwenwen committed
397
398
399
                    # if self.use_fa_pad and (re.findall(qkv_words, layername)):
                    #     if not gemm_bank_conf(weight.data.shape[0]):
                    #         weight.data = pad_weight(weight.data, 32)
400
                        
gaoqiong's avatar
gaoqiong committed
401
402
403
                    _weight = torch.zeros_like(weight.data)
                    ori_shape =_weight.shape
                    
zhuwenwen's avatar
zhuwenwen committed
404
                    ops.trans_w16_gemm(_weight, weight.data, _weight.shape[0], _weight.shape[1])
gaoqiong's avatar
gaoqiong committed
405
406
407
408
                    weight.data.copy_(_weight)
                    
                    weight.data=weight.data.reshape(ori_shape[1],-1)
                    
zhuwenwen's avatar
zhuwenwen committed
409
        if self.quant_method == "awq":
zhuwenwen's avatar
zhuwenwen committed
410
            os.environ['LM_NN'] = '0'
gaoqiong's avatar
gaoqiong committed
411
412
413
414
415
416
417
418
            lay_key_words = [
                "attn.c_attn.qweight",
                "attn.c_proj.qweight",
                "mlp.gate_up_proj.qweight",
                "mlp.c_proj.qweight"
            ]
            combined_words = "|".join(lay_key_words)
            
zhuwenwen's avatar
zhuwenwen committed
419
420
            for layername in loaded_params:
                weight = params_dict[layername]
gaoqiong's avatar
gaoqiong committed
421
422
423
424
425
426
427
428
429
430
431
432
433
434
                
                matches = re.findall(combined_words, layername)
                if matches:
                    qweight =params_dict[layername]
                    qzeros=params_dict[layername.replace("qweight", "qzeros")]
                    scales=params_dict[layername.replace("qweight", "scales")]
                    zeros_and_scalse =params_dict[layername.replace("qweight", "zeros_and_scales")]
                    
                    group_size= self.quant_config.group_size 
                   
                    dim_n = scales.data.shape[1]
                    dim_k = qweight.data.shape[0]
                    pad_group=2              
                    
gaoqiong's avatar
gaoqiong committed
435
                    _qw, _sz=ops.convert_s4(qweight,qzeros,scales,int(group_size)) 
gaoqiong's avatar
gaoqiong committed
436
                    
gaoqiong's avatar
gaoqiong committed
437
                    sz = ops.sz_permute(_sz).reshape(-1,dim_n)       
gaoqiong's avatar
gaoqiong committed
438
439
440
441
442
443
444
445
                    
                    zeros_and_scalse.data.copy_(sz)
                    qweight.data.copy_(_qw)
                    
                    #reshape
                    zeros_and_scalse.data=zeros_and_scalse.reshape(dim_n,-1)    #[k/greop_size,n]------>[n,k/group_size]
                    qweight.data=qweight.data.reshape(dim_n,-1)                      #[k,n/8]---->[n,k/8]  
                
zhuwenwen's avatar
zhuwenwen committed
446
                    if dim_k % 4096==0 and self.use_awq_pad:
gaoqiong's avatar
gaoqiong committed
447
448
449
450
                        zeros_and_scalse_pad= torch.zeros(dim_n,pad_group,dtype=torch.int32).cuda()
                        zeros_and_scalse.data=torch.cat((zeros_and_scalse.data,zeros_and_scalse_pad),dim=1).contiguous()
                        qweight_pad= torch.zeros(dim_n,int(group_size//4),dtype=torch.int32).cuda()
                        qweight.data=torch.cat((qweight.data,qweight_pad),dim=1).contiguous()
zhuwenwen's avatar
zhuwenwen committed
451
452
                       
        if self.quant_method == "compressed_tensors":
453
            os.environ['LM_NN'] = '0'
zhuwenwen's avatar
zhuwenwen committed
454
455
456
457
458
459
460
            lay_key_words = [
                "attn.c_attn.weight",
                "attn.c_proj.weight",
                "mlp.gate_up_proj.weight",
                "mlp.c_proj.weight",
            ]
            combined_words = "|".join(lay_key_words)
gaoqiong's avatar
gaoqiong committed
461
462
            weight_shapes=[]
            all_json={}
463
            matched_key_words=set()
zhuwenwen's avatar
zhuwenwen committed
464
            
zhuwenwen's avatar
zhuwenwen committed
465
466
            for layername in loaded_params:
                weight = params_dict[layername] 
zhuwenwen's avatar
zhuwenwen committed
467
                matches = re.findall(combined_words, layername)
gaoqiong's avatar
gaoqiong committed
468
                if matches and "scale" not in layername:
zhuwenwen's avatar
zhuwenwen committed
469
                    weight_data =params_dict[layername]
gaoqiong's avatar
gaoqiong committed
470
471
472
473
474
475
476
477
                    n=weight_data.shape[0]
                    
                    #rocblas和cutlass目前都需要weight做处理,但是triton不用
                    if self.w8a8_strategy!=1:
                        _weight=weight_data.T.contiguous().reshape(n,-1)
                        weight_data.data.copy_(_weight)  
                    
                    #下面是针对模型记录模型出现k和n值 
478
479
                    elif len(matched_key_words) < 4 and matches[0] not in matched_key_words:
                        matched_key_words.add(matches[0]) 
gaoqiong's avatar
gaoqiong committed
480
481
482
483
484
                        k=weight_data.shape[1]
                        weight_shapes.append({n,k})
                
                        json_file=self.tritonsingleton.get_w8a8json_name(n,k)
                        configs_dict=self.tritonsingleton.get_triton_cache(json_file,n,k)
gaoqiong's avatar
gaoqiong committed
485
486
                        if configs_dict:
                            all_json.update(configs_dict)
gaoqiong's avatar
gaoqiong committed
487
488
489
490
491
492
493
494
                                              
            if self.w8a8_strategy==1:
                self.tritonsingleton.triton_json_dict.append(all_json)
                #找到的所有config都进行一次warmup
                for key, value in all_json.items():
                    m=int(key.split('_')[0])
                    n=int(key.split('_')[1])
                    k=int(key.split('_')[2])
zhuwenwen's avatar
zhuwenwen committed
495
                    ops.triton_int8_gemm_helper(m=m,n=n,k=k,per_token_act_quant=True,per_out_channel_weight_quant=True,use_bias=False,best_config=value)
zhuwenwen's avatar
zhuwenwen committed
496
            
497
        return loaded_params
498
499


500
class QWenLMHeadModel(QWenBaseModel, SupportsPP, SupportsLoRA):
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
    packed_modules_mapping = {
        "c_attn": ["c_attn"],
        "gate_up_proj": [
            "w2",
            "w1",
        ],
    }
    # LoRA specific attributes
    supported_lora_modules = [
        "c_attn",
        "gate_up_proj",
        "c_proj",
    ]

    embedding_modules = {}
    embedding_padding_modules = []

518
    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
519
        config = vllm_config.model_config.hf_config
520
521
522
523
524
525
526
527
528
529
530
        if hasattr(config, "visual"):
            hf_overrides = {
                "architectures": ["QwenVLForConditionalGeneration"]
            }
            raise RuntimeError(
                "The configuration of this model indicates that it supports "
                "vision inputs, but you instantiated the text-only version "
                "of this model. Please use the vision model by setting "
                f"`--hf-overrides {hf_overrides!r}`")

        super().__init__(vllm_config=vllm_config, prefix=prefix)
531

532
533
534
535
536
537
538
539
540
541
542
543
544
    def forward(
        self,
        input_ids: torch.Tensor,
        positions: torch.Tensor,
        kv_caches: List[torch.Tensor],
        attn_metadata: AttentionMetadata,
        intermediate_tensors: Optional[IntermediateTensors] = None,
        inputs_embeds: Optional[torch.Tensor] = None,
    ) -> Union[torch.Tensor, IntermediateTensors]:
        hidden_states = self.transformer(input_ids, positions, kv_caches,
                                         attn_metadata, intermediate_tensors,
                                         inputs_embeds)
        return hidden_states