step3_text.py 19.4 KB
Newer Older
Song's avatar
Song committed
1
2
3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only Jurassic model."""
4

Song's avatar
Song committed
5
from collections.abc import Iterable
6
from itertools import islice
7
from typing import Any
Song's avatar
Song committed
8
9
10
11
12
13

import torch
from torch import nn

from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, VllmConfig
14
15
16
17
from vllm.distributed import (
    get_pp_group,
    get_tensor_model_parallel_world_size,
)
Song's avatar
Song committed
18
19
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
20
from vllm.model_executor.layers.attention import Attention
21
22
23
from vllm.model_executor.layers.fused_moe import (
    FusedMoE,
)
Song's avatar
Song committed
24
from vllm.model_executor.layers.layernorm import RMSNorm
25
26
27
28
29
30
from vllm.model_executor.layers.linear import (
    ColumnParallelLinear,
    MergedColumnParallelLinear,
    ReplicatedLinear,
    RowParallelLinear,
)
Song's avatar
Song committed
31
from vllm.model_executor.layers.logits_processor import LogitsProcessor
32
from vllm.model_executor.layers.quantization import QuantizationConfig
Song's avatar
Song committed
33
34
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import (
35
36
37
    ParallelLMHead,
    VocabParallelEmbedding,
)
Song's avatar
Song committed
38
39
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.sequence import IntermediateTensors
40
from vllm.transformers_utils.configs.step3_vl import Step3TextConfig
Song's avatar
Song committed
41
42

from .interfaces import SupportsPP
43
44
45
46
47
48
49
from .utils import (
    PPMissingLayer,
    is_pp_missing_parameter,
    make_empty_intermediate_tensors_factory,
    make_layers,
    maybe_prefix,
)
Song's avatar
Song committed
50
51
52
53
54

logger = init_logger(__name__)


class FusedMoEBlock(nn.Module):
55
56
57
    def __init__(
        self,
        config: ModelConfig,
58
        quant_config: QuantizationConfig | None = None,
59
60
        prefix: str = "",
    ):
Song's avatar
Song committed
61
62
63
64
65
66
        super().__init__()
        self.tp_size = get_tensor_model_parallel_world_size()

        if self.tp_size > config.moe_num_experts:
            raise ValueError(
                f"Tensor parallel size {self.tp_size} is greater than "
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
                f"the number of experts {config.moe_num_experts}."
            )

        self.experts = FusedMoE(
            num_experts=config.moe_num_experts,
            top_k=config.moe_top_k,
            hidden_size=config.hidden_size,
            intermediate_size=config.moe_intermediate_size,
            renormalize=config.norm_expert_weight,
            quant_config=quant_config,
            prefix=f"{prefix}.experts",
        )
        self.gate = ReplicatedLinear(
            config.hidden_size,
            config.moe_num_experts,
            bias=False,
            quant_config=None,
            prefix=f"{prefix}.gate",
        )
Song's avatar
Song committed
86
87
88
89
90
91
92
93

    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
        orig_shape = hidden_states.shape
        hidden_dim = hidden_states.shape[-1]
        hidden_states = hidden_states.view(-1, hidden_dim)

        router_logits, _ = self.gate(hidden_states)

94
95
96
        final_hidden_states = self.experts(
            hidden_states=hidden_states, router_logits=router_logits
        )
Song's avatar
Song committed
97
98
99
100
101
102
103
104
105
106

        return final_hidden_states.view(orig_shape)


class Step3TextMLP(nn.Module):
    def __init__(
        self,
        hidden_size: int,
        intermediate_size: int,
        hidden_act: str,
107
        quant_config: QuantizationConfig | None = None,
Song's avatar
Song committed
108
109
110
111
        prefix: str = "",
    ) -> None:
        super().__init__()
        self.gate_up_proj = MergedColumnParallelLinear(
112
113
            hidden_size,
            [intermediate_size] * 2,
Song's avatar
Song committed
114
115
            bias=False,
            quant_config=quant_config,
116
117
118
119
120
121
122
123
124
            prefix=f"{prefix}.gate_up_proj",
        )
        self.down_proj = RowParallelLinear(
            intermediate_size,
            hidden_size,
            bias=False,
            quant_config=quant_config,
            prefix=f"{prefix}.down_proj",
        )
Song's avatar
Song committed
125
        if hidden_act != "silu":
126
127
128
            raise ValueError(
                f"Unsupported activation: {hidden_act}. Only silu is supported for now."
            )
Song's avatar
Song committed
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
        self.act_fn = SiluAndMul()
        self.hidden_size = hidden_size

    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
        gate_up, _ = self.gate_up_proj(hidden_states)
        intermediate_act = self.act_fn(gate_up)
        output, _ = self.down_proj(intermediate_act)
        return output


class Step3TextAttention(nn.Module):
    def __init__(
        self,
        hidden_size: int,
        num_heads: int,
        num_kv_heads: int,
        norm_eps: float,
146
        rope_parameters: dict[str, Any],
147
        share_q_dim: int | None = None,
Song's avatar
Song committed
148
149
        max_position_embedding: int = 8192,
        head_dim: int = 256,
150
151
        cache_config: CacheConfig | None = None,
        quant_config: QuantizationConfig | None = None,
Song's avatar
Song committed
152
153
154
155
156
157
158
159
160
161
162
        prefix: str = "",
    ):
        super().__init__()
        self.hidden_size = hidden_size
        tp_size = get_tensor_model_parallel_world_size()

        self.total_num_heads = num_heads
        assert self.total_num_heads % tp_size == 0
        self.num_heads = self.total_num_heads // tp_size

        if num_kv_heads != 1:
163
164
165
            raise ValueError(
                f"Step3TextAttention num_kv_heads must be 1, but got {num_kv_heads}."
            )
Song's avatar
Song committed
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
        self.num_kv_heads = num_kv_heads

        self.head_dim = head_dim
        self.kv_size = self.num_kv_heads * self.head_dim
        self.q_size = share_q_dim if share_q_dim else self.head_dim

        self.qkv_proj = ReplicatedLinear(
            hidden_size,
            self.q_size + self.kv_size * 2,
            bias=False,
            quant_config=quant_config,
            prefix=f"{prefix}.qkv_proj",
        )

        self.o_proj = RowParallelLinear(
            self.total_num_heads * self.head_dim,
            hidden_size,
            bias=False,
            quant_config=quant_config,
            prefix=f"{prefix}.o_proj",
        )
        self.inter_norm = RMSNorm(self.q_size, eps=norm_eps)
        self.wq = ColumnParallelLinear(
            self.q_size,
            self.head_dim * self.total_num_heads,
            bias=False,
            quant_config=quant_config,
            prefix=f"{prefix}.wq",
        )
195
196
197
        self.rotary_emb = get_rope(
            self.head_dim,
            max_position=max_position_embedding,
198
            rope_parameters=rope_parameters,
199
        )
Song's avatar
Song committed
200
        scaling = self.head_dim**-0.5
201
202
203
204
205
206
207
208
209
210
211
212
        self.attn = Attention(
            self.num_heads,
            self.head_dim,
            scaling,
            self.num_kv_heads,
            cache_config=cache_config,
            prefix=f"{prefix}.attn",
        )

    def forward(
        self, positions: torch.Tensor, hidden_states: torch.Tensor
    ) -> torch.Tensor:
Song's avatar
Song committed
213
214
215
216
217
218
219
220
221
222
223
        qkv, _ = self.qkv_proj(hidden_states)
        q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
        q = self.inter_norm(q)
        q = self.wq(q)[0]
        q, k = self.rotary_emb(positions, q, k)
        attn_output = self.attn(q, k, v)
        residual, _ = self.o_proj(attn_output)
        return residual


class Step3TextDecoderLayer(nn.Module):
224
225
    def __init__(
        self,
226
        config: Step3TextConfig,
227
228
        cache_config: CacheConfig | None = None,
        quant_config: QuantizationConfig | None = None,
229
230
        prefix: str = "",
    ) -> None:
Song's avatar
Song committed
231
232
233
234
235
236
237
238
239
240
241
242
243
        super().__init__()
        self.hidden_size = config.hidden_size

        self.self_attn = Step3TextAttention(
            hidden_size=self.hidden_size,
            num_heads=config.num_attention_heads,
            num_kv_heads=1,
            cache_config=cache_config,
            quant_config=quant_config,
            norm_eps=config.rms_norm_eps,
            max_position_embedding=config.max_position_embedding,
            head_dim=config.head_dim,
            share_q_dim=config.share_q_dim,
244
            rope_parameters=config.rope_parameters,
245
246
            prefix=f"{prefix}.self_attn",
        )
Song's avatar
Song committed
247
248
249
250

        layer_idx = int(prefix.split("layers.")[1].split(".")[0])
        moe_layers_enum = getattr(config, "moe_layers_enum", None)
        if moe_layers_enum is not None:
251
            moe_layers_idx = [int(i) for i in moe_layers_enum.strip().split(",")]
Song's avatar
Song committed
252
253
254
255
256
        else:
            # Default to 1dense.
            moe_layers_idx = [i for i in range(1, config.num_hidden_layers)]

        if layer_idx in moe_layers_idx:
257
258
259
            self.moe = FusedMoEBlock(
                config=config, quant_config=quant_config, prefix=f"{prefix}.moe"
            )
Song's avatar
Song committed
260
261
262
263
264
            self.share_expert = Step3TextMLP(
                hidden_size=self.hidden_size,
                intermediate_size=config.share_expert_dim,
                hidden_act="silu",
                quant_config=quant_config,
265
266
                prefix=f"{prefix}.share_expert",
            )
Song's avatar
Song committed
267
268
            self.use_moe = True
        else:
269
270
271
272
273
274
275
            self.mlp = Step3TextMLP(
                hidden_size=config.hidden_size,
                intermediate_size=config.intermediate_size,
                hidden_act="silu",
                quant_config=quant_config,
                prefix=f"{prefix}.mlp",
            )
Song's avatar
Song committed
276
            self.use_moe = False
277
278
279
280
        self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
        self.post_attention_layernorm = RMSNorm(
            config.hidden_size, eps=config.rms_norm_eps
        )
Song's avatar
Song committed
281
282

    def forward(
283
284
285
        self,
        positions: torch.Tensor,
        hidden_states: torch.Tensor,
286
        residual: torch.Tensor | None,
Song's avatar
Song committed
287
288
289
290
291
    ) -> tuple[torch.Tensor, torch.Tensor]:
        if residual is None:
            residual = hidden_states
            hidden_states = self.input_layernorm(hidden_states)
        else:
292
            hidden_states, residual = self.input_layernorm(hidden_states, residual)
Song's avatar
Song committed
293
294
295
296
297
298

        hidden_states = self.self_attn(
            positions=positions,
            hidden_states=hidden_states,
        )

299
        hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
Song's avatar
Song committed
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320

        if self.use_moe:
            share_output = self.share_expert(hidden_states)
            moe_output = self.moe(hidden_states)
            hidden_states = share_output + moe_output
        else:
            hidden_states = self.mlp(hidden_states)

        return hidden_states, residual


@support_torch_compile
class Step3TextModel(nn.Module):
    def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
        super().__init__()
        config = vllm_config.model_config.hf_config
        cache_config = vllm_config.cache_config
        quant_config = vllm_config.quant_config
        self.vocab_size = config.vocab_size
        self.config = config

321
322
323
        if get_pp_group().is_first_rank or (
            config.tie_word_embeddings and get_pp_group().is_last_rank
        ):
Song's avatar
Song committed
324
325
326
327
328
329
330
331
332
            self.embed_tokens = VocabParallelEmbedding(
                self.vocab_size,
                config.hidden_size,
            )
        else:
            self.embed_tokens = PPMissingLayer()

        self.start_layer, self.end_layer, self.layers = make_layers(
            config.num_hidden_layers,
333
            lambda prefix: Step3TextDecoderLayer(
334
                config=config,
335
336
337
338
                cache_config=cache_config,
                quant_config=quant_config,
                prefix=prefix,
            ),
Song's avatar
Song committed
339
340
341
342
343
344
345
            prefix=f"{prefix}.layers",
        )
        if get_pp_group().is_last_rank:
            self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
        else:
            self.norm = PPMissingLayer()

346
347
348
        self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
            ["hidden_states"], config.hidden_size
        )
Song's avatar
Song committed
349

350
    def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
Song's avatar
Song committed
351
352
353
354
        return self.embed_tokens(input_ids)

    def forward(
        self,
355
        input_ids: torch.Tensor | None,
Song's avatar
Song committed
356
        positions: torch.Tensor,
357
358
        intermediate_tensors: IntermediateTensors | None = None,
        inputs_embeds: torch.Tensor | None = None,
Song's avatar
Song committed
359
360
361
362
363
    ) -> torch.Tensor:
        if get_pp_group().is_first_rank:
            if inputs_embeds is not None:
                hidden_states = inputs_embeds
            else:
364
                hidden_states = self.embed_input_ids(input_ids)
Song's avatar
Song committed
365
366
367
368
369
370
            residual = None
        else:
            assert intermediate_tensors is not None
            hidden_states = intermediate_tensors["hidden_states"]
            residual = intermediate_tensors["residual"]

371
        for layer in islice(self.layers, self.start_layer, self.end_layer):
Song's avatar
Song committed
372
373
374
            hidden_states, residual = layer(positions, hidden_states, residual)

        if not get_pp_group().is_last_rank:
375
376
377
378
379
380
            return IntermediateTensors(
                {
                    "hidden_states": hidden_states,
                    "residual": residual,
                }
            )
Song's avatar
Song committed
381
382
383
384
385
386
387
388
389
390
391
392
393
394

        hidden_states, _ = self.norm(hidden_states, residual)
        return hidden_states


class Step3TextForCausalLM(nn.Module, SupportsPP):
    def __init__(
        self,
        *,
        vllm_config: VllmConfig,
        prefix: str = "",
    ):
        super().__init__()
        config = vllm_config.model_config.hf_config
395

Song's avatar
Song committed
396
397
398
399
400
401
402
        self.config = config
        self.vllm_config = vllm_config

        self.model = Step3TextModel(vllm_config=vllm_config, prefix=prefix)

        if get_pp_group().is_last_rank:
            self.lm_head = ParallelLMHead(
403
                config.vocab_size,
Song's avatar
Song committed
404
                config.hidden_size,
405
                prefix=maybe_prefix(prefix, "lm_head"),
Song's avatar
Song committed
406
            )
407
            self.logits_processor = LogitsProcessor(config.vocab_size)
Song's avatar
Song committed
408
409
410
411
        else:
            self.lm_head = PPMissingLayer()

        self.make_empty_intermediate_tensors = (
412
413
            self.model.make_empty_intermediate_tensors
        )
Song's avatar
Song committed
414

415
416
    def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
        return self.model.embed_input_ids(input_ids)
417

418
419
    def forward(
        self,
420
        input_ids: torch.Tensor | None,
421
        positions: torch.Tensor,
422
423
        intermediate_tensors: IntermediateTensors | None = None,
        inputs_embeds: torch.Tensor | None = None,
424
425
426
427
    ):
        hidden_states = self.model(
            input_ids, positions, intermediate_tensors, inputs_embeds
        )
Song's avatar
Song committed
428
429
        return hidden_states

430
431
    def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
        logits = self.logits_processor(self.lm_head, hidden_states)
Song's avatar
Song committed
432
433
        return logits

434
    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
Song's avatar
Song committed
435
436
        qkv_params_mapping = [
            # (param_name, shard_name, relative_start_idx, relative_end_idx)
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
            (
                ".qkv_proj",
                ".q_proj",
                0,
                self.config.share_q_dim
                / (self.config.share_q_dim + self.config.head_dim * 2),
            ),
            (
                ".qkv_proj",
                ".k_proj",
                self.config.share_q_dim
                / (self.config.share_q_dim + self.config.head_dim * 2),
                (self.config.share_q_dim + self.config.head_dim)
                / (self.config.share_q_dim + self.config.head_dim * 2),
            ),
            (
                ".qkv_proj",
                ".v_proj",
                (self.config.share_q_dim + self.config.head_dim)
                / (self.config.share_q_dim + self.config.head_dim * 2),
                (self.config.share_q_dim + self.config.head_dim * 2)
                / (self.config.share_q_dim + self.config.head_dim * 2),
            ),
Song's avatar
Song committed
460
461
462
463
464
465
466
467
        ]
        stacked_params_mapping = [
            # (param_name, shard_name, shard_id)
            (".gate_up_proj", ".gate_proj", 0),
            (".gate_up_proj", ".up_proj", 1),
        ]
        params_dict = dict(self.named_parameters())
        loaded_params: set[str] = set()
468
469
470
        base_layer = (
            "base_layer." if any(".base_layer." in name for name in params_dict) else ""
        )
Song's avatar
Song committed
471
472

        expert_params_mapping = [
473
474
475
            (f".moe.experts.{base_layer}w13_weight", ".moe.gate_proj.weight", "w1"),
            (f".moe.experts.{base_layer}w13_weight", ".moe.up_proj.weight", "w3"),
            (f".moe.experts.{base_layer}w2_weight", ".moe.down_proj.weight", "w2"),
Song's avatar
Song committed
476
477
        ]

478
        disable_moe_stacked_params = [data[1] for data in expert_params_mapping]
Song's avatar
Song committed
479
480

        for name, loaded_weight in weights:
481
            for param_name, weight_name, shard_id in stacked_params_mapping:
Song's avatar
Song committed
482
483
                if weight_name not in name:
                    continue
484
485
486
487
                if any(
                    disable_moe_stacked_param in name
                    for disable_moe_stacked_param in disable_moe_stacked_params
                ):
Song's avatar
Song committed
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
                    continue
                name = name.replace(weight_name, param_name)
                if is_pp_missing_parameter(name, self):
                    continue
                param = params_dict[name]
                weight_loader = param.weight_loader
                weight_loader(param, loaded_weight, shard_id)
                loaded_params.add(name)
                break
            else:
                for mapping in expert_params_mapping:
                    param_name, weight_name, shard_id = mapping
                    if weight_name not in name:
                        continue
                    name = name.replace(weight_name, param_name)
                    # Skip layers on other devices.
                    if is_pp_missing_parameter(name, self):
                        continue
                    # Skip loading extra bias for GPTQ models.
507
508
509
                    if (
                        name.endswith(".bias") or name.endswith("_bias")
                    ) and name not in params_dict:
Song's avatar
Song committed
510
511
512
513
514
                        continue
                    param = params_dict[name]
                    weight_loader = param.weight_loader
                    for expert_id in range(loaded_weight.shape[0]):
                        loaded_weight_expert = loaded_weight[expert_id]
515
516
517
518
519
520
521
                        weight_loader(
                            param,
                            loaded_weight_expert,
                            name,
                            shard_id=shard_id,
                            expert_id=expert_id,
                        )
Song's avatar
Song committed
522
523
524
                    loaded_params.add(name)
                    break
                else:
525
526
527
528
529
530
                    for (
                        param_name,
                        weight_name,
                        start_idx,
                        end_idx,
                    ) in qkv_params_mapping:
Song's avatar
Song committed
531
532
533
534
535
536
537
538
539
                        if weight_name not in name:
                            continue
                        name = name.replace(weight_name, param_name)
                        if is_pp_missing_parameter(name, self):
                            continue
                        param = params_dict[name]
                        dim = param.shape[param.output_dim]
                        begin_idx = int(start_idx * dim)
                        end_idx = int(end_idx * dim)
540
541
542
                        param_slice = param.narrow(
                            param.output_dim, begin_idx, end_idx - begin_idx
                        )
Song's avatar
Song committed
543
544
545
546
547
548
549
                        param_slice.copy_(loaded_weight)
                        loaded_params.add(name)
                        break
                    else:
                        if is_pp_missing_parameter(name, self):
                            continue
                        param = params_dict[name]
550
551
552
                        weight_loader = getattr(
                            param, "weight_loader", default_weight_loader
                        )
Song's avatar
Song committed
553
554
555
                        weight_loader(param, loaded_weight)
                        loaded_params.add(name)
        return loaded_params