llama_eagle3.py 12.6 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
5
from collections.abc import Iterable
from typing import Optional
6
7
8
9
10

import torch
import torch.nn as nn
from transformers import LlamaConfig

11
from vllm.compilation.decorators import support_torch_compile
12
from vllm.config import VllmConfig, get_current_vllm_config
13
14
15
16
from vllm.logger import init_logger
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import QKVParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
17
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
18
from vllm.model_executor.layers.vocab_parallel_embedding import (
19
20
21
22
    DEFAULT_VOCAB_PADDING_SIZE,
    ParallelLMHead,
    VocabParallelEmbedding,
)
23
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
24
from vllm.model_executor.models.llama import LlamaDecoderLayer, LlamaForCausalLM
25
from vllm.multimodal import MULTIMODAL_REGISTRY
26
from vllm.multimodal.inputs import NestedTensors
27
28
29
30
31
32
33

from .utils import AutoWeightsLoader, maybe_prefix

logger = init_logger(__name__)


class LlamaDecoderLayer(LlamaDecoderLayer):
34
35
36
37
38
    def __init__(
        self,
        vllm_config: VllmConfig,
        prefix: str = "",
        config: Optional[LlamaConfig] = None,
39
        layer_idx: int = 0,
40
    ) -> None:
41
42
43
        super().__init__(vllm_config, prefix=prefix, config=config)

        config = config or vllm_config.model_config.hf_config
44
        quant_config = self.get_quant_config(vllm_config)
45

46
47
48
49
        # First layer uses 2*hidden_size (embeds + hidden_states concatenated)
        # Subsequent layers use hidden_size (only hidden_states, no embeds)
        qkv_input_size = 2 * self.hidden_size if layer_idx == 0 else self.hidden_size

50
51
        # override qkv
        self.self_attn.qkv_proj = QKVParallelLinear(
52
            qkv_input_size,
53
54
55
56
57
58
59
60
61
            self.self_attn.head_dim,
            self.self_attn.total_num_heads,
            self.self_attn.total_num_kv_heads,
            bias=False,
            quant_config=quant_config,
            prefix=maybe_prefix(prefix, "qkv_proj"),
        )

        self.hidden_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
62
        self.layer_idx = layer_idx
63

64
65
66
67
68
        if getattr(config, "norm_before_residual", False):
            self._residual_norm = self._norm_before_residual
        else:
            self._residual_norm = self._norm_after_residual

69
    def get_quant_config(self, vllm_config: VllmConfig) -> Optional[QuantizationConfig]:
70
71
72
73
        """Use drafter's quantization config instead of verifier's."""
        draft_model_config = vllm_config.speculative_config.draft_model_config
        draft_load_config = vllm_config.load_config

74
75
76
77
78
        return (
            VllmConfig.get_quantization_config(draft_model_config, draft_load_config)
            if draft_model_config
            else None
        )
79

80
    def _norm_before_residual(
81
82
        self, hidden_states: torch.Tensor
    ) -> tuple[torch.Tensor, torch.Tensor]:
83
84
85
86
87
        hidden_states = self.hidden_norm(hidden_states)
        residual = hidden_states
        return hidden_states, residual

    def _norm_after_residual(
88
89
        self, hidden_states: torch.Tensor
    ) -> tuple[torch.Tensor, torch.Tensor]:
90
91
92
93
        residual = hidden_states
        hidden_states = self.hidden_norm(hidden_states)
        return hidden_states, residual

94
95
96
97
98
99
    def forward(
        self,
        positions: torch.Tensor,
        embeds: torch.Tensor,
        hidden_states: torch.Tensor,
        residual: Optional[torch.Tensor],
100
    ) -> tuple[torch.Tensor, torch.Tensor]:
101
102
103
104
105
106
107
108
        if self.layer_idx == 0:
            # First layer: concatenate embeds with hidden_states
            embeds = self.input_layernorm(embeds)
            hidden_states, residual = self._residual_norm(hidden_states=hidden_states)
            hidden_states = torch.cat([embeds, hidden_states], dim=-1)
        else:
            # Subsequent layers: process hidden_states and residuals only
            hidden_states, residual = self.input_layernorm(hidden_states, residual)
109
110
111
112
113
114
115

        # Self Attention
        hidden_states = self.self_attn(
            positions=positions,
            hidden_states=hidden_states,
        )

116
        hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
117
118
119
120
121
122
123

        # Fully Connected
        hidden_states = self.mlp(hidden_states)

        return hidden_states, residual


124
125
126
127
128
129
130
131
132
@support_torch_compile(
    # torch.compile is disabled for multimodal EAGLE3 models due to constraint
    # violations with dynamic shapes during tensor concatenation operations.
    # See: https://github.com/vllm-project/vllm/pull/22872/files#r2362028132
    # Non-multimodal EAGLE3 models can still use torch.compile safely.
    enable_if=lambda vllm_config: not MULTIMODAL_REGISTRY.supports_multimodal_inputs(
        vllm_config.model_config
    ),
)
133
134
135
136
class LlamaModel(nn.Module):
    def __init__(
        self,
        *,
137
        vllm_config: VllmConfig,
138
139
140
141
        start_layer_id: int = 0,
        prefix: str = "",
    ) -> None:
        super().__init__()
142
        self.config = vllm_config.speculative_config.draft_model_config.hf_config
143
        self.vocab_size = self.config.vocab_size
144

145
146
        current_vllm_config = get_current_vllm_config()

147
148
149
150
151
        self.embed_tokens = VocabParallelEmbedding(
            self.config.vocab_size,
            self.config.hidden_size,
            prefix=maybe_prefix(prefix, "embed_tokens"),
        )
152

153
154
155
156
        self.layers = nn.ModuleList(
            [
                LlamaDecoderLayer(
                    current_vllm_config,
157
                    prefix=maybe_prefix(prefix, f"layers.{layer_idx + start_layer_id}"),
158
                    config=self.config,
159
                    layer_idx=layer_idx,
160
                )
161
                for layer_idx in range(self.config.num_hidden_layers)
162
163
            ]
        )
164
        if hasattr(self.config, "target_hidden_size"):
165
166
167
            self.fc = torch.nn.Linear(
                self.config.target_hidden_size * 3, self.config.hidden_size, bias=False
            )
168
        else:
169
170
171
            self.fc = torch.nn.Linear(
                self.config.hidden_size * 3, self.config.hidden_size, bias=False
            )
172
173
174
175
176
        self.norm = RMSNorm(
            self.config.hidden_size,
            eps=self.config.rms_norm_eps,
        )

177
    def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
178
179
        return self.embed_tokens(input_ids)

180
181
182
183
184
    def forward(
        self,
        input_ids: torch.Tensor,
        positions: torch.Tensor,
        hidden_states: torch.Tensor,
185
        input_embeds: Optional[torch.Tensor] = None,
186
    ) -> tuple[torch.Tensor, torch.Tensor]:
187
188
        if input_embeds is None:
            input_embeds = self.get_input_embeddings(input_ids)
189
        assert hidden_states.shape[-1] == input_embeds.shape[-1]
190
191

        residual = None
192
193
194
195
196
197
198
        for layer in self.layers:
            hidden_states, residual = layer(
                positions=positions,
                embeds=input_embeds,
                hidden_states=hidden_states,
                residual=residual,
            )
199
200
201
        hidden_states, hidden_prenorm = self.norm(hidden_states, residual)
        return hidden_states, hidden_prenorm

202
    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
203
204
205
206
207
208
209
210
211
        stacked_params_mapping = [
            # (param_name, shard_name, shard_id)
            (".qkv_proj", ".q_proj", "q"),
            (".qkv_proj", ".k_proj", "k"),
            (".qkv_proj", ".v_proj", "v"),
            (".gate_up_proj", ".gate_proj", 0),
            (".gate_up_proj", ".up_proj", 1),
        ]
        params_dict = dict(self.named_parameters())
212
        loaded_params: set[str] = set()
213
        for name, loaded_weight in weights:
214
215
            if "midlayer." in name:
                name = name.replace("midlayer.", "layers.0.")
216
217
218
219
220
221
222
223
224
225
            for param_name, weight_name, shard_id in stacked_params_mapping:
                if weight_name not in name:
                    continue
                name = name.replace(weight_name, param_name)
                param = params_dict[name]
                weight_loader = param.weight_loader
                weight_loader(param, loaded_weight, shard_id)
                break
            else:
                param = params_dict[name]
226
                weight_loader = getattr(param, "weight_loader", default_weight_loader)
227
228
229
230
231
232
                weight_loader(param, loaded_weight)
            loaded_params.add(name)
        return loaded_params


class Eagle3LlamaForCausalLM(LlamaForCausalLM):
233
    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
234
        nn.Module.__init__(self)
235
        self.config = vllm_config.speculative_config.draft_model_config.hf_config
236
237
238
239
240
        # Ensure draft_vocab_size is set
        # default to the base vocab size when absent
        if getattr(self.config, "draft_vocab_size", None) is None:
            base_vocab_size = getattr(self.config, "vocab_size", None)
            self.config.draft_vocab_size = base_vocab_size
241
        target_layer_num = vllm_config.model_config.get_num_layers(
242
243
            vllm_config.parallel_config
        )
244
245
246
247

        # Store target layer count in draft config for
        # proper layer_types indexing in draft models
        self.config.target_layer_count = target_layer_num
248
249
250
        self.model = LlamaModel(
            vllm_config=vllm_config, prefix="model", start_layer_id=target_layer_num
        )
251
252
253
254
255
256
257

        logit_scale = getattr(self.config, "logit_scale", 1.0)
        self.lm_head = ParallelLMHead(
            self.config.draft_vocab_size,
            self.config.hidden_size,
            org_num_embeddings=self.config.draft_vocab_size,
            padding_size=(DEFAULT_VOCAB_PADDING_SIZE),
258
259
260
261
262
            prefix=maybe_prefix(prefix, "lm_head"),
        )
        self.logits_processor = LogitsProcessor(
            self.config.draft_vocab_size, scale=logit_scale
        )
263
        self.draft_id_to_target_id = nn.Parameter(
264
            torch.zeros(self.config.draft_vocab_size, dtype=torch.long),
265
266
267
            requires_grad=False,
        )

268
269
270
271
272
273
    def get_input_embeddings(
        self,
        input_ids: torch.Tensor,
        multimodal_embeddings: Optional[NestedTensors] = None,
        is_multimodal: Optional[torch.Tensor] = None,
    ) -> torch.Tensor:
274
275
        return self.model.get_input_embeddings(input_ids)

276
277
278
279
280
    def forward(
        self,
        input_ids: torch.Tensor,
        positions: torch.Tensor,
        hidden_states: torch.Tensor,
281
        inputs_embeds: Optional[torch.Tensor] = None,
282
    ) -> tuple[torch.Tensor, torch.Tensor]:
283
        return self.model(input_ids, positions, hidden_states, inputs_embeds)
284
285
286
287
288

    def compute_logits(
        self,
        hidden_states: torch.Tensor,
    ) -> Optional[torch.Tensor]:
289
        logits = self.logits_processor(self.lm_head, hidden_states)
290
        if self.draft_id_to_target_id is None:
291
292
            assert logits.shape[1] == self.config.vocab_size, (
                "Expected logits to have shape "
293
                f"(*, {self.config.vocab_size}), but got {logits.shape}"
294
            )
295
296
            return logits

297
298
        base = torch.arange(self.config.draft_vocab_size, device=logits.device)
        targets = base + self.draft_id_to_target_id
299
300
301
302
303
304
305
        logits_new = logits.new_full(
            (
                logits.shape[0],
                self.config.vocab_size,
            ),
            float("-inf"),
        )
306
307
308
        logits_new[:, targets] = logits
        return logits_new

309
310
311
312
313
314
315
    def combine_hidden_states(
        self,
        hidden_states: torch.Tensor,
    ) -> torch.Tensor:
        # combine multiple auxiliary hidden states returned by eagle3
        return self.model.fc(hidden_states)

316
    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
317
        model_weights = {}
318
        includes_draft_id_mapping = False
319
        includes_embed_tokens = False
320
321
322
323
324
        for name, loaded_weight in weights:
            if "t2d" in name:
                continue
            if "d2t" in name:
                name = name.replace("d2t", "draft_id_to_target_id")
325
                includes_draft_id_mapping = True
326
327
            elif "lm_head" not in name:
                name = "model." + name
328
329
            if "embed_tokens" in name:
                includes_embed_tokens = True
330
331
            model_weights[name] = loaded_weight

332
333
334
335
336
        skip_substrs = []
        if not includes_draft_id_mapping:
            skip_substrs.append("draft_id_to_target_id")
        if not includes_embed_tokens:
            skip_substrs.append("embed_tokens")
337
338
339
        loader = AutoWeightsLoader(
            self,
            skip_prefixes=None,
340
            skip_substrs=skip_substrs,
341
342
        )
        loader.load_weights(model_weights.items())