mxfp4.py 15.7 KB
Newer Older
1
2
3
4
5
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

import torch

6
from vllm.config import get_current_vllm_config
7
from vllm.logger import init_logger
8
from vllm.model_executor.layers.attention import Attention
9
10
11
12
13
from vllm.model_executor.layers.fused_moe import (
    FusedMoE,
    FusedMoEConfig,
    FusedMoEMethodBase,
)
14
from vllm.model_executor.layers.fused_moe import modular_kernel as mk
15
from vllm.model_executor.layers.fused_moe.config import (
16
    FusedMoEParallelConfig,
17
    FusedMoEQuantConfig,
18
)
19
20
21
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
    TRITON_BACKENDS,
    Mxfp4MoeBackend,
22
    convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
23
24
25
    make_mxfp4_moe_kernel,
    make_mxfp4_moe_quant_config,
    mxfp4_round_up_hidden_size_and_intermediate_size,
26
    select_gpt_oss_mxfp4_moe_backend,
27
28
)
from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
29
30
from vllm.model_executor.layers.quantization import QuantizationMethods
from vllm.model_executor.layers.quantization.base_config import (
31
32
33
34
    QuantizationConfig,
    QuantizeMethodBase,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped
35
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
36

37
38
39
logger = init_logger(__name__)


40
class Mxfp4Config(QuantizationConfig):
41
42
43
44
45
46
    """Canonical base config for MXFP4 quantization.

    Subclasses override get_name() and override_quantization_method() to
    register themselves as the handler for a specific checkpoint format.
    """

47
    def __init__(self, ignored_layers: list[str] | None = None):
48
49
50
51
52
53
54
55
56
        super().__init__()
        self.ignored_layers = ignored_layers

    @classmethod
    def from_config(cls, config):
        return cls()

    @classmethod
    def get_min_capability(cls) -> int:
57
        return 80
58
59
60
61
62
63
64
65
66
67
68
69
70

    @classmethod
    def get_name(cls) -> QuantizationMethods:
        return "mxfp4"

    @classmethod
    def get_supported_act_dtypes(cls) -> list[torch.dtype]:
        return [torch.bfloat16]

    @classmethod
    def get_config_filenames(cls) -> list[str]:
        return []

71
72
    # TODO (zyongye) This is only temporaty fallback.
    # We should have `Mxfp4MoEMethod` after this migration is complete.
73
74
    def get_quant_method(
        self, layer: torch.nn.Module, prefix: str
75
    ) -> "QuantizeMethodBase | None":
76
77
        if isinstance(layer, LinearBase):
            if self.ignored_layers and is_layer_skipped(
78
79
80
81
                prefix=prefix,
                ignored_layers=self.ignored_layers,
                fused_mapping=self.packed_modules_mapping,
            ):
82
                return UnquantizedLinearMethod()
83
            logger.debug_once(
84
                "MXFP4 linear layer is not implemented - falling back to "
85
                "UnquantizedLinearMethod.",
86
87
            )
            return UnquantizedLinearMethod()
88
        elif isinstance(layer, FusedMoE):
89
            return GptOssMxfp4MoEMethod(layer.moe_config)
90
        elif isinstance(layer, Attention):
91
            logger.debug_once(
92
                "MXFP4 attention layer is not implemented. "
93
                "Skipping quantization for this layer.",
94
            )
95
96
        return None

97
98
99
100
    def is_mxfp4_quant(self, prefix: str, layer: torch.nn.Module) -> bool:
        """MXFP4 config always uses MXFP4 quantization."""
        return True

101

102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
class GptOssMxfp4Config(Mxfp4Config):
    """MXFP4 config for GPT-OSS checkpoints.

    Checkpoints carry ``"quant_method": "mxfp4"`` in their JSON config.
    override_quantization_method() maps that to the canonical internal name
    so that the rest of the loading path uses "gpt_oss_mxfp4" consistently.
    """

    @classmethod
    def get_name(cls) -> QuantizationMethods:
        return "gpt_oss_mxfp4"

    @classmethod
    def override_quantization_method(
        cls, hf_quant_cfg, user_quant, hf_config=None
    ) -> QuantizationMethods | None:
        # Match both "mxfp4" (original checkpoint value) and "gpt_oss_mxfp4"
        # (already normalized by verify_and_update_model_config) so that
        # explicit --quantization mxfp4 from the user doesn't cause a mismatch.
        if not (
            isinstance(hf_quant_cfg, dict)
            and hf_quant_cfg.get("quant_method") in ("mxfp4", "gpt_oss_mxfp4")
        ):
            return None
        # Require explicit confirmation that this is a GPT-OSS model.
        # Do NOT fall back to returning the override when hf_config is None,
        # as that would silently claim all mxfp4 checkpoints.
        model_type = getattr(hf_config, "model_type", None)
        if model_type != "gpt_oss":
            return None
        return "gpt_oss_mxfp4"


class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
136
137
    """MXFP4 MoE quantization method."""

138
    def __init__(self, moe: FusedMoEConfig):
139
        super().__init__(moe)
140
141
        self.weight_dtype = "gpt_oss_mxfp4"
        self.mxfp4_backend, self.experts_cls = select_gpt_oss_mxfp4_moe_backend(moe)
142

143
        self.max_capture_size = (
144
            get_current_vllm_config().compilation_config.max_cudagraph_capture_size
145
        )
146

147
        self._cache_permute_indices: dict[torch.Size, torch.Tensor] = {}
148
        self.moe_kernel: mk.FusedMoEKernel | None = None
149

150
151
152
153
        # Used for triton kernel precision configs
        self.w13_precision_config = None
        self.w2_precision_config = None

154
155
156
157
    @property
    def skip_forward_padding(self) -> bool:
        # SM100_FI_MXFP4_MXFP8_TRTLLM supports padding with mxfp8 quant
        # so can skip the padding in the forward before applying the moe method
158
        return self.mxfp4_backend == Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8
159

160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
    def maybe_roundup_sizes(
        self,
        hidden_size: int,
        intermediate_size_per_partition: int,
        act_dtype: torch.dtype,
        moe_parallel_config: FusedMoEParallelConfig,
    ) -> tuple[int, int]:
        hidden_size, intermediate_size_per_partition = super().maybe_roundup_sizes(
            hidden_size=hidden_size,
            intermediate_size_per_partition=intermediate_size_per_partition,
            act_dtype=act_dtype,
            moe_parallel_config=moe_parallel_config,
        )
        return mxfp4_round_up_hidden_size_and_intermediate_size(
            self.mxfp4_backend, hidden_size, intermediate_size_per_partition
        )

177
178
179
180
181
182
183
184
185
    def create_weights(
        self,
        layer: torch.nn.Module,
        num_experts: int,
        hidden_size: int,
        intermediate_size_per_partition: int,
        params_dtype: torch.dtype,
        **extra_weight_attrs,
    ):
186
187
188
189
190
        self.num_experts = num_experts
        weight_dtype = torch.uint8
        scale_dtype = torch.uint8
        mxfp4_block = 32

191
192
        layer.params_dtype = params_dtype
        layer.num_experts = num_experts
193
194
        self.intermediate_size = intermediate_size_per_partition
        self.hidden_size = hidden_size
195

196
        # Fused gate_up_proj (column parallel)
197
198
199
        w13_weight = torch.nn.Parameter(
            torch.zeros(
                num_experts,
200
                2 * intermediate_size_per_partition,
201
202
203
204
205
                hidden_size // 2,
                dtype=weight_dtype,
            ),
            requires_grad=False,
        )
206
207
208
        layer.register_parameter("w13_weight", w13_weight)
        set_weight_attrs(w13_weight, extra_weight_attrs)

209
210
211
        w13_weight_scale = torch.nn.Parameter(
            torch.zeros(
                num_experts,
212
                2 * intermediate_size_per_partition,
213
214
215
216
217
                hidden_size // mxfp4_block,
                dtype=scale_dtype,
            ),
            requires_grad=False,
        )
218
219
220
221
        layer.register_parameter("w13_weight_scale", w13_weight_scale)
        set_weight_attrs(w13_weight_scale, extra_weight_attrs)

        # down_proj (row parallel)
222
223
224
225
        w2_weight = torch.nn.Parameter(
            torch.zeros(
                num_experts,
                hidden_size,
226
                intermediate_size_per_partition // 2,
227
228
229
230
                dtype=weight_dtype,
            ),
            requires_grad=False,
        )
231
232
233
        layer.register_parameter("w2_weight", w2_weight)
        set_weight_attrs(w2_weight, extra_weight_attrs)

234
235
236
237
        w2_weight_scale = torch.nn.Parameter(
            torch.zeros(
                num_experts,
                hidden_size,
238
                intermediate_size_per_partition // mxfp4_block,
239
240
241
242
                dtype=scale_dtype,
            ),
            requires_grad=False,
        )
243
244
245
        layer.register_parameter("w2_weight_scale", w2_weight_scale)
        set_weight_attrs(w2_weight_scale, extra_weight_attrs)

246
247
248
249
        if self.moe.has_bias:
            w13_bias = torch.nn.Parameter(
                torch.zeros(
                    num_experts,
250
                    2 * intermediate_size_per_partition,
251
                    dtype=torch.bfloat16,
252
                ),
253
254
                requires_grad=False,
            )
255
256
            layer.register_parameter("w13_bias", w13_bias)
            set_weight_attrs(w13_bias, extra_weight_attrs)
257

258
259
260
261
262
263
            w2_bias = torch.nn.Parameter(
                torch.zeros(
                    num_experts,
                    hidden_size,
                    dtype=torch.bfloat16,
                ),
264
265
                requires_grad=False,
            )
266
267
            layer.register_parameter("w2_bias", w2_bias)
            set_weight_attrs(w2_bias, extra_weight_attrs)
268

269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
    def _setup_kernel(
        self,
        layer: FusedMoE,
        w13: torch.Tensor,
        w2: torch.Tensor,
        w13_scale: torch.Tensor,
        w2_scale: torch.Tensor,
        w13_bias: torch.Tensor | None = None,
        w2_bias: torch.Tensor | None = None,
    ) -> None:
        num_experts = self.num_experts
        intermediate_size = self.intermediate_size
        hidden_size = self.hidden_size
        sf_block_size = 32

        # Shape assertions
        assert (
            w13.dim() == 3
            and w13.shape[0] == num_experts
            and w13.shape[1] == intermediate_size * 2
            and w13.shape[2] == hidden_size // 2
        )
        assert (
            w13_scale.dim() == 3
            and w13_scale.shape[0] == num_experts
            and w13_scale.shape[1] == intermediate_size * 2
            and w13_scale.shape[2] == hidden_size // sf_block_size
        )
        assert (
            w2.dim() == 3
            and w2.shape[0] == num_experts
            and w2.shape[1] == hidden_size
            and w2.shape[2] == intermediate_size // 2
        )
        assert (
            w2_scale.dim() == 3
            and w2_scale.shape[1] == hidden_size
            and w2_scale.shape[2] == intermediate_size // sf_block_size
        )
        if w13_bias is not None:
309
            assert (
310
311
312
                w13_bias.dim() == 2
                and w13_bias.shape[0] == num_experts
                and w13_bias.shape[1] == intermediate_size * 2
313
            )
314
        if w2_bias is not None:
315
            assert (
316
317
318
319
320
321
322
                w2_bias.dim() == 2
                and w2_bias.shape[0] == num_experts
                and w2_bias.shape[1] == hidden_size
            )

        # Convert weights to kernel format
        w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = (
323
            convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
324
325
326
327
328
329
330
331
332
                mxfp4_backend=self.mxfp4_backend,
                layer=layer,
                w13_weight=w13,
                w2_weight=w2,
                w13_weight_scale=w13_scale,
                w2_weight_scale=w2_scale,
                w13_bias=w13_bias,
                w2_bias=w2_bias,
                _cache_permute_indices=self._cache_permute_indices,
333
            )
334
        )
335

336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
        # For TRITON backends, weights are wrapped tensors from triton_kernels
        # that don't support .detach(). Manually assign parameters.
        if self.mxfp4_backend not in TRITON_BACKENDS:
            replace_parameter(layer, "w13_weight", w13)
            replace_parameter(layer, "w2_weight", w2)
            replace_parameter(layer, "w13_weight_scale", w13_scale)
            replace_parameter(layer, "w2_weight_scale", w2_scale)
        else:
            layer.w13_weight = w13
            layer.w2_weight = w2
            self.w13_precision_config = w13_scale
            self.w2_precision_config = w2_scale

        if w13_bias is not None and w2_bias is not None:
            replace_parameter(layer, "w13_bias", w13_bias)
            replace_parameter(layer, "w2_bias", w2_bias)

        # Build quant config
        self.moe_quant_config = self.get_fused_moe_quant_config(layer)

        # Build kernel (modular or monolithic)
        if self.moe_quant_config is not None and self.experts_cls is not None:
            self.moe_kernel = make_mxfp4_moe_kernel(
                moe_quant_config=self.moe_quant_config,
                moe_config=self.moe,
                mxfp4_backend=self.mxfp4_backend,
                experts_cls=self.experts_cls,
363
                routing_tables=layer._maybe_init_expert_routing_tables(),
364
                shared_experts=layer.shared_experts,
365
366
            )

367
368
369
370
371
372
373
    def process_weights_after_loading(self, layer):
        w13 = layer.w13_weight
        w2 = layer.w2_weight
        w13_scale = layer.w13_weight_scale
        w2_scale = layer.w2_weight_scale
        w13_bias = getattr(layer, "w13_bias", None)
        w2_bias = getattr(layer, "w2_bias", None)
374

375
376
        if self.mxfp4_backend == Mxfp4MoeBackend.NONE:
            return
377

378
        self._setup_kernel(layer, w13, w2, w13_scale, w2_scale, w13_bias, w2_bias)
379

380
    def get_fused_moe_quant_config(
381
        self, layer: torch.nn.Module
382
    ) -> FusedMoEQuantConfig | None:
383
384
385
386
387
388
389
390
        w1_scale = layer.w13_weight_scale
        w2_scale = layer.w2_weight_scale
        w1_bias = getattr(layer, "w13_bias", None)
        w2_bias = getattr(layer, "w2_bias", None)

        if self.mxfp4_backend in TRITON_BACKENDS:
            assert self.w13_precision_config is not None
            assert self.w2_precision_config is not None
391
392
            w1_scale = self.w13_precision_config
            w2_scale = self.w2_precision_config
393
394
395
396
397
398
399
400

        return make_mxfp4_moe_quant_config(
            mxfp4_backend=self.mxfp4_backend,
            w1_scale=w1_scale,
            w2_scale=w2_scale,
            w1_bias=w1_bias,
            w2_bias=w2_bias,
        )
401

402
403
    def select_gemm_impl(
        self,
404
        prepare_finalize: mk.FusedMoEPrepareAndFinalize,
405
        layer: torch.nn.Module,
406
    ) -> mk.FusedMoEExpertsModular:
407
408
409
        raise ValueError(
            f"{self.__class__.__name__} uses the new modular kernel "
            "initialization logic. This function should not be called."
410
411
        )

412
413
    def apply(
        self,
414
        layer: FusedMoE,
415
        x: torch.Tensor,
416
417
        topk_weights: torch.Tensor,
        topk_ids: torch.Tensor,
418
        shared_experts_input: torch.Tensor | None,
419
    ) -> torch.Tensor:
420
        assert not self.is_monolithic
421
422
        assert self.moe_kernel is not None
        return self.moe_kernel.apply(
423
424
425
426
427
428
429
430
431
432
            hidden_states=x,
            w1=layer.w13_weight,
            w2=layer.w2_weight,
            topk_weights=topk_weights,
            topk_ids=topk_ids,
            activation=layer.activation,
            global_num_experts=layer.global_num_experts,
            apply_router_weight_on_input=layer.apply_router_weight_on_input,
            expert_map=layer.expert_map,
            shared_experts_input=shared_experts_input,
433
434
435
436
437
438
439
        )

    def apply_monolithic(
        self,
        layer: FusedMoE,
        x: torch.Tensor,
        router_logits: torch.Tensor,
440
    ) -> torch.Tensor:
441
        assert self.is_monolithic
442
443
444
445
446
447
448
449
450
451
452
        assert self.moe_kernel is not None
        return self.moe_kernel.apply_monolithic(
            hidden_states=x,
            w1=layer.w13_weight,
            w2=layer.w2_weight,
            router_logits=router_logits,
            activation=layer.activation,
            global_num_experts=layer.global_num_experts,
            expert_map=layer.expert_map,
            apply_router_weight_on_input=layer.apply_router_weight_on_input,
        )