mxfp4.py 15.8 KB
Newer Older
1
2
3
4
5
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

import torch

6
from vllm.config import get_current_vllm_config
7
from vllm.logger import init_logger
8
from vllm.model_executor.layers.attention import Attention
9
10
11
12
13
from vllm.model_executor.layers.fused_moe import (
    FusedMoE,
    FusedMoEConfig,
    FusedMoEMethodBase,
)
14
from vllm.model_executor.layers.fused_moe import modular_kernel as mk
15
from vllm.model_executor.layers.fused_moe.config import (
16
    FusedMoEParallelConfig,
17
    FusedMoEQuantConfig,
18
)
19
20
21
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
    TRITON_BACKENDS,
    Mxfp4MoeBackend,
22
    convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
23
24
25
    make_mxfp4_moe_kernel,
    make_mxfp4_moe_quant_config,
    mxfp4_round_up_hidden_size_and_intermediate_size,
26
    select_gpt_oss_mxfp4_moe_backend,
27
28
)
from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
29
30
from vllm.model_executor.layers.quantization import QuantizationMethods
from vllm.model_executor.layers.quantization.base_config import (
31
32
33
34
    QuantizationConfig,
    QuantizeMethodBase,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped
35
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
36

37
38
39
logger = init_logger(__name__)


40
class Mxfp4Config(QuantizationConfig):
41
42
43
44
45
46
    """Canonical base config for MXFP4 quantization.

    Subclasses override get_name() and override_quantization_method() to
    register themselves as the handler for a specific checkpoint format.
    """

47
    def __init__(self, ignored_layers: list[str] | None = None):
48
49
50
51
52
53
54
55
56
        super().__init__()
        self.ignored_layers = ignored_layers

    @classmethod
    def from_config(cls, config):
        return cls()

    @classmethod
    def get_min_capability(cls) -> int:
57
        return 80
58
59
60
61
62
63
64
65
66
67
68
69
70

    @classmethod
    def get_name(cls) -> QuantizationMethods:
        return "mxfp4"

    @classmethod
    def get_supported_act_dtypes(cls) -> list[torch.dtype]:
        return [torch.bfloat16]

    @classmethod
    def get_config_filenames(cls) -> list[str]:
        return []

71
72
    # TODO (zyongye) This is only temporaty fallback.
    # We should have `Mxfp4MoEMethod` after this migration is complete.
73
74
    def get_quant_method(
        self, layer: torch.nn.Module, prefix: str
75
    ) -> "QuantizeMethodBase | None":
76
77
        if isinstance(layer, LinearBase):
            if self.ignored_layers and is_layer_skipped(
78
79
80
81
                prefix=prefix,
                ignored_layers=self.ignored_layers,
                fused_mapping=self.packed_modules_mapping,
            ):
82
                return UnquantizedLinearMethod()
83
            logger.debug_once(
84
                "MXFP4 linear layer is not implemented - falling back to "
85
86
                "UnquantizedLinearMethod.",
                scope="local",
87
88
            )
            return UnquantizedLinearMethod()
89
        elif isinstance(layer, FusedMoE):
90
            return GptOssMxfp4MoEMethod(layer.moe_config)
91
        elif isinstance(layer, Attention):
92
            logger.debug_once(
93
                "MXFP4 attention layer is not implemented. "
94
95
                "Skipping quantization for this layer.",
                scope="local",
96
            )
97
98
        return None

99
100
101
102
    def is_mxfp4_quant(self, prefix: str, layer: torch.nn.Module) -> bool:
        """MXFP4 config always uses MXFP4 quantization."""
        return True

103

104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
class GptOssMxfp4Config(Mxfp4Config):
    """MXFP4 config for GPT-OSS checkpoints.

    Checkpoints carry ``"quant_method": "mxfp4"`` in their JSON config.
    override_quantization_method() maps that to the canonical internal name
    so that the rest of the loading path uses "gpt_oss_mxfp4" consistently.
    """

    @classmethod
    def get_name(cls) -> QuantizationMethods:
        return "gpt_oss_mxfp4"

    @classmethod
    def override_quantization_method(
        cls, hf_quant_cfg, user_quant, hf_config=None
    ) -> QuantizationMethods | None:
        # Match both "mxfp4" (original checkpoint value) and "gpt_oss_mxfp4"
        # (already normalized by verify_and_update_model_config) so that
        # explicit --quantization mxfp4 from the user doesn't cause a mismatch.
        if not (
            isinstance(hf_quant_cfg, dict)
            and hf_quant_cfg.get("quant_method") in ("mxfp4", "gpt_oss_mxfp4")
        ):
            return None
        # Require explicit confirmation that this is a GPT-OSS model.
        # Do NOT fall back to returning the override when hf_config is None,
        # as that would silently claim all mxfp4 checkpoints.
        model_type = getattr(hf_config, "model_type", None)
        if model_type != "gpt_oss":
            return None
        return "gpt_oss_mxfp4"


class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
138
139
    """MXFP4 MoE quantization method."""

140
    def __init__(self, moe: FusedMoEConfig):
141
        super().__init__(moe)
142
143
        self.weight_dtype = "gpt_oss_mxfp4"
        self.mxfp4_backend, self.experts_cls = select_gpt_oss_mxfp4_moe_backend(moe)
144

145
        self.max_capture_size = (
146
            get_current_vllm_config().compilation_config.max_cudagraph_capture_size
147
        )
148

149
        self._cache_permute_indices: dict[torch.Size, torch.Tensor] = {}
150
        self.moe_kernel: mk.FusedMoEKernel | None = None
151

152
153
154
155
        # Used for triton kernel precision configs
        self.w13_precision_config = None
        self.w2_precision_config = None

156
157
158
159
    @property
    def skip_forward_padding(self) -> bool:
        # SM100_FI_MXFP4_MXFP8_TRTLLM supports padding with mxfp8 quant
        # so can skip the padding in the forward before applying the moe method
160
        return self.mxfp4_backend == Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8
161

162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
    def maybe_roundup_sizes(
        self,
        hidden_size: int,
        intermediate_size_per_partition: int,
        act_dtype: torch.dtype,
        moe_parallel_config: FusedMoEParallelConfig,
    ) -> tuple[int, int]:
        hidden_size, intermediate_size_per_partition = super().maybe_roundup_sizes(
            hidden_size=hidden_size,
            intermediate_size_per_partition=intermediate_size_per_partition,
            act_dtype=act_dtype,
            moe_parallel_config=moe_parallel_config,
        )
        return mxfp4_round_up_hidden_size_and_intermediate_size(
            self.mxfp4_backend, hidden_size, intermediate_size_per_partition
        )

179
180
181
182
183
184
185
186
187
    def create_weights(
        self,
        layer: torch.nn.Module,
        num_experts: int,
        hidden_size: int,
        intermediate_size_per_partition: int,
        params_dtype: torch.dtype,
        **extra_weight_attrs,
    ):
188
189
190
191
192
        self.num_experts = num_experts
        weight_dtype = torch.uint8
        scale_dtype = torch.uint8
        mxfp4_block = 32

193
194
        layer.params_dtype = params_dtype
        layer.num_experts = num_experts
195
196
        self.intermediate_size = intermediate_size_per_partition
        self.hidden_size = hidden_size
197

198
        # Fused gate_up_proj (column parallel)
199
200
201
        w13_weight = torch.nn.Parameter(
            torch.zeros(
                num_experts,
202
                2 * intermediate_size_per_partition,
203
204
205
206
207
                hidden_size // 2,
                dtype=weight_dtype,
            ),
            requires_grad=False,
        )
208
209
210
        layer.register_parameter("w13_weight", w13_weight)
        set_weight_attrs(w13_weight, extra_weight_attrs)

211
212
213
        w13_weight_scale = torch.nn.Parameter(
            torch.zeros(
                num_experts,
214
                2 * intermediate_size_per_partition,
215
216
217
218
219
                hidden_size // mxfp4_block,
                dtype=scale_dtype,
            ),
            requires_grad=False,
        )
220
221
222
223
        layer.register_parameter("w13_weight_scale", w13_weight_scale)
        set_weight_attrs(w13_weight_scale, extra_weight_attrs)

        # down_proj (row parallel)
224
225
226
227
        w2_weight = torch.nn.Parameter(
            torch.zeros(
                num_experts,
                hidden_size,
228
                intermediate_size_per_partition // 2,
229
230
231
232
                dtype=weight_dtype,
            ),
            requires_grad=False,
        )
233
234
235
        layer.register_parameter("w2_weight", w2_weight)
        set_weight_attrs(w2_weight, extra_weight_attrs)

236
237
238
239
        w2_weight_scale = torch.nn.Parameter(
            torch.zeros(
                num_experts,
                hidden_size,
240
                intermediate_size_per_partition // mxfp4_block,
241
242
243
244
                dtype=scale_dtype,
            ),
            requires_grad=False,
        )
245
246
247
        layer.register_parameter("w2_weight_scale", w2_weight_scale)
        set_weight_attrs(w2_weight_scale, extra_weight_attrs)

248
249
250
251
        if self.moe.has_bias:
            w13_bias = torch.nn.Parameter(
                torch.zeros(
                    num_experts,
252
                    2 * intermediate_size_per_partition,
253
                    dtype=torch.bfloat16,
254
                ),
255
256
                requires_grad=False,
            )
257
258
            layer.register_parameter("w13_bias", w13_bias)
            set_weight_attrs(w13_bias, extra_weight_attrs)
259

260
261
262
263
264
265
            w2_bias = torch.nn.Parameter(
                torch.zeros(
                    num_experts,
                    hidden_size,
                    dtype=torch.bfloat16,
                ),
266
267
                requires_grad=False,
            )
268
269
            layer.register_parameter("w2_bias", w2_bias)
            set_weight_attrs(w2_bias, extra_weight_attrs)
270

271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
    def _setup_kernel(
        self,
        layer: FusedMoE,
        w13: torch.Tensor,
        w2: torch.Tensor,
        w13_scale: torch.Tensor,
        w2_scale: torch.Tensor,
        w13_bias: torch.Tensor | None = None,
        w2_bias: torch.Tensor | None = None,
    ) -> None:
        num_experts = self.num_experts
        intermediate_size = self.intermediate_size
        hidden_size = self.hidden_size
        sf_block_size = 32

        # Shape assertions
        assert (
            w13.dim() == 3
            and w13.shape[0] == num_experts
            and w13.shape[1] == intermediate_size * 2
            and w13.shape[2] == hidden_size // 2
        )
        assert (
            w13_scale.dim() == 3
            and w13_scale.shape[0] == num_experts
            and w13_scale.shape[1] == intermediate_size * 2
            and w13_scale.shape[2] == hidden_size // sf_block_size
        )
        assert (
            w2.dim() == 3
            and w2.shape[0] == num_experts
            and w2.shape[1] == hidden_size
            and w2.shape[2] == intermediate_size // 2
        )
        assert (
            w2_scale.dim() == 3
            and w2_scale.shape[1] == hidden_size
            and w2_scale.shape[2] == intermediate_size // sf_block_size
        )
        if w13_bias is not None:
311
            assert (
312
313
314
                w13_bias.dim() == 2
                and w13_bias.shape[0] == num_experts
                and w13_bias.shape[1] == intermediate_size * 2
315
            )
316
        if w2_bias is not None:
317
            assert (
318
319
320
321
322
323
324
                w2_bias.dim() == 2
                and w2_bias.shape[0] == num_experts
                and w2_bias.shape[1] == hidden_size
            )

        # Convert weights to kernel format
        w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = (
325
            convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
326
327
328
329
330
331
332
333
334
                mxfp4_backend=self.mxfp4_backend,
                layer=layer,
                w13_weight=w13,
                w2_weight=w2,
                w13_weight_scale=w13_scale,
                w2_weight_scale=w2_scale,
                w13_bias=w13_bias,
                w2_bias=w2_bias,
                _cache_permute_indices=self._cache_permute_indices,
335
            )
336
        )
337

338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
        # For TRITON backends, weights are wrapped tensors from triton_kernels
        # that don't support .detach(). Manually assign parameters.
        if self.mxfp4_backend not in TRITON_BACKENDS:
            replace_parameter(layer, "w13_weight", w13)
            replace_parameter(layer, "w2_weight", w2)
            replace_parameter(layer, "w13_weight_scale", w13_scale)
            replace_parameter(layer, "w2_weight_scale", w2_scale)
        else:
            layer.w13_weight = w13
            layer.w2_weight = w2
            self.w13_precision_config = w13_scale
            self.w2_precision_config = w2_scale

        if w13_bias is not None and w2_bias is not None:
            replace_parameter(layer, "w13_bias", w13_bias)
            replace_parameter(layer, "w2_bias", w2_bias)

        # Build quant config
        self.moe_quant_config = self.get_fused_moe_quant_config(layer)

        # Build kernel (modular or monolithic)
        if self.moe_quant_config is not None and self.experts_cls is not None:
            self.moe_kernel = make_mxfp4_moe_kernel(
                moe_quant_config=self.moe_quant_config,
                moe_config=self.moe,
                mxfp4_backend=self.mxfp4_backend,
                experts_cls=self.experts_cls,
365
                routing_tables=layer._maybe_init_expert_routing_tables(),
366
                shared_experts=layer.shared_experts,
367
368
            )

369
370
371
372
373
374
375
    def process_weights_after_loading(self, layer):
        w13 = layer.w13_weight
        w2 = layer.w2_weight
        w13_scale = layer.w13_weight_scale
        w2_scale = layer.w2_weight_scale
        w13_bias = getattr(layer, "w13_bias", None)
        w2_bias = getattr(layer, "w2_bias", None)
376

377
378
        if self.mxfp4_backend == Mxfp4MoeBackend.NONE:
            return
379

380
        self._setup_kernel(layer, w13, w2, w13_scale, w2_scale, w13_bias, w2_bias)
381

382
    def get_fused_moe_quant_config(
383
        self, layer: torch.nn.Module
384
    ) -> FusedMoEQuantConfig | None:
385
386
387
388
389
390
391
392
        w1_scale = layer.w13_weight_scale
        w2_scale = layer.w2_weight_scale
        w1_bias = getattr(layer, "w13_bias", None)
        w2_bias = getattr(layer, "w2_bias", None)

        if self.mxfp4_backend in TRITON_BACKENDS:
            assert self.w13_precision_config is not None
            assert self.w2_precision_config is not None
393
394
            w1_scale = self.w13_precision_config
            w2_scale = self.w2_precision_config
395
396
397
398
399
400
401
402

        return make_mxfp4_moe_quant_config(
            mxfp4_backend=self.mxfp4_backend,
            w1_scale=w1_scale,
            w2_scale=w2_scale,
            w1_bias=w1_bias,
            w2_bias=w2_bias,
        )
403

404
405
    def select_gemm_impl(
        self,
406
        prepare_finalize: mk.FusedMoEPrepareAndFinalize,
407
        layer: torch.nn.Module,
408
    ) -> mk.FusedMoEExpertsModular:
409
410
411
        raise ValueError(
            f"{self.__class__.__name__} uses the new modular kernel "
            "initialization logic. This function should not be called."
412
413
        )

414
415
    def apply(
        self,
416
        layer: FusedMoE,
417
        x: torch.Tensor,
418
419
        topk_weights: torch.Tensor,
        topk_ids: torch.Tensor,
420
        shared_experts_input: torch.Tensor | None,
421
    ) -> torch.Tensor:
422
        assert not self.is_monolithic
423
424
        assert self.moe_kernel is not None
        return self.moe_kernel.apply(
425
426
427
428
429
430
431
432
433
434
            hidden_states=x,
            w1=layer.w13_weight,
            w2=layer.w2_weight,
            topk_weights=topk_weights,
            topk_ids=topk_ids,
            activation=layer.activation,
            global_num_experts=layer.global_num_experts,
            apply_router_weight_on_input=layer.apply_router_weight_on_input,
            expert_map=layer.expert_map,
            shared_experts_input=shared_experts_input,
435
436
437
438
439
440
441
        )

    def apply_monolithic(
        self,
        layer: FusedMoE,
        x: torch.Tensor,
        router_logits: torch.Tensor,
442
    ) -> torch.Tensor:
443
        assert self.is_monolithic
444
445
446
447
448
449
450
451
452
453
454
        assert self.moe_kernel is not None
        return self.moe_kernel.apply_monolithic(
            hidden_states=x,
            w1=layer.w13_weight,
            w2=layer.w2_weight,
            router_logits=router_logits,
            activation=layer.activation,
            global_num_experts=layer.global_num_experts,
            expert_map=layer.expert_map,
            apply_router_weight_on_input=layer.apply_router_weight_on_input,
        )