test_fusion_attn.py 17.1 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import copy
4
5
6
7

import pytest
import torch._dynamo

8
from tests.compile.backend import LazyInitPass, TestBackend
9
from tests.utils import flat_product
10
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
11
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
12
from vllm.attention.backends.abstract import AttentionMetadata
13
from vllm.attention.backends.registry import AttentionBackendEnum
14
from vllm.attention.layer import Attention
15
16
from vllm.compilation.fusion_attn import ATTN_OP, AttnFusionPass
from vllm.compilation.fx_utils import find_op_nodes
17
from vllm.compilation.matcher_utils import QUANT_OPS
18
from vllm.compilation.noop_elimination import NoOpEliminationPass
19
from vllm.compilation.post_cleanup import PostCleanupPass
20
from vllm.config import (
21
    AttentionConfig,
22
23
    CacheConfig,
    CompilationConfig,
24
    CompilationMode,
25
26
27
28
29
30
    ModelConfig,
    PassConfig,
    SchedulerConfig,
    VllmConfig,
    set_current_vllm_config,
)
31
32
from vllm.forward_context import get_forward_context, set_forward_context
from vllm.model_executor.layers.quantization.utils.quant_utils import (
33
    QuantKey,
34
35
36
37
    kFp8StaticTensorSym,
    kNvfp4Quant,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import Fp8LinearOp
38
from vllm.platforms import current_platform
39
from vllm.utils.flashinfer import has_flashinfer
40
41
42
from vllm.v1.kv_cache_interface import AttentionSpec

FP8_DTYPE = current_platform.fp8_dtype()
43
FP4_DTYPE = torch.uint8
44
45


46
47
class AttentionQuantPatternModel(torch.nn.Module):
    """Base model for AttentionQuantPattern fusion."""
48

49
50
51
52
53
54
55
56
57
58
    def __init__(
        self,
        num_qo_heads: int,
        num_kv_heads: int,
        head_size: int,
        kv_cache_dtype: torch.dtype,
        device: torch.device,
        vllm_config: VllmConfig,
        **kwargs,
    ):
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
        super().__init__()
        self.num_qo_heads = num_qo_heads
        self.num_kv_heads = num_kv_heads
        self.head_size = head_size
        self.kv_cache_dtype = kv_cache_dtype
        self.device = device
        self.vllm_config = vllm_config

        self.attn = Attention(
            num_heads=self.num_qo_heads,
            head_size=self.head_size,
            scale=1.0 / (self.head_size**0.5),
            num_kv_heads=self.num_kv_heads,
            cache_config=vllm_config.cache_config,
            prefix="model.layers.0.self_attn.attn",
        )
75
76
        self.attn._k_scale = self.attn._k_scale.to(device)
        self.attn._v_scale = self.attn._v_scale.to(device)
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92

        self.block_size = 16

        # Initialize attn MetadataBuilder
        self.builder = self.attn.attn_backend.get_builder_cls()(
            kv_cache_spec=AttentionSpec(
                block_size=self.block_size,
                num_kv_heads=self.num_kv_heads,
                head_size=self.head_size,
                dtype=self.kv_cache_dtype,
            ),
            layer_names=[self.attn.layer_name],
            vllm_config=self.vllm_config,
            device=self.device,
        )

93
    def build_attn_metadata(self, batch_size: int) -> AttentionMetadata:
94
95
96
        """Initialize attention metadata."""

        # Create common attn metadata
97
        batch_spec = BatchSpec(seq_lens=[1] * batch_size, query_lens=[1] * batch_size)
98
        common_attn_metadata = create_common_attn_metadata(
99
100
            batch_spec, self.block_size, self.device, arange_block_indices=True
        )
101

102
        max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size
103
        num_blocks = batch_size * max_blocks
104
        backend = self.attn.backend
105

106
        # TODO(luka) use get_kv_cache_stride_order
107
        # Create dummy KV cache for the selected backend
108
        if backend == AttentionBackendEnum.ROCM_ATTN:
109
            # k/v as 1st dimention
110
111
112
113
114
115
116
117
118
119
            # HND: [num_blocks, num_kv_heads, block_size, head_size]
            kv_cache = torch.zeros(
                2,
                num_blocks,
                self.num_kv_heads,
                self.block_size,
                self.head_size,
                dtype=self.kv_cache_dtype,
                device=self.device,
            )
120
        elif backend == AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN:
121
122
123
124
125
126
127
128
129
130
131
            # k/v as 1st dimention
            # NHD: [num_blocks, block_size, num_kv_heads, head_size]
            kv_cache = torch.zeros(
                2,
                num_blocks,
                self.block_size,
                self.num_kv_heads,
                self.head_size,
                dtype=self.kv_cache_dtype,
                device=self.device,
            )
132
        elif backend == AttentionBackendEnum.TRITON_ATTN:
133
            # k/v as 2nd dimention
134
135
136
137
138
139
140
141
142
143
            # NHD: [num_blocks, block_size, num_kv_heads, head_size]
            kv_cache = torch.zeros(
                num_blocks,
                2,
                self.num_kv_heads,
                self.block_size,
                self.head_size,
                dtype=self.kv_cache_dtype,
                device=self.device,
            )
144
        elif backend == AttentionBackendEnum.FLASHINFER:
145
146
147
148
149
150
151
152
153
154
155
            kv_cache = torch.zeros(
                num_blocks,
                2,
                self.num_kv_heads,
                self.block_size,
                self.head_size,
                dtype=self.kv_cache_dtype,
                device=self.device,
            ).permute(0, 1, 3, 2, 4)
        else:
            raise ValueError(f"Unsupported backend: {backend}")
156
157
158
159
        self.attn.kv_cache = [kv_cache]

        # Build attn metadata
        self.attn_metadata = self.builder.build(
160
161
            common_prefix_len=0, common_attn_metadata=common_attn_metadata
        )
162
163
164

        return self.attn_metadata

165
166
167
168
169
170
171
172
173
174
175

class TestAttentionFp8StaticQuantPatternModel(AttentionQuantPatternModel):
    """Test model for AttentionFp8StaticQuantPattern fusion."""

    quant_key = kFp8StaticTensorSym

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.fp8_linear = Fp8LinearOp(
            act_quant_static=self.quant_key.scale.static,
176
177
            act_quant_group_shape=self.quant_key.scale.group_shape,
        )
178
179
180

        hidden_size = self.num_qo_heads * self.head_size
        self.w = kwargs.get(
181
182
183
184
185
186
187
188
189
            "w",
            {
                "weight": torch.randn(hidden_size, hidden_size)
                .to(dtype=FP8_DTYPE, device=self.device)
                .t(),
                "wscale": torch.tensor([1.0], dtype=torch.float32, device=self.device),
                "scale": torch.tensor([1.0], dtype=torch.float32, device=self.device),
            },
        )
190
191

    def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor):
192
193
        """Forward pass that creates the pattern to be fused."""
        attn_output = self.attn(q, k, v)
194
195
196
197
198
199
        return self.fp8_linear.apply(
            input=attn_output,
            weight=self.w["weight"],
            weight_scale=self.w["wscale"],
            input_scale=self.w["scale"],
        )
200
201
202
203
204
205
206
207
208
209
210
211


class TestAttentionNvfp4QuantPatternModel(AttentionQuantPatternModel):
    """Test model for AttentionNvfp4QuantPattern fusion."""

    quant_key = kNvfp4Quant

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        hidden_size = self.num_qo_heads * self.head_size
        self.w = kwargs.get(
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
            "w",
            {
                "weight": torch.randint(
                    256,
                    (hidden_size, hidden_size // 2),
                    dtype=FP4_DTYPE,
                    device=self.device,
                ),
                "wscale_swizzled": torch.randn(hidden_size, hidden_size // 16).to(
                    dtype=FP8_DTYPE, device=self.device
                ),
                "wscale": torch.tensor([500], dtype=torch.float32, device=self.device),
                "scale": torch.tensor([0.002], dtype=torch.float32, device=self.device),
            },
        )
227
228
229
230
231

    def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor):
        """Forward pass that creates the pattern to be fused."""
        attn_output = self.attn(q, k, v)
        quant_output, output_block_scale = scaled_fp4_quant(
232
233
234
235
236
237
238
239
240
241
            attn_output, 1 / self.w["scale"]
        )
        return cutlass_scaled_fp4_mm(
            a=quant_output,
            b=self.w["weight"],
            block_scale_a=output_block_scale,
            block_scale_b=self.w["wscale_swizzled"],
            alpha=self.w["scale"] * self.w["wscale"],
            out_dtype=attn_output.dtype,
        )
242
243


244
245
246
247
MODELS_FP8: list[tuple[str, type]] = []
MODELS_FP4: list[tuple[str, type]] = []
HEADS: list[tuple[int, int]] = []
SPLIT_ATTENTION: list[bool] = []
248
249
BACKENDS_FP8: list[AttentionBackendEnum] = []
BACKENDS_FP4: list[AttentionBackendEnum] = []
250

251
if current_platform.is_cuda():
252
253
    HEADS = [(64, 8), (40, 8)]
    MODELS_FP8 = [
254
255
256
        (
            "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8",
            TestAttentionFp8StaticQuantPatternModel,
257
258
259
        )
    ]
    MODELS_FP4 = [
260
261
262
        (
            "nvidia/Llama-4-Scout-17B-16E-Instruct-FP4",
            TestAttentionNvfp4QuantPatternModel,
263
        )
264
    ]
265
266
    BACKENDS_FP8 = [AttentionBackendEnum.TRITON_ATTN, AttentionBackendEnum.FLASHINFER]
    BACKENDS_FP4 = [AttentionBackendEnum.FLASHINFER]
267

268
elif current_platform.is_rocm():
269
270
    HEADS = [(32, 8), (40, 8)]
    MODELS_FP8 = [
271
272
        ("amd/Llama-3.1-8B-Instruct-FP8-KV", TestAttentionFp8StaticQuantPatternModel)
    ]
273
    BACKENDS = [
274
275
276
        AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
        AttentionBackendEnum.ROCM_ATTN,
        AttentionBackendEnum.TRITON_ATTN,
277
    ]
278
279
280


@pytest.mark.parametrize("num_qo_heads, num_kv_heads", HEADS)
281
@pytest.mark.parametrize("head_size", [128])
282
283
284
@pytest.mark.parametrize(
    "batch_size", [7, 256, 533] if current_platform.is_cuda() else [8]
)
285
286
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize(
287
288
289
290
291
    "backend, model_name, model_class, custom_ops",
    # Test attention+quant_fp8 fusion with custom and torch impls of QuantFP8
    list(flat_product(BACKENDS_FP8, MODELS_FP8, ["+quant_fp8", "-quant_fp8"]))
    # quant_fp4 only has the custom impl
    + list(flat_product(BACKENDS_FP4, MODELS_FP4, [""])),
292
293
294
295
)
@pytest.mark.skipif(
    not current_platform.is_cuda_alike(), reason="Only test ROCm or CUDA"
)
296
@pytest.mark.skipif(not current_platform.supports_fp8(), reason="Need FP8")
297
298
299
300
301
302
def test_attention_quant_pattern(
    num_qo_heads: int,
    num_kv_heads: int,
    head_size: int,
    batch_size: int,
    dtype: torch.dtype,
303
    custom_ops: str,
304
305
    model_name: str,
    model_class: type[AttentionQuantPatternModel],
306
    backend: AttentionBackendEnum,
307
308
    dist_init,
):
309
    """Test AttentionStaticQuantPattern fusion pass"""
310
    if backend == AttentionBackendEnum.FLASHINFER and (
311
312
313
        not current_platform.is_device_capability((10, 0)) or not has_flashinfer()
    ):
        pytest.skip("FlashInfer attn fusion requires Blackwell and flashinfer")
314

315
    custom_ops_list = custom_ops.split(",") if custom_ops else []
316

317
    device = torch.device("cuda:0")
318
    torch.set_default_dtype(dtype)
319
320
    torch.manual_seed(42)

321
322
323
324
325
    model_config = ModelConfig(
        model=model_name,
        max_model_len=2048,
        dtype=dtype,
    )
326
    vllm_config = VllmConfig(
327
328
329
330
331
        model_config=model_config,
        scheduler_config=SchedulerConfig(
            max_num_seqs=1024,
            max_model_len=model_config.max_model_len,
            is_encoder_decoder=model_config.is_encoder_decoder,
332
333
        ),
        compilation_config=CompilationConfig(
334
            mode=CompilationMode.VLLM_COMPILE,
335
            custom_ops=custom_ops_list,
336
        ),
337
        cache_config=CacheConfig(cache_dtype="fp8"),
338
        attention_config=AttentionConfig(backend=backend),
339
    )
340
341

    # Create test inputs
342
343
344
    q = torch.randn(batch_size, num_qo_heads * head_size, dtype=dtype, device=device)
    k = torch.randn(batch_size, num_kv_heads * head_size, dtype=dtype, device=device)
    v = torch.randn(batch_size, num_kv_heads * head_size, dtype=dtype, device=device)
345
346
347
348
349
350
351
352

    # Mark first dimension as dynamic for realistic testing
    torch._dynamo.mark_dynamic(q, 0)
    torch._dynamo.mark_dynamic(k, 0)
    torch._dynamo.mark_dynamic(v, 0)

    # Run model directly without compilation and fusion
    vllm_config_unfused = copy.deepcopy(vllm_config)
353
354
355
356
357
358
359
360
361
362
363
364
    with (
        set_current_vllm_config(vllm_config_unfused),
        set_forward_context(attn_metadata=None, vllm_config=vllm_config_unfused),
    ):
        model_unfused = model_class(
            num_qo_heads=num_qo_heads,
            num_kv_heads=num_kv_heads,
            head_size=head_size,
            kv_cache_dtype=FP8_DTYPE,
            device=device,
            vllm_config=vllm_config_unfused,
        )
365
366
367
        model_unfused = model_unfused.to(device)

        forward_ctx = get_forward_context()
368
        forward_ctx.attn_metadata = model_unfused.build_attn_metadata(batch_size)
369

370
371
372
        # Run model directly without fusion
        # Still compile so query QuantFP8 has closer numerics
        result_unfused = torch.compile(model_unfused, fullgraph=True)(q, k, v)
373
374
375

    # Run model with attn fusion enabled
    vllm_config.compilation_config.pass_config = PassConfig(
376
        fuse_attn_quant=True, eliminate_noops=True
377
378
379
380
381
382
383
384
385
386
387
388
389
390
    )
    with (
        set_current_vllm_config(vllm_config),
        set_forward_context(attn_metadata=None, vllm_config=vllm_config),
    ):
        model_fused = model_class(
            num_qo_heads=num_qo_heads,
            num_kv_heads=num_kv_heads,
            head_size=head_size,
            kv_cache_dtype=FP8_DTYPE,
            device=device,
            vllm_config=vllm_config,
            w=model_unfused.w,
        )
391
392
393
        model_fused = model_fused.to(device)

        forward_ctx = get_forward_context()
394
        forward_ctx.attn_metadata = model_fused.build_attn_metadata(batch_size)
395
396
397

        # Create test backend with fusion passes enabled
        noop_pass = NoOpEliminationPass(vllm_config)
398
399
400
401
        attn_pass = LazyInitPass(AttnFusionPass, vllm_config)
        cleanup_pass = PostCleanupPass(vllm_config)

        test_backend = TestBackend(noop_pass, attn_pass, cleanup_pass)
402
403

        # Compile model with fusion enabled
404
405
406
        model_compiled = torch.compile(
            model_fused, backend=test_backend, fullgraph=True
        )
407
        assert model_compiled.attn._o_scale_float is None
408

409
        result_fused_1 = model_compiled(q, k, v)
410

411
        if backend == AttentionBackendEnum.FLASHINFER:
412
413
414
415
416
417
            # With the Flashinfer backend after the 1st round of the forward
            # pass, output quant scale should be loaded into the attn layer's
            # _o_scale_float, the 2nd round should reuse the loaded
            # _o_scale_float
            assert model_compiled.attn._o_scale_float is not None
            result_fused_2 = model_compiled(q, k, v)
418

419
420
            assert model_compiled.attn._o_scale_float is not None

421
422
423
            torch.testing.assert_close(
                result_unfused, result_fused_2, atol=1e-2, rtol=1e-2
            )
424
425

    # Check attn fusion support
426
    quant_key: QuantKey = model_class.quant_key
427
    attn_fusion_supported = [
428
429
        layer.impl.fused_output_quant_supported(quant_key)
        for key, layer in vllm_config.compilation_config.static_forward_context.items()
430
    ]
431
432
433
434
435
436
437
438
439
440
441
442
443
444
    assert sum(attn_fusion_supported) == len(attn_fusion_supported), (
        "All layers should support attention fusion"
    )

    # Check quantization ops in the graph before and after fusion
    quant_op = (
        torch.ops.aten.reciprocal
        if "-quant_fp8" in custom_ops_list
        else QUANT_OPS[quant_key]
    )

    # Note: for fp8, fully_replaced=False because query quant ops remain in graph.
    # Only output quant ops are fused into attention.
    test_backend.check_before_ops([quant_op], fully_replaced=quant_key is kNvfp4Quant)
445

446
447
448
    # access the underlying `AttnFusionPass` on the `LazyInitPass`
    assert attn_pass.pass_.matched_count == sum(attn_fusion_supported)

449
450
    # Check attention ops in the graph before and after fusion
    attn_nodes_pre = list(find_op_nodes(ATTN_OP, test_backend.graph_pre_pass))
451
    attn_nodes_post = list(find_op_nodes(ATTN_OP, test_backend.graph_post_pass))
452
453

    assert len(attn_nodes_pre) > 0, "Should have attention nodes before fusion"
454
    assert len(attn_nodes_pre) == len(attn_nodes_post), (
455
        "Should have same number of attention nodes before and after fusion"
456
457
    )
    assert attn_nodes_pre[0].kwargs.get("output_scale") is None, (
458
        "Attention should not have output_scale before fusion"
459
460
    )
    assert attn_nodes_post[0].kwargs.get("output_scale") is not None, (
461
        "Attention should have output_scale after fusion"
462
    )
463

464
    assert attn_nodes_pre[0].kwargs.get("output_block_scale") is None, (
465
        "Attention should not have output_block_scale before fusion"
466
    )
467
    if quant_key.dtype == FP8_DTYPE:
468
        assert attn_nodes_post[0].kwargs.get("output_block_scale") is None, (
469
            "Attention should not have output_block_scale after FP8 fusion"
470
        )
471
    elif quant_key.dtype == FP4_DTYPE:
472
473
        assert attn_nodes_post[0].kwargs.get("output_block_scale") is not None, (
            "Attention should have output_block_scale after FP4 fusion"
474
        )
475

476
    # Check that results are close
477
    torch.testing.assert_close(result_unfused, result_fused_1, atol=1e-2, rtol=1e-2)