import argparse import math import random import torch import triton from flash_mla import flash_mla_with_kvcache_fp8, get_mla_decoding_metadata_dense_fp8 torch.set_printoptions(precision=4, profile="default", sci_mode=False) def scaled_dot_product_attention(query, key, value, h_q, h_kv, is_causal=False, k_scale=1.0): query = query.float() key = key.float() * k_scale value = value.float() * k_scale key = key.repeat_interleave(h_q // h_kv, dim=0) value = value.repeat_interleave(h_q // h_kv, dim=0) attn_weight = query @ key.transpose(-2, -1) / math.sqrt(query.size(-1)) if is_causal: s_q = query.shape[-2] s_k = key.shape[-2] attn_bias = torch.zeros(s_q, s_k, dtype=query.dtype) temp_mask = torch.ones(s_q, s_k, dtype=torch.bool).tril(diagonal=s_k - s_q) attn_bias.masked_fill_(temp_mask.logical_not(), float("-inf")) attn_bias.to(query.dtype) attn_weight += attn_bias lse = attn_weight.logsumexp(dim=-1) attn_weight = torch.softmax(attn_weight, dim=-1, dtype=torch.float32) return attn_weight @ value, lse def cal_diff(x: torch.Tensor, y: torch.Tensor, name: str) -> None: torch_dtype = x.dtype x, y = x.double(), y.double() RMSE = ((x - y) * (x - y)).mean().sqrt().item() cos_diff = 1 - 2 * (x * y).sum().item() / max((x * x + y * y).sum().item(), 1e-12) amax_diff = (x - y).abs().max().item() print(f"{name}: {cos_diff=}, {RMSE=}, {amax_diff=}") assert cos_diff < (1e-4 if torch_dtype == torch.bfloat16 else 1e-5) @torch.inference_mode() def test_flash_mla_fp8_e5m2(b, s_q, mean_sk, h_q, h_kv, d, dv, causal, varlen, is_prof=False): print( f"{b=}, {s_q=}, {mean_sk=}, {h_q=}, {h_kv=}, {d=}, {dv=}, {causal=}, {varlen=}" ) cache_seqlens = torch.full((b,), mean_sk, dtype=torch.int32) if varlen: for i in range(b): cache_seqlens[i] = max(random.normalvariate(mean_sk, mean_sk / 2), s_q) total_seqlens = cache_seqlens.sum().item() mean_seqlens = cache_seqlens.float().mean().int().item() max_seqlen = cache_seqlens.max().item() max_seqlen_pad = triton.cdiv(max_seqlen, 256) * 256 print(f"{total_seqlens=}, {mean_seqlens=}, {max_seqlen=}, {max_seqlen_pad=}") q = torch.randn(b, s_q, h_q, d) # q = torch.ones(b, s_q, h_q, d) block_size = 64 block_table = torch.arange( b * max_seqlen_pad // block_size, dtype=torch.int32 ).view(b, max_seqlen_pad // block_size) # blocked_k = torch.randint(low=0, high=4, size = (block_table.numel(), block_size, h_kv, d), dtype = torch.int8) # blocked_k = torch.ones(size = (block_table.numel(), block_size, h_kv, d), dtype = torch.int8) blocked_k = (torch.randn(block_table.numel(), block_size, h_kv, d)).to(torch.float8_e5m2) # blocked_k[0, 0, 0, 56] = 1 # blocked_k[0, 1, 0, 8] = 2 # blocked_k[0, 2, 0, 8] = 5 # blocked_k[0, 3, 0, 8] = 4 # for i in range(64): # for j in range(64): # blocked_k[0, i, 0, j] = j # blocked_k[0, i, 0, j] = (i * 50 + j) % 128 # print("blocked_k ", blocked_k[0, 0, 0, 0:10]) # for i in range(b): # blocked_k.view(b, max_seqlen_pad, h_kv, d)[i, cache_seqlens[i].item():] = ( # -128 # ) blocked_v = blocked_k[..., :dv] tile_scheduler_metadata, num_splits = get_mla_decoding_metadata_dense_fp8( cache_seqlens, s_q * h_q // h_kv, h_kv ) # print("q:", q.shape, q.dtype, q) # print("cache_seqlens:", cache_seqlens.shape, cache_seqlens) # print("block_table:", block_table.shape, block_table) # print("blocked_k:", blocked_k.shape, blocked_k[0]) # print("blocked_v:", blocked_v.shape) # torch.set_printoptions(precision=4, profile="full", sci_mode=False) # print("tile_scheduler_metadata:", tile_scheduler_metadata.shape, tile_scheduler_metadata) # torch.set_printoptions(precision=4, profile="default", sci_mode=False) # print("num_splits:", num_splits.shape, num_splits) # k_scale = torch.tensor(1.0).to(torch.float32).to("cuda:0") # k_scale = torch.tensor(2.1).to(torch.float32).to("cuda:0") descale_q = torch.ones((1), dtype=torch.float32) descale_k = torch.ones((1), dtype=torch.float32) def flash_mla(): return flash_mla_with_kvcache_fp8( q, blocked_k, block_table, cache_seqlens, dv, tile_scheduler_metadata, num_splits, causal=causal, descale_q = descale_q, descale_k = descale_k, ) def ref_mla(): out = torch.empty(b, s_q, h_q, dv, dtype=torch.float32) lse = torch.empty(b, h_q, s_q, dtype=torch.float32) for i in range(b): begin = i * max_seqlen_pad end = begin + cache_seqlens[i] O, LSE = scaled_dot_product_attention( q[i].transpose(0, 1), blocked_k.view(-1, h_kv, d)[begin:end].transpose(0, 1), blocked_v.view(-1, h_kv, dv)[begin:end].transpose(0, 1), h_q=h_q, h_kv=h_kv, is_causal=causal, k_scale = k_scale ) out[i] = O.transpose(0, 1) lse[i] = LSE return out, lse out_flash, lse_flash = flash_mla() out_torch, lse_torch = ref_mla() # print("out_flash ", out_flash[0, 0, 0, 0:14]) # print("out_torch ", out_torch[0, 0, 0, 0:14]) # print("lse_flash ", lse_flash[0, 0, 0:10]) # print("lse_torch ", lse_torch[0, 0, 0:10]) # print("out max_diff ", (out_flash - out_torch).abs().max()) # print("lse max_diff ", (lse_flash - lse_torch).abs().max()) # print(" out ", torch.nonzero((out_flash - out_torch).abs())) # print(" out_torch", out_torch) cal_diff(lse_flash, lse_torch, "lse") cal_diff(out_flash, out_torch, "out") t = triton.testing.do_bench(flash_mla) FLOPS = s_q * total_seqlens * h_q * (d + dv) * 2 bytes = ( b * s_q * h_q * d + b * s_q * h_q * dv) * ( torch.finfo(q.dtype).bits // 8 ) + total_seqlens * h_kv * d print( f"{t:.3f} ms, {FLOPS / 10 ** 9 / t:.0f} TFLOPS, {bytes / 10 ** 6 / t:.0f} GB/s" ) def main(torch_dtype, is_prof=False): device = torch.device("cuda:0") torch.set_default_dtype(torch_dtype) torch.set_default_device(device) torch.cuda.set_device(device) torch.manual_seed(0) random.seed(0) ''' h_kv = 1 d, dv = 576, 512 causal = True for b in [128]: for s in [4096, 8192]: for h_q in [16, 32, 64, 128]: # TP = 8, 4, 2, 1 for s_q in [1, 2]: # MTP = 1, 2 for varlen in [False, True]: test_flash_mla(b, s_q, s, h_q, h_kv, d, dv, causal, varlen) # b, s_q, s, h_q, h_kv, d, dv, causal, varlen''' # test_flash_mla( 1, 1, 64, 16, 1, 576, 512, True, False, is_prof=is_prof) # test_flash_mla_fp8( 1, 1, 1000, 1, 1, 576, 512, True, False, is_prof=is_prof) # test_flash_mla_fp8( 1, 1, 4096, 8, 1, 576, 512, True, False, is_prof=is_prof) # test_flash_mla_fp8(32, 1, 4096, 16, 1, 576, 512, False, False, is_prof=is_prof) # ''' h_kv = 1 d, dv = 576, 512 causal = True # for b in [1, 32]: # for s in [200, 1002, 2002, 1024, 2000, 4000, 32768, 65536]: # for h_q in [4, 16, 32, 64]: # for s_q in [1, 2]: # MTP = 1, 2 # for varlen in [True]: # test_flash_mla(b, s_q, s, h_q, h_kv, d, dv, causal, varlen) # for b in [32]: # for s in [16384, 32768, 65536*2]: # for h_q in [16]: # for s_q in [1]: # MTP = 1, 2 # for varlen in [False]: # # for varlen in [True]: # test_flash_mla_fp8_e5m2(b, s_q, s, h_q, h_kv, d, dv, causal, varlen) # test_flash_mla_fp8_e4m3(b, s_q, s, h_q, h_kv, d, dv, causal, varlen) # ''' for b in [3, 6, 9, 12, 15, 18, 21, 24]: for s in [111, 112, 123, 1234, 432, 4325, 4000, 8192, 11111]: for h_q in [16, 128]: for s_q in [1, 2, 3]: # MTP = 1, 2 for varlen in [False, True]: test_flash_mla_fp8_e5m2(b, s_q, s, h_q, h_kv, d, dv, causal, varlen,True) for b in [3, 6, 9, 12, 15, 18, 21, 24, 32, 64, 128, 256]: for s in [4000]: for h_q in [16, 128]: for s_q in [1]: # MTP = 1, 2 for varlen in [False]: test_flash_mla_fp8_e5m2(b, s_q, s, h_q, h_kv, d, dv, causal, varlen) for b in [3, 6, 9, 12, 15, 18, 21, 24, 32, 64, 128, 256]: for s in [4000]: for h_q in [16]: for s_q in [1, 2, 3, 4]: # MTP = 1, 2 for varlen in [False]: test_flash_mla_fp8_e5m2(b, s_q, s, h_q, h_kv, d, dv, causal, varlen) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "--dtype", type=str, choices=["bf16", "fp16"], default="bf16", help="Data type to use for testing (bf16 or fp16)", ) parser.add_argument('--prof', default=False, action='store_true', help='prof or not') args = parser.parse_args() torch_dtype = torch.bfloat16 if args.dtype == "fp16": torch_dtype = torch.float16 main(torch_dtype, args.prof)