
/******************************************/
/* Begin Kernel                           */
/******************************************/
.amdgcn_target "amdgcn-amd-amdhsa--gfx936"
.text
.protected Cijk_Alik_Bljk_HHS_BH_UserArgs_MT32x32x128_SN_K1_PGR4_TT2_2_WG16_16_2
.globl Cijk_Alik_Bljk_HHS_BH_UserArgs_MT32x32x128_SN_K1_PGR4_TT2_2_WG16_16_2
.p2align 8
.type Cijk_Alik_Bljk_HHS_BH_UserArgs_MT32x32x128_SN_K1_PGR4_TT2_2_WG16_16_2,@function
.section .rodata,#alloc
.p2align 6
.amdhsa_kernel Cijk_Alik_Bljk_HHS_BH_UserArgs_MT32x32x128_SN_K1_PGR4_TT2_2_WG16_16_2
  .amdhsa_user_sgpr_kernarg_segment_ptr 1
  .amdhsa_next_free_vgpr 94 // vgprs
  .amdhsa_next_free_sgpr 101 // sgprs
  .amdhsa_group_segment_fixed_size 65536 // lds bytes
  .amdhsa_private_segment_fixed_size 0
  .amdhsa_system_sgpr_workgroup_id_x 1
  .amdhsa_system_sgpr_workgroup_id_y 1
  .amdhsa_system_sgpr_workgroup_id_z 1
  .amdhsa_system_vgpr_workitem_id 0
  .amdhsa_float_denorm_mode_32 3
  .amdhsa_float_denorm_mode_16_64 3
.end_amdhsa_kernel
.text
/* Num VGPR   =36 */
/* Num AccVGPR=0 */
/* Num SGPR   =101 */

/******************************************/
/* Optimizations and Config:              */
/******************************************/
/* ThreadTile= 2 x 2 */
/* SubGroup= 16 x 16 */
/* VectorWidthA=-1 */
/* VectorWidthB=-1 */
/* GlobalReadVectorWidthA=1, GlobalReadVectorWidthB=1 */
/* DirectToLdsA=1 */
/* DirectToLdsB=1 */
/* UseSgprForGRO=0 */
.amdgpu_metadata
---
custom.config:
  InternalSupportParams:
    KernArgsVersion: 2
amdhsa.version:
  - 1
  - 1
amdhsa.kernels:
  - .name: Cijk_Alik_Bljk_HHS_BH_UserArgs_MT32x32x128_SN_K1_PGR4_TT2_2_WG16_16_2
    .symbol: 'Cijk_Alik_Bljk_HHS_BH_UserArgs_MT32x32x128_SN_K1_PGR4_TT2_2_WG16_16_2.kd'
    .language:                   OpenCL C
    .language_version:
      - 2
      - 0
    .args:
      - .name:            Gemm info
        .size:            4
        .offset:          0
        .value_kind:      by_value
        .value_type:      u32
      - .name:            kernel info0
        .size:            4
        .offset:          4
        .value_kind:      by_value
        .value_type:      u32
      - .name:            kernel info1
        .size:            4
        .offset:          8
        .value_kind:      by_value
        .value_type:      u32
      - .name:            numWG
        .size:            4
        .offset:          12
        .value_kind:      by_value
        .value_type:      u32
      - .name:            SizesFree0
        .size:            4
        .offset:          16
        .value_kind:      by_value
        .value_type:      u32
      - .name:            SizesFree1
        .size:            4
        .offset:          20
        .value_kind:      by_value
        .value_type:      u32
      - .name:            SizesFree2
        .size:            4
        .offset:          24
        .value_kind:      by_value
        .value_type:      u32
      - .name:            SizesSum0
        .size:            4
        .offset:          28
        .value_kind:      by_value
        .value_type:      u32
      - .name:            D
        .size:            8
        .offset:          32
        .value_kind:      global_buffer
        .value_type:      f16
        .address_space:   generic
      - .name:            C
        .size:            8
        .offset:          40
        .value_kind:      global_buffer
        .value_type:      f16
        .address_space:   generic
      - .name:            A
        .size:            8
        .offset:          48
        .value_kind:      global_buffer
        .value_type:      f16
        .address_space:   generic
      - .name:            B
        .size:            8
        .offset:          56
        .value_kind:      global_buffer
        .value_type:      f16
        .address_space:   generic
      - .name:            strideD0
        .size:            4
        .offset:          64
        .value_kind:      by_value
        .value_type:      u32
      - .name:            strideD1
        .size:            4
        .offset:          68
        .value_kind:      by_value
        .value_type:      u32
      - .name:            strideC0
        .size:            4
        .offset:          72
        .value_kind:      by_value
        .value_type:      u32
      - .name:            strideC1
        .size:            4
        .offset:          76
        .value_kind:      by_value
        .value_type:      u32
      - .name:            strideA0
        .size:            4
        .offset:          80
        .value_kind:      by_value
        .value_type:      u32
      - .name:            strideA1
        .size:            4
        .offset:          84
        .value_kind:      by_value
        .value_type:      u32
      - .name:            strideB0
        .size:            4
        .offset:          88
        .value_kind:      by_value
        .value_type:      u32
      - .name:            strideB1
        .size:            4
        .offset:          92
        .value_kind:      by_value
        .value_type:      u32
      - .name:            alpha
        .size:            4
        .offset:          96
        .value_kind:      by_value
        .value_type:      f32
      - .name:            beta
        .size:            4
        .offset:          100
        .value_kind:      by_value
        .value_type:      f32
      - .name:            AddressDbg
        .size:            8
        .offset:          104
        .value_kind:      global_buffer
        .value_type:      struct
        .address_space:   generic
      - .name:            dstD
        .size:            8
        .offset:          112
        .value_kind:      global_buffer
        .value_type:      f16
        .address_space:   generic
      - .name:            Synchronizer
        .size:            8
        .offset:          120
        .value_kind:      global_buffer
        .value_type:      f32
        .address_space:   generic
      - .name:            GSUSync
        .size:            4
        .offset:          128
        .value_kind:      by_value
        .value_type:      u32
    .group_segment_fixed_size:   65536
    .kernarg_segment_align:      8
    .kernarg_segment_size:       136
    .max_flat_workgroup_size:    512
    .private_segment_fixed_size: 0
    .sgpr_count:                 101
    .sgpr_spill_count:           0
    .vgpr_count:                 90
    .vgpr_spill_count:           0
    .wavefront_size:             64
...
.end_amdgpu_metadata
Cijk_Alik_Bljk_HHS_BH_UserArgs_MT32x32x128_SN_K1_PGR4_TT2_2_WG16_16_2:
label_ASM_Start:  /// Main body of the asm kernel

.set vgprValuC, 0
.set vgprValuA_X0_I0, 4
.set vgprValuA_X1_I0, 4
.set vgprValuB_X0_I0, 36
.set vgprValuB_X1_I0, 36

//user define
.set vgprTemp0,      68
.set vgprTemp1,      69
.set vgprTemp2,      70
.set vgprTemp3,      71
.set vgprGlobalWriteOffsetD,    72
.set vgprLocalReadAddrA,        88
.set vgprLocalReadAddrB,        91
.set vgprKeepSgprValue,         79
.set vgprSerial,     80
.set vgprAddressDbg, 81        //debugbuffer
.set vgprDebugTmp,   83        //debugbuffer
.set vgprGlobalReadOffsetA,     84
.set vgprGlobalReadOffsetB,     86


.set BufferLimit, 0xffffffff   //0xffffffff
.set BufferOOB, 0x80000000
.set Srd127_96, 0x00020000
.set laneSrdA0, 0
.set laneSrdA1, 1
.set laneSrdA2, 2
.set laneAddressB0, 4
.set laneAddressB1, 5
.set laneAddressC0, 6
.set laneAddressC1, 7
.set laneAddressD0, 8
.set laneAddressD1, 9
.set laneSizesFree1, 12
.set laneWorkGroup1, 13
.set laneStrideD, 14
.set laneLoopCnt, 15

.set laneAddressWSA0, 16
.set laneAddressWSA1, 17
.set laneAddressWSA2, 18
.set laneAddressWSA3, 19
.set laneAddressWSB0, 20
.set laneAddressWSB1, 21
.set laneAddressWSB2, 22
.set laneAddressWSB3, 23

.set laneAddressDout0, 24
.set laneAddressDout1, 25
.set laneAddressSync0, 26
.set laneAddressSync1, 27
.set laneNumGroup, 28
.set laneIncA, 29
.set laneIncB, 30
.set laneWorkGroup1, 31

.set laneSrdB0, 32
.set laneSrdB1, 33
.set laneSrdB2, 34
.set laneldsWrA, 35
.set laneldsWrB, 36
.set lanelimitA0, 37
.set lanelimitA1, 38
.set lanelimitB0, 39
.set lanelimitB1, 40
.set laneAddressWSD0, 41
.set laneAddressWSD1, 42
.set laneAddressWSD2, 43
.set laneAddressWSD3, 44


//Args define
.set sgprKernArgAddress, 0
.set sgprWorkGroup0, 2
.set sgprWorkGroup1, 3
.set sgprWorkGroup2, 4
.set sgprArgType, 5
.set sgprGSUSumIdx, 6
.set sgprNumWorkGroups1, 7
.set sgprGSULog2BpeC, 8
.set sgprGSULog2BpeD, 9
.set sgprStaggerU, 10
.set sgprLoopCounterL, 11
.set sgprGemmCount, 12
.set sgprGSU, 13
.set sgprWGM, 14
.set sgprNumWorkGroups0, 15
.set sgprSrdD, 16
.set sgprSrdC, 20

.set sgprSizesFree, 24
.set sgprSizesSum, 27
.set sgprAddressD, 28
.set sgprAddressC, 30
.set sgprAddressA, 32
.set sgprAddressB, 34
.set sgprStridesD, 36
.set sgprStridesC, 38
.set sgprStridesA, 40
.set sgprStridesB, 42
.set sgprAlpha, 44
.set sgprBeta, 45
.set sgprWGMBuffer, 46
.set sgprAddressDbg, 48     //定义debug buffer address


//user define
.set sgprWorkGroup0Ori, 50
.set sgprLoopCntCommon, 51
.set sgprSrdA, 52
.set sgprSrdB, 56
.set sgprShadowLimitA, 60
.set sgprShadowLimitB, 62
//.set sgprStaggerUIter, 49
//.set sgprWrapUA, 62
//.set sgprWrapUB, 64
.set sgprGlobalReadIncsA, 66
.set sgprGlobalReadIncsB, 67
.set sgprPackKForV0, 68
.set sgprPackKForV1, 69
.set sgprPackKForV2, 70
.set sgprPackKForV3, 71
.set sgprLocalWriteAddrA, 72
.set sgprLocalWriteAddrB, 73
.set sgprWaveID, 74
.set sgprLDSMask, 75
.set sgprLoopforPfIter, 76

.set sgprLDSWriteIter, 78
.set sgprTemp0, 80
.set sgprTemp1, 81
.set sgprTemp2, 82
.set sgprTemp3, 83
.set sgprTensor2dSizeA, 84
.set sgprTensor2dSizeB, 86
.set sgprWaveID_M, 88
.set sgprWaveID_N, 89
.set sgprLocalWriteAddrAori, 90
.set sgprLocalWriteAddrBori, 91
.set sgprGlWaveID, 92
.set sgprD_MEdge, 93
.set sgprTemp4, 96
.set sgprTemp5, 97
.set sgprTemp6, 98
.set sgprTemp7, 99
.set sgprStrideStructOffset, 100
.set sgprStructStrideA, 76
.set sgprStructStrideB, 77



.macro MMAC_16x16_part0_0
v_mmac_f32_16x16x16_f16 v[vgprValuC+0+0:vgprValuC+0+1:vgprValuC+0+2:vgprValuC+0+3] v[vgprValuA_X0_I0+0:vgprValuA_X0_I0+1] v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+1] v[vgprValuC+0+0:vgprValuC+0+1:vgprValuC+0+2:vgprValuC+0+3]//
s_setprio 1
v_mmac_f32_16x16x16_f16 v[vgprValuC+0+0:vgprValuC+0+1:vgprValuC+0+2:vgprValuC+0+3] v[vgprValuA_X0_I0+2:vgprValuA_X0_I0+3] v[vgprValuB_X0_I0+2:vgprValuB_X0_I0+3] v[vgprValuC+0+0:vgprValuC+0+1:vgprValuC+0+2:vgprValuC+0+3]// 
v_mmac_f32_16x16x16_f16 v[vgprValuC+0+0:vgprValuC+0+1:vgprValuC+0+2:vgprValuC+0+3] v[vgprValuA_X0_I0+4:vgprValuA_X0_I0+5] v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+5] v[vgprValuC+0+0:vgprValuC+0+1:vgprValuC+0+2:vgprValuC+0+3]// 
v_mmac_f32_16x16x16_f16 v[vgprValuC+0+0:vgprValuC+0+1:vgprValuC+0+2:vgprValuC+0+3] v[vgprValuA_X0_I0+6:vgprValuA_X0_I0+7] v[vgprValuB_X0_I0+6:vgprValuB_X0_I0+7] v[vgprValuC+0+0:vgprValuC+0+1:vgprValuC+0+2:vgprValuC+0+3]// 
v_mmac_f32_16x16x16_f16 v[vgprValuC+0+0:vgprValuC+0+1:vgprValuC+0+2:vgprValuC+0+3] v[vgprValuA_X0_I0+8:vgprValuA_X0_I0+9] v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+9] v[vgprValuC+0+0:vgprValuC+0+1:vgprValuC+0+2:vgprValuC+0+3]//
v_mmac_f32_16x16x16_f16 v[vgprValuC+0+0:vgprValuC+0+1:vgprValuC+0+2:vgprValuC+0+3] v[vgprValuA_X0_I0+10:vgprValuA_X0_I0+11] v[vgprValuB_X0_I0+10:vgprValuB_X0_I0+11] v[vgprValuC+0+0:vgprValuC+0+1:vgprValuC+0+2:vgprValuC+0+3]// 
v_mmac_f32_16x16x16_f16 v[vgprValuC+0+0:vgprValuC+0+1:vgprValuC+0+2:vgprValuC+0+3] v[vgprValuA_X0_I0+12:vgprValuA_X0_I0+13] v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+13] v[vgprValuC+0+0:vgprValuC+0+1:vgprValuC+0+2:vgprValuC+0+3]// 
v_mmac_f32_16x16x16_f16 v[vgprValuC+0+0:vgprValuC+0+1:vgprValuC+0+2:vgprValuC+0+3] v[vgprValuA_X0_I0+14:vgprValuA_X0_I0+15] v[vgprValuB_X0_I0+14:vgprValuB_X0_I0+15] v[vgprValuC+0+0:vgprValuC+0+1:vgprValuC+0+2:vgprValuC+0+3]// 
s_setprio 0
.endm

.set MT0, 32
.set MT1, 32
//.set LDS_B_OFFSET, 4096
//.set LDS_BLK_OFFSET, 8192
//.set LDS_BLK_OFFSET_64Kmasked, 8192
.set LDS_B_OFFSET, 8192
.set LDS_BLK_OFFSET, 16384
.set LDS_BLK_OFFSET_64Kmasked, 16384
.set LOG2BPE, 1
.set BPE, 2
.set Log2BpeDest, 1
.set LOG2BpeCompute, 2
.set DEPTHU, 128
.set LOG2DEPTHU, 7
.set PFTLOOPS, 4
.set GLWAVES, 4
.set LOG2GLWAVES, 2
.set MperWAVE, 16
.set NperWAVE, 16

/* Load num of Gemms */
s_load_dword s[sgprGemmCount], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x0

/* Load GSU data */
s_load_dword s[sgprGSU], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x4

/* Load WGM data */
s_load_dword s[sgprWGM], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x8

s_add_u32 s[sgprKernArgAddress], s[sgprKernArgAddress], 16 // Shift common args
s_addc_u32 s[sgprKernArgAddress+1], s[sgprKernArgAddress+1], 0x0
s_load_dwordx16 s[24:39], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x0
s_load_dwordx4 s[40:43], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x40
s_load_dwordx2 s[44:45], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x50
s_load_dwordx2 s[sgprAddressDbg:sgprAddressDbg+1], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x58
s_waitcnt lgkmcnt(0)
s_mov_b32 s[sgprWorkGroup0Ori], s[sgprWorkGroup0]

s_and_b32 s[sgprStaggerU], s[sgprGSU], 0xffff0000  // Restore StaggerU related vars
s_lshr_b32 s[sgprStaggerU], s[sgprStaggerU], 0x10
s_and_b32 s[sgprGSU], s[sgprGSU], 0xffff           // Restore GSUConfig and GSU
v_mov_b32 v[vgprSerial], v0
v_readfirstlane_b32 s[sgprWaveID], v[vgprSerial]
s_lshr_b32 s[sgprWaveID], s[sgprWaveID], 6
s_and_b32 s[sgprGlWaveID], s[sgprWaveID], GLWAVES-1
s_and_b32 s[sgprWaveID_M], s[sgprWaveID], 1
s_lshr_b32 s[sgprWaveID_N], s[sgprWaveID], 1
s_mov_b32 s[sgprLDSMask], 0x10000


.set debug_buffer, 1
.if debug_buffer     //计算每个 workgroup 的 debugbuffer 的地址偏移
v_mov_b32 v4, s[sgprWorkGroup0Ori]                    // v1=wg1*nwg0+wg0
v_lshlrev_b32 v5, 0x8, v4                          // v1 = v1 * 256  //这里是 thread 总数，随着wave数量自行更改
v_mul_lo_u32 v6, 2, v5
v_add_u32 v7, v6, v[vgprSerial]                    // v1=tid+NT*(wg1*nwg0+wg0)=serial
v_mul_lo_u32 v8, 0x40, v7                          // v1=serial*nipt*4
v_mov_b32 v2, 0                                    // 
v_mov_b32 v3, s[sgprAddressDbg+1]                  // v3=AddressD1
v_add_co_u32 v[vgprAddressDbg], vcc, s[sgprAddressDbg], v8 // v[vgprAddressDbg]=AddrD* + serial*nipt*4
v_addc_co_u32 v[vgprAddressDbg+1], vcc, v3, v2, vcc // v[vgprAddressDbg]=AddrD* + serial*nipt*4
.endif

.if debug_buffer
v_mov_b32 v[vgprDebugTmp], v[vgprSerial]
flat_store_dword v[vgprAddressDbg:vgprAddressDbg+1], v[vgprDebugTmp] // debug dump store
v_add_u32 v[vgprAddressDbg], v[vgprAddressDbg], 0x4 // debug dump inc
.endif

.if debug_buffer
v_mov_b32 v[vgprDebugTmp], s[sgprWorkGroup0]
flat_store_dword v[vgprAddressDbg:vgprAddressDbg+1], v[vgprDebugTmp] // debug dump store
v_add_u32 v[vgprAddressDbg], v[vgprAddressDbg], 0x4 // debug dump inc
.endif

.if debug_buffer
v_mov_b32 v[vgprDebugTmp], s[sgprWorkGroup1]
flat_store_dword v[vgprAddressDbg:vgprAddressDbg+1], v[vgprDebugTmp] // debug dump store
v_add_u32 v[vgprAddressDbg], v[vgprAddressDbg], 0x4 // debug dump inc
.endif

.if debug_buffer
v_mov_b32 v[vgprDebugTmp], 0xaaaa
flat_store_dword v[vgprAddressDbg:vgprAddressDbg+1], v[vgprDebugTmp] // debug dump store
v_add_u32 v[vgprAddressDbg], v[vgprAddressDbg], 0x4 // debug dump inc
.endif
/******************************************/
/* Compute GroupID                        */
/******************************************/

v_mov_b32 v8, MT0                                  // set MT0 into sgpr
v_mov_b32 v7, s[sgprSizesFree+0]                   // set Free0 size
v_cvt_f32_u32 v6, v8                               // v6 = ceil(v7 / v8)
v_rcp_iflag_f32 v6, v6                             // v6 = ceil(v7 / v8)
v_cvt_f32_u32 v9, v7                               // v6 = ceil(v7 / v8)
v_mul_f32 v6, v6, v9                               // v6 = ceil(v7 / v8)
v_cvt_u32_f32 v6, v6                               // v6 = ceil(v7 / v8)
v_mul_u32_u24 v9, v6, v8                           // v6 = ceil(v7 / v8)
v_sub_u32 v9, v7, v9                               // v6 = ceil(v7 / v8)
v_cmp_ne_u32 vcc, v9, 0                            // v6 = ceil(v7 / v8)
v_addc_co_u32 v6, vcc, v6, 0, vcc                  // ceil
v_mov_b32 v8, MT1                                  // set MT1 into sgpr
v_mov_b32 v7, s[sgprSizesFree+1]                   // set Free1 size
v_readfirstlane_b32 s[sgprNumWorkGroups0], v6      // set back to numWorkGroup0
v_cvt_f32_u32 v6, v8                               // v6 = ceil(v7 / v8)
v_rcp_iflag_f32 v6, v6                             // v6 = ceil(v7 / v8)
v_cvt_f32_u32 v9, v7                               // v6 = ceil(v7 / v8)
v_mul_f32 v6, v6, v9                               // v6 = ceil(v7 / v8)
v_cvt_u32_f32 v6, v6                               // v6 = ceil(v7 / v8)
v_mul_u32_u24 v9, v6, v8                           // v6 = ceil(v7 / v8)
v_sub_u32 v9, v7, v9                               // v6 = ceil(v7 / v8)
v_cmp_ne_u32 vcc, v9, 0                            // v6 = ceil(v7 / v8)
v_addc_co_u32 v6, vcc, v6, 0, vcc                  // ceil
v_readfirstlane_b32 s[sgprNumWorkGroups1], v6      // set back to numWorkGroup1

/* remap wg from 1D(idxWG012) to 3D(wg2,wg1,wg0) */
/* wg2 = idxWG012 * smallMagicNumber(1/(numWG0*numWG1)) */
s_mul_i32 s78, s[sgprNumWorkGroups0], s[sgprNumWorkGroups1]
s_and_b32 s79, s[sgprGSU], 0x3fff                  // Restore GSU
s_mul_i32 s78, s78, s79
v_cvt_f32_u32 v6, s78                              // s78 = s[sgprWorkGroup0] / s78
v_rcp_iflag_f32 v6, v6                             // s78 = s[sgprWorkGroup0] / s78
v_cvt_f32_u32 v7, s[sgprWorkGroup0]                // s78 = s[sgprWorkGroup0] / s78
v_mul_f32 v6, v6, v7                               // s78 = s[sgprWorkGroup0] / s78
v_cvt_u32_f32 v6, v6                               // s78 = s[sgprWorkGroup0] / s78
v_mul_u32_u24 v7, v6, s78                          // s78 = s[sgprWorkGroup0] / s78
v_sub_u32 v7, s[sgprWorkGroup0], v7                // s78 = s[sgprWorkGroup0] / s78
v_cmpx_eq_u32 exec, v7, s78                        // s78 = s[sgprWorkGroup0] / s78
v_add_u32 v6, 1, v6                                // s78 = s[sgprWorkGroup0] / s78
s_mov_b64 exec, -1                                 // s78 = s[sgprWorkGroup0] / s78
v_readfirstlane_b32 s78, v6                        // quotient
s_mov_b32 s[sgprWorkGroup2], s78
/* idxWG01 = idxWG012 - wg2 * numWG0 * numWG1 */
s_mul_i32 s78, s[sgprNumWorkGroups1], s[sgprNumWorkGroups0]
s_mul_i32 s78, s78, s[sgprWorkGroup2]
s_mul_i32 s78, s78, s79
s_sub_u32 s[sgprWorkGroup0], s[sgprWorkGroup0], s78
/* wg1 = idxWG01 * smallMagicNumber(1/numWG0) */
v_cvt_f32_u32 v6, s[sgprNumWorkGroups0]            // s78 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0]
v_rcp_iflag_f32 v6, v6                             // s78 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0]
v_cvt_f32_u32 v7, s[sgprWorkGroup0]                // s78 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0]
v_mul_f32 v6, v6, v7                               // s78 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0]
v_cvt_u32_f32 v6, v6                               // s78 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0]
v_mul_u32_u24 v7, v6, s[sgprNumWorkGroups0]        // s78 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0]
v_sub_u32 v7, s[sgprWorkGroup0], v7                // s78 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0]
v_cmpx_eq_u32 exec, v7, s[sgprNumWorkGroups0]      // s78 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0]
v_add_u32 v6, 1, v6                                // s78 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0]
s_mov_b64 exec, -1                                 // s78 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0]
v_readfirstlane_b32 s78, v6                        // quotient
s_mov_b32 s[sgprWorkGroup1], s78
/* wg0 = idxWG01 - wg1 * numWG0 */
s_mul_i32 s78, s[sgprWorkGroup1], s[sgprNumWorkGroups0]
s_sub_u32 s[sgprWorkGroup0], s[sgprWorkGroup0], s78

/******************************************/
/* WrokGroup Mapping                      */
/******************************************/

/* graWorkGroup mapping */

s_and_b32 s80, s[sgprGSU], 0x3fff                  // Restore GSU
s_cmp_eq_u32 s80, 1                                // GSU == 1 ?
s_cbranch_scc1 label_GSU                           // branch if GSU == 1
// GSU-not-WGMapRR :nwg1 = (size1J + MT1J - 1) / MT1J;
s_and_b32 s80, s[sgprGSU], 0x4000                  // SCC = (GSUWGMRR == 1) ?
s_cbranch_scc1 label_GSUWGMRR                      // branch if GSUWGMRR == 1
s_and_b32 s80, s[sgprGSU], 0x3fff                  // Restore GSU
v_cvt_f32_u32 v6, s80                              // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s80
v_rcp_iflag_f32 v6, v6                             // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s80
v_cvt_f32_u32 v7, s[sgprWorkGroup1]                // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s80
v_mul_f32 v6, v6, v7                               // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s80
v_cvt_u32_f32 v6, v6                               // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s80
v_mul_u32_u24 v7, v6, s80                          // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s80
v_sub_u32 v7, s[sgprWorkGroup1], v7                // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s80
v_cmpx_eq_u32 exec, v7, s80                        // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s80
v_add_u32 v6, 1, v6                                // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s80
v_mov_b32 v7, 0                                    // s[sgprGSUSumIdx] = s[sgprWorkGroup1] % s80
s_mov_b64 exec, -1                                 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s80
v_readfirstlane_b32 s[sgprWorkGroup1], v6          // quotient
v_readfirstlane_b32 s[sgprGSUSumIdx], v7           // remainder
s_branch label_GSUWGMRR_End
label_GSUWGMRR:
v_cvt_f32_u32 v6, s[sgprNumWorkGroups1]            // s[sgprGSUSumIdx] = s[sgprWorkGroup1] / s[sgprNumWorkGroups1]
v_rcp_iflag_f32 v6, v6                             // s[sgprGSUSumIdx] = s[sgprWorkGroup1] / s[sgprNumWorkGroups1]
v_cvt_f32_u32 v7, s[sgprWorkGroup1]                // s[sgprGSUSumIdx] = s[sgprWorkGroup1] / s[sgprNumWorkGroups1]
v_mul_f32 v6, v6, v7                               // s[sgprGSUSumIdx] = s[sgprWorkGroup1] / s[sgprNumWorkGroups1]
v_cvt_u32_f32 v6, v6                               // s[sgprGSUSumIdx] = s[sgprWorkGroup1] / s[sgprNumWorkGroups1]
v_mul_u32_u24 v7, v6, s[sgprNumWorkGroups1]        // s[sgprGSUSumIdx] = s[sgprWorkGroup1] / s[sgprNumWorkGroups1]
v_sub_u32 v7, s[sgprWorkGroup1], v7                // s[sgprGSUSumIdx] = s[sgprWorkGroup1] / s[sgprNumWorkGroups1]
v_cmpx_eq_u32 exec, v7, s[sgprNumWorkGroups1]      // s[sgprGSUSumIdx] = s[sgprWorkGroup1] / s[sgprNumWorkGroups1]
v_add_u32 v6, 1, v6                                // s[sgprGSUSumIdx] = s[sgprWorkGroup1] / s[sgprNumWorkGroups1]
v_mov_b32 v7, 0                                    // s[sgprWorkGroup1] = s[sgprWorkGroup1] % s[sgprNumWorkGroups1]
s_mov_b64 exec, -1                                 // s[sgprGSUSumIdx] = s[sgprWorkGroup1] / s[sgprNumWorkGroups1]
v_readfirstlane_b32 s[sgprGSUSumIdx], v6           // quotient
v_readfirstlane_b32 s[sgprWorkGroup1], v7          // remainder
label_GSUWGMRR_End:
s_mov_b32 s[sgprGSULog2BpeC], Log2BpeDest
s_mov_b32 s[sgprGSULog2BpeD], LOG2BpeCompute
s_branch label_GSU_End
label_GSU:
s_mov_b64 s[sgprGSUSumIdx:sgprGSUSumIdx+1], 0      // Set GSUSumIdx to 0
s_mov_b32 s[sgprGSULog2BpeC], Log2BpeDest
s_mov_b32 s[sgprGSULog2BpeD], Log2BpeDest
label_GSU_End:


s_cmp_le_i32 s[sgprWGM], 1
s_cbranch_scc1 label_WGM
v_cvt_f32_u32 v6, s[sgprWGM]                       // WGM
v_rcp_iflag_f32 v6, v6                             // WGM
v_cvt_f32_u32 v7, s[sgprWorkGroup1]                // WGM
v_mul_f32 v6, v6, v7                               // WGM
v_cvt_u32_f32 v6, v6                               // WGM
v_mul_u32_u24 v7, v6, s[sgprWGM]                   // WGM
v_sub_u32 v7, s[sgprWorkGroup1], v7                // WGM
v_cmpx_eq_u32 exec, v7, s[sgprWGM]                 // WGM
v_add_u32 v6, 1, v6                                // WGM
s_mov_b64 exec, -1                                 // WGM
v_readfirstlane_b32 s76, v6                        // quotient
s_mul_i32 s77, s76, s[sgprWGM]                     // quotient * non-magic divisor
s_sub_u32 s77, s[sgprWorkGroup1], s77              // WorkGroup1=remainder
s_mul_i32 s77, s77, s[sgprNumWorkGroups0]          // (wg1 % WGM)*NumWorkGroups0
s_add_u32 s77, s77, s[sgprWorkGroup0]              // wgSerial = wg0 + (wg1 % WGM)*NumWorkGroups0
v_cvt_f32_u32 v6, s[sgprWGM]                       // WGM
v_rcp_iflag_f32 v6, v6                             // WGM
v_cvt_f32_u32 v7, s[sgprNumWorkGroups1]            // WGM
v_mul_f32 v6, v6, v7                               // WGM
v_cvt_u32_f32 v6, v6                               // WGM
v_mul_u32_u24 v7, v6, s[sgprWGM]                   // WGM
v_sub_u32 v7, s[sgprNumWorkGroups1], v7            // WGM
v_cmpx_eq_u32 exec, v7, s[sgprWGM]                 // WGM
v_add_u32 v6, 1, v6                                // WGM
s_mov_b64 exec, -1                                 // WGM
v_readfirstlane_b32 s78, v6                        // quotient
s_mul_i32 s79, s[sgprWGM], s78                     // quotient * non-magic divisor
s_sub_u32 s79, s[sgprNumWorkGroups1], s79          // NumWorkGroups1=remainder
s_cmp_eq_u32 s79, 0                                // remainder == 0 ?
s_cmov_b32 s79, s[sgprWGM]                         // remainder = WGM if remainder == 0
s_cmp_ge_u32 s76, s78                              // blockId >= numFullBlocks ?
s_cselect_b32 s78, s79, s[sgprWGM]
v_cvt_f32_u32 v6, s78                              // s[sgprWorkGroup0] = s77 / s78
v_rcp_iflag_f32 v6, v6                             // s[sgprWorkGroup0] = s77 / s78
v_cvt_f32_u32 v7, s77                              // s[sgprWorkGroup0] = s77 / s78
v_mul_f32 v6, v6, v7                               // s[sgprWorkGroup0] = s77 / s78
v_cvt_u32_f32 v6, v6                               // s[sgprWorkGroup0] = s77 / s78
v_mul_u32_u24 v7, v6, s78                          // s[sgprWorkGroup0] = s77 / s78
v_sub_u32 v7, s77, v7                              // s[sgprWorkGroup0] = s77 / s78
v_cmpx_eq_u32 exec, v7, s78                        // s[sgprWorkGroup0] = s77 / s78
v_add_u32 v6, 1, v6                                // s[sgprWorkGroup0] = s77 / s78
v_mov_b32 v7, 0                                    // s[sgprWorkGroup1] = s77 % s78
s_mov_b64 exec, -1                                 // s[sgprWorkGroup0] = s77 / s78
v_readfirstlane_b32 s[sgprWorkGroup0], v6          // quotient
v_readfirstlane_b32 s[sgprWorkGroup1], v7          // remainder
s_mul_i32 s[sgprWorkGroup1], s[sgprWorkGroup0], s78 // quotient * non-magic divisor
s_sub_u32 s[sgprWorkGroup1], s77, s[sgprWorkGroup1] // WorkGroup1=remainder
s_mul_i32 s76, s76, s[sgprWGM]                     // blockId * WGM
s_add_u32 s[sgprWorkGroup1], s[sgprWorkGroup1], s76 // wg1 += blockId * WGM
label_WGM:

/******************************************/
/* Generate Global A parameters ...       */
/******************************************/
.set COALESCE_THREAD_A, 8     //x4 load
.set LOG2_COALESCE_THREAD_A, 3     //x4 load


v_and_b32 v[vgprTemp0], 255, v[vgprSerial]        //
v_lshrrev_b32 v[vgprTemp1], 6, v[vgprTemp0]             //wave ID
v_and_b32 v[vgprTemp0], v[vgprSerial], 63               //set to 0~63
v_lshrrev_b32 v[vgprTemp0], 4, v[vgprTemp0]             //0..0 1..1 2..2 3..3 ... 7..7
v_lshlrev_b32 v[vgprTemp0], 2, v[vgprTemp0]             //0..0 4..4 8..8 12..12 ... 28..28
v_add_u32 v0, v[vgprTemp1], v[vgprTemp0]

v_and_b32 v1, 15, v[vgprSerial]          //COALESCE_THREAD
v_lshlrev_b32 v1, 4, v1


/* global read addresses: Perp offsets*/
v_mov_b32 v2 v0

/* global read addresses: Coalesce offsets*/
v_mov_b32 v3 v1

/* global read addresses: final offsets*/

v_mul_lo_u32 v[vgprTemp0], s[sgprStridesA], v2
v_lshlrev_b32 v[vgprTemp0], 1, v[vgprTemp0] 
v_add_co_u32 v[vgprGlobalReadOffsetA+0], vcc, v3, v[vgprTemp0]

v_add_u32 v2, 16, v2
v_mul_lo_u32 v[vgprTemp0], s[sgprStridesA], v2
v_lshlrev_b32 v[vgprTemp0], 1, v[vgprTemp0] 
v_add_co_u32 v[vgprGlobalReadOffsetA+1], vcc, v3, v[vgprTemp0]



/******************************************/
/* Generate Global B parameters ...       */
/******************************************/
.set COALESCE_THREAD_B, 8     //x4 load
.set LOG2_COALESCE_THREAD_B, 3     //x4 load


v_and_b32 v[vgprTemp0], 255, v[vgprSerial]        //
v_lshrrev_b32 v[vgprTemp1], 6, v[vgprTemp0]             //wave ID
v_and_b32 v[vgprTemp0], v[vgprSerial], 63               //set to 0~63
v_lshrrev_b32 v[vgprTemp0], 4, v[vgprTemp0]             //0..0 1..1 2..2 3..3 ... 7..7
v_lshlrev_b32 v[vgprTemp0], 2, v[vgprTemp0]             //0..0 4..4 8..8 12..12 ... 28..28
v_add_u32 v0, v[vgprTemp1], v[vgprTemp0]

v_and_b32 v1, 15, v[vgprSerial]          //COALESCE_THREAD
v_lshlrev_b32 v1, 4, v1



/* global read addresses: Perp offsets*/
v_mov_b32 v2 v0

/* global read addresses: Coalesce offsets*/
v_mov_b32 v3 v1

/* global read addresses: final offsets*/

v_mul_lo_u32 v[vgprTemp0], s[sgprStridesB], v2
v_lshlrev_b32 v[vgprTemp0], 1, v[vgprTemp0] 
v_add_co_u32 v[vgprGlobalReadOffsetB+0], vcc, v3, v[vgprTemp0]

v_add_u32 v2, 16, v2
v_mul_lo_u32 v[vgprTemp0], s[sgprStridesB], v2
v_lshlrev_b32 v[vgprTemp0], 1, v[vgprTemp0] 
v_add_co_u32 v[vgprGlobalReadOffsetB+1], vcc, v3, v[vgprTemp0]

//v_add_u32 v2, 8, v2
//v_mul_lo_u32 v[vgprTemp0], s[sgprStridesB], v2
//v_lshlrev_b32 v[vgprTemp0], 1, v[vgprTemp0] 
//v_add_co_u32 v[vgprGlobalReadOffsetB+2], vcc, v3, v[vgprTemp0]

//v_add_u32 v2, 8, v2
//v_mul_lo_u32 v[vgprTemp0], s[sgprStridesB], v2
//v_lshlrev_b32 v[vgprTemp0], 1, v[vgprTemp0] 
//v_add_co_u32 v[vgprGlobalReadOffsetB+3], vcc, v3, v[vgprTemp0]


/******************************************/
/* Generate Srd A/B parameters ...        */
/******************************************/

s_mul_hi_u32 s[sgprTemp1], s[sgprWorkGroup0], MT0                       // WorkGroup[00] * MT
s_mul_i32 s[sgprTemp0], s[sgprWorkGroup0], MT0                          // WorkGroup[00] * MT
s_mul_hi_u32 s[sgprTemp1], s[sgprTemp1], s[sgprStridesA+0]              // tlu=0, scaled tile-offset by stride
s_mul_i32 s[sgprTemp0], s[sgprTemp0], s[sgprStridesA+0]                 // tlu=0, scaled tile-offset by stride

// GSU processing
s_and_b32 s[sgprTemp2], s[sgprGSU], 0x8000                      // SCC = (GSUC == 1) ?
s_cbranch_scc1 label_GSUC_A                                     // branch if GSUC == 1
s_mul_hi_u32 s[sgprTemp3], DEPTHU, s[sgprGSUSumIdx]             // gsuOffset = DepthU*GSUSumIdx
s_mul_i32 s[sgprTemp2], DEPTHU, s[sgprGSUSumIdx]                // gsuOffset = DepthU*GSUSumIdx
s_branch label_GSUC_A_End
label_GSUC_A:
s_lshr_b32 s[sgprLoopCounterL], s[sgprSizesSum], LOG2DEPTHU     // s[LoopCounterL] = s[sgprSizesSum] / DEPTHU
s_and_b32 s[sgprGSUSumIdx+1], s[sgprGSU], 0x3fff                // Restore GSU
v_cvt_f32_u32 v0, s[sgprGSUSumIdx+1]                            // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_rcp_iflag_f32 v0, v0                                          // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_cvt_f32_u32 v1, s[sgprLoopCounterL]                           // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_mul_f32 v0, v0, v1                                            // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_cvt_u32_f32 v0, v0                                            // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_mul_u32_u24 v1, v0, s[sgprGSUSumIdx+1]                        // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_sub_u32 v1, s[sgprLoopCounterL], v1                           // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_cmpx_eq_u32 exec, v1, s[sgprGSUSumIdx+1]                      // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_add_u32 v0, 1, v0                                             // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_mov_b32 v1, 0                                                 // s[sgprGSUSumIdx+1] = s[sgprLoopCounterL] % s[sgprGSUSumIdx+1]
s_mov_b64 exec, -1                                              // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_readfirstlane_b32 s[sgprLoopCounterL], v0                     // quotient
v_readfirstlane_b32 s[sgprGSUSumIdx+1], v1                      // remainder
s_mul_i32 s[sgprTemp3], s[sgprLoopCounterL], s[sgprGSUSumIdx]   // quotient*GSUSumIdx
s_add_u32 s[sgprTemp2], 1, s[sgprLoopCounterL]                  // quotient+1
s_add_u32 s[sgprTemp3], s[sgprTemp3], s[sgprGSUSumIdx+1]        // quotient*GSUSumIdx+remainder
s_mul_i32 s[sgprTemp2], s[sgprTemp2], s[sgprGSUSumIdx]          // (quotient+1)*GSUSumIdx
s_cmp_lt_u32 s[sgprGSUSumIdx], s[sgprGSUSumIdx+1]               // gsuSumIdx < numIterPerWgRemainder
s_cselect_b32 s[sgprTemp2], s[sgprTemp2], s[sgprTemp3]          // (quotient+1)*GSUSumIdx if needed
s_mul_hi_u32 s[sgprTemp3], s[sgprTemp2], DEPTHU                 // gsuOffset = DepthU*accumulatedNumOfLoopCounterL
s_mul_i32 s[sgprTemp2], s[sgprTemp2], DEPTHU                    // gsuOffset = DepthU*accumulatedNumOfLoopCounterL
label_GSUC_A_End:
s_add_u32 s[sgprTemp0], s[sgprTemp0], s[sgprTemp2]              // accum GsuOffset term to tilestart
s_addc_u32 s[sgprTemp1], s[sgprTemp1], s[sgprTemp3]             // accum GsuOffset term to tilestart


s_mul_hi_u32 s[sgprTensor2dSizeA+1], s[sgprStridesA+0], s[sgprSizesFree+0]
s_mul_i32    s[sgprTensor2dSizeA+0], s[sgprStridesA+0], s[sgprSizesFree+0]
s_sub_u32 s[sgprShadowLimitA+0], s[sgprTensor2dSizeA], s[sgprTemp0]
s_subb_u32 s[sgprShadowLimitA+1], s[sgprTensor2dSizeA+1], s[sgprTemp1]
s_mul_hi_u32 s[sgprTemp3], s[sgprWorkGroup2], s[sgprStridesA+1]         // Stride*WG
s_mul_i32    s[sgprTemp2], s[sgprWorkGroup2], s[sgprStridesA+1]         // Stride*WG
s_add_u32 s[sgprTemp0], s[sgprTemp0], s[sgprTemp2]                      // accum wg term to tilestart
s_addc_u32 s[sgprTemp1], s[sgprTemp1], s[sgprTemp3]                     // accum wg term to tilestart
// Set limit to use bytes fp16 = 1
s_lshl_b64 s[sgprShadowLimitA:sgprShadowLimitA+1], s[sgprShadowLimitA:sgprShadowLimitA+1], LOG2BPE


s_cmp_eq_u32 s[sgprShadowLimitA+1], 0                           // are we within 2^32?
s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32
s_lshl_b64 s[sgprTemp0:sgprTemp1], s[sgprTemp0:sgprTemp1], LOG2BPE
s_add_u32 s[sgprSrdA+0], s[sgprAddressA+0], s[sgprTemp0]        // SRD base = Address+ tileStart0
s_addc_u32 s[sgprSrdA+1], s[sgprAddressA+1], s[sgprTemp1]       // SRD base = Address+ tileStart1
s_mov_b32 s[sgprSrdA+3], Srd127_96                              // Set bits 127_96 in SRD


s_and_b32 s80, s[sgprGSU], 0x3fff                  // Restore GSU
s_mul_i32 s80, s80, DEPTHU*BPE                     // GSU*DEPTHU*BPE
s_and_b32 s81, s[sgprGSU], 0x8000                  // SCC = (GSUC == 1) ?
s_cselect_b32 s[sgprGlobalReadIncsA+0], DEPTHU*BPE, s80  //depthU*PEB


s_mul_hi_u32 s[sgprTemp1], s[sgprWorkGroup1], MT1                       // WorkGroup[01] * MT
s_mul_i32 s[sgprTemp0], s[sgprWorkGroup1], MT1                          // WorkGroup[01] * MT
s_mul_hi_u32 s[sgprTemp1], s[sgprTemp1], s[sgprStridesB+0]              // tlu=0, scaled tile-offset by stride
s_mul_i32 s[sgprTemp0], s[sgprTemp0], s[sgprStridesB+0]                 // tlu=0, scaled tile-offset by stride

// GSU processing
s_and_b32 s[sgprTemp2], s[sgprGSU], 0x8000                      // SCC = (GSUC == 1) ?
s_cbranch_scc1 label_GSUC_B                                     // branch if GSUC == 1
s_mul_hi_u32 s[sgprTemp3], DEPTHU, s[sgprGSUSumIdx]             // gsuOffset = DepthU*GSUSumIdx
s_mul_i32 s[sgprTemp2], DEPTHU, s[sgprGSUSumIdx]                // gsuOffset = DepthU*GSUSumIdx
s_branch label_GSUC_B_End
label_GSUC_B:
s_lshr_b32 s[sgprLoopCounterL], s[sgprSizesSum], LOG2DEPTHU     // s[LoopCounterL] = s[sgprSizesSum] / DEPTHU
s_and_b32 s[sgprGSUSumIdx+1], s[sgprGSU], 0x3fff                // Restore GSU
v_cvt_f32_u32 v0, s[sgprGSUSumIdx+1]                            // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_rcp_iflag_f32 v0, v0                                          // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_cvt_f32_u32 v1, s[sgprLoopCounterL]                           // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_mul_f32 v0, v0, v1                                            // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_cvt_u32_f32 v0, v0                                            // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_mul_u32_u24 v1, v0, s[sgprGSUSumIdx+1]                        // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_sub_u32 v1, s[sgprLoopCounterL], v1                           // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_cmpx_eq_u32 exec, v1, s[sgprGSUSumIdx+1]                      // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_add_u32 v0, 1, v0                                             // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_mov_b32 v1, 0                                                 // s[sgprGSUSumIdx+1] = s[sgprLoopCounterL] % s[sgprGSUSumIdx+1]
s_mov_b64 exec, -1                                              // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_readfirstlane_b32 s[sgprLoopCounterL], v0                     // quotient
v_readfirstlane_b32 s[sgprGSUSumIdx+1], v1                      // remainder
s_mul_i32 s[sgprTemp3], s[sgprLoopCounterL], s[sgprGSUSumIdx]   // quotient*GSUSumIdx
s_add_u32 s[sgprTemp2], 1, s[sgprLoopCounterL]                  // quotient+1
s_add_u32 s[sgprTemp3], s[sgprTemp3], s[sgprGSUSumIdx+1]        // quotient*GSUSumIdx+remainder
s_mul_i32 s[sgprTemp2], s[sgprTemp2], s[sgprGSUSumIdx]          // (quotient+1)*GSUSumIdx
s_cmp_lt_u32 s[sgprGSUSumIdx], s[sgprGSUSumIdx+1]               // gsuSumIdx < numIterPerWgRemainder
s_cselect_b32 s[sgprTemp2], s[sgprTemp2], s[sgprTemp3]          // (quotient+1)*GSUSumIdx if needed
s_mul_hi_u32 s[sgprTemp3], s[sgprTemp2], DEPTHU                 // gsuOffset = DepthU*accumulatedNumOfLoopCounterL
s_mul_i32 s[sgprTemp2], s[sgprTemp2], DEPTHU                    // gsuOffset = DepthU*accumulatedNumOfLoopCounterL
label_GSUC_B_End:
s_add_u32 s[sgprTemp0], s[sgprTemp0], s[sgprTemp2]              // accum GsuOffset term to tilestart
s_addc_u32 s[sgprTemp1], s[sgprTemp1], s[sgprTemp3]             // accum GsuOffset term to tilestart

s_mul_hi_u32 s[sgprTensor2dSizeB+1], s[sgprStridesB+0], s[sgprSizesFree+1]
s_mul_i32    s[sgprTensor2dSizeB+0], s[sgprStridesB+0], s[sgprSizesFree+1]
s_sub_u32 s[sgprShadowLimitB+0], s[sgprTensor2dSizeB], s[sgprTemp0]
s_subb_u32 s[sgprShadowLimitB+1], s[sgprTensor2dSizeB+1], s[sgprTemp1]
s_mul_hi_u32 s[sgprTemp3], s[sgprWorkGroup2], s[sgprStridesB+1]         // Stride*WG
s_mul_i32    s[sgprTemp2], s[sgprWorkGroup2], s[sgprStridesB+1]         // Stride*WG
s_add_u32 s[sgprTemp0], s[sgprTemp0], s[sgprTemp2]                      // accum wg term to tilestart
s_addc_u32 s[sgprTemp1], s[sgprTemp1], s[sgprTemp3]                     // accum wg term to tilestart
// Set limit to use bytes fp16 = 1
s_lshl_b64 s[sgprShadowLimitB:sgprShadowLimitB+1], s[sgprShadowLimitB:sgprShadowLimitB+1], LOG2BPE 

s_cmp_eq_u32 s[sgprShadowLimitB+1], 0                           // are we within 2^32?
s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32
s_lshl_b64 s[sgprTemp0:sgprTemp1], s[sgprTemp0:sgprTemp1], LOG2BPE
s_add_u32 s[sgprSrdB+0], s[sgprAddressB+0], s[sgprTemp0]        // SRD base = Address+ tileStart0
s_addc_u32 s[sgprSrdB+1], s[sgprAddressB+1], s[sgprTemp1]       // SRD base = Address+ tileStart1
s_mov_b32 s[sgprSrdB+3], Srd127_96                              // Set bits 127_96 in SRD

s_and_b32 s80, s[sgprGSU], 0x3fff                  // Restore GSU
s_mul_i32 s80, s80, DEPTHU*BPE                     // GSU*DEPTHU*BPE
s_and_b32 s81, s[sgprGSU], 0x8000                  // SCC = (GSUC == 1) ?
s_cselect_b32 s[sgprGlobalReadIncsB+0], DEPTHU*BPE, s80  //depthU*PEB

/******************************************/
/* Generate LDS A parameters ...          */
/******************************************/
.set WAVE_LDS_OFFSET_A, 64*16     //x4 load
.set WAVE_LDS_OFFSET, WAVE_LDS_OFFSET_A     //x4 load
.set LOADxWAVES_K_A, 64/COALESCE_THREAD_A*GLWAVES
.set LOADxWAVES_K_A_LOG2, 5
.set LOADxWAVES_LDS_OFFSET_A, WAVE_LDS_OFFSET_A*GLWAVES

//Wrap Lds
s_mul_i32 s[sgprLocalWriteAddrA], s[sgprGlWaveID], WAVE_LDS_OFFSET_A+0                    // can add lds pad
s_and_b32 s[sgprTemp0], s[sgprGlWaveID], 3
s_mul_i32 s[sgprTemp0], s[sgprTemp0], 2
//s_mul_i32 s[sgprTemp0], s[sgprTemp0], 0
s_lshl_b32 s[sgprTemp0], s[sgprTemp0], 16
s_or_b32 s[sgprLocalWriteAddrA], s[sgprLocalWriteAddrA], s[sgprTemp0]
s_mov_b32 s[sgprLocalWriteAddrAori], s[sgprLocalWriteAddrA]

s_cmp_ge_i32 s[sgprWaveID], 4
s_cbranch_scc1 skip_MacWaveALdsR

//get lds read addrA  dwordx4 load, DepthU=128, ldspad=0  ldswarp=32
v_and_b32 v[vgprTemp0], v[vgprSerial], 63
v_and_b32 v0, 3, v[vgprTemp0]                      // (0~3) 4 waves as a group
v_lshlrev_b32 v1, 10, v0                           // (0~3) * 1024
v_lshlrev_b32 v2, 5, v0                            // (0~3) * 32, per wave 8 lds bank warp
v_add_u32 v0, v1, v2                               // add
//v_add_u32 v0, v1, 0                               // add

v_and_b32 v1, 15, v[vgprTemp0]                     // 0~15 0~15 0~15 0~15
v_lshrrev_b32 v1, 2, v1                            // 0000 1111 ~ 3333
v_lshlrev_b32 v1, 8, v1                            // (0000 1111 ~ 3333) * 128
v_add_u32 v0, v0, v1                               // add

v_lshrrev_b32 v1, 4, v[vgprTemp0]                  // (0~63) / 16
v_lshlrev_b32 v1, 4, v1                            // (00..00 11..11 ~ 33..33) * 16
v_add_u32 v[vgprLocalReadAddrA], v0, v1            // add

//lds wrap A

v_mov_b32 v0, 0
v_mov_b32 v1, 0
s_mov_b32 s[sgprTemp0], 1024
v_writelane_b32 v0, s[sgprTemp0], 47
v_writelane_b32 v0, s[sgprTemp0], 63

v_writelane_b32 v1, s[sgprTemp0], 14
v_writelane_b32 v1, s[sgprTemp0], 15
v_writelane_b32 v1, s[sgprTemp0], 30
v_writelane_b32 v1, s[sgprTemp0], 31
v_writelane_b32 v1, s[sgprTemp0], 45
v_writelane_b32 v1, s[sgprTemp0], 46
v_writelane_b32 v1, s[sgprTemp0], 47
v_writelane_b32 v1, s[sgprTemp0], 61
v_writelane_b32 v1, s[sgprTemp0], 62
v_writelane_b32 v1, s[sgprTemp0], 63

v_sub_u32 v[vgprLocalReadAddrA+1], v[vgprLocalReadAddrA], v0 // add
v_sub_u32 v[vgprLocalReadAddrA+2], v[vgprLocalReadAddrA], v1 // add

s_lshl_b32 s[sgprTemp1], s[sgprWaveID_M], 12
v_add_u32 v[vgprLocalReadAddrA], v[vgprLocalReadAddrA], s[sgprTemp1]
v_add_u32 v[vgprLocalReadAddrA+1], v[vgprLocalReadAddrA+1], s[sgprTemp1]
v_add_u32 v[vgprLocalReadAddrA+2], v[vgprLocalReadAddrA+2], s[sgprTemp1]

skip_MacWaveALdsR:
.set WAVE_LDS_OFFSET, UNDEF     //x4 load

/******************************************/
/* Generate LDS B parameters ...          */
/******************************************/
.set WAVE_LDS_OFFSET_B, 64*16     //x4 load
.set WAVE_LDS_OFFSET, WAVE_LDS_OFFSET_B     //x4 load
.set LOADxWAVES_K_B, 64/COALESCE_THREAD_B*GLWAVES
.set LOADxWAVES_K_B_LOG2, 5
.set LOADxWAVES_LDS_OFFSET_B, WAVE_LDS_OFFSET_B*GLWAVES

//Wrap Lds
s_mul_i32 s[sgprLocalWriteAddrB], s[sgprGlWaveID], WAVE_LDS_OFFSET_B+0                    // can add lds pad
s_and_b32 s[sgprTemp0], s[sgprGlWaveID], 3
s_mul_i32 s[sgprTemp0], s[sgprTemp0], 2
//s_mul_i32 s[sgprTemp0], s[sgprTemp0], 0
s_lshl_b32 s[sgprTemp0], s[sgprTemp0], 16
s_or_b32 s[sgprLocalWriteAddrB], s[sgprLocalWriteAddrB], s[sgprTemp0]
s_add_u32 s[sgprLocalWriteAddrB], s[sgprLocalWriteAddrB], LDS_B_OFFSET
s_mov_b32 s[sgprLocalWriteAddrBori], s[sgprLocalWriteAddrB]

s_cmp_ge_i32 s[sgprWaveID], 4
s_cbranch_scc1 skip_MacWaveBLdsR

//get lds read addrB  dwordx4 load, DepthU=128, ldspad=0  ldswarp=32
v_and_b32 v[vgprTemp0], v[vgprSerial], 63
v_and_b32 v0, 3, v[vgprTemp0]                      // (0~3) 4 waves as a group
v_lshlrev_b32 v1, 10, v0                           // (0~3) * 1024
v_lshlrev_b32 v2, 5, v0                            // (0~3) * 32, per wave 8 lds bank warp
v_add_u32 v0, v1, v2                               // add
//v_add_u32 v0, v1, 0                               // add

v_and_b32 v1, 15, v[vgprTemp0]                     // 0~15 0~15 0~15 0~15
v_lshrrev_b32 v1, 2, v1                            // 0000 1111 ~ 3333
v_lshlrev_b32 v1, 8, v1                            // (0000 1111 ~ 3333) * 128
v_add_u32 v0, v0, v1                               // add

v_lshrrev_b32 v1, 4, v[vgprTemp0]                  // (0~63) / 16
v_lshlrev_b32 v1, 4, v1                            // (00..00 11..11 ~ 33..33) * 16
v_add_u32 v[vgprLocalReadAddrB], v0, v1            // add

//lds wrap B

v_mov_b32 v0, 0
v_mov_b32 v1, 0
s_mov_b32 s[sgprTemp0], 1024
v_writelane_b32 v0, s[sgprTemp0], 47
v_writelane_b32 v0, s[sgprTemp0], 63

v_writelane_b32 v1, s[sgprTemp0], 14
v_writelane_b32 v1, s[sgprTemp0], 15
v_writelane_b32 v1, s[sgprTemp0], 30
v_writelane_b32 v1, s[sgprTemp0], 31
v_writelane_b32 v1, s[sgprTemp0], 45
v_writelane_b32 v1, s[sgprTemp0], 46
v_writelane_b32 v1, s[sgprTemp0], 47
v_writelane_b32 v1, s[sgprTemp0], 61
v_writelane_b32 v1, s[sgprTemp0], 62
v_writelane_b32 v1, s[sgprTemp0], 63

v_add_u32 v[vgprLocalReadAddrB], LDS_B_OFFSET, v[vgprLocalReadAddrB]
v_sub_u32 v[vgprLocalReadAddrB+1], v[vgprLocalReadAddrB], v0 // add
v_sub_u32 v[vgprLocalReadAddrB+2], v[vgprLocalReadAddrB], v1 // add

s_lshl_b32 s[sgprTemp1], s[sgprWaveID_N], 12
v_add_u32 v[vgprLocalReadAddrB], v[vgprLocalReadAddrB], s[sgprTemp1]
v_add_u32 v[vgprLocalReadAddrB+1], v[vgprLocalReadAddrB+1], s[sgprTemp1]
v_add_u32 v[vgprLocalReadAddrB+2], v[vgprLocalReadAddrB+2], s[sgprTemp1]


skip_MacWaveBLdsR:
.set WAVE_LDS_OFFSET, UNDEF     //x4 load

/******************************************/
/* Keep Sgpr Values for use later ...     */
/******************************************/

//store sgprs to keep value
v_writelane_b32 v[vgprKeepSgprValue], s[sgprSrdA+0], laneSrdA0
v_writelane_b32 v[vgprKeepSgprValue], s[sgprSrdA+1], laneSrdA1
v_writelane_b32 v[vgprKeepSgprValue], s[sgprSrdA+2], laneSrdA2
v_writelane_b32 v[vgprKeepSgprValue], s[sgprSrdB+0], laneSrdB0
v_writelane_b32 v[vgprKeepSgprValue], s[sgprSrdB+1], laneSrdB1
v_writelane_b32 v[vgprKeepSgprValue], s[sgprSrdB+2], laneSrdB2
v_writelane_b32 v[vgprKeepSgprValue], s[sgprGlobalReadIncsA], laneIncA
v_writelane_b32 v[vgprKeepSgprValue], s[sgprGlobalReadIncsB], laneIncB

/******************************************/
/* Define Global Load...                  */
/******************************************/

.macro GLOBAL_LOADA offset:req

s_add_u32 m0, s[sgprLocalWriteAddrA], \offset
buffer_load_dwordx4 v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 lds
s_add_u32 m0, m0, 4096
buffer_load_dwordx4 v[vgprGlobalReadOffsetA+1], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 lds

.endm

.macro GLOBAL_LOADB offset:req

s_add_u32 m0, s[sgprLocalWriteAddrB], \offset
buffer_load_dwordx4 v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:0 lds
s_add_u32 m0, m0, 4096
buffer_load_dwordx4 v[vgprGlobalReadOffsetB+1], s[sgprSrdB:sgprSrdB+3], 0 offen offset:0 lds

.endm

/******************************************/
/* Define Global Load adress Increase...  */
/******************************************/

.macro GLOBAL_INCA 

s_mov_b32 s[sgprTemp0], s[sgprGlobalReadIncsA+0]
s_mov_b32 s[sgprTemp1], 0
s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0],  s[sgprTemp0]
s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s[sgprTemp1]
s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0],  s[sgprTemp0]
s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s[sgprTemp1]
s_cmp_eq_u32 s[sgprShadowLimitA+1], 0                            // are we within 2^32?
s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32

.endm

.macro GLOBAL_INCB 

s_mov_b32 s[sgprTemp0], s[sgprGlobalReadIncsB+0]
s_mov_b32 s[sgprTemp1], 0
s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0],  s[sgprTemp0]
s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s[sgprTemp1]
s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0],  s[sgprTemp0]
s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s[sgprTemp1]
s_cmp_eq_u32 s[sgprShadowLimitB+1], 0                            // are we within 2^32?
s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32

.endm

/******************************************/
/* Define LDS Load...                     */
/******************************************/

.macro LDS_LOADA off:req

ds_read_b128 v[vgprValuA_X0_I0+ 0 +0:vgprValuA_X0_I0+ 0 +3], v[vgprLocalReadAddrA + 0] offset:0 + 0 + \off
ds_read_b128 v[vgprValuA_X0_I0+ 0 +4:vgprValuA_X0_I0+ 0 +7], v[vgprLocalReadAddrA + 0] offset:64 + 0 + \off
ds_read_b128 v[vgprValuA_X0_I0+ 0 +8:vgprValuA_X0_I0+ 0 +11], v[vgprLocalReadAddrA + 1] offset:128 + 0 + \off
ds_read_b128 v[vgprValuA_X0_I0+ 0 +12:vgprValuA_X0_I0+ 0 +15], v[vgprLocalReadAddrA + 2] offset:192 + 0 + \off

.endm

.macro LDS_LOADB off:req

ds_read_b128 v[vgprValuB_X0_I0+ 0 +0:vgprValuB_X0_I0+ 0 +3], v[vgprLocalReadAddrB + 0] offset:0 + 0 + \off
ds_read_b128 v[vgprValuB_X0_I0+ 0 +4:vgprValuB_X0_I0+ 0 +7], v[vgprLocalReadAddrB + 0] offset:64 + 0 + \off
ds_read_b128 v[vgprValuB_X0_I0+ 0 +8:vgprValuB_X0_I0+ 0 +11], v[vgprLocalReadAddrB + 1] offset:128 + 0 + \off
ds_read_b128 v[vgprValuB_X0_I0+ 0 +12:vgprValuB_X0_I0+ 0 +15], v[vgprLocalReadAddrB + 2] offset:192 + 0 + \off

.endm

s_lshr_b32 s[sgprLoopCounterL], s[sgprSizesSum+0], LOG2DEPTHU   // s[sgprLoopCounterL] = s[sgprSizesSum+0] / DEPTHU
s_and_b32 s80, s[sgprGSU], 0x3fff                               // Restore GSU
s_cmp_eq_u32 s80, 1                                             // GSU == 1 ?
s_cbranch_scc1 label_GSU_1                                      // branch if GSU == 1
s_and_b32 s[sgprGSUSumIdx+1], s[sgprGSU], 0x3fff                // Restore GSU
v_cvt_f32_u32 v0, s[sgprGSUSumIdx+1]                            // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_rcp_iflag_f32 v0, v0                                          // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_cvt_f32_u32 v1, s[sgprLoopCounterL]                           // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_mul_f32 v0, v0, v1                                            // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_cvt_u32_f32 v0, v0                                            // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_mul_u32_u24 v1, v0, s[sgprGSUSumIdx+1]                        // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_sub_u32 v1, s[sgprLoopCounterL], v1                           // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_cmpx_eq_u32 exec, v1, s[sgprGSUSumIdx+1]                      // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_add_u32 v0, 1, v0                                             // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_mov_b32 v1, 0                                                 // s[sgprGSUSumIdx+1] = s[sgprLoopCounterL] % s[sgprGSUSumIdx+1]
s_mov_b64 exec, -1                                              // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSUSumIdx+1]
v_readfirstlane_b32 s[sgprLoopCounterL], v0                     // quotient
v_readfirstlane_b32 s[sgprGSUSumIdx+1], v1                      // remainder
s_add_u32 s80, 1, s[sgprLoopCounterL]                           // tmp<-numIterMyWg+1
s_cmp_lt_u32 s[sgprGSUSumIdx], s[sgprGSUSumIdx+1]               // gsuSumIdx < numIterPerWgRemainder
s_cmov_b32 s[sgprLoopCounterL], s80                             // numIterMyWg++ if needed
label_GSU_1:


/******************************************/
/* Use Global Load Wave process ...       */
/******************************************/
s_cmp_lt_u32 s[sgprWaveID], 4 
s_cbranch_scc1 SkipGL 
s_min_u32 s[sgprLoopCntCommon], 3, s[sgprLoopCounterL] 

s_cmp_le_i32 s[sgprLoopCntCommon], 0
s_cbranch_scc1 PreLoad_END
PreLoad_BEGIN:

GLOBAL_LOADA LDS_BLK_OFFSET*0
GLOBAL_LOADB LDS_BLK_OFFSET*0
GLOBAL_INCA
GLOBAL_INCB

s_add_u32 s[sgprLocalWriteAddrA], s[sgprLocalWriteAddrA], LDS_BLK_OFFSET_64Kmasked
s_add_u32 s[sgprTemp0], s[sgprLocalWriteAddrAori], s[sgprLDSMask]
s_cmp_ge_u32 s[sgprLocalWriteAddrA], s[sgprTemp0]
s_cmov_b32  s[sgprLocalWriteAddrA], s[sgprLocalWriteAddrAori]

s_add_u32 s[sgprLocalWriteAddrB], s[sgprLocalWriteAddrB], LDS_BLK_OFFSET_64Kmasked
s_add_u32 s[sgprTemp0], s[sgprLocalWriteAddrBori], s[sgprLDSMask]
s_cmp_ge_u32 s[sgprLocalWriteAddrB], s[sgprTemp0]
s_cmov_b32  s[sgprLocalWriteAddrB], s[sgprLocalWriteAddrBori]


s_sub_u32 s[sgprLoopCntCommon], s[sgprLoopCntCommon], 1
s_cmp_gt_i32 s[sgprLoopCntCommon], 0
s_cbranch_scc1 PreLoad_BEGIN
PreLoad_END:
s_waitcnt vmcnt(4)

s_cmp_gt_i32 s[sgprLoopCounterL], 3
s_cbranch_scc1 skip_vmcnt0
s_waitcnt vmcnt(0)
skip_vmcnt0:
s_barrier
s_mov_b32 s[sgprLoopCntCommon], s[sgprLoopCounterL]

s_cmp_le_i32 s[sgprLoopCntCommon], 0
s_cbranch_scc1 MainLoop_END
MainLoop_BEGIN:

s_cmp_le_i32 s[sgprLoopCntCommon], 3 
s_cmov_b32 s[sgprSrdA+2], 0 
s_cmov_b32 s[sgprGlobalReadIncsA+0], 0 

s_cmov_b32 s[sgprSrdB+2], 0 
s_cmov_b32 s[sgprGlobalReadIncsB+0], 0 

GLOBAL_LOADA LDS_BLK_OFFSET*0
GLOBAL_LOADB LDS_BLK_OFFSET*0

GLOBAL_INCA
GLOBAL_INCB

s_add_u32 s[sgprLocalWriteAddrA], s[sgprLocalWriteAddrA], LDS_BLK_OFFSET_64Kmasked
s_add_u32 s[sgprTemp0], s[sgprLocalWriteAddrAori], s[sgprLDSMask]
s_cmp_ge_u32 s[sgprLocalWriteAddrA], s[sgprTemp0]
s_cmov_b32  s[sgprLocalWriteAddrA], s[sgprLocalWriteAddrAori]

s_add_u32 s[sgprLocalWriteAddrB], s[sgprLocalWriteAddrB], LDS_BLK_OFFSET_64Kmasked
s_add_u32 s[sgprTemp0], s[sgprLocalWriteAddrBori], s[sgprLDSMask]
s_cmp_ge_u32 s[sgprLocalWriteAddrB], s[sgprTemp0]
s_cmov_b32  s[sgprLocalWriteAddrB], s[sgprLocalWriteAddrBori]

s_waitcnt vmcnt(4)
s_barrier

s_sub_u32 s[sgprLoopCntCommon], s[sgprLoopCntCommon], 1
s_cmp_gt_i32 s[sgprLoopCntCommon], 0
s_cbranch_scc1 MainLoop_BEGIN
MainLoop_END:
s_endpgm
SkipGL:

/******************************************/
/* GL END                                 */
/******************************************/

/******************************************/
/* Generate SrcD ...                      */
/******************************************/

s_and_b32 s[sgprTemp2], s[sgprGSU], 0x3fff
s_cmp_gt_u32 s[sgprTemp2], 1
s_cselect_b32 s[sgprTemp2], 2, 1
s_mov_b32 s[sgprSrdD+0], s[sgprAddressD+0]         // init SRD base address (lower)
s_mov_b32 s[sgprSrdD+1], s[sgprAddressD+1]         // init SRD base address (upper) + other fields
s_mov_b32 s[sgprSrdC+0], s[sgprAddressC+0]         // init SRD base address (lower)
s_mov_b32 s[sgprSrdC+1], s[sgprAddressC+1]         // init SRD base address (upper) + other fields
s_mul_i32 s[sgprTemp0], s[sgprWorkGroup0], MT0
s_mul_i32 s[sgprTemp1], s[sgprWorkGroup1], MT1
s_mul_i32 s[sgprTemp1], s[sgprTemp1], s[sgprStridesD]
s_add_u32 s[sgprTemp0], s[sgprTemp0], s[sgprTemp1]
s_lshl_b32  s[sgprTemp0], s[sgprTemp0], s[sgprTemp2]
s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s[sgprTemp0]
s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0
s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s[sgprTemp0]
s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0

#s_mul_i32 s[sgprTemp0], s[sgprSizesFree+0], s[sgprSizesFree+1]
s_mul_i32 s[sgprTemp0], s[sgprStridesD+0], s[sgprSizesFree+1]
s_sub_u32 s[sgprTemp1], s[sgprTemp0], s[sgprTemp1]
s_lshl_b32 s[sgprSrdD+2], s[sgprTemp1], s[sgprTemp2]
s_mov_b32 s[sgprSrdD+3], Srd127_96                 // Set bits 127_96 in post-loop SRD
s_lshl_b32 s[sgprSrdC+2], s[sgprTemp1], s[sgprTemp2]
s_mov_b32 s[sgprSrdC+3], Srd127_96                 // Set bits 127_96 in post-loop SRD


s_and_b32 s[sgprTemp3], s[sgprGSU], 0x3fff
s_cmp_gt_u32 s[sgprTemp3], 1
s_cselect_b32 s[sgprTemp3], 2, 1
v_and_b32 v[vgprTemp0], v[vgprSerial], 63
v_lshrrev_b32 v[vgprTemp1], 4, v[vgprTemp0]                 
v_and_b32 v[vgprTemp2], 15, v[vgprTemp0]                    
v_lshlrev_b32 v[vgprTemp2], s[sgprTemp3], v[vgprTemp2]                          
v_mul_lo_u32 v[vgprTemp1], v[vgprTemp1], s[sgprStridesD]   
v_lshlrev_b32 v[vgprTemp1], s[sgprTemp3], v[vgprTemp1]        
v_add_u32 v[vgprGlobalWriteOffsetD], v[vgprTemp1], v[vgprTemp2]
s_mul_i32 s[sgprTemp1], s[sgprWaveID_M], MperWAVE
s_mul_i32 s[sgprTemp0], s[sgprWaveID_N], NperWAVE
s_mul_i32 s[sgprTemp0], s[sgprTemp0], s[sgprStridesD]
s_add_u32 s[sgprTemp0], s[sgprTemp0], s[sgprTemp1]
s_lshl_b32 s[sgprTemp0], s[sgprTemp0], s[sgprTemp3]
v_add_u32 v[vgprGlobalWriteOffsetD], v[vgprGlobalWriteOffsetD], s[sgprTemp0]

/******************************************/
/* Init ValueC ...                        */
/******************************************/

v_mov_b32 v[vgprValuC+0], 0x0
v_mov_b32 v[vgprValuC+1], 0x0
v_mov_b32 v[vgprValuC+2], 0x0
v_mov_b32 v[vgprValuC+3], 0x0

/******************************************/
/* LoopCounter == 0, Skip to tail/last loop ... */
/******************************************/
s_cmp_le_i32 s[sgprLoopCounterL], 0
s_cbranch_scc1 TAIL_LOOP
s_barrier
LDS_LOADA 0
LDS_LOADB 0


/******************************************/
/* Main Loop Process ...                  */
/******************************************/
s_mov_b32 s[sgprLoopCntCommon], s[sgprLoopCounterL]

/* Unrolled Loop 1/4 - Begin              */


s_cmp_le_i32 s[sgprLoopCntCommon], 0
s_cbranch_scc1 MainLoopBeginW0_3_END
MainLoopBeginW0_3_BEGIN:

s_waitcnt lgkmcnt(0)
MMAC_16x16_part0_0
s_barrier

LDS_LOADA 16384
LDS_LOADB 16384

s_sub_u32 s[sgprLoopCntCommon], s[sgprLoopCntCommon], 1
s_cmp_le_i32 s[sgprLoopCntCommon], 0
s_cbranch_scc1 TAIL_LOOP

/* Unrolled Loop 2/4 - Begin              */

s_waitcnt lgkmcnt(0)
MMAC_16x16_part0_0
s_barrier

LDS_LOADA 32768
LDS_LOADB 32768
s_sub_u32 s[sgprLoopCntCommon], s[sgprLoopCntCommon], 1
s_cmp_le_i32 s[sgprLoopCntCommon], 0
s_cbranch_scc1 TAIL_LOOP

/* Unrolled Loop 3/4 - Begin              */

s_waitcnt lgkmcnt(0)
MMAC_16x16_part0_0
s_barrier

LDS_LOADA 49152
LDS_LOADB 49152

s_sub_u32 s[sgprLoopCntCommon], s[sgprLoopCntCommon], 1
s_cmp_le_i32 s[sgprLoopCntCommon], 0
s_cbranch_scc1 TAIL_LOOP

/* Unrolled Loop 4/4 - Begin              */

s_waitcnt lgkmcnt(0)
MMAC_16x16_part0_0
s_barrier

LDS_LOADA 0
LDS_LOADB 0


s_sub_u32 s[sgprLoopCntCommon], s[sgprLoopCntCommon], 1
s_cmp_gt_i32 s[sgprLoopCntCommon], 0
s_cbranch_scc1 MainLoopBeginW0_3_BEGIN
MainLoopBeginW0_3_END:

/******************************************/
/* Tail Loop Process ...                  */
/******************************************/
TAIL_LOOP:


/******************************************/
/* Global Write Process ...               */
/******************************************/

s_cmp_eq_u32 s[sgprBeta], 0
s_cbranch_scc1 Beta_eqcase
s_endpgm
s_branch Beta_EndSwitch
Beta_eqcase:

s_mul_i32 s[sgprD_MEdge], s[sgprWorkGroup0], MT0
s_sub_u32 s[sgprD_MEdge], s[sgprSizesFree+0], s[sgprD_MEdge]
s_lshl_b32 s[sgprD_MEdge], s[sgprD_MEdge], 1
s_min_u32 s[sgprD_MEdge], s[sgprD_MEdge], MT0*2
v_and_b32 v[vgprTemp2], v[vgprSerial], 15  
s_mul_i32 s[sgprTemp1], s[sgprWaveID_M], MperWAVE
v_add_u32 v[vgprTemp2], v[vgprTemp2], s[sgprTemp1]         
v_lshlrev_b32  v[vgprTemp2], 1, v[vgprTemp2]                 
v_mov_b32 v[vgprTemp3], v[vgprTemp2]               //store inittial addr
v_mov_b32 v[vgprTemp0], v[vgprGlobalWriteOffsetD]  //store inittial addr

.set Nvoff, 0
v_mov_b32 v[vgprTemp1], v[vgprTemp0]   //v[vgprTemp1] <- v[vgprGlobalWriteOffsetD] with strideD
v_mov_b32 v[vgprTemp2], v[vgprTemp3]   //v[vgprTemp2] is just M offset without strideD for compare edge 
v_cmp_ge_u32 s[sgprTemp2:sgprTemp2+1], v[vgprTemp2], s[sgprD_MEdge]
v_cndmask_b32 v[vgprGlobalWriteOffsetD], v[vgprTemp1], -1, s[sgprTemp2:sgprTemp2+1]
v_mul_f32 v[vgprValuC+Nvoff+0], s[sgprAlpha], v[vgprValuC+Nvoff+0]
v_cvt_f16_f32 v[vgprValuC+Nvoff+0], v[vgprValuC+Nvoff+0]
buffer_store_short v[vgprValuC+Nvoff+0],  v[vgprGlobalWriteOffsetD], s[sgprSrdD:sgprSrdD+3], 0 offen offset:0   // store D 
v_add_u32 v[vgprTemp1], 32, v[vgprTemp1]
v_add_u32 v[vgprTemp2], 32, v[vgprTemp2]


.set Nvoff, UNDEF
s_mul_i32  s[sgprTemp0], s[sgprStridesD], 4*2                
s_add_u32  s[sgprSrdD+0], s[sgprSrdD+0], s[sgprTemp0]        
s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0        
s_subb_u32 s[sgprSrdD+2], s[sgprSrdD+2],  s[sgprTemp0]
s_cmov_b32 s[sgprSrdD+2], 0
s_add_u32  s[sgprSrdC+0], s[sgprSrdC+0], s[sgprTemp0]        
s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0        
s_subb_u32 s[sgprSrdC+2], s[sgprSrdC+2],  s[sgprTemp0]
s_cmov_b32 s[sgprSrdC+2], 0

.set Nvoff, 1
v_mov_b32 v[vgprTemp1], v[vgprTemp0]   //v[vgprTemp1] <- v[vgprGlobalWriteOffsetD] with strideD
v_mov_b32 v[vgprTemp2], v[vgprTemp3]   //v[vgprTemp2] is just M offset without strideD for compare edge 
v_cmp_ge_u32 s[sgprTemp2:sgprTemp2+1], v[vgprTemp2], s[sgprD_MEdge]
v_cndmask_b32 v[vgprGlobalWriteOffsetD], v[vgprTemp1], -1, s[sgprTemp2:sgprTemp2+1]
v_mul_f32 v[vgprValuC+Nvoff+0], s[sgprAlpha], v[vgprValuC+Nvoff+0]
v_cvt_f16_f32 v[vgprValuC+Nvoff+0], v[vgprValuC+Nvoff+0]
buffer_store_short v[vgprValuC+Nvoff+0],  v[vgprGlobalWriteOffsetD], s[sgprSrdD:sgprSrdD+3], 0 offen offset:0   // store D 
v_add_u32 v[vgprTemp1], 32, v[vgprTemp1]
v_add_u32 v[vgprTemp2], 32, v[vgprTemp2]


.set Nvoff, UNDEF
s_mul_i32  s[sgprTemp0], s[sgprStridesD], 4*2                
s_add_u32  s[sgprSrdD+0], s[sgprSrdD+0], s[sgprTemp0]        
s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0        
s_subb_u32 s[sgprSrdD+2], s[sgprSrdD+2],  s[sgprTemp0]
s_cmov_b32 s[sgprSrdD+2], 0
s_add_u32  s[sgprSrdC+0], s[sgprSrdC+0], s[sgprTemp0]        
s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0        
s_subb_u32 s[sgprSrdC+2], s[sgprSrdC+2],  s[sgprTemp0]
s_cmov_b32 s[sgprSrdC+2], 0

.set Nvoff, 2
v_mov_b32 v[vgprTemp1], v[vgprTemp0]   //v[vgprTemp1] <- v[vgprGlobalWriteOffsetD] with strideD
v_mov_b32 v[vgprTemp2], v[vgprTemp3]   //v[vgprTemp2] is just M offset without strideD for compare edge 
v_cmp_ge_u32 s[sgprTemp2:sgprTemp2+1], v[vgprTemp2], s[sgprD_MEdge]
v_cndmask_b32 v[vgprGlobalWriteOffsetD], v[vgprTemp1], -1, s[sgprTemp2:sgprTemp2+1]
v_mul_f32 v[vgprValuC+Nvoff+0], s[sgprAlpha], v[vgprValuC+Nvoff+0]
v_cvt_f16_f32 v[vgprValuC+Nvoff+0], v[vgprValuC+Nvoff+0]
buffer_store_short v[vgprValuC+Nvoff+0],  v[vgprGlobalWriteOffsetD], s[sgprSrdD:sgprSrdD+3], 0 offen offset:0   // store D 
v_add_u32 v[vgprTemp1], 32, v[vgprTemp1]
v_add_u32 v[vgprTemp2], 32, v[vgprTemp2]


.set Nvoff, UNDEF
s_mul_i32  s[sgprTemp0], s[sgprStridesD], 4*2                
s_add_u32  s[sgprSrdD+0], s[sgprSrdD+0], s[sgprTemp0]        
s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0        
s_subb_u32 s[sgprSrdD+2], s[sgprSrdD+2],  s[sgprTemp0]
s_cmov_b32 s[sgprSrdD+2], 0
s_add_u32  s[sgprSrdC+0], s[sgprSrdC+0], s[sgprTemp0]        
s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0        
s_subb_u32 s[sgprSrdC+2], s[sgprSrdC+2],  s[sgprTemp0]
s_cmov_b32 s[sgprSrdC+2], 0

.set Nvoff, 3
v_mov_b32 v[vgprTemp1], v[vgprTemp0]   //v[vgprTemp1] <- v[vgprGlobalWriteOffsetD] with strideD
v_mov_b32 v[vgprTemp2], v[vgprTemp3]   //v[vgprTemp2] is just M offset without strideD for compare edge 
v_cmp_ge_u32 s[sgprTemp2:sgprTemp2+1], v[vgprTemp2], s[sgprD_MEdge]
v_cndmask_b32 v[vgprGlobalWriteOffsetD], v[vgprTemp1], -1, s[sgprTemp2:sgprTemp2+1]
v_mul_f32 v[vgprValuC+Nvoff+0], s[sgprAlpha], v[vgprValuC+Nvoff+0]
v_cvt_f16_f32 v[vgprValuC+Nvoff+0], v[vgprValuC+Nvoff+0]
buffer_store_short v[vgprValuC+Nvoff+0],  v[vgprGlobalWriteOffsetD], s[sgprSrdD:sgprSrdD+3], 0 offen offset:0   // store D 
v_add_u32 v[vgprTemp1], 32, v[vgprTemp1]
v_add_u32 v[vgprTemp2], 32, v[vgprTemp2]


.set Nvoff, UNDEF
s_mul_i32  s[sgprTemp0], s[sgprStridesD], 4*2                
s_add_u32  s[sgprSrdD+0], s[sgprSrdD+0], s[sgprTemp0]        
s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0        
s_subb_u32 s[sgprSrdD+2], s[sgprSrdD+2],  s[sgprTemp0]
s_cmov_b32 s[sgprSrdD+2], 0
s_add_u32  s[sgprSrdC+0], s[sgprSrdC+0], s[sgprTemp0]        
s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0        
s_subb_u32 s[sgprSrdC+2], s[sgprSrdC+2],  s[sgprTemp0]
s_cmov_b32 s[sgprSrdC+2], 0

Beta_EndSwitch:
s_endpgm
