Restructure gridwise and blockwise GEMM, add tensor contraction and FWD-v4r5 (#36)

* experimenting magic number division

* overhauling fwd-v4r4 to clearly reflect transformation graph

* added fwd-v4r5

* bug fix for make_dynamic_naive_tensor_descriptor_aligned_v2

* bug fix and added sanity-check in transform_dynamic_tensor_descriptor

* added conv_driver_v2

composable_kernel/include/driver/driver_dynamic_contraction_v1r1.hpp

+#ifndef CK_DRIVER_DYNAMIC_CONTRACTION_V1R1_HPP
+#define CK_DRIVER_DYNAMIC_CONTRACTION_V1R1_HPP
+
+#include "common_header.hpp"
+#include "dynamic_tensor_descriptor.hpp"
+#include "dynamic_tensor_descriptor_helper.hpp"
+#include "gridwise_dynamic_contraction_v1r1.hpp"
+
+namespace ck {
+
+template <index_t BlockSize,
+          typename FloatAB,
+          typename FloatAcc,
+          typename FloatC,
+          InMemoryDataOperation CGlobalMemoryDataOperation,
+          typename AGKGM0GM1GridDesc,
+          typename BGKGN0GN1GridDesc,
+          typename CGM0GM1GN0GN1GridDesc,
+          index_t GM1PerBlockGM11,
+          index_t GN1PerBlockGN11,
+          index_t KPerBlock,
+          index_t M1PerThread,
+          index_t N1PerThread,
+          index_t KPerThread,
+          index_t M1N1ThreadClusterM10,
+          index_t M1N1ThreadClusterN10,
+          index_t M1N1ThreadClusterM11,
+          index_t M1N1ThreadClusterN11,
+          typename ABlockTransferThreadSliceLengths_GK_GM0_GM10_GM11,
+          typename ABlockTransferThreadClusterLengths_GK_GM0_GM10_GM11,
+          typename ABlockTransferThreadClusterArrangeOrder,
+          typename ABlockTransferSrcAccessOrder,
+          index_t ABlockTransferSrcVectorDim,
+          index_t ABlockTransferSrcScalarPerVector,
+          index_t ABlockTransferDstScalarPerVector_GM11,
+          bool AThreadTransferSrcResetCoordinateAfterRun,
+          typename BBlockTransferThreadSliceLengths_GK_GN0_GN10_GN11,
+          typename BBlockTransferThreadClusterLengths_GK_GN0_GN10_GN11,
+          typename BBlockTransferThreadClusterArrangeOrder,
+          typename BBlockTransferSrcAccessOrder,
+          index_t BBlockTransferSrcVectorDim,
+          index_t BBlockTransferSrcScalarPerVector,
+          index_t BBlockTransferDstScalarPerVector_GN11,
+          bool BThreadTransferSrcResetCoordinateAfterRun,
+          typename CThreadTransferSrcDstAccessOrder,
+          index_t CThreadTransferSrcDstVectorDim,
+          index_t CThreadTransferDstScalarPerVector,
+          typename AGridIteratorHacks,
+          typename BGridIteratorHacks,
+          typename CGridIteratorHacks,
+          typename AGridMoveSliceWindowIteratorHacks,
+          typename BGridMoveSliceWindowIteratorHacks>
+__host__ float
+driver_dynamic_contraction_v1r1(const FloatAB* p_a_grid,
+                                const FloatAB* p_b_grid,
+                                FloatC* p_c_grid,
+                                const AGKGM0GM1GridDesc& a_gk_gm0_gm1_grid_desc,
+                                const BGKGN0GN1GridDesc& b_gk_gn0_gn1_grid_desc,
+                                const CGM0GM1GN0GN1GridDesc& c_gm0_gm1_gn0_gn1_grid_desc,
+                                AGridIteratorHacks,
+                                BGridIteratorHacks,
+                                CGridIteratorHacks,
+                                AGridMoveSliceWindowIteratorHacks,
+                                BGridMoveSliceWindowIteratorHacks,
+                                index_t nrepeat)
+
+{
+    constexpr auto I0 = Number<0>{};
+    constexpr auto I1 = Number<1>{};
+    constexpr auto I2 = Number<2>{};
+    constexpr auto I3 = Number<3>{};
+    constexpr auto I4 = Number<4>{};
+    constexpr auto I5 = Number<5>{};
+
+    // GEMM
+    using GridwiseContraction = GridwiseDynamicContraction_km0m1_kn0n1_m0m1n0n1_v1r1<
+        BlockSize,
+        FloatAB,
+        FloatAcc,
+        FloatC,
+        CGlobalMemoryDataOperation,
+        AGKGM0GM1GridDesc,
+        BGKGN0GN1GridDesc,
+        CGM0GM1GN0GN1GridDesc,
+        GM1PerBlockGM11,
+        GN1PerBlockGN11,
+        KPerBlock,
+        M1PerThread,
+        N1PerThread,
+        KPerThread,
+        M1N1ThreadClusterM10,
+        M1N1ThreadClusterN10,
+        M1N1ThreadClusterM11,
+        M1N1ThreadClusterN11,
+        ABlockTransferThreadSliceLengths_GK_GM0_GM10_GM11,
+        ABlockTransferThreadClusterLengths_GK_GM0_GM10_GM11,
+        ABlockTransferThreadClusterArrangeOrder,
+        ABlockTransferSrcAccessOrder,
+        ABlockTransferSrcVectorDim,
+        ABlockTransferSrcScalarPerVector,
+        ABlockTransferDstScalarPerVector_GM11,
+        AThreadTransferSrcResetCoordinateAfterRun,
+        BBlockTransferThreadSliceLengths_GK_GN0_GN10_GN11,
+        BBlockTransferThreadClusterLengths_GK_GN0_GN10_GN11,
+        BBlockTransferThreadClusterArrangeOrder,
+        BBlockTransferSrcAccessOrder,
+        BBlockTransferSrcVectorDim,
+        BBlockTransferSrcScalarPerVector,
+        BBlockTransferDstScalarPerVector_GN11,
+        BThreadTransferSrcResetCoordinateAfterRun,
+        CThreadTransferSrcDstAccessOrder,
+        CThreadTransferSrcDstVectorDim,
+        CThreadTransferDstScalarPerVector,
+        AGridIteratorHacks,
+        BGridIteratorHacks,
+        CGridIteratorHacks,
+        AGridMoveSliceWindowIteratorHacks,
+        BGridMoveSliceWindowIteratorHacks>;
+
+    const auto K = a_gk_gm0_gm1_grid_desc.GetLength(I0);
+
+    if(!GridwiseContraction::CheckValidity(
+           a_gk_gm0_gm1_grid_desc, b_gk_gn0_gn1_grid_desc, c_gm0_gm1_gn0_gn1_grid_desc))
+    {
+        throw std::runtime_error(
+            "wrong! GridwiseDynamicContraction_km_kn0n1_mn0n1_v1r1 has invalid setting");
+    }
+
+    const auto a_gk_gm0_gm10_gm11_grid_desc =
+        GridwiseContraction::MakeAGKGM0GM10GM11GridDescriptor(a_gk_gm0_gm1_grid_desc);
+    const auto b_gk_gn0_gn10_gn11_grid_desc =
+        GridwiseContraction::MakeBGKGN0GN10GN11GridDescriptor(b_gk_gn0_gn1_grid_desc);
+
+    using AGKGM0GM10GM11GridDesc = decltype(a_gk_gm0_gm10_gm11_grid_desc);
+    using BGKGN0GN10GN11GridDesc = decltype(b_gk_gn0_gn10_gn11_grid_desc);
+
+    // c_gm10_bm0_bm1_gn10_bn0_bn1_grid_desc
+    const auto c_gm10_bm0_bm1_gn10_bn0_bn1_grid_desc =
+        GridwiseContraction::MakeCGM10BM0BM1GN10BN0BN1GridDescriptor(c_gm0_gm1_gn0_gn1_grid_desc);
+
+    using CGM10BM0BM1GN10BN0BN1GridDesc = decltype(c_gm10_bm0_bm1_gn10_bn0_bn1_grid_desc);
+
+    // c_blockid_to_gm10_gn10_block_cluster_adaptor
+    const auto c_blockid_to_gm10_gn10_block_cluster_adaptor =
+        GridwiseContraction::MakeCBlockIdToGM10GN10BlockClusterAdaptor(c_gm0_gm1_gn0_gn1_grid_desc);
+
+    using CBlockIdToGM10GN10BlockClusterAdaptor =
+        decltype(c_blockid_to_gm10_gn10_block_cluster_adaptor);
+
+    const index_t grid_size = GridwiseContraction::CalculateGridSize(c_gm0_gm1_gn0_gn1_grid_desc);
+
+    const bool has_main_k_block_loop = GridwiseContraction::CalculateHasMainKBlockLoop(K);
+
+    const bool has_double_tail_k_block_loop =
+        GridwiseContraction::CalculateHasDoubleTailKBlockLoop(K);
+
+    {
+        std::cout << "a_gk_gm0_gm10_gm11_grid_desc{" << a_gk_gm0_gm10_gm11_grid_desc.GetLength(I0)
+                  << ", " << a_gk_gm0_gm10_gm11_grid_desc.GetLength(I1) << ", "
+                  << a_gk_gm0_gm10_gm11_grid_desc.GetLength(I2) << ", "
+                  << a_gk_gm0_gm10_gm11_grid_desc.GetLength(I3) << "}" << std::endl;
+
+        std::cout << "b_gk_gn0_gn10_gn11_grid_desc{" << b_gk_gn0_gn10_gn11_grid_desc.GetLength(I0)
+                  << ", " << b_gk_gn0_gn10_gn11_grid_desc.GetLength(I1) << ", "
+                  << b_gk_gn0_gn10_gn11_grid_desc.GetLength(I2) << ", "
+                  << b_gk_gn0_gn10_gn11_grid_desc.GetLength(I3) << "}" << std::endl;
+
+        std::cout << "c_gm10_bm0_bm1_gn10_bn0_bn1_grid_desc{ "
+                  << c_gm10_bm0_bm1_gn10_bn0_bn1_grid_desc.GetLength(I0) << ", "
+                  << c_gm10_bm0_bm1_gn10_bn0_bn1_grid_desc.GetLength(I1) << ", "
+                  << c_gm10_bm0_bm1_gn10_bn0_bn1_grid_desc.GetLength(I2) << ", "
+                  << c_gm10_bm0_bm1_gn10_bn0_bn1_grid_desc.GetLength(I3) << ", "
+                  << c_gm10_bm0_bm1_gn10_bn0_bn1_grid_desc.GetLength(I4) << ", "
+                  << c_gm10_bm0_bm1_gn10_bn0_bn1_grid_desc.GetLength(I5) << "}" << std::endl;
+    }
+
+    float ave_time = 0;
+
+    if(has_main_k_block_loop && has_double_tail_k_block_loop)
+    {
+        const auto kernel = kernel_dynamic_contraction_v1r1<
+            GridwiseContraction,
+            FloatAB,
+            FloatC,
+            remove_reference_t<AGKGM0GM10GM11GridDesc>,
+            remove_reference_t<BGKGN0GN10GN11GridDesc>,
+            remove_reference_t<CGM10BM0BM1GN10BN0BN1GridDesc>,
+            remove_reference_t<CBlockIdToGM10GN10BlockClusterAdaptor>,
+            true,
+            true>;
+
+        ave_time = launch_and_time_kernel(kernel,
+                                          nrepeat,
+                                          dim3(grid_size),
+                                          dim3(BlockSize),
+                                          0,
+                                          0,
+                                          p_a_grid,
+                                          p_b_grid,
+                                          p_c_grid,
+                                          a_gk_gm0_gm10_gm11_grid_desc,
+                                          b_gk_gn0_gn10_gn11_grid_desc,
+                                          c_gm10_bm0_bm1_gn10_bn0_bn1_grid_desc,
+                                          c_blockid_to_gm10_gn10_block_cluster_adaptor);
+    }
+    else if(has_main_k_block_loop && !has_double_tail_k_block_loop)
+    {
+        const auto kernel = kernel_dynamic_contraction_v1r1<
+            GridwiseContraction,
+            FloatAB,
+            FloatC,
+            remove_reference_t<AGKGM0GM10GM11GridDesc>,
+            remove_reference_t<BGKGN0GN10GN11GridDesc>,
+            remove_reference_t<CGM10BM0BM1GN10BN0BN1GridDesc>,
+            remove_reference_t<CBlockIdToGM10GN10BlockClusterAdaptor>,
+            true,
+            false>;
+
+        ave_time = launch_and_time_kernel(kernel,
+                                          nrepeat,
+                                          dim3(grid_size),
+                                          dim3(BlockSize),
+                                          0,
+                                          0,
+                                          p_a_grid,
+                                          p_b_grid,
+                                          p_c_grid,
+                                          a_gk_gm0_gm10_gm11_grid_desc,
+                                          b_gk_gn0_gn10_gn11_grid_desc,
+                                          c_gm10_bm0_bm1_gn10_bn0_bn1_grid_desc,
+                                          c_blockid_to_gm10_gn10_block_cluster_adaptor);
+    }
+    else if(!has_main_k_block_loop && has_double_tail_k_block_loop)
+    {
+        const auto kernel = kernel_dynamic_contraction_v1r1<
+            GridwiseContraction,
+            FloatAB,
+            FloatC,
+            remove_reference_t<AGKGM0GM10GM11GridDesc>,
+            remove_reference_t<BGKGN0GN10GN11GridDesc>,
+            remove_reference_t<CGM10BM0BM1GN10BN0BN1GridDesc>,
+            remove_reference_t<CBlockIdToGM10GN10BlockClusterAdaptor>,
+            false,
+            true>;
+
+        ave_time = launch_and_time_kernel(kernel,
+                                          nrepeat,
+                                          dim3(grid_size),
+                                          dim3(BlockSize),
+                                          0,
+                                          0,
+                                          p_a_grid,
+                                          p_b_grid,
+                                          p_c_grid,
+                                          a_gk_gm0_gm10_gm11_grid_desc,
+                                          b_gk_gn0_gn10_gn11_grid_desc,
+                                          c_gm10_bm0_bm1_gn10_bn0_bn1_grid_desc,
+                                          c_blockid_to_gm10_gn10_block_cluster_adaptor);
+    }
+    else
+    {
+        const auto kernel = kernel_dynamic_contraction_v1r1<
+            GridwiseContraction,
+            FloatAB,
+            FloatC,
+            remove_reference_t<AGKGM0GM10GM11GridDesc>,
+            remove_reference_t<BGKGN0GN10GN11GridDesc>,
+            remove_reference_t<CGM10BM0BM1GN10BN0BN1GridDesc>,
+            remove_reference_t<CBlockIdToGM10GN10BlockClusterAdaptor>,
+            false,
+            false>;
+
+        ave_time = launch_and_time_kernel(kernel,
+                                          nrepeat,
+                                          dim3(grid_size),
+                                          dim3(BlockSize),
+                                          0,
+                                          0,
+                                          p_a_grid,
+                                          p_b_grid,
+                                          p_c_grid,
+                                          a_gk_gm0_gm10_gm11_grid_desc,
+                                          b_gk_gn0_gn10_gn11_grid_desc,
+                                          c_gm10_bm0_bm1_gn10_bn0_bn1_grid_desc,
+                                          c_blockid_to_gm10_gn10_block_cluster_adaptor);
+    }
+
+    return ave_time;
+}
+
+} // namespace ck
+#endif

composable_kernel/include/driver/driver_dynamic_gemm_v1r2.hpp

+#ifndef CK_DRIVER_DYNAMIC_GEMM_V1R2
+#define CK_DRIVER_DYNAMIC_GEMM_V1R2
+
+#include "common_header.hpp"
+#include "dynamic_tensor_descriptor.hpp"
+#include "dynamic_tensor_descriptor_helper.hpp"
+#include "gridwise_dynamic_gemm_v1r2.hpp"
+
+namespace ck {
+
+template <index_t BlockSize,
+          typename FloatAB,
+          typename FloatAcc,
+          typename FloatC,
+          InMemoryDataOperation CGlobalMemoryDataOperation,
+          typename AKMGridDesc,
+          typename BKNGridDesc,
+          typename CMNGridDesc,
+          index_t MPerBlock,
+          index_t NPerBlock,
+          index_t KPerBlock,
+          index_t M1PerThread,
+          index_t N1PerThread,
+          index_t KPerThread,
+          index_t M1N1ThreadClusterM10,
+          index_t M1N1ThreadClusterN10,
+          index_t M1N1ThreadClusterM11,
+          index_t M1N1ThreadClusterN11,
+          typename ABlockTransferThreadSliceLengths_K_M0_M1,
+          typename ABlockTransferThreadClusterLengths_K_M0_M1,
+          typename ABlockTransferThreadClusterArrangeOrder,
+          typename ABlockTransferSrcAccessOrder,
+          index_t ABlockTransferSrcVectorDim,
+          index_t ABlockTransferSrcScalarPerVector,
+          index_t ABlockTransferDstScalarPerVector_M1,
+          bool AThreadTransferSrcResetCoordinateAfterRun,
+          typename BBlockTransferThreadSliceLengths_K_N0_N1,
+          typename BBlockTransferThreadClusterLengths_K_N0_N1,
+          typename BBlockTransferThreadClusterArrangeOrder,
+          typename BBlockTransferSrcAccessOrder,
+          index_t BBlockTransferSrcVectorDim,
+          index_t BBlockTransferSrcScalarPerVector,
+          index_t BBlockTransferDstScalarPerVector_N1,
+          bool BThreadTransferSrcResetCoordinateAfterRun,
+          typename CThreadTransferSrcDstAccessOrder,
+          index_t CThreadTransferSrcDstVectorDim,
+          index_t CThreadTransferDstScalarPerVector,
+          typename AGridIteratorHacks,
+          typename BGridIteratorHacks,
+          typename CGridIteratorHacks,
+          typename AGridMoveSliceWindowIteratorHacks,
+          typename BGridMoveSliceWindowIteratorHacks>
+__host__ float driver_dynamic_gemm_v1r2(const FloatAB* p_a_grid,
+                                        const FloatAB* p_b_grid,
+                                        FloatC* p_c_grid,
+                                        const AKMGridDesc& a_k_m_grid_desc,
+                                        const BKNGridDesc& b_k_n_grid_desc,
+                                        const CMNGridDesc& c_m_n_grid_desc,
+                                        AGridIteratorHacks,
+                                        BGridIteratorHacks,
+                                        CGridIteratorHacks,
+                                        AGridMoveSliceWindowIteratorHacks,
+                                        BGridMoveSliceWindowIteratorHacks,
+                                        index_t nrepeat)
+
+{
+    constexpr auto I0 = Number<0>{};
+    constexpr auto I1 = Number<1>{};
+    constexpr auto I2 = Number<2>{};
+    constexpr auto I3 = Number<3>{};
+    constexpr auto I4 = Number<4>{};
+    constexpr auto I5 = Number<5>{};
+
+    // GEMM
+    using GridwiseGemm =
+        GridwiseDynamicGemm_km_kn_mn_v1r2<BlockSize,
+                                          FloatAB,
+                                          FloatAcc,
+                                          FloatC,
+                                          CGlobalMemoryDataOperation,
+                                          AKMGridDesc,
+                                          BKNGridDesc,
+                                          CMNGridDesc,
+                                          MPerBlock,
+                                          NPerBlock,
+                                          KPerBlock,
+                                          M1PerThread,
+                                          N1PerThread,
+                                          KPerThread,
+                                          M1N1ThreadClusterM10,
+                                          M1N1ThreadClusterN10,
+                                          M1N1ThreadClusterM11,
+                                          M1N1ThreadClusterN11,
+                                          ABlockTransferThreadSliceLengths_K_M0_M1,
+                                          ABlockTransferThreadClusterLengths_K_M0_M1,
+                                          ABlockTransferThreadClusterArrangeOrder,
+                                          ABlockTransferSrcAccessOrder,
+                                          ABlockTransferSrcVectorDim,
+                                          ABlockTransferSrcScalarPerVector,
+                                          ABlockTransferDstScalarPerVector_M1,
+                                          AThreadTransferSrcResetCoordinateAfterRun,
+                                          BBlockTransferThreadSliceLengths_K_N0_N1,
+                                          BBlockTransferThreadClusterLengths_K_N0_N1,
+                                          BBlockTransferThreadClusterArrangeOrder,
+                                          BBlockTransferSrcAccessOrder,
+                                          BBlockTransferSrcVectorDim,
+                                          BBlockTransferSrcScalarPerVector,
+                                          BBlockTransferDstScalarPerVector_N1,
+                                          BThreadTransferSrcResetCoordinateAfterRun,
+                                          CThreadTransferSrcDstAccessOrder,
+                                          CThreadTransferSrcDstVectorDim,
+                                          CThreadTransferDstScalarPerVector,
+                                          AGridIteratorHacks,
+                                          BGridIteratorHacks,
+                                          CGridIteratorHacks,
+                                          AGridMoveSliceWindowIteratorHacks,
+                                          BGridMoveSliceWindowIteratorHacks>;
+
+    const auto M = a_k_m_grid_desc.GetLength(I1);
+    const auto N = b_k_n_grid_desc.GetLength(I1);
+    const auto K = a_k_m_grid_desc.GetLength(I0);
+
+    if(!GridwiseGemm::CheckValidity(a_k_m_grid_desc, b_k_n_grid_desc, c_m_n_grid_desc))
+    {
+        throw std::runtime_error("wrong! GridwiseDynamicGemm_km_kn_mn_v1r2 has invalid setting");
+    }
+
+    const auto a_k_m0_m1_grid_desc = GridwiseGemm::MakeAKM0M1GridDescriptor(a_k_m_grid_desc);
+    const auto b_k_n0_n1_grid_desc = GridwiseGemm::MakeBKN0N1GridDescriptor(b_k_n_grid_desc);
+
+    using AKM0M1GridDesc = decltype(a_k_m0_m1_grid_desc);
+    using BKN0N1GridDesc = decltype(b_k_n0_n1_grid_desc);
+
+    // c_m0_m10_m11_n0_n10_n11_grid_desc
+    const auto c_m0_m10_m11_n0_n10_n11_grid_desc =
+        GridwiseGemm::MakeCM0M10M11N0N10N11GridDescriptor(c_m_n_grid_desc);
+
+    using CM0M10M11N0N10N11GridDesc = decltype(c_m0_m10_m11_n0_n10_n11_grid_desc);
+
+    // c_blockid_to_m0_n0_block_cluster_adaptor
+    const auto c_blockid_to_m0_n0_block_cluster_adaptor =
+        GridwiseGemm::MakeCBlockIdToM0N0BlockClusterAdaptor(c_m_n_grid_desc);
+
+    using CBlockIdToM0N0BlockClusterAdaptor = decltype(c_blockid_to_m0_n0_block_cluster_adaptor);
+
+    const index_t grid_size = GridwiseGemm::CalculateGridSize(M, N);
+
+    const bool has_main_k_block_loop = GridwiseGemm::CalculateHasMainKBlockLoop(K);
+
+    const bool has_double_tail_k_block_loop = GridwiseGemm::CalculateHasDoubleTailKBlockLoop(K);
+
+    {
+        std::cout << "a_k_m0_m1_grid_desc{" << a_k_m0_m1_grid_desc.GetLength(I0) << ", "
+                  << a_k_m0_m1_grid_desc.GetLength(I1) << ", " << a_k_m0_m1_grid_desc.GetLength(I2)
+                  << "}" << std::endl;
+
+        std::cout << "b_k_n0_n1_grid_desc{" << b_k_n0_n1_grid_desc.GetLength(I0) << ", "
+                  << b_k_n0_n1_grid_desc.GetLength(I1) << ", " << b_k_n0_n1_grid_desc.GetLength(I2)
+                  << "}" << std::endl;
+
+        std::cout << "c_m0_m10_m11_n0_n10_n11_grid_desc{ "
+                  << c_m0_m10_m11_n0_n10_n11_grid_desc.GetLength(I0) << ", "
+                  << c_m0_m10_m11_n0_n10_n11_grid_desc.GetLength(I1) << ", "
+                  << c_m0_m10_m11_n0_n10_n11_grid_desc.GetLength(I2) << ", "
+                  << c_m0_m10_m11_n0_n10_n11_grid_desc.GetLength(I3) << ", "
+                  << c_m0_m10_m11_n0_n10_n11_grid_desc.GetLength(I4) << ", "
+                  << c_m0_m10_m11_n0_n10_n11_grid_desc.GetLength(I5) << "}" << std::endl;
+    }
+
+    float ave_time = 0;
+
+    if(has_main_k_block_loop && has_double_tail_k_block_loop)
+    {
+        const auto kernel =
+            kernel_dynamic_gemm_v1r2<GridwiseGemm,
+                                     FloatAB,
+                                     FloatC,
+                                     remove_reference_t<AKM0M1GridDesc>,
+                                     remove_reference_t<BKN0N1GridDesc>,
+                                     remove_reference_t<CM0M10M11N0N10N11GridDesc>,
+                                     remove_reference_t<CBlockIdToM0N0BlockClusterAdaptor>,
+                                     true,
+                                     true>;
+
+        ave_time = launch_and_time_kernel(kernel,
+                                          nrepeat,
+                                          dim3(grid_size),
+                                          dim3(BlockSize),
+                                          0,
+                                          0,
+                                          p_a_grid,
+                                          p_b_grid,
+                                          p_c_grid,
+                                          a_k_m0_m1_grid_desc,
+                                          b_k_n0_n1_grid_desc,
+                                          c_m0_m10_m11_n0_n10_n11_grid_desc,
+                                          c_blockid_to_m0_n0_block_cluster_adaptor);
+    }
+    else if(has_main_k_block_loop && !has_double_tail_k_block_loop)
+    {
+        const auto kernel =
+            kernel_dynamic_gemm_v1r2<GridwiseGemm,
+                                     FloatAB,
+                                     FloatC,
+                                     remove_reference_t<AKM0M1GridDesc>,
+                                     remove_reference_t<BKN0N1GridDesc>,
+                                     remove_reference_t<CM0M10M11N0N10N11GridDesc>,
+                                     remove_reference_t<CBlockIdToM0N0BlockClusterAdaptor>,
+                                     true,
+                                     false>;
+
+        ave_time = launch_and_time_kernel(kernel,
+                                          nrepeat,
+                                          dim3(grid_size),
+                                          dim3(BlockSize),
+                                          0,
+                                          0,
+                                          p_a_grid,
+                                          p_b_grid,
+                                          p_c_grid,
+                                          a_k_m0_m1_grid_desc,
+                                          b_k_n0_n1_grid_desc,
+                                          c_m0_m10_m11_n0_n10_n11_grid_desc,
+                                          c_blockid_to_m0_n0_block_cluster_adaptor);
+    }
+    else if(!has_main_k_block_loop && has_double_tail_k_block_loop)
+    {
+        const auto kernel =
+            kernel_dynamic_gemm_v1r2<GridwiseGemm,
+                                     FloatAB,
+                                     FloatC,
+                                     remove_reference_t<AKM0M1GridDesc>,
+                                     remove_reference_t<BKN0N1GridDesc>,
+                                     remove_reference_t<CM0M10M11N0N10N11GridDesc>,
+                                     remove_reference_t<CBlockIdToM0N0BlockClusterAdaptor>,
+                                     false,
+                                     true>;
+
+        ave_time = launch_and_time_kernel(kernel,
+                                          nrepeat,
+                                          dim3(grid_size),
+                                          dim3(BlockSize),
+                                          0,
+                                          0,
+                                          p_a_grid,
+                                          p_b_grid,
+                                          p_c_grid,
+                                          a_k_m0_m1_grid_desc,
+                                          b_k_n0_n1_grid_desc,
+                                          c_m0_m10_m11_n0_n10_n11_grid_desc,
+                                          c_blockid_to_m0_n0_block_cluster_adaptor);
+    }
+    else
+    {
+        const auto kernel =
+            kernel_dynamic_gemm_v1r2<GridwiseGemm,
+                                     FloatAB,
+                                     FloatC,
+                                     remove_reference_t<AKM0M1GridDesc>,
+                                     remove_reference_t<BKN0N1GridDesc>,
+                                     remove_reference_t<CM0M10M11N0N10N11GridDesc>,
+                                     remove_reference_t<CBlockIdToM0N0BlockClusterAdaptor>,
+                                     false,
+                                     false>;
+
+        ave_time = launch_and_time_kernel(kernel,
+                                          nrepeat,
+                                          dim3(grid_size),
+                                          dim3(BlockSize),
+                                          0,
+                                          0,
+                                          p_a_grid,
+                                          p_b_grid,
+                                          p_c_grid,
+                                          a_k_m0_m1_grid_desc,
+                                          b_k_n0_n1_grid_desc,
+                                          c_m0_m10_m11_n0_n10_n11_grid_desc,
+                                          c_blockid_to_m0_n0_block_cluster_adaptor);
+    }
+
+    return ave_time;
+}
+
+} // namespace ck
+#endif

composable_kernel/include/kernel_algorithm/transform_forward_convolution_into_gemm_v4r4_nchw_kcyx_nkhw.hpp

@@ -10,11 +10,7 @@ namespace ck {
 // GemmM = K
 // GemmN = N * Ho * Wo
 // GemmK = C * Y * X
-template <index_t GemmMPerBlock,
-          index_t GemmNPerBlock,
-          index_t GemmM1,
-          index_t GemmN1,
-          typename... Wei,
+template <typename... Wei,
           typename... In,
           typename... Out,
           typename ConvStrides,
@@ -100,74 +96,11 @@ __host__ __device__ constexpr auto transform_forward_convolution_into_gemm_v4r4_
         make_tuple(Sequence<1>{}, Sequence<0, 2>{}),
         make_tuple(Sequence<0>{}, Sequence<1>{}));
 
-    const auto GemmM = out_gemmm_gemmn_global_desc.GetLength(I0);
-    const auto GemmN = out_gemmm_gemmn_global_desc.GetLength(I1);
-    const auto GemmK = wei_gemmk_gemmm_global_desc.GetLength(I0);
-
-    assert(GemmM % GemmMPerBlock == 0 && GemmN % GemmNPerBlock == 0 && GemmK % GemmKPerBlock == 0);
-
-    const auto GemmM0 = GemmM / Number<GemmM1>{};
-    const auto GemmN0 = GemmN / Number<GemmN1>{};
-
-    const auto out_gemmm0_gemmm1_gemmn0_gemmn1_global_desc = transform_dynamic_tensor_descriptor(
-        out_gemmm_gemmn_global_desc,
-        make_tuple(make_unmerge_transform(make_tuple(GemmM0, GemmM1)),
-                   make_unmerge_transform(make_tuple(GemmN0, GemmN1))),
-        make_tuple(Sequence<0>{}, Sequence<1>{}),
-        make_tuple(Sequence<0, 1>{}, Sequence<2, 3>{}));
-
-    // out_gemm_block_cluster_desc
-    const auto out_gemm_block_cluster_desc = make_cluster_descriptor_v2(
-        make_tuple(GemmM / Number<GemmMPerBlock>{}, GemmN / Number<GemmNPerBlock>{}));
-
-    // hack to control index calculation when iterating over wei_gemmk_gemmm_global tensor
-    constexpr auto wei_gemmk_gemmm_global_iterator_hacks =
-        make_tuple(make_tuple(Sequence<0, 0, 0>{}, Sequence<0, 0, 0>{}),
-                   make_tuple(Sequence<0, 0, 0>{}, Sequence<0, 0, 0>{}));
-
-    constexpr auto wei_gemmk_gemmm_global_move_slice_window_iterator_hacks = Sequence<0, 0, 0>{};
-
-    // hack to control index calculation when iterating over in_gemmk_gemmn_global tensor
-    constexpr auto in_gemmk_gemmn_global_iterator_hacks =
-        make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0>{},
-                              Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1>{}),
-                   make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0>{},
-                              Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2>{}));
-
-    constexpr auto in_gemmk_gemmn_global_move_slice_window_iterator_hacks =
-        Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2>{};
-
-    // hack to control index calculation when iterating over out_gemmm0_gemmm1_gemmn0_gemmn1_global
-    // tensor hack for NKHW format
-    constexpr auto out_gemmm0_gemmm1_gemmn0_gemmn1_global_iterator_hacks =
-        make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0>{},
-                              Sequence<0, 0, 0, 0, 0>{},
-                              Sequence<0, 0, 1, 0, 0>{},
-                              Sequence<0, 0, 1, 0, 0>{}),
-                   make_tuple(Sequence<0, 0, 0, 0, 0>{},
-                              Sequence<0, 0, 0, 0, 0>{},
-                              Sequence<0, 0, 2, 0, 0>{},
-                              Sequence<0, 0, 2, 0, 0>{}));
-
-    return make_tuple(wei_gemmk_gemmm_global_desc,
-                      in_gemmk_gemmn_global_desc,
-                      out_gemmm0_gemmm1_gemmn0_gemmn1_global_desc,
-                      out_gemm_block_cluster_desc,
-                      wei_gemmk_gemmm_global_iterator_hacks,
-                      in_gemmk_gemmn_global_iterator_hacks,
-                      out_gemmm0_gemmm1_gemmn0_gemmn1_global_iterator_hacks,
-                      wei_gemmk_gemmm_global_move_slice_window_iterator_hacks,
-                      in_gemmk_gemmn_global_move_slice_window_iterator_hacks);
+    return make_tuple(
+        wei_gemmk_gemmm_global_desc, in_gemmk_gemmn_global_desc, out_gemmm_gemmn_global_desc);
 }
 
-// GemmM = K
-// GemmN = N * Ho * Wo
-// GemmK = C * Y * X
-template <index_t GemmMPerBlock,
-          index_t GemmNPerBlock,
-          index_t GemmM1,
-          index_t GemmN1,
-          typename... Wei,
+template <typename... Wei,
           typename... In,
           typename... Out,
           typename ConvStrides,
@@ -247,72 +180,11 @@ transform_forward_convolution_into_gemm_v4r4_nchw_kcyx_nkhw_no_pad(
         make_tuple(Sequence<1>{}, Sequence<0, 2>{}),
         make_tuple(Sequence<0>{}, Sequence<1>{}));
 
-    const auto GemmM = out_gemmm_gemmn_global_desc.GetLength(I0);
-    const auto GemmN = out_gemmm_gemmn_global_desc.GetLength(I1);
-    const auto GemmK = wei_gemmk_gemmm_global_desc.GetLength(I0);
-
-    assert(GemmM % GemmMPerBlock == 0 && GemmN % GemmNPerBlock == 0 && GemmK % GemmKPerBlock == 0);
-
-    const auto GemmM0 = GemmM / Number<GemmM1>{};
-    const auto GemmN0 = GemmN / Number<GemmN1>{};
-
-    const auto out_gemmm0_gemmm1_gemmn0_gemmn1_global_desc = transform_dynamic_tensor_descriptor(
-        out_gemmm_gemmn_global_desc,
-        make_tuple(make_unmerge_transform(make_tuple(GemmM0, GemmM1)),
-                   make_unmerge_transform(make_tuple(GemmN0, GemmN1))),
-        make_tuple(Sequence<0>{}, Sequence<1>{}),
-        make_tuple(Sequence<0, 1>{}, Sequence<2, 3>{}));
-
-    // out_gemm_block_cluster_desc
-    const auto out_gemm_block_cluster_desc = make_cluster_descriptor_v2(
-        make_tuple(GemmM / Number<GemmMPerBlock>{}, GemmN / Number<GemmNPerBlock>{}));
-
-    // hack to control index calculation when iterating over a_k_m_global tensor
-    constexpr auto wei_gemmk_gemmm_global_iterator_hacks =
-        make_tuple(make_tuple(Sequence<0, 0, 0>{}, Sequence<0, 0, 0>{}),
-                   make_tuple(Sequence<0, 0, 0>{}, Sequence<0, 0, 0>{}));
-
-    constexpr auto wei_gemmk_gemmm_global_move_slice_window_iterator_hacks = Sequence<0, 0, 0>{};
-
-    // hack to control index calculation when iterating over b_k_n_global tensor
-    constexpr auto in_gemmk_gemmn_global_iterator_hacks =
-        make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 1, 0>{}, Sequence<0, 0, 0, 0, 0, 0, 1>{}),
-                   make_tuple(Sequence<0, 0, 0, 0, 0, 2, 0>{}, Sequence<0, 0, 0, 0, 0, 0, 2>{}));
-
-    constexpr auto in_gemmk_gemmn_global_move_slice_window_iterator_hacks =
-        Sequence<0, 0, 0, 0, 0, 1, 2>{};
-
-    // hack to control index calculation when iterating over c_m0_m1_n0_n1_global tensor
-    // hack for NKHW format
-    constexpr auto out_gemmm0_gemmm1_gemmn0_gemmn1_global_iterator_hacks =
-        make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0>{},
-                              Sequence<0, 0, 0, 0, 0>{},
-                              Sequence<0, 0, 1, 0, 0>{},
-                              Sequence<0, 0, 1, 0, 0>{}),
-                   make_tuple(Sequence<0, 0, 0, 0, 0>{},
-                              Sequence<0, 0, 0, 0, 0>{},
-                              Sequence<0, 0, 2, 0, 0>{},
-                              Sequence<0, 0, 2, 0, 0>{}));
-
-    return make_tuple(wei_gemmk_gemmm_global_desc,
-                      in_gemmk_gemmn_global_desc,
-                      out_gemmm0_gemmm1_gemmn0_gemmn1_global_desc,
-                      out_gemm_block_cluster_desc,
-                      wei_gemmk_gemmm_global_iterator_hacks,
-                      in_gemmk_gemmn_global_iterator_hacks,
-                      out_gemmm0_gemmm1_gemmn0_gemmn1_global_iterator_hacks,
-                      wei_gemmk_gemmm_global_move_slice_window_iterator_hacks,
-                      in_gemmk_gemmn_global_move_slice_window_iterator_hacks);
+    return make_tuple(
+        wei_gemmk_gemmm_global_desc, in_gemmk_gemmn_global_desc, out_gemmm_gemmn_global_desc);
 }
 
-// GemmM = K
-// GemmN = N * Ho * Wo
-// GemmK = C * Y * X
-template <index_t GemmMPerBlock,
-          index_t GemmNPerBlock,
-          index_t GemmM1,
-          index_t GemmN1,
-          typename... Wei,
+template <typename... Wei,
           typename... In,
           typename... Out,
           typename ConvStrides,
@@ -383,60 +255,8 @@ __host__ __device__ constexpr auto transform_forward_convolution_into_gemm_v4r4_
         make_tuple(Sequence<1>{}, Sequence<0, 2>{}),
         make_tuple(Sequence<0>{}, Sequence<1>{}));
 
-    const auto GemmM = out_gemmm_gemmn_global_desc.GetLength(I0);
-    const auto GemmN = out_gemmm_gemmn_global_desc.GetLength(I1);
-    const auto GemmK = wei_gemmk_gemmm_global_desc.GetLength(I0);
-
-    assert(GemmM % GemmMPerBlock == 0 && GemmN % GemmNPerBlock == 0 && GemmK % GemmKPerBlock == 0);
-
-    const auto GemmM0 = GemmM / Number<GemmM1>{};
-    const auto GemmN0 = GemmN / Number<GemmN1>{};
-
-    const auto out_gemmm0_gemmm1_gemmn0_gemmn1_global_desc = transform_dynamic_tensor_descriptor(
-        out_gemmm_gemmn_global_desc,
-        make_tuple(make_unmerge_transform(make_tuple(GemmM0, GemmM1)),
-                   make_unmerge_transform(make_tuple(GemmN0, GemmN1))),
-        make_tuple(Sequence<0>{}, Sequence<1>{}),
-        make_tuple(Sequence<0, 1>{}, Sequence<2, 3>{}));
-
-    // out_gemm_block_cluster_desc
-    const auto out_gemm_block_cluster_desc = make_cluster_descriptor_v2(
-        make_tuple(GemmM / Number<GemmMPerBlock>{}, GemmN / Number<GemmNPerBlock>{}));
-
-    // hack to control index calculation when iterating over a_k_m_global tensor
-    constexpr auto wei_gemmk_gemmm_global_iterator_hacks =
-        make_tuple(make_tuple(Sequence<0, 0, 0>{}, Sequence<0, 0, 0>{}),
-                   make_tuple(Sequence<0, 0, 0>{}, Sequence<0, 0, 0>{}));
-
-    constexpr auto wei_gemmk_gemmm_global_move_slice_window_iterator_hacks = Sequence<0, 0, 0>{};
-
-    // hack to control index calculation when iterating over b_k_n_global tensor
-    constexpr auto in_gemmk_gemmn_global_iterator_hacks =
-        make_tuple(make_tuple(Sequence<0, 1, 0>{}, Sequence<0, 0, 1>{}),
-                   make_tuple(Sequence<0, 2, 0>{}, Sequence<0, 0, 2>{}));
-
-    constexpr auto in_gemmk_gemmn_global_move_slice_window_iterator_hacks = Sequence<0, 1, 2>{};
-
-    // hack to control index calculation when iterating over c_m0_m1_n0_n1_global tensor
-    constexpr auto out_gemmm0_gemmm1_gemmn0_gemmn1_global_iterator_hacks =
-        make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0>{},
-                              Sequence<0, 0, 0, 0, 0>{},
-                              Sequence<0, 0, 1, 0, 0>{},
-                              Sequence<0, 0, 1, 0, 0>{}),
-                   make_tuple(Sequence<0, 0, 0, 0, 0>{},
-                              Sequence<0, 0, 0, 0, 0>{},
-                              Sequence<0, 0, 2, 0, 0>{},
-                              Sequence<0, 0, 2, 0, 0>{}));
-
-    return make_tuple(wei_gemmk_gemmm_global_desc,
-                      in_gemmk_gemmn_global_desc,
-                      out_gemmm0_gemmm1_gemmn0_gemmn1_global_desc,
-                      out_gemm_block_cluster_desc,
-                      wei_gemmk_gemmm_global_iterator_hacks,
-                      in_gemmk_gemmn_global_iterator_hacks,
-                      out_gemmm0_gemmm1_gemmn0_gemmn1_global_iterator_hacks,
-                      wei_gemmk_gemmm_global_move_slice_window_iterator_hacks,
-                      in_gemmk_gemmn_global_move_slice_window_iterator_hacks);
+    return make_tuple(
+        wei_gemmk_gemmm_global_desc, in_gemmk_gemmn_global_desc, out_gemmm_gemmn_global_desc);
 }
 
 } // namespace ck

composable_kernel/include/kernel_algorithm/transform_forward_convolution_into_gemm_v4r4_nhwc_kyxc_nhwk.hpp

@@ -10,11 +10,7 @@ namespace ck {
 // GemmM = K
 // GemmN = N * Ho * Wo
 // GemmK = C * Y * X
-template <index_t GemmMPerBlock,
-          index_t GemmNPerBlock,
-          index_t GemmM1,
-          index_t GemmN1,
-          typename... Wei,
+template <typename... Wei,
           typename... In,
           typename... Out,
           typename ConvStrides,
@@ -100,74 +96,11 @@ __host__ __device__ constexpr auto transform_forward_convolution_into_gemm_v4r4_
         make_tuple(Sequence<0>{}, Sequence<1>{}),
         make_tuple(Sequence<1>{}, Sequence<0>{}));
 
-    const auto GemmM = out_gemmm_gemmn_global_desc.GetLength(I0);
-    const auto GemmN = out_gemmm_gemmn_global_desc.GetLength(I1);
-    const auto GemmK = wei_gemmk_gemmm_global_desc.GetLength(I0);
-
-    assert(GemmM % GemmMPerBlock == 0 && GemmN % GemmNPerBlock == 0 && GemmK % GemmKPerBlock == 0);
-
-    const auto GemmM0 = GemmM / Number<GemmM1>{};
-    const auto GemmN0 = GemmN / Number<GemmN1>{};
-
-    const auto out_gemmm0_gemmm1_gemmn0_gemmn1_global_desc = transform_dynamic_tensor_descriptor(
-        out_gemmm_gemmn_global_desc,
-        make_tuple(make_unmerge_transform(make_tuple(GemmM0, GemmM1)),
-                   make_unmerge_transform(make_tuple(GemmN0, GemmN1))),
-        make_tuple(Sequence<0>{}, Sequence<1>{}),
-        make_tuple(Sequence<0, 1>{}, Sequence<2, 3>{}));
-
-    // out_gemm_block_cluster_desc
-    const auto out_gemm_block_cluster_desc = make_cluster_descriptor_v2(
-        make_tuple(GemmM / Number<GemmMPerBlock>{}, GemmN / Number<GemmNPerBlock>{}));
-
-    // hack to control index calculation when iterating over a_k_m_global tensor
-    constexpr auto wei_gemmk_gemmm_global_iterator_hacks =
-        make_tuple(make_tuple(Sequence<0, 0, 0>{}, Sequence<0, 0, 0>{}),
-                   make_tuple(Sequence<0, 0, 0>{}, Sequence<0, 0, 0>{}));
-
-    constexpr auto wei_gemmk_gemmm_global_move_slice_window_iterator_hacks = Sequence<0, 0, 0>{};
-
-    // hack to control index calculation when iterating over b_k_n_global tensor
-    constexpr auto in_gemmk_gemmn_global_iterator_hacks =
-        make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0>{},
-                              Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1>{}),
-                   make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0>{},
-                              Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2>{}));
-
-    constexpr auto in_gemmk_gemmn_global_move_slice_window_iterator_hacks =
-        Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2>{};
-
-    // hack to control index calculation when iterating over c_m0_m1_n0_n1_global tensor
-    // hack for NKHW format
-    constexpr auto out_gemmm0_gemmm1_gemmn0_gemmn1_global_iterator_hacks =
-        make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0>{},
-                              Sequence<0, 0, 0, 0, 0>{},
-                              Sequence<0, 0, 1, 0, 0>{},
-                              Sequence<0, 0, 1, 0, 0>{}),
-                   make_tuple(Sequence<0, 0, 0, 0, 0>{},
-                              Sequence<0, 0, 0, 0, 0>{},
-                              Sequence<0, 0, 2, 0, 0>{},
-                              Sequence<0, 0, 2, 0, 0>{}));
-
-    return make_tuple(wei_gemmk_gemmm_global_desc,
-                      in_gemmk_gemmn_global_desc,
-                      out_gemmm0_gemmm1_gemmn0_gemmn1_global_desc,
-                      out_gemm_block_cluster_desc,
-                      wei_gemmk_gemmm_global_iterator_hacks,
-                      in_gemmk_gemmn_global_iterator_hacks,
-                      out_gemmm0_gemmm1_gemmn0_gemmn1_global_iterator_hacks,
-                      wei_gemmk_gemmm_global_move_slice_window_iterator_hacks,
-                      in_gemmk_gemmn_global_move_slice_window_iterator_hacks);
+    return make_tuple(
+        wei_gemmk_gemmm_global_desc, in_gemmk_gemmn_global_desc, out_gemmm_gemmn_global_desc);
 }
 
-// GemmM = K
-// GemmN = N * Ho * Wo
-// GemmK = C * Y * X
-template <index_t GemmMPerBlock,
-          index_t GemmNPerBlock,
-          index_t GemmM1,
-          index_t GemmN1,
-          typename... Wei,
+template <typename... Wei,
           typename... In,
           typename... Out,
           typename ConvStrides,
@@ -238,61 +171,8 @@ __host__ __device__ constexpr auto transform_forward_convolution_into_gemm_v4r4_
         make_tuple(Sequence<0>{}, Sequence<1>{}),
         make_tuple(Sequence<1>{}, Sequence<0>{}));
 
-    const auto GemmM = out_gemmm_gemmn_global_desc.GetLength(I0);
-    const auto GemmN = out_gemmm_gemmn_global_desc.GetLength(I1);
-    const auto GemmK = wei_gemmk_gemmm_global_desc.GetLength(I0);
-
-    assert(GemmM % GemmMPerBlock == 0 && GemmN % GemmNPerBlock == 0 && GemmK % GemmKPerBlock == 0);
-
-    const auto GemmM0 = GemmM / Number<GemmM1>{};
-    const auto GemmN0 = GemmN / Number<GemmN1>{};
-
-    const auto out_gemmm0_gemmm1_gemmn0_gemmn1_global_desc = transform_dynamic_tensor_descriptor(
-        out_gemmm_gemmn_global_desc,
-        make_tuple(make_unmerge_transform(make_tuple(GemmM0, GemmM1)),
-                   make_unmerge_transform(make_tuple(GemmN0, GemmN1))),
-        make_tuple(Sequence<0>{}, Sequence<1>{}),
-        make_tuple(Sequence<0, 1>{}, Sequence<2, 3>{}));
-
-    // out_gemm_block_cluster_desc
-    const auto out_gemm_block_cluster_desc = make_cluster_descriptor_v2(
-        make_tuple(GemmM / Number<GemmMPerBlock>{}, GemmN / Number<GemmNPerBlock>{}));
-
-    // hack to control index calculation when iterating over wei_gemmk_gemmm_global_iterator_hacks
-    // tensor
-    constexpr auto wei_gemmk_gemmm_global_iterator_hacks =
-        make_tuple(make_tuple(Sequence<0, 0, 0>{}, Sequence<0, 0, 0>{}),
-                   make_tuple(Sequence<0, 0, 0>{}, Sequence<0, 0, 0>{}));
-
-    constexpr auto wei_gemmk_gemmm_global_move_slice_window_iterator_hacks = Sequence<0, 0, 0>{};
-
-    // hack to control index calculation when iterating over b_k_n_global tensor
-    constexpr auto in_gemmk_gemmn_global_iterator_hacks =
-        make_tuple(make_tuple(Sequence<0, 0, 0>{}, Sequence<0, 0, 0>{}),
-                   make_tuple(Sequence<0, 0, 0>{}, Sequence<0, 0, 0>{}));
-
-    constexpr auto in_gemmk_gemmn_global_move_slice_window_iterator_hacks = Sequence<0, 0, 0>{};
-
-    // hack to control index calculation when iterating over c_m0_m1_n0_n1_global tensor
-    constexpr auto out_gemmm0_gemmm1_gemmn0_gemmn1_global_iterator_hacks =
-        make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0>{},
-                              Sequence<0, 0, 0, 0, 0>{},
-                              Sequence<0, 0, 0, 0, 0>{},
-                              Sequence<0, 0, 0, 0, 0>{}),
-                   make_tuple(Sequence<0, 0, 0, 0, 0>{},
-                              Sequence<0, 0, 0, 0, 0>{},
-                              Sequence<0, 0, 0, 0, 0>{},
-                              Sequence<0, 0, 0, 0, 0>{}));
-
-    return make_tuple(wei_gemmk_gemmm_global_desc,
-                      in_gemmk_gemmn_global_desc,
-                      out_gemmm0_gemmm1_gemmn0_gemmn1_global_desc,
-                      out_gemm_block_cluster_desc,
-                      wei_gemmk_gemmm_global_iterator_hacks,
-                      in_gemmk_gemmn_global_iterator_hacks,
-                      out_gemmm0_gemmm1_gemmn0_gemmn1_global_iterator_hacks,
-                      wei_gemmk_gemmm_global_move_slice_window_iterator_hacks,
-                      in_gemmk_gemmn_global_move_slice_window_iterator_hacks);
+    return make_tuple(
+        wei_gemmk_gemmm_global_desc, in_gemmk_gemmn_global_desc, out_gemmm_gemmn_global_desc);
 }
 
 } // namespace ck

composable_kernel/include/kernel_algorithm/transform_forward_convolution_into_gemm_v4r5_nchw_kcyx_nkhw.hpp

+#ifndef CK_TRANSFORM_FORWARD_CONVOLUTION_INTO_CONTRACTION_V4R5_NCHW_KCYX_NKHW_HPP
+#define CK_TRANSFORM_FORWARD_CONVOLUTION_INTO_CONTRACTION_V4R5_NCHW_KCYX_NKHW_HPP
+
+#include "common_header.hpp"
+#include "dynamic_tensor_descriptor.hpp"
+#include "dynamic_tensor_descriptor_helper.hpp"
+
+namespace ck {
+
+// GemmM = K
+// GemmN = N * Ho * Wo
+// GemmK = C * Y * X
+template <index_t N0_,
+          typename... Wei,
+          typename... In,
+          typename... Out,
+          typename ConvStrides,
+          typename ConvDilations,
+          typename InLeftPads,
+          typename InRightPads>
+__host__ __device__ constexpr auto
+transform_forward_convolution_into_contraction_v4r5_nchw_kcyx_nkhw_pad(
+    const DynamicTensorDescriptor<Wei...>& wei_k_c_y_x_grid_desc,
+    const DynamicTensorDescriptor<In...>& in_n_c_hi_wi_grid_desc,
+    const DynamicTensorDescriptor<Out...>& out_n_k_ho_wo_grid_desc,
+    const ConvStrides& conv_strides,
+    const ConvDilations& conv_dilations,
+    const InLeftPads& in_left_pads,
+    const InRightPads& in_right_pads)
+{
+    constexpr auto I0 = Number<0>{};
+    constexpr auto I1 = Number<1>{};
+    constexpr auto I2 = Number<2>{};
+    constexpr auto I3 = Number<3>{};
+
+    const auto N = in_n_c_hi_wi_grid_desc.GetLength(I0);
+    const auto C = in_n_c_hi_wi_grid_desc.GetLength(I1);
+    const auto K = out_n_k_ho_wo_grid_desc.GetLength(I1);
+
+    const auto Hi = in_n_c_hi_wi_grid_desc.GetLength(I2);
+    const auto Wi = in_n_c_hi_wi_grid_desc.GetLength(I3);
+
+    const auto Ho = out_n_k_ho_wo_grid_desc.GetLength(I2);
+    const auto Wo = out_n_k_ho_wo_grid_desc.GetLength(I3);
+
+    const auto Y = wei_k_c_y_x_grid_desc.GetLength(I2);
+    const auto X = wei_k_c_y_x_grid_desc.GetLength(I3);
+
+    const auto ConvStrideH = conv_strides[I0];
+    const auto ConvStrideW = conv_strides[I1];
+
+    const auto ConvDilationH = conv_dilations[I0];
+    const auto ConvDilationW = conv_dilations[I1];
+
+    const auto InLeftPadH = in_left_pads[I0];
+    const auto InLeftPadW = in_left_pads[I1];
+
+    const auto InRightPadH = in_right_pads[I0];
+    const auto InRightPadW = in_right_pads[I1];
+
+    // weight tensor
+    const auto wei_gk_gm0_gm1_grid_desc = transform_dynamic_tensor_descriptor(
+        make_dynamic_naive_tensor_descriptor_packed_v2(make_tuple(K, C * Y * X)),
+        make_tuple(make_unmerge_transform(make_tuple(I1, K)),
+                   make_pass_through_transform(C * Y * X)),
+        make_tuple(Sequence<0>{}, Sequence<1>{}),
+        make_tuple(Sequence<1, 2>{}, Sequence<0>{}));
+
+    // input tensor
+    const auto in_n_c_hip_wip_grid_desc = transform_dynamic_tensor_descriptor(
+        in_n_c_hi_wi_grid_desc,
+        make_tuple(make_pass_through_transform(N),
+                   make_pass_through_transform(C),
+                   make_pad_transform(Hi, InLeftPadH, InRightPadH),
+                   make_pad_transform(Wi, InLeftPadW, InRightPadW)),
+        make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}),
+        make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}));
+
+    constexpr auto N0 = Number<N0_>{};
+    const auto N1     = N / N0;
+
+    const auto in_n0_n1_c_y_ho_x_wo_grid_desc = transform_dynamic_tensor_descriptor(
+        in_n_c_hip_wip_grid_desc,
+        make_tuple(make_unmerge_transform(make_tuple(N0, N1)),
+                   make_pass_through_transform(C),
+                   make_embed_transform(make_tuple(Y, Ho), make_tuple(ConvDilationH, ConvStrideH)),
+                   make_embed_transform(make_tuple(X, Wo), make_tuple(ConvDilationW, ConvStrideW))),
+        make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}),
+        make_tuple(Sequence<0, 1>{}, Sequence<2>{}, Sequence<3, 4>{}, Sequence<5, 6>{}));
+
+    const auto in_gk_gn0_gn1_grid_desc = transform_dynamic_tensor_descriptor(
+        in_n0_n1_c_y_ho_x_wo_grid_desc,
+        make_tuple(make_merge_transform(make_tuple(C, Y, X)),
+                   make_pass_through_transform(N0),
+                   make_merge_transform(make_tuple(N1, Ho, Wo))),
+        make_tuple(Sequence<2, 3, 5>{}, Sequence<0>{}, Sequence<1, 4, 6>{}),
+        make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}));
+
+    // output tensor
+    const auto out_n_k_howo_grid_desc =
+        make_dynamic_naive_tensor_descriptor_packed_v2(make_tuple(N, K, Ho * Wo));
+
+    const auto out_n0_n1_1_k_howo_grid_desc = transform_dynamic_tensor_descriptor(
+        out_n_k_howo_grid_desc,
+        make_tuple(make_unmerge_transform(make_tuple(Number<N0>{}, N1)),
+                   make_unmerge_transform(make_tuple(I1, K)),
+                   make_pass_through_transform(Ho * Wo)),
+        make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}),
+        make_tuple(Sequence<0, 1>{}, Sequence<2, 3>{}, Sequence<4>{}));
+
+    const auto out_gm0_gm1_gn0_gn1_grid_desc = transform_dynamic_tensor_descriptor(
+        out_n0_n1_1_k_howo_grid_desc,
+        make_tuple(make_pass_through_transform(I1),
+                   make_pass_through_transform(K),
+                   make_pass_through_transform(Number<N0>{}),
+                   make_merge_transform_v2_magic_division(make_tuple(N1, Ho * Wo))),
+        make_tuple(Sequence<2>{}, Sequence<3>{}, Sequence<0>{}, Sequence<1, 4>{}),
+        make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}));
+
+    return make_tuple(
+        wei_gk_gm0_gm1_grid_desc, in_gk_gn0_gn1_grid_desc, out_gm0_gm1_gn0_gn1_grid_desc);
+}
+
+} // namespace ck
+#endif

composable_kernel/include/tensor_description/dynamic_multi_index_transform.hpp

@@ -1164,6 +1164,165 @@ struct DynamicMerge_v2_magic_division
     }
 };
 
+// Implementation of "Merge" transformation primitive that uses magic-number-division to do lowering
+// of both multi-index and delta of multi-index
+// Caution:
+//   1. The magic number division implementation being used would produce correct result if the
+//   dividended is uint32_t and its value is with in 31-bit value range of uint32_t.
+//   2. The magic number division for int32_t dividened has not been implemented, the int32_t
+//   dividend would be bit-wise interpreted as uint32_t and magic number division implementation for
+//   uint32_t is then used.
+//   3. For Merge primitive, upper-index is the dividend.
+//   4. When upper-index is uint32_t, its value need to be within 31-bit range.
+//   5. When upper-index is int32_t type (when index_t is int32_t), its value need to be
+//   non-negative.
+template <typename LowLengths>
+struct DynamicMerge_v2r2_magic_division
+{
+    static constexpr index_t NDimLow = LowLengths::Size();
+
+    using LowerIndex = MultiIndex<NDimLow>;
+    using UpperIndex = MultiIndex<1>;
+
+    using LowLengthsScan = decltype(
+        container_reverse_exclusive_scan(LowLengths{}, math::multiplies_v2{}, Number<1>{}));
+
+    using UpLengths =
+        decltype(make_tuple(container_reduce(LowLengths{}, math::multiplies_v2{}, Number<1>{})));
+
+    using LowLengthsScanMagicDivisorMultipiler = decltype(generate_tuple(
+        lambda_merge_generate_MagicDivision_calculate_magic_multiplier<LowLengthsScan>{},
+        Number<NDimLow>{}));
+
+    using LowLengthsScanMagicDivisorShift = decltype(
+        generate_tuple(lambda_merge_generate_MagicDivision_calculate_magic_shift<LowLengthsScan>{},
+                       Number<NDimLow>{}));
+
+    LowLengths low_lengths_;
+    LowLengthsScan low_lengths_scan_;
+    LowLengthsScanMagicDivisorMultipiler low_lengths_scan_magic_divisor_multiplier_;
+    LowLengthsScanMagicDivisorShift low_lengths_scan_magic_divisor_shift_;
+    UpLengths up_lengths_;
+
+    __host__ __device__ constexpr DynamicMerge_v2r2_magic_division() = default;
+
+    __host__ __device__ constexpr DynamicMerge_v2r2_magic_division(const LowLengths& low_lengths)
+        : low_lengths_{low_lengths},
+          low_lengths_scan_{
+              container_reverse_exclusive_scan(low_lengths, math::multiplies_v2{}, Number<1>{})},
+          low_lengths_scan_magic_divisor_multiplier_{generate_tuple(
+              [&](auto i) { return MagicDivision::CalculateMagicMultiplier(low_lengths_scan_[i]); },
+              Number<NDimLow>{})},
+          low_lengths_scan_magic_divisor_shift_{generate_tuple(
+              [&](auto i) { return MagicDivision::CalculateMagicShift(low_lengths_scan_[i]); },
+              Number<NDimLow>{})},
+          up_lengths_{make_tuple(container_reduce(low_lengths, math::multiplies_v2{}, Number<1>{}))}
+    {
+        static_assert(LowerIndex::Size() == NDimLow, "wrong!");
+    }
+
+    __host__ __device__ static constexpr index_t GetNumOfLowerDimension() { return NDimLow; }
+
+    __host__ __device__ static constexpr index_t GetNumOfUpperDimension() { return 1; }
+
+    __host__ __device__ constexpr const auto& GetUpperLengths() const { return up_lengths_; }
+
+    template <typename LowIdx, typename UpIdx>
+    __host__ __device__ constexpr void CalculateLowerIndex(LowIdx& idx_low,
+                                                           const UpIdx& idx_up) const
+    {
+        static_assert(LowIdx::Size() == NDimLow && UpIdx::Size() == 1,
+                      "wrong! inconsistent # of dimension");
+
+        index_t tmp = idx_up[Number<0>{}];
+
+        static_for<0, NDimLow - 1, 1>{}([&, this](auto i) {
+            idx_low(i) =
+                MagicDivision::DoMagicDivision(tmp,
+                                               this->low_lengths_scan_magic_divisor_multiplier_[i],
+                                               this->low_lengths_scan_magic_divisor_shift_[i]);
+
+            tmp -= idx_low[i] * this->low_lengths_scan_[i];
+        });
+
+        idx_low(Number<NDimLow - 1>{}) = tmp;
+    }
+
+    template <typename LowIdxDiff,
+              typename UpIdxDiff,
+              typename LowIdx,
+              typename UpIdx,
+              index_t Hack>
+    __host__ __device__ void UpdateLowerIndex(LowIdxDiff& idx_diff_low,
+                                              const UpIdxDiff& idx_diff_up,
+                                              LowIdx& idx_low,
+                                              const UpIdx& idx_up_new,
+                                              Number<Hack>) const
+    {
+        static_assert(LowIdxDiff::Size() == NDimLow && UpIdxDiff::Size() == 1 &&
+                          LowIdx::Size() == NDimLow && UpIdx::Size() == 1,
+                      "wrong! inconsistent # of dimension");
+
+        index_t tmp = idx_up_new[Number<0>{}];
+
+        static_for<0, NDimLow - 1, 1>{}([&, this](auto i) {
+            index_t idx_low_old = idx_low[i];
+
+            idx_low(i) =
+                MagicDivision::DoMagicDivision(tmp,
+                                               this->low_lengths_scan_magic_divisor_multiplier_[i],
+                                               this->low_lengths_scan_magic_divisor_shift_[i]);
+
+            idx_diff_low(i) = idx_low[i] - idx_low_old;
+
+            tmp -= idx_low[i] * this->low_lengths_scan_[i];
+        });
+
+        idx_diff_low(Number<NDimLow - 1>{}) = tmp - idx_low[Number<NDimLow - 1>{}];
+
+        idx_low(Number<NDimLow - 1>{}) = tmp;
+    }
+
+    __host__ __device__ static constexpr bool IsLinearTransform() { return false; }
+
+    __host__ __device__ static constexpr bool IsValidUpperIndexAlwaysMappedToValidLowerIndex()
+    {
+        return true;
+    }
+
+    __host__ __device__ static constexpr bool IsKnownAtCompileTime()
+    {
+        return is_known_at_compile_time<LowLengths>::value &&
+               is_known_at_compile_time<LowLengthsScanMagicDivisorMultipiler>::value &&
+               is_known_at_compile_time<LowLengthsScanMagicDivisorShift>::value &&
+               is_known_at_compile_time<UpLengths>::value;
+    }
+
+    template <typename UpIdx>
+    __host__ __device__ static constexpr bool
+    IsValidUpperIndexMappedToValidLowerIndex(const UpIdx& /* idx_up */)
+    {
+        return true;
+    }
+
+    __host__ __device__ void Print() const
+    {
+        printf("{");
+        printf("DynamicMerge_v2r2_magic_division, ");
+        printf("low_lengths_ ");
+        print_multi_index(low_lengths_);
+        printf("low_lengths_scan ");
+        print_multi_index(low_lengths_scan_);
+        printf("low_lengths_scan_magic_divisor_multiplier_ ");
+        print_multi_index(low_lengths_scan_magic_divisor_multiplier_);
+        printf("low_lengths_scan_magic_divisor_shift_ ");
+        print_multi_index(low_lengths_scan_magic_divisor_shift_);
+        printf("up_lengths_ ");
+        print_multi_index(up_lengths_);
+        printf("}");
+    }
+};
+
 template <typename UpLengths, bool Use24BitIntegerCalculation>
 struct DynamicUnMerge
 {

composable_kernel/include/tensor_description/dynamic_multi_index_transform_helper.hpp

@@ -56,10 +56,21 @@ __host__ __device__ constexpr auto make_merge_transform(const LowLengths& low_le
 #if !CK_EXPERIMENTAL_MERGE_USE_MAGIC_DIVISION
     return DynamicMerge_v1_carry_check<LowLengths>{low_lengths};
 #else
+#if 1
     return DynamicMerge_v2_magic_division<LowLengths>{low_lengths};
+#else
+    return DynamicMerge_v2r2_magic_division<LowLengths>{low_lengths};
+#endif
 #endif
 }
 
+template <typename LowLengths>
+__host__ __device__ constexpr auto
+make_merge_transform_v2_magic_division(const LowLengths& low_lengths)
+{
+    return DynamicMerge_v2_magic_division<LowLengths>{low_lengths};
+}
+
 template <typename UpLengths, bool Use24BitIntegerCalculation = false>
 __host__ __device__ constexpr auto make_unmerge_transform(
     const UpLengths& up_lengths,

composable_kernel/include/tensor_description/dynamic_tensor_descriptor.hpp

@@ -308,6 +308,19 @@ transform_dynamic_tensor_descriptor(const OldTensorDescriptor& old_tensor_desc,
                                     NewLowerDimensionOldVisibleIdss,
                                     NewUpperDimensionNewVisibleIdss)
 {
+    // sanity check
+    {
+        constexpr auto all_old_top_ids = unpack([](auto... xs) { return merge_sequences(xs...); },
+                                                NewLowerDimensionOldVisibleIdss{});
+
+        constexpr auto all_new_top_ids = unpack([](auto... xs) { return merge_sequences(xs...); },
+                                                NewUpperDimensionNewVisibleIdss{});
+
+        static_assert(is_valid_sequence_map<decltype(all_old_top_ids)>::value &&
+                          is_valid_sequence_map<decltype(all_old_top_ids)>::value,
+                      "wrong!");
+    }
+
     // lower dimension's hidden idss
     // convert lower dimension visible idss (tuple of sequences) to hidden idss (tuple of
     // sequences)

composable_kernel/include/tensor_description/dynamic_tensor_descriptor_helper.hpp

@@ -115,26 +115,30 @@ template <typename... Lengths, typename Align>
 __host__ __device__ constexpr auto
 make_dynamic_naive_tensor_descriptor_aligned_v2(const Tuple<Lengths...>& lengths, Align align)
 {
+    constexpr auto I1 = Number<1>{};
+
     constexpr index_t N = sizeof...(Lengths);
 
+    const auto stride_n_minus_2 = math::integer_least_multiple(lengths[Number<N - 1>{}], align);
+
     auto strides = generate_tuple(
         [&](auto i) {
             if constexpr(i.value == N - 1)
             {
-                return Number<1>{};
+                return I1;
             }
             else if constexpr(i.value == N - 2)
             {
-                return math::lcm(lengths[Number<N - 1>{}], align);
+                return Number<stride_n_minus_2>{};
             }
             else
             {
                 return container_reduce(lengths,
                                         math::multiplies_v2{},
-                                        math::lcm(lengths[Number<N - 1>{}], align),
-                                        i,
+                                        Number<stride_n_minus_2>{},
+                                        i + I1,
                                         Number<N - 2>{},
-                                        Number<1>{});
+                                        I1);
             }
         },
         Number<N>{});

composable_kernel/include/tensor_operation/blockwise_dynamic_tensor_slice_transfer.hpp

@@ -31,8 +31,8 @@ template <index_t BlockSize,
           index_t DstScalarPerVector,
           index_t SrcScalarStrideInVector,
           index_t DstScalarStrideInVector,
-          index_t ThreadTransferSrcResetCoordinateAfterRun,
-          index_t ThreadTransferDstResetCoordinateAfterRun>
+          bool ThreadTransferSrcResetCoordinateAfterRun,
+          bool ThreadTransferDstResetCoordinateAfterRun>
 struct BlockwiseDynamicTensorSliceTransfer_v4
 {
     static constexpr index_t nDim = remove_reference_t<SrcDesc>::GetNumOfDimension();

composable_kernel/include/tensor_operation/blockwise_gemm_v2.hpp

@@ -29,10 +29,10 @@ template <index_t BlockSize,
           index_t M1PerThread,
           index_t N1PerThread,
           index_t KPerThread,
-          index_t MLevel0ThreadCluster,
-          index_t NLevel0ThreadCluster,
-          index_t MLevel1ThreadCluster,
-          index_t NLevel1ThreadCluster,
+          index_t M1N1ThreadClusterM10,
+          index_t M1N1ThreadClusterN10,
+          index_t M1N1ThreadClusterM11,
+          index_t M1N1ThreadClusterN11,
           index_t AThreadCopyScalarPerVector_M1,
           index_t BThreadCopyScalarPerVector_N1,
           typename std::enable_if<ABlockDesc::IsKnownAtCompileTime() &&
@@ -62,8 +62,8 @@ struct BlockwiseGemm_km0m1_kn0n1_m0m1n0n1_v1
                           CThreadDesc::IsKnownAtCompileTime(),
                       "wrong! Desc should be known at compile-time");
 
-        static_assert(BlockSize == MLevel0ThreadCluster * MLevel1ThreadCluster *
-                                       NLevel0ThreadCluster * NLevel1ThreadCluster,
+        static_assert(BlockSize == M1N1ThreadClusterM11 * M1N1ThreadClusterM10 *
+                                       M1N1ThreadClusterN11 * M1N1ThreadClusterN10,
                       "wrong! blocksize and cluster size not consistent");
 
         static_assert(ABlockDesc{}.GetLength(I0) == BBlockDesc{}.GetLength(I0),
@@ -78,6 +78,8 @@ struct BlockwiseGemm_km0m1_kn0n1_m0m1n0n1_v1
         constexpr index_t N1 = BBlockDesc{}.GetLength(I2);
 
         // 4-d data space into 4-d thread space
+        // upper: {1, M1N1ThreadClusterM10 * M1N1ThreadClusterM11, 1, M1N1ThreadClusterN10 *
+        // M1N1ThreadClusterN11} lower: {M0, M1, N0, N1}
         constexpr auto adaptor0 = make_single_stage_tensor_adaptor(
             make_tuple(make_vectorize_transform(M0, 1),
                        make_vectorize_transform(M1PerThread, M1 / M1PerThread),
@@ -87,21 +89,27 @@ struct BlockwiseGemm_km0m1_kn0n1_m0m1n0n1_v1
             make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}));
 
         // thread position 4-d thread space
+        // upper: {M1N1ThreadClusterM10, M1N1ThreadClusterM11, M1N1ThreadClusterN10,
+        // M1N1ThreadClusterN11} lower: {1, M1N1ThreadClusterM10 * M1N1ThreadClusterM11, 1,
+        // M1N1ThreadClusterN10 * M1N1ThreadClusterN11}
         constexpr auto adaptor1 = make_single_stage_tensor_adaptor(
             make_tuple(
                 make_freeze_transform(make_multi_index(0)),
-                make_unmerge_transform(make_tuple(MLevel1ThreadCluster, MLevel0ThreadCluster)),
+                make_unmerge_transform(make_tuple(M1N1ThreadClusterM10, M1N1ThreadClusterM11)),
                 make_freeze_transform(make_multi_index(0)),
-                make_unmerge_transform(make_tuple(NLevel1ThreadCluster, NLevel0ThreadCluster))),
+                make_unmerge_transform(make_tuple(M1N1ThreadClusterN10, M1N1ThreadClusterN11))),
             make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}),
             make_tuple(Sequence<>{}, Sequence<0, 1>{}, Sequence<>{}, Sequence<2, 3>{}));
 
         // 4-d thread space to 1-d thread space
+        // upper: {BlockSize}
+        // lower: {M1N1ThreadClusterM10, M1N1ThreadClusterM11, M1N1ThreadClusterN10,
+        // M1N1ThreadClusterN11}
         constexpr auto adaptor2 = make_single_stage_tensor_adaptor(
-            make_tuple(make_merge_transform(make_tuple(MLevel1ThreadCluster,
-                                                       NLevel1ThreadCluster,
-                                                       MLevel0ThreadCluster,
-                                                       NLevel0ThreadCluster))),
+            make_tuple(make_merge_transform(make_tuple(M1N1ThreadClusterM10,
+                                                       M1N1ThreadClusterN10,
+                                                       M1N1ThreadClusterM11,
+                                                       M1N1ThreadClusterN11))),
             make_tuple(Sequence<0, 2, 1, 3>{}),
             make_tuple(Sequence<0>{}));
 
@@ -221,10 +229,10 @@ template <index_t BlockSize,
           index_t M1PerThread,
           index_t N1PerThread,
           index_t KPerThread,
-          index_t MLevel0ThreadCluster,
-          index_t NLevel0ThreadCluster,
-          index_t MLevel1ThreadCluster,
-          index_t NLevel1ThreadCluster,
+          index_t M1N1ThreadClusterM10,
+          index_t M1N1ThreadClusterN10,
+          index_t M1N1ThreadClusterM11,
+          index_t M1N1ThreadClusterN11,
           index_t AThreadCopyScalarPerVector_M1,
           index_t BThreadCopyScalarPerVector_N1,
           typename std::enable_if<ABlockDesc::IsKnownAtCompileTime() &&
@@ -254,8 +262,8 @@ struct BlockwiseGemm_km0m1_kn0n1_m0m1n0n1_v2_pipeline_2x2
                           CThreadDesc::IsKnownAtCompileTime(),
                       "wrong! Desc should be known at compile-time");
 
-        static_assert(BlockSize == MLevel0ThreadCluster * MLevel1ThreadCluster *
-                                       NLevel0ThreadCluster * NLevel1ThreadCluster,
+        static_assert(BlockSize == M1N1ThreadClusterM11 * M1N1ThreadClusterM10 *
+                                       M1N1ThreadClusterN11 * M1N1ThreadClusterN10,
                       "wrong! blocksize and cluster size not consistent");
 
         static_assert(ABlockDesc{}.GetLength(I0) == BBlockDesc{}.GetLength(I0),
@@ -287,18 +295,18 @@ struct BlockwiseGemm_km0m1_kn0n1_m0m1n0n1_v2_pipeline_2x2
         constexpr auto adaptor1 = make_single_stage_tensor_adaptor(
             make_tuple(
                 make_freeze_transform(make_multi_index(0)),
-                make_unmerge_transform(make_tuple(MLevel1ThreadCluster, MLevel0ThreadCluster)),
+                make_unmerge_transform(make_tuple(M1N1ThreadClusterM10, M1N1ThreadClusterM11)),
                 make_freeze_transform(make_multi_index(0)),
-                make_unmerge_transform(make_tuple(NLevel1ThreadCluster, NLevel0ThreadCluster))),
+                make_unmerge_transform(make_tuple(M1N1ThreadClusterN10, M1N1ThreadClusterN11))),
             make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}),
             make_tuple(Sequence<>{}, Sequence<0, 1>{}, Sequence<>{}, Sequence<2, 3>{}));
 
         // 4-d thread space to 1-d thread space
         constexpr auto adaptor2 = make_single_stage_tensor_adaptor(
-            make_tuple(make_merge_transform(make_tuple(MLevel1ThreadCluster,
-                                                       NLevel1ThreadCluster,
-                                                       MLevel0ThreadCluster,
-                                                       NLevel0ThreadCluster))),
+            make_tuple(make_merge_transform(make_tuple(M1N1ThreadClusterM10,
+                                                       M1N1ThreadClusterN10,
+                                                       M1N1ThreadClusterM11,
+                                                       M1N1ThreadClusterN11))),
             make_tuple(Sequence<0, 2, 1, 3>{}),
             make_tuple(Sequence<0>{}));
 

composable_kernel/include/tensor_operation/blockwise_gemm_v2r2.hpp

+#ifndef CK_BLOCKWISE_GEMM_V2R2_HPP
+#define CK_BLOCKWISE_GEMM_V2R2_HPP
+
+#include "common_header.hpp"
+#include "tensor_adaptor.hpp"
+#include "threadwise_dynamic_tensor_slice_transfer.hpp"
+#include "threadwise_gemm_v2.hpp"
+
+namespace ck {
+
+// C[M0, M1, N0, N1] += transpose(A[K, M0, M1]) * B[K, N0, N1]
+// A and B are visable to the whole block, C is distributed among each thread
+// Assume:
+//   1. A:
+//     1. AKMBlockDesc is known at compile-time
+//     2. ABlockBuffer is DynamicBuffer
+//   2. B:
+//     1. BKNBlockDesc is known at compile-time
+//     2. BBlockBuffer is DynamicBuffer
+//   3. C:
+//     1. CM0M1N0N1ThreadDesc is known at compile-time
+//     2. CThreadBuffer is StaticBuffer
+// Also assume:
+//   M0 = N0 = 2. It will do 2x2 pipelined read and fma (ABBA optimization)
+template <index_t BlockSize,
+          typename FloatA,
+          typename FloatB,
+          typename FloatC,
+          typename AKMBlockDesc,
+          typename BKNBlockDesc,
+          index_t M1PerThreadM11,
+          index_t N1PerThreadN11,
+          index_t KPerThread,
+          index_t M1N1ThreadClusterM100,
+          index_t M1N1ThreadClusterN100,
+          index_t M1N1ThreadClusterM101,
+          index_t M1N1ThreadClusterN101,
+          index_t AThreadCopyScalarPerVector_M11,
+          index_t BThreadCopyScalarPerVector_N11,
+          typename std::enable_if<AKMBlockDesc::IsKnownAtCompileTime() &&
+                                      BKNBlockDesc::IsKnownAtCompileTime(),
+                                  bool>::type = false>
+struct BlockwiseGemm_km_kn_m0m1n0n1_v2r2_pipeline_2x2
+{
+    using AIndex = MultiIndex<3>;
+    using BIndex = MultiIndex<3>;
+    using CIndex = MultiIndex<4>;
+
+    static constexpr auto I0 = Number<0>{};
+    static constexpr auto I1 = Number<1>{};
+    static constexpr auto I2 = Number<2>{};
+    static constexpr auto I3 = Number<3>{};
+
+    static constexpr index_t K = AKMBlockDesc{}.GetLength(I0);
+    static constexpr index_t M = AKMBlockDesc{}.GetLength(I1);
+    static constexpr index_t N = BKNBlockDesc{}.GetLength(I1);
+
+    static constexpr index_t M100 = M1N1ThreadClusterM100;
+    static constexpr index_t N100 = M1N1ThreadClusterN100;
+
+    static constexpr index_t M101 = M1N1ThreadClusterM101;
+    static constexpr index_t N101 = M1N1ThreadClusterN101;
+
+    static constexpr index_t M11 = M1PerThreadM11;
+    static constexpr index_t N11 = N1PerThreadN11;
+
+    static constexpr index_t M1 = M1N1ThreadClusterM100 * M1N1ThreadClusterM101 * M1PerThreadM11;
+    static constexpr index_t N1 = M1N1ThreadClusterN100 * M1N1ThreadClusterN101 * N1PerThreadN11;
+
+    static constexpr index_t M0 = M / M1;
+    static constexpr index_t N0 = N / N1;
+
+    __host__ __device__ static constexpr auto
+    MakeAKM0M1BlockDescriptor(const AKMBlockDesc& a_k_m_block_desc)
+    {
+        const auto a_k_m0_m1_block_desc = transform_dynamic_tensor_descriptor(
+            AKMBlockDesc{},
+            make_tuple(make_pass_through_transform(Number<K>{}),
+                       make_unmerge_transform(make_tuple(Number<M0>{}, Number<M1>{}))),
+            make_tuple(Sequence<0>{}, Sequence<1>{}),
+            make_tuple(Sequence<0>{}, Sequence<1, 2>{}));
+
+        return a_k_m0_m1_block_desc;
+    }
+
+    __host__ __device__ static constexpr auto
+    MakeBKN0N1BlockDescriptor(const BKNBlockDesc& b_k_n_block_desc)
+    {
+        const auto b_k_n0_n1_block_desc = transform_dynamic_tensor_descriptor(
+            BKNBlockDesc{},
+            make_tuple(make_pass_through_transform(Number<K>{}),
+                       make_unmerge_transform(make_tuple(Number<N0>{}, Number<N1>{}))),
+            make_tuple(Sequence<0>{}, Sequence<1>{}),
+            make_tuple(Sequence<0>{}, Sequence<1, 2>{}));
+
+        return b_k_n0_n1_block_desc;
+    }
+
+    __host__ __device__ static constexpr auto MakeCM0M100M101M11N0N100N101N11ToMNBlockAdaptor()
+    {
+        // upper: [M0, M100, M101, M11, N0, N100, N101, N11]
+        // lower: [M, N]
+        constexpr auto c_m0_m100_m101_m11_n0_n100_n101_n11_to_m_n_block_adaptor =
+            make_single_stage_tensor_adaptor(
+                make_tuple(make_unmerge_transform(make_tuple(
+                               Number<M0>{}, Number<M100>{}, Number<M101>{}, Number<M11>{})),
+                           make_unmerge_transform(make_tuple(
+                               Number<N0>{}, Number<N100>{}, Number<N101>{}, Number<N11>{}))),
+                make_tuple(Sequence<0>{}, Sequence<1>{}),
+                make_tuple(Sequence<0, 1, 2, 3>{}, Sequence<4, 5, 6, 7>{}));
+
+        return c_m0_m100_m101_m11_n0_n100_n101_n11_to_m_n_block_adaptor;
+    }
+
+    __host__ __device__ static constexpr auto
+    MakeCM0M100M101M11N0N100N101N11ToM0M1N0N1BlockAdaptor()
+    {
+        // upper: [M0, M100, M101, M11, N0, N100, N101, N11]
+        // lower: [M0, M1, N0, N1]
+        constexpr auto c_m0_m100_m101_m11_n0_n100_n101_n11_to_m0_m1_n0_n1_block_adaptor =
+            make_single_stage_tensor_adaptor(
+                make_tuple(make_pass_through_transform(Number<M0>{}),
+                           make_unmerge_transform(
+                               make_tuple(Number<M100>{}, Number<M101>{}, Number<M11>{})),
+                           make_pass_through_transform(Number<N0>{}),
+                           make_unmerge_transform(
+                               make_tuple(Number<N100>{}, Number<N101>{}, Number<N11>{}))),
+                make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}),
+                make_tuple(Sequence<0>{}, Sequence<1, 2, 3>{}, Sequence<4>{}, Sequence<5, 6, 7>{}));
+
+        return c_m0_m100_m101_m11_n0_n100_n101_n11_to_m0_m1_n0_n1_block_adaptor;
+    }
+
+    __host__ __device__ static constexpr auto GetCM0M1N0N1ThreadTensorLengths()
+    {
+        return Sequence<M0, M11, N0, N11>{};
+    }
+
+    static constexpr auto a_k_m0_m1_block_desc_ = MakeAKM0M1BlockDescriptor(AKMBlockDesc{});
+    static constexpr auto b_k_n0_n1_block_desc_ = MakeBKN0N1BlockDescriptor(BKNBlockDesc{});
+
+    public:
+    __device__ BlockwiseGemm_km_kn_m0m1n0n1_v2r2_pipeline_2x2()
+        : c_thread_origin_data_idx_{CalculateCM0M1N0N1ThreadOriginOnBlock(
+              get_thread_local_1d_id())},
+          a_thread_copy_{
+              make_tuple(0, c_thread_origin_data_idx_[I0], c_thread_origin_data_idx_[I1])},
+          b_thread_copy_{
+              make_tuple(0, c_thread_origin_data_idx_[I2], c_thread_origin_data_idx_[I3])}
+    {
+        static_assert(AKMBlockDesc::IsKnownAtCompileTime() && BKNBlockDesc::IsKnownAtCompileTime(),
+                      "wrong! Desc should be known at compile-time");
+
+        static_assert(BlockSize == M101 * M100 * N101 * N100,
+                      "wrong! blocksize and cluster size not consistent");
+
+        static_assert(M % M1 == 0 && N % N1 == 0, "wrong!");
+
+        static_assert(AKMBlockDesc{}.GetLength(I0) == BKNBlockDesc{}.GetLength(I0),
+                      "wrong! K dimension not consistent");
+
+        // TODO: remove this restriction
+        static_assert(M0 == 2 && N0 == 2, "wrong");
+    }
+
+    __device__ static CIndex CalculateCM0M1N0N1ThreadOriginOnBlock(index_t thread_id)
+    {
+        // lower: [M0, M1, N0, N1]
+        // upper: [M0, M100, M101, M11, N0, N100, N101, N11]
+        constexpr auto adaptor0 = MakeCM0M100M101M11N0N100N101N11ToM0M1N0N1BlockAdaptor();
+
+        // lower: [M0, M100, M101, M11, N0, N100, N101, N11]
+        // upper: [Tid, M0, M11, N0, N11]
+        constexpr auto adaptor1 = make_single_stage_tensor_adaptor(
+            make_tuple(make_merge_transform(make_tuple(M100, N100, M101, N101)),
+                       make_pass_through_transform(M0),
+                       make_pass_through_transform(M11),
+                       make_pass_through_transform(N0),
+                       make_pass_through_transform(N11)),
+            make_tuple(
+                Sequence<1, 5, 2, 6>{}, Sequence<0>{}, Sequence<3>{}, Sequence<4>{}, Sequence<7>{}),
+            make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}, Sequence<4>{}));
+
+        constexpr auto adaptor = chain_tensor_adaptors(adaptor0, adaptor1);
+
+        return adaptor.CalculateBottomIndex(make_multi_index(get_thread_local_1d_id(), 0, 0, 0, 0));
+    }
+
+    __host__ __device__ static constexpr index_t GetABlockAlignment() { return M1PerThreadM11; }
+
+    __host__ __device__ static constexpr auto GetBBlockAlignment() { return N1PerThreadN11; }
+
+    template <typename CM0M1N0N1ThreadDesc,
+              typename ABlockBuffer,
+              typename BBlockBuffer,
+              typename CThreadBuffer>
+    __device__ void Run(const CM0M1N0N1ThreadDesc& c_m0_m1_n0_n1_thread_desc,
+                        const ABlockBuffer& a_block_buf,
+                        const BBlockBuffer& b_block_buf,
+                        CThreadBuffer& c_thread_buf) const
+    {
+        static_assert(CM0M1N0N1ThreadDesc::IsKnownAtCompileTime(),
+                      "wrong! Desc should be known at compile-time");
+
+        // TODO: remove this restriction
+        static_assert(M0 == 2 && N0 == 2 && CM0M1N0N1ThreadDesc{}.GetLength(I0) == M0 &&
+                          CM0M1N0N1ThreadDesc{}.GetLength(I2) == N0,
+                      "wrong");
+
+        auto a_thread_buf = make_static_buffer<AddressSpace::Vgpr, FloatA>(
+            a_k_m0_m1_thread_desc_.GetElementSpaceSize());
+        auto b_thread_buf = make_static_buffer<AddressSpace::Vgpr, FloatB>(
+            b_k_n0_n1_thread_desc_.GetElementSpaceSize());
+
+        constexpr auto threadwise_gemm =
+            ThreadwiseGemm_km0m1_kn0n1_m0m1n0n1<FloatA,
+                                                FloatB,
+                                                FloatC,
+                                                decltype(a_k_m0_m1_thread_desc_),
+                                                decltype(b_k_n0_n1_thread_desc_),
+                                                CM0M1N0N1ThreadDesc,
+                                                Sequence<KPerThread>,
+                                                Sequence<1, M1PerThreadM11>,
+                                                Sequence<1, N1PerThreadN11>>{};
+
+        // read A_sub_0
+        a_thread_copy_.Run(a_k_m0_m1_block_desc_,
+                           make_tuple(I0, I0, I0),
+                           a_block_buf,
+                           a_k_m0_m1_thread_desc_,
+                           make_tuple(I0, I0, I0),
+                           a_thread_buf);
+
+        // read B_sub_0
+        b_thread_copy_.Run(b_k_n0_n1_block_desc_,
+                           make_tuple(I0, I0, I0),
+                           b_block_buf,
+                           b_k_n0_n1_thread_desc_,
+                           make_tuple(I0, I0, I0),
+                           b_thread_buf);
+
+        // read B_sub_1
+        b_thread_copy_.Run(b_k_n0_n1_block_desc_,
+                           make_tuple(I0, I1, I0),
+                           b_block_buf,
+                           b_k_n0_n1_thread_desc_,
+                           make_tuple(I0, I1, I0),
+                           b_thread_buf);
+
+        // read A_sub_1
+        a_thread_copy_.Run(a_k_m0_m1_block_desc_,
+                           make_tuple(I0, I1, I0),
+                           a_block_buf,
+                           a_k_m0_m1_thread_desc_,
+                           make_tuple(I0, I1, I0),
+                           a_thread_buf);
+
+        // C_sub_00 += transpose(A_sub_0) * B_sub_0
+        threadwise_gemm.Run(a_thread_buf,
+                            make_tuple(I0, I0, I0),
+                            b_thread_buf,
+                            make_tuple(I0, I0, I0),
+                            c_thread_buf,
+                            make_tuple(I0, I0, I0, I0));
+
+        // C_sub_01 += transpose(A_sub_0) * B_sub_1
+        threadwise_gemm.Run(a_thread_buf,
+                            make_tuple(I0, I0, I0),
+                            b_thread_buf,
+                            make_tuple(I0, I1, I0),
+                            c_thread_buf,
+                            make_tuple(I0, I0, I1, I0));
+
+        // loop over rest of k
+        static_for<KPerThread, K, KPerThread>{}([&](auto k) {
+            // read A_sub_0
+            a_thread_copy_.Run(a_k_m0_m1_block_desc_,
+                               make_tuple(k, I0, I0),
+                               a_block_buf,
+                               a_k_m0_m1_thread_desc_,
+                               make_tuple(I0, I0, I0),
+                               a_thread_buf);
+
+            // C_sub_10 += transpose(A_sub_1) * B_sub_0
+            threadwise_gemm.Run(a_thread_buf,
+                                make_tuple(I0, I1, I0),
+                                b_thread_buf,
+                                make_tuple(I0, I0, I0),
+                                c_thread_buf,
+                                make_tuple(I1, I0, I0, I0));
+
+            // read B_sub_0
+            b_thread_copy_.Run(b_k_n0_n1_block_desc_,
+                               make_tuple(k, I0, I0),
+                               b_block_buf,
+                               b_k_n0_n1_thread_desc_,
+                               make_tuple(I0, I0, I0),
+                               b_thread_buf);
+
+            // C_sub_11 += transpose(A_sub_1) * B_sub_1
+            threadwise_gemm.Run(a_thread_buf,
+                                make_tuple(I0, I1, I0),
+                                b_thread_buf,
+                                make_tuple(I0, I1, I0),
+                                c_thread_buf,
+                                make_tuple(I1, I0, I1, I0));
+
+            // read B_sub_1
+            b_thread_copy_.Run(b_k_n0_n1_block_desc_,
+                               make_tuple(k, I1, I0),
+                               b_block_buf,
+                               b_k_n0_n1_thread_desc_,
+                               make_tuple(I0, I1, I0),
+                               b_thread_buf);
+
+            // read A_sub_1
+            a_thread_copy_.Run(a_k_m0_m1_block_desc_,
+                               make_tuple(k, I1, I0),
+                               a_block_buf,
+                               a_k_m0_m1_thread_desc_,
+                               make_tuple(I0, I1, I0),
+                               a_thread_buf);
+
+            // C_sub_00 += transpose(A_sub_0) * B_sub_0
+            threadwise_gemm.Run(a_thread_buf,
+                                make_tuple(I0, I0, I0),
+                                b_thread_buf,
+                                make_tuple(I0, I0, I0),
+                                c_thread_buf,
+                                make_tuple(I0, I0, I0, I0));
+
+            // C_sub_01 += transpose(A_sub_0) * B_sub_1
+            threadwise_gemm.Run(a_thread_buf,
+                                make_tuple(I0, I0, I0),
+                                b_thread_buf,
+                                make_tuple(I0, I1, I0),
+                                c_thread_buf,
+                                make_tuple(I0, I0, I1, I0));
+        });
+
+        // C_sub_10 += transpose(A_sub_1) * B_sub_0
+        threadwise_gemm.Run(a_thread_buf,
+                            make_tuple(I0, I1, I0),
+                            b_thread_buf,
+                            make_tuple(I0, I0, I0),
+                            c_thread_buf,
+                            make_tuple(I1, I0, I0, I0));
+
+        // C_sub_11 += transpose(A_sub_1) * B_sub_1
+        threadwise_gemm.Run(a_thread_buf,
+                            make_tuple(I0, I1, I0),
+                            b_thread_buf,
+                            make_tuple(I0, I1, I0),
+                            c_thread_buf,
+                            make_tuple(I1, I0, I1, I0));
+    }
+
+    private:
+    // A[K, M0, M1]
+    static constexpr auto a_k_m0_m1_thread_desc_ = make_dynamic_naive_tensor_descriptor_packed_v2(
+        make_tuple(Number<KPerThread>{}, Number<M0>{}, Number<M1PerThreadM11>{}));
+
+    // B[K, N0, N1]
+    static constexpr auto b_k_n0_n1_thread_desc_ = make_dynamic_naive_tensor_descriptor_packed_v2(
+        make_tuple(Number<KPerThread>{}, Number<N0>{}, Number<N1PerThreadN11>{}));
+
+    using AThreadCopy =
+        ThreadwiseDynamicTensorSliceTransfer_v4<FloatA,
+                                                FloatA,
+                                                decltype(a_k_m0_m1_block_desc_),
+                                                decltype(a_k_m0_m1_thread_desc_),
+                                                Sequence<KPerThread, 1, M1PerThreadM11>,
+                                                Sequence<0, 1, 2>,
+                                                2,
+                                                AThreadCopyScalarPerVector_M11,
+                                                1>;
+
+    using BThreadCopy =
+        ThreadwiseDynamicTensorSliceTransfer_v4<FloatB,
+                                                FloatB,
+                                                decltype(b_k_n0_n1_block_desc_),
+                                                decltype(b_k_n0_n1_thread_desc_),
+                                                Sequence<KPerThread, 1, N1PerThreadN11>,
+                                                Sequence<0, 1, 2>,
+                                                2,
+                                                BThreadCopyScalarPerVector_N11,
+                                                1>;
+
+    CIndex c_thread_origin_data_idx_;
+
+    AThreadCopy a_thread_copy_;
+    BThreadCopy b_thread_copy_;
+};
+
+} // namespace ck
+#endif

composable_kernel/include/tensor_operation/gridwise_dynamic_gemm.hpp → composable_kernel/include/tensor_operation/gridwise_dynamic_gemm_v1r1.hpp

@@ -27,13 +27,13 @@ __global__ void
 #if CK_USE_LAUNCH_BOUNDS
     __launch_bounds__(CK_MAX_THREAD_PER_BLOCK, CK_MIN_BLOCK_PER_CU)
 #endif
-        kernel_dynamic_gemm_v1(const FloatA* __restrict__ p_a_global,
-                               const FloatB* __restrict__ p_b_global,
-                               FloatC* __restrict__ p_c_global,
-                               const AGlobalDesc a_k_m_global_desc,
-                               const BGlobalDesc b_k_n_global_desc,
-                               const CGlobalDesc c_m0_m1_n0_n1_global_desc,
-                               const CBlockClusterDesc c_block_cluster_desc)
+        kernel_dynamic_gemm_v1r1(const FloatA* __restrict__ p_a_global,
+                                 const FloatB* __restrict__ p_b_global,
+                                 FloatC* __restrict__ p_c_global,
+                                 const AGlobalDesc a_k_m_global_desc,
+                                 const BGlobalDesc b_k_n_global_desc,
+                                 const CGlobalDesc c_m0_m1_n0_n1_global_desc,
+                                 const CBlockClusterDesc c_block_cluster_desc)
 {
     GridwiseGemm::Run(p_a_global,
                       p_b_global,
@@ -63,13 +63,13 @@ __global__ void
 #if CK_USE_LAUNCH_BOUNDS
     __launch_bounds__(CK_MAX_THREAD_PER_BLOCK, CK_MIN_BLOCK_PER_CU)
 #endif
-        kernel_dynamic_gemm_v1(const FloatA* __restrict__ p_a_global,
-                               const FloatB* __restrict__ p_b_global,
-                               FloatC* __restrict__ p_c_global,
-                               const void __CONSTANT__* p_a_k_m_global_desc,
-                               const void __CONSTANT__* p_b_k_n_global_desc,
-                               const void __CONSTANT__* p_c_m0_m1_n0_n1_global_desc,
-                               const void __CONSTANT__* p_c_block_cluster_desc)
+        kernel_dynamic_gemm_v1r1(const FloatA* __restrict__ p_a_global,
+                                 const FloatB* __restrict__ p_b_global,
+                                 FloatC* __restrict__ p_c_global,
+                                 const void __CONSTANT__* p_a_k_m_global_desc,
+                                 const void __CONSTANT__* p_b_k_n_global_desc,
+                                 const void __CONSTANT__* p_c_m0_m1_n0_n1_global_desc,
+                                 const void __CONSTANT__* p_c_block_cluster_desc)
 {
     // first cast void __CONSTANT__ void* to void*
     // second cast void* to Desc*
@@ -108,13 +108,13 @@ template <index_t BlockSize,
           index_t MPerBlock,
           index_t NPerBlock,
           index_t KPerBlock,
-          index_t MPerThread,
-          index_t NPerThread,
+          index_t M1PerThread,
+          index_t N1PerThread,
           index_t KPerThread,
-          index_t MLevel0Cluster,
-          index_t NLevel0Cluster,
-          index_t MLevel1Cluster,
-          index_t NLevel1Cluster,
+          index_t M1N1ThreadClusterM10,
+          index_t M1N1ThreadClusterN10,
+          index_t M1N1ThreadClusterM11,
+          index_t M1N1ThreadClusterN11,
           typename ABlockTransferThreadSliceLengths_K_M,
           typename ABlockTransferThreadClusterLengths_K_M,
           typename ABlockTransferThreadClusterArrangeOrder,
@@ -139,14 +139,14 @@ template <index_t BlockSize,
           typename CGlobalIteratorHacks,
           typename AGlobalMoveSliceWindowIteratorHacks,
           typename BGlobalMoveSliceWindowIteratorHacks>
-struct GridwiseDynamicGemm_km_kn_m0m1n0n1_v1
+struct GridwiseDynamicGemm_km_kn_m0m1n0n1_v1r1
 {
     __host__ __device__ static constexpr index_t GetSharedMemoryNumberOfByte()
     {
         constexpr auto max_lds_align = math::lcm(Number<ABlockTransferDstScalarPerVector_M>{},
                                                  Number<BBlockTransferDstScalarPerVector_N>{},
-                                                 Number<MPerThread>{},
-                                                 Number<NPerThread>{});
+                                                 Number<M1PerThread>{},
+                                                 Number<N1PerThread>{});
 
         // A matrix in LDS memory, dst of blockwise copy
         //   be careful of LDS alignment
@@ -210,8 +210,8 @@ struct GridwiseDynamicGemm_km_kn_m0m1n0n1_v1
         // lds max alignment
         constexpr auto max_lds_align = math::lcm(Number<ABlockTransferDstScalarPerVector_M>{},
                                                  Number<BBlockTransferDstScalarPerVector_N>{},
-                                                 Number<MPerThread>{},
-                                                 Number<NPerThread>{});
+                                                 Number<M1PerThread>{},
+                                                 Number<N1PerThread>{});
 
         // A matrix in LDS memory, dst of blockwise copy
         //   be careful of LDS alignment
@@ -284,34 +284,39 @@ struct GridwiseDynamicGemm_km_kn_m0m1n0n1_v1
         //     c_mtx[MPerBlock, NPerBlock] is distributed among threads, and saved in
         //       register
         // sanity check
-        static_assert(MPerBlock % (MPerThread * MLevel0Cluster * MLevel1Cluster) == 0 &&
-                          NPerBlock % (NPerThread * NLevel0Cluster * NLevel1Cluster) == 0,
-                      "wrong!");
+        static_assert(
+            MPerBlock % (M1PerThread * M1N1ThreadClusterM11 * M1N1ThreadClusterM10) == 0 &&
+                NPerBlock % (N1PerThread * M1N1ThreadClusterN11 * M1N1ThreadClusterN10) == 0,
+            "wrong!");
 
-        constexpr index_t MRepeat = MPerBlock / (MPerThread * MLevel0Cluster * MLevel1Cluster);
-        constexpr index_t NRepeat = NPerBlock / (NPerThread * NLevel0Cluster * NLevel1Cluster);
+        constexpr index_t M0PerThread =
+            MPerBlock / (M1PerThread * M1N1ThreadClusterM11 * M1N1ThreadClusterM10);
+        constexpr index_t N0PerThread =
+            NPerBlock / (N1PerThread * M1N1ThreadClusterN11 * M1N1ThreadClusterN10);
 
         constexpr auto a_k_m0_m1_block_desc = transform_dynamic_tensor_descriptor(
             a_k_m_block_desc,
-            make_tuple(
-                make_pass_through_transform(Number<KPerBlock>{}),
-                make_unmerge_transform(make_tuple(
-                    Number<MRepeat>{}, Number<MPerThread * MLevel0Cluster * MLevel1Cluster>{}))),
+            make_tuple(make_pass_through_transform(Number<KPerBlock>{}),
+                       make_unmerge_transform(make_tuple(
+                           Number<M0PerThread>{},
+                           Number<M1PerThread * M1N1ThreadClusterM11 * M1N1ThreadClusterM10>{}))),
             make_tuple(Sequence<0>{}, Sequence<1>{}),
             make_tuple(Sequence<0>{}, Sequence<1, 2>{}));
 
         constexpr auto b_k_n0_n1_block_desc = transform_dynamic_tensor_descriptor(
             b_k_n_block_desc,
-            make_tuple(
-                make_pass_through_transform(Number<KPerBlock>{}),
-                make_unmerge_transform(make_tuple(
-                    Number<NRepeat>{}, Number<NPerThread * NLevel0Cluster * NLevel1Cluster>{}))),
+            make_tuple(make_pass_through_transform(Number<KPerBlock>{}),
+                       make_unmerge_transform(make_tuple(
+                           Number<N0PerThread>{},
+                           Number<N1PerThread * M1N1ThreadClusterN11 * M1N1ThreadClusterN10>{}))),
             make_tuple(Sequence<0>{}, Sequence<1>{}),
             make_tuple(Sequence<0>{}, Sequence<1, 2>{}));
 
         constexpr auto c_m0_m1_n0_n1_thread_desc =
-            make_dynamic_naive_tensor_descriptor_packed_v2(make_tuple(
-                Number<MRepeat>{}, Number<MPerThread>{}, Number<NRepeat>{}, Number<NPerThread>{}));
+            make_dynamic_naive_tensor_descriptor_packed_v2(make_tuple(Number<M0PerThread>{},
+                                                                      Number<M1PerThread>{},
+                                                                      Number<N0PerThread>{},
+                                                                      Number<N1PerThread>{}));
 
         const auto blockwise_gemm =
             BlockwiseGemm_km0m1_kn0n1_m0m1n0n1_v2_pipeline_2x2<BlockSize,
@@ -321,15 +326,15 @@ struct GridwiseDynamicGemm_km_kn_m0m1n0n1_v1
                                                                decltype(a_k_m0_m1_block_desc),
                                                                decltype(b_k_n0_n1_block_desc),
                                                                decltype(c_m0_m1_n0_n1_thread_desc),
-                                                               MPerThread,
-                                                               NPerThread,
+                                                               M1PerThread,
+                                                               N1PerThread,
                                                                KPerThread,
-                                                               MLevel0Cluster,
-                                                               NLevel0Cluster,
-                                                               MLevel1Cluster,
-                                                               NLevel1Cluster,
-                                                               MPerThread,
-                                                               NPerThread>{};
+                                                               M1N1ThreadClusterM10,
+                                                               M1N1ThreadClusterN10,
+                                                               M1N1ThreadClusterM11,
+                                                               M1N1ThreadClusterN11,
+                                                               M1PerThread,
+                                                               N1PerThread>{};
 
         // LDS allocation for A and B: be careful of alignment
         constexpr auto a_block_space_size =
@@ -345,9 +350,10 @@ struct GridwiseDynamicGemm_km_kn_m0m1n0n1_v1
         auto c_thread_buf = make_static_buffer<AddressSpace::Vgpr, FloatAcc>(
             c_m0_m1_n0_n1_thread_desc.GetElementSpaceSize());
 
-        ThreadwiseDynamicTensorSliceSet_v1<FloatAcc,
-                                           decltype(c_m0_m1_n0_n1_thread_desc),
-                                           Sequence<MRepeat, MPerThread, NRepeat, NPerThread>>{}
+        ThreadwiseDynamicTensorSliceSet_v1<
+            FloatAcc,
+            decltype(c_m0_m1_n0_n1_thread_desc),
+            Sequence<M0PerThread, M1PerThread, N0PerThread, N1PerThread>>{}
             .Run(c_m0_m1_n0_n1_thread_desc, make_tuple(I0, I0, I0, I0), c_thread_buf, FloatAcc{0});
 
         constexpr auto a_block_slice_copy_step = make_multi_index(KPerBlock, 0);
@@ -479,8 +485,8 @@ struct GridwiseDynamicGemm_km_kn_m0m1n0n1_v1
 
         // output: register to global memory
         {
-            constexpr auto M1 = Number<MPerThread * MLevel0Cluster * MLevel1Cluster>{};
-            constexpr auto N1 = Number<NPerThread * NLevel0Cluster * NLevel1Cluster>{};
+            constexpr auto M1 = Number<M1PerThread * M1N1ThreadClusterM10 * M1N1ThreadClusterM11>{};
+            constexpr auto N1 = Number<N1PerThread * M1N1ThreadClusterN10 * M1N1ThreadClusterN11>{};
 
             // hack to control index calculation when iterating over c_m0_m1_n0_n1_global tensor
             constexpr auto c_m0_m1_n0_n1_global_tensor_iterator_hacks = CGlobalIteratorHacks{};
@@ -493,7 +499,7 @@ struct GridwiseDynamicGemm_km_kn_m0m1n0n1_v1
                 FloatC,
                 decltype(c_m0_m1_n0_n1_thread_desc),
                 decltype(c_m0_m1_n0_n1_global_desc),
-                Sequence<MRepeat, MPerThread, NRepeat, NPerThread>,
+                Sequence<M0PerThread, M1PerThread, N0PerThread, N1PerThread>,
                 CThreadTransferSrcDstAccessOrder,
                 CThreadTransferSrcDstVectorDim,
                 CThreadTransferDstScalarPerVector,

composable_kernel/include/utility/common_header.hpp

@@ -6,6 +6,7 @@
 #include "container_helper.hpp"
 #include "statically_indexed_array.hpp"
 #include "container_element_picker.hpp"
+#include "multi_index.hpp"
 #include "data_type.hpp"
 #include "float_type.hpp"
 #include "functional.hpp"

composable_kernel/include/utility/config.amd.hpp.in

@@ -14,7 +14,7 @@
 #define CK_DEVICE_BACKEND_AMD 1
 
 // GPU ID
-#if 0
+#if 1
 #define CK_AMD_GPU_GFX906 1
 #elif 0
 #define CK_AMD_GPU_GFX908 1
@@ -28,7 +28,7 @@
 #endif
 
 // launch bounds
-#define CK_USE_LAUNCH_BOUNDS 1
+#define CK_USE_LAUNCH_BOUNDS 0
 
 #ifdef CK_USE_LAUNCH_BOUNDS
 #define CK_MAX_THREAD_PER_BLOCK 256

composable_kernel/include/utility/magic_division.hpp

@@ -118,6 +118,7 @@ struct MagicDivision
         return (tmp + dividend) >> shift;
     }
 
+#if 1 // debug
     // HACK: magic division for int32_t
     // HACK: use dividend_i32 as if it's uint32_t, dividend_i32 need to be
     // non-negative for result to be correct
@@ -127,8 +128,25 @@ struct MagicDivision
     {
         uint32_t dividend_u32 = as_type<uint32_t>(dividend_i32);
         uint32_t tmp          = ((uint64_t)dividend_u32 * (uint64_t)multiplier) >> 32;
-        return (tmp + dividend_i32) >> shift;
+        return (tmp + dividend_u32) >> shift;
     }
+#else
+    // the inline ASM is producing wrong result
+    __host__ __device__ static int32_t
+    DoMagicDivision(int32_t dividend_i32, uint32_t multiplier, uint32_t shift)
+    {
+        uint32_t r;
+        asm volatile("\n \
+                v_mul_hi_u32 %0, %1, %2 \n \
+                v_add_u32_e32 %0, %1, %0 \n \
+                v_lshrrev_b32_e32 %0, %3, %0 \n \
+                "
+                     : "=v"(r)
+                     : "v"(as_type<uint32_t>(dividend_i32)), "s"(multiplier), "s"(shift));
+
+        return as_type<int32_t>(r);
+    }
+#endif
 };
 
 } // namespace ck