overhauling fwd-v4r4

composable_kernel/include/driver/driver_dynamic_gemm_v1r2.hpp

+#ifndef CK_DRIVER_DYNAMIC_GEMM_V1R2
+#define CK_DRIVER_DYNAMIC_GEMM_V1R2
+
+#include "common_header.hpp"
+#include "dynamic_tensor_descriptor.hpp"
+#include "dynamic_tensor_descriptor_helper.hpp"
+#include "gridwise_dynamic_gemm_v1r2.hpp"
+
+namespace ck {
+
+template <index_t BlockSize,
+          typename FloatAB,
+          typename FloatAcc,
+          typename FloatC,
+          InMemoryDataOperation CGlobalMemoryDataOperation,
+          typename AGlobalDesc,
+          typename BGlobalDesc,
+          typename CGlobalDesc,
+          typename CBlockClusterDesc,
+          index_t MPerBlock,
+          index_t NPerBlock,
+          index_t KPerBlock,
+          index_t M1PerThread,
+          index_t N1PerThread,
+          index_t KPerThread,
+          index_t M1N1ThreadClusterM10,
+          index_t M1N1ThreadClusterN10,
+          index_t M1N1ThreadClusterM11,
+          index_t M1N1ThreadClusterN11,
+          typename ABlockTransferThreadSliceLengths_K_M,
+          typename ABlockTransferThreadClusterLengths_K_M,
+          typename ABlockTransferThreadClusterArrangeOrder,
+          typename ABlockTransferSrcAccessOrder,
+          index_t ABlockTransferSrcVectorDim,
+          index_t ABlockTransferSrcScalarPerVector,
+          index_t ABlockTransferDstScalarPerVector_M,
+          bool AThreadTransferSrcResetCoordinateAfterRun,
+          typename BBlockTransferThreadSliceLengths_K_N,
+          typename BBlockTransferThreadClusterLengths_K_N,
+          typename BBlockTransferThreadClusterArrangeOrder,
+          typename BBlockTransferSrcAccessOrder,
+          index_t BBlockTransferSrcVectorDim,
+          index_t BBlockTransferSrcScalarPerVector,
+          index_t BBlockTransferDstScalarPerVector_N,
+          bool BThreadTransferSrcResetCoordinateAfterRun,
+          typename CThreadTransferSrcDstAccessOrder,
+          index_t CThreadTransferSrcDstVectorDim,
+          index_t CThreadTransferDstScalarPerVector,
+          typename AGlobalIteratorHacks,
+          typename BGlobalIteratorHacks,
+          typename CGlobalIteratorHacks,
+          typename AGlobalMoveSliceWindowIteratorHacks,
+          typename BGlobalMoveSliceWindowIteratorHacks>
+__host__ float launch_kernel_dynamic_gemm_v1r2(const FloatAB* p_a_global,
+                                               const FloatAB* p_b_global,
+                                               FloatC* p_c_global,
+                                               const AGlobalDesc& a_k_m_global_desc,
+                                               const BGlobalDesc& b_k_n_global_desc,
+                                               const CGlobalDesc& c_m0_m1_n0_n1_global_desc,
+                                               const CBlockClusterDesc& c_block_cluster_desc,
+                                               AGlobalIteratorHacks,
+                                               BGlobalIteratorHacks,
+                                               CGlobalIteratorHacks,
+                                               AGlobalMoveSliceWindowIteratorHacks,
+                                               BGlobalMoveSliceWindowIteratorHacks,
+                                               index_t nrepeat)
+
+{
+    constexpr auto I0 = Number<0>{};
+    constexpr auto I1 = Number<1>{};
+    constexpr auto I2 = Number<2>{};
+    constexpr auto I3 = Number<3>{};
+
+    const auto M = a_k_m_global_desc.GetLength(I1);
+    const auto N = b_k_n_global_desc.GetLength(I1);
+    const auto K = a_k_m_global_desc.GetLength(I0);
+
+    if(!(M % MPerBlock == 0 && N % NPerBlock == 0 && K % KPerBlock == 0))
+    {
+        throw std::runtime_error("wrong! GEMM size no divisible");
+    }
+
+    constexpr auto M1 = Number<M1PerThread * M1N1ThreadClusterM11 * M1N1ThreadClusterM10>{};
+    constexpr auto N1 = Number<N1PerThread * M1N1ThreadClusterN11 * M1N1ThreadClusterN10>{};
+
+    if(!(MPerBlock % M1 == 0 && NPerBlock % N1 == 0))
+    {
+        throw std::runtime_error("wrong! GEMM size no divisible");
+    }
+
+    // GEMM
+    using gridwise_gemm =
+        GridwiseDynamicGemm_km_kn_m0m1n0n1_v1r2<BlockSize,
+                                                FloatAB,
+                                                FloatAcc,
+                                                FloatC,
+                                                CGlobalMemoryDataOperation,
+                                                AGlobalDesc,
+                                                BGlobalDesc,
+                                                CGlobalDesc,
+                                                CBlockClusterDesc,
+                                                MPerBlock,
+                                                NPerBlock,
+                                                KPerBlock,
+                                                M1PerThread,
+                                                N1PerThread,
+                                                KPerThread,
+                                                M1N1ThreadClusterM10,
+                                                M1N1ThreadClusterN10,
+                                                M1N1ThreadClusterM11,
+                                                M1N1ThreadClusterN11,
+                                                ABlockTransferThreadSliceLengths_K_M,
+                                                ABlockTransferThreadClusterLengths_K_M,
+                                                ABlockTransferThreadClusterArrangeOrder,
+                                                ABlockTransferSrcAccessOrder,
+                                                ABlockTransferSrcVectorDim,
+                                                ABlockTransferSrcScalarPerVector,
+                                                ABlockTransferDstScalarPerVector_M,
+                                                AThreadTransferSrcResetCoordinateAfterRun,
+                                                BBlockTransferThreadSliceLengths_K_N,
+                                                BBlockTransferThreadClusterLengths_K_N,
+                                                BBlockTransferThreadClusterArrangeOrder,
+                                                BBlockTransferSrcAccessOrder,
+                                                BBlockTransferSrcVectorDim,
+                                                BBlockTransferSrcScalarPerVector,
+                                                BBlockTransferDstScalarPerVector_N,
+                                                BThreadTransferSrcResetCoordinateAfterRun,
+                                                CThreadTransferSrcDstAccessOrder,
+                                                CThreadTransferSrcDstVectorDim,
+                                                CThreadTransferDstScalarPerVector,
+                                                AGlobalIteratorHacks,
+                                                BGlobalIteratorHacks,
+                                                CGlobalIteratorHacks,
+                                                AGlobalMoveSliceWindowIteratorHacks,
+                                                BGlobalMoveSliceWindowIteratorHacks>;
+
+    const auto GridSize = (M / MPerBlock) * (N / NPerBlock);
+
+    const bool has_main_k_block_loop = (K + KPerBlock) / (2 * KPerBlock) > 1;
+
+    const bool has_double_tail_k_block_loop = (K / KPerBlock) % 2 == 0;
+
+#if CK_EXPERIMENTAL_PASS_TENSOR_DESCRIPTOR_BY_VALUE
+    float ave_time = 0;
+
+    if(has_main_k_block_loop && has_double_tail_k_block_loop)
+    {
+        const auto kernel = kernel_dynamic_gemm_v1r2<gridwise_gemm,
+                                                     FloatAB,
+                                                     FloatAB,
+                                                     FloatC,
+                                                     remove_reference_t<AGlobalDesc>,
+                                                     remove_reference_t<BGlobalDesc>,
+                                                     remove_reference_t<CGlobalDesc>,
+                                                     remove_reference_t<CBlockClusterDesc>,
+                                                     true,
+                                                     true>;
+
+        ave_time = launch_and_time_kernel(kernel,
+                                          nrepeat,
+                                          dim3(GridSize),
+                                          dim3(BlockSize),
+                                          0,
+                                          0,
+                                          p_a_global,
+                                          p_b_global,
+                                          p_c_global,
+                                          a_k_m_global_desc,
+                                          b_k_n_global_desc,
+                                          c_m0_m1_n0_n1_global_desc,
+                                          c_block_cluster_desc);
+    }
+    else if(has_main_k_block_loop && !has_double_tail_k_block_loop)
+    {
+        const auto kernel = kernel_dynamic_gemm_v1r2<gridwise_gemm,
+                                                     FloatAB,
+                                                     FloatAB,
+                                                     FloatC,
+                                                     remove_reference_t<AGlobalDesc>,
+                                                     remove_reference_t<BGlobalDesc>,
+                                                     remove_reference_t<CGlobalDesc>,
+                                                     remove_reference_t<CBlockClusterDesc>,
+                                                     true,
+                                                     false>;
+
+        ave_time = launch_and_time_kernel(kernel,
+                                          nrepeat,
+                                          dim3(GridSize),
+                                          dim3(BlockSize),
+                                          0,
+                                          0,
+                                          p_a_global,
+                                          p_b_global,
+                                          p_c_global,
+                                          a_k_m_global_desc,
+                                          b_k_n_global_desc,
+                                          c_m0_m1_n0_n1_global_desc,
+                                          c_block_cluster_desc);
+    }
+    else if(!has_main_k_block_loop && has_double_tail_k_block_loop)
+    {
+        const auto kernel = kernel_dynamic_gemm_v1r2<gridwise_gemm,
+                                                     FloatAB,
+                                                     FloatAB,
+                                                     FloatC,
+                                                     remove_reference_t<AGlobalDesc>,
+                                                     remove_reference_t<BGlobalDesc>,
+                                                     remove_reference_t<CGlobalDesc>,
+                                                     remove_reference_t<CBlockClusterDesc>,
+                                                     false,
+                                                     true>;
+
+        ave_time = launch_and_time_kernel(kernel,
+                                          nrepeat,
+                                          dim3(GridSize),
+                                          dim3(BlockSize),
+                                          0,
+                                          0,
+                                          p_a_global,
+                                          p_b_global,
+                                          p_c_global,
+                                          a_k_m_global_desc,
+                                          b_k_n_global_desc,
+                                          c_m0_m1_n0_n1_global_desc,
+                                          c_block_cluster_desc);
+    }
+    else
+    {
+        const auto kernel = kernel_dynamic_gemm_v1r2<gridwise_gemm,
+                                                     FloatAB,
+                                                     FloatAB,
+                                                     FloatC,
+                                                     remove_reference_t<AGlobalDesc>,
+                                                     remove_reference_t<BGlobalDesc>,
+                                                     remove_reference_t<CGlobalDesc>,
+                                                     remove_reference_t<CBlockClusterDesc>,
+                                                     false,
+                                                     false>;
+
+        ave_time = launch_and_time_kernel(kernel,
+                                          nrepeat,
+                                          dim3(GridSize),
+                                          dim3(BlockSize),
+                                          0,
+                                          0,
+                                          p_a_global,
+                                          p_b_global,
+                                          p_c_global,
+                                          a_k_m_global_desc,
+                                          b_k_n_global_desc,
+                                          c_m0_m1_n0_n1_global_desc,
+                                          c_block_cluster_desc);
+    }
+
+    return ave_time;
+#elif CK_EXPERIMENTAL_PASS_TENSOR_DESCRIPTOR_BY_VOID_POINTER
+    DeviceMem a_k_m_global_desc_device_buf(sizeof(AGlobalDesc));
+    DeviceMem b_k_n_global_desc_device_buf(sizeof(BGlobalDesc));
+    DeviceMem c_m0_m1_n0_n1_global_desc_device_buf(sizeof(CGlobalDesc));
+    DeviceMem c_block_cluster_desc_device_buf(sizeof(c_block_cluster_desc));
+
+    a_k_m_global_desc_device_buf.ToDevice(&a_k_m_global_desc);
+    b_k_n_global_desc_device_buf.ToDevice(&b_k_n_global_desc);
+    c_m0_m1_n0_n1_global_desc_device_buf.ToDevice(&c_m0_m1_n0_n1_global_desc);
+    c_block_cluster_desc_device_buf.ToDevice(&c_block_cluster_desc);
+
+    float ave_time = 0;
+
+    if(has_main_k_block_loop && has_double_tail_k_block_loop)
+    {
+        const auto kernel = kernel_dynamic_gemm_v1r2<gridwise_gemm,
+                                                     FloatAB,
+                                                     FloatAB,
+                                                     FloatC,
+                                                     remove_reference_t<AGlobalDesc>,
+                                                     remove_reference_t<BGlobalDesc>,
+                                                     remove_reference_t<CGlobalDesc>,
+                                                     remove_reference_t<CBlockClusterDesc>,
+                                                     true,
+                                                     true>;
+
+        ave_time = launch_and_time_kernel(
+            kernel,
+            nrepeat,
+            dim3(GridSize),
+            dim3(BlockSize),
+            0,
+            0,
+            p_a_global,
+            p_b_global,
+            p_c_global,
+            (void __CONSTANT__*)a_k_m_global_desc_device_buf.GetDeviceBuffer(),
+            (void __CONSTANT__*)b_k_n_global_desc_device_buf.GetDeviceBuffer(),
+            (void __CONSTANT__*)c_m0_m1_n0_n1_global_desc_device_buf.GetDeviceBuffer(),
+            (void __CONSTANT__*)c_block_cluster_desc_device_buf.GetDeviceBuffer());
+    }
+    else if(has_main_k_block_loop && !has_double_tail_k_block_loop)
+    {
+        const auto kernel = kernel_dynamic_gemm_v1r2<gridwise_gemm,
+                                                     FloatAB,
+                                                     FloatAB,
+                                                     FloatC,
+                                                     remove_reference_t<AGlobalDesc>,
+                                                     remove_reference_t<BGlobalDesc>,
+                                                     remove_reference_t<CGlobalDesc>,
+                                                     remove_reference_t<CBlockClusterDesc>,
+                                                     true,
+                                                     false>;
+
+        ave_time = launch_and_time_kernel(
+            kernel,
+            nrepeat,
+            dim3(GridSize),
+            dim3(BlockSize),
+            0,
+            0,
+            p_a_global,
+            p_b_global,
+            p_c_global,
+            (void __CONSTANT__*)a_k_m_global_desc_device_buf.GetDeviceBuffer(),
+            (void __CONSTANT__*)b_k_n_global_desc_device_buf.GetDeviceBuffer(),
+            (void __CONSTANT__*)c_m0_m1_n0_n1_global_desc_device_buf.GetDeviceBuffer(),
+            (void __CONSTANT__*)c_block_cluster_desc_device_buf.GetDeviceBuffer());
+    }
+    else if(!has_main_k_block_loop && has_double_tail_k_block_loop)
+    {
+        const auto kernel = kernel_dynamic_gemm_v1r2<gridwise_gemm,
+                                                     FloatAB,
+                                                     FloatAB,
+                                                     FloatC,
+                                                     remove_reference_t<AGlobalDesc>,
+                                                     remove_reference_t<BGlobalDesc>,
+                                                     remove_reference_t<CGlobalDesc>,
+                                                     remove_reference_t<CBlockClusterDesc>,
+                                                     false,
+                                                     true>;
+
+        ave_time = launch_and_time_kernel(
+            kernel,
+            nrepeat,
+            dim3(GridSize),
+            dim3(BlockSize),
+            0,
+            0,
+            p_a_global,
+            p_b_global,
+            p_c_global,
+            (void __CONSTANT__*)a_k_m_global_desc_device_buf.GetDeviceBuffer(),
+            (void __CONSTANT__*)b_k_n_global_desc_device_buf.GetDeviceBuffer(),
+            (void __CONSTANT__*)c_m0_m1_n0_n1_global_desc_device_buf.GetDeviceBuffer(),
+            (void __CONSTANT__*)c_block_cluster_desc_device_buf.GetDeviceBuffer());
+    }
+    else
+    {
+        const auto kernel = kernel_dynamic_gemm_v1r2<gridwise_gemm,
+                                                     FloatAB,
+                                                     FloatAB,
+                                                     FloatC,
+                                                     remove_reference_t<AGlobalDesc>,
+                                                     remove_reference_t<BGlobalDesc>,
+                                                     remove_reference_t<CGlobalDesc>,
+                                                     remove_reference_t<CBlockClusterDesc>,
+                                                     false,
+                                                     false>;
+
+        ave_time = launch_and_time_kernel(
+            kernel,
+            nrepeat,
+            dim3(GridSize),
+            dim3(BlockSize),
+            0,
+            0,
+            p_a_global,
+            p_b_global,
+            p_c_global,
+            (void __CONSTANT__*)a_k_m_global_desc_device_buf.GetDeviceBuffer(),
+            (void __CONSTANT__*)b_k_n_global_desc_device_buf.GetDeviceBuffer(),
+            (void __CONSTANT__*)c_m0_m1_n0_n1_global_desc_device_buf.GetDeviceBuffer(),
+            (void __CONSTANT__*)c_block_cluster_desc_device_buf.GetDeviceBuffer());
+    }
+
+    return ave_time;
+#endif
+}
+
+} // namespace ck
+#endif

composable_kernel/include/kernel_algorithm/transform_forward_convolution_into_gemm_v4r4r2_nchw_kcyx_nkhw.hpp

+#ifndef CK_TRANSFORM_FORWARD_CONVOLUTION_INTO_GEMM_V4R4R2_NCHW_KCYX_NKHW_HPP
+#define CK_TRANSFORM_FORWARD_CONVOLUTION_INTO_GEMM_V4R4R2_NCHW_KCYX_NKHW_HPP
+
+#include "common_header.hpp"
+#include "dynamic_tensor_descriptor.hpp"
+#include "dynamic_tensor_descriptor_helper.hpp"
+
+namespace ck {
+
+// GemmM = K
+// GemmN = N * Ho * Wo
+// GemmK = C * Y * X
+template <index_t GemmMPerBlock,
+          index_t GemmNPerBlock,
+          index_t GemmM1,
+          index_t GemmN1,
+          typename... Wei,
+          typename... In,
+          typename... Out,
+          typename ConvStrides,
+          typename ConvDilations,
+          typename InLeftPads,
+          typename InRightPads>
+__host__ __device__ constexpr auto
+transform_forward_convolution_into_gemm_v4r4r2_nchw_kcyx_nkhw_pad(
+    const DynamicTensorDescriptor<Wei...>& wei_k_c_y_x_global_desc,
+    const DynamicTensorDescriptor<In...>& in_n_c_hi_wi_global_desc,
+    const DynamicTensorDescriptor<Out...>& out_n_k_ho_wo_global_desc,
+    const ConvStrides& conv_strides,
+    const ConvDilations& conv_dilations,
+    const InLeftPads& in_left_pads,
+    const InRightPads& in_right_pads)
+{
+    constexpr auto I0 = Number<0>{};
+    constexpr auto I1 = Number<1>{};
+    constexpr auto I2 = Number<2>{};
+    constexpr auto I3 = Number<3>{};
+
+    const auto N = in_n_c_hi_wi_global_desc.GetLength(I0);
+    const auto C = in_n_c_hi_wi_global_desc.GetLength(I1);
+    const auto K = out_n_k_ho_wo_global_desc.GetLength(I1);
+
+    const auto Hi = in_n_c_hi_wi_global_desc.GetLength(I2);
+    const auto Wi = in_n_c_hi_wi_global_desc.GetLength(I3);
+
+    const auto Ho = out_n_k_ho_wo_global_desc.GetLength(I2);
+    const auto Wo = out_n_k_ho_wo_global_desc.GetLength(I3);
+
+    const auto Y = wei_k_c_y_x_global_desc.GetLength(I2);
+    const auto X = wei_k_c_y_x_global_desc.GetLength(I3);
+
+    const auto ConvStrideH = conv_strides[I0];
+    const auto ConvStrideW = conv_strides[I1];
+
+    const auto ConvDilationH = conv_dilations[I0];
+    const auto ConvDilationW = conv_dilations[I1];
+
+    const auto InLeftPadH = in_left_pads[I0];
+    const auto InLeftPadW = in_left_pads[I1];
+
+    const auto InRightPadH = in_right_pads[I0];
+    const auto InRightPadW = in_right_pads[I1];
+
+    // weight tensor
+    const auto wei_gemmk_gemmm_global_desc = transform_dynamic_tensor_descriptor(
+        make_dynamic_naive_tensor_descriptor_packed_v2(make_tuple(K, C * Y * X)),
+        make_tuple(make_pass_through_transform(K), make_pass_through_transform(C * Y * X)),
+        make_tuple(Sequence<0>{}, Sequence<1>{}),
+        make_tuple(Sequence<1>{}, Sequence<0>{}));
+
+    // input tensor
+    const auto in_n_c_hip_wip_global_desc = transform_dynamic_tensor_descriptor(
+        in_n_c_hi_wi_global_desc,
+        make_tuple(make_pass_through_transform(N),
+                   make_pass_through_transform(C),
+                   make_pad_transform(Hi, InLeftPadH, InRightPadH),
+                   make_pad_transform(Wi, InLeftPadW, InRightPadW)),
+        make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}),
+        make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}));
+
+    const auto in_n_c_y_ho_x_wo_global_desc = transform_dynamic_tensor_descriptor(
+        in_n_c_hip_wip_global_desc,
+        make_tuple(make_pass_through_transform(N),
+                   make_pass_through_transform(C),
+                   make_embed_transform(make_tuple(Y, Ho), make_tuple(ConvDilationH, ConvStrideH)),
+                   make_embed_transform(make_tuple(X, Wo), make_tuple(ConvDilationW, ConvStrideW))),
+        make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}),
+        make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2, 3>{}, Sequence<4, 5>{}));
+
+    const auto in_gemmk_gemmn_global_desc =
+        transform_dynamic_tensor_descriptor(in_n_c_y_ho_x_wo_global_desc,
+                                            make_tuple(make_merge_transform(make_tuple(C, Y, X)),
+                                                       make_merge_transform(make_tuple(N, Ho, Wo))),
+                                            make_tuple(Sequence<1, 2, 4>{}, Sequence<0, 3, 5>{}),
+                                            make_tuple(Sequence<0>{}, Sequence<1>{}));
+
+    // output tensor
+    const auto out_gemmm_gemmn_global_desc = transform_dynamic_tensor_descriptor(
+        make_dynamic_naive_tensor_descriptor_packed_v2(make_tuple(N, K, Ho * Wo)),
+        make_tuple(make_pass_through_transform(K), make_merge_transform(make_tuple(N, Ho * Wo))),
+        make_tuple(Sequence<1>{}, Sequence<0, 2>{}),
+        make_tuple(Sequence<0>{}, Sequence<1>{}));
+
+    const auto GemmM = out_gemmm_gemmn_global_desc.GetLength(I0);
+    const auto GemmN = out_gemmm_gemmn_global_desc.GetLength(I1);
+    const auto GemmK = wei_gemmk_gemmm_global_desc.GetLength(I0);
+
+    assert(GemmM % GemmMPerBlock == 0 && GemmN % GemmNPerBlock == 0 && GemmK % GemmKPerBlock == 0);
+
+    const auto GemmM0 = GemmM / Number<GemmM1>{};
+    const auto GemmN0 = GemmN / Number<GemmN1>{};
+
+    const auto out_gemmm0_gemmm1_gemmn0_gemmn1_global_desc = transform_dynamic_tensor_descriptor(
+        out_gemmm_gemmn_global_desc,
+        make_tuple(make_unmerge_transform(make_tuple(GemmM0, GemmM1)),
+                   make_unmerge_transform(make_tuple(GemmN0, GemmN1))),
+        make_tuple(Sequence<0>{}, Sequence<1>{}),
+        make_tuple(Sequence<0, 1>{}, Sequence<2, 3>{}));
+
+    // out_gemm_block_cluster_desc
+    const auto out_gemm_block_cluster_desc = make_cluster_descriptor_v2(
+        make_tuple(GemmM / Number<GemmMPerBlock>{}, GemmN / Number<GemmNPerBlock>{}));
+
+    // hack to control index calculation when iterating over wei_gemmk_gemmm_global tensor
+    constexpr auto wei_gemmk_gemmm_global_iterator_hacks =
+        make_tuple(make_tuple(Sequence<0, 0, 0>{}, Sequence<0, 0, 0>{}),
+                   make_tuple(Sequence<0, 0, 0>{}, Sequence<0, 0, 0>{}));
+
+    constexpr auto wei_gemmk_gemmm_global_move_slice_window_iterator_hacks = Sequence<0, 0, 0>{};
+
+    // hack to control index calculation when iterating over in_gemmk_gemmn_global tensor
+    constexpr auto in_gemmk_gemmn_global_iterator_hacks =
+        make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0>{},
+                              Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1>{}),
+                   make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0>{},
+                              Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2>{}));
+
+    constexpr auto in_gemmk_gemmn_global_move_slice_window_iterator_hacks =
+        Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2>{};
+
+    // hack to control index calculation when iterating over out_gemmm0_gemmm1_gemmn0_gemmn1_global
+    // tensor hack for NKHW format
+    constexpr auto out_gemmm0_gemmm1_gemmn0_gemmn1_global_iterator_hacks =
+        make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0>{},
+                              Sequence<0, 0, 0, 0, 0>{},
+                              Sequence<0, 0, 1, 0, 0>{},
+                              Sequence<0, 0, 1, 0, 0>{}),
+                   make_tuple(Sequence<0, 0, 0, 0, 0>{},
+                              Sequence<0, 0, 0, 0, 0>{},
+                              Sequence<0, 0, 2, 0, 0>{},
+                              Sequence<0, 0, 2, 0, 0>{}));
+
+    return make_tuple(wei_gemmk_gemmm_global_desc,
+                      in_gemmk_gemmn_global_desc,
+                      out_gemmm0_gemmm1_gemmn0_gemmn1_global_desc,
+                      out_gemm_block_cluster_desc,
+                      wei_gemmk_gemmm_global_iterator_hacks,
+                      in_gemmk_gemmn_global_iterator_hacks,
+                      out_gemmm0_gemmm1_gemmn0_gemmn1_global_iterator_hacks,
+                      wei_gemmk_gemmm_global_move_slice_window_iterator_hacks,
+                      in_gemmk_gemmn_global_move_slice_window_iterator_hacks);
+}
+
+} // namespace ck
+#endif

composable_kernel/include/utility/config.amd.hpp.in

@@ -14,7 +14,7 @@
 #define CK_DEVICE_BACKEND_AMD 1
 
 // GPU ID
-#if 1
+#if 0
 #define CK_AMD_GPU_GFX906 1
 #elif 0
 #define CK_AMD_GPU_GFX908 1

driver/src/conv_driver.cpp

@@ -15,6 +15,7 @@
 #include "device_convolution_forward_implicit_gemm_v4r4_nchw_kcyx_nkhw.hpp"
 #include "device_convolution_forward_implicit_gemm_v4r4_nhwc_kyxc_nhwk.hpp"
 #include "device_dynamic_convolution_forward_implicit_gemm_v4r4_nchw_kcyx_nkhw.hpp"
+#include "device_dynamic_convolution_forward_implicit_gemm_v4r4r2_nchw_kcyx_nkhw.hpp"
 #include "device_dynamic_convolution_forward_implicit_gemm_v4r4_nhwc_kyxc_nhwk.hpp"
 #include "device_dynamic_convolution_forward_implicit_gemm_v5r1_nchw_kcyx_nkhw.hpp"
 
@@ -724,7 +725,7 @@ int main(int argc, char* argv[])
                                                                  LeftPads{},
                                                                  RightPads{},
                                                                  nrepeat);
-#elif 1
+#elif 0
     device_dynamic_convolution_forward_implicit_gemm_v4r4_nchw_kcyx_nkhw<in_data_t,
                                                                          in_vector_size,
                                                                          acc_data_t,
@@ -740,6 +741,22 @@ int main(int argc, char* argv[])
         LeftPads{},
         RightPads{},
         nrepeat);
+#elif 1
+    device_dynamic_convolution_forward_implicit_gemm_v4r4r2_nchw_kcyx_nkhw<in_data_t,
+                                                                           in_vector_size,
+                                                                           acc_data_t,
+                                                                           out_data_t>(
+        in_nchw_desc,
+        in_nchw,
+        wei_kcyx_desc,
+        wei_kcyx,
+        out_nkhw_desc,
+        out_nkhw_device,
+        ConvStrides{},
+        ConvDilations{},
+        LeftPads{},
+        RightPads{},
+        nrepeat);
 #elif 1
     device_dynamic_convolution_forward_implicit_gemm_v4r4_nhwc_kyxc_nhwk<in_data_t,
                                                                          in_vector_size,