Merge remote-tracking branch 'origin/develop' into bwroblew/direct_loads

profiler/include/profiler/profile_gemm_splitk_impl.hpp

@@ -143,8 +143,7 @@ bool profile_gemm_splitk_impl(int do_verification,
     // profile device GEMM instances
     for(auto& op_ptr : op_ptrs)
     {
-        std::vector<int> kbatch_list = {1,  2,  4,  8,  12, 16,  20,  24,  32,  36,  40, 60,
-                                        64, 72, 80, 88, 96, 128, 144, 160, 176, 192, 256};
+        std::vector<int> kbatch_list = {1, 2, 4, 8, 12, 16, 20, 32, 36, 40, 64, 96, 128};
 
         if(KBatch > 0)
         {

profiler/include/profiler/profile_grouped_conv_fwd_impl.hpp

@@ -198,18 +198,18 @@ bool profile_grouped_conv_fwd_impl(int do_verification,
         }
     };
 
-    using DeviceOp = ck::tensor_operation::device::DeviceGroupedConvFwdMultipleD<NDimSpatial,
-                                                                                 InLayout,
-                                                                                 WeiLayout,
-                                                                                 ck::Tuple<>,
-                                                                                 OutLayout,
-                                                                                 InDataType,
-                                                                                 WeiDataType,
-                                                                                 ck::Tuple<>,
-                                                                                 OutDataType,
-                                                                                 InElementOp,
-                                                                                 WeiElementOp,
-                                                                                 OutElementOp>;
+    using DeviceOp = ck::tensor_operation::device::DeviceGroupedConvFwdMultipleABD<NDimSpatial,
+                                                                                   InLayout,
+                                                                                   WeiLayout,
+                                                                                   ck::Tuple<>,
+                                                                                   OutLayout,
+                                                                                   InDataType,
+                                                                                   WeiDataType,
+                                                                                   ck::Tuple<>,
+                                                                                   OutDataType,
+                                                                                   InElementOp,
+                                                                                   WeiElementOp,
+                                                                                   OutElementOp>;
 
     // get device op instances
     const auto op_ptrs = ck::tensor_operation::device::instance::DeviceOperationInstanceFactory<

profiler/include/profiler/profile_groupnorm_impl.hpp → profiler/include/profiler/profile_groupnorm_fwd_impl.hpp

@@ -7,7 +7,7 @@
 
 #include "ck/ck.hpp"
 
-#include "ck/library/tensor_operation_instance/gpu/normalization.hpp"
+#include "ck/library/tensor_operation_instance/gpu/normalization_fwd.hpp"
 
 #include "ck/library/utility/check_err.hpp"
 #include "ck/library/utility/device_memory.hpp"
@@ -88,14 +88,14 @@ bool profile_groupnorm_impl(int do_verification,
     beta_dev.ToDevice(beta.mData.data());
 
     // add device normalization instances
-    using DeviceOp = ck::tensor_operation::device::DeviceNormalization<XDataType,
-                                                                       GammaDataType,
-                                                                       BetaDataType,
-                                                                       YDataType,
-                                                                       SaveMeanInvStdDataType,
-                                                                       PassThrough,
-                                                                       5,
-                                                                       3>;
+    using DeviceOp = ck::tensor_operation::device::DeviceNormalizationFwd<XDataType,
+                                                                          GammaDataType,
+                                                                          BetaDataType,
+                                                                          YDataType,
+                                                                          SaveMeanInvStdDataType,
+                                                                          PassThrough,
+                                                                          5,
+                                                                          3>;
 
     // get device op instances
     const auto instance_ptrs =

profiler/include/profiler/profile_layernorm_impl.hpp → profiler/include/profiler/profile_layernorm_fwd_impl.hpp

@@ -6,7 +6,7 @@
 #include <iomanip>
 
 #include "ck/ck.hpp"
-#include "ck/library/tensor_operation_instance/gpu/normalization.hpp"
+#include "ck/library/tensor_operation_instance/gpu/normalization_fwd.hpp"
 #include "ck/library/utility/check_err.hpp"
 #include "ck/library/utility/device_memory.hpp"
 #include "ck/library/utility/host_tensor.hpp"
@@ -94,14 +94,14 @@ bool profile_layernorm_impl(int do_verification,
     constexpr int NumReduceDim = Rank - 1;
 
     // add device normalization instances
-    using DeviceOp = ck::tensor_operation::device::DeviceNormalization<XDataType,
-                                                                       GammaDataType,
-                                                                       BetaDataType,
-                                                                       YDataType,
-                                                                       SaveMeanInvStdDataType,
-                                                                       PassThrough,
-                                                                       Rank,
-                                                                       NumReduceDim>;
+    using DeviceOp = ck::tensor_operation::device::DeviceNormalizationFwd<XDataType,
+                                                                          GammaDataType,
+                                                                          BetaDataType,
+                                                                          YDataType,
+                                                                          SaveMeanInvStdDataType,
+                                                                          PassThrough,
+                                                                          Rank,
+                                                                          NumReduceDim>;
 
     // get device op instances
     const auto instance_ptrs =

profiler/include/profiler/profile_transpose_impl.hpp

+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved.
+
+#pragma once
+
+#include <iomanip>
+#include <iostream>
+#include <typeinfo>
+
+#include "ck/ck.hpp"
+#include "ck/tensor_operation/gpu/device/tensor_layout.hpp"
+#include "ck/tensor_operation/gpu/device/device_elementwise.hpp"
+#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp"
+#include "ck/tensor_operation/gpu/device/impl/device_elementwise_3d_impl.hpp"
+
+#include "ck/library/tensor_operation_instance/gpu/transpose_3d.hpp"
+
+#include "ck/library/utility/check_err.hpp"
+#include "ck/library/utility/device_memory.hpp"
+#include "ck/library/utility/host_tensor.hpp"
+#include "ck/library/utility/host_tensor_generator.hpp"
+#include "ck/library/utility/literals.hpp"
+
+namespace ck {
+namespace profiler {
+
+template <typename HostTensorA, typename HostTensorB, typename Functor>
+void host_elementwise4D(HostTensorB& B_nchwd, const HostTensorA& A_ncdhw, Functor functor)
+{
+    for(std::size_t n = 0; n < A_ncdhw.mDesc.GetLengths()[0]; ++n)
+        for(std::size_t c = 0; c < A_ncdhw.mDesc.GetLengths()[1]; ++c)
+            for(std::size_t d = 0; d < A_ncdhw.mDesc.GetLengths()[2]; ++d)
+                for(std::size_t h = 0; h < A_ncdhw.mDesc.GetLengths()[3]; ++h)
+                    for(std::size_t w = 0; w < A_ncdhw.mDesc.GetLengths()[4]; ++w)
+                    {
+                        auto a_val = A_ncdhw(n, c, d, h, w);
+                        functor(B_nchwd(n, c, h, w, d), a_val);
+                    }
+}
+
+template <typename ADataType, typename BDataType, index_t NumDim>
+bool profile_transpose_impl(int do_verification,
+                            int init_method,
+                            bool do_log,
+                            bool time_kernel,
+                            std::vector<index_t> lengths)
+{
+    bool pass = true;
+
+    index_t N = lengths[0];
+    index_t C = lengths[1];
+    index_t D = lengths[2];
+    index_t H = lengths[3];
+    index_t W = lengths[4];
+
+    std::vector<ck::index_t> ncdhw = {N, C, D, H, W};
+    std::vector<ck::index_t> ndhwc = {N, D, H, W, C};
+    Tensor<ADataType> a(ncdhw);
+    Tensor<BDataType> b(ndhwc);
+    Tensor<BDataType> host_b(ndhwc);
+
+    // a.GenerateTensorValue(GeneratorTensor_3<ADataType>{0.0, 1.0});
+
+    std::array<ck::index_t, 5> ab_lengths{N, C, H, W, D};
+    std::array<ck::index_t, 5> a_strides = {C * D * H * W, H * W, W, 1, D * H * W}; // N, C, D, H, W
+    std::array<ck::index_t, 5> b_strides = {C * H * W * D, H * W * D, W * D, D, 1}; // N, D, H, W, C
+
+    std::cout << "A: " << a.mDesc << std::endl;
+    std::cout << "B: " << b.mDesc << std::endl;
+
+    switch(init_method)
+    {
+    case 0: break;
+    case 1: a.GenerateTensorValue(GeneratorTensor_2<ADataType>{-1, 2}); break;
+    default: a.GenerateTensorValue(GeneratorTensor_3<ADataType>{0.0, 1.0});
+    }
+
+    using ElementOp = ck::tensor_operation::element_wise::PassThrough;
+
+    // const auto element_op = ElementOp{};
+
+    DeviceMem a_device_buf(sizeof(ADataType) * a.mDesc.GetElementSpaceSize());
+    DeviceMem b_device_buf(sizeof(BDataType) * b.mDesc.GetElementSpaceSize());
+
+    a_device_buf.ToDevice(a.mData.data());
+
+    std::array<const void*, 1> input = {a_device_buf.GetDeviceBuffer()};
+    std::array<void*, 1> output      = {b_device_buf.GetDeviceBuffer()};
+    using DeviceOp                   = ck::tensor_operation::device::
+        DeviceElementwise<ck::Tuple<ADataType>, ck::Tuple<BDataType>, ElementOp, NumDim>;
+
+    // get device op instances
+    const auto op_ptrs = ck::tensor_operation::device::instance::DeviceOperationInstanceFactory<
+        DeviceOp>::GetInstances();
+
+    std::cout << "found " << op_ptrs.size() << " instances" << std::endl;
+
+    if(do_verification)
+    {
+        host_elementwise4D(host_b, a, ElementOp{});
+    }
+
+    std::string best_op_name;
+    float best_ave_time   = 0;
+    float best_tflops     = 0;
+    float best_gb_per_sec = 0;
+
+    for(auto& op_ptr : op_ptrs)
+    {
+        auto argument_ptr = op_ptr->MakeArgumentPointer(
+            ab_lengths, {a_strides}, {b_strides}, input, output, ElementOp{});
+
+        auto invoker_ptr = op_ptr->MakeInvokerPointer();
+
+        if(op_ptr->IsSupportedArgument(argument_ptr.get()))
+        {
+
+            // re-init C to zero before profiling next kernel
+            b_device_buf.SetZero();
+
+            invoker_ptr->Run(argument_ptr.get(), StreamConfig{nullptr, false});
+
+            if(do_verification)
+            {
+                b_device_buf.FromDevice(b.mData.data());
+
+                pass &= ck::utils::check_err(
+                    b.mData, host_b.mData, "Error: Incorrect results b", 1e-3, 1e-3);
+
+                if(do_log)
+                {
+                    LogRangeAsType<float>(std::cout << "a : ", a.mData, ",") << std::endl;
+                    LogRangeAsType<float>(std::cout << "b: ", b.mData, ",") << std::endl;
+                }
+            }
+
+            std::string op_name = op_ptr->GetTypeString();
+
+            float ave_time =
+                invoker_ptr->Run(argument_ptr.get(), StreamConfig{nullptr, time_kernel});
+
+            std::size_t flop =
+                std::size_t(2) * ncdhw[0] * ncdhw[1] * ncdhw[2] * ncdhw[3] * ncdhw[4];
+
+            std::size_t num_btype =
+                sizeof(ADataType) * (ncdhw[0] * ncdhw[1] * ncdhw[2] * ncdhw[3] * ncdhw[4]) +
+                sizeof(BDataType) * (ncdhw[0] * ncdhw[1] * ncdhw[2] * ncdhw[3] * ncdhw[4]);
+
+            float tflops = static_cast<float>(flop) / 1.E9 / ave_time;
+
+            float gb_per_sec = num_btype / 1.E6 / ave_time;
+
+            std::cout << "Perf: " << std::setw(10) << ave_time << " ms, " << tflops << " TFlops, "
+                      << gb_per_sec << " GB/s, " << op_name << std::endl;
+
+            // pass = pass & ck::utils::check_err(b_device_result, b_host_result);
+            pass &= ck::utils::check_err(
+                b.mData, host_b.mData, "Error: Incorrect results b", 1e-3, 1e-3);
+
+            if(tflops > best_tflops)
+            {
+                best_op_name    = op_name;
+                best_tflops     = tflops;
+                best_ave_time   = ave_time;
+                best_gb_per_sec = gb_per_sec;
+            }
+        }
+        else
+        {
+            std::cout << op_ptr->GetTypeString() << " does not support this problem" << std::endl;
+        }
+    }
+
+    std::cout << " N = " << N << " C = " << C << " D = " << D << " H = " << H << " W = " << W
+              << " : " << best_ave_time << " ms, " << best_tflops << " TFlops, " << best_gb_per_sec
+              << " GB/s, " << best_op_name << std::endl;
+
+    return pass;
+}
+
+} // namespace profiler
+} // namespace ck

profiler/src/CMakeLists.txt

@@ -16,8 +16,8 @@ set(PROFILER_SOURCES
     profile_grouped_conv_fwd.cpp
     profile_grouped_conv_bwd_weight.cpp
     profile_reduce.cpp
-    profile_groupnorm.cpp
-    profile_layernorm.cpp
+    profile_groupnorm_fwd.cpp
+    profile_layernorm_fwd.cpp
     profile_max_pool3d_fwd.cpp
     profile_avg_pool3d_bwd.cpp
     profile_max_pool3d_bwd.cpp
@@ -28,9 +28,11 @@ set(PROFILER_SOURCES
     profile_grouped_conv_bwd_data.cpp
     profile_conv_tensor_rearrange.cpp
 )
+
 if(DL_KERNELS)
   list(APPEND PROFILER_SOURCES profile_batched_gemm_multi_d.cpp)
 endif()
+
 if(DTYPES MATCHES "fp16" OR NOT DEFINED DTYPES)
   list(APPEND PROFILER_SOURCES profile_batched_gemm_gemm.cpp)
   list(APPEND PROFILER_SOURCES profile_gemm_fastgelu.cpp)
@@ -75,7 +77,7 @@ target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_grouped_conv2d_bwd_w
 target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_grouped_conv3d_bwd_weight_instance)
 target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_conv2d_fwd_bias_relu_instance)
 target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_conv2d_fwd_bias_relu_add_instance)
-target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_normalization_instance)
+target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_normalization_fwd_instance)
 target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_softmax_instance)
 target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_reduce_instance)
 target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_batchnorm_instance)
@@ -110,4 +112,5 @@ if(DTYPES MATCHES "fp16" OR NOT DEFINED DTYPES)
   target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_grouped_gemm_instance)
   target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_grouped_gemm_fastgelu_instance)
 endif()
+
 rocm_install(TARGETS ${PROFILER_EXECUTABLE} COMPONENT profiler)

profiler/src/profile_contraction_bilinear.cpp

@@ -17,8 +17,9 @@
 static void print_helper_msg()
 {
     std::cout << "arg1: tensor operation (" OP_NAME ": " OP_DESC ")\n"
-              << "arg2: data type (0: fp32; 1: f64)\n"
-              << "arg3: matrix layout (0: A[m0, m1, k0, k1] * B[k0, k1, n0, n1] + "
+              << "arg2: data type (0: fp32; 1: f64; 2: f16; 3: bf16)\n"
+              << "arg3: compute data type (0: fp32; 1: f64; 2: f16; 3: bf16)\n"
+              << "arg4: matrix layout (0: A[m0, m1, k0, k1] * B[k0, k1, n0, n1] + "
                  "D[m0, m1, n0, n1] = E[m0, m1, n0, n1];\n"
               << "                     1: A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + "
                  "D[m0, m1, n0, n1] = E[m0, m1, n0, n1];\n"
@@ -26,40 +27,42 @@ static void print_helper_msg()
                  "D[m0, m1, n0, n1] = E[m0, m1, n0, n1];\n"
               << "                     3: A[k0, k1, m0, m1] * B[n0, n1, k0, k1] + "
                  "D[m0, m1, n0, n1] = E[m0, m1, n0, n1])\n"
-              << "arg4: verification (0: no; 1: yes)\n"
-              << "arg5: initialization (0: no init; 1: integer value; 2: decimal "
+              << "arg5: verification (0: no; 1: yes)\n"
+              << "arg6: initialization (0: no init; 1: integer value; 2: decimal "
               << "value)\n"
-              << "arg6: print tensor value (0: no; 1: yes)\n"
-              << "arg7: time kernel (0: no, 1: yes)\n"
-              << "arg8 and arg9: alpha and beta\n"
-              << "arg10 to 15: M0, M1, N0, N1, K0, K1\n"
-              << "arg16 to 31: Strides for A, B, D and E (skip for default)\n"
+              << "arg7: print tensor value (0: no; 1: yes)\n"
+              << "arg8: time kernel (0: no, 1: yes)\n"
+              << "arg9: alpha\n"
+              << "arg10: beta\n"
+              << "arg11 to 16: M0, M1, N0, N1, K0, K1\n"
+              << "arg17 to 32: Strides for A, B, D and E (skip for default)\n"
               << std::endl;
 }
 
 int profile_contraction_bilinear(int argc, char* argv[])
 {
-    const bool default_strides = argc == 16;
+    const bool default_strides = argc == 17;
 
-    if(argc != 32 && argc != 16)
+    if(argc != 33 && argc != 17)
     {
         print_helper_msg();
         exit(1);
     }
 
     const auto data_type          = static_cast<ContractionDataType>(std::stoi(argv[2]));
-    const auto layout             = static_cast<ContractionMatrixLayout>(std::stoi(argv[3]));
-    const bool do_verification    = std::stoi(argv[4]);
-    const ck::index_t init_method = std::stoi(argv[5]);
-    const bool do_log             = std::stoi(argv[6]);
-    const bool time_kernel        = std::stoi(argv[7]);
-    const float alpha             = std::stof(argv[8]);
-    const float beta              = std::stof(argv[9]);
+    const auto compute_data_type  = static_cast<ContractionComputeDataType>(std::stoi(argv[3]));
+    const auto layout             = static_cast<ContractionMatrixLayout>(std::stoi(argv[4]));
+    const bool do_verification    = std::stoi(argv[5]);
+    const ck::index_t init_method = std::stoi(argv[6]);
+    const bool do_log             = std::stoi(argv[7]);
+    const bool time_kernel        = std::stoi(argv[8]);
+    const float alpha             = std::stof(argv[9]);
+    const float beta              = std::stof(argv[10]);
 
     std::vector<ck::index_t> M;
     std::vector<ck::index_t> N;
     std::vector<ck::index_t> K;
-    const ck::index_t dims_arg_num = 10;
+    const ck::index_t dims_arg_num = 11;
     collect_index_params(argv, M, dims_arg_num, 2);
     collect_index_params(argv, N, dims_arg_num + 2, 2);
     collect_index_params(argv, K, dims_arg_num + 4, 2);
@@ -76,90 +79,130 @@ int profile_contraction_bilinear(int argc, char* argv[])
         collect_index_params(argv, StridesD, dims_arg_num + 18, 4);
     }
 
-    using F32 = float;
-    using F64 = double;
-
-    auto profile = [&](auto a_layout, auto b_layout, auto cde_layout, auto type) {
-        using ALayout   = decltype(a_layout);
-        using BLayout   = decltype(b_layout);
-        using CDELayout = decltype(cde_layout);
-
-        using DataType = decltype(type);
-
-        if(default_strides)
+    using F16  = ck::half_t;
+    using BF16 = ck::bhalf_t;
+    using F32  = float;
+    using F64  = double;
+
+    auto profile =
+        [&](auto a_layout, auto b_layout, auto cde_layout, auto type, auto compute_type) {
+            using ALayout   = decltype(a_layout);
+            using BLayout   = decltype(b_layout);
+            using CDELayout = decltype(cde_layout);
+
+            using DataType        = decltype(type);
+            using ComputeDataType = decltype(compute_type);
+
+            if(default_strides)
+            {
+                assign_default_strides(a_layout, StridesA, {M[0], M[1], K[0], K[1]});
+                assign_default_strides(b_layout, StridesB, {N[0], N[1], K[0], K[1]});
+                assign_default_strides(cde_layout, StridesE, {M[0], M[1], N[0], N[1]});
+                assign_default_strides(cde_layout, StridesD, {M[0], M[1], N[0], N[1]});
+            }
+            bool pass = ck::profiler::profile_contraction_impl<ALayout,
+                                                               BLayout,
+                                                               CDELayout,
+                                                               DataType,
+                                                               ComputeDataType,
+                                                               ck::Tuple<DataType>,
+                                                               Bilinear>(do_verification,
+                                                                         init_method,
+                                                                         do_log,
+                                                                         time_kernel,
+                                                                         Bilinear{alpha, beta},
+                                                                         M,
+                                                                         N,
+                                                                         K,
+                                                                         StridesA,
+                                                                         StridesB,
+                                                                         StridesE,
+                                                                         StridesD);
+
+            return pass;
+        };
+
+    auto run_profile_for_datatype = [&](auto type, auto compute_type) {
+        if(layout == ContractionMatrixLayout::MK_KN_MN_MN)
         {
-            assign_default_strides(a_layout, StridesA, {M[0], M[1], K[0], K[1]});
-            assign_default_strides(b_layout, StridesB, {K[0], K[1], N[0], N[1]});
-            assign_default_strides(cde_layout, StridesE, {M[0], M[1], N[0], N[1]});
-            assign_default_strides(cde_layout, StridesD, {M[0], M[1], N[0], N[1]});
+            return profile(Row{}, Row{}, Row{}, type, compute_type);
         }
-        bool pass = ck::profiler::profile_contraction_impl<ALayout,
-                                                           BLayout,
-                                                           CDELayout,
-                                                           DataType,
-                                                           ck::Tuple<DataType>,
-                                                           Bilinear>(do_verification,
-                                                                     init_method,
-                                                                     do_log,
-                                                                     time_kernel,
-                                                                     Bilinear{alpha, beta},
-                                                                     M,
-                                                                     N,
-                                                                     K,
-                                                                     StridesA,
-                                                                     StridesB,
-                                                                     StridesE,
-                                                                     StridesD);
-
-        return pass;
+        else if(layout == ContractionMatrixLayout::MK_NK_MN_MN)
+        {
+            return profile(Row{}, Col{}, Row{}, type, compute_type);
+        }
+        else if(layout == ContractionMatrixLayout::KM_KN_MN_MN)
+        {
+            return profile(Col{}, Row{}, Row{}, type, compute_type);
+        }
+        else if(layout == ContractionMatrixLayout::KM_NK_MN_MN)
+        {
+            return profile(Col{}, Col{}, Row{}, type, compute_type);
+        }
+        return false;
     };
 
-    if(data_type == ContractionDataType::F32_F32_F32_F32 &&
-       layout == ContractionMatrixLayout::MK_KN_MN_MN)
-    {
-        return profile(Row{}, Row{}, Row{}, F32{});
-    }
-    else if(data_type == ContractionDataType::F32_F32_F32_F32 &&
-            layout == ContractionMatrixLayout::MK_NK_MN_MN)
+    if(data_type == ContractionDataType::F32_F32_F32_F32)
     {
-        return profile(Row{}, Col{}, Row{}, F32{});
-    }
-    else if(data_type == ContractionDataType::F32_F32_F32_F32 &&
-            layout == ContractionMatrixLayout::KM_KN_MN_MN)
-    {
-        return profile(Col{}, Row{}, Row{}, F32{});
-    }
-    else if(data_type == ContractionDataType::F32_F32_F32_F32 &&
-            layout == ContractionMatrixLayout::KM_NK_MN_MN)
-    {
-        return profile(Col{}, Col{}, Row{}, F32{});
-    }
-    else if(data_type == ContractionDataType::F64_F64_F64_F64 &&
-            layout == ContractionMatrixLayout::MK_KN_MN_MN)
-    {
-        return profile(Row{}, Row{}, Row{}, F64{});
-    }
-    else if(data_type == ContractionDataType::F64_F64_F64_F64 &&
-            layout == ContractionMatrixLayout::MK_NK_MN_MN)
-    {
-        return profile(Row{}, Col{}, Row{}, F64{});
+        if(compute_data_type == ContractionComputeDataType::F32)
+        {
+            return run_profile_for_datatype(F32{}, F32{});
+        }
+        else if(compute_data_type == ContractionComputeDataType::F16)
+        {
+            return run_profile_for_datatype(F32{}, F16{});
+        }
+        else if(compute_data_type == ContractionComputeDataType::BF16)
+        {
+            return run_profile_for_datatype(F32{}, BF16{});
+        }
+        else
+        {
+            std::cout << "Incorrect combination of data type and compute data type." << std::endl;
+            return 1;
+        }
     }
-    else if(data_type == ContractionDataType::F64_F64_F64_F64 &&
-            layout == ContractionMatrixLayout::KM_KN_MN_MN)
+    else if(data_type == ContractionDataType::F64_F64_F64_F64)
     {
-        return profile(Col{}, Row{}, Row{}, F64{});
+        if(compute_data_type == ContractionComputeDataType::F64)
+        {
+            return run_profile_for_datatype(F64{}, F64{});
+        }
+        else if(compute_data_type == ContractionComputeDataType::F32)
+        {
+            return run_profile_for_datatype(F64{}, F32{});
+        }
+        else
+        {
+            std::cout << "Incorrect combination of data type and compute data type." << std::endl;
+            return 1;
+        }
     }
-    else if(data_type == ContractionDataType::F64_F64_F64_F64 &&
-            layout == ContractionMatrixLayout::KM_NK_MN_MN)
+    else if(data_type == ContractionDataType::F16_F16_F16_F16)
     {
-        return profile(Col{}, Col{}, Row{}, F64{});
+        if(compute_data_type == ContractionComputeDataType::F32)
+        {
+            return run_profile_for_datatype(F16{}, F32{});
+        }
+        else
+        {
+            std::cout << "Incorrect combination of data type and compute data type." << std::endl;
+            return 1;
+        }
     }
-    else
+    else if(data_type == ContractionDataType::BF16_BF16_BF16_BF16)
     {
-        std::cout << "this data_type & layout is not implemented" << std::endl;
-
-        return 1;
+        if(compute_data_type == ContractionComputeDataType::F32)
+        {
+            return run_profile_for_datatype(BF16{}, F32{});
+        }
+        else
+        {
+            std::cout << "Incorrect combination of data type and compute data type." << std::endl;
+            return 1;
+        }
     }
+    return 1;
 }
 
 REGISTER_PROFILER_OPERATION(OP_NAME, OP_DESC, profile_contraction_bilinear);

profiler/src/profile_contraction_scale.cpp

@@ -17,8 +17,9 @@
 static void print_helper_msg()
 {
     std::cout << "arg1: tensor operation (" OP_NAME ": " OP_DESC ")\n"
-              << "arg2: data type (0: fp32; 1: f64)\n"
-              << "arg3: matrix layout (0: A[m0, m1, k0, k1] * B[k0, k1, n0, n1] + "
+              << "arg2: data type (0: fp32; 1: f64; 2: f16; 3: bf16)\n"
+              << "arg3: compute data type (0: fp32; 1: f64; 2: f16; 3: bf16)\n"
+              << "arg4: matrix layout (0: A[m0, m1, k0, k1] * B[k0, k1, n0, n1] + "
                  "D[m0, m1, n0, n1] = E[m0, m1, n0, n1];\n"
               << "                     1: A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + "
                  "D[m0, m1, n0, n1] = E[m0, m1, n0, n1];\n"
@@ -26,39 +27,40 @@ static void print_helper_msg()
                  "D[m0, m1, n0, n1] = E[m0, m1, n0, n1];\n"
               << "                     3: A[k0, k1, m0, m1] * B[n0, n1, k0, k1] + "
                  "D[m0, m1, n0, n1] = E[m0, m1, n0, n1])\n"
-              << "arg4: verification (0: no; 1: yes)\n"
-              << "arg5: initialization (0: no init; 1: integer value; 2: decimal "
+              << "arg5: verification (0: no; 1: yes)\n"
+              << "arg6: initialization (0: no init; 1: integer value; 2: decimal "
               << "value)\n"
-              << "arg6: print tensor value (0: no; 1: yes)\n"
-              << "arg7: time kernel (0: no, 1: yes)\n"
-              << "arg8: alpha\n"
-              << "arg9 to 14: M0, M1, N0, N1, K0, K1\n"
-              << "arg15 to 30: Strides for A, B, D and E (skip for default)\n"
+              << "arg7: print tensor value (0: no; 1: yes)\n"
+              << "arg8: time kernel (0: no, 1: yes)\n"
+              << "arg9: alpha\n"
+              << "arg10 to 15: M0, M1, N0, N1, K0, K1\n"
+              << "arg16 to 31: Strides for A, B, D and E (skip for default)\n"
               << std::endl;
 }
 
 int profile_contraction_scale(int argc, char* argv[])
 {
-    const bool default_strides = argc == 15;
+    const bool default_strides = argc == 16;
 
-    if(argc != 31 && argc != 15)
+    if(argc != 32 && argc != 16)
     {
         print_helper_msg();
         exit(1);
     }
 
     const auto data_type          = static_cast<ContractionDataType>(std::stoi(argv[2]));
-    const auto layout             = static_cast<ContractionMatrixLayout>(std::stoi(argv[3]));
-    const bool do_verification    = std::stoi(argv[4]);
-    const ck::index_t init_method = std::stoi(argv[5]);
-    const bool do_log             = std::stoi(argv[6]);
-    const bool time_kernel        = std::stoi(argv[7]);
-    const float alpha             = std::stof(argv[8]);
+    const auto compute_data_type  = static_cast<ContractionComputeDataType>(std::stoi(argv[3]));
+    const auto layout             = static_cast<ContractionMatrixLayout>(std::stoi(argv[4]));
+    const bool do_verification    = std::stoi(argv[5]);
+    const ck::index_t init_method = std::stoi(argv[6]);
+    const bool do_log             = std::stoi(argv[7]);
+    const bool time_kernel        = std::stoi(argv[8]);
+    const float alpha             = std::stof(argv[9]);
 
     std::vector<ck::index_t> M;
     std::vector<ck::index_t> N;
     std::vector<ck::index_t> K;
-    const ck::index_t dims_arg_num = 9;
+    const ck::index_t dims_arg_num = 10;
     collect_index_params(argv, M, dims_arg_num, 2);
     collect_index_params(argv, N, dims_arg_num + 2, 2);
     collect_index_params(argv, K, dims_arg_num + 4, 2);
@@ -75,88 +77,131 @@ int profile_contraction_scale(int argc, char* argv[])
         collect_index_params(argv, StridesD, dims_arg_num + 18, 4);
     }
 
-    using F32 = float;
-    using F64 = double;
-
-    auto profile = [&](auto a_layout, auto b_layout, auto cde_layout, auto type) {
-        using ALayout   = decltype(a_layout);
-        using BLayout   = decltype(b_layout);
-        using CDELayout = decltype(cde_layout);
-
-        using DataType = decltype(type);
-
-        if(default_strides)
+    using F16  = ck::half_t;
+    using BF16 = ck::bhalf_t;
+    using F32  = float;
+    using F64  = double;
+
+    auto profile =
+        [&](auto a_layout, auto b_layout, auto cde_layout, auto type, auto compute_type) {
+            using ALayout   = decltype(a_layout);
+            using BLayout   = decltype(b_layout);
+            using CDELayout = decltype(cde_layout);
+
+            using DataType        = decltype(type);
+            using ComputeDataType = decltype(compute_type);
+
+            if(default_strides)
+            {
+                assign_default_strides(a_layout, StridesA, {M[0], M[1], K[0], K[1]});
+                assign_default_strides(b_layout, StridesB, {N[0], N[1], K[0], K[1]});
+                assign_default_strides(cde_layout, StridesE, {M[0], M[1], N[0], N[1]});
+                assign_default_strides(cde_layout, StridesD, {M[0], M[1], N[0], N[1]});
+            }
+
+            bool pass = ck::profiler::profile_contraction_impl<ALayout,
+                                                               BLayout,
+                                                               CDELayout,
+                                                               DataType,
+                                                               ComputeDataType,
+                                                               ck::Tuple<>,
+                                                               Scale>(do_verification,
+                                                                      init_method,
+                                                                      do_log,
+                                                                      time_kernel,
+                                                                      Scale{alpha},
+                                                                      M,
+                                                                      N,
+                                                                      K,
+                                                                      StridesA,
+                                                                      StridesB,
+                                                                      StridesE,
+                                                                      StridesD);
+
+            return pass;
+        };
+
+    auto run_profile_for_datatype = [&](auto type, auto compute_type) {
+        if(layout == ContractionMatrixLayout::MK_KN_MN_MN)
         {
-            assign_default_strides(a_layout, StridesA, {M[0], M[1], K[0], K[1]});
-            assign_default_strides(b_layout, StridesB, {K[0], K[1], N[0], N[1]});
-            assign_default_strides(cde_layout, StridesE, {M[0], M[1], N[0], N[1]});
-            assign_default_strides(cde_layout, StridesD, {M[0], M[1], N[0], N[1]});
+            return profile(Row{}, Row{}, Row{}, type, compute_type);
         }
-
-        bool pass = ck::profiler::
-            profile_contraction_impl<ALayout, BLayout, CDELayout, DataType, ck::Tuple<>, Scale>(
-                do_verification,
-                init_method,
-                do_log,
-                time_kernel,
-                Scale{alpha},
-                M,
-                N,
-                K,
-                StridesA,
-                StridesB,
-                StridesE,
-                StridesD);
-
-        return pass;
+        else if(layout == ContractionMatrixLayout::MK_NK_MN_MN)
+        {
+            return profile(Row{}, Col{}, Row{}, type, compute_type);
+        }
+        else if(layout == ContractionMatrixLayout::KM_KN_MN_MN)
+        {
+            return profile(Col{}, Row{}, Row{}, type, compute_type);
+        }
+        else if(layout == ContractionMatrixLayout::KM_NK_MN_MN)
+        {
+            return profile(Col{}, Col{}, Row{}, type, compute_type);
+        }
+        return false;
     };
 
-    if(data_type == ContractionDataType::F32_F32_F32_F32 &&
-       layout == ContractionMatrixLayout::MK_KN_MN_MN)
-    {
-        return profile(Row{}, Row{}, Row{}, F32{});
-    }
-    else if(data_type == ContractionDataType::F32_F32_F32_F32 &&
-            layout == ContractionMatrixLayout::MK_NK_MN_MN)
-    {
-        return profile(Row{}, Col{}, Row{}, F32{});
-    }
-    else if(data_type == ContractionDataType::F32_F32_F32_F32 &&
-            layout == ContractionMatrixLayout::KM_KN_MN_MN)
+    if(data_type == ContractionDataType::F32_F32_F32_F32)
     {
-        return profile(Col{}, Row{}, Row{}, F32{});
-    }
-    else if(data_type == ContractionDataType::F32_F32_F32_F32 &&
-            layout == ContractionMatrixLayout::KM_NK_MN_MN)
-    {
-        return profile(Col{}, Col{}, Row{}, F32{});
-    }
-    else if(data_type == ContractionDataType::F64_F64_F64_F64 &&
-            layout == ContractionMatrixLayout::MK_KN_MN_MN)
-    {
-        return profile(Row{}, Row{}, Row{}, F64{});
-    }
-    else if(data_type == ContractionDataType::F64_F64_F64_F64 &&
-            layout == ContractionMatrixLayout::MK_NK_MN_MN)
-    {
-        return profile(Row{}, Col{}, Row{}, F64{});
+        if(compute_data_type == ContractionComputeDataType::F32)
+        {
+            return run_profile_for_datatype(F32{}, F32{});
+        }
+        else if(compute_data_type == ContractionComputeDataType::F16)
+        {
+            return run_profile_for_datatype(F32{}, F16{});
+        }
+        else if(compute_data_type == ContractionComputeDataType::BF16)
+        {
+            return run_profile_for_datatype(F32{}, BF16{});
+        }
+        else
+        {
+            std::cout << "Incorrect combination of data type and compute data type." << std::endl;
+            return 1;
+        }
     }
-    else if(data_type == ContractionDataType::F64_F64_F64_F64 &&
-            layout == ContractionMatrixLayout::KM_KN_MN_MN)
+    else if(data_type == ContractionDataType::F64_F64_F64_F64)
     {
-        return profile(Col{}, Row{}, Row{}, F64{});
+        if(compute_data_type == ContractionComputeDataType::F64)
+        {
+            return run_profile_for_datatype(F64{}, F64{});
+        }
+        else if(compute_data_type == ContractionComputeDataType::F32)
+        {
+            return run_profile_for_datatype(F64{}, F32{});
+        }
+        else
+        {
+            std::cout << "Incorrect combination of data type and compute data type." << std::endl;
+            return 1;
+        }
     }
-    else if(data_type == ContractionDataType::F64_F64_F64_F64 &&
-            layout == ContractionMatrixLayout::KM_NK_MN_MN)
+    else if(data_type == ContractionDataType::F16_F16_F16_F16)
     {
-        return profile(Col{}, Col{}, Row{}, F64{});
+        if(compute_data_type == ContractionComputeDataType::F32)
+        {
+            return run_profile_for_datatype(F16{}, F32{});
+        }
+        else
+        {
+            std::cout << "Incorrect combination of data type and compute data type." << std::endl;
+            return 1;
+        }
     }
-    else
+    else if(data_type == ContractionDataType::BF16_BF16_BF16_BF16)
     {
-        std::cout << "this data_type & layout is not implemented" << std::endl;
-
-        return 1;
+        if(compute_data_type == ContractionComputeDataType::F32)
+        {
+            return run_profile_for_datatype(BF16{}, F32{});
+        }
+        else
+        {
+            std::cout << "Incorrect combination of data type and compute data type." << std::endl;
+            return 1;
+        }
     }
+    return 1;
 }
 
 REGISTER_PROFILER_OPERATION(OP_NAME, OP_DESC, profile_contraction_scale);

profiler/src/profile_grouped_gemm.cpp

@@ -27,6 +27,8 @@ enum struct GemmDataType
     F16_F16_F16,    // 1
     BF16_BF16_BF16, // 2
     INT8_INT8_INT8, // 3
+    F8_F16_F16,     // 4
+    F16_F8_F16,     // 5
 };
 
 #define OP_NAME "grouped_gemm"
@@ -56,7 +58,7 @@ int profile_grouped_gemm(int argc, char* argv[])
     {
         std::cout
             << "arg1: tensor operation (" OP_NAME ": " OP_DESC ")\n"
-            << "arg2: data type (0: fp32; 1: fp16; 2: bf16; 3: int8)\n"
+            << "arg2: data type (0: fp32; 1: fp16; 2: bf16; 3: int8; 4: fp8@fp6; 5: f16@f8)\n"
             << "arg3: matrix layout (0: A[m, k] * B[k, n] = C[m, n];\n"
             << "                     1: A[m, k] * B[n, k] = C[m, n];\n"
             << "                     2: A[k, m] * B[k, n] = C[m, n];\n"
@@ -169,6 +171,46 @@ int profile_grouped_gemm(int argc, char* argv[])
                                                                                    StrideCs,
                                                                                    kbatch);
     }
+    else if(data_type == GemmDataType::F8_F16_F16 && layout == GemmMatrixLayout::MK_KN_MN)
+    {
+        ck::profiler::profile_grouped_gemm_impl<ck::f8_t,
+                                                ck::half_t,
+                                                ck::half_t,
+                                                float,
+                                                ck::tensor_layout::gemm::RowMajor,
+                                                ck::tensor_layout::gemm::RowMajor,
+                                                ck::tensor_layout::gemm::RowMajor>(do_verification,
+                                                                                   init_method,
+                                                                                   do_log,
+                                                                                   time_kernel,
+                                                                                   Ms,
+                                                                                   Ns,
+                                                                                   Ks,
+                                                                                   StrideAs,
+                                                                                   StrideBs,
+                                                                                   StrideCs,
+                                                                                   kbatch);
+    }
+    else if(data_type == GemmDataType::F16_F8_F16 && layout == GemmMatrixLayout::MK_KN_MN)
+    {
+        ck::profiler::profile_grouped_gemm_impl<ck::half_t,
+                                                ck::f8_t,
+                                                ck::half_t,
+                                                float,
+                                                ck::tensor_layout::gemm::RowMajor,
+                                                ck::tensor_layout::gemm::RowMajor,
+                                                ck::tensor_layout::gemm::RowMajor>(do_verification,
+                                                                                   init_method,
+                                                                                   do_log,
+                                                                                   time_kernel,
+                                                                                   Ms,
+                                                                                   Ns,
+                                                                                   Ks,
+                                                                                   StrideAs,
+                                                                                   StrideBs,
+                                                                                   StrideCs,
+                                                                                   kbatch);
+    }
     else
     {
         throw std::runtime_error("wrong! this GEMM data_type & layout is not implemented");

profiler/src/profile_groupnorm.cpp → profiler/src/profile_groupnorm_fwd.cpp

@@ -6,7 +6,7 @@
 #include <unordered_map>
 
 #include "profiler/data_type_enum.hpp"
-#include "profiler/profile_groupnorm_impl.hpp"
+#include "profiler/profile_groupnorm_fwd_impl.hpp"
 #include "profiler_operation_registry.hpp"
 
 using ck::index_t;

profiler/src/profile_layernorm.cpp → profiler/src/profile_layernorm_fwd.cpp

@@ -6,7 +6,7 @@
 #include <unordered_map>
 
 #include "profiler/data_type_enum.hpp"
-#include "profiler/profile_layernorm_impl.hpp"
+#include "profiler/profile_layernorm_fwd_impl.hpp"
 #include "profiler_operation_registry.hpp"
 
 using ck::index_t;
@@ -76,19 +76,46 @@ int profile_layernorm(int argc, char* argv[])
     arg_parser(argc, argv);
     const std::vector<index_t> length = arg_parser.long_opts["length"];
 
-    using F16          = ck::half_t;
-    using F32          = float;
-    constexpr int rank = 2;
+    using F16 = ck::half_t;
+    using F32 = float;
 
-    if(data_type == ck::DataTypeEnum::Half)
+    if(length.size() == 2)
     {
-        ck::profiler::profile_layernorm_impl<F16, F16, F16, F32, F16, F32, false, rank>(
-            do_verification, init_method, do_log, time_kernel, length);
+        constexpr int rank = 2;
+
+        if(data_type == ck::DataTypeEnum::Half)
+        {
+            ck::profiler::profile_layernorm_impl<F16, F16, F16, F32, F16, F32, false, rank>(
+                do_verification, init_method, do_log, time_kernel, length);
+        }
+        else if(data_type == ck::DataTypeEnum::Float)
+        {
+            ck::profiler::profile_layernorm_impl<F32, F32, F32, F32, F32, F32, false, rank>(
+                do_verification, init_method, do_log, time_kernel, length);
+        }
+        else
+        {
+            throw std::runtime_error("not implemented yet");
+        }
     }
-    else if(data_type == ck::DataTypeEnum::Float)
+    else if(length.size() == 4)
     {
-        ck::profiler::profile_layernorm_impl<F32, F32, F32, F32, F32, F32, false, rank>(
-            do_verification, init_method, do_log, time_kernel, length);
+        constexpr int rank = 4;
+
+        if(data_type == ck::DataTypeEnum::Half)
+        {
+            ck::profiler::profile_layernorm_impl<F16, F16, F16, F32, F16, F32, false, rank>(
+                do_verification, init_method, do_log, time_kernel, length);
+        }
+        else if(data_type == ck::DataTypeEnum::Float)
+        {
+            ck::profiler::profile_layernorm_impl<F32, F32, F32, F32, F32, F32, false, rank>(
+                do_verification, init_method, do_log, time_kernel, length);
+        }
+        else
+        {
+            throw std::runtime_error("not implemented yet");
+        }
     }
     else
     {

profiler/src/profile_transpose.cpp

+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved.
+
+#include <iostream>
+#include <numeric>
+#include <initializer_list>
+#include <cstdlib>
+
+#include "profiler/profile_transpose_impl.hpp"
+#include "profiler_operation_registry.hpp"
+
+enum struct MatrixLayout
+{
+    NCDHW, // 0
+    NCHWD, // 1
+};
+
+enum struct DataType
+{
+    F32_F32_F32_F32_F32, // 0
+    F16_F16_F16_F16_F16, // 1
+};
+
+#define OP_NAME "transpose"
+#define OP_DESC "Transpose"
+
+int profile_transpose(int argc, char* argv[])
+{
+    if(argc != 15)
+    {
+        printf("arg1: tensor operation (" OP_NAME ": " OP_DESC ")\n");
+        printf("arg2: data type (0: fp32; 1: fp16)\n");
+        // printf("arg3: matrix layout (NCDHW -> NDCHW);\n");
+        printf("arg4: verification (0: no; 1: yes)\n");
+        printf("arg5: initialization (0: no init; 1: integer value; 2: decimal value)\n");
+        printf("arg6: print tensor value (0: no; 1: yes)\n");
+        printf("arg7: time kernel (0=no, 1=yes)\n");
+        printf("arg8 to 13: N, C, D, H, W\n");
+        exit(1);
+    }
+
+    const auto data_type = static_cast<DataType>(std::stoi(argv[2]));
+    // const auto layout          = static_cast<MatrixLayout>(std::stoi(argv[3]));
+    const bool do_verification   = std::stoi(argv[3]);
+    const int init_method        = std::stoi(argv[4]);
+    const bool do_log            = std::stoi(argv[5]);
+    const bool time_kernel       = std::stoi(argv[6]);
+    std::vector<index_t> lengths = std::stoi(argv[7]);
+
+    /**const int N = std::stoi(argv[7]);
+    const int C = std::stoi(argv[8]);
+    const int D = std::stoi(argv[9]);
+    const int H = std::stoi(argv[10]);
+    const int W = std::stoi(argv[11]);**/
+
+    using F32 = float;
+    using F16 = ck::half_t;
+
+    auto profile = [&](auto a_type, auto b_type) {
+        using ADataType = decltype(a_type);
+        using BDataType = decltype(b_type);
+
+        bool pass = ck::profiler::profile_transpose_impl<ADataType, BDataType>(
+            do_verification, init_method, do_log, time_kernel, lengths);
+
+        return pass ? 0 : 1;
+    };
+
+    if(data_type == GemmDataType::F32_F32_F32_F32_F32)
+    {
+        return profile(F32{}, F32{});
+    }
+    else if(data_type == GemmDataType::F16_F16_F16_F16_F16)
+    {
+        return profile(F16{}, F16{});
+    }
+    else
+    {
+        std::cout << "this data_type & layout is not implemented" << std::endl;
+
+        return 1;
+    }
+}
+
+REGISTER_PROFILER_OPERATION(OP_NAME, OP_DESC, profile_gemm_transpose);

script/cmake-ck-dev.sh

@@ -8,8 +8,7 @@ MY_PROJECT_SOURCE=$1
 cmake                                                                                             \
 -D CMAKE_PREFIX_PATH=/opt/rocm                                                                    \
 -D CMAKE_CXX_COMPILER=/opt/rocm/bin/hipcc                                                         \
--D CMAKE_CXX_FLAGS="-std=c++17 -O3 -ftemplate-backtrace-limit=0  -fPIE  -Wno-gnu-line-marker      \
--save-temps=$PWD"                                                                                 \
+-D CMAKE_CXX_FLAGS="-std=c++17 -O3 -ftemplate-backtrace-limit=0  -fPIE  -Wno-gnu-line-marker"     \
 -D CMAKE_BUILD_TYPE=Release                                                                       \
 -D BUILD_DEV=ON                                                                                   \
 -D GPU_TARGETS="gfx908;gfx90a;gfx940"                                                             \

script/hip_fatbin_insert

+SECTIONS {
+ .hipFatBinSegment : { *(.hipFatBinSegment) }
+} INSERT AFTER .bss
+
+SECTIONS {
+  .hip_fatbin : { *(.hip_fatbin) }
+} INSERT AFTER .hipFatBinSegment

test/CMakeLists.txt

@@ -139,7 +139,7 @@ add_subdirectory(grouped_convnd_fwd)
 add_subdirectory(grouped_convnd_bwd_weight)
 add_subdirectory(block_to_ctile_map)
 add_subdirectory(softmax)
-add_subdirectory(normalization)
+add_subdirectory(normalization_fwd)
 add_subdirectory(data_type)
 add_subdirectory(elementwise_normalization)
 add_subdirectory(batchnorm)
@@ -148,6 +148,7 @@ add_subdirectory(pool)
 add_subdirectory(batched_gemm_multi_d)
 add_subdirectory(grouped_convnd_bwd_data)
 add_subdirectory(conv_tensor_rearrange)
+add_subdirectory(transpose)
 if(GPU_TARGETS MATCHES "gfx11")
     add_subdirectory(wmma_op)
 endif()

test/contraction/test_contraction.cpp

@@ -10,9 +10,12 @@
 #include <gtest/gtest.h>
 
 #include "profiler/profile_contraction_impl.hpp"
+#include "profiler/profile_contraction_utils.hpp"
 
-using F32 = float;
-using F64 = double;
+using F16  = ck::half_t;
+using BF16 = ck::bhalf_t;
+using F32  = float;
+using F64  = double;
 
 using Row = ck::tensor_layout::gemm::RowMajor;
 using Col = ck::tensor_layout::gemm::ColumnMajor;
@@ -20,49 +23,49 @@ using Col = ck::tensor_layout::gemm::ColumnMajor;
 using Bilinear = ck::tensor_operation::element_wise::Bilinear;
 using Scale    = ck::tensor_operation::element_wise::Scale;
 
-struct MemoryParams
+struct Dimensions
 {
     std::vector<ck::index_t> M;
     std::vector<ck::index_t> N;
     std::vector<ck::index_t> K;
-    std::vector<ck::index_t> StridesA;
-    std::vector<ck::index_t> StridesB;
-    std::vector<ck::index_t> StridesC;
-    std::vector<ck::index_t> StridesD;
 };
 
 template <typename Tuple>
 class TestContraction : public ::testing::Test
 {
     protected:
-    using ALayout        = std::tuple_element_t<0, Tuple>;
-    using BLayout        = std::tuple_element_t<1, Tuple>;
-    using CDLayout       = std::tuple_element_t<2, Tuple>;
-    using DataType       = std::tuple_element_t<3, Tuple>;
-    using DTupleDataType = std::tuple_element_t<4, Tuple>;
-    using CDElementOp    = std::tuple_element_t<5, Tuple>;
-
-    std::vector<MemoryParams> list_of_memory_params = {{{32, 32},
-                                                        {32, 32},
-                                                        {32, 32},
-                                                        {32768, 1024, 32, 1},
-                                                        {32768, 1024, 32, 1},
-                                                        {32768, 1024, 32, 1},
-                                                        {32768, 1024, 32, 1}},
-                                                       {{16, 16},
-                                                        {32, 32},
-                                                        {16, 16},
-                                                        {4096, 256, 16, 1},
-                                                        {16, 1, 8192, 256},
-                                                        {16384, 1024, 32, 1},
-                                                        {16384, 1024, 32, 1}}};
-
-    std::vector<ck::index_t> init_methods = {0, 1, 2};
+    using ALayout         = std::tuple_element_t<0, Tuple>;
+    using BLayout         = std::tuple_element_t<1, Tuple>;
+    using CDLayout        = std::tuple_element_t<2, Tuple>;
+    using DataType        = std::tuple_element_t<3, Tuple>;
+    using DTupleDataType  = std::tuple_element_t<4, Tuple>;
+    using ComputeDataType = std::tuple_element_t<5, Tuple>;
+    using CDElementOp     = std::tuple_element_t<6, Tuple>;
+
+    std::vector<Dimensions> dimension_list = {{{32, 32}, {32, 32}, {32, 32}},
+                                              {{16, 16}, {32, 32}, {16, 16}}};
+
+    std::vector<ck::index_t> init_methods = {1, 2};
     std::unique_ptr<CDElementOp> p_cd_element_op;
+
     void Run()
     {
-        for(auto& memory_params : list_of_memory_params)
+        for(auto& dimension_params : dimension_list)
         {
+            std::vector<ck::index_t> StridesA;
+            std::vector<ck::index_t> StridesB;
+            std::vector<ck::index_t> StridesC;
+            std::vector<ck::index_t> StridesD;
+
+            const auto& M = dimension_params.M;
+            const auto& N = dimension_params.N;
+            const auto& K = dimension_params.K;
+
+            assign_default_strides(ALayout{}, StridesA, {M[0], M[1], K[0], K[1]});
+            assign_default_strides(BLayout{}, StridesB, {N[0], N[1], K[0], K[1]});
+            assign_default_strides(CDLayout{}, StridesC, {M[0], M[1], N[0], N[1]});
+            assign_default_strides(CDLayout{}, StridesD, {M[0], M[1], N[0], N[1]});
+
             for(const ck::index_t init_method : init_methods)
             {
                 bool pass =
@@ -70,19 +73,20 @@ class TestContraction : public ::testing::Test
                                                            BLayout,
                                                            CDLayout,
                                                            DataType,
+                                                           ComputeDataType,
                                                            DTupleDataType,
                                                            CDElementOp>(true /*do_verification*/,
                                                                         init_method,
                                                                         false /*do_logs*/,
                                                                         false /*time_kernel*/,
                                                                         *p_cd_element_op,
-                                                                        memory_params.M,
-                                                                        memory_params.N,
-                                                                        memory_params.K,
-                                                                        memory_params.StridesA,
-                                                                        memory_params.StridesB,
-                                                                        memory_params.StridesC,
-                                                                        memory_params.StridesD);
+                                                                        dimension_params.M,
+                                                                        dimension_params.N,
+                                                                        dimension_params.K,
+                                                                        StridesA,
+                                                                        StridesB,
+                                                                        StridesC,
+                                                                        StridesD);
                 EXPECT_TRUE(pass);
             }
         }
@@ -99,24 +103,18 @@ class TestContractionBilinear : public TestContraction<Tuple>
 {
 };
 
+#define ALL_LAYOUT_COMBINATIONS(dt, tuple_dt, compute_dt, op)    \
+    std::tuple<Row, Row, Row, dt, tuple_dt, compute_dt, op>,     \
+        std::tuple<Row, Col, Row, dt, tuple_dt, compute_dt, op>, \
+        std::tuple<Col, Row, Row, dt, tuple_dt, compute_dt, op>, \
+        std::tuple<Col, Col, Row, dt, tuple_dt, compute_dt, op>
+
 using BilinearKernelTypes =
-    ::testing::Types<std::tuple<Row, Row, Row, F32, ck::Tuple<F32>, Bilinear>,
-                     std::tuple<Row, Col, Row, F32, ck::Tuple<F32>, Bilinear>,
-                     std::tuple<Col, Row, Row, F32, ck::Tuple<F32>, Bilinear>,
-                     std::tuple<Col, Col, Row, F32, ck::Tuple<F32>, Bilinear>,
-                     std::tuple<Row, Row, Row, F64, ck::Tuple<F32>, Bilinear>,
-                     std::tuple<Row, Col, Row, F64, ck::Tuple<F32>, Bilinear>,
-                     std::tuple<Col, Row, Row, F64, ck::Tuple<F32>, Bilinear>,
-                     std::tuple<Col, Col, Row, F64, ck::Tuple<F32>, Bilinear>>;
-
-using ScaleKernelTypes = ::testing::Types<std::tuple<Row, Row, Row, F32, ck::Tuple<>, Scale>,
-                                          std::tuple<Row, Col, Row, F32, ck::Tuple<>, Scale>,
-                                          std::tuple<Col, Row, Row, F32, ck::Tuple<>, Scale>,
-                                          std::tuple<Col, Col, Row, F32, ck::Tuple<>, Scale>,
-                                          std::tuple<Row, Row, Row, F64, ck::Tuple<>, Scale>,
-                                          std::tuple<Row, Col, Row, F64, ck::Tuple<>, Scale>,
-                                          std::tuple<Col, Row, Row, F64, ck::Tuple<>, Scale>,
-                                          std::tuple<Col, Col, Row, F64, ck::Tuple<>, Scale>>;
+    ::testing::Types<ALL_LAYOUT_COMBINATIONS(F32, ck::Tuple<F32>, F32, Bilinear),
+                     ALL_LAYOUT_COMBINATIONS(F64, ck::Tuple<F64>, F64, Bilinear)>;
+
+using ScaleKernelTypes = ::testing::Types<ALL_LAYOUT_COMBINATIONS(F32, ck::Tuple<>, F32, Scale),
+                                          ALL_LAYOUT_COMBINATIONS(F64, ck::Tuple<>, F64, Scale)>;
 
 TYPED_TEST_SUITE(TestContractionBilinear, BilinearKernelTypes);
 TYPED_TEST_SUITE(TestContractionScale, ScaleKernelTypes);
@@ -136,3 +134,46 @@ TYPED_TEST(TestContractionScale, scale)
     this->p_cd_element_op = std::make_unique<Scale>(0.5f);
     this->Run();
 }
+
+template <typename Tuple>
+class TestContractionScaleMixedPrecision : public TestContraction<Tuple>
+{
+};
+
+template <typename Tuple>
+class TestContractionBilinearMixedPrecision : public TestContraction<Tuple>
+{
+};
+
+using BilinearKernelTypesMixedPrecision =
+    ::testing::Types<ALL_LAYOUT_COMBINATIONS(F32, ck::Tuple<F32>, F16, Bilinear),
+                     ALL_LAYOUT_COMBINATIONS(F32, ck::Tuple<F32>, BF16, Bilinear),
+                     ALL_LAYOUT_COMBINATIONS(F64, ck::Tuple<F64>, F32, Bilinear),
+                     ALL_LAYOUT_COMBINATIONS(F16, ck::Tuple<F16>, F32, Bilinear),
+                     ALL_LAYOUT_COMBINATIONS(BF16, ck::Tuple<BF16>, F32, Bilinear)>;
+
+using ScaleKernelTypesMixedPrecision =
+    ::testing::Types<ALL_LAYOUT_COMBINATIONS(F32, ck::Tuple<>, F16, Scale),
+                     ALL_LAYOUT_COMBINATIONS(F32, ck::Tuple<>, BF16, Scale),
+                     ALL_LAYOUT_COMBINATIONS(F64, ck::Tuple<>, F32, Scale),
+                     ALL_LAYOUT_COMBINATIONS(F16, ck::Tuple<>, F32, Scale),
+                     ALL_LAYOUT_COMBINATIONS(BF16, ck::Tuple<>, F32, Scale)>;
+
+TYPED_TEST_SUITE(TestContractionBilinearMixedPrecision, BilinearKernelTypesMixedPrecision);
+TYPED_TEST_SUITE(TestContractionScaleMixedPrecision, ScaleKernelTypesMixedPrecision);
+
+TYPED_TEST(TestContractionBilinearMixedPrecision, bilinear)
+{
+    this->p_cd_element_op = std::make_unique<Bilinear>(1.f, 1.f);
+    this->Run();
+    this->p_cd_element_op = std::make_unique<Bilinear>(-0.5f, 0.5f);
+    this->Run();
+}
+
+TYPED_TEST(TestContractionScaleMixedPrecision, scale)
+{
+    this->p_cd_element_op = std::make_unique<Scale>(1.f);
+    this->Run();
+    this->p_cd_element_op = std::make_unique<Scale>(0.5f);
+    this->Run();
+}

test/contraction/test_contraction_interface.cpp

@@ -34,11 +34,11 @@ class ContractionInstanceWrapper
     static constexpr ck::index_t NumDim = 2;
     // clang-format off
     using ContractionDeviceInstance = ck::tensor_operation::device::
-        //#####################################| NumDimM| NumDimN| NumDimK| AData| BData| AccData| CShuffle|         DsData| EData|            A|           B|          CDE|           GEMM| NumGemmK| Block|  MPer|  NPer|  KPer| AK1| BK1| MPer| NPer| MXdl| NXdl|  ABlockTransfer| ABlockTransfer| ABlockTransfer|             ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds|  BBlockTransfer| BBlockTransfer| BBlockTransfer|              BlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds|    CShuffle|    CShuffle| CBlockTransferClusterLengths|                  CBlockTransfer|
-        //#####################################|        |        |        |  Type|  Type|    Type| DataType|           Type|  Type|  Elementwise| Elementwise|  Elementwise| Spacialization| Prefetch|  Size| Block| Block| Block|    |    |  XDL|  XDL|  Per|  Per|   ThreadCluster|  ThreadCluster| SrcAccessOrder|               SrcVectorDim|      SrcScalar|      DstScalar| AddExtraM|   ThreadCluster|  ThreadCluster| SrcAccessOrder|               SrcVectorDim|      SrcScalar|      DstScalar| AddExtraN| MXdlPerWave| NXdlPerWave|         _MBlock_MWaveMPerXdl|                 ScalarPerVector|
-        //#####################################|        |        |        |      |      |        |         |               |      |    Operation|   Operation|    Operation|               |    Stage|      |      |      |      |    |    |     |     | Wave| Wave| Lengths_K0_M_K1|   ArrangeOrder|               |                           |      PerVector|   PerVector_K1|          | Lengths_K0_N_K1|   ArrangeOrder|               |                           |      PerVector|   PerVector_K1|          |  PerShuffle|  PerShuffle|         _NBlock_NWaveNPerXdl|                   _NWaveNPerXdl|
-        //#####################################|        |        |        |      |      |        |         |               |      |             |            |             |               |         |      |      |      |      |    |    |     |     |     |     |                |               |               |                           |               |               |          |                |               |               |                           |               |               |          |            |            |                             |                                |
-        DeviceContractionMultipleD_Xdl_CShuffle<  NumDim,  NumDim,  NumDim,   F32,   F32,     F32,      F32, ck::Tuple<F32>,   F32,         Pass,        Pass,     Bilinear,       GemmSpec,        1,   256,   256,   128,    16,   4,   4,   32,   32,    4,    2,     S<4, 32, 1>,     S<1, 0, 2>,     S<1, 0, 2>, ABlockTransferSrcVectorDim,              4,              4,         1,     S<4, 32, 1>,     S<1, 0, 2>,     S<1, 0, 2>, BBlockTransferSrcVectorDim,              4,              4,         1,           1,           1,              S<1, 16, 1, 16>, CDEBlockTransferScalarPerVector>;
+        //#####################################| NumDimM| NumDimN| NumDimK| AData| BData| AccData| CShuffle|         DsData| EData|           A|           B|          CDE|           GEMM| NumGemmK| Block|  MPer|  NPer|  KPer| AK1| BK1| MPer| NPer| MXdl| NXdl|  ABlockTransfer| ABlockTransfer| ABlockTransfer|             ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds|  BBlockTransfer| BBlockTransfer| BBlockTransfer|              BlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds|    CShuffle|    CShuffle| CBlockTransferClusterLengths|                  CBlockTransfer| Compute|
+        //#####################################|        |        |        |  Type|  Type|    Type| DataType|           Type|  Type| Elementwise| Elementwise|  Elementwise| Spacialization| Prefetch|  Size| Block| Block| Block|    |    |  XDL|  XDL|  Per|  Per|   ThreadCluster|  ThreadCluster| SrcAccessOrder|               SrcVectorDim|      SrcScalar|      DstScalar| AddExtraM|   ThreadCluster|  ThreadCluster| SrcAccessOrder|               SrcVectorDim|      SrcScalar|      DstScalar| AddExtraN| MXdlPerWave| NXdlPerWave|         _MBlock_MWaveMPerXdl|                 ScalarPerVector|    Data|
+        //#####################################|        |        |        |      |      |        |         |               |      |   Operation|   Operation|    Operation|               |    Stage|      |      |      |      |    |    |     |     | Wave| Wave| Lengths_K0_M_K1|   ArrangeOrder|               |                           |      PerVector|   PerVector_K1|          | Lengths_K0_N_K1|   ArrangeOrder|               |                           |      PerVector|   PerVector_K1|          |  PerShuffle|  PerShuffle|         _NBlock_NWaveNPerXdl|                   _NWaveNPerXdl|    Type|
+        //#####################################|        |        |        |      |      |        |         |               |      |            |            |             |               |         |      |      |      |      |    |    |     |     |     |     |                |               |               |                           |               |               |          |                |               |               |                           |               |               |          |            |            |                             |                                |        |
+        DeviceContractionMultipleD_Xdl_CShuffle<  NumDim,  NumDim,  NumDim,   F32,   F32,     F32,      F32, ck::Tuple<F32>,   F32,        Pass,        Pass,     Bilinear,       GemmSpec,        1,   256,   256,   128,    16,   4,   4,   32,   32,    4,    2,     S<4, 32, 1>,     S<1, 0, 2>,     S<1, 0, 2>, ABlockTransferSrcVectorDim,              4,              4,         1,     S<4, 32, 1>,     S<1, 0, 2>,     S<1, 0, 2>, BBlockTransferSrcVectorDim,              4,              4,         1,           1,           1,              S<1, 16, 1, 16>, CDEBlockTransferScalarPerVector,     F32>;
     // clang-format on
 
     bool isSupported(std::vector<ck::index_t>& ADims,

test/grouped_convnd_fwd/CMakeLists.txt

 add_gtest_executable(test_grouped_convnd_fwd test_grouped_convnd_fwd.cpp)
 target_link_libraries(test_grouped_convnd_fwd PRIVATE utility device_grouped_conv1d_fwd_instance device_grouped_conv2d_fwd_instance device_grouped_conv3d_fwd_instance)
 
+add_gtest_executable(test_grouped_convnd_fwd_multi_ab_interface test_grouped_convnd_fwd_multi_ab_interface.cpp)
+target_link_libraries(test_grouped_convnd_fwd_multi_ab_interface PRIVATE utility)
+
+add_gtest_executable(test_grouped_convnd_fwd_multi_d_interface_compatibility test_grouped_convnd_fwd_multi_d_interface_compatibility.cpp)
+target_link_libraries(test_grouped_convnd_fwd_multi_d_interface_compatibility PRIVATE utility device_grouped_conv3d_fwd_instance)

test/grouped_convnd_fwd/test_grouped_convnd_fwd_multi_ab_interface.cpp

+// SPDX-License-Identifier: MIT
+// Copyright (c) 2023, Advanced Micro Devices, Inc. All rights reserved.
+
+#include <cstdlib>
+#include <iostream>
+#include <initializer_list>
+#include <tuple>
+#include <vector>
+
+#include "ck/ck.hpp"
+#include "ck/tensor_operation/gpu/device/tensor_layout.hpp"
+#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp"
+#include "ck/tensor_operation/gpu/device/impl/device_grouped_conv_fwd_multiple_abd_xdl_cshuffle.hpp"
+
+#include "ck/host_utility/device_prop.hpp"
+
+#include "ck/library/utility/convolution_parameter.hpp"
+#include "ck/library/utility/algorithm.hpp"
+#include "ck/library/utility/convolution_host_tensor_descriptor_helper.hpp"
+
+#include <gtest/gtest.h>
+
+template <ck::index_t... Is>
+using S = ck::Sequence<Is...>;
+
+using ScaleAdd    = ck::tensor_operation::element_wise::ScaleAdd;
+using PassThrough = ck::tensor_operation::element_wise::PassThrough;
+
+template <typename DataType,
+          typename InDataTypes,
+          typename WeiDataTypes,
+          typename InElementOp,
+          typename WeiElementOp>
+class TestGroupedConvndFwdMultiABInterfaceBase : public ::testing::Test
+{
+    protected:
+    static constexpr ck::index_t NDimSpatial = 3;
+    static constexpr ck::index_t NumAs       = 2;
+    static constexpr ck::index_t NumBs       = 2;
+    static constexpr auto ConvSpec =
+        ck::tensor_operation::device::ConvolutionForwardSpecialization::Default;
+    static constexpr auto GemmSpec = ck::tensor_operation::device::GemmSpecialization::MNKPadding;
+    using InLayout                 = ck::tensor_layout::convolution::GNDHWC;
+    using WeiLayout                = ck::tensor_layout::convolution::GKZYXC;
+    using OutLayout                = ck::tensor_layout::convolution::GNDHWK;
+    using OutElementOp             = PassThrough;
+
+    using DeviceGroupedConvNDMultiABFwdInstance =
+        ck::tensor_operation::device::DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<
+            NDimSpatial,
+            InLayout,
+            WeiLayout,
+            ck::Tuple<>,
+            OutLayout,
+            InDataTypes,
+            WeiDataTypes,
+            DataType,
+            DataType,
+            ck::Tuple<>,
+            DataType,
+            InElementOp,
+            WeiElementOp,
+            OutElementOp,
+            ConvSpec,    // ConvForwardSpecialization
+            GemmSpec,    // GemmSpecialization
+            1,           //
+            256,         // BlockSize
+            128,         // MPerBlock
+            256,         // NPerBlock
+            32,          // KPerBlock
+            8,           // AK1
+            8,           // BK1
+            32,          // MPerXdl
+            32,          // NPerXdl
+            2,           // MXdlPerWave
+            4,           // NXdlPerWave
+            S<4, 64, 1>, // ABlockTransferThreadClusterLengths_AK0_M_AK1
+            S<1, 0, 2>,  // ABlockTransferThreadClusterArrangeOrder
+            S<1, 0, 2>,  // ABlockTransferSrcAccessOrder
+            2,           // ABlockTransferSrcVectorDim
+            8,           // ABlockTransferSrcScalarPerVector
+            8,           // ABlockTransferDstScalarPerVector_AK1
+            1,           // ABlockLdsExtraM
+            S<4, 64, 1>, // BBlockTransferThreadClusterLengths_BK0_N_BK1
+            S<1, 0, 2>,  // BBlockTransferThreadClusterArrangeOrder
+            S<1, 0, 2>,  // BBlockTransferSrcAccessOrder
+            2,           // BBlockTransferSrcVectorDim
+            8,           // BBlockTransferSrcScalarPerVector
+            8,           // BBlockTransferDstScalarPerVector_BK1
+            1,           // BBlockLdsExtraN
+            1,
+            1,
+            S<1, 32, 1, 8>,
+            8>;
+
+    const ck::utils::conv::ConvParam conv_param{
+        3, 1, 16, 16, 8, {3, 3, 3}, {17, 17, 17}, {2, 2, 2}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}};
+
+    void SetUp() override
+    {
+        if(!ck::is_xdl_supported())
+        {
+            GTEST_SKIP();
+        }
+    }
+
+    template <typename ADataType, typename BDataType>
+    bool Run(ADataType as, BDataType bs)
+    {
+        const auto in_g_n_c_wis_desc =
+            ck::utils::conv::make_input_host_tensor_descriptor_g_n_c_wis_packed<InLayout>(
+                conv_param);
+        const auto wei_g_k_c_xs_desc =
+            ck::utils::conv::make_weight_host_tensor_descriptor_g_k_c_xs_packed<WeiLayout>(
+                conv_param);
+        const auto out_g_n_k_wos_desc =
+            ck::utils::conv::make_output_host_tensor_descriptor_g_n_k_wos_packed<OutLayout>(
+                conv_param);
+
+        std::array<ck::index_t, NDimSpatial + 3> a_g_n_c_wis_lengths{};
+        std::array<ck::index_t, NDimSpatial + 3> a_g_n_c_wis_strides{};
+        std::array<ck::index_t, NDimSpatial + 3> b_g_k_c_xs_lengths{};
+        std::array<ck::index_t, NDimSpatial + 3> b_g_k_c_xs_strides{};
+        std::array<ck::index_t, NDimSpatial + 3> e_g_n_k_wos_lengths{};
+        std::array<ck::index_t, NDimSpatial + 3> e_g_n_k_wos_strides{};
+        std::array<ck::index_t, NDimSpatial> conv_filter_strides{};
+        std::array<ck::index_t, NDimSpatial> conv_filter_dilations{};
+        std::array<ck::index_t, NDimSpatial> input_left_pads{};
+        std::array<ck::index_t, NDimSpatial> input_right_pads{};
+
+        auto copy = [](const auto& x, auto& y) { ck::ranges::copy(x, y.begin()); };
+
+        copy(in_g_n_c_wis_desc.GetLengths(), a_g_n_c_wis_lengths);
+        copy(in_g_n_c_wis_desc.GetStrides(), a_g_n_c_wis_strides);
+        copy(wei_g_k_c_xs_desc.GetLengths(), b_g_k_c_xs_lengths);
+        copy(wei_g_k_c_xs_desc.GetStrides(), b_g_k_c_xs_strides);
+        copy(out_g_n_k_wos_desc.GetLengths(), e_g_n_k_wos_lengths);
+        copy(out_g_n_k_wos_desc.GetStrides(), e_g_n_k_wos_strides);
+        copy(conv_param.conv_filter_strides_, conv_filter_strides);
+        copy(conv_param.conv_filter_dilations_, conv_filter_dilations);
+        copy(conv_param.input_left_pads_, input_left_pads);
+        copy(conv_param.input_right_pads_, input_right_pads);
+
+        std::array<const void*, 0> ds{};
+
+        // do Conv
+        auto conv     = DeviceGroupedConvNDMultiABFwdInstance{};
+        auto invoker  = conv.MakeInvoker();
+        auto argument = conv.MakeArgument(as,
+                                          bs,
+                                          ds,
+                                          nullptr,
+                                          a_g_n_c_wis_lengths,
+                                          a_g_n_c_wis_strides,
+                                          b_g_k_c_xs_lengths,
+                                          b_g_k_c_xs_strides,
+                                          {},
+                                          {},
+                                          e_g_n_k_wos_lengths,
+                                          e_g_n_k_wos_strides,
+                                          conv_filter_strides,
+                                          conv_filter_dilations,
+                                          input_left_pads,
+                                          input_right_pads,
+                                          InElementOp{},
+                                          WeiElementOp{},
+                                          OutElementOp{});
+
+        return conv.IsSupportedArgument(argument);
+    }
+};
+
+class TestGroupedConvndFwdMultiAInterface
+    : public TestGroupedConvndFwdMultiABInterfaceBase<float,
+                                                      ck::Tuple<float, float>,
+                                                      float,
+                                                      ScaleAdd,
+                                                      PassThrough>
+{
+};
+
+class TestGroupedConvndFwdMultiBInterface
+    : public TestGroupedConvndFwdMultiABInterfaceBase<float,
+                                                      float,
+                                                      ck::Tuple<float, float>,
+                                                      PassThrough,
+                                                      ScaleAdd>
+{
+};
+
+class TestGroupedConvndFwdMultiABInterface
+    : public TestGroupedConvndFwdMultiABInterfaceBase<float,
+                                                      ck::Tuple<float, float>,
+                                                      ck::Tuple<float, float>,
+                                                      ScaleAdd,
+                                                      ScaleAdd>
+{
+};
+
+class TestGroupedConvndFwdInterface
+    : public TestGroupedConvndFwdMultiABInterfaceBase<float, float, float, PassThrough, PassThrough>
+{
+};
+
+TEST_F(TestGroupedConvndFwdMultiAInterface, MultiA)
+{
+    std::array<const void*, NumAs> as{nullptr, nullptr};
+    const void* b = nullptr;
+
+    EXPECT_TRUE(this->template Run(as, b));
+}
+
+TEST_F(TestGroupedConvndFwdMultiBInterface, MultiB)
+{
+    const void* a = nullptr;
+    std::array<const void*, NumBs> bs{nullptr, nullptr};
+
+    EXPECT_TRUE(this->template Run(a, bs));
+}
+
+TEST_F(TestGroupedConvndFwdMultiABInterface, MultiAB)
+{
+    std::array<const void*, NumAs> as{nullptr, nullptr};
+    std::array<const void*, NumBs> bs{nullptr, nullptr};
+
+    EXPECT_TRUE(this->template Run(as, bs));
+}
+
+TEST_F(TestGroupedConvndFwdInterface, SingleAB)
+{
+    const void* a = nullptr;
+    const void* b = nullptr;
+
+    EXPECT_TRUE(this->template Run(a, b));
+}

test/grouped_convnd_fwd/test_grouped_convnd_fwd_multi_d_interface_compatibility.cpp

+// SPDX-License-Identifier: MIT
+// Copyright (c) 2023, Advanced Micro Devices, Inc. All rights reserved.
+
+#include <cstdlib>
+#include <iostream>
+#include <initializer_list>
+#include <tuple>
+#include <vector>
+
+#include "ck/ck.hpp"
+#include "ck/tensor_operation/gpu/device/tensor_layout.hpp"
+#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp"
+#include "ck/tensor_operation/gpu/device/device_grouped_conv_fwd_multiple_d.hpp"
+#include "ck/library/tensor_operation_instance/gpu/grouped_convolution_forward.hpp"
+
+#include <gtest/gtest.h>
+
+template <ck::index_t... Is>
+using S = ck::Sequence<Is...>;
+
+using PassThrough = ck::tensor_operation::element_wise::PassThrough;
+
+class TestGroupedConvndFwdMultiDInterfaceCompatibility : public ::testing::Test
+{
+    protected:
+    static constexpr ck::index_t NDimSpatial = 3;
+
+    using InDataType  = float;
+    using WeiDataType = float;
+    using OutDataType = float;
+    using InLayout    = ck::tensor_layout::convolution::GNDHWC;
+    using WeiLayout   = ck::tensor_layout::convolution::GKZYXC;
+    using OutLayout   = ck::tensor_layout::convolution::GNDHWK;
+
+    using DeviceOp = ck::tensor_operation::device::DeviceGroupedConvFwdMultipleD<NDimSpatial,
+                                                                                 InLayout,
+                                                                                 WeiLayout,
+                                                                                 ck::Tuple<>,
+                                                                                 OutLayout,
+                                                                                 InDataType,
+                                                                                 WeiDataType,
+                                                                                 ck::Tuple<>,
+                                                                                 OutDataType,
+                                                                                 PassThrough,
+                                                                                 PassThrough,
+                                                                                 PassThrough>;
+
+    bool Run()
+    {
+        const auto op_ptrs = ck::tensor_operation::device::instance::DeviceOperationInstanceFactory<
+            DeviceOp>::GetInstances();
+        return op_ptrs.size() != 0;
+    }
+};
+
+TEST_F(TestGroupedConvndFwdMultiDInterfaceCompatibility, CompatibilityTest)
+{
+    EXPECT_TRUE(this->Run());
+}