merge develop

profiler/src/profile_conv_bwd_data.cpp

+#include <iostream>
+#include <numeric>
+#include <initializer_list>
+#include <cstdlib>
+#include <stdlib.h>
+#include <half.hpp>
+#include "profile_conv_bwd_data_impl.hpp"
+
+enum ConvDataType
+{
+    F32_F32_F32,    // 0
+    F16_F16_F16,    // 1
+    BF16_BF16_BF16, // 2
+    INT8_INT8_INT8, // 3
+};
+
+enum ConvInputLayout
+{
+    NCHW, // 0
+    NHWC, // 1
+};
+
+enum ConvWeightLayout
+{
+    KCYX, // 0
+    KYXC, // 1
+};
+
+enum ConvOutputLayout
+{
+    NKHW, // 0
+    NHWK, // 1
+};
+
+int profile_conv_bwd_data(int argc, char* argv[])
+{
+    if(argc != 25)
+    {
+        printf("arg1: tensor operation (conv_bwd: BackwardConvolution)\n");
+        printf("arg2: data type (0: fp32; 1: fp16)\n");
+        printf("arg3: input tensor layout (0: NCHW; 1: NHWC)\n");
+        printf("arg4: weight tensor layout (0: KCYX; 1: KYXC)\n");
+        printf("arg5: output tensor layout (0: NKHW; 1: NHWK)\n");
+        printf("arg6: verification (0: no; 1: yes)\n");
+        printf("arg7: initialization (0: no init; 1: integer value; 2: decimal value)\n");
+        printf("arg8: print tensor value (0: no; 1: yes)\n");
+        printf("arg9: run kernel # of times (>1)\n");
+        printf("arg10 to 24: N, K, C, Y, X, Hi, Wi, Sy, Sx, Dy, Dx, LeftPy, LeftPx, RightPy, "
+               "RightPx\n");
+        exit(1);
+    }
+
+    const int data_type        = static_cast<ConvDataType>(std::stoi(argv[2]));
+    const int in_layout        = static_cast<ConvInputLayout>(std::stoi(argv[3]));
+    const int wei_layout       = static_cast<ConvWeightLayout>(std::stoi(argv[4]));
+    const int out_layout       = static_cast<ConvOutputLayout>(std::stoi(argv[5]));
+    const bool do_verification = std::stoi(argv[6]);
+    const int init_method      = std::stoi(argv[7]);
+    const bool do_log          = std::stoi(argv[8]);
+    const int nrepeat          = std::stoi(argv[9]);
+
+    const ck::index_t N  = std::stoi(argv[10]);
+    const ck::index_t K  = std::stoi(argv[11]);
+    const ck::index_t C  = std::stoi(argv[12]);
+    const ck::index_t Y  = std::stoi(argv[13]);
+    const ck::index_t X  = std::stoi(argv[14]);
+    const ck::index_t Hi = std::stoi(argv[15]);
+    const ck::index_t Wi = std::stoi(argv[16]);
+
+    const ck::index_t conv_stride_h   = std::stoi(argv[17]);
+    const ck::index_t conv_stride_w   = std::stoi(argv[18]);
+    const ck::index_t conv_dilation_h = std::stoi(argv[19]);
+    const ck::index_t conv_dilation_w = std::stoi(argv[20]);
+    const ck::index_t in_left_pad_h   = std::stoi(argv[21]);
+    const ck::index_t in_left_pad_w   = std::stoi(argv[22]);
+    const ck::index_t in_right_pad_h  = std::stoi(argv[23]);
+    const ck::index_t in_right_pad_w  = std::stoi(argv[24]);
+
+    const ck::index_t YEff = (Y - 1) * conv_dilation_h + 1;
+    const ck::index_t XEff = (X - 1) * conv_dilation_w + 1;
+
+    const ck::index_t Ho = (Hi + in_left_pad_h + in_right_pad_h - YEff) / conv_stride_h + 1;
+    const ck::index_t Wo = (Wi + in_left_pad_w + in_right_pad_w - XEff) / conv_stride_w + 1;
+
+    if(data_type == ConvDataType::F32_F32_F32 && in_layout == ConvInputLayout::NHWC &&
+       wei_layout == ConvWeightLayout::KYXC && out_layout == ConvOutputLayout::NHWK)
+    {
+        ck::profiler::profile_conv_bwd_data_impl<2,
+                                                 float,
+                                                 float,
+                                                 float,
+                                                 ck::tensor_layout::convolution::NHWC,
+                                                 ck::tensor_layout::convolution::KYXC,
+                                                 ck::tensor_layout::convolution::NHWK>(
+            do_verification,
+            init_method,
+            do_log,
+            nrepeat,
+            N,
+            K,
+            C,
+            std::vector<ck::index_t>{Hi, Wi},
+            std::vector<ck::index_t>{Y, X},
+            std::vector<ck::index_t>{Ho, Wo},
+            std::vector<ck::index_t>{conv_stride_h, conv_stride_w},
+            std::vector<ck::index_t>{conv_dilation_h, conv_dilation_w},
+            std::vector<ck::index_t>{in_left_pad_h, in_left_pad_w},
+            std::vector<ck::index_t>{in_right_pad_h, in_right_pad_w});
+    }
+    else if(data_type == ConvDataType::F16_F16_F16 && in_layout == ConvInputLayout::NHWC &&
+            wei_layout == ConvWeightLayout::KYXC && out_layout == ConvOutputLayout::NHWK)
+    {
+        ck::profiler::profile_conv_bwd_data_impl<2,
+                                                 ck::half_t,
+                                                 ck::half_t,
+                                                 ck::half_t,
+                                                 ck::tensor_layout::convolution::NHWC,
+                                                 ck::tensor_layout::convolution::KYXC,
+                                                 ck::tensor_layout::convolution::NHWK>(
+            do_verification,
+            init_method,
+            do_log,
+            nrepeat,
+            N,
+            K,
+            C,
+            std::vector<ck::index_t>{Hi, Wi},
+            std::vector<ck::index_t>{Y, X},
+            std::vector<ck::index_t>{Ho, Wo},
+            std::vector<ck::index_t>{conv_stride_h, conv_stride_w},
+            std::vector<ck::index_t>{conv_dilation_h, conv_dilation_w},
+            std::vector<ck::index_t>{in_left_pad_h, in_left_pad_w},
+            std::vector<ck::index_t>{in_right_pad_h, in_right_pad_w});
+    }
+    else if(data_type == ConvDataType::BF16_BF16_BF16 && in_layout == ConvInputLayout::NHWC &&
+            wei_layout == ConvWeightLayout::KYXC && out_layout == ConvOutputLayout::NHWK)
+    {
+        ck::profiler::profile_conv_bwd_data_impl<2,
+                                                 uint16_t,
+                                                 uint16_t,
+                                                 uint16_t,
+                                                 ck::tensor_layout::convolution::NHWC,
+                                                 ck::tensor_layout::convolution::KYXC,
+                                                 ck::tensor_layout::convolution::NHWK>(
+            do_verification,
+            init_method,
+            do_log,
+            nrepeat,
+            N,
+            K,
+            C,
+            std::vector<ck::index_t>{Hi, Wi},
+            std::vector<ck::index_t>{Y, X},
+            std::vector<ck::index_t>{Ho, Wo},
+            std::vector<ck::index_t>{conv_stride_h, conv_stride_w},
+            std::vector<ck::index_t>{conv_dilation_h, conv_dilation_w},
+            std::vector<ck::index_t>{in_left_pad_h, in_left_pad_w},
+            std::vector<ck::index_t>{in_right_pad_h, in_right_pad_w});
+    }
+    else if(data_type == ConvDataType::INT8_INT8_INT8 && in_layout == ConvInputLayout::NHWC &&
+            wei_layout == ConvWeightLayout::KYXC && out_layout == ConvOutputLayout::NHWK)
+    {
+        ck::profiler::profile_conv_bwd_data_impl<2,
+                                                 int8_t,
+                                                 int8_t,
+                                                 int8_t,
+                                                 ck::tensor_layout::convolution::NHWC,
+                                                 ck::tensor_layout::convolution::KYXC,
+                                                 ck::tensor_layout::convolution::NHWK>(
+            do_verification,
+            init_method,
+            do_log,
+            nrepeat,
+            N,
+            K,
+            C,
+            std::vector<ck::index_t>{Hi, Wi},
+            std::vector<ck::index_t>{Y, X},
+            std::vector<ck::index_t>{Ho, Wo},
+            std::vector<ck::index_t>{conv_stride_h, conv_stride_w},
+            std::vector<ck::index_t>{conv_dilation_h, conv_dilation_w},
+            std::vector<ck::index_t>{in_left_pad_h, in_left_pad_w},
+            std::vector<ck::index_t>{in_right_pad_h, in_right_pad_w});
+    }
+    else
+    {
+        throw std::runtime_error("wrong! this Conv data_type & layout is not implemented");
+    }
+
+    return 1;
+}

profiler/src/profile_gemm.cpp

@@ -20,8 +20,10 @@ enum GemmMatrixLayout
 
 enum GemmDataType
 {
-    F32_F32_F32, // 0
-    F16_F16_F16, // 1
+    F32_F32_F32,    // 0
+    F16_F16_F16,    // 1
+    BF16_BF16_BF16, // 2
+    INT8_INT8_INT8, // 3
 };
 
 int profile_gemm(int argc, char* argv[])
@@ -29,7 +31,7 @@ int profile_gemm(int argc, char* argv[])
     if(!(argc == 14 || argc == 15))
     {
         printf("arg1: tensor operation (gemm: GEMM)\n");
-        printf("arg2: data type (0: fp32; 1: fp16)\n");
+        printf("arg2: data type (0: fp32; 1: fp16; 2: bf16; 3: int8)\n");
         printf("arg3: matrix layout (0: A[m, k] * B[k, n] = C[m, n];\n");
         printf("                     1: A[m, k] * B[n, k] = C[m, n];\n");
         printf("                     2: A[k, m] * B[k, n] = C[m, n];\n");
@@ -221,6 +223,46 @@ int profile_gemm(int argc, char* argv[])
             (StrideC < 0) ? N : StrideC,
             KBatch);
     }
+    else if(data_type == GemmDataType::INT8_INT8_INT8 && layout == GemmMatrixLayout::MK_NK_MN)
+    {
+        ck::profiler::profile_gemm_impl<int8_t,
+                                        int8_t,
+                                        int8_t,
+                                        ck::tensor_layout::gemm::RowMajor,
+                                        ck::tensor_layout::gemm::ColumnMajor,
+                                        ck::tensor_layout::gemm::RowMajor>(
+            do_verification,
+            init_method,
+            do_log,
+            nrepeat,
+            M,
+            N,
+            K,
+            (StrideA < 0) ? M : StrideA,
+            (StrideB < 0) ? K : StrideB,
+            (StrideC < 0) ? N : StrideC,
+            KBatch);
+    }
+    else if(data_type == GemmDataType::BF16_BF16_BF16 && layout == GemmMatrixLayout::MK_NK_MN)
+    {
+        ck::profiler::profile_gemm_impl<ck::bhalf_t,
+                                        ck::bhalf_t,
+                                        ck::bhalf_t,
+                                        ck::tensor_layout::gemm::RowMajor,
+                                        ck::tensor_layout::gemm::ColumnMajor,
+                                        ck::tensor_layout::gemm::RowMajor>(
+            do_verification,
+            init_method,
+            do_log,
+            nrepeat,
+            M,
+            N,
+            K,
+            (StrideA < 0) ? M : StrideA,
+            (StrideB < 0) ? K : StrideB,
+            (StrideC < 0) ? N : StrideC,
+            KBatch);
+    }
     else
     {
         throw std::runtime_error("wrong! this GEMM data_type & layout is not implemented");

profiler/src/profile_gemm_bias_2d.cpp

@@ -28,7 +28,7 @@ int profile_gemm_bias_2d(int argc, char* argv[])
 {
     if(!(argc == 16 || argc == 17))
     {
-        printf("arg1: tensor operation (gemm: GEMM+Bias)\n");
+        printf("arg1: tensor operation (gemm: GEMM+Bias_2d)\n");
         printf("arg2: data type (0: fp32; 1: fp16)\n");
         printf("arg3: matrix layout (0: A[m, k] * B[k, n] = C[m, n];\n");
         printf("                     1: A[m, k] * B[n, k] = C[m, n];\n");
@@ -63,11 +63,6 @@ int profile_gemm_bias_2d(int argc, char* argv[])
     const float alpha = std::stof(argv[14]);
     const float beta  = std::stof(argv[15]);
 
-    int KBatch = 1;
-
-    if(argc == 17)
-        KBatch = std::stoi(argv[16]);
-
     if(data_type == GemmDataType::F32_F32_F32 && layout == GemmMatrixLayout::MK_KN_MN)
     {
         ck::profiler::profile_gemm_bias_2d_impl<float,

profiler/src/profile_gemm_bias_relu.cpp

@@ -58,11 +58,6 @@ int profile_gemm_bias_relu(int argc, char* argv[])
     const int StrideB = std::stoi(argv[12]);
     const int StrideC = std::stoi(argv[13]);
 
-    int KBatch = 1;
-
-    if(argc == 15)
-        KBatch = std::stoi(argv[14]);
-
     if(data_type == GemmDataType::F16_F16_F16 && layout == GemmMatrixLayout::MK_KN_MN)
     {
         ck::profiler::profile_gemm_bias_relu_impl<ck::half_t,

profiler/src/profile_gemm_bias_relu_add.cpp

@@ -59,11 +59,6 @@ int profile_gemm_bias_relu_add(int argc, char* argv[])
     const int StrideC  = std::stoi(argv[13]);
     const int StrideC1 = std::stoi(argv[14]);
 
-    int KBatch = 1;
-
-    if(argc == 16)
-        KBatch = std::stoi(argv[15]);
-
     if(data_type == GemmDataType::F16_F16_F16 && layout == GemmMatrixLayout::MK_KN_MN)
     {
         ck::profiler::profile_gemm_bias_relu_add_impl<ck::half_t,

profiler/src/profile_reduce.cpp

+#include <iostream>
+#include <fstream>
+#include <numeric>
+#include <initializer_list>
+#include <cstdlib>
+#include <vector>
+#include <stdexcept>
+#include <sstream>
+#include <getopt.h>
+
+#include "config.hpp"
+#include "print.hpp"
+#include "device.hpp"
+#include "host_tensor.hpp"
+#include "host_tensor_generator.hpp"
+#include "device_tensor.hpp"
+#include "reduction_enums.hpp"
+
+#include "profile_reduce_impl.hpp"
+
+using namespace std;
+
+using ck::NanPropagation_t;
+using ck::ReduceTensorIndices_t;
+using ck::ReduceTensorOp_t;
+
+static struct option long_options[] = {{"inLengths", required_argument, nullptr, 'D'},
+                                       {"toReduceDims", required_argument, nullptr, 'R'},
+                                       {"reduceOp", required_argument, nullptr, 'O'},
+                                       {"compType", required_argument, nullptr, 'C'},
+                                       {"outType", required_argument, nullptr, 'W'},
+                                       {"nanOpt", required_argument, nullptr, 'N'},
+                                       {"indicesOpt", required_argument, nullptr, 'I'},
+                                       {"scales", required_argument, nullptr, 'S'},
+                                       {"half", no_argument, nullptr, '?'},
+                                       {"double", no_argument, nullptr, '?'},
+                                       {"dumpout", required_argument, nullptr, 'o'},
+                                       {"verify", required_argument, nullptr, 'v'},
+                                       {"log", required_argument, nullptr, 'l'},
+                                       {"help", no_argument, nullptr, '?'},
+                                       {nullptr, 0, nullptr, 0}};
+
+template <typename T>
+static T getSingleValueFromString(const string& valueStr)
+{
+    std::istringstream iss(valueStr);
+
+    T val;
+
+    iss >> val;
+
+    return (val);
+};
+
+template <typename T>
+static std::vector<T> getTypeValuesFromString(const char* cstr_values)
+{
+    std::string valuesStr(cstr_values);
+
+    std::vector<T> values;
+    std::size_t pos = 0;
+    std::size_t new_pos;
+
+    new_pos = valuesStr.find(',', pos);
+    while(new_pos != std::string::npos)
+    {
+        const std::string sliceStr = valuesStr.substr(pos, new_pos - pos);
+
+        T val = getSingleValueFromString<T>(sliceStr);
+
+        values.push_back(val);
+
+        pos     = new_pos + 1;
+        new_pos = valuesStr.find(',', pos);
+    };
+
+    std::string sliceStr = valuesStr.substr(pos);
+    T val                = getSingleValueFromString<T>(sliceStr);
+
+    values.push_back(val);
+
+    return (values);
+}
+
+typedef enum
+{
+    appHalf     = 0,
+    appFloat    = 1,
+    appInt32    = 2,
+    appInt8     = 3,
+    appInt8x4   = 4,
+    appBFloat16 = 5,
+    appDouble   = 6,
+} appDataType_t;
+
+static void check_reduce_dims(const int rank, const std::vector<int>& toReduceDims)
+{
+    for(auto dim : toReduceDims)
+    {
+        if(dim < 0 || dim >= rank)
+            throw std::runtime_error("Invalid dimension index specified for Reducing");
+    };
+
+    unsigned int flag = 0;
+
+    for(auto dim : toReduceDims)
+    {
+        if(flag & (0x1 << dim))
+            throw std::runtime_error("All toReduce dimensions should be different!");
+        flag = flag | (0x1 << dim);
+    };
+};
+
+class AppArgs
+{
+    private:
+    int option_index = 0;
+
+    public:
+    bool use_half   = false;
+    bool use_double = false;
+
+    std::vector<size_t> inLengths;
+    std::vector<size_t> outLengths;
+    std::vector<int> toReduceDims;
+
+    std::vector<float> scales;
+
+    ReduceTensorOp_t reduceOp = ReduceTensorOp_t::ADD;
+    appDataType_t compTypeId  = appFloat;
+    appDataType_t outTypeId   = appFloat;
+
+    bool compType_assigned = false;
+    bool outType_assigned  = false;
+
+    NanPropagation_t nanOpt          = NanPropagation_t::NOT_PROPAGATE_NAN;
+    ReduceTensorIndices_t indicesOpt = ReduceTensorIndices_t::NO_INDICES;
+    bool do_log                      = false;
+    bool do_verification             = false;
+    bool do_dumpout                  = false;
+
+    int init_method;
+    int nrepeat;
+
+    bool need_indices = false;
+
+    AppArgs()  = default;
+    ~AppArgs() = default;
+
+    void show_usage(const char* cmd)
+    {
+        std::cout << "Usage of " << cmd << std::endl;
+        std::cout << "--inLengths or -D, comma separated list of input tensor dimension lengths"
+                  << std::endl;
+        std::cout << "--toReduceDims or -R, comma separated list of to-reduce dimensions"
+                  << std::endl;
+        std::cout << "--reduceOp or -O, enum value indicating the reduction operations"
+                  << std::endl;
+        std::cout << "--compType or -C, enum value indicating the type of accumulated values used "
+                     "during the reduction"
+                  << std::endl;
+        std::cout << "--outType or -W, optional enum value indicating the type of the reduced "
+                     "output, which could be float when the input data is half"
+                  << std::endl;
+        std::cout << "--nanOpt or -N, enum value indicates the selection for NanOpt" << std::endl;
+        std::cout << "--indicesOpt or -I, enum value indicates the selection for IndicesOpt"
+                  << std::endl;
+        std::cout << "--scales or -S, comma separated two float values for alpha and beta"
+                  << std::endl;
+        std::cout << "--half, use fp16 for the input and output tensor data types" << std::endl;
+        std::cout << "--double, use fp64 for the input and output tensor data types" << std::endl;
+        std::cout << "--verify or -v, 1/0 to indicate whether to verify the reduction result by "
+                     "comparing with the host-based reduction"
+                  << std::endl;
+        std::cout << "--dumpout or -o, 1/0 to indicate where to save the reduction result to files "
+                     "for further analysis"
+                  << std::endl;
+        std::cout << "--log or -l, 1/0 to indicate whether to log some information" << std::endl;
+    };
+
+    int processArgs(int argc, char* argv[])
+    {
+        unsigned int ch;
+
+        optind++; // to skip the "reduce" module name
+
+        while(1)
+        {
+            ch = getopt_long(argc, argv, "D:R:O:C:W:N:I:S:v:o:l:", long_options, &option_index);
+            if(ch == -1)
+                break;
+            switch(ch)
+            {
+            case 'D':
+                if(!optarg)
+                    throw std::runtime_error("Invalid option format!");
+
+                inLengths = getTypeValuesFromString<size_t>(optarg);
+                break;
+            case 'R':
+                if(!optarg)
+                    throw std::runtime_error("Invalid option format!");
+
+                toReduceDims = getTypeValuesFromString<int>(optarg);
+                break;
+            case 'O':
+                if(!optarg)
+                    throw std::runtime_error("Invalid option format!");
+
+                reduceOp = static_cast<ReduceTensorOp_t>(std::atoi(optarg));
+                break;
+            case 'C':
+                if(!optarg)
+                    throw std::runtime_error("Invalid option format!");
+
+                compTypeId        = static_cast<appDataType_t>(std::atoi(optarg));
+                compType_assigned = true;
+                break;
+            case 'W':
+                if(!optarg)
+                    throw std::runtime_error("Invalid option format!");
+
+                outTypeId        = static_cast<appDataType_t>(std::atoi(optarg));
+                outType_assigned = true;
+                break;
+            case 'N':
+                if(!optarg)
+                    throw std::runtime_error("Invalid option format!");
+
+                nanOpt = static_cast<NanPropagation_t>(std::atoi(optarg));
+                break;
+            case 'I':
+                if(!optarg)
+                    throw std::runtime_error("Invalid option format!");
+
+                indicesOpt = static_cast<ReduceTensorIndices_t>(std::atoi(optarg));
+                break;
+            case 'S':
+                if(!optarg)
+                    throw std::runtime_error("Invalid option format!");
+
+                scales = getTypeValuesFromString<float>(optarg);
+
+                if(scales.size() != 2)
+                    throw std::runtime_error("Invalid option format!");
+                break;
+            case 'v':
+                if(!optarg)
+                    throw std::runtime_error("Invalid option format!");
+
+                do_verification = static_cast<bool>(std::atoi(optarg));
+                break;
+            case 'o':
+                if(!optarg)
+                    throw std::runtime_error("Invalid option format!");
+
+                do_dumpout = static_cast<bool>(std::atoi(optarg));
+                break;
+            case 'l':
+                if(!optarg)
+                    throw std::runtime_error("Invalid option format!");
+
+                do_log = static_cast<bool>(std::atoi(optarg));
+                break;
+            case '?':
+                if(std::string(long_options[option_index].name) == "half")
+                    use_half = true;
+                else if(std::string(long_options[option_index].name) == "double")
+                    use_double = true;
+                else if(std::string(long_options[option_index].name) == "help")
+                {
+                    show_usage(argv[0]);
+                    return (-1);
+                };
+                break;
+
+            default:
+                show_usage(argv[0]);
+                std::cerr << "Invalid cmd-line options!" << std::endl;
+                return (-1);
+            };
+        };
+
+        if(optind + 2 > argc)
+            throw std::runtime_error("Invalid cmd-line arguments, more argumetns are needed!");
+
+        init_method = std::atoi(argv[optind++]);
+        nrepeat     = std::atoi(argv[optind]);
+
+        if(scales.empty())
+        {
+            scales.push_back(1.0f);
+            scales.push_back(0.0f);
+        };
+
+        if(reduceOp == ReduceTensorOp_t::MIN || reduceOp == ReduceTensorOp_t::MAX ||
+           reduceOp == ReduceTensorOp_t::AMAX)
+        {
+            if(indicesOpt != ReduceTensorIndices_t::NO_INDICES)
+                need_indices = true;
+
+            // for indexable operations, no need to assign compType and outType, just let them be
+            // same as inType
+            compType_assigned = false;
+            outType_assigned  = false;
+        };
+
+        return (0);
+    };
+
+}; // end of class AppArgs
+
+int profile_reduce(int argc, char* argv[])
+{
+    using namespace ck::profiler;
+
+    AppArgs args;
+
+    if(args.processArgs(argc, argv) < 0)
+        return (-1);
+
+    int rank = args.inLengths.size();
+
+    check_reduce_dims(rank, args.toReduceDims);
+
+    if(args.reduceOp == ReduceTensorOp_t::MUL || args.reduceOp == ReduceTensorOp_t::NORM1)
+        throw std::runtime_error("MUL and NORM1 are not supported by composable kernel!");
+
+    if(args.use_half)
+    {
+        if(!args.compType_assigned)
+            args.compTypeId = appHalf;
+
+        if(args.outType_assigned && (args.outTypeId != appHalf && args.outTypeId != appFloat))
+            args.outTypeId = appFloat;
+
+        if(!args.outType_assigned)
+            args.outTypeId = appHalf;
+
+        if(args.compTypeId == appHalf)
+        {
+            profile_reduce_impl<ck::half_t, ck::half_t, ck::half_t>(args.do_verification,
+                                                                    args.init_method,
+                                                                    args.do_log,
+                                                                    args.do_dumpout,
+                                                                    args.nrepeat,
+                                                                    args.inLengths,
+                                                                    args.toReduceDims,
+                                                                    args.reduceOp,
+                                                                    args.nanOpt,
+                                                                    args.indicesOpt,
+                                                                    args.scales[0],
+                                                                    args.scales[1]);
+        }
+        else if(args.compTypeId == appFloat)
+        {
+            profile_reduce_impl<ck::half_t, float, ck::half_t>(args.do_verification,
+                                                               args.init_method,
+                                                               args.do_log,
+                                                               args.do_dumpout,
+                                                               args.nrepeat,
+                                                               args.inLengths,
+                                                               args.toReduceDims,
+                                                               args.reduceOp,
+                                                               args.nanOpt,
+                                                               args.indicesOpt,
+                                                               args.scales[0],
+                                                               args.scales[1]);
+        }
+        else
+            throw std::runtime_error("Invalid compType assignment!");
+    }
+    else if(args.use_double)
+    {
+        profile_reduce_impl<double, double, double>(args.do_verification,
+                                                    args.init_method,
+                                                    args.do_log,
+                                                    args.do_dumpout,
+                                                    args.nrepeat,
+                                                    args.inLengths,
+                                                    args.toReduceDims,
+                                                    args.reduceOp,
+                                                    args.nanOpt,
+                                                    args.indicesOpt,
+                                                    args.scales[0],
+                                                    args.scales[1]);
+    }
+    else
+    {
+        if(args.compTypeId == appFloat)
+        {
+            profile_reduce_impl<float, float, float>(args.do_verification,
+                                                     args.init_method,
+                                                     args.do_log,
+                                                     args.do_dumpout,
+                                                     args.nrepeat,
+                                                     args.inLengths,
+                                                     args.toReduceDims,
+                                                     args.reduceOp,
+                                                     args.nanOpt,
+                                                     args.indicesOpt,
+                                                     args.scales[0],
+                                                     args.scales[1]);
+        }
+        else if(args.compTypeId == appDouble)
+        {
+            profile_reduce_impl<float, double, float>(args.do_verification,
+                                                      args.init_method,
+                                                      args.do_log,
+                                                      args.do_dumpout,
+                                                      args.nrepeat,
+                                                      args.inLengths,
+                                                      args.toReduceDims,
+                                                      args.reduceOp,
+                                                      args.nanOpt,
+                                                      args.indicesOpt,
+                                                      args.scales[0],
+                                                      args.scales[1]);
+        }
+        else
+            throw std::runtime_error("Invalid compType assignment!");
+    };
+
+    return (0);
+};

profiler/src/profiler.cpp

@@ -2,8 +2,7 @@
 #include <numeric>
 #include <initializer_list>
 #include <cstdlib>
-#include <stdlib.h>
-#include <half.hpp>
+#include <cstring>
 
 int profile_gemm(int, char*[]);
 int profile_batched_gemm(int, char*[]);
@@ -14,6 +13,8 @@ int profile_conv_fwd(int, char*[]);
 int profile_conv_fwd_bias_relu(int, char*[]);
 int profile_conv_fwd_bias_relu_add(int, char*[]);
 int profile_conv_fwd_bias_relu_atomic_add(int, char*[]);
+int profile_conv_bwd_data(int, char*[]);
+int profile_reduce(int, char*[]);
 
 int main(int argc, char* argv[])
 {
@@ -53,6 +54,14 @@ int main(int argc, char* argv[])
     {
         return profile_conv_fwd_bias_relu_atomic_add(argc, argv);
     }
+    else if(strcmp(argv[1], "conv_bwd") == 0)
+    {
+        return profile_conv_bwd_data(argc, argv);
+    }
+    else if(strcmp(argv[1], "reduce") == 0)
+    {
+        return profile_reduce(argc, argv);
+    }
     else
     {
         // clang-format off
@@ -63,7 +72,9 @@ int main(int argc, char* argv[])
                "                        conv_fwd: ForwardConvolution\n"
                "                        conv_fwd_bias_relu: ForwardConvolution+Bias+ReLU\n"
                "                        conv_fwd_bias_relu_add: ForwardConvolution+Bias+ReLU+Add\n"
-               "                        conv_fwd_bias_relu_atomic_add: ForwardConvolution+Bias+ReLU+AtomicAdd\n");
+               "                        conv_fwd_bias_relu_atomic_add: ForwardConvolution+Bias+ReLU+AtomicAdd\n"
+               "                        conv_bwd: BackwardConvolution\n"
+               "                        reduce: REDUCE\n");
         // clang-format on
 
         return 0;

rbuild.ini

@@ -5,4 +5,4 @@ ignore = pcre
 deps =
     -f dev-requirements.txt
 define =
-    BUILD_DEV=On
\ No newline at end of file
+    BUILD_DEV=On

requirements.txt

-half,https://github.com/pfultz2/half/archive/1.12.0.tar.gz -X header -H sha256:0a08660b68abb176ebc2a0cdf8de46e3182a7f46c66443bb80dbfaaec98cf969 --build
 danmar/cppcheck@dd05839a7e63ef04afd34711cb3e1e0ef742882f

script/profile_reduce_no_index.sh

+#!/bin/bash
+
+PRECISION=   ##--half
+
+if test -n $PRECISION && test "$PRECISION" = "--half"; then 
+   CTYPE="-C 1"
+else
+   CTYPE=""
+fi
+
+WTYPE=
+
+if [ $# -ge 1 ] ; then
+    NREPEAT=$1
+else
+    NREPEAT=1
+fi
+
+Operation=7
+
+## for generic validation
+for op in $Operation; do
+    set -x
+    ./bin/ckProfiler reduce $PRECISION -D 64,4,280,82 -R 0 -O $op $CTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 4,64,280,82 -R 0 -O $op $CTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 280,4,64,82 -R 0 -O $op $CTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 64,4,280,82  -R 0,1,2 -O $op $CTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 4,64,280,82  -R 0,1,2 -O $op $CTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 64,280,82,4  -R 0,1,2 -O $op $CTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 700,8192  -R 1 -O $op $CTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 700,1024  -R 1 -O $op $CTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 700,4  -R 1 -O $op $CTYPE -v 1 1 $NREPEAT
+    set +x
+done
+
+Operation=5
+
+## for performance evaluation (resnet50 NHWC => C)
+for op in $Operation; do
+    set -x
+    ./bin/ckProfiler reduce $PRECISION -D 256,14,14,1024 -R 0,1,2 -O $op $CTYPE $WTYPE -v 1 1 $NREPEAT 
+    ./bin/ckProfiler reduce $PRECISION -D 256,28,28,128 -R 0,1,2 -O $op $CTYPE $WTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 256,58,58,128 -R 0,1,2 -O $op $CTYPE $WTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 256,7,7,2048 -R 0,1,2 -O $op $CTYPE $WTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 256,14,14,256 -R 0,1,2 -O $op $CTYPE $WTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 256,30,30,256 -R 0,1,2 -O $op $CTYPE $WTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 256,56,56,256 -R 0,1,2 -O $op $CTYPE $WTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 256,16,16,512 -R 0,1,2 -O $op $CTYPE $WTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 256,28,28,512 -R 0,1,2 -O $op $CTYPE $WTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 256,7,7,512 -R 0,1,2 -O $op $CTYPE $WTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 256,56,56,64 -R 0,1,2 -O $op $CTYPE $WTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 256,230,230,3 -R 0,1,2 -O $op $CTYPE $WTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 128,14,14,1024 -R 0,1,2 -O $op $CTYPE $WTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 128,28,28,128 -R 0,1,2 -O $op $CTYPE $WTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 128,58,58,128 -R 0,1,2 -O $op $CTYPE $WTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 128,7,7,2048 -R 0,1,2 -O $op $CTYPE $WTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 128,14,14,256 -R 0,1,2 -O $op $CTYPE $WTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 128,30,30,256 -R 0,1,2 -O $op $CTYPE $WTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 128,56,56,256 -R 0,1,2 -O $op $CTYPE $WTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 128,16,16,512 -R 0,1,2 -O $op $CTYPE $WTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 128,28,28,512 -R 0,1,2 -O $op $CTYPE $WTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 128,7,7,512 -R 0,1,2 -O $op $CTYPE $WTYPE -v 1 1 $NREPEAT
+    ./bin/ckProfiler reduce $PRECISION -D 128,56,56,64 -R 0,1,2 -O $op $CTYPE $WTYPE -v 1 1 $NREPEAT
+    set +x
+done 
+

script/profile_reduce_with_index.sh

+#!/bin/bash
+
+PRECISION=  ##--half
+
+if [ $# -ge 1 ] ; then
+    NREPEAT=$1
+else
+    NREPEAT=1
+fi
+
+Operation=4
+
+LENGTHS=64,4,280,82
+
+## for generic validation
+for op in $Operation; do
+    for use_idx in 0 1; do
+        set -x
+        ./bin/ckProfiler reduce $PRECISION -D 64,4,280,82 -R 0 -O $op $CTYPE -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 4,64,280,82 -R 0 -O $op $CTYPE -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 280,4,64,82 -R 0 -O $op $CTYPE -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 64,4,280,82  -R 0,1,2 -O $op $CTYPE -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 4,64,280,82  -R 0,1,2 -O $op $CTYPE -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 64,280,82,4  -R 0,1,2 -O $op $CTYPE -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 700,8192  -R 1 -O $op $CTYPE -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 700,1024  -R 1 -O $op $CTYPE -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 700,4  -R 1 -O $op $CTYPE -v 1 1 $NREPEAT
+        set +x
+    done
+done
+
+## for performance evaluation (resnet50 NHWC => C)
+for op in $Operation; do
+    for use_idx in 0 1; do
+        set -x
+        ./bin/ckProfiler reduce $PRECISION -D 256,14,14,1024 -R 0,1,2 -O $op -I $use_idx  -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 256,28,28,128 -R 0,1,2 -O $op -I $use_idx  -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 256,58,58,128 -R 0,1,2 -O $op -I $use_idx  -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 256,7,7,2048 -R 0,1,2 -O $op -I $use_idx  -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 256,14,14,256 -R 0,1,2 -O $op -I $use_idx  -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 256,30,30,256 -R 0,1,2 -O $op -I $use_idx  -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 256,56,56,256 -R 0,1,2 -O $op -I $use_idx  -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 256,16,16,512 -R 0,1,2 -O $op -I $use_idx  -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 256,28,28,512 -R 0,1,2 -O $op -I $use_idx  -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 256,7,7,512 -R 0,1,2 -O $op -I $use_idx  -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 256,56,56,64 -R 0,1,2 -O $op -I $use_idx  -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 256,230,230,3 -R 0,1,2 -O $op -I $use_idx  -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 128,14,14,1024 -R 0,1,2 -O $op -I $use_idx  -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 128,28,28,128 -R 0,1,2 -O $op -I $use_idx  -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 128,58,58,128 -R 0,1,2 -O $op -I $use_idx  -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 128,7,7,2048 -R 0,1,2 -O $op -I $use_idx  -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 128,14,14,256 -R 0,1,2 -O $op -I $use_idx  -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 128,30,30,256 -R 0,1,2 -O $op -I $use_idx  -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 128,56,56,256 -R 0,1,2 -O $op -I $use_idx  -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 128,16,16,512 -R 0,1,2 -O $op -I $use_idx  -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 128,28,28,512 -R 0,1,2 -O $op -I $use_idx  -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 128,7,7,512 -R 0,1,2 -O $op -I $use_idx  -v 1 1 $NREPEAT
+        ./bin/ckProfiler reduce $PRECISION -D 128,56,56,64 -R 0,1,2 -O $op -I $use_idx  -v 1 1 $NREPEAT
+        set +x
+    done
+done 
+

test/CMakeLists.txt

 include_directories(BEFORE
-    include
-    ${PROJECT_SOURCE_DIR}/host/host_tensor/include
-    ${PROJECT_SOURCE_DIR}/host/device/include
-    ${PROJECT_SOURCE_DIR}/device_operation/include
-    ${PROJECT_SOURCE_DIR}/composable_kernel/include
-    ${PROJECT_SOURCE_DIR}/composable_kernel/include/utility
-    ${PROJECT_SOURCE_DIR}/composable_kernel/include/tensor_description
-    ${PROJECT_SOURCE_DIR}/composable_kernel/include/tensor_operation
-    ${PROJECT_SOURCE_DIR}/composable_kernel/include/problem_transform
-    ${PROJECT_SOURCE_DIR}/external/rocm/include
-    ${PROJECT_SOURCE_DIR}/reference_operation/include
+    ${PROJECT_SOURCE_DIR}/include/ck
+    ${PROJECT_SOURCE_DIR}/include/ck/utility
+    ${PROJECT_SOURCE_DIR}/include/ck/tensor_description
+    ${PROJECT_SOURCE_DIR}/include/ck/tensor
+    ${PROJECT_SOURCE_DIR}/include/ck/problem_transform
+    ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/device
+    ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/grid
+    ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/block
+    ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/warp
+    ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/thread
+    ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/element
+    ${PROJECT_SOURCE_DIR}/library/include/ck/library/host_tensor
+    ${PROJECT_SOURCE_DIR}/library/include/ck/library/tensor_operation_instance
+    ${PROJECT_SOURCE_DIR}/library/include/ck/library/tensor_operation_instance/gpu/reduce
+    ${PROJECT_SOURCE_DIR}/library/include/ck/library/reference_tensor_operation/cpu
+    ${PROJECT_SOURCE_DIR}/library/include/ck/library/reference_tensor_operation/gpu
     ${PROJECT_SOURCE_DIR}/test/include
+    ${PROJECT_SOURCE_DIR}/external/include/half
 )
 
-# test_magic_number_division
-set(MAGIC_NUMBER_DIVISISON_SOURCE magic_number_division/main.cpp)
-add_executable(test_magic_number_division ${MAGIC_NUMBER_DIVISISON_SOURCE})
-target_link_libraries(test_magic_number_division PRIVATE host_tensor)
-
-
-set(CONV2D_FWD_SOURCE conv2d_fwd/main.cpp)
-
-add_executable(test_conv2d_fwd ${CONV2D_FWD_SOURCE})
-target_link_libraries(test_conv2d_fwd PRIVATE host_tensor)
-target_link_libraries(test_conv2d_fwd PRIVATE device_conv2d_fwd_instance)
-
-# test_split_k
-set(SPLIT_K_SOURCE split_k/main.cpp)
-add_executable(test_split_k ${SPLIT_K_SOURCE})
-target_link_libraries(test_split_k PRIVATE host_tensor)
-target_link_libraries(test_split_k PRIVATE device_gemm_instance)
-
-# test_conv_util
-set(CONV_UTIL_SOURCE conv_util/main.cpp)
-add_executable(test_conv_util ${CONV_UTIL_SOURCE})
-target_link_libraries(test_conv_util PRIVATE host_tensor)
-
-# test_reference_conv_fwd
-set(REFERENCE_CONV_FWD_SOURCE reference_conv_fwd/main.cpp)
-add_executable(test_reference_conv_fwd ${REFERENCE_CONV_FWD_SOURCE})
-target_link_libraries(test_reference_conv_fwd PRIVATE host_tensor)
-
-# test_convnd_fwd_xdl
-set(CONVND_FWD_XDL_SOURCE convnd_fwd_xdl/main.cpp)
-add_executable(test_convnd_fwd_xdl ${CONVND_FWD_XDL_SOURCE})
-target_link_libraries(test_convnd_fwd_xdl PRIVATE host_tensor)
-
-# test space_filling_curve_
-set(SPACE_FILLING_CURVE_SOURCE space_filling_curve/space_filling_curve.cpp)
-add_executable(space_filling_curve ${SPACE_FILLING_CURVE_SOURCE})
-target_link_libraries(space_filling_curve PRIVATE host_tensor)
+add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure -C ${CMAKE_CFG_INTDIR})
+add_custom_target(tests)
+
+function(add_test_executable TEST_NAME)
+    message("adding test ${TEST_NAME}")
+    add_executable(${TEST_NAME} ${ARGN})
+    add_test(NAME ${TEST_NAME} COMMAND $<TARGET_FILE:${TEST_NAME}> )
+    add_dependencies(tests ${TEST_NAME})
+    add_dependencies(check ${TEST_NAME})
+endfunction(add_test_executable TEST_NAME)
+
+add_subdirectory(magic_number_division)
+add_subdirectory(space_filling_curve)
+add_subdirectory(conv_util)
+add_subdirectory(reference_conv_fwd)
+add_subdirectory(gemm)
+add_subdirectory(gemm_split_k)
+add_subdirectory(conv2d_fwd)
+add_subdirectory(convnd_fwd)
+add_subdirectory(conv2d_bwd_data)

test/conv2d_bwd_data/CMakeLists.txt

+add_test_executable(test_conv2d_bwd_data conv2d_bwd_data.cpp)
+target_link_libraries(test_conv2d_bwd_data PRIVATE host_tensor)
+target_link_libraries(test_conv2d_bwd_data PRIVATE device_conv2d_bwd_data_instance)

test/conv2d_bwd_data/conv2d_bwd_data.cpp

+#include "config.hpp"
+#include "device.hpp"
+#include "host_tensor.hpp"
+#include "host_tensor_generator.hpp"
+#include "host_conv.hpp"
+#include "tensor_layout.hpp"
+#include "device_tensor.hpp"
+#include "device_conv_bwd_data.hpp"
+#include "element_wise_operation.hpp"
+#include "reference_conv_bwd_data.hpp"
+
+using F16  = ck::half_t;
+using F32  = float;
+using BF16 = ck::bhalf_t;
+using INT8 = int8_t;
+
+namespace ck {
+namespace tensor_operation {
+namespace device {
+namespace device_conv2d_bwd_data_instance {
+
+using DeviceConvBwdDataNoOpPtr =
+    DeviceConvBwdDataPtr<ck::tensor_operation::element_wise::PassThrough,
+                         ck::tensor_operation::element_wise::PassThrough,
+                         ck::tensor_operation::element_wise::PassThrough>;
+
+void add_device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_f32_instances(
+    std::vector<DeviceConvBwdDataNoOpPtr>&);
+void add_device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_f16_instances(
+    std::vector<DeviceConvBwdDataNoOpPtr>&);
+void add_device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_bf16_instances(
+    std::vector<DeviceConvBwdDataNoOpPtr>&);
+void add_device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_int8_instances(
+    std::vector<DeviceConvBwdDataNoOpPtr>&);
+
+} // namespace device_conv2d_bwd_data_instance
+} // namespace device
+} // namespace tensor_operation
+} // namespace ck
+
+using InElementOp  = ck::tensor_operation::element_wise::PassThrough;
+using WeiElementOp = ck::tensor_operation::element_wise::PassThrough;
+using OutElementOp = ck::tensor_operation::element_wise::PassThrough;
+
+template <typename T>
+static bool check_out(const Tensor<T>& ref, const Tensor<T>& result)
+{
+    float max_diff = 1e-6;
+
+    for(int i = 0; i < ref.mData.size(); ++i)
+    {
+        float diff = std::abs(double(ref.mData[i]) - double(result.mData[i]));
+        if(max_diff < diff)
+        {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+int main(int argc, char* argv[])
+{
+    int data_type   = 0;
+    int init_method = 0;
+
+    // Conv shape
+    ck::index_t N               = 128;
+    ck::index_t K               = 256;
+    ck::index_t C               = 192;
+    ck::index_t Y               = 3;
+    ck::index_t X               = 3;
+    ck::index_t Hi              = 71;
+    ck::index_t Wi              = 71;
+    ck::index_t conv_stride_h   = 2;
+    ck::index_t conv_stride_w   = 2;
+    ck::index_t conv_dilation_h = 1;
+    ck::index_t conv_dilation_w = 1;
+    ck::index_t in_left_pad_h   = 1;
+    ck::index_t in_left_pad_w   = 1;
+    ck::index_t in_right_pad_h  = 1;
+    ck::index_t in_right_pad_w  = 1;
+
+    if(argc == 1)
+    {
+        data_type   = 1;
+        init_method = 1;
+    }
+    else if(argc == 3)
+    {
+        data_type   = std::stoi(argv[1]);
+        init_method = std::stoi(argv[2]);
+    }
+    else if(argc == 18)
+    {
+        data_type   = std::stoi(argv[1]);
+        init_method = std::stoi(argv[2]);
+
+        N               = std::stoi(argv[3]);
+        K               = std::stoi(argv[4]);
+        C               = std::stoi(argv[5]);
+        Y               = std::stoi(argv[6]);
+        X               = std::stoi(argv[7]);
+        Hi              = std::stoi(argv[8]);
+        Wi              = std::stoi(argv[9]);
+        conv_stride_h   = std::stoi(argv[10]);
+        conv_stride_w   = std::stoi(argv[11]);
+        conv_dilation_h = std::stoi(argv[12]);
+        conv_dilation_w = std::stoi(argv[13]);
+        in_left_pad_h   = std::stoi(argv[14]);
+        in_left_pad_w   = std::stoi(argv[15]);
+        in_right_pad_h  = std::stoi(argv[16]);
+        in_right_pad_w  = std::stoi(argv[17]);
+    }
+    else
+    {
+        printf("arg1: data type (0=fp32, 1=fp16, 2= bfp16, 3= int8_t )\n");
+        printf("arg2: initialization (0=no init, 1=integer value, 2=decimal value)\n");
+        printf("arg3 to 17: N, K, C, Y, X, Hi, Wi, Sy, Sx, Dy, Dx, LeftPy, LeftPx, RightPy, "
+               "RightPx\n");
+        exit(1);
+    }
+
+    auto Run = [&](auto input_type, auto wei_type, auto out_type) {
+        using InDataType  = decltype(input_type);
+        using WeiDataType = decltype(wei_type);
+        using OutDataType = decltype(out_type);
+
+        using ReferenceConvBwdInstance =
+            ck::tensor_operation::host::ReferenceConvBwdData<InDataType,
+                                                             WeiDataType,
+                                                             OutDataType,
+                                                             InElementOp,
+                                                             WeiElementOp,
+                                                             OutElementOp>;
+
+        const ck::index_t YEff = (Y - 1) * conv_dilation_h + 1;
+        const ck::index_t XEff = (X - 1) * conv_dilation_w + 1;
+
+        const ck::index_t Ho = (Hi + in_left_pad_h + in_right_pad_h - YEff) / conv_stride_h + 1;
+        const ck::index_t Wo = (Wi + in_left_pad_w + in_right_pad_w - XEff) / conv_stride_w + 1;
+
+        const std::vector<ck::index_t> input_spatial_lengths{{Hi, Wi}};
+        const std::vector<ck::index_t> filter_spatial_lengths{{Y, X}};
+        const std::vector<ck::index_t> output_spatial_lengths{{Ho, Wo}};
+        const std::vector<ck::index_t> conv_filter_strides{{conv_stride_h, conv_stride_w}};
+        const std::vector<ck::index_t> conv_filter_dilations{{conv_dilation_h, conv_dilation_w}};
+        const std::vector<ck::index_t> input_left_pads{{in_left_pad_h, in_left_pad_w}};
+        const std::vector<ck::index_t> input_right_pads{{in_right_pad_h, in_right_pad_w}};
+
+        auto f_host_tensor_descriptor =
+            [](std::size_t N_, std::size_t C_, std::size_t H, std::size_t W) {
+                return HostTensorDescriptor(std::vector<std::size_t>({N_, C_, H, W}),
+                                            std::vector<std::size_t>({C_ * H * W, 1, W * C_, C_}));
+            };
+
+        Tensor<OutDataType> out_n_k_ho_wo(f_host_tensor_descriptor(N, K, Ho, Wo));
+        Tensor<WeiDataType> wei_k_c_y_x(f_host_tensor_descriptor(K, C, Y, X));
+        Tensor<InDataType> in_n_c_hi_wi_host_result(f_host_tensor_descriptor(N, C, Hi, Wi));
+        Tensor<InDataType> in_n_c_hi_wi_device_result(f_host_tensor_descriptor(N, C, Hi, Wi));
+
+        std::cout << "in_n_c_hi_wi: " << in_n_c_hi_wi_host_result.mDesc << std::endl;
+        std::cout << "wei_k_c_y_x: " << wei_k_c_y_x.mDesc << std::endl;
+        std::cout << "out_n_k_ho_wo: " << out_n_k_ho_wo.mDesc << std::endl;
+
+        switch(init_method)
+        {
+        case 0: break;
+        case 1:
+            out_n_k_ho_wo.GenerateTensorValue(GeneratorTensor_2<OutDataType>{-5, 5});
+            wei_k_c_y_x.GenerateTensorValue(GeneratorTensor_2<WeiDataType>{-5, 5});
+            break;
+        default:
+            out_n_k_ho_wo.GenerateTensorValue(GeneratorTensor_1<OutDataType>{1});
+            wei_k_c_y_x.GenerateTensorValue(GeneratorTensor_1<WeiDataType>{1});
+        }
+
+        DeviceMem in_device_buf(sizeof(InDataType) *
+                                in_n_c_hi_wi_device_result.mDesc.GetElementSpace());
+        DeviceMem wei_device_buf(sizeof(WeiDataType) * wei_k_c_y_x.mDesc.GetElementSpace());
+        DeviceMem out_device_buf(sizeof(OutDataType) * out_n_k_ho_wo.mDesc.GetElementSpace());
+
+        out_device_buf.ToDevice(out_n_k_ho_wo.mData.data());
+        wei_device_buf.ToDevice(wei_k_c_y_x.mData.data());
+
+        in_n_c_hi_wi_device_result.GenerateTensorValue(GeneratorTensor_1<InDataType>{5});
+        in_device_buf.ToDevice(in_n_c_hi_wi_device_result.mData.data());
+
+        // get host result
+        {
+            auto ref_conv    = ReferenceConvBwdInstance{};
+            auto ref_invoker = ref_conv.MakeInvoker();
+
+            auto ref_argument = ref_conv.MakeArgument(in_n_c_hi_wi_host_result,
+                                                      wei_k_c_y_x,
+                                                      out_n_k_ho_wo,
+                                                      conv_filter_strides,
+                                                      conv_filter_dilations,
+                                                      input_left_pads,
+                                                      input_right_pads,
+                                                      InElementOp{},
+                                                      WeiElementOp{},
+                                                      OutElementOp{});
+            ref_invoker.Run(ref_argument);
+        }
+
+        using PassThrough              = ck::tensor_operation::element_wise::PassThrough;
+        using DeviceConvBwdDataNoOpPtr = ck::tensor_operation::device::
+            DeviceConvBwdDataPtr<PassThrough, PassThrough, PassThrough>;
+
+        // add device Conv instances
+        std::vector<DeviceConvBwdDataNoOpPtr> conv_ptrs;
+
+        if constexpr(ck::is_same_v<ck::remove_cv_t<InDataType>, float> &&
+                     ck::is_same_v<ck::remove_cv_t<WeiDataType>, float> &&
+                     ck::is_same_v<ck::remove_cv_t<OutDataType>, float>)
+        {
+            ck::tensor_operation::device::device_conv2d_bwd_data_instance::
+                add_device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_f32_instances(conv_ptrs);
+        }
+        else if constexpr(ck::is_same_v<ck::remove_cv_t<InDataType>, ck::half_t> &&
+                          ck::is_same_v<ck::remove_cv_t<WeiDataType>, ck::half_t> &&
+                          ck::is_same_v<ck::remove_cv_t<OutDataType>, ck::half_t>)
+        {
+            ck::tensor_operation::device::device_conv2d_bwd_data_instance::
+                add_device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_f16_instances(conv_ptrs);
+        }
+        else if constexpr(ck::is_same_v<ck::remove_cv_t<InDataType>, ushort> &&
+                          ck::is_same_v<ck::remove_cv_t<WeiDataType>, ushort> &&
+                          ck::is_same_v<ck::remove_cv_t<OutDataType>, ushort>)
+        {
+            ck::tensor_operation::device::device_conv2d_bwd_data_instance::
+                add_device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_bf16_instances(conv_ptrs);
+        }
+        else if constexpr(ck::is_same_v<ck::remove_cv_t<InDataType>, int8_t> &&
+                          ck::is_same_v<ck::remove_cv_t<WeiDataType>, int8_t> &&
+                          ck::is_same_v<ck::remove_cv_t<OutDataType>, int8_t>)
+        {
+            ck::tensor_operation::device::device_conv2d_bwd_data_instance::
+                add_device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_int8_instances(conv_ptrs);
+        }
+
+        if(conv_ptrs.size() <= 0)
+        {
+            throw std::runtime_error("wrong! no device Conv instance found");
+        }
+
+        // profile device Conv instances
+        bool success = true;
+        for(auto& conv_ptr : conv_ptrs)
+        {
+            auto argument_ptr = conv_ptr->MakeArgumentPointer(
+                static_cast<InDataType*>(in_device_buf.GetDeviceBuffer()),
+                static_cast<WeiDataType*>(wei_device_buf.GetDeviceBuffer()),
+                static_cast<OutDataType*>(out_device_buf.GetDeviceBuffer()),
+                N,
+                K,
+                C,
+                input_spatial_lengths,
+                filter_spatial_lengths,
+                output_spatial_lengths,
+                conv_filter_strides,
+                conv_filter_dilations,
+                input_left_pads,
+                input_right_pads,
+                InElementOp{},
+                WeiElementOp{},
+                OutElementOp{});
+
+            if(conv_ptr->IsSupportedArgument(argument_ptr.get()))
+            {
+                auto invoker_ptr = conv_ptr->MakeInvokerPointer();
+                invoker_ptr->Run(argument_ptr.get(), 1);
+
+                in_device_buf.FromDevice(in_n_c_hi_wi_device_result.mData.data());
+
+                if(!check_out(in_n_c_hi_wi_host_result, in_n_c_hi_wi_device_result))
+                {
+                    std::cout << "Fail Info: " << conv_ptr->GetTypeString() << std::endl;
+                    success = false;
+                }
+                else
+                {
+                    std::cout << "Pass Info: " << conv_ptr->GetTypeString() << std::endl;
+                }
+            }
+            else
+            {
+                std::cout << "Not support Info: " << conv_ptr->GetTypeString() << std::endl;
+            }
+        }
+
+        if(success)
+        {
+            std::cout << "test conv2d bwd : Pass" << std::endl;
+        }
+        else
+        {
+            std::cout << "test conv2d bwd: Fail " << std::endl;
+        }
+    };
+
+    if(data_type == 0)
+    {
+        Run(F32(), F32(), F32());
+    }
+    else if(data_type == 1)
+    {
+        Run(F16(), F16(), F16());
+    }
+    else if(data_type == 2)
+    {
+        Run(BF16(), BF16(), BF16());
+    }
+    else if(data_type == 3)
+    {
+        Run(INT8(), INT8(), INT8());
+    }
+    else
+    {
+        return 1;
+    }
+
+    return 0;
+}

test/conv2d_fwd/CMakeLists.txt

+add_test_executable(test_conv2d_fwd conv2d_fwd.cpp)
+target_link_libraries(test_conv2d_fwd PRIVATE host_tensor)
+target_link_libraries(test_conv2d_fwd PRIVATE device_conv2d_fwd_instance)

test/conv2d_fwd/main.cpp → test/conv2d_fwd/conv2d_fwd.cpp

@@ -75,8 +75,12 @@ int main(int argc, char* argv[])
     ck::index_t in_left_pad_w   = 1;
     ck::index_t in_right_pad_h  = 1;
     ck::index_t in_right_pad_w  = 1;
-
-    if(argc == 3)
+    if(argc == 1)
+    {
+        data_type   = 1;
+        init_method = 1;
+    }
+    else if(argc == 3)
     {
         data_type   = std::stoi(argv[1]);
         init_method = std::stoi(argv[2]);
@@ -104,10 +108,9 @@ int main(int argc, char* argv[])
     }
     else
     {
-        printf("arg1: verification (0=no, 1=yes)\n");
+        printf("arg1: data type (0=fp32, 1=fp16, 2= bfp16, 3= int8_t )\n");
         printf("arg2: initialization (0=no init, 1=integer value, 2=decimal value)\n");
-        printf("arg3: run kernel # of times (>1)\n");
-        printf("arg4 to 18: N, K, C, Y, X, Hi, Wi, Sy, Sx, Dy, Dx, LeftPy, LeftPx, RightPy, "
+        printf("arg3 to 17: N, K, C, Y, X, Hi, Wi, Sy, Sx, Dy, Dx, LeftPy, LeftPx, RightPy, "
                "RightPx\n");
         exit(1);
     }
@@ -198,9 +201,9 @@ int main(int argc, char* argv[])
             ck::tensor_operation::device::device_conv2d_fwd_instance::
                 add_device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk_f16_instances(conv_ptrs);
         }
-        else if constexpr(ck::is_same_v<ck::remove_cv_t<InDataType>, ushort> &&
-                          ck::is_same_v<ck::remove_cv_t<WeiDataType>, ushort> &&
-                          ck::is_same_v<ck::remove_cv_t<OutDataType>, ushort>)
+        else if constexpr(ck::is_same_v<ck::remove_cv_t<InDataType>, ck::bhalf_t> &&
+                          ck::is_same_v<ck::remove_cv_t<WeiDataType>, ck::bhalf_t> &&
+                          ck::is_same_v<ck::remove_cv_t<OutDataType>, ck::bhalf_t>)
         {
             ck::tensor_operation::device::device_conv2d_fwd_instance::
                 add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instances(conv_ptrs);
@@ -275,33 +278,31 @@ int main(int argc, char* argv[])
         if(success)
         {
             std::cout << "test conv2d fwd : Pass" << std::endl;
+            return 0;
         }
         else
         {
             std::cout << "test conv2d fwd: Fail " << std::endl;
+            return -1;
         }
     };
-
+    int res = -1;
     if(data_type == 0)
     {
-        Run(float(), float(), float());
+        res = Run(float(), float(), float());
     }
     else if(data_type == 1)
     {
-        Run(ck::half_t(), ck::half_t(), ck::half_t());
+        res = Run(ck::half_t(), ck::half_t(), ck::half_t());
     }
     else if(data_type == 2)
     {
-        Run(ushort(), ushort(), ushort());
+        Run(ck::bhalf_t(), ck::bhalf_t(), ck::bhalf_t());
     }
     else if(data_type == 3)
     {
-        Run(int8_t(), int8_t(), int8_t());
-    }
-    else
-    {
-        return 1;
+        res = Run(int8_t(), int8_t(), int8_t());
     }
 
-    return 0;
+    return res;
 }

test/conv_util/CMakeLists.txt

+add_test_executable(test_conv_util conv_util.cpp)
+target_link_libraries(test_conv_util PRIVATE host_tensor)

test/convnd_fwd/CMakeLists.txt

+add_test_executable(test_convnd_fwd convnd_fwd.cpp)
+target_link_libraries(test_convnd_fwd PRIVATE host_tensor)