merge changes for upstream/latest update

profiler/include/profile_gemm_add_add_fastgelu_impl.hpp

+#pragma once
+
+#include <iomanip>
+
+#include "check_err.hpp"
+#include "config.hpp"
+#include "device.hpp"
+#include "host_tensor.hpp"
+#include "host_tensor_generator.hpp"
+#include "host_conv.hpp"
+#include "tensor_layout.hpp"
+#include "device_tensor.hpp"
+#include "element_wise_operation.hpp"
+#include "reference_gemm.hpp"
+#include "device_gemm_multiple_d.hpp"
+
+namespace ck {
+namespace tensor_operation {
+namespace device {
+namespace device_gemm_instance {
+
+using DeviceGemmAddAddFastGeluPtr = ck::tensor_operation::device::DeviceGemmMultipleDPtr<
+    2,
+    ck::tensor_operation::element_wise::PassThrough,
+    ck::tensor_operation::element_wise::PassThrough,
+    ck::tensor_operation::element_wise::AddAddFastGelu>;
+
+void add_device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_mk_kn_mn_instances(
+    std::vector<DeviceGemmAddAddFastGeluPtr>&);
+void add_device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_mk_nk_mn_instances(
+    std::vector<DeviceGemmAddAddFastGeluPtr>&);
+void add_device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_km_kn_mn_instances(
+    std::vector<DeviceGemmAddAddFastGeluPtr>&);
+void add_device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_km_nk_mn_instances(
+    std::vector<DeviceGemmAddAddFastGeluPtr>&);
+
+} // namespace device_gemm_instance
+} // namespace device
+} // namespace tensor_operation
+} // namespace ck
+
+namespace ck {
+namespace profiler {
+
+template <typename ADataType,
+          typename BDataType,
+          typename AccDataType,
+          typename D0DataType,
+          typename D1DataType,
+          typename EDataType,
+          typename ALayout,
+          typename BLayout,
+          typename D0Layout,
+          typename D1Layout,
+          typename ELayout>
+int profile_gemm_add_add_fastgelu_impl(int do_verification,
+                                       int init_method,
+                                       bool /*do_log*/,
+                                       bool time_kernel,
+                                       int M,
+                                       int N,
+                                       int K,
+                                       int StrideA,
+                                       int StrideB,
+                                       int StrideD0,
+                                       int StrideD1,
+                                       int StrideE)
+{
+    auto f_host_tensor_descriptor =
+        [](std::size_t row, std::size_t col, std::size_t stride, auto layout) {
+            if(is_same<decltype(layout), tensor_layout::gemm::RowMajor>::value)
+            {
+                return HostTensorDescriptor(std::vector<std::size_t>({row, col}),
+                                            std::vector<std::size_t>({stride, 1}));
+            }
+            else
+            {
+                return HostTensorDescriptor(std::vector<std::size_t>({row, col}),
+                                            std::vector<std::size_t>({1, stride}));
+            }
+        };
+
+    Tensor<ADataType> a_m_k(f_host_tensor_descriptor(M, K, StrideA, ALayout{}));
+    Tensor<BDataType> b_k_n(f_host_tensor_descriptor(K, N, StrideB, BLayout{}));
+    Tensor<D0DataType> d0_m_n(f_host_tensor_descriptor(M, N, StrideD0, D0Layout{}));
+    Tensor<D1DataType> d1_m_n(f_host_tensor_descriptor(M, N, StrideD1, D1Layout{}));
+    Tensor<EDataType> e_m_n_device_result(f_host_tensor_descriptor(M, N, StrideE, ELayout{}));
+    Tensor<EDataType> e_m_n_host_result(f_host_tensor_descriptor(M, N, StrideE, ELayout{}));
+
+    std::cout << "a_m_k: " << a_m_k.mDesc << std::endl;
+    std::cout << "b_k_n: " << b_k_n.mDesc << std::endl;
+    std::cout << "d0_m_n: " << d0_m_n.mDesc << std::endl;
+    std::cout << "d1_m_n: " << d1_m_n.mDesc << std::endl;
+    std::cout << "e_m_n: " << e_m_n_device_result.mDesc << std::endl;
+
+    switch(init_method)
+    {
+    case 0: break;
+    case 1:
+        a_m_k.GenerateTensorValue(GeneratorTensor_2<ADataType>{-5, 5});
+        b_k_n.GenerateTensorValue(GeneratorTensor_2<BDataType>{-5, 5});
+        d0_m_n.GenerateTensorValue(GeneratorTensor_2<D0DataType>{-5, 5});
+        d1_m_n.GenerateTensorValue(GeneratorTensor_2<D1DataType>{-5, 5});
+        break;
+    default:
+        a_m_k.GenerateTensorValue(GeneratorTensor_3<ADataType>{0.0, 1.0});
+        b_k_n.GenerateTensorValue(GeneratorTensor_3<BDataType>{-0.5, 0.5});
+        d0_m_n.GenerateTensorValue(GeneratorTensor_3<D0DataType>{0.0, 1.0});
+        d1_m_n.GenerateTensorValue(GeneratorTensor_3<D1DataType>{0.0, 1.0});
+    }
+
+    using PassThrough    = ck::tensor_operation::element_wise::PassThrough;
+    using AddAddFastGelu = ck::tensor_operation::element_wise::AddAddFastGelu;
+
+    using AElementOp   = PassThrough;
+    using BElementOp   = PassThrough;
+    using CDEElementOp = AddAddFastGelu;
+
+    const auto a_element_op   = AElementOp{};
+    const auto b_element_op   = BElementOp{};
+    const auto cde_element_op = CDEElementOp{};
+
+    // add device GEMM instances
+    std::vector<ck::tensor_operation::device::device_gemm_instance::DeviceGemmAddAddFastGeluPtr>
+        device_op_ptrs;
+
+    if constexpr(is_same_v<ADataType, half_t> && is_same_v<BDataType, half_t> &&
+                 is_same_v<EDataType, half_t>)
+    {
+        if constexpr(is_same_v<ALayout, tensor_layout::gemm::RowMajor> &&
+                     is_same_v<BLayout, tensor_layout::gemm::RowMajor> &&
+                     is_same_v<ELayout, tensor_layout::gemm::RowMajor>)
+        {
+            ck::tensor_operation::device::device_gemm_instance::
+                add_device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_mk_kn_mn_instances(
+                    device_op_ptrs);
+        }
+        else if constexpr(is_same_v<ALayout, tensor_layout::gemm::RowMajor> &&
+                          is_same_v<BLayout, tensor_layout::gemm::ColumnMajor> &&
+                          is_same_v<ELayout, tensor_layout::gemm::RowMajor>)
+        {
+            ck::tensor_operation::device::device_gemm_instance::
+                add_device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_mk_nk_mn_instances(
+                    device_op_ptrs);
+        }
+        else if constexpr(is_same_v<ALayout, tensor_layout::gemm::ColumnMajor> &&
+                          is_same_v<BLayout, tensor_layout::gemm::RowMajor> &&
+                          is_same_v<ELayout, tensor_layout::gemm::RowMajor>)
+        {
+            ck::tensor_operation::device::device_gemm_instance::
+                add_device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_km_kn_mn_instances(
+                    device_op_ptrs);
+        }
+        else if constexpr(is_same_v<ALayout, tensor_layout::gemm::ColumnMajor> &&
+                          is_same_v<BLayout, tensor_layout::gemm::ColumnMajor> &&
+                          is_same_v<ELayout, tensor_layout::gemm::RowMajor>)
+        {
+            ck::tensor_operation::device::device_gemm_instance::
+                add_device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_km_nk_mn_instances(
+                    device_op_ptrs);
+        }
+    }
+
+    std::cout << "found " << device_op_ptrs.size() << " instances" << std::endl;
+
+    // run reference
+    if(do_verification)
+    {
+        Tensor<AccDataType> c_m_n(HostTensorDescriptor(
+            std::vector<std::size_t>{static_cast<std::size_t>(M), static_cast<std::size_t>(N)}));
+
+        using ReferenceGemmInstance = ck::tensor_operation::host::ReferenceGemm<ADataType,
+                                                                                BDataType,
+                                                                                AccDataType,
+                                                                                AccDataType,
+                                                                                AElementOp,
+                                                                                BElementOp,
+                                                                                PassThrough>;
+
+        auto ref_gemm    = ReferenceGemmInstance{};
+        auto ref_invoker = ref_gemm.MakeInvoker();
+
+        auto ref_argument =
+            ref_gemm.MakeArgument(a_m_k, b_k_n, c_m_n, a_element_op, b_element_op, PassThrough{});
+
+        ref_invoker.Run(ref_argument);
+
+        for(int m = 0; m < M; ++m)
+        {
+            for(int n = 0; n < N; ++n)
+            {
+                cde_element_op(e_m_n_host_result(m, n), c_m_n(m, n), d0_m_n(m, n), d1_m_n(m, n));
+            }
+        }
+    }
+
+    DeviceMem a_device_buf(sizeof(ADataType) * a_m_k.mDesc.GetElementSpace());
+    DeviceMem b_device_buf(sizeof(BDataType) * b_k_n.mDesc.GetElementSpace());
+    DeviceMem d0_m_n_device_buf(sizeof(D0DataType) * d0_m_n.mDesc.GetElementSpace());
+    DeviceMem d1_m_n_device_buf(sizeof(D1DataType) * d1_m_n.mDesc.GetElementSpace());
+    DeviceMem e_device_buf(sizeof(EDataType) * e_m_n_device_result.mDesc.GetElementSpace());
+
+    a_device_buf.ToDevice(a_m_k.mData.data());
+    b_device_buf.ToDevice(b_k_n.mData.data());
+    d0_m_n_device_buf.ToDevice(d0_m_n.mData.data());
+    d1_m_n_device_buf.ToDevice(d1_m_n.mData.data());
+
+    std::string best_device_op_name;
+    float best_ave_time   = 0;
+    float best_tflops     = 0;
+    float best_gb_per_sec = 0;
+
+    bool pass = true;
+
+    // profile device operation instances
+    for(auto& device_op_ptr : device_op_ptrs)
+    {
+        auto argument_ptr = device_op_ptr->MakeArgumentPointer(
+            a_device_buf.GetDeviceBuffer(),
+            b_device_buf.GetDeviceBuffer(),
+            std::array<const void*, 2>{d0_m_n_device_buf.GetDeviceBuffer(),
+                                       d1_m_n_device_buf.GetDeviceBuffer()},
+            static_cast<EDataType*>(e_device_buf.GetDeviceBuffer()),
+            M,
+            N,
+            K,
+            StrideA,
+            StrideB,
+            std::array<ck::index_t, 2>{StrideD0, StrideD1},
+            StrideE,
+            a_element_op,
+            b_element_op,
+            cde_element_op);
+
+        auto invoker_ptr = device_op_ptr->MakeInvokerPointer();
+
+        std::string device_op_name = device_op_ptr->GetTypeString();
+
+        if(device_op_ptr->IsSupportedArgument(argument_ptr.get()))
+        {
+            // re-init E to zero before profiling a kernel
+            e_device_buf.SetZero();
+
+            float ave_time =
+                invoker_ptr->Run(argument_ptr.get(), StreamConfig{nullptr, time_kernel});
+
+            std::size_t flop = std::size_t(2) * M * N * K;
+
+            std::size_t num_btype =
+                sizeof(ADataType) * M * K + sizeof(BDataType) * K * N + sizeof(EDataType) * M * N;
+
+            float tflops = static_cast<float>(flop) / 1.E9 / ave_time;
+
+            float gb_per_sec = num_btype / 1.E6 / ave_time;
+
+            std::cout << "Perf: " << std::setw(10) << ave_time << " ms, " << tflops << " TFlops, "
+                      << gb_per_sec << " GB/s, " << device_op_name << std::endl;
+
+            if(tflops > best_tflops)
+            {
+                best_device_op_name = device_op_name;
+                best_tflops         = tflops;
+                best_ave_time       = ave_time;
+                best_gb_per_sec     = gb_per_sec;
+            }
+
+            if(do_verification)
+            {
+                e_device_buf.FromDevice(e_m_n_device_result.mData.data());
+
+                pass = pass &&
+                       ck::utils::check_err(e_m_n_device_result.mData, e_m_n_host_result.mData);
+            }
+        }
+        else
+        {
+            std::cout << device_op_name << " does not support this problem" << std::endl;
+        }
+    }
+
+    std::cout << "Best Perf: " << best_ave_time << " ms, " << best_tflops << " TFlops, "
+              << best_gb_per_sec << " GB/s, " << best_device_op_name << std::endl;
+
+    return pass ? 0 : 1;
+}
+
+} // namespace profiler
+} // namespace ck

profiler/include/profile_gemm_bias_add_reduce_impl.hpp

+#pragma once
+#include "check_err.hpp"
+#include "config.hpp"
+#include "device.hpp"
+#include "host_tensor.hpp"
+#include "host_tensor_generator.hpp"
+#include "host_conv.hpp"
+#include "tensor_layout.hpp"
+#include "device_tensor.hpp"
+#include "element_wise_operation.hpp"
+#include "reduction_operator.hpp"
+#include "device_gemm_reduce.hpp"
+#include "reference_gemm.hpp"
+
+namespace ck {
+namespace tensor_operation {
+namespace device {
+namespace device_gemm_instance {
+
+using F32            = float;
+using F16            = ck::half_t;
+using DPtrsGlobal    = ck::Tuple<F32*, F32*>;
+using Div            = ck::tensor_operation::element_wise::UnaryDivide;
+using Identity       = ck::tensor_operation::element_wise::PassThrough;
+using Square         = ck::tensor_operation::element_wise::UnarySquare;
+using DInElementOps  = ck::Tuple<Identity, Square>;
+using DOutElementOps = ck::Tuple<Div, Div>;
+
+using DeviceGemmBiasAddReduceNoOpPtr = ck::tensor_operation::device::DeviceGemmBiasAddReducePtr<
+    ck::tensor_operation::element_wise::PassThrough,
+    ck::tensor_operation::element_wise::PassThrough,
+    ck::tensor_operation::element_wise::PassThrough,
+    ck::tensor_operation::element_wise::PassThrough,
+    DInElementOps,
+    DOutElementOps>;
+
+void add_device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f16_f16_f32_f32_mk_kn_mn_instances(
+    std::vector<DeviceGemmBiasAddReduceNoOpPtr>&);
+
+void add_device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f16_f16_f32_f32_mk_nk_mn_instances(
+    std::vector<DeviceGemmBiasAddReduceNoOpPtr>&);
+
+void add_device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f16_f16_f32_f32_km_kn_mn_instances(
+    std::vector<DeviceGemmBiasAddReduceNoOpPtr>&);
+
+void add_device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f16_f16_f32_f32_km_nk_mn_instances(
+    std::vector<DeviceGemmBiasAddReduceNoOpPtr>&);
+
+} // namespace device_gemm_instance
+} // namespace device
+} // namespace tensor_operation
+} // namespace ck
+
+namespace ck {
+namespace profiler {
+
+template <typename ADataType,
+          typename BDataType,
+          typename CDataType,
+          typename C0DataType,
+          typename C1DataType,
+          typename DDataType,
+          typename ALayout,
+          typename BLayout,
+          typename CLayout>
+void profile_gemm_bias_add_reduce_impl(int do_verification,
+                                       int init_method,
+                                       bool do_log,
+                                       bool time_kernel,
+                                       int M,
+                                       int N,
+                                       int K,
+                                       int StrideA,
+                                       int StrideB,
+                                       int StrideC,
+                                       int StrideC1)
+{
+    auto f_host_tensor_descriptor1d = [](std::size_t len, std::size_t stride) {
+        return HostTensorDescriptor(std::vector<std::size_t>({len}),
+                                    std::vector<std::size_t>({stride}));
+    };
+
+    auto f_host_tensor_descriptor2d =
+        [](std::size_t row, std::size_t col, std::size_t stride, auto layout) {
+            if(is_same<decltype(layout), tensor_layout::gemm::RowMajor>::value)
+            {
+                return HostTensorDescriptor(std::vector<std::size_t>({row, col}),
+                                            std::vector<std::size_t>({stride, 1}));
+            }
+            else
+            {
+                return HostTensorDescriptor(std::vector<std::size_t>({row, col}),
+                                            std::vector<std::size_t>({1, stride}));
+            }
+        };
+
+    Tensor<ADataType> a_m_k(f_host_tensor_descriptor2d(M, K, StrideA, ALayout{}));
+    Tensor<BDataType> b_k_n(f_host_tensor_descriptor2d(K, N, StrideB, BLayout{}));
+
+    Tensor<CDataType> c_m_n_host_result(f_host_tensor_descriptor2d(M, N, StrideC, CLayout{}));
+    Tensor<C0DataType> bias_n(f_host_tensor_descriptor1d(N, 1));
+    Tensor<C1DataType> c1_m_n(f_host_tensor_descriptor2d(M, N, StrideC, CLayout{}));
+    Tensor<DDataType> d0_m_host_result(
+        HostTensorDescriptor(std::vector<std::size_t>({static_cast<std::size_t>(M)})));
+    Tensor<DDataType> d1_m_host_result(
+        HostTensorDescriptor(std::vector<std::size_t>({static_cast<std::size_t>(M)})));
+
+    Tensor<CDataType> c_m_n_device_result(f_host_tensor_descriptor2d(M, N, StrideC, CLayout{}));
+    Tensor<DDataType> d0_m_device_result(
+        HostTensorDescriptor(std::vector<std::size_t>({static_cast<std::size_t>(M)})));
+    Tensor<DDataType> d1_m_device_result(
+        HostTensorDescriptor(std::vector<std::size_t>({static_cast<std::size_t>(M)})));
+
+    std::cout << "a_m_k: " << a_m_k.mDesc << std::endl;
+    std::cout << "b_k_n: " << b_k_n.mDesc << std::endl;
+    std::cout << "c_m_n: " << c_m_n_host_result.mDesc << std::endl;
+    std::cout << "d0_m: " << d0_m_host_result.mDesc << std::endl;
+    std::cout << "d1_m: " << d1_m_host_result.mDesc << std::endl;
+
+    std::size_t num_thread = 1;
+    switch(init_method)
+    {
+    case 0: break;
+    case 1:
+        std::srand(0);
+        a_m_k.GenerateTensorValue(GeneratorTensor_2<ADataType>{-5, 5}, num_thread);
+        b_k_n.GenerateTensorValue(GeneratorTensor_2<BDataType>{-5, 5}, num_thread);
+        bias_n.GenerateTensorValue(GeneratorTensor_2<BDataType>{-5, 5}, num_thread);
+        c1_m_n.GenerateTensorValue(GeneratorTensor_2<BDataType>{-5, 5}, num_thread);
+        break;
+    default:
+        std::srand(0);
+        a_m_k.GenerateTensorValue(GeneratorTensor_3<ADataType>{0.0, 1.0}, num_thread);
+        b_k_n.GenerateTensorValue(GeneratorTensor_3<BDataType>{-0.5, 0.5}, num_thread);
+        bias_n.GenerateTensorValue(GeneratorTensor_3<ADataType>{-0.5, 0.5}, num_thread);
+        c1_m_n.GenerateTensorValue(GeneratorTensor_3<BDataType>{-0.5, 0.5}, num_thread);
+    }
+
+    using PassThrough           = ck::tensor_operation::element_wise::PassThrough;
+    using AElementOp            = PassThrough;
+    using BElementOp            = PassThrough;
+    using CElementOp            = PassThrough;
+    using C1ElementOp           = PassThrough;
+    using D0ReduceOp            = ck::reduce::Add;
+    using D1ReduceOp            = ck::reduce::Add;
+    using UnaryDivElementOp     = ck::tensor_operation::element_wise::UnaryDivide;
+    using UnaryIdenticElementOp = ck::tensor_operation::element_wise::PassThrough;
+    using UnarySquareElementOp  = ck::tensor_operation::element_wise::UnarySquare;
+    using DxsInElementOps       = ck::Tuple<UnaryIdenticElementOp, UnarySquareElementOp>;
+    using DxsOutElementOps      = ck::Tuple<UnaryDivElementOp, UnaryDivElementOp>;
+
+    const auto a_element_op  = AElementOp{};
+    const auto b_element_op  = BElementOp{};
+    const auto c_element_op  = CElementOp{};
+    const auto c1_element_op = C1ElementOp{};
+    const auto d0_reduce_op  = D0ReduceOp{};
+    const auto d1_reduce_op  = D1ReduceOp{};
+
+    auto dxs_in_element_op  = DxsInElementOps{};
+    auto dxs_out_element_op = DxsOutElementOps{N, N};
+
+    if(do_verification)
+    {
+        using ReferenceGemmInstance = ck::tensor_operation::host::ReferenceGemm<ADataType,
+                                                                                BDataType,
+                                                                                CDataType,
+                                                                                DDataType,
+                                                                                AElementOp,
+                                                                                BElementOp,
+                                                                                CElementOp>;
+
+        using ReduceAccDataType = DDataType;
+
+        auto ref_gemm    = ReferenceGemmInstance{};
+        auto ref_invoker = ref_gemm.MakeInvoker();
+
+        auto ref_argument = ref_gemm.MakeArgument(
+            a_m_k, b_k_n, c_m_n_host_result, a_element_op, b_element_op, PassThrough{});
+
+        ref_invoker.Run(ref_argument);
+
+        for(int m = 0; m < M; ++m)
+            for(int n = 0; n < N; ++n)
+            {
+                ReduceAccDataType acc = static_cast<ReduceAccDataType>(c_m_n_host_result(m, n)) +
+                                        static_cast<ReduceAccDataType>(bias_n(n));
+
+                ReduceAccDataType c1 = static_cast<ReduceAccDataType>(c1_m_n(m, n));
+                c_element_op(acc, acc);
+                c1_element_op(c1, c1);
+                acc += c1;
+                c_m_n_host_result(m, n) = static_cast<CDataType>(acc);
+            }
+
+        for(int m = 0; m < M; ++m)
+        {
+            auto d0_acc = d0_reduce_op.GetIdentityValue<ReduceAccDataType>();
+            auto d1_acc = d1_reduce_op.GetIdentityValue<ReduceAccDataType>();
+
+            for(int n = 0; n < N; ++n)
+            {
+                ReduceAccDataType c_val =
+                    ck::type_convert<ReduceAccDataType>(c_m_n_host_result(m, n));
+                ReduceAccDataType d0_val;
+                ReduceAccDataType d1_val;
+
+                dxs_in_element_op(ck::Number<0>{})(d0_val, c_val);
+                dxs_in_element_op(ck::Number<1>{})(d1_val, c_val);
+                d0_reduce_op(d0_acc, d0_val);
+                d1_reduce_op(d1_acc, d1_val);
+            }
+
+            dxs_out_element_op(ck::Number<0>{})(d0_acc, d0_acc);
+            dxs_out_element_op(ck::Number<1>{})(d1_acc, d1_acc);
+            d0_m_host_result(m) = ck::type_convert<DDataType>(d0_acc);
+            d1_m_host_result(m) = ck::type_convert<DDataType>(d1_acc);
+        }
+    }
+
+    DeviceMem a_device_buf(sizeof(ADataType) * a_m_k.mDesc.GetElementSpace());
+    DeviceMem b_device_buf(sizeof(BDataType) * b_k_n.mDesc.GetElementSpace());
+    DeviceMem c_device_buf(sizeof(CDataType) * c_m_n_device_result.mDesc.GetElementSpace());
+    DeviceMem bias_device_buf(sizeof(C0DataType) * bias_n.mDesc.GetElementSpace());
+    DeviceMem c1_device_buf(sizeof(C1DataType) * c1_m_n.mDesc.GetElementSpace());
+    DeviceMem d0_device_buf(sizeof(DDataType) * d0_m_device_result.mDesc.GetElementSpace());
+    DeviceMem d1_device_buf(sizeof(DDataType) * d1_m_device_result.mDesc.GetElementSpace());
+
+    auto dxs_global = ck::make_tuple(static_cast<DDataType*>(d0_device_buf.GetDeviceBuffer()),
+                                     static_cast<DDataType*>(d1_device_buf.GetDeviceBuffer()));
+
+    a_device_buf.ToDevice(a_m_k.mData.data());
+    b_device_buf.ToDevice(b_k_n.mData.data());
+    bias_device_buf.ToDevice(bias_n.mData.data());
+    c1_device_buf.ToDevice(c1_m_n.mData.data());
+
+    // add device GEMM instances
+    std::vector<ck::tensor_operation::device::device_gemm_instance::DeviceGemmBiasAddReduceNoOpPtr>
+        gemm_ptrs;
+
+    if constexpr(is_same<ADataType, half_t>::value && is_same<BDataType, half_t>::value &&
+                 is_same<CDataType, half_t>::value)
+    {
+        if constexpr(is_same<ALayout, tensor_layout::gemm::RowMajor>::value &&
+                     is_same<BLayout, tensor_layout::gemm::RowMajor>::value &&
+                     is_same<CLayout, tensor_layout::gemm::RowMajor>::value)
+        {
+            ck::tensor_operation::device::device_gemm_instance::
+                add_device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f16_f16_f32_f32_mk_kn_mn_instances(
+                    gemm_ptrs);
+        }
+        else if constexpr(is_same<ALayout, tensor_layout::gemm::RowMajor>::value &&
+                          is_same<BLayout, tensor_layout::gemm::ColumnMajor>::value &&
+                          is_same<CLayout, tensor_layout::gemm::RowMajor>::value)
+        {
+            ck::tensor_operation::device::device_gemm_instance::
+                add_device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f16_f16_f32_f32_mk_nk_mn_instances(
+                    gemm_ptrs);
+        }
+        else if constexpr(is_same<ALayout, tensor_layout::gemm::ColumnMajor>::value &&
+                          is_same<BLayout, tensor_layout::gemm::RowMajor>::value &&
+                          is_same<CLayout, tensor_layout::gemm::RowMajor>::value)
+        {
+            ck::tensor_operation::device::device_gemm_instance::
+                add_device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f16_f16_f32_f32_km_kn_mn_instances(
+                    gemm_ptrs);
+        }
+        else if constexpr(is_same<ALayout, tensor_layout::gemm::ColumnMajor>::value &&
+                          is_same<BLayout, tensor_layout::gemm::ColumnMajor>::value &&
+                          is_same<CLayout, tensor_layout::gemm::RowMajor>::value)
+        {
+            ck::tensor_operation::device::device_gemm_instance::
+                add_device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f16_f16_f32_f32_km_nk_mn_instances(
+                    gemm_ptrs);
+        }
+    }
+
+    if(gemm_ptrs.size() <= 0)
+    {
+        throw std::runtime_error("wrong! no device GEMM instance found");
+    }
+
+    std::string best_gemm_name;
+    float best_ave_time   = 0;
+    float best_tflops     = 0;
+    float best_gb_per_sec = 0;
+
+    // profile device GEMM instances
+    for(auto& gemm_ptr : gemm_ptrs)
+    {
+        auto argument_ptr = gemm_ptr->MakeArgumentPointer(
+            static_cast<ADataType*>(a_device_buf.GetDeviceBuffer()),
+            static_cast<BDataType*>(b_device_buf.GetDeviceBuffer()),
+            static_cast<CDataType*>(c_device_buf.GetDeviceBuffer()),
+            static_cast<C0DataType*>(bias_device_buf.GetDeviceBuffer()),
+            static_cast<C1DataType*>(c1_device_buf.GetDeviceBuffer()),
+            &dxs_global,
+            M,
+            N,
+            K,
+            StrideA,
+            StrideB,
+            StrideC,
+            StrideC1,
+            a_element_op,
+            b_element_op,
+            c_element_op,
+            c1_element_op,
+            dxs_in_element_op,
+            dxs_out_element_op);
+
+        auto invoker_ptr = gemm_ptr->MakeInvokerPointer();
+
+        if(gemm_ptr->IsSupportedArgument(argument_ptr.get()))
+        {
+            // init DO, D1 to 0
+            d0_device_buf.SetZero();
+            d1_device_buf.SetZero();
+
+            float ave_time =
+                invoker_ptr->Run(argument_ptr.get(), StreamConfig{nullptr, time_kernel});
+
+            std::string gemm_name = gemm_ptr->GetTypeString();
+
+            std::size_t flop = std::size_t(2) * M * N * K + std::size_t(2) * M * N;
+
+            std::size_t num_byte = sizeof(ADataType) * M * K + sizeof(BDataType) * K * N +
+                                   sizeof(CDataType) * M * N + sizeof(C0DataType) * M * N +
+                                   sizeof(C1DataType) * M * N + sizeof(DDataType) * M +
+                                   sizeof(DDataType) * M;
+
+            float tflops = static_cast<float>(flop) / 1.E9 / ave_time;
+
+            float gb_per_sec = num_byte / 1.E6 / ave_time;
+
+            std::cout << "Perf: " << ave_time << " ms, " << tflops << " TFlops, " << gb_per_sec
+                      << " GB/s, " << gemm_name << std::endl;
+
+            if(tflops > best_tflops)
+            {
+                best_gemm_name  = gemm_name;
+                best_tflops     = tflops;
+                best_ave_time   = ave_time;
+                best_gb_per_sec = gb_per_sec;
+            }
+
+            if(do_verification)
+            {
+                c_device_buf.FromDevice(c_m_n_device_result.mData.data());
+                d0_device_buf.FromDevice(d0_m_device_result.mData.data());
+                d1_device_buf.FromDevice(d1_m_device_result.mData.data());
+
+                ck::utils::check_err(c_m_n_device_result.mData, c_m_n_host_result.mData);
+                ck::utils::check_err(d0_m_device_result.mData, d0_m_host_result.mData);
+                ck::utils::check_err(d1_m_device_result.mData, d1_m_host_result.mData);
+
+                if(do_log)
+                {
+                    LogRangeAsType<float>(std::cout << "a : ", a_m_k.mData, ",") << std::endl;
+                    LogRangeAsType<float>(std::cout << "b: ", b_k_n.mData, ",") << std::endl;
+                    LogRangeAsType<float>(std::cout << "c_host: ", c_m_n_host_result.mData, ",")
+                        << std::endl;
+                    LogRangeAsType<float>(std::cout << "c_device: ", c_m_n_device_result.mData, ",")
+                        << std::endl;
+                    LogRangeAsType<float>(std::cout << "d0_host: ", d0_m_host_result.mData, ",")
+                        << std::endl;
+                    LogRangeAsType<float>(std::cout << "d0_device: ", d0_m_device_result.mData, ",")
+                        << std::endl;
+                    LogRangeAsType<float>(std::cout << "d1_host: ", d1_m_host_result.mData, ",")
+                        << std::endl;
+                    LogRangeAsType<float>(std::cout << "d1_device: ", d1_m_device_result.mData, ",")
+                        << std::endl;
+                }
+            }
+        }
+        else
+        {
+            std::cout << "does not support this GEMM problem" << std::endl;
+        }
+    }
+
+    std::cout << "Best Perf: " << best_ave_time << " ms, " << best_tflops << " TFlops, "
+              << best_gb_per_sec << " GB/s, " << best_gemm_name << std::endl;
+}
+
+} // namespace profiler
+} // namespace ck

profiler/include/profile_gemm_reduce_impl.hpp

 #pragma once
+#include "check_err.hpp"
 #include "config.hpp"
 #include "device.hpp"
 #include "host_tensor.hpp"
@@ -19,14 +20,13 @@ namespace device_gemm_instance {
 using F32            = float;
 using F16            = ck::half_t;
 using DPtrsGlobal    = ck::Tuple<F32*, F32*>;
-using Div            = ck::tensor_operation::element_wise::UnaryIdentic<F32, F32, true>;
-using Identity       = ck::tensor_operation::element_wise::UnaryIdentic<F32, F32, false>;
-using Square         = ck::tensor_operation::element_wise::UnarySquare<F32, F32, false>;
+using Div            = ck::tensor_operation::element_wise::UnaryDivide;
+using Identity       = ck::tensor_operation::element_wise::PassThrough;
+using Square         = ck::tensor_operation::element_wise::UnarySquare;
 using DInElementOps  = ck::Tuple<Identity, Square>;
 using DOutElementOps = ck::Tuple<Div, Div>;
 
 using DeviceGemmReduceNoOpPtr = ck::tensor_operation::device::DeviceGemmReducePtr<
-    DPtrsGlobal,
     ck::tensor_operation::element_wise::PassThrough,
     ck::tensor_operation::element_wise::PassThrough,
     ck::tensor_operation::element_wise::PassThrough,
@@ -123,18 +123,16 @@ bool profile_gemm_reduce_impl(int do_verification,
         b_k_n.GenerateTensorValue(GeneratorTensor_3<BDataType>{-0.5, 0.5}, num_thread);
     }
 
-    using AElementOp        = ck::tensor_operation::element_wise::PassThrough;
-    using BElementOp        = ck::tensor_operation::element_wise::PassThrough;
-    using CElementOp        = ck::tensor_operation::element_wise::PassThrough;
-    using D0ReduceOp        = ck::reduce::Add<float>;
-    using D1ReduceOp        = ck::reduce::Add<float>;
-    using UnaryDivElementOp = ck::tensor_operation::element_wise::UnaryIdentic<float, float, true>;
-    using UnaryIdenticElementOp =
-        ck::tensor_operation::element_wise::UnaryIdentic<float, float, false>;
-    using UnarySquareElementOp =
-        ck::tensor_operation::element_wise::UnarySquare<float, float, false>;
-    using DxsInElementOps  = ck::Tuple<UnaryIdenticElementOp, UnarySquareElementOp>;
-    using DxsOutElementOps = ck::Tuple<UnaryDivElementOp, UnaryDivElementOp>;
+    using AElementOp            = ck::tensor_operation::element_wise::PassThrough;
+    using BElementOp            = ck::tensor_operation::element_wise::PassThrough;
+    using CElementOp            = ck::tensor_operation::element_wise::PassThrough;
+    using D0ReduceOp            = ck::reduce::Add;
+    using D1ReduceOp            = ck::reduce::Add;
+    using UnaryDivElementOp     = ck::tensor_operation::element_wise::UnaryDivide;
+    using UnaryIdenticElementOp = ck::tensor_operation::element_wise::PassThrough;
+    using UnarySquareElementOp  = ck::tensor_operation::element_wise::UnarySquare;
+    using DxsInElementOps       = ck::Tuple<UnaryIdenticElementOp, UnarySquareElementOp>;
+    using DxsOutElementOps      = ck::Tuple<UnaryDivElementOp, UnaryDivElementOp>;
 
     const auto a_element_op = AElementOp{};
     const auto b_element_op = BElementOp{};
@@ -143,7 +141,7 @@ bool profile_gemm_reduce_impl(int do_verification,
     const auto d1_reduce_op = D1ReduceOp{};
 
     auto dxs_in_element_op  = DxsInElementOps{};
-    auto dxs_out_element_op = DxsOutElementOps{M, M};
+    auto dxs_out_element_op = DxsOutElementOps{N, N};
 
     if(do_verification)
     {
@@ -155,6 +153,8 @@ bool profile_gemm_reduce_impl(int do_verification,
                                                                                 BElementOp,
                                                                                 CElementOp>;
 
+        using ReduceAccDataType = DDataType;
+
         auto ref_gemm    = ReferenceGemmInstance{};
         auto ref_invoker = ref_gemm.MakeInvoker();
 
@@ -165,14 +165,15 @@ bool profile_gemm_reduce_impl(int do_verification,
 
         for(int m = 0; m < M; ++m)
         {
-            float d0_acc = d0_reduce_op.GetIdentityValue();
-            float d1_acc = d1_reduce_op.GetIdentityValue();
+            auto d0_acc = d0_reduce_op.GetIdentityValue<ReduceAccDataType>();
+            auto d1_acc = d1_reduce_op.GetIdentityValue<ReduceAccDataType>();
 
             for(int n = 0; n < N; ++n)
             {
-                float c_val  = ck::type_convert<float>(c_m_n_host_result(m, n));
-                float d0_val = 0;
-                float d1_val = 0;
+                ReduceAccDataType c_val =
+                    ck::type_convert<ReduceAccDataType>(c_m_n_host_result(m, n));
+                ReduceAccDataType d0_val;
+                ReduceAccDataType d1_val;
 
                 dxs_in_element_op(ck::Number<0>{})(d0_val, c_val);
                 dxs_in_element_op(ck::Number<1>{})(d1_val, c_val);
@@ -257,7 +258,7 @@ bool profile_gemm_reduce_impl(int do_verification,
             gemm_ptr->MakeArgumentPointer(static_cast<ADataType*>(a_device_buf.GetDeviceBuffer()),
                                           static_cast<BDataType*>(b_device_buf.GetDeviceBuffer()),
                                           static_cast<CDataType*>(c_device_buf.GetDeviceBuffer()),
-                                          dxs_global,
+                                          &dxs_global,
                                           M,
                                           N,
                                           K,
@@ -309,13 +310,9 @@ bool profile_gemm_reduce_impl(int do_verification,
                 d0_device_buf.FromDevice(d0_m_device_result.mData.data());
                 d1_device_buf.FromDevice(d1_m_device_result.mData.data());
 
-                float c_error  = check_error(c_m_n_host_result, c_m_n_device_result);
-                float d0_error = check_error(d0_m_host_result, d0_m_device_result);
-                float d1_error = check_error(d1_m_host_result, d1_m_device_result);
-
-                pass = pass && (c_error < 1E-6);
-                pass = pass && (d0_error < 1E-6);
-                pass = pass && (d1_error < 1E-6);
+                ck::utils::check_err(c_m_n_device_result.mData, c_m_n_host_result.mData);
+                ck::utils::check_err(d0_m_device_result.mData, d0_m_host_result.mData);
+                ck::utils::check_err(d1_m_device_result.mData, d1_m_host_result.mData);
 
                 if(do_log)
                 {

profiler/include/profile_reduce_impl.hpp

@@ -261,13 +261,18 @@ bool profile_reduce_impl_impl(bool do_verification,
         float best_gb_per_sec = 0;
 
         using InElementwiseOperation =
-            typename reduce_unary_operator<AccDataType, ReduceOpId, true, true>::
-                InElementwiseOperation;
+            typename reduce_unary_operator<ReduceOpId, true, true>::InElementwiseOperation;
         using AccElementwiseOperation =
-            typename reduce_unary_operator<AccDataType, ReduceOpId, true, true>::
-                AccElementwiseOperation;
+            typename reduce_unary_operator<ReduceOpId, true, true>::AccElementwiseOperation;
 
-        using ReduceOperation = typename reduce_binary_operator<AccDataType, ReduceOpId>::opType;
+        using ReduceOperation = typename reduce_binary_operator<ReduceOpId>::opType;
+
+        InElementwiseOperation in_elementwise_op;
+        AccElementwiseOperation acc_elementwise_op;
+
+        std::tie(in_elementwise_op, acc_elementwise_op) =
+            reduce_unary_operator<ReduceOpId, true, true>::GetElementwiseOperator(
+                static_cast<int32_t>(reduce_total_length));
 
         using DeviceReduceInstPtr0 =
             DeviceReducePtr<InElementwiseOperation, AccElementwiseOperation>;
@@ -323,8 +328,13 @@ bool profile_reduce_impl_impl(bool do_verification,
                           OutputIndex>
                 hostReduce(in.mDesc, out_ref.mDesc, invariantDims, reduceDims);
 
-            hostReduce.Run(
-                alpha, in.mData.data(), beta, out_ref.mData.data(), out_indices_ref.mData.data());
+            hostReduce.Run(alpha,
+                           in.mData.data(),
+                           beta,
+                           out_ref.mData.data(),
+                           out_indices_ref.mData.data(),
+                           in_elementwise_op,
+                           acc_elementwise_op);
         };
 
         std::vector<ck::index_t> i_inLengths;
@@ -339,10 +349,6 @@ bool profile_reduce_impl_impl(bool do_verification,
 
         for(auto& reduce_ptr : reduce0_ptrs)
         {
-
-            InElementwiseOperation in_elementwise_op(static_cast<int32_t>(reduce_total_length));
-            AccElementwiseOperation acc_elementwise_op(static_cast<int32_t>(reduce_total_length));
-
             auto argument_ptr = reduce_ptr->MakeArgumentPointer(i_inLengths,
                                                                 i_inStrides,
                                                                 i_outLengths,

profiler/src/profile_convnd_fwd.cpp

 #include <cstdlib>
+#include <functional>
 #include <iostream>
 #include <memory>
 #include <string>
@@ -150,9 +151,12 @@ void profile_convnd_instances_impl(const ck::utils::conv::ConvParams& params,
                                     ck::tensor_operation::element_wise::PassThrough,
                                     ck::tensor_operation::element_wise::PassThrough,
                                     ck::tensor_operation::element_wise::PassThrough,
-                                    ck::utils::FillUniform<int>,
-                                    ck::utils::FillUniform<int>>>(
-            params, true, ck::utils::FillUniform<int>{}, ck::utils::FillUniform<int>{});
+                                    ck::utils::FillUniformDistributionIntegerValue<int>,
+                                    ck::utils::FillUniformDistributionIntegerValue<int>>>(
+            params,
+            true,
+            ck::utils::FillUniformDistributionIntegerValue<int>{},
+            ck::utils::FillUniformDistributionIntegerValue<int>{});
         break;
     case 2:
         conv_instance = std::make_unique<
@@ -165,12 +169,12 @@ void profile_convnd_instances_impl(const ck::utils::conv::ConvParams& params,
                                     ck::tensor_operation::element_wise::PassThrough,
                                     ck::tensor_operation::element_wise::PassThrough,
                                     ck::tensor_operation::element_wise::PassThrough,
-                                    ck::utils::FillUniform<InDataType>,
-                                    ck::utils::FillUniform<WeiDataType>>>(
+                                    ck::utils::FillUniformDistribution<InDataType>,
+                                    ck::utils::FillUniformDistribution<WeiDataType>>>(
             params,
             true,
-            ck::utils::FillUniform<InDataType>{},
-            ck::utils::FillUniform<WeiDataType>{});
+            ck::utils::FillUniformDistribution<InDataType>{},
+            ck::utils::FillUniformDistribution<WeiDataType>{});
         break;
     default: throw std::runtime_error("Unsupported init method!");
     }
@@ -181,8 +185,10 @@ void profile_convnd_instances_impl(const ck::utils::conv::ConvParams& params,
         _1,
         _2,
         _3);
-    OpInstanceRunEngine<InDataType, WeiDataType, OutDataType> run_engine(*conv_instance,
-                                                                         reference_conv_fwd_fun);
+
+    OpInstanceRunEngine<InDataType, WeiDataType, OutDataType> run_engine(
+        *conv_instance, reference_conv_fwd_fun, do_verification);
+
     auto best_conf = run_engine.Profile(
         conv::ConvolutionFwdInstances<InDataType, WeiDataType, OutDataType>::template Get<NDim>(),
         time_kernel,

profiler/src/profile_gemm_add_add_fastgelu.cpp

+#include <iostream>
+#include <numeric>
+#include <initializer_list>
+#include <cstdlib>
+#include <stdlib.h>
+
+#include "profile_gemm_add_add_fastgelu_impl.hpp"
+
+int profile_gemm_add_add_fastgelu(int argc, char* argv[])
+{
+    enum struct MatrixLayout
+    {
+        MK_KN_MN_MN_MN, // 0
+        MK_NK_MN_MN_MN, // 1
+        KM_KN_MN_MN_MN, // 2
+        KM_NK_MN_MN_MN, // 3
+        MK_KN_NM_MN_MN, // 4
+        MK_NK_NM_MN_MN, // 5
+        KM_KN_NM_MN_MN, // 6
+        KM_NK_NM_MN_MN, // 7
+    };
+
+    enum struct MatrixDataType
+    {
+        F32_F32_F32_F32_F32,      // 0
+        F16_F16_F16_F16_F16,      // 1
+        BF16_BF16_BF16_BF16_BF16, // 2
+        INT8_INT8_INT8_INT8_INT8, // 3
+    };
+
+    if(argc != 16)
+    {
+        // clang-format off
+        printf("arg1: tensor operation (gemm_add_add_fastgelu: GEMM+Add+Add+GeLU)\n");
+        printf("arg2: data type (0: fp32; 1: fp16; 2: bf16; 3: int8)\n");
+        printf("arg3: matrix layout (0: E[m, n] = FastGeLU(A[m, k] * B[k, n] + D0[m, n] + D1[m, n]);\n");
+        printf("                     1: E[m, n] = FastGeLU(A[m, k] * B[n, k] + D0[m, n] + D1[m, n]);\n");
+        printf("                     2: E[m, n] = FastGeLU(A[k, m] * B[k, n] + D0[m, n] + D1[m, n]);\n");
+        printf("                     3: E[m, n] = FastGeLU(A[k, m] * B[n, k] + D0[m, n] + D1[m, n]))\n");
+        printf("arg4: verification (0: no; 1: yes)\n");
+        printf("arg5: initialization (0: no init; 1: integer value; 2: decimal value)\n");
+        printf("arg6: print tensor value (0: no; 1: yes)\n");
+        printf("arg7: time kernel (0=no, 1=yes)\n");
+        printf("arg8 to 13: M, N, K, StrideA, StrideB, StrideD0, StrideD1, StrideE\n");
+        // clang-format on
+        exit(1);
+    }
+
+    const auto data_type       = static_cast<MatrixDataType>(std::stoi(argv[2]));
+    const auto layout          = static_cast<MatrixLayout>(std::stoi(argv[3]));
+    const bool do_verification = std::stoi(argv[4]);
+    const int init_method      = std::stoi(argv[5]);
+    const bool do_log          = std::stoi(argv[6]);
+    const bool time_kernel     = std::stoi(argv[7]);
+
+    const int M = std::stoi(argv[8]);
+    const int N = std::stoi(argv[9]);
+    const int K = std::stoi(argv[10]);
+
+    const int StrideA  = std::stoi(argv[11]);
+    const int StrideB  = std::stoi(argv[12]);
+    const int StrideD0 = std::stoi(argv[13]);
+    const int StrideD1 = std::stoi(argv[14]);
+    const int StrideE  = std::stoi(argv[15]);
+
+    using F16 = ck::half_t;
+    using F32 = float;
+
+    using Row = ck::tensor_layout::gemm::RowMajor;
+    using Col = ck::tensor_layout::gemm::ColumnMajor;
+
+    auto profile = [&](auto a_type,
+                       auto b_type,
+                       auto acc_type,
+                       auto d0_type,
+                       auto d1_type,
+                       auto e_type,
+                       auto a_layout,
+                       auto b_layout,
+                       auto d0_layout,
+                       auto d1_layout,
+                       auto e_layout) {
+        using ADataType   = decltype(a_type);
+        using BDataType   = decltype(b_type);
+        using AccDataType = decltype(acc_type);
+        using D0DataType  = decltype(d0_type);
+        using D1DataType  = decltype(d1_type);
+        using EDataType   = decltype(e_type);
+
+        using ALayout  = decltype(a_layout);
+        using BLayout  = decltype(b_layout);
+        using D0Layout = decltype(d0_layout);
+        using D1Layout = decltype(d1_layout);
+        using ELayout  = decltype(e_layout);
+
+        const int DefaultStrideA  = ck::is_same_v<ALayout, Row> ? K : M;
+        const int DefaultStrideB  = ck::is_same_v<BLayout, Row> ? N : K;
+        const int DefaultStrideD0 = ck::is_same_v<D0Layout, Row> ? N : M;
+        const int DefaultStrideD1 = ck::is_same_v<D1Layout, Row> ? N : M;
+        const int DefaultStrideE  = ck::is_same_v<ELayout, Row> ? N : M;
+
+        return ck::profiler::profile_gemm_add_add_fastgelu_impl<ADataType,
+                                                                BDataType,
+                                                                AccDataType,
+                                                                D0DataType,
+                                                                D1DataType,
+                                                                EDataType,
+                                                                ALayout,
+                                                                BLayout,
+                                                                D0Layout,
+                                                                D1Layout,
+                                                                ELayout>(
+            do_verification,
+            init_method,
+            do_log,
+            time_kernel,
+            M,
+            N,
+            K,
+            (StrideA < 0) ? DefaultStrideA : StrideA,
+            (StrideB < 0) ? DefaultStrideB : StrideB,
+            (StrideD0 < 0) ? DefaultStrideD0 : StrideD0,
+            (StrideD1 < 0) ? DefaultStrideD1 : StrideD1,
+            (StrideE < 0) ? DefaultStrideE : StrideE);
+    };
+
+    if(data_type == MatrixDataType::F16_F16_F16_F16_F16 && layout == MatrixLayout::MK_KN_MN_MN_MN)
+    {
+        return profile(F16{}, F16{}, F32{}, F16{}, F16{}, F16{}, Row{}, Row{}, Row{}, Row{}, Row{});
+    }
+    else if(data_type == MatrixDataType::F16_F16_F16_F16_F16 &&
+            layout == MatrixLayout::MK_NK_MN_MN_MN)
+    {
+        return profile(F16{}, F16{}, F32{}, F16{}, F16{}, F16{}, Row{}, Col{}, Row{}, Row{}, Row{});
+    }
+    else if(data_type == MatrixDataType::F16_F16_F16_F16_F16 &&
+            layout == MatrixLayout::KM_KN_MN_MN_MN)
+    {
+        return profile(F16{}, F16{}, F32{}, F16{}, F16{}, F16{}, Col{}, Row{}, Row{}, Row{}, Row{});
+    }
+    else if(data_type == MatrixDataType::F16_F16_F16_F16_F16 &&
+            layout == MatrixLayout::KM_NK_MN_MN_MN)
+    {
+        return profile(F16{}, F16{}, F32{}, F16{}, F16{}, F16{}, Col{}, Col{}, Row{}, Row{}, Row{});
+    }
+    else
+    {
+        std::cout << "this data_type & layout is not implemented" << std::endl;
+
+        return 0;
+    }
+}

profiler/src/profile_gemm_bias_add_reduce.cpp

+#include <iostream>
+#include <numeric>
+#include <initializer_list>
+#include <cstdlib>
+#include <stdlib.h>
+#include <half.hpp>
+#include "profile_gemm_bias_add_reduce_impl.hpp"
+
+int profile_gemm_bias_add_reduce(int argc, char* argv[])
+{
+    enum struct GemmMatrixLayout
+    {
+        MK_KN_MN, // 0
+        MK_NK_MN, // 1
+        KM_KN_MN, // 2
+        KM_NK_MN, // 3
+    };
+
+    enum struct GemmReduceDataType
+    {
+        F32_F32_F32_F32_F32_F32_F32, // 0
+        F16_F16_F16_F16_F16_F32_F32, // 1
+    };
+
+    if(!(argc == 14 || argc == 15))
+    {
+        printf("arg1: tensor operation (gemm: GEMM+bias+add+Reduce)\n");
+        printf("arg2: data type (0: fp32; 1: fp16)\n");
+        printf("arg3: matrix layout (0: A[m, k] * B[k, n] = C[m, n];\n");
+        printf("                     1: A[m, k] * B[n, k] = C[m, n];\n");
+        printf("                     2: A[k, m] * B[k, n] = C[m, n];\n");
+        printf("                     3: A[k, m] * B[n, k] = C[m, n])\n");
+        printf("arg4: verification (0: no; 1: yes)\n");
+        printf("arg5: initialization (0: no init; 1: integer value; 2: decimal value)\n");
+        printf("arg6: print tensor value (0: no; 1: yes)\n");
+        printf("arg7: time kernel (0=n0, 1=yes)\n");
+        printf("arg8 to 14: M, N, K, StrideA, StrideB, StrideC, StrideC1\n");
+        exit(1);
+    }
+
+    const auto data_type       = static_cast<GemmReduceDataType>(std::stoi(argv[2]));
+    const auto layout          = static_cast<GemmMatrixLayout>(std::stoi(argv[3]));
+    const bool do_verification = std::stoi(argv[4]);
+    const int init_method      = std::stoi(argv[5]);
+    const bool do_log          = std::stoi(argv[6]);
+    const bool time_kernel     = std::stoi(argv[7]);
+
+    const int M = std::stoi(argv[8]);
+    const int N = std::stoi(argv[9]);
+    const int K = std::stoi(argv[10]);
+
+    const int StrideA  = std::stoi(argv[11]);
+    const int StrideB  = std::stoi(argv[12]);
+    const int StrideC  = std::stoi(argv[13]);
+    const int StrideC1 = std::stoi(argv[14]);
+
+    if(data_type == GemmReduceDataType::F16_F16_F16_F16_F16_F32_F32 &&
+       layout == GemmMatrixLayout::MK_KN_MN)
+    {
+        ck::profiler::profile_gemm_bias_add_reduce_impl<ck::half_t,
+                                                        ck::half_t,
+                                                        ck::half_t,
+                                                        ck::half_t,
+                                                        ck::half_t,
+                                                        float,
+                                                        ck::tensor_layout::gemm::RowMajor,
+                                                        ck::tensor_layout::gemm::RowMajor,
+                                                        ck::tensor_layout::gemm::RowMajor>(
+            do_verification,
+            init_method,
+            do_log,
+            time_kernel,
+            M,
+            N,
+            K,
+            (StrideA < 0) ? M : StrideA,
+            (StrideB < 0) ? N : StrideB,
+            (StrideC < 0) ? N : StrideC,
+            (StrideC1 < 0) ? N : StrideC1);
+    }
+    else if(data_type == GemmReduceDataType::F16_F16_F16_F16_F16_F32_F32 &&
+            layout == GemmMatrixLayout::MK_NK_MN)
+    {
+        ck::profiler::profile_gemm_bias_add_reduce_impl<ck::half_t,
+                                                        ck::half_t,
+                                                        ck::half_t,
+                                                        ck::half_t,
+                                                        ck::half_t,
+                                                        float,
+                                                        ck::tensor_layout::gemm::RowMajor,
+                                                        ck::tensor_layout::gemm::ColumnMajor,
+                                                        ck::tensor_layout::gemm::RowMajor>(
+            do_verification,
+            init_method,
+            do_log,
+            time_kernel,
+            M,
+            N,
+            K,
+            (StrideA < 0) ? M : StrideA,
+            (StrideB < 0) ? N : StrideB,
+            (StrideC < 0) ? N : StrideC,
+            (StrideC1 < 0) ? N : StrideC1);
+    }
+    else if(data_type == GemmReduceDataType::F16_F16_F16_F16_F16_F32_F32 &&
+            layout == GemmMatrixLayout::KM_KN_MN)
+    {
+        ck::profiler::profile_gemm_bias_add_reduce_impl<ck::half_t,
+                                                        ck::half_t,
+                                                        ck::half_t,
+                                                        ck::half_t,
+                                                        ck::half_t,
+                                                        float,
+                                                        ck::tensor_layout::gemm::ColumnMajor,
+                                                        ck::tensor_layout::gemm::RowMajor,
+                                                        ck::tensor_layout::gemm::RowMajor>(
+            do_verification,
+            init_method,
+            do_log,
+            time_kernel,
+            M,
+            N,
+            K,
+            (StrideA < 0) ? M : StrideA,
+            (StrideB < 0) ? N : StrideB,
+            (StrideC < 0) ? N : StrideC,
+            (StrideC1 < 0) ? N : StrideC1);
+    }
+    else if(data_type == GemmReduceDataType::F16_F16_F16_F16_F16_F32_F32 &&
+            layout == GemmMatrixLayout::KM_NK_MN)
+    {
+        ck::profiler::profile_gemm_bias_add_reduce_impl<ck::half_t,
+                                                        ck::half_t,
+                                                        ck::half_t,
+                                                        ck::half_t,
+                                                        ck::half_t,
+                                                        float,
+                                                        ck::tensor_layout::gemm::ColumnMajor,
+                                                        ck::tensor_layout::gemm::ColumnMajor,
+                                                        ck::tensor_layout::gemm::RowMajor>(
+            do_verification,
+            init_method,
+            do_log,
+            time_kernel,
+            M,
+            N,
+            K,
+            (StrideA < 0) ? M : StrideA,
+            (StrideB < 0) ? N : StrideB,
+            (StrideC < 0) ? N : StrideC,
+            (StrideC1 < 0) ? N : StrideC1);
+    }
+    else
+    {
+        throw std::runtime_error("wrong! this data_type & layout is not implemented");
+    }
+
+    return 0;
+}

profiler/src/profiler.cpp

@@ -11,6 +11,7 @@ int profile_gemm_bias_2d(int, char*[]);
 int profile_gemm_bias_relu(int, char*[]);
 int profile_gemm_bias_relu_add(int, char*[]);
 int profile_gemm_reduce(int, char*[]);
+int profile_gemm_bias_add_reduce(int, char*[]);
 int profile_batched_gemm(int, char*[]);
 int profile_grouped_gemm(int, char*[]);
 int profile_conv_fwd(int, char*[]);
@@ -21,9 +22,39 @@ int profile_convnd_bwd_data(int, char*[], int);
 int profile_reduce(int, char*[]);
 int profile_conv_bwd_weight(int, char*[]);
 int profile_batched_gemm_reduce(int, char*[]);
+int profile_gemm_add_add_fastgelu(int, char*[]);
+
+static void print_helper_message()
+{
+    // clang-format off
+        printf("arg1: tensor operation (gemm: GEMM\n"
+               "                        gemm_bias_2d: GEMM+Bias(2D)\n"
+               "                        gemm_bias_relu: GEMM+Bias+ReLU\n"
+               "                        gemm_bias_relu_add: GEMM+Bias+ReLU+Add\n"
+               "                        gemm_reduce: GEMM+Reduce\n"
+               "                        grouped_gemm: Grouped GEMM\n"
+               "                        conv_fwd: ForwardConvolution\n"
+               "                        conv_fwd_bias_relu: ForwardConvolution+Bias+ReLU\n"
+               "                        conv_fwd_bias_relu_add: ForwardConvolution+Bias+ReLU+Add\n"
+               "                        conv_fwd_bias_relu_atomic_add: ForwardConvolution+Bias+ReLU+AtomicAdd\n"
+               "                        conv1d_bwd_data: BackwardConvolution data 1 dim\n"
+               "                        conv2d_bwd_data: BackwardConvolution data 2 dim\n"
+               "                        conv3d_bwd_data: BackwardConvolution data 3 dim\n"
+               "                        reduce: Reduce\n"
+               "                        conv2d_bwd_weight: Backward Weight Convolution 2d\n"
+               "                        gemm_add_add_fastgelu: GEMM+Add+Add+FastGeLU\n");
+    // clang-format on
+}
 
 int main(int argc, char* argv[])
 {
+    if(argc == 1)
+    {
+        print_helper_message();
+
+        return 0;
+    }
+
     if(strcmp(argv[1], "gemm") == 0)
     {
         return profile_gemm(argc, argv);
@@ -44,6 +75,10 @@ int main(int argc, char* argv[])
     {
         return profile_gemm_reduce(argc, argv);
     }
+    else if(strcmp(argv[1], "gemm_bias_add_reduce") == 0)
+    {
+        return profile_gemm_bias_add_reduce(argc, argv);
+    }
     else if(strcmp(argv[1], "batched_gemm") == 0)
     {
         return profile_batched_gemm(argc, argv);
@@ -92,25 +127,14 @@ int main(int argc, char* argv[])
     {
         return profile_conv_bwd_weight(argc, argv);
     }
+    else if(strcmp(argv[1], "gemm_add_add_fastgelu") == 0)
+    {
+        return profile_gemm_add_add_fastgelu(argc, argv);
+    }
     else
     {
-        // clang-format off
-        printf("arg1: tensor operation (gemm: GEMM\n"
-               "                        gemm_bias_2d: GEMM+Bias(2D)\n"
-               "                        gemm_bias_relu: GEMM+Bias+ReLU\n"
-               "                        gemm_bias_relu_add: GEMM+Bias+ReLU+Add\n"
-               "                        gemm_reduce: GEMM+Reduce\n"
-               "                        grouped_gemm: Grouped GEMM\n"
-               "                        conv_fwd: ForwardConvolution\n"
-               "                        conv_fwd_bias_relu: ForwardConvolution+Bias+ReLU\n"
-               "                        conv_fwd_bias_relu_add: ForwardConvolution+Bias+ReLU+Add\n"
-               "                        conv_fwd_bias_relu_atomic_add: ForwardConvolution+Bias+ReLU+AtomicAdd\n"
-               "                        conv1d_bwd_data: BackwardConvolution data 1 dim\n"
-               "                        conv2d_bwd_data: BackwardConvolution data 2 dim\n"
-               "                        conv3d_bwd_data: BackwardConvolution data 3 dim\n"
-               "                        reduce: Reduce\n"
-               "                        conv2d_bwd_weight: Backward Weight Convolution 2d\n");
-        // clang-format on
+        print_helper_message();
+
+        return 0;
     }
-    return 0;
 }

script/profile_conv.sh

@@ -26,7 +26,7 @@ REPEAT=$9
 
  N=${10}
 
-# Resnet50 from Bing
+# Resnet50 (no duplicated layer)
 ########  op  datatype  in_layout   wei_layout  out_layout  verify  init  log  repeat  N__ K___ C___ Y X Hi__ Wi__ Strides Dilations LeftPads RightPads
 #$DRIVER $OP $DATATYPE $IN_LAYOUT  $WEI_LAYOUT $OUT_LAYOUT $VERIFY $INIT $LOG $REPEAT   $N  256 1024 1 1   14   14     1 1       1 1      0 0       0 0
 #$DRIVER $OP $DATATYPE $IN_LAYOUT  $WEI_LAYOUT $OUT_LAYOUT $VERIFY $INIT $LOG $REPEAT   $N  512 1024 1 1   14   14     1 1       1 1      0 0       0 0
@@ -47,10 +47,10 @@ REPEAT=$9
 #$DRIVER $OP $DATATYPE $IN_LAYOUT  $WEI_LAYOUT $OUT_LAYOUT $VERIFY $INIT $LOG $REPEAT   $N  256   64 1 1   56   56     1 1       1 1      0 0       0 0
 #$DRIVER $OP $DATATYPE $IN_LAYOUT  $WEI_LAYOUT $OUT_LAYOUT $VERIFY $INIT $LOG $REPEAT   $N   64   64 1 1   56   56     1 1       1 1      0 0       0 0
 #$DRIVER $OP $DATATYPE $IN_LAYOUT  $WEI_LAYOUT $OUT_LAYOUT $VERIFY $INIT $LOG $REPEAT   $N   64   64 3 3   56   56     1 1       1 1      1 1       1 1
-#$DRIVER $OP $DATATYPE $IN_LAYOUT  $WEI_LAYOUT $OUT_LAYOUT $VERIFY $INIT $LOG $REPEAT   $N   64    8 7 7  224  224     2 2       1 1      3 3       3 3
+#$DRIVER $OP $DATATYPE $IN_LAYOUT  $WEI_LAYOUT $OUT_LAYOUT $VERIFY $INIT $LOG $REPEAT   $N   64    3 7 7  224  224     2 2       1 1      3 3       3 3
 
 
-# Resnet50 from Bing
+# Resnet50 fusion
 ####### op_________________    datatype  in_layout   wei_layout  out_layout  verify  init  log  repeat    N__  K___ C_ Y X  Hi_ Wi__ Strides Dilations LeftPads RightPads
 $DRIVER conv_fwd_bias_relu     $DATATYPE $IN_LAYOUT  $WEI_LAYOUT $OUT_LAYOUT $VERIFY $INIT $LOG $REPEAT   $N   64    3 7 7  224 224    2   2     1   1    3   3     3   3
 $DRIVER conv_fwd_bias_relu     $DATATYPE $IN_LAYOUT  $WEI_LAYOUT $OUT_LAYOUT $VERIFY $INIT $LOG $REPEAT   $N   64   64 1 1   56  56    1   1     1   1    0   0     0   0

script/run_performance_tests.sh

-#!/bin/bash 
-#
-# in order to run this script you'd first need to build the ckProfiler executable in ../build/bin/
-# and make sure the following python packages are installed in your environment:
-# pip3 install --upgrade pip
-# pip3 install sqlalchemy
-# pip3 install pymysql
-# pip3 install pandas
-# pip3 install sshtunnel
-# you would also need to set up some environment variables in order to 
-# post your new test results to the database and compare them to the baseline
-# please contact Illia.Silin@amd.com for more details
-#
-
-export gemm_log="perf_gemm.log"
-rm -f $gemm_log
-git status | grep -e 'On branch' > ${gemm_log}
-echo -n 'Node name: ' >>${gemm_log}; hostname >> ${gemm_log}
-#get GPU_arch and number of compute units from rocminfo
-echo -n "GPU_arch: " >> ${gemm_log}; rocminfo | grep "Name:" | grep "gfx" >> ${gemm_log} 
-rocminfo | grep "Compute Unit:" >> ${gemm_log} 
-hipcc --version | grep -e 'HIP version'  >> ${gemm_log}
-/opt/rocm/bin/amdclang++ --version | grep -e 'InstalledDir' >> ${gemm_log}
-./profile_gemm.sh gemm 0 0 0 1 0 5 | tee -a ${gemm_log}
-./profile_gemm.sh gemm 1 0 0 1 0 5 | tee -a $gemm_log
-./profile_gemm.sh gemm 2 0 0 1 0 5 | tee -a $gemm_log
-./profile_gemm.sh gemm 3 0 0 1 0 5 | tee -a $gemm_log
-./profile_gemm.sh gemm 0 1 0 1 0 5 | tee -a $gemm_log
-./profile_gemm.sh gemm 1 1 0 1 0 5 | tee -a $gemm_log
-./profile_gemm.sh gemm 2 1 0 1 0 5 | tee -a $gemm_log
-./profile_gemm.sh gemm 3 1 0 1 0 5 | tee -a $gemm_log
-./profile_gemm.sh gemm 0 2 0 1 0 5 | tee -a $gemm_log
-./profile_gemm.sh gemm 1 2 0 1 0 5 | tee -a $gemm_log
-./profile_gemm.sh gemm 2 2 0 1 0 5 | tee -a $gemm_log
-./profile_gemm.sh gemm 3 2 0 1 0 5 | tee -a $gemm_log
-./profile_gemm.sh gemm 0 3 0 1 0 5 | tee -a $gemm_log
-./profile_gemm.sh gemm 1 3 0 1 0 5 | tee -a $gemm_log
-./profile_gemm.sh gemm 2 3 0 1 0 5 | tee -a $gemm_log
-./profile_gemm.sh gemm 3 3 0 1 0 5 | tee -a $gemm_log
-
-python3 parse_perf_data.py ${gemm_log}
-
-#run resnet50 test
-export resnet_log="perf_resnet50.log"
-rm -f $resnet_log
-git status | grep -e 'On branch' > ${resnet_log}
-echo -n 'Node name: '>>${resnet_log}; hostname >>${resnet_log}
-#get GPU_arch and number of compute units from rocminfo
-echo -n "GPU_arch: " >> ${resnet_log}; rocminfo | grep "Name:" | grep "gfx" >> ${resnet_log}
-rocminfo | grep "Compute Unit:" >> ${resnet_log} 
-hipcc --version | grep -e 'HIP version'  >> ${resnet_log}
-/opt/rocm/bin/amdclang++ --version | grep -e 'InstalledDir' >> ${resnet_log}
-#first run tests with N=256
-./profile_conv.sh conv_fwd_bias_relu 1 1 1 1 0 2 0 1 256 | tee -a ${resnet_log}
-#then run with N=4
-./profile_conv.sh conv_fwd_bias_relu 1 1 1 1 0 2 0 1 4 | tee -a ${resnet_log}
-#the script will put the results from N=256 and N=4 runs into separate tables
-python3 parse_perf_data.py ${resnet_log}
+#!/bin/bash 
+#
+# in order to run this script you'd first need to build the ckProfiler executable in ../build/bin/
+# and make sure the following python packages are installed in your environment:
+
+pip3 install --upgrade pip
+pip3 install sqlalchemy pymysql pandas sshtunnel
+
+# you would also need to set up some environment variables in order to 
+# post your new test results to the database and compare them to the baseline
+# please contact Illia.Silin@amd.com for more details
+#
+
+export gemm_log="perf_gemm.log"
+rm -f $gemm_log
+git status | grep -e 'On branch' > ${gemm_log}
+echo -n 'Node name: ' >>${gemm_log}; hostname >> ${gemm_log}
+#get GPU_arch and number of compute units from rocminfo
+echo -n "GPU_arch: " >> ${gemm_log}; rocminfo | grep "Name:" | grep "gfx" >> ${gemm_log} 
+rocminfo | grep "Compute Unit:" >> ${gemm_log} 
+hipcc --version | grep -e 'HIP version'  >> ${gemm_log}
+/opt/rocm/bin/amdclang++ --version | grep -e 'InstalledDir' >> ${gemm_log}
+./profile_gemm.sh gemm 0 0 0 1 0 5 | tee -a ${gemm_log}
+./profile_gemm.sh gemm 1 0 0 1 0 5 | tee -a $gemm_log
+./profile_gemm.sh gemm 2 0 0 1 0 5 | tee -a $gemm_log
+./profile_gemm.sh gemm 3 0 0 1 0 5 | tee -a $gemm_log
+./profile_gemm.sh gemm 0 1 0 1 0 5 | tee -a $gemm_log
+./profile_gemm.sh gemm 1 1 0 1 0 5 | tee -a $gemm_log
+./profile_gemm.sh gemm 2 1 0 1 0 5 | tee -a $gemm_log
+./profile_gemm.sh gemm 3 1 0 1 0 5 | tee -a $gemm_log
+./profile_gemm.sh gemm 0 2 0 1 0 5 | tee -a $gemm_log
+./profile_gemm.sh gemm 1 2 0 1 0 5 | tee -a $gemm_log
+./profile_gemm.sh gemm 2 2 0 1 0 5 | tee -a $gemm_log
+./profile_gemm.sh gemm 3 2 0 1 0 5 | tee -a $gemm_log
+./profile_gemm.sh gemm 0 3 0 1 0 5 | tee -a $gemm_log
+./profile_gemm.sh gemm 1 3 0 1 0 5 | tee -a $gemm_log
+./profile_gemm.sh gemm 2 3 0 1 0 5 | tee -a $gemm_log
+./profile_gemm.sh gemm 3 3 0 1 0 5 | tee -a $gemm_log
+
+python3 parse_perf_data.py ${gemm_log}
+
+#run resnet50 test
+export resnet_log="perf_resnet50.log"
+rm -f $resnet_log
+git status | grep -e 'On branch' > ${resnet_log}
+echo -n 'Node name: '>>${resnet_log}; hostname >>${resnet_log}
+#get GPU_arch and number of compute units from rocminfo
+echo -n "GPU_arch: " >> ${resnet_log}; rocminfo | grep "Name:" | grep "gfx" >> ${resnet_log}
+rocminfo | grep "Compute Unit:" >> ${resnet_log} 
+hipcc --version | grep -e 'HIP version'  >> ${resnet_log}
+/opt/rocm/bin/amdclang++ --version | grep -e 'InstalledDir' >> ${resnet_log}
+#first run tests with N=256
+./profile_conv.sh conv_fwd_bias_relu 1 1 1 1 0 2 0 1 256 | tee -a ${resnet_log}
+#then run with N=4
+./profile_conv.sh conv_fwd_bias_relu 1 1 1 1 0 2 0 1 4 | tee -a ${resnet_log}
+#the script will put the results from N=256 and N=4 runs into separate tables
+python3 parse_perf_data.py ${resnet_log}

test/CMakeLists.txt

@@ -65,4 +65,5 @@ add_subdirectory(reduce)
 add_subdirectory(conv2d_bwd_weight)
 add_subdirectory(convnd_bwd_data)
 add_subdirectory(block_to_ctile_map)
+add_subdirectory(softmax)
 # DONOT add client_app, that is tested via CI independently

test/convnd_fwd/CMakeLists.txt

@@ -5,7 +5,7 @@ target_link_libraries(test_conv1d_fwd PRIVATE host_tensor device_conv1d_fwd_inst
 add_dependencies(test_convnd_fwd test_conv1d_fwd)
 
 add_gtest_executable(test_conv2d_fwd conv2d_fwd.cpp)
-target_link_libraries(test_conv2d_fwd PRIVATE host_tensor device_conv2d_fwd_instance conv_util)
+target_link_libraries(test_conv2d_fwd PRIVATE host_tensor device_conv2d_fwd_instance device_convnd_2d_fwd_instance conv_util)
 add_dependencies(test_convnd_fwd test_conv2d_fwd)
 
 add_gtest_executable(test_conv3d_fwd conv3d_fwd.cpp)

test/convnd_fwd/conv1d_fwd.cpp

 #include <iostream>
-#include <stdexcept>
 #include <tuple>
 #include <vector>
 #include "gtest/gtest.h"
@@ -11,83 +10,180 @@
 
 namespace {
 
-template <typename T>
-bool test_conv1d_nwc_instances(const std::vector<test::conv::DeviceConvFwdNoOpPtr>& conv_ptrs)
+class Conv1dFwdNWCInstances : public ::testing::Test
+{
+    public:
+    template <typename T>
+    bool test_conv1d_nwc_instances(const std::vector<test::conv::DeviceConvFwdNoOpPtr>& conv_ptrs,
+                                   const ck::utils::conv::ConvParams& params)
+    {
+        using namespace std::placeholders;
+        using namespace ck::utils;
+        namespace ctl = ck::tensor_layout::convolution;
+
+        conv::ConvFwdOpInstance<T,
+                                T,
+                                T,
+                                ctl::NWC,
+                                ctl::KXC,
+                                ctl::NWK,
+                                ck::tensor_operation::element_wise::PassThrough,
+                                ck::tensor_operation::element_wise::PassThrough,
+                                ck::tensor_operation::element_wise::PassThrough,
+                                FillUniformDistributionIntegerValue<T>,
+                                FillUniformDistributionIntegerValue<T>>
+            conv_instance(params,
+                          true,
+                          FillUniformDistributionIntegerValue<T>{},
+                          FillUniformDistributionIntegerValue<T>{});
+        auto reference_conv_fwd_fun =
+            std::bind(conv::run_reference_convolution_forward<1, T, T, T>, params, _1, _2, _3);
+        OpInstanceRunEngine<T, T, T> run_engine(conv_instance, reference_conv_fwd_fun);
+        run_engine.SetAtol(atol_);
+        run_engine.SetRtol(rtol_);
+        return run_engine.Test(conv_ptrs);
+    }
+
+    template <typename T>
+    bool test_default()
+    {
+        return test_conv1d_nwc_instances<T>(
+            ck::utils::conv::ConvolutionFwdInstances<T, T, T>::template Get<1>(), params_default_);
+    }
+
+    template <typename T>
+    bool test_filter1x1_stride1_pad0()
+    {
+        return test_conv1d_nwc_instances<T>(
+            ck::utils::conv::ConvolutionFwdInstances<T, T, T>::template Get<1>(),
+            params_filter1x1_stride1_pad0_);
+    }
+
+    template <typename T>
+    bool test_filter1x1_pad0()
+    {
+        return test_conv1d_nwc_instances<T>(
+            ck::utils::conv::ConvolutionFwdInstances<T, T, T>::template Get<1>(),
+            params_filter1x1_pad0_);
+    }
+
+    static inline ck::utils::conv::ConvParams params_default_{
+        1, 4, 256, 64, {3}, {71}, {2}, {2}, {2}, {2}};
+    static inline ck::utils::conv::ConvParams params_filter1x1_stride1_pad0_{
+        1, 4, 256, 64, {1}, {28}, {1}, {1}, {0}, {0}};
+    static inline ck::utils::conv::ConvParams params_filter1x1_pad0_{
+        1, 4, 256, 64, {1}, {28}, {2}, {1}, {0}, {0}};
+
+    private:
+    double atol_{1e-5};
+    double rtol_{1e-4};
+};
+
+} // anonymous namespace
+
+TEST(Conv1DFwdNWC, IntegerValues)
 {
     using namespace std::placeholders;
     using namespace ck::utils;
     namespace ctl = ck::tensor_layout::convolution;
+    using T       = float;
 
-    ck::utils::conv::ConvParams params;
-    params.num_dim_spatial_        = 1;
-    params.filter_spatial_lengths_ = std::vector<ck::index_t>{3};
-    params.input_spatial_lengths_  = std::vector<ck::index_t>{71};
-    params.conv_filter_strides_    = std::vector<ck::index_t>{2};
-    params.conv_filter_dilations_  = std::vector<ck::index_t>{1};
-    params.input_left_pads_        = std::vector<ck::index_t>{1};
-    params.input_right_pads_       = std::vector<ck::index_t>{1};
+    ck::utils::conv::ConvParams params{1, 4, 256, 64, {3}, {36}, {1}, {2}, {2}, {2}};
 
-    conv::ConvFwdOpInstance<T, T, T, ctl::NWC, ctl::KCX, ctl::NWK> conv_instance(params);
+    std::vector<test::conv::DeviceConvFwdNoOpPtr> conv_ptrs;
+    test::conv::get_test_convolution_fwd_instance<1, T, T, T, T>(conv_ptrs);
+    conv::ConvFwdOpInstance<T,
+                            T,
+                            T,
+                            ctl::NWC,
+                            ctl::KXC,
+                            ctl::NWK,
+                            ck::tensor_operation::element_wise::PassThrough,
+                            ck::tensor_operation::element_wise::PassThrough,
+                            ck::tensor_operation::element_wise::PassThrough,
+                            FillUniformDistributionIntegerValue<T>,
+                            FillUniformDistributionIntegerValue<T>>
+        conv_instance(params,
+                      true,
+                      FillUniformDistributionIntegerValue<T>{},
+                      FillUniformDistributionIntegerValue<T>{});
 
     auto reference_conv_fwd_fun =
         std::bind(conv::run_reference_convolution_forward<1, T, T, T>, params, _1, _2, _3);
     OpInstanceRunEngine<T, T, T> run_engine(conv_instance, reference_conv_fwd_fun);
-    return run_engine.Test(conv_ptrs);
+    run_engine.SetAtol(1e-5);
+    run_engine.SetRtol(1e-4);
+    EXPECT_TRUE(run_engine.Test(conv_ptrs));
 }
 
-} // anonymous namespace
-
-TEST(Conv1DFwdNWC, TestConv1D)
+TEST(Conv1DFwdNWC, FloatingPointValues)
 {
     using namespace std::placeholders;
     using namespace ck::utils;
     namespace ctl = ck::tensor_layout::convolution;
+    using T       = ck::half_t;
 
-    ck::utils::conv::ConvParams params;
-    params.num_dim_spatial_        = 1;
-    params.N_                      = 2;
-    params.K_                      = 16;
-    params.C_                      = 4;
-    params.filter_spatial_lengths_ = std::vector<ck::index_t>{3};
-    params.input_spatial_lengths_  = std::vector<ck::index_t>{16};
-    params.conv_filter_strides_    = std::vector<ck::index_t>{1};
-    params.conv_filter_dilations_  = std::vector<ck::index_t>{1};
-    params.input_left_pads_        = std::vector<ck::index_t>{1};
-    params.input_right_pads_       = std::vector<ck::index_t>{1};
+    ck::utils::conv::ConvParams params{1, 4, 256, 64, {3}, {36}, {1}, {2}, {2}, {2}};
 
     std::vector<test::conv::DeviceConvFwdNoOpPtr> conv_ptrs;
-    test::conv::get_test_convolution_fwd_instance<1>(conv_ptrs);
-    conv::ConvFwdOpInstance<float, float, float, ctl::NWC, ctl::KCX, ctl::NWK> conv_instance(
-        params);
+    test::conv::get_test_convolution_fwd_instance<1, T, T, T, float>(conv_ptrs);
+    conv::ConvFwdOpInstance<T,
+                            T,
+                            T,
+                            ctl::NWC,
+                            ctl::KXC,
+                            ctl::NWK,
+                            ck::tensor_operation::element_wise::PassThrough,
+                            ck::tensor_operation::element_wise::PassThrough,
+                            ck::tensor_operation::element_wise::PassThrough,
+                            FillUniformDistribution<T>,
+                            FillUniformDistribution<T>>
+        conv_instance(params, true, FillUniformDistribution<T>{}, FillUniformDistribution<T>{});
 
-    auto reference_conv_fwd_fun = std::bind(
-        conv::run_reference_convolution_forward<1, float, float, float>, params, _1, _2, _3);
-    OpInstanceRunEngine<float, float, float> run_engine(conv_instance, reference_conv_fwd_fun);
-    run_engine.SetAtol(1e-5);
-    run_engine.SetRtol(1e-4);
+    auto reference_conv_fwd_fun =
+        std::bind(conv::run_reference_convolution_forward<1, T, T, T>, params, _1, _2, _3);
+    OpInstanceRunEngine<T, T, T> run_engine(conv_instance, reference_conv_fwd_fun);
+    run_engine.SetAtol(0.1);
+    run_engine.SetRtol(1e-2);
     EXPECT_TRUE(run_engine.Test(conv_ptrs));
 }
 
-TEST(Conv1DFwdNWC, Bf16Iinstances)
+TEST_F(Conv1dFwdNWCInstances, BF16_default) { EXPECT_TRUE(this->test_default<ck::bhalf_t>()); }
+TEST_F(Conv1dFwdNWCInstances, BF16_filter1x1_stride1_pad0)
+{
+    EXPECT_TRUE(this->test_filter1x1_stride1_pad0<ck::bhalf_t>());
+}
+TEST_F(Conv1dFwdNWCInstances, BF16_filter1x1_pad0)
 {
-    EXPECT_TRUE(test_conv1d_nwc_instances<ck::bhalf_t>(
-        ck::utils::conv::ConvolutionFwdInstances<ck::bhalf_t, ck::bhalf_t, ck::bhalf_t>::Get<1>()));
+    EXPECT_TRUE(this->test_filter1x1_pad0<ck::bhalf_t>());
 }
 
-TEST(Conv1DFwdNWC, F16Instances)
+TEST_F(Conv1dFwdNWCInstances, F16_default) { EXPECT_TRUE(this->test_default<ck::half_t>()); }
+TEST_F(Conv1dFwdNWCInstances, F16_filter1x1_stride1_pad0)
+{
+    EXPECT_TRUE(this->test_filter1x1_stride1_pad0<ck::half_t>());
+}
+TEST_F(Conv1dFwdNWCInstances, F16_filter1x1_pad0)
 {
-    EXPECT_TRUE(test_conv1d_nwc_instances<ck::half_t>(
-        ck::utils::conv::ConvolutionFwdInstances<ck::half_t, ck::half_t, ck::half_t>::Get<1>()));
+    EXPECT_TRUE(this->test_filter1x1_pad0<ck::half_t>());
 }
 
-TEST(Conv1DFwdNWC, F32Instances)
+TEST_F(Conv1dFwdNWCInstances, F32_default) { EXPECT_TRUE(this->test_default<float>()); }
+TEST_F(Conv1dFwdNWCInstances, F32_filter1x1_stride1_pad0)
 {
-    EXPECT_TRUE(test_conv1d_nwc_instances<float>(
-        ck::utils::conv::ConvolutionFwdInstances<float, float, float>::Get<1>()));
+    EXPECT_TRUE(this->test_filter1x1_stride1_pad0<float>());
+}
+TEST_F(Conv1dFwdNWCInstances, F32_filter1x1_pad0)
+{
+    EXPECT_TRUE(this->test_filter1x1_pad0<float>());
 }
 
-TEST(Conv1DFwdNWC, Int8Instances)
+TEST_F(Conv1dFwdNWCInstances, I8_default) { EXPECT_TRUE(this->test_default<int8_t>()); }
+TEST_F(Conv1dFwdNWCInstances, I8_filter1x1_stride1_pad0)
+{
+    EXPECT_TRUE(this->test_filter1x1_stride1_pad0<int8_t>());
+}
+TEST_F(Conv1dFwdNWCInstances, I8_filter1x1_pad0)
 {
-    EXPECT_TRUE(test_conv1d_nwc_instances<int8_t>(
-        ck::utils::conv::ConvolutionFwdInstances<int8_t, int8_t, int8_t>::Get<1>()));
+    EXPECT_TRUE(this->test_filter1x1_pad0<int8_t>());
 }

test/convnd_fwd/conv2d_fwd.cpp

-#include <half.hpp>
-#include <iostream>
 #include <tuple>
 #include <vector>
 #include "gtest/gtest.h"
 
-#include "data_type.hpp"
-#include "element_wise_operation.hpp"
 #include "ck/library/utility/conv_util.hpp"
+#include "config.hpp"
 #include "conv_util.hpp"
+#include "data_type.hpp"
+#include "element_wise_operation.hpp"
+#include "fill.hpp"
 
 namespace {
 
-template <typename T>
-bool test_conv2d_nhwc_instances(const std::vector<test::conv::DeviceConvFwdNoOpPtr>& conv_ptrs)
+class Conv2dFwdNHWCInstances : public ::testing::Test
+{
+    public:
+    template <typename T>
+    bool test_conv2d_nhwc_instances(const std::vector<test::conv::DeviceConvFwdNoOpPtr>& conv_ptrs,
+                                    const ck::utils::conv::ConvParams& params)
+    {
+        using namespace std::placeholders;
+        using namespace ck::utils;
+
+        conv::ConvFwdOpInstance<T,
+                                T,
+                                T,
+                                ck::tensor_layout::convolution::NHWC,
+                                ck::tensor_layout::convolution::KYXC,
+                                ck::tensor_layout::convolution::NHWK,
+                                ck::tensor_operation::element_wise::PassThrough,
+                                ck::tensor_operation::element_wise::PassThrough,
+                                ck::tensor_operation::element_wise::PassThrough,
+                                FillUniformDistributionIntegerValue<T>,
+                                FillUniformDistributionIntegerValue<T>>
+            conv_instance(params,
+                          true,
+                          FillUniformDistributionIntegerValue<T>{},
+                          FillUniformDistributionIntegerValue<T>{});
+        auto reference_conv_fwd_fun =
+            std::bind(conv::run_reference_convolution_forward<2, T, T, T>, params, _1, _2, _3);
+        OpInstanceRunEngine<T, T, T> run_engine(conv_instance, reference_conv_fwd_fun);
+        run_engine.SetAtol(atol_);
+        run_engine.SetRtol(rtol_);
+        return run_engine.Test(conv_ptrs);
+    }
+
+    template <typename T>
+    bool test_default(bool use_convnd = false)
+    {
+        if(use_convnd)
+        {
+            return test_conv2d_nhwc_instances<T>(
+                test::conv::ConvolutionNDFwdInstances<T, T, T>::Get(2), params_default_);
+        }
+        else
+        {
+            return test_conv2d_nhwc_instances<T>(
+                ck::utils::conv::ConvolutionFwdInstances<T, T, T>::template Get<2>(),
+                params_default_);
+        }
+    }
+
+    template <typename T>
+    bool test_filter1x1_stride1_pad0(bool use_convnd = false)
+    {
+        if(use_convnd)
+        {
+            return test_conv2d_nhwc_instances<T>(
+                test::conv::ConvolutionNDFwdInstances<T, T, T>::Get(2),
+                params_filter1x1_stride1_pad0_);
+        }
+        else
+        {
+            return test_conv2d_nhwc_instances<T>(
+                ck::utils::conv::ConvolutionFwdInstances<T, T, T>::template Get<2>(),
+                params_filter1x1_stride1_pad0_);
+        }
+    }
+
+    template <typename T>
+    bool test_filter1x1_pad0(bool use_convnd = false)
+    {
+        if(use_convnd)
+        {
+            return test_conv2d_nhwc_instances<T>(
+                test::conv::ConvolutionNDFwdInstances<T, T, T>::Get(2), params_filter1x1_pad0_);
+        }
+        else
+        {
+            return test_conv2d_nhwc_instances<T>(
+                ck::utils::conv::ConvolutionFwdInstances<T, T, T>::template Get<2>(),
+                params_filter1x1_pad0_);
+        }
+    }
+
+    template <typename T>
+    bool test_oddC()
+    {
+        return test_conv2d_nhwc_instances<T>(
+            ck::utils::conv::ConvolutionFwdInstances<T, T, T>::template Get<2>(), params_oddC_);
+    }
+
+    static inline ck::utils::conv::ConvParams params_default_{
+        2, 4, 256, 64, {3, 3}, {36, 36}, {2, 2}, {2, 2}, {2, 2}, {2, 2}};
+    static inline ck::utils::conv::ConvParams params_filter1x1_stride1_pad0_{
+        2, 4, 256, 64, {1, 1}, {28, 28}, {1, 1}, {1, 1}, {0, 0}, {0, 0}};
+    static inline ck::utils::conv::ConvParams params_filter1x1_pad0_{
+        2, 4, 256, 64, {1, 1}, {28, 28}, {2, 2}, {1, 1}, {0, 0}, {0, 0}};
+    static inline ck::utils::conv::ConvParams params_oddC_{
+        2, 4, 256, 3, {3, 3}, {28, 28}, {1, 1}, {1, 1}, {0, 0}, {0, 0}};
+
+    private:
+    double atol_{1e-5};
+    double rtol_{1e-4};
+};
+
+} // anonymous namespace
+
+TEST(Conv2DFwdNHWC, IntegerValues)
 {
     using namespace std::placeholders;
     using namespace ck::utils;
+    using T = float;
 
-    conv::ConvParams params;
-    params.num_dim_spatial_        = 2;
-    params.filter_spatial_lengths_ = std::vector<ck::index_t>{3, 3};
-    params.input_spatial_lengths_  = std::vector<ck::index_t>{71, 71};
-    params.conv_filter_strides_    = std::vector<ck::index_t>{2, 2};
-    params.conv_filter_dilations_  = std::vector<ck::index_t>{1, 1};
-    params.input_left_pads_        = std::vector<ck::index_t>{1, 1};
-    params.input_right_pads_       = std::vector<ck::index_t>{1, 1};
+    ck::utils::conv::ConvParams params{
+        2, 4, 256, 64, {3, 3}, {36, 36}, {1, 1}, {2, 2}, {2, 2}, {2, 2}};
 
-    conv::ConvFwdOpInstance<T, T, T> conv_instance(params);
+    std::vector<test::conv::DeviceConvFwdNoOpPtr> conv_ptrs;
+    test::conv::get_test_convolution_fwd_instance<2, T, T, T, T>(conv_ptrs);
+    conv::ConvFwdOpInstance<T,
+                            T,
+                            T,
+                            ck::tensor_layout::convolution::NHWC,
+                            ck::tensor_layout::convolution::KYXC,
+                            ck::tensor_layout::convolution::NHWK,
+                            ck::tensor_operation::element_wise::PassThrough,
+                            ck::tensor_operation::element_wise::PassThrough,
+                            ck::tensor_operation::element_wise::PassThrough,
+                            FillUniformDistributionIntegerValue<T>,
+                            FillUniformDistributionIntegerValue<T>>
+        conv_instance(params,
+                      true,
+                      FillUniformDistributionIntegerValue<T>{},
+                      FillUniformDistributionIntegerValue<T>{});
 
     auto reference_conv_fwd_fun =
         std::bind(conv::run_reference_convolution_forward<2, T, T, T>, params, _1, _2, _3);
     OpInstanceRunEngine<T, T, T> run_engine(conv_instance, reference_conv_fwd_fun);
-    return run_engine.Test(conv_ptrs);
+    run_engine.SetAtol(1e-5);
+    run_engine.SetRtol(1e-4);
+    EXPECT_TRUE(run_engine.Test(conv_ptrs));
 }
 
-} // anonymous namespace
-
-TEST(Conv2DFwdNHWC, TestConv2D)
+TEST(Conv2DFwdNHWC, FloatingPointValues)
 {
     using namespace std::placeholders;
     using namespace ck::utils;
+    using T = ck::half_t;
 
-    ck::utils::conv::ConvParams params;
-    params.N_                     = 2;
-    params.K_                     = 16;
-    params.C_                     = 4;
-    params.input_spatial_lengths_ = std::vector<ck::index_t>{16, 16};
-    params.conv_filter_strides_   = std::vector<ck::index_t>{1, 1};
+    ck::utils::conv::ConvParams params{
+        2, 4, 256, 64, {3, 3}, {36, 36}, {2, 2}, {2, 2}, {2, 2}, {2, 2}};
 
     std::vector<test::conv::DeviceConvFwdNoOpPtr> conv_ptrs;
-    test::conv::get_test_convolution_fwd_instance<2>(conv_ptrs);
-    conv::ConvFwdOpInstance<float, float, float> conv_instance(params);
+    test::conv::get_test_convolution_fwd_instance<2, T, T, T, float>(conv_ptrs);
+    conv::ConvFwdOpInstance<T,
+                            T,
+                            T,
+                            ck::tensor_layout::convolution::NHWC,
+                            ck::tensor_layout::convolution::KYXC,
+                            ck::tensor_layout::convolution::NHWK,
+                            ck::tensor_operation::element_wise::PassThrough,
+                            ck::tensor_operation::element_wise::PassThrough,
+                            ck::tensor_operation::element_wise::PassThrough,
+                            FillUniformDistribution<T>,
+                            FillUniformDistribution<T>>
+        conv_instance(params, true, FillUniformDistribution<T>{}, FillUniformDistribution<T>{});
 
-    auto reference_conv_fwd_fun = std::bind(
-        conv::run_reference_convolution_forward<2, float, float, float>, params, _1, _2, _3);
-    OpInstanceRunEngine<float, float, float> run_engine(conv_instance, reference_conv_fwd_fun);
-    run_engine.SetAtol(1e-5);
-    run_engine.SetRtol(1e-4);
+    auto reference_conv_fwd_fun =
+        std::bind(conv::run_reference_convolution_forward<2, T, T, T>, params, _1, _2, _3);
+    OpInstanceRunEngine<T, T, T> run_engine(conv_instance, reference_conv_fwd_fun);
+    run_engine.SetAtol(2e-4);
+    run_engine.SetRtol(1e-3);
     EXPECT_TRUE(run_engine.Test(conv_ptrs));
 }
 
-TEST(Conv2DFwdNHWC, Bf16Instances)
+TEST_F(Conv2dFwdNHWCInstances, BF16_default) { EXPECT_TRUE(this->test_default<ck::bhalf_t>()); }
+TEST_F(Conv2dFwdNHWCInstances, BF16_filter1x1_stride1_pad0)
 {
-    EXPECT_TRUE(test_conv2d_nhwc_instances<ck::bhalf_t>(
-        ck::utils::conv::ConvolutionFwdInstances<ck::bhalf_t, ck::bhalf_t, ck::bhalf_t>::Get<2>()));
+    EXPECT_TRUE(this->test_filter1x1_stride1_pad0<ck::bhalf_t>());
 }
-
-TEST(Conv2DFwdNHWC, F16Instances)
+TEST_F(Conv2dFwdNHWCInstances, BF16_filter1x1_pad0)
 {
-    EXPECT_TRUE(test_conv2d_nhwc_instances<ck::half_t>(
-        ck::utils::conv::ConvolutionFwdInstances<ck::half_t, ck::half_t, ck::half_t>::Get<2>()));
+    EXPECT_TRUE(this->test_filter1x1_pad0<ck::bhalf_t>());
 }
-
-TEST(Conv2DFwdNHWC, BF32Instances)
+TEST_F(Conv2dFwdNHWCInstances, F16_default) { EXPECT_TRUE(this->test_default<ck::half_t>()); }
+TEST_F(Conv2dFwdNHWCInstances, F16_filter1x1_stride1_pad0)
 {
-    EXPECT_TRUE(test_conv2d_nhwc_instances<float>(
-        ck::utils::conv::ConvolutionFwdInstances<float, float, float>::Get<2>()));
+    EXPECT_TRUE(this->test_filter1x1_stride1_pad0<ck::half_t>());
 }
-
-TEST(Conv2DFwdNHWC, F32Instances)
+TEST_F(Conv2dFwdNHWCInstances, F16_filter1x1_pad0)
+{
+    EXPECT_TRUE(this->test_filter1x1_pad0<ck::half_t>());
+}
+TEST_F(Conv2dFwdNHWCInstances, F16_oddC) { EXPECT_TRUE(this->test_oddC<ck::half_t>()); }
+TEST_F(Conv2dFwdNHWCInstances, F32_default) { EXPECT_TRUE(this->test_default<float>()); }
+TEST_F(Conv2dFwdNHWCInstances, F32_filter1x1_stride1_pad0)
+{
+    EXPECT_TRUE(this->test_filter1x1_stride1_pad0<float>());
+}
+TEST_F(Conv2dFwdNHWCInstances, F32_filter1x1_pad0)
+{
+    EXPECT_TRUE(this->test_filter1x1_pad0<float>());
+}
+TEST_F(Conv2dFwdNHWCInstances, I8_default) { EXPECT_TRUE(this->test_default<int8_t>()); }
+TEST_F(Conv2dFwdNHWCInstances, I8_filter1x1_stride1_pad0)
 {
-    EXPECT_TRUE(test_conv2d_nhwc_instances<float>(
-        ck::utils::conv::ConvolutionFwdInstances<float, float, float>::Get<2>()));
+    EXPECT_TRUE(this->test_filter1x1_stride1_pad0<int8_t>());
+}
+TEST_F(Conv2dFwdNHWCInstances, I8_filter1x1_pad0)
+{
+    EXPECT_TRUE(this->test_filter1x1_pad0<int8_t>());
 }
 
-TEST(Conv2DFwdNHWC, Int8Instances)
+TEST_F(Conv2dFwdNHWCInstances, ND_BF16_default)
+{
+    EXPECT_TRUE(this->test_default<ck::bhalf_t>(true));
+}
+TEST_F(Conv2dFwdNHWCInstances, ND_BF16_filter1x1_stride1_pad0)
+{
+    EXPECT_TRUE(this->test_filter1x1_stride1_pad0<ck::bhalf_t>(true));
+}
+TEST_F(Conv2dFwdNHWCInstances, ND_BF16_filter1x1_pad0)
+{
+    EXPECT_TRUE(this->test_filter1x1_pad0<ck::bhalf_t>(true));
+}
+TEST_F(Conv2dFwdNHWCInstances, ND_F16_default)
+{
+    EXPECT_TRUE(this->test_default<ck::half_t>(true));
+}
+TEST_F(Conv2dFwdNHWCInstances, ND_F16_filter1x1_stride1_pad0)
+{
+    EXPECT_TRUE(this->test_filter1x1_stride1_pad0<ck::half_t>(true));
+}
+TEST_F(Conv2dFwdNHWCInstances, ND_F16_filter1x1_pad0)
+{
+    EXPECT_TRUE(this->test_filter1x1_pad0<ck::half_t>(true));
+}
+TEST_F(Conv2dFwdNHWCInstances, ND_F32_default) { EXPECT_TRUE(this->test_default<float>(true)); }
+TEST_F(Conv2dFwdNHWCInstances, ND_F32_filter1x1_stride1_pad0)
+{
+    EXPECT_TRUE(this->test_filter1x1_stride1_pad0<float>(true));
+}
+TEST_F(Conv2dFwdNHWCInstances, ND_F32_filter1x1_pad0)
+{
+    EXPECT_TRUE(this->test_filter1x1_pad0<float>(true));
+}
+TEST_F(Conv2dFwdNHWCInstances, ND_I8_default) { EXPECT_TRUE(this->test_default<int8_t>(true)); }
+TEST_F(Conv2dFwdNHWCInstances, ND_I8_filter1x1_stride1_pad0)
+{
+    EXPECT_TRUE(this->test_filter1x1_stride1_pad0<int8_t>(true));
+}
+TEST_F(Conv2dFwdNHWCInstances, ND_I8_filter1x1_pad0)
 {
-    EXPECT_TRUE(test_conv2d_nhwc_instances<int8_t>(
-        ck::utils::conv::ConvolutionFwdInstances<int8_t, int8_t, int8_t>::Get<2>()));
+    EXPECT_TRUE(this->test_filter1x1_pad0<int8_t>(true));
 }

test/convnd_fwd/conv3d_fwd.cpp

@@ -12,61 +12,143 @@
 
 namespace {
 
-template <typename T>
-bool test_conv3d_ndhwc_instances(const std::vector<test::conv::DeviceConvFwdNoOpPtr>& conv_ptrs)
+class Conv3dFwdNDHWCInstances : public ::testing::Test
+{
+    public:
+    template <typename T>
+    bool test_conv3d_nwc_instances(const std::vector<test::conv::DeviceConvFwdNoOpPtr>& conv_ptrs,
+                                   const ck::utils::conv::ConvParams& params)
+    {
+        using namespace std::placeholders;
+        using namespace ck::utils;
+        namespace ctl = ck::tensor_layout::convolution;
+
+        conv::ConvFwdOpInstance<T,
+                                T,
+                                T,
+                                ctl::NDHWC,
+                                ctl::KZYXC,
+                                ctl::NDHWK,
+                                ck::tensor_operation::element_wise::PassThrough,
+                                ck::tensor_operation::element_wise::PassThrough,
+                                ck::tensor_operation::element_wise::PassThrough,
+                                FillUniformDistributionIntegerValue<T>,
+                                FillUniformDistributionIntegerValue<T>>
+            conv_instance(params,
+                          true,
+                          FillUniformDistributionIntegerValue<T>{},
+                          FillUniformDistributionIntegerValue<T>{});
+        auto reference_conv_fwd_fun =
+            std::bind(conv::run_reference_convolution_forward<3, T, T, T>, params, _1, _2, _3);
+        OpInstanceRunEngine<T, T, T> run_engine(conv_instance, reference_conv_fwd_fun);
+        run_engine.SetAtol(atol_);
+        run_engine.SetRtol(rtol_);
+        return run_engine.Test(conv_ptrs);
+    }
+
+    template <typename T>
+    bool test_default()
+    {
+        return test_conv3d_nwc_instances<T>(
+            ck::utils::conv::ConvolutionFwdInstances<T, T, T>::template Get<3>(), params_default_);
+    }
+
+    template <typename T>
+    bool test_filter1x1_stride1_pad0()
+    {
+        return test_conv3d_nwc_instances<T>(
+            ck::utils::conv::ConvolutionFwdInstances<T, T, T>::template Get<3>(),
+            params_filter1x1_stride1_pad0_);
+    }
+
+    template <typename T>
+    bool test_filter1x1_pad0()
+    {
+        return test_conv3d_nwc_instances<T>(
+            ck::utils::conv::ConvolutionFwdInstances<T, T, T>::template Get<3>(),
+            params_filter1x1_pad0_);
+    }
+
+    static inline ck::utils::conv::ConvParams params_default_{
+        3, 4, 256, 64, {3, 3, 3}, {28, 28, 28}, {2, 2, 2}, {2, 2, 2}, {2, 2, 2}, {2, 2, 2}};
+    static inline ck::utils::conv::ConvParams params_filter1x1_stride1_pad0_{
+        3, 4, 256, 64, {1, 1, 1}, {28, 28, 28}, {1, 1, 1}, {1, 1, 1}, {0, 0, 0}, {0, 0, 0}};
+    static inline ck::utils::conv::ConvParams params_filter1x1_pad0_{
+        3, 4, 256, 64, {1, 1, 1}, {28, 28, 28}, {2, 2, 2}, {1, 1, 1}, {0, 0, 0}, {0, 0, 0}};
+
+    private:
+    double atol_{1e-5};
+    double rtol_{1e-4};
+};
+
+} // anonymous namespace
+
+TEST(Conv3DFwdNDHWC, IntegerValues)
 {
     using namespace std::placeholders;
     using namespace ck::utils;
     namespace ctl = ck::tensor_layout::convolution;
+    using T       = float;
 
-    conv::ConvParams params;
-    params.N_                      = 64;
-    params.num_dim_spatial_        = 3;
-    params.filter_spatial_lengths_ = std::vector<ck::index_t>{3, 3, 2};
-    params.input_spatial_lengths_  = std::vector<ck::index_t>{32, 32, 2};
-    params.conv_filter_strides_    = std::vector<ck::index_t>{2, 2, 2};
-    params.conv_filter_dilations_  = std::vector<ck::index_t>{1, 1, 1};
-    params.input_left_pads_        = std::vector<ck::index_t>{1, 1, 1};
-    params.input_right_pads_       = std::vector<ck::index_t>{1, 1, 1};
+    ck::utils::conv::ConvParams params{
+        3, 4, 256, 64, {3, 3, 3}, {18, 18, 18}, {1, 1, 1}, {2, 2, 2}, {2, 2, 2}, {2, 2, 2}};
 
-    conv::ConvFwdOpInstance<T, T, T, ctl::NDHWC, ctl::KZYXC, ctl::NDHWK> conv_instance(params);
+    std::vector<test::conv::DeviceConvFwdNoOpPtr> conv_ptrs;
+    test::conv::get_test_convolution_fwd_instance<3, T, T, T, T>(conv_ptrs);
+    conv::ConvFwdOpInstance<T,
+                            T,
+                            T,
+                            ctl::NDHWC,
+                            ctl::KZYXC,
+                            ctl::NDHWK,
+                            ck::tensor_operation::element_wise::PassThrough,
+                            ck::tensor_operation::element_wise::PassThrough,
+                            ck::tensor_operation::element_wise::PassThrough,
+                            FillUniformDistributionIntegerValue<T>,
+                            FillUniformDistributionIntegerValue<T>>
+        conv_instance(params,
+                      true,
+                      FillUniformDistributionIntegerValue<T>{},
+                      FillUniformDistributionIntegerValue<T>{});
 
     auto reference_conv_fwd_fun =
         std::bind(conv::run_reference_convolution_forward<3, T, T, T>, params, _1, _2, _3);
     OpInstanceRunEngine<T, T, T> run_engine(conv_instance, reference_conv_fwd_fun);
-    return run_engine.Test(conv_ptrs);
+    run_engine.SetAtol(1e-5);
+    run_engine.SetRtol(1e-3);
+    EXPECT_TRUE(run_engine.Test(conv_ptrs));
 }
 
-} // anonymous namespace
-
-TEST(Conv3DFwdNDHWC, TestConv3D)
+TEST(Conv3DFwdNDHWC, FloatingPointValues)
 {
     using namespace std::placeholders;
     using namespace ck::utils;
     namespace ctl = ck::tensor_layout::convolution;
+    using T       = ck::half_t;
 
-    conv::ConvParams params;
-    params.num_dim_spatial_        = 3;
-    params.N_                      = 2;
-    params.K_                      = 16;
-    params.C_                      = 4;
-    params.filter_spatial_lengths_ = std::vector<ck::index_t>{3, 3, 3};
-    params.input_spatial_lengths_  = std::vector<ck::index_t>{16, 16, 16};
-    params.conv_filter_strides_    = std::vector<ck::index_t>{1, 1, 1};
-    params.conv_filter_dilations_  = std::vector<ck::index_t>{1, 1, 1};
-    params.input_left_pads_        = std::vector<ck::index_t>{1, 1, 1};
-    params.input_right_pads_       = std::vector<ck::index_t>{1, 1, 1};
+    ck::utils::conv::ConvParams params{
+        3, 4, 256, 64, {3, 3, 3}, {18, 18, 18}, {1, 1, 1}, {2, 2, 2}, {2, 2, 2}, {2, 2, 2}};
 
     std::vector<test::conv::DeviceConvFwdNoOpPtr> conv_ptrs;
-    test::conv::get_test_convolution_fwd_instance<3>(conv_ptrs);
-    conv::ConvFwdOpInstance<float, float, float, ctl::NDHWC, ctl::KZYXC, ctl::NDHWK> conv_instance(
-        params);
+    test::conv::get_test_convolution_fwd_instance<3, T, T, T, float>(conv_ptrs);
+    conv::ConvFwdOpInstance<T,
+                            T,
+                            T,
+                            ctl::NDHWC,
+                            ctl::KZYXC,
+                            ctl::NDHWK,
+                            ck::tensor_operation::element_wise::PassThrough,
+                            ck::tensor_operation::element_wise::PassThrough,
+                            ck::tensor_operation::element_wise::PassThrough,
+                            FillUniformDistribution<T>,
+                            FillUniformDistribution<T>>
+        conv_instance(params, true, FillUniformDistribution<T>{}, FillUniformDistribution<T>{});
 
-    auto reference_conv_fwd_fun = std::bind(
-        conv::run_reference_convolution_forward<3, float, float, float>, params, _1, _2, _3);
-    OpInstanceRunEngine<float, float, float> run_engine(conv_instance, reference_conv_fwd_fun);
-    run_engine.SetAtol(1e-5);
-    run_engine.SetRtol(1e-4);
+    auto reference_conv_fwd_fun =
+        std::bind(conv::run_reference_convolution_forward<3, T, T, T>, params, _1, _2, _3);
+    OpInstanceRunEngine<T, T, T> run_engine(conv_instance, reference_conv_fwd_fun);
+    run_engine.SetAtol(1e-3);
+    run_engine.SetRtol(1e-3);
     EXPECT_TRUE(run_engine.Test(conv_ptrs));
 }
 
@@ -74,6 +156,7 @@ TEST(Conv3DFwdNDHWC, InputOver2GB)
 {
     using PassThrough = ck::tensor_operation::element_wise::PassThrough;
     using namespace ck::utils;
+    using T = float;
 
     // >2GB Input
     conv::ConvParams params;
@@ -89,8 +172,7 @@ TEST(Conv3DFwdNDHWC, InputOver2GB)
     params.input_right_pads_       = std::vector<ck::index_t>{1, 1, 1};
 
     std::vector<test::conv::DeviceConvFwdNoOpPtr> conv_ptrs;
-    test::conv::get_test_convolution_fwd_instance<3>(conv_ptrs);
-
+    test::conv::get_test_convolution_fwd_instance<3, T, T, T, T>(conv_ptrs);
     auto arg = conv_ptrs.back()->MakeArgumentPointer(nullptr,
                                                      nullptr,
                                                      nullptr,
@@ -114,6 +196,7 @@ TEST(Conv3DFwdNDHWC, FiltersOver2GB)
 {
     using PassThrough = ck::tensor_operation::element_wise::PassThrough;
     using namespace ck::utils;
+    using T = float;
 
     // >2GB Filters
     conv::ConvParams params;
@@ -129,8 +212,7 @@ TEST(Conv3DFwdNDHWC, FiltersOver2GB)
     params.input_right_pads_       = std::vector<ck::index_t>{1, 1, 1};
 
     std::vector<test::conv::DeviceConvFwdNoOpPtr> conv_ptrs;
-    test::conv::get_test_convolution_fwd_instance<3>(conv_ptrs);
-
+    test::conv::get_test_convolution_fwd_instance<3, T, T, T, T>(conv_ptrs);
     auto arg = conv_ptrs.back()->MakeArgumentPointer(nullptr,
                                                      nullptr,
                                                      nullptr,
@@ -154,6 +236,7 @@ TEST(Conv3DFwdNDHWC, OutputOver2GB)
 {
     using PassThrough = ck::tensor_operation::element_wise::PassThrough;
     using namespace ck::utils;
+    using T = float;
 
     // >2GB Output
     conv::ConvParams params;
@@ -169,7 +252,7 @@ TEST(Conv3DFwdNDHWC, OutputOver2GB)
     params.input_right_pads_       = std::vector<ck::index_t>{2, 2, 2};
 
     std::vector<test::conv::DeviceConvFwdNoOpPtr> conv_ptrs;
-    test::conv::get_test_convolution_fwd_instance<3>(conv_ptrs);
+    test::conv::get_test_convolution_fwd_instance<3, T, T, T, T>(conv_ptrs);
     auto arg = conv_ptrs.back()->MakeArgumentPointer(nullptr,
                                                      nullptr,
                                                      nullptr,
@@ -189,26 +272,42 @@ TEST(Conv3DFwdNDHWC, OutputOver2GB)
     EXPECT_FALSE(conv_ptrs.back()->IsSupportedArgument(arg.get()));
 }
 
-TEST(Conv3DFwdNDHWC, Bf16Instances)
+TEST_F(Conv3dFwdNDHWCInstances, BF16_default) { EXPECT_TRUE(this->test_default<ck::bhalf_t>()); }
+TEST_F(Conv3dFwdNDHWCInstances, BF16_filter1x1_stride1_pad0)
 {
-    EXPECT_TRUE(test_conv3d_ndhwc_instances<ck::bhalf_t>(
-        ck::utils::conv::ConvolutionFwdInstances<ck::bhalf_t, ck::bhalf_t, ck::bhalf_t>::Get<3>()));
+    EXPECT_TRUE(this->test_filter1x1_stride1_pad0<ck::bhalf_t>());
+}
+TEST_F(Conv3dFwdNDHWCInstances, BF16_filter1x1_pad0)
+{
+    EXPECT_TRUE(this->test_filter1x1_pad0<ck::bhalf_t>());
 }
 
-TEST(Conv3DFwdNDHWC, F16Instances)
+TEST_F(Conv3dFwdNDHWCInstances, F16_default) { EXPECT_TRUE(this->test_default<ck::half_t>()); }
+TEST_F(Conv3dFwdNDHWCInstances, F16_filter1x1_stride1_pad0)
 {
-    EXPECT_TRUE(test_conv3d_ndhwc_instances<ck::half_t>(
-        ck::utils::conv::ConvolutionFwdInstances<ck::half_t, ck::half_t, ck::half_t>::Get<3>()));
+    EXPECT_TRUE(this->test_filter1x1_stride1_pad0<ck::half_t>());
+}
+TEST_F(Conv3dFwdNDHWCInstances, F16_filter1x1_pad0)
+{
+    EXPECT_TRUE(this->test_filter1x1_pad0<ck::half_t>());
 }
 
-TEST(Conv3DFwdNDHWC, F32Instances)
+TEST_F(Conv3dFwdNDHWCInstances, F32_default) { EXPECT_TRUE(this->test_default<float>()); }
+TEST_F(Conv3dFwdNDHWCInstances, F32_filter1x1_stride1_pad0)
+{
+    EXPECT_TRUE(this->test_filter1x1_stride1_pad0<float>());
+}
+TEST_F(Conv3dFwdNDHWCInstances, F32_filter1x1_pad0)
 {
-    EXPECT_TRUE(test_conv3d_ndhwc_instances<float>(
-        ck::utils::conv::ConvolutionFwdInstances<float, float, float>::Get<3>()));
+    EXPECT_TRUE(this->test_filter1x1_pad0<float>());
 }
 
-TEST(Conv3DFwdNDHWC, Int8Instances)
+TEST_F(Conv3dFwdNDHWCInstances, I8_default) { EXPECT_TRUE(this->test_default<int8_t>()); }
+TEST_F(Conv3dFwdNDHWCInstances, I8_filter1x1_stride1_pad0)
+{
+    EXPECT_TRUE(this->test_filter1x1_stride1_pad0<int8_t>());
+}
+TEST_F(Conv3dFwdNDHWCInstances, I8_filter1x1_pad0)
 {
-    EXPECT_TRUE(test_conv3d_ndhwc_instances<int8_t>(
-        ck::utils::conv::ConvolutionFwdInstances<int8_t, int8_t, int8_t>::Get<3>()));
+    EXPECT_TRUE(this->test_filter1x1_pad0<int8_t>());
 }

test/convnd_fwd/conv_util.hpp

-#ifndef TEST_CONV_UTIL_HPP
-#define TEST_CONV_UTIL_HPP
+#pragma once
 
 #include <tuple>
 
 #include "config.hpp"
+#include "data_type.hpp"
 #include "device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp"
 #include "element_wise_operation.hpp"
 #include "host_tensor.hpp"
 #include "sequence.hpp"
 
+namespace ck {
+namespace tensor_operation {
+namespace device {
+
+using DeviceConvFwdNoOpPtr = DeviceConvFwdPtr<element_wise::PassThrough,
+                                              element_wise::PassThrough,
+                                              element_wise::PassThrough>;
+namespace device_conv2d_fwd_instance {
+
+void add_device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instances(std::vector<DeviceConvFwdNoOpPtr>&);
+void add_device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instances(std::vector<DeviceConvFwdNoOpPtr>&);
+void add_device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instances(std::vector<DeviceConvFwdNoOpPtr>&);
+void add_device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instances(std::vector<DeviceConvFwdNoOpPtr>&);
+
+} // namespace device_conv2d_fwd_instance
+} // namespace device
+} // namespace tensor_operation
+} // namespace ck
+
 namespace test {
 namespace conv {
 
@@ -25,57 +44,128 @@ using DeviceConvFwdNoOpPtr =
 static constexpr auto ConvFwdDefault =
     ck::tensor_operation::device::ConvolutionForwardSpecialization::Default;
 
-template <ck::index_t SpatialDims, typename InDataType, typename WeiDataType, typename OutDataType>
+template <ck::index_t SpatialDims,
+          typename InDataType,
+          typename WeiDataType,
+          typename OutDataType,
+          typename AccDataType>
 using DeviceConvNDFwdInstance = ck::tensor_operation::device::
     DeviceConvNDFwdXdl_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K<
         // clang-format off
         InDataType,         // 
         WeiDataType,        //
         OutDataType,        //
-        InDataType,         // 
+        AccDataType,        // Accumulator data type.
         InElementOp,        // Input Elementwise Operation
         WeiElementOp,       // Weights Elementwise Operation
         OutElementOp,       // Output Elementwise Operation
         ConvFwdDefault,     // ConvForwardSpecialization
         SpatialDims,        // SptialDims
-        64,                 // BlockSize
-        16,                 // MPerBlock
-        16,                 // NPerBlock
+        256,                // BlockSize
+        128,                // MPerBlock
+        256,                // NPerBlock
         4,                  // K0PerBlock
-        1,                  // K1                                           
-        16,                 // MPerXDL
-        16,                 // NPerXDL
-        1,                  // MXdlPerWave
-        1,                  // NXdlPerWave
-        S<1, 16, 1>,        // ABlockTransferThreadClusterLengths_K0_M_K1
+        8,                  // K1
+        32,                 // MPerXdl
+        32,                 // NPerXdl
+        2,                  // MXdlPerWave
+        4,                  // NXdlPerWave
+        S<4, 64, 1>,        // ABlockTransferThreadClusterLengths_K0_M_K1
         S<1, 0, 2>,         // ABlockTransferThreadClusterArrangeOrder
         S<1, 0, 2>,         // ABlockTransferSrcAccessOrder
         2,                  // ABlockTransferSrcVectorDim
-        1,                  // ABlockTransferSrcScalarPerVector
-        1,                  // ABlockTransferDstScalarPerVector_K1
+        8,                  // ABlockTransferSrcScalarPerVector
+        8,                  // ABlockTransferDstScalarPerVector_K1
         true,               // ABlockLdsAddExtraM
-        S<1, 16, 1>,        // BBlockTransferThreadClusterLengths_K0_N_K1
+        S<4, 64, 1>,        // BBlockTransferThreadClusterLengths_K0_N_K1
         S<1, 0, 2>,         // BBlockTransferThreadClusterArrangeOrder
         S<1, 0, 2>,         // BBlockTransferSrcAccessOrder
         2,                  // BBlockTransferSrcVectorDim
-        1,                  // BBlockTransferSrcScalarPerVector
-        1,                  // BBlockTransferDstScalarPerVector_K1
-        true,               // BBlockTransferAddExtraN
+        8,                  // BBlockTransferSrcScalarPerVector
+        8,                  // BBlockTransferDstScalarPerVector_K1
+        true,               // BBlockLdsAddExtraN
         7,                  // CThreadTransferSrcDstVectorDim
-        1>;                 // CThreadTransferDstScalarPerVector
+        1>;                // CThreadTransferDstScalarPerVector
 // clang-format on
 
 template <ck::index_t NDim,
-          typename InDataType  = float,
-          typename WeiDataType = float,
-          typename OutDataType = float>
+          typename InDataType,
+          typename WeiDataType,
+          typename OutDataType,
+          typename AccDataType>
 void get_test_convolution_fwd_instance(std::vector<DeviceConvFwdNoOpPtr>& instances)
 {
-    using ConvInstanceT = DeviceConvNDFwdInstance<NDim, InDataType, WeiDataType, OutDataType>;
+    using ConvInstanceT =
+        DeviceConvNDFwdInstance<NDim, InDataType, WeiDataType, OutDataType, AccDataType>;
     instances.emplace_back(std::make_unique<ConvInstanceT>());
 }
 
+// TODO (aosewski)
+// Temporary solution to get all DeviceConvNDFwdXdl_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K
+// instances. When switched over to DeviceConvNDFwdXdl for 2D remove ConvolutionNDFwdInstances
+// structures.
+template <typename InDataType, typename WeiDataType, typename OutDataType>
+struct ConvolutionNDFwdInstances;
+
+template <>
+struct ConvolutionNDFwdInstances<float, float, float>
+{
+    static std::vector<DeviceConvFwdNoOpPtr> Get(std::size_t num_dim_spatial)
+    {
+        std::vector<DeviceConvFwdNoOpPtr> conv_ptrs;
+        if(num_dim_spatial == 2)
+        {
+            ck::tensor_operation::device::device_conv2d_fwd_instance::
+                add_device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instances(conv_ptrs);
+        }
+        return conv_ptrs;
+    }
+};
+
+template <>
+struct ConvolutionNDFwdInstances<ck::half_t, ck::half_t, ck::half_t>
+{
+    static std::vector<DeviceConvFwdNoOpPtr> Get(std::size_t num_dim_spatial)
+    {
+        std::vector<DeviceConvFwdNoOpPtr> conv_ptrs;
+        if(num_dim_spatial == 2)
+        {
+            ck::tensor_operation::device::device_conv2d_fwd_instance::
+                add_device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instances(conv_ptrs);
+        }
+        return conv_ptrs;
+    }
+};
+
+template <>
+struct ConvolutionNDFwdInstances<ck::bhalf_t, ck::bhalf_t, ck::bhalf_t>
+{
+    static std::vector<DeviceConvFwdNoOpPtr> Get(std::size_t num_dim_spatial)
+    {
+        std::vector<DeviceConvFwdNoOpPtr> conv_ptrs;
+        if(num_dim_spatial == 2)
+        {
+            ck::tensor_operation::device::device_conv2d_fwd_instance::
+                add_device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instances(conv_ptrs);
+        }
+        return conv_ptrs;
+    }
+};
+
+template <>
+struct ConvolutionNDFwdInstances<int8_t, int8_t, int8_t>
+{
+    static std::vector<DeviceConvFwdNoOpPtr> Get(std::size_t num_dim_spatial)
+    {
+        std::vector<DeviceConvFwdNoOpPtr> conv_ptrs;
+        if(num_dim_spatial == 2)
+        {
+            ck::tensor_operation::device::device_conv2d_fwd_instance::
+                add_device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instances(conv_ptrs);
+        }
+        return conv_ptrs;
+    }
+};
+
 } // namespace conv
 } // namespace test
-
-#endif

test/softmax/CMakeLists.txt

+add_custom_target(test_softmax)
+
+add_gtest_executable(test_softmax_fp32 test_softmax_fp32.cpp)
+add_gtest_executable(test_softmax_fp16 test_softmax_fp16.cpp)
+target_link_libraries(test_softmax_fp32 PRIVATE host_tensor)
+target_link_libraries(test_softmax_fp16 PRIVATE host_tensor)
+add_dependencies(test_softmax test_softmax_fp32)
+add_dependencies(test_softmax test_softmax_fp16)
\ No newline at end of file

test/softmax/test_softmax_fp16.cpp

+#include "gtest/gtest.h"
+#include "test_softmax_util.hpp"
+
+template <ck::index_t N>
+using I = ck::Number<N>;
+
+template <typename Tuple>
+class TestSoftmaxFP16 : public ck::TestSoftmax<Tuple>
+{
+};
+
+// clang-format off
+using KernelTypes = ::testing::Types<
+// InDataType, AccDataType, OutDataType, Rank, NumReduceDim, BlockSize, MThreadClusterSize, KThreadClusterSize, MThreadSliceSize, KThreadSliceSize, InSrcVectorDim, InSrcVectorSize, OutDstVectorSize>
+    std::tuple<ck::half_t, float, ck::half_t, I<3>, I<1>, I<256>, I<8>, I<32>, I<1>, I<8>, I<1>, I<8>, I<8>>,
+    std::tuple<ck::half_t, float, ck::half_t, I<3>, I<1>, I<256>, I<4>, I<64>, I<1>, I<8>, I<1>, I<8>, I<8>>,
+    std::tuple<ck::half_t, float, ck::half_t, I<3>, I<1>, I<256>, I<2>, I<128>, I<1>, I<8>, I<1>, I<8>, I<8>>,
+    std::tuple<ck::half_t, float, ck::half_t, I<3>, I<1>, I<256>, I<1>, I<256>, I<1>, I<8>, I<1>, I<8>, I<8>>,
+    std::tuple<ck::half_t, float, ck::half_t, I<3>, I<2>, I<256>, I<8>, I<32>, I<1>, I<8>, I<1>, I<8>, I<8>>,
+    std::tuple<ck::half_t, float, ck::half_t, I<3>, I<2>, I<256>, I<4>, I<64>, I<1>, I<8>, I<1>, I<8>, I<8>>,
+    std::tuple<ck::half_t, float, ck::half_t, I<3>, I<2>, I<256>, I<2>, I<128>, I<1>, I<8>, I<1>, I<8>, I<8>>,
+    std::tuple<ck::half_t, float, ck::half_t, I<3>, I<2>, I<256>, I<1>, I<256>, I<1>, I<8>, I<1>, I<8>, I<8>>
+    >;
+// clang-format on
+TYPED_TEST_SUITE(TestSoftmaxFP16, KernelTypes);
+TYPED_TEST(TestSoftmaxFP16, Test_FP16) { this->Run(); }

test/softmax/test_softmax_fp32.cpp

+#include "gtest/gtest.h"
+#include "test_softmax_util.hpp"
+
+template <ck::index_t N>
+using I = ck::Number<N>;
+
+template <typename Tuple>
+class TestSoftmaxFP32 : public ck::TestSoftmax<Tuple>
+{
+};
+
+// clang-format off
+using KernelTypes = ::testing::Types<
+// InDataType, AccDataType, OutDataType, Rank, NumReduceDim, BlockSize, MThreadClusterSize, KThreadClusterSize, MThreadSliceSize, KThreadSliceSize, InSrcVectorDim, InSrcVectorSize, OutDstVectorSize>
+    std::tuple<float, float, float, I<3>, I<1>, I<256>, I<8>, I<32>, I<1>, I<4>, I<1>, I<4>, I<4>>,
+    std::tuple<float, float, float, I<3>, I<1>, I<256>, I<4>, I<64>, I<1>, I<4>, I<1>, I<4>, I<4>>,
+    std::tuple<float, float, float, I<3>, I<1>, I<256>, I<2>, I<128>, I<1>, I<4>, I<1>, I<4>, I<4>>,
+    std::tuple<float, float, float, I<3>, I<1>, I<256>, I<1>, I<256>, I<1>, I<4>, I<1>, I<4>, I<4>>,
+    std::tuple<float, float, float, I<3>, I<2>, I<256>, I<8>, I<32>, I<1>, I<4>, I<1>, I<4>, I<4>>,
+    std::tuple<float, float, float, I<3>, I<2>, I<256>, I<4>, I<64>, I<1>, I<4>, I<1>, I<4>, I<4>>,
+    std::tuple<float, float, float, I<3>, I<2>, I<256>, I<2>, I<128>, I<1>, I<4>, I<1>, I<4>, I<4>>,
+    std::tuple<float, float, float, I<3>, I<2>, I<256>, I<1>, I<256>, I<1>, I<4>, I<1>, I<4>, I<4>>
+    >;
+// clang-format on
+TYPED_TEST_SUITE(TestSoftmaxFP32, KernelTypes);
+TYPED_TEST(TestSoftmaxFP32, Test_FP32) { this->Run(); }

test/softmax/test_softmax_util.hpp

+#include <vector>
+#include <iostream>
+#include "gtest/gtest.h"
+
+#include "config.hpp"
+#include "host_tensor.hpp"
+#include "check_err.hpp"
+#include "number.hpp"
+#include "reference_softmax.hpp"
+#include "device_softmax.hpp"
+
+namespace ck {
+
+template <typename Tuple>
+class TestSoftmax : public ::testing::Test
+{
+    protected:
+    using InDataType                            = std::tuple_element_t<0, Tuple>;
+    using AccDataType                           = std::tuple_element_t<1, Tuple>;
+    using OutDataType                           = std::tuple_element_t<2, Tuple>;
+    static constexpr index_t Rank               = std::tuple_element_t<3, Tuple>{}.value;
+    static constexpr index_t NumReduceDim       = std::tuple_element_t<4, Tuple>{}.value;
+    static constexpr index_t BlockSize          = std::tuple_element_t<5, Tuple>{}.value;
+    static constexpr index_t MThreadClusterSize = std::tuple_element_t<6, Tuple>{}.value;
+    static constexpr index_t KThreadClusterSize = std::tuple_element_t<7, Tuple>{}.value;
+    static constexpr index_t MThreadSliceSize   = std::tuple_element_t<8, Tuple>{}.value;
+    static constexpr index_t KThreadSliceSize   = std::tuple_element_t<9, Tuple>{}.value;
+    static constexpr index_t InSrcVectorDim     = std::tuple_element_t<10, Tuple>{}.value;
+    static constexpr index_t InSrcVectorSize    = std::tuple_element_t<11, Tuple>{}.value;
+    static constexpr index_t OutDstVectorSize   = std::tuple_element_t<12, Tuple>{}.value;
+
+    using ReferenceInstance =
+        tensor_operation::host::ReferenceSoftmax<InDataType, OutDataType, AccDataType>;
+
+    using DeviceInstance = tensor_operation::device::DeviceSoftmax<InDataType,
+                                                                   AccDataType,
+                                                                   OutDataType,
+                                                                   Rank,
+                                                                   NumReduceDim,
+                                                                   BlockSize,
+                                                                   MThreadClusterSize,
+                                                                   KThreadClusterSize,
+                                                                   MThreadSliceSize,
+                                                                   KThreadSliceSize,
+                                                                   InSrcVectorDim,
+                                                                   InSrcVectorSize,
+                                                                   OutDstVectorSize>;
+
+    TestSoftmax() : ref_instance_invoker_(ReferenceInstance{}.MakeInvoker()) {}
+
+    void RunSingle(std::vector<index_t> in_length, AccDataType alpha, AccDataType beta)
+    {
+        std::vector<index_t> reduce_dims(NumReduceDim);
+        std::iota(reduce_dims.begin(), reduce_dims.end(), Rank - NumReduceDim);
+
+        Tensor<InDataType> in(in_length);
+        Tensor<OutDataType> out(in_length);
+
+        in.GenerateTensorValue(GeneratorTensor_2<InDataType>{-5, 5});
+        out.GenerateTensorValue(GeneratorTensor_2<OutDataType>{-5, 5});
+
+        Tensor<OutDataType> out_ref(out);
+
+        DeviceMem in_dev(sizeof(InDataType) * in.mDesc.GetElementSpace());
+        DeviceMem out_dev(sizeof(OutDataType) * out.mDesc.GetElementSpace());
+        in_dev.ToDevice(in.mData.data());
+        out_dev.ToDevice(out.mData.data());
+
+        std::vector<index_t> i_in_lengths(in.mDesc.GetLengths().begin(),
+                                          in.mDesc.GetLengths().end());
+        std::vector<index_t> i_in_strides(in.mDesc.GetStrides().begin(),
+                                          in.mDesc.GetStrides().end());
+
+        auto device_instance = DeviceInstance{};
+        auto argument_ptr    = device_instance.MakeArgumentPointer(i_in_lengths,
+                                                                i_in_strides,
+                                                                reduce_dims,
+                                                                alpha,
+                                                                beta,
+                                                                in_dev.GetDeviceBuffer(),
+                                                                out_dev.GetDeviceBuffer());
+
+        if(!device_instance.IsSupportedArgument(argument_ptr.get()))
+        {
+            FAIL() << "Unsupported argument";
+        }
+
+        auto invoker_ptr = device_instance.MakeInvokerPointer();
+        invoker_ptr->Run(argument_ptr.get());
+
+        ref_instance_invoker_.Run({in, out_ref, alpha, beta, Rank, reduce_dims});
+
+        out_dev.FromDevice(out.mData.data());
+        EXPECT_TRUE(ck::utils::check_err(out.mData, out_ref.mData));
+    }
+
+    void Run()
+    {
+        for(auto in_length : this->in_lengths_)
+        {
+            for(auto scale : this->scales_)
+            {
+                this->RunSingle(in_length, std::get<0>(scale), std::get<1>(scale));
+            }
+        }
+    }
+
+    std::vector<std::vector<index_t>> in_lengths_ = {{1, 8, 128}, {2, 128, 1024}, {3, 9, 1032}};
+    std::vector<std::tuple<AccDataType, AccDataType>> scales_ = {{1, 0}, {2, 2}, {0, 1}};
+
+    typename ReferenceInstance::Invoker ref_instance_invoker_;
+};
+} // namespace ck