added test for batched_gemm

include/ck/tensor_operation/gpu/device/device_batched_gemm_xdl.hpp

@@ -249,7 +249,9 @@ struct DeviceBatchedGemmXdl
 
     struct ComputeBasePtrOfStridedBatch
     {
-        ComputeBasePtrOfStridedBatch(index_t BatchStrideA, index_t BatchStrideB, index_t BatchStrideC)
+        ComputeBasePtrOfStridedBatch(index_t BatchStrideA,
+                                     index_t BatchStrideB,
+                                     index_t BatchStrideC)
             : BatchStrideA_(BatchStrideA), BatchStrideB_(BatchStrideB), BatchStrideC_(BatchStrideC)
         {
         }

library/include/ck/library/host_tensor/host_tensor.hpp

@@ -317,7 +317,7 @@ float bf16_to_f32_(ck::bhalf_t src_val);
 void bf16_to_f32_(const Tensor<ck::bhalf_t>& src, Tensor<float>& dst);
 
 template <typename T>
-void check_error(const Tensor<T>& ref, const Tensor<T>& result)
+float check_error(const Tensor<T>& ref, const Tensor<T>& result)
 {
     float error     = 0;
     float max_diff  = -1;
@@ -354,6 +354,7 @@ void check_error(const Tensor<T>& ref, const Tensor<T>& result)
 
     std::cout << "error: " << error << std::endl;
     std::cout << "max_diff: " << max_diff << ", " << ref_value << ", " << result_value << std::endl;
+    return max_diff;
 }
 
 template <typename T>

library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f16_f16_f16_gmk_gkn_gmn_instance.cpp

@@ -21,9 +21,8 @@ using S = ck::Sequence<Is...>;
 using PassThrough = ck::tensor_operation::element_wise::PassThrough;
 
 // Compilation parameters for a[m, k] * b[k, n] = c[m, n]
-using device_batched_gemm_xdl_f16_f16_f16_gmk_gkn_gmn_instances =
-    std::tuple<
-        // clang-format off
+using device_batched_gemm_xdl_f16_f16_f16_gmk_gkn_gmn_instances = std::tuple<
+    // clang-format off
         //#################| AData| BData| CData| AccData| ALayout| BLayout| CLayout|           A|           B|           C| Block|  MPer|  NPer| K0Per| K1| MPer| NPer| MXdl| NXdl|  ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds|  BBlockTransfer| BBlockTransfer| BBlockTransfer| BlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CThreadTransfer| CThreadTransfer|
         //#################|  Type|  Type|  Type|    Type|        |        |        | Elementwise| Elementwise| Elementwise|  Size| Block| Block| Block|   |  XDL|  XDL|  Per|  Per|   ThreadCluster|  ThreadCluster| SrcAccessOrder|   SrcVectorDim|      SrcScalar|      DstScalar| AddExtraM|   ThreadCluster|  ThreadCluster| SrcAccessOrder|  SrcVectorDim|      SrcScalar|      DstScalar| AddExtraN| SrcDstVectorDim|       DstScalar|
         //#################|      |      |      |        |        |        |        |   Operation|   Operation|   Operation|      |      |      |      |   |     |     | Wave| Wave| Lengths_K0_M_K1|   ArrangeOrder|               |               |      PerVector|   PerVector_K1|          | Lengths_K0_N_K1|   ArrangeOrder|               |              |      PerVector|   PerVector_K1|          |                |       PerVector|
@@ -45,8 +44,8 @@ using device_batched_gemm_xdl_f16_f16_f16_gmk_gkn_gmn_instances =
         DeviceBatchedGemmXdl<  F16,   F16,   F16,     F32,     Row,      Row,    Row, PassThrough, PassThrough, PassThrough,   128,    16,    64,     4,  8,   16,   16,    1,    2,     S<4, 16, 1>,     S<1, 0, 2>,     S<1, 0, 2>,              2,              8,              8,      true,     S<4, 32, 1>,     S<0, 2, 1>,     S<0, 2, 1>,             1,              2,              8,      true,               7,               1>,
         DeviceBatchedGemmXdl<  F16,   F16,   F16,     F32,     Row,      Row,    Row, PassThrough, PassThrough, PassThrough,   128,    16,    32,     4,  8,   16,   16,    1,    1,     S<4, 16, 1>,     S<1, 0, 2>,     S<1, 0, 2>,              2,              8,              8,      true,     S<4, 32, 1>,     S<0, 2, 1>,     S<0, 2, 1>,             1,              1,              8,      true,               7,               1>,
         DeviceBatchedGemmXdl<  F16,   F16,   F16,     F32,     Row,      Row,    Row, PassThrough, PassThrough, PassThrough,    64,    16,    16,     4,  8,   16,   16,    1,    1,     S<4, 16, 1>,     S<1, 0, 2>,     S<1, 0, 2>,              2,              8,              8,      true,     S<4, 16, 1>,     S<0, 2, 1>,     S<0, 2, 1>,             1,              1,              8,      true,               7,               1>
-        // clang-format on
-        >;
+    // clang-format on
+    >;
 
 void add_device_batched_gemm_xdl_f16_f16_f16_gmk_gkn_gmn_instances(
     std::vector<DeviceGemmPtr<PassThrough, PassThrough, PassThrough>>& instances)

test/CMakeLists.txt

@@ -39,3 +39,4 @@ add_subdirectory(gemm_split_k)
 add_subdirectory(conv2d_fwd)
 add_subdirectory(convnd_fwd)
 add_subdirectory(conv2d_bwd_data)
+add_subdirectory(batched_gemm)

test/batched_gemm/CMakeLists.txt

+add_test_executable(test_batched_gemm_fp16 batched_gemm_fp16.cpp)
+target_link_libraries(test_batched_gemm_fp16 PRIVATE host_tensor)
+target_link_libraries(test_batched_gemm_fp16 PRIVATE device_batched_gemm_instance)
+

test/batched_gemm/batched_gemm_fp16.cpp

+#include <half.hpp>
+#include <tuple>
+#include <vector>
+
+#include "batched_gemm_util.hpp"
+#include "reference_batched_gemm.hpp"
+#include "config.hpp"
+#include "device.hpp"
+#include "host_tensor.hpp"
+#include "host_tensor_generator.hpp"
+#include "device_tensor.hpp"
+#include "device_batched_gemm_xdl.hpp"
+#include "element_wise_operation.hpp"
+#include "test_util.hpp"
+
+using PassThrough = ck::tensor_operation::element_wise::PassThrough;
+
+using DeviceBatchedGemmPtr =
+    ck::tensor_operation::device::DeviceGemmPtr<ck::tensor_operation::element_wise::PassThrough,
+                                                ck::tensor_operation::element_wise::PassThrough,
+                                                ck::tensor_operation::element_wise::PassThrough>;
+
+namespace ck {
+namespace tensor_operation {
+namespace device {
+namespace device_batched_gemm_instance {
+void add_device_batched_gemm_xdl_f16_f16_f16_gmk_gnk_gmn_instances(
+    std::vector<DeviceBatchedGemmPtr>& instances);
+}
+} // namespace device
+} // namespace tensor_operation
+} // namespace ck
+
+namespace {
+using ADataType   = ck::half_t;
+using BDataType   = ck::half_t;
+using CDataType   = ck::half_t;
+using AccDataType = float;
+
+using ALayout = ck::tensor_layout::gemm::RowMajor;
+using BLayout = ck::tensor_layout::gemm::ColumnMajor;
+using CLayout = ck::tensor_layout::gemm::RowMajor;
+
+auto PrepareGemmTensor(const std::size_t batch_count,
+                       const ck::batched_gemm_util::GemmParams& params)
+{
+    auto f_host_tensor_descriptor =
+        [batch_count](std::size_t row, std::size_t col, std::size_t stride, auto layout) {
+            if(std::is_same<decltype(layout), ck::tensor_layout::gemm::RowMajor>::value)
+            {
+                return HostTensorDescriptor(std::vector<std::size_t>({batch_count, row, col}),
+                                            std::vector<std::size_t>({row * stride, stride, 1}));
+            }
+            else
+            {
+                return HostTensorDescriptor(std::vector<std::size_t>({batch_count, row, col}),
+                                            std::vector<std::size_t>({col * stride, 1, stride}));
+            }
+        };
+
+    Tensor<ADataType> a_g_m_k(
+        f_host_tensor_descriptor(params.M, params.K, params.StrideA, ALayout{}));
+    Tensor<BDataType> b_g_k_n(
+        f_host_tensor_descriptor(params.K, params.N, params.StrideB, BLayout{}));
+    Tensor<CDataType> c_g_m_n_host_result(
+        f_host_tensor_descriptor(params.M, params.N, params.StrideC, CLayout{}));
+    Tensor<CDataType> c_g_m_n_device_result(
+        f_host_tensor_descriptor(params.M, params.N, params.StrideC, CLayout{}));
+
+    a_g_m_k.GenerateTensorValue(GeneratorTensor_3<ADataType>{-0.5, 0.5});
+    b_g_k_n.GenerateTensorValue(GeneratorTensor_3<BDataType>{-0.5, 0.5});
+
+    return std::make_tuple(a_g_m_k, b_g_k_n, c_g_m_n_host_result, c_g_m_n_device_result);
+}
+
+bool TestBatchedGemm(const std::size_t batch_count, DeviceBatchedGemmPtr& gemmPtr)
+{
+    // Arrange
+    ck::batched_gemm_util::GemmParams params;
+    params.M       = 1024;
+    params.N       = 1024;
+    params.K       = 1024;
+    params.StrideA = 1024;
+    params.StrideB = 1024;
+    params.StrideC = 1024;
+
+    auto host_tensors           = PrepareGemmTensor(batch_count, params);
+    const Tensor<ADataType>& a  = std::get<0>(host_tensors);
+    const Tensor<BDataType>& b  = std::get<1>(host_tensors);
+    Tensor<CDataType>& c_host   = std::get<2>(host_tensors);
+    Tensor<CDataType>& c_device = std::get<3>(host_tensors);
+
+    auto a_element_op = PassThrough{};
+    auto b_element_op = PassThrough{};
+    auto c_element_op = PassThrough{};
+
+    using ReferenceBatchedGemmInstance =
+        ck::tensor_operation::host::ReferenceBatchedGemm<ADataType,
+                                                         BDataType,
+                                                         CDataType,
+                                                         PassThrough,
+                                                         PassThrough,
+                                                         PassThrough>;
+    ck::batched_gemm_util::RunHostBatchedGemm<ReferenceBatchedGemmInstance>(
+        a, b, c_host, a_element_op, b_element_op, c_element_op);
+
+    // Act
+    ck::batched_gemm_util::RunDeviceBatchedGemm(
+        gemmPtr, params, a, b, c_device, a_element_op, b_element_op, c_element_op);
+
+    // Assert
+    // bool res = test_util::check_err(
+    // c_device.mData, c_host.mData, "Error: incorrect results!", 1e-5f, 1e-4f);
+    bool res = check_error(c_device, c_host) < 0.007815f;
+
+    std::cout << (res ? "SUCCESS" : "FAILURE") << std::endl;
+
+    return res;
+}
+} // namespace
+
+int main()
+{
+    std::vector<DeviceBatchedGemmPtr> batched_gemm_ptrs;
+    ck::tensor_operation::device::device_batched_gemm_instance::
+        add_device_batched_gemm_xdl_f16_f16_f16_gmk_gnk_gmn_instances(batched_gemm_ptrs);
+
+    bool res = true;
+
+    const std::size_t batch_count = 4;
+    for(auto& gemmPtr : batched_gemm_ptrs)
+    {
+        res &= TestBatchedGemm(batch_count, gemmPtr);
+    }
+
+    std::cout << "TestGemm ..... " << (res ? "SUCCESS" : "FAILURE") << std::endl;
+}

test/batched_gemm/batched_gemm_util.hpp

+#ifndef BATCHED_GEMM_UTILS_HPP
+#define BATCHED_GEMM_UTILS_HPP
+
+#include "config.hpp"
+#include "device.hpp"
+#include "host_tensor.hpp"
+
+namespace ck {
+namespace batched_gemm_util {
+
+struct GemmParams
+{
+    GemmParams()
+        : M(1024), N(1024), K(1024), StrideA(1024), StrideB(1024), StrideC(1024), alpha(1), beta(0)
+    {
+    }
+
+    ck::index_t M;
+    ck::index_t N;
+    ck::index_t K;
+
+    ck::index_t StrideA;
+    ck::index_t StrideB;
+    ck::index_t StrideC;
+
+    float alpha;
+    float beta;
+};
+
+template <typename BatchedGemmInstance,
+          typename ADataType,
+          typename BDataType,
+          typename CDataType,
+          typename AElementwiseOperation,
+          typename BElementwiseOperation,
+          typename CElementwiseOperation>
+void RunHostBatchedGemm(const Tensor<ADataType>& A,
+                        const Tensor<BDataType>& B,
+                        Tensor<CDataType>& C,
+                        AElementwiseOperation a_element_op,
+                        BElementwiseOperation b_element_op,
+                        CElementwiseOperation c_element_op)
+{
+    auto ref_batched_gemm = BatchedGemmInstance{};
+    auto ref_invoker      = ref_batched_gemm.MakeInvoker();
+
+    auto ref_argument =
+        ref_batched_gemm.MakeArgument(A, B, C, a_element_op, b_element_op, c_element_op);
+
+    ref_invoker.Run(ref_argument);
+}
+
+template <typename DeviceGemmPtr,
+          typename ADataType,
+          typename BDataType,
+          typename CDataType,
+          typename AElementwiseOperation,
+          typename BElementwiseOperation,
+          typename CElementwiseOperation>
+void RunDeviceBatchedGemm(DeviceGemmPtr& batched_gemm_ptr,
+                          const ck::batched_gemm_util::GemmParams& params,
+                          const Tensor<ADataType>& A,
+                          const Tensor<BDataType>& B,
+                          Tensor<CDataType>& C,
+                          AElementwiseOperation a_element_op,
+                          BElementwiseOperation b_element_op,
+                          CElementwiseOperation c_element_op)
+{
+    DeviceMem a_g_m_k_device_buf(sizeof(ADataType) * A.mDesc.GetElementSpace());
+    DeviceMem b_g_k_n_device_buf(sizeof(BDataType) * B.mDesc.GetElementSpace());
+    DeviceMem c_g_m_n_device_buf(sizeof(CDataType) * C.mDesc.GetElementSpace());
+
+    a_g_m_k_device_buf.ToDevice(A.mData.data());
+    b_g_k_n_device_buf.ToDevice(B.mData.data());
+
+    const auto batch_count = A.mDesc.GetLengths()[0];
+    auto invoker_ptr       = batched_gemm_ptr->MakeInvokerPointer();
+    auto argument_ptr      = batched_gemm_ptr->MakeArgumentPointer(
+        static_cast<ADataType*>(a_g_m_k_device_buf.GetDeviceBuffer()),
+        static_cast<BDataType*>(b_g_k_n_device_buf.GetDeviceBuffer()),
+        static_cast<CDataType*>(c_g_m_n_device_buf.GetDeviceBuffer()),
+        params.M,
+        params.N,
+        params.K,
+        params.StrideA,
+        params.StrideB,
+        params.StrideC,
+        a_element_op,
+        b_element_op,
+        c_element_op,
+        batch_count);
+
+    if(!batched_gemm_ptr->IsSupportedArgument(argument_ptr.get()))
+    {
+        throw std::runtime_error(
+            "wrong! device_gemm with the specified compilation parameters does "
+            "not support this GEMM problem");
+    }
+
+    invoker_ptr->Run(argument_ptr.get());
+    c_g_m_n_device_buf.FromDevice(C.mData.data());
+}
+
+} // namespace batched_gemm_util
+} // namespace ck
+#endif