Merge branch 'develop' into lwpck-911

profiler/src/profile_gemm_splitk.cpp

@@ -67,7 +67,9 @@ int profile_gemm_splitk(int argc, char* argv[])
 
     using F32 = float;
     using F16 = ck::half_t;
+#if defined CK_ENABLE_FP8
     using F8 = ck::f8_t;
+#endif
 
     using Row = ck::tensor_layout::gemm::RowMajor;
     using Col = ck::tensor_layout::gemm::ColumnMajor;
@@ -146,6 +148,7 @@ int profile_gemm_splitk(int argc, char* argv[])
     {
         return profile(F16{}, F16{}, F32{}, F16{}, Col{}, Col{}, Row{});
     }
+#if defined CK_ENABLE_FP8
     else if(data_type == GemmDataType::F8_F16_F16 && layout == GemmMatrixLayout::MK_KN_MN)
     {
         return profile(F8{}, F16{}, F32{}, F16{}, Row{}, Row{}, Row{});
@@ -178,6 +181,7 @@ int profile_gemm_splitk(int argc, char* argv[])
     {
         return profile(F16{}, F8{}, F32{}, F16{}, Col{}, Col{}, Row{});
     }
+#endif
     else
     {
         std::cout << "this data_type & layout is not implemented" << std::endl;

test/data_type/CMakeLists.txt

@@ -3,5 +3,12 @@ if (USE_BITINT_EXTENSION_INT4)
   target_link_libraries(test_int4 PRIVATE utility)
 endif()
 
-add_gtest_executable(test_fp8 fp8.cpp)
-target_link_libraries(test_fp8 PRIVATE utility)
+if(DTYPES MATCHES "fp8" OR NOT DEFINED DTYPES)
+  add_gtest_executable(test_f8 f8.cpp)
+  target_link_libraries(test_f8 PRIVATE utility)
+endif()
+
+if(DTYPES MATCHES "bf8" OR NOT DEFINED DTYPES)
+  add_gtest_executable(test_bf8 bf8.cpp)
+  target_link_libraries(test_bf8 PRIVATE utility)
+endif()

test/data_type/bf8.cpp

+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved.
+
+#include "gtest/gtest.h"
+#include "ck/utility/data_type.hpp"
+#include "ck/utility/type_convert.hpp"
+
+using ck::bf8_t;
+using ck::f8_convert_sr;
+using ck::half_t;
+using ck::type_convert;
+
+TEST(BF8, NumericLimits)
+{
+    // constants given for negative zero nan mode
+    EXPECT_EQ(ck::NumericLimits<bf8_t>::Min(), type_convert<bf8_t>(0x04));
+    EXPECT_EQ(ck::NumericLimits<bf8_t>::Max(), type_convert<bf8_t>(0x7F));
+    EXPECT_EQ(ck::NumericLimits<bf8_t>::Lowest(), type_convert<bf8_t>(0xFF));
+    EXPECT_EQ(ck::NumericLimits<bf8_t>::QuietNaN(), type_convert<bf8_t>(0x80));
+}
+
+TEST(BF8, ConvertFP32Nearest)
+{
+    // fix the tolerance value
+    float abs_tol = 1e-6;
+    // convert 0 float to bf8 and back, check if holds
+    ASSERT_NEAR(0.0f, type_convert<float>(type_convert<bf8_t>(0.0f)), abs_tol);
+    // convert minimal float to bf8 and back, check if holds
+    ASSERT_NEAR(std::numeric_limits<float>::min(),
+                type_convert<float>(type_convert<bf8_t>(std::numeric_limits<float>::min())),
+                abs_tol);
+    // convert maximal bf8_t to float and check if equal to 57344.0
+    ASSERT_NEAR(57344.0f, type_convert<float>(type_convert<bf8_t>(57344.0f)), abs_tol);
+    // convert maximal float to bf8 and back, check if clipped to 57344.0
+    ASSERT_NEAR(57344.0f,
+                type_convert<float>(type_convert<bf8_t>(std::numeric_limits<float>::max())),
+                abs_tol);
+    // convert inf float to bf8_t and check if it is qNan
+    ASSERT_NEAR(type_convert<bf8_t>(0x80),
+                type_convert<bf8_t>(std::numeric_limits<float>::infinity()),
+                abs_tol);
+    // positive norm float value to bf8 and back, check if holds
+    float pos_float = 0.0000762939f;
+    ASSERT_NEAR(pos_float, type_convert<float>(type_convert<bf8_t>(pos_float)), abs_tol);
+    // negative norm float value to bf8 and back, check if holds
+    float neg_float = -0.0000610351f;
+    ASSERT_NEAR(neg_float, type_convert<float>(type_convert<bf8_t>(neg_float)), abs_tol);
+    // positive subnorm float value to bf8 and back, check if holds
+    pos_float = 0.0000305175f;
+    ASSERT_NEAR(pos_float, type_convert<float>(type_convert<bf8_t>(pos_float)), abs_tol);
+    // negative subnorm float value to bf8 and back, check if holds
+    neg_float = -0.0000152587f;
+    ASSERT_NEAR(neg_float, type_convert<float>(type_convert<bf8_t>(neg_float)), abs_tol);
+}
+
+TEST(BF8, ConvertFP32Stochastic)
+{
+    // fix the tolerance value
+    float abs_tol = 1e-6;
+    // convert 0 float to bf8 and back, check if holds
+    ASSERT_NEAR(0.0f, type_convert<float>(f8_convert_sr<bf8_t>(0.0f)), abs_tol);
+    // convert minimal float to bf8 and back, check if holds
+    ASSERT_NEAR(std::numeric_limits<float>::min(),
+                type_convert<float>(f8_convert_sr<bf8_t>(std::numeric_limits<float>::min())),
+                abs_tol);
+    // convert maximal bf8_t to float and check if equal to 57344.0
+    ASSERT_NEAR(57344.0f, type_convert<float>(f8_convert_sr<bf8_t>(57344.0f)), abs_tol);
+    // convert maximal float to bf8 and back, check if clipped to 57344.0
+    ASSERT_NEAR(57344.0f,
+                type_convert<float>(f8_convert_sr<bf8_t>(std::numeric_limits<float>::max())),
+                abs_tol);
+    // convert inf float to bf8_t and check if it is qNan
+    ASSERT_NEAR(type_convert<bf8_t>(0x80),
+                f8_convert_sr<bf8_t>(std::numeric_limits<float>::infinity()),
+                abs_tol);
+    // positive norm float value to bf8 and back, check if holds
+    float pos_float = 0.0000762939f;
+    ASSERT_NEAR(pos_float, type_convert<float>(f8_convert_sr<bf8_t>(pos_float)), abs_tol);
+    // negative norm float value to bf8 and back, check if holds
+    float neg_float = -0.0000610351f;
+    ASSERT_NEAR(neg_float, type_convert<float>(f8_convert_sr<bf8_t>(neg_float)), abs_tol);
+    // positive subnorm float value to bf8 and back, check if holds
+    pos_float = 0.0000305175f;
+    ASSERT_NEAR(pos_float, type_convert<float>(f8_convert_sr<bf8_t>(pos_float)), abs_tol);
+    // negative subnorm float value to bf8 and back, check if holds
+    neg_float = -0.0000152587f;
+    ASSERT_NEAR(neg_float, type_convert<float>(f8_convert_sr<bf8_t>(neg_float)), abs_tol);
+}
+
+TEST(BF8, ConvertFP16Nearest)
+{
+    // fix the tolerance value
+    float abs_tol = 1e-3;
+    // convert 0 fp16 to bf8 and back, check if holds
+    ASSERT_NEAR(half_t{0.0}, type_convert<half_t>(type_convert<bf8_t>(half_t{0.0})), abs_tol);
+    // convert minimal fp16 to bf8 and back, check if holds
+    ASSERT_NEAR(ck::NumericLimits<half_t>::Min(),
+                type_convert<half_t>(type_convert<bf8_t>(ck::NumericLimits<half_t>::Min())),
+                abs_tol);
+    // convert maximal bf8_t to fp16 and check if equal to 57344.0
+    ASSERT_NEAR(
+        half_t{57344.0}, type_convert<half_t>(type_convert<bf8_t>(half_t{57344.0})), abs_tol);
+    // convert maximal fp16 to bf8 and back, check if clipped to 57344.0
+    ASSERT_NEAR(half_t{57344.0},
+                type_convert<half_t>(type_convert<bf8_t>(ck::NumericLimits<half_t>::Max())),
+                abs_tol);
+    // convert QuietNaN fp16 to bf8_t and check if it is QuietNaN
+    ASSERT_NEAR(type_convert<bf8_t>(0x80),
+                type_convert<bf8_t>(ck::NumericLimits<half_t>::QuietNaN()),
+                abs_tol);
+    // positive norm fp16 value to bf8 and back, check if holds
+    half_t pos_half = half_t{0.0000762939};
+    ASSERT_NEAR(pos_half, type_convert<half_t>(type_convert<bf8_t>(pos_half)), abs_tol);
+    // negative norm fp16 value to bf8 and back, check if holds
+    half_t neg_half = half_t{-0.0000610351};
+    ASSERT_NEAR(neg_half, type_convert<half_t>(type_convert<bf8_t>(neg_half)), abs_tol);
+    // positive subnorm fp16 value to bf8 and back, check if holds
+    pos_half = half_t{0.0000305175};
+    ASSERT_NEAR(pos_half, type_convert<half_t>(type_convert<bf8_t>(pos_half)), abs_tol);
+    // negative subnorm fp16 value to bf8 and back, check if holds
+    neg_half = half_t{-0.0000152587};
+    ASSERT_NEAR(neg_half, type_convert<half_t>(type_convert<bf8_t>(neg_half)), abs_tol);
+}
+
+TEST(BF8, ConvertFP16Stochastic)
+{
+    // fix the tolerance value
+    float abs_tol = 1e-3;
+    // convert 0 fp16 to bf8 and back, check if holds
+    ASSERT_NEAR(half_t{0.0}, type_convert<half_t>(f8_convert_sr<bf8_t>(half_t{0.0})), abs_tol);
+    // convert minimal fp16 to bf8 and back, check if holds
+    ASSERT_NEAR(ck::NumericLimits<half_t>::Min(),
+                type_convert<half_t>(f8_convert_sr<bf8_t>(ck::NumericLimits<half_t>::Min())),
+                abs_tol);
+    // convert maximal bf8_t to fp16 and check if equal to 57344.0
+    ASSERT_NEAR(
+        half_t{57344.0}, type_convert<half_t>(f8_convert_sr<bf8_t>(half_t{57344.0})), abs_tol);
+    // convert maximal fp16 to bf8 and back, check if clipped to 57344.0
+    ASSERT_NEAR(half_t{57344.0},
+                type_convert<half_t>(f8_convert_sr<bf8_t>(ck::NumericLimits<half_t>::Max())),
+                abs_tol);
+    // convert QuietNaN fp16 to bf8_t and check if it is QuietNaN
+    ASSERT_NEAR(type_convert<bf8_t>(0x80),
+                f8_convert_sr<bf8_t>(ck::NumericLimits<half_t>::QuietNaN()),
+                abs_tol);
+    // positive norm fp16 value to bf8 and back, check if holds
+    half_t pos_half = half_t{0.0000762939};
+    ASSERT_NEAR(pos_half, type_convert<half_t>(f8_convert_sr<bf8_t>(pos_half)), abs_tol);
+    // negative norm fp16 value to bf8 and back, check if holds
+    half_t neg_half = half_t{-0.0000610351};
+    ASSERT_NEAR(neg_half, type_convert<half_t>(f8_convert_sr<bf8_t>(neg_half)), abs_tol);
+    // positive subnorm fp16 value to bf8 and back, check if holds
+    pos_half = half_t{0.0000305175};
+    ASSERT_NEAR(pos_half, type_convert<half_t>(f8_convert_sr<bf8_t>(pos_half)), abs_tol);
+    // negative subnorm fp16 value to bf8 and back, check if holds
+    neg_half = half_t{-0.0000152587};
+    ASSERT_NEAR(neg_half, type_convert<half_t>(f8_convert_sr<bf8_t>(neg_half)), abs_tol);
+}

test/grouped_convnd_bwd_weight/test_grouped_convnd_bwd_weight.cpp

@@ -14,6 +14,8 @@
 
 #include "profiler/profile_grouped_conv_bwd_weight_impl.hpp"
 
+using namespace ck::tensor_layout::convolution;
+
 template <typename Tuple>
 class TestGroupedConvndBwdWeight : public ::testing::Test
 {
@@ -27,14 +29,43 @@ class TestGroupedConvndBwdWeight : public ::testing::Test
     using NDimSpatial = std::tuple_element_t<6, Tuple>;
 
     std::vector<ck::utils::conv::ConvParam> conv_params;
-    ck::index_t split_k{2};
+    std::vector<ck::index_t> split_ks{1, 2};
+
+    bool skip_case(const ck::utils::conv::ConvParam& params, const ck::index_t split_k)
+    {
+        // K or C are odd is supported only by DL kernel (only applies to fp16)
+        // DL kernel is only supported for split_k=1
+        if constexpr(std::is_same_v<InDataType, ck::half_t>)
+        {
+            if(split_k != 1 && (params.K_ % 2 != 0 || params.C_ % 2 != 0))
+            {
+                return true;
+            }
+        }
+
+        // 1d NWGC is only supported by DL kernel
+        // DL kernel is only supported for split_k=1
+        if constexpr(std::is_same_v<InLayout, NWGC> && std::is_same_v<OutLayout, NWGK>)
+        {
+            if(split_k != 1)
+            {
+                return true;
+            }
+        }
+
+        return false;
+    }
 
     void Run()
     {
         EXPECT_FALSE(conv_params.empty());
         bool pass = true;
 
+        for(auto split_k : split_ks)
+        {
             for(auto& param : conv_params)
+            {
+                if(!skip_case(param, split_k))
                 {
                     pass = pass && ck::profiler::profile_grouped_conv_bwd_weight_impl<NDimSpatial{},
                                                                                       InLayout,
@@ -50,6 +81,8 @@ class TestGroupedConvndBwdWeight : public ::testing::Test
                                        param,
                                        split_k);
                 }
+            }
+        }
         EXPECT_TRUE(pass);
     }
 };
@@ -69,12 +102,13 @@ class TestGroupedConvndBwdWeight3d : public TestGroupedConvndBwdWeight<Tuple>
 {
 };
 
-using namespace ck::tensor_layout::convolution;
-
 using KernelTypes1d = ::testing::Types<
     std::tuple<float, float, float, GNWC, GKXC, GNWK, ck::Number<1>>,
     std::tuple<ck::half_t, ck::half_t, ck::half_t, GNWC, GKXC, GNWK, ck::Number<1>>,
-    std::tuple<ck::bhalf_t, float, ck::bhalf_t, GNWC, GKXC, GNWK, ck::Number<1>>>;
+    std::tuple<ck::bhalf_t, float, ck::bhalf_t, GNWC, GKXC, GNWK, ck::Number<1>>,
+    std::tuple<float, float, float, NWGC, GKXC, NWGK, ck::Number<1>>,
+    std::tuple<ck::half_t, ck::half_t, ck::half_t, NWGC, GKXC, NWGK, ck::Number<1>>,
+    std::tuple<ck::bhalf_t, float, ck::bhalf_t, NWGC, GKXC, NWGK, ck::Number<1>>>;
 using KernelTypes2d = ::testing::Types<
     std::tuple<float, float, float, GNHWC, GKYXC, GNHWK, ck::Number<2>>,
     std::tuple<ck::half_t, ck::half_t, ck::half_t, GNHWC, GKYXC, GNHWK, ck::Number<2>>,