Merge remote-tracking branch 'origin/develop' into fp16_fp8_gemm_scale_bias

library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_i8_i8_i8_mk_nk_mn_irregular_instance.cpp

@@ -8,7 +8,7 @@
 #include "ck/tensor_operation/gpu/device/gemm_specialization.hpp"
 #include "ck/tensor_operation/gpu/device/impl/device_gemm_dl.hpp"
 #include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp"
-#ifdef __int8__
+#ifdef CK_ENABLE_INT8
 namespace ck {
 namespace tensor_operation {
 namespace device {

library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_i8_i8_i8_km_kn_mn_instance.cpp

@@ -8,7 +8,7 @@
 #include "ck/tensor_operation/gpu/device/gemm_specialization.hpp"
 #include "ck/tensor_operation/gpu/device/impl/device_gemm_xdl_cshuffle.hpp"
 #include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp"
-#ifdef __int8__
+#ifdef CK_ENABLE_INT8
 namespace ck {
 namespace tensor_operation {
 namespace device {

library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_i8_i8_i8_km_nk_mn_instance.cpp

@@ -8,7 +8,7 @@
 #include "ck/tensor_operation/gpu/device/gemm_specialization.hpp"
 #include "ck/tensor_operation/gpu/device/impl/device_gemm_xdl_cshuffle.hpp"
 #include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp"
-#ifdef __int8__
+#ifdef CK_ENABLE_INT8
 namespace ck {
 namespace tensor_operation {
 namespace device {

library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_i8_i8_i8_mk_kn_mn_instance.cpp

@@ -8,7 +8,7 @@
 #include "ck/tensor_operation/gpu/device/gemm_specialization.hpp"
 #include "ck/tensor_operation/gpu/device/impl/device_gemm_xdl_cshuffle.hpp"
 #include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp"
-#ifdef __int8__
+#ifdef CK_ENABLE_INT8
 namespace ck {
 namespace tensor_operation {
 namespace device {

library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_i8_i8_i8_mk_nk_mn_instance.cpp

@@ -8,7 +8,7 @@
 #include "ck/tensor_operation/gpu/device/gemm_specialization.hpp"
 #include "ck/tensor_operation/gpu/device/impl/device_gemm_xdl_cshuffle.hpp"
 #include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp"
-#ifdef __int8__
+#ifdef CK_ENABLE_INT8
 namespace ck {
 namespace tensor_operation {
 namespace device {

profiler/src/profile_batched_gemm_multi_d.cpp

@@ -71,7 +71,7 @@ int profile_batched_gemm_multi_d(int argc, char* argv[])
     const int BatchCount = std::stoi(argv[17]);
 
     using F16 = ck::half_t;
-#ifdef __int8__
+#ifdef CK_ENABLE_INT8
     using INT8 = int8_t;
 #endif
 
@@ -165,7 +165,7 @@ int profile_batched_gemm_multi_d(int argc, char* argv[])
     {
         return profile(F16{}, F16{}, F16{}, Col{}, Col{}, Row{});
     }
-#ifdef __int8__
+#ifdef CK_ENABLE_INT8
     else if(data_type == GemmDataType::INT8_INT8_INT8 && layout == GemmMatrixLayout::MK_KN_MN)
     {
         return profile(INT8{}, INT8{}, INT8{}, Row{}, Row{}, Row{});

profiler/src/profile_conv_bwd_data.cpp

@@ -77,7 +77,7 @@ int profile_conv_bwd_data(int argc, char* argv[])
     using F32  = float;
     using F16  = ck::half_t;
     using BF16 = ck::bhalf_t;
-#ifdef __int8__
+#ifdef CK_ENABLE_INT8
     using INT8 = int8_t;
 #endif
 
@@ -140,7 +140,7 @@ int profile_conv_bwd_data(int argc, char* argv[])
         {
             return profile(I1, NWC{}, KXC{}, NWK{}, BF16{}, BF16{}, BF16{});
         }
-#ifdef __int8__
+#ifdef CK_ENABLE_INT8
         else if(data_type == ConvDataType::INT8_INT8_INT8)
         {
             return profile(I1, NWC{}, KXC{}, NWK{}, INT8{}, INT8{}, INT8{});
@@ -161,7 +161,7 @@ int profile_conv_bwd_data(int argc, char* argv[])
         {
             return profile(I2, NHWC{}, KYXC{}, NHWK{}, BF16{}, BF16{}, BF16{});
         }
-#ifdef __int8__
+#ifdef CK_ENABLE_INT8
         else if(data_type == ConvDataType::INT8_INT8_INT8)
         {
             return profile(I2, NHWC{}, KYXC{}, NHWK{}, INT8{}, INT8{}, INT8{});
@@ -182,7 +182,7 @@ int profile_conv_bwd_data(int argc, char* argv[])
         {
             return profile(I3, NDHWC{}, KZYXC{}, NDHWK{}, BF16{}, BF16{}, BF16{});
         }
-#ifdef __int8__
+#ifdef CK_ENABLE_INT8
         else if(data_type == ConvDataType::INT8_INT8_INT8)
         {
             return profile(I3, NDHWC{}, KZYXC{}, NDHWK{}, INT8{}, INT8{}, INT8{});

profiler/src/profile_gemm.cpp

@@ -69,10 +69,10 @@ int profile_gemm(int argc, char* argv[])
 
     using F32 = float;
     using F16 = ck::half_t;
-#ifdef __bf16__
+#ifdef CK_ENABLE_BF16
     using BF16 = ck::bhalf_t;
 #endif
-#ifdef __int8__
+#ifdef CK_ENABLE_INT8
     using INT8  = int8_t;
     using INT32 = int32_t;
 #endif
@@ -123,7 +123,7 @@ int profile_gemm(int argc, char* argv[])
 
     if(false)
         ;
-#ifdef __fp32__
+#ifdef CK_ENABLE_FP32
     else if(data_type == GemmDataType::F32_F32_F32 && layout == GemmMatrixLayout::MK_KN_MN)
     {
         return profile(Row{}, Row{}, Row{}, F32{}, F32{}, F32{}, F32{});
@@ -141,7 +141,7 @@ int profile_gemm(int argc, char* argv[])
         return profile(Col{}, Col{}, Row{}, F32{}, F32{}, F32{}, F32{});
     }
 #endif
-#ifdef __fp16__
+#ifdef CK_ENABLE_FP16
     else if(data_type == GemmDataType::F16_F16_F16 && layout == GemmMatrixLayout::MK_KN_MN)
     {
         return profile(Row{}, Row{}, Row{}, F16{}, F16{}, F32{}, F16{});
@@ -159,7 +159,7 @@ int profile_gemm(int argc, char* argv[])
         return profile(Col{}, Col{}, Row{}, F16{}, F16{}, F32{}, F16{});
     }
 #endif
-#ifdef __bf16__
+#ifdef CK_ENABLE_BF16
     else if(data_type == GemmDataType::BF16_BF16_BF16 && layout == GemmMatrixLayout::MK_KN_MN)
     {
         return profile(Row{}, Row{}, Row{}, BF16{}, BF16{}, F32{}, BF16{});
@@ -177,7 +177,7 @@ int profile_gemm(int argc, char* argv[])
         return profile(Col{}, Col{}, Row{}, BF16{}, BF16{}, F32{}, BF16{});
     }
 #endif
-#ifdef __int8__
+#ifdef CK_ENABLE_INT8
     else if(data_type == GemmDataType::INT8_INT8_INT8 && layout == GemmMatrixLayout::MK_KN_MN)
     {
         return profile(Row{}, Row{}, Row{}, INT8{}, INT8{}, INT32{}, INT8{});

profiler/src/profile_grouped_gemm.cpp

@@ -88,7 +88,7 @@ int profile_grouped_gemm(int argc, char* argv[])
     const auto StrideBs = argToIntArray(argv[12]);
     const auto StrideCs = argToIntArray(argv[13]);
     const int kbatch    = argc == 15 ? std::stoi(argv[14]) : 1;
-#ifdef __fp16__
+#ifdef CK_ENABLE_FP16
     if(data_type == GemmDataType::F16_F16_F16 && layout == GemmMatrixLayout::MK_KN_MN)
     {
         ck::profiler::profile_grouped_gemm_impl<ck::half_t,

script/clang-format-overwrite.sh

-#find . -name deps -prune -o -name build -prune -o -iname '*.h' -o -iname '*.hpp' -o -iname '*.cpp' -o -iname '*.h.in' -o -iname '*.hpp.in' -o -iname '*.cpp.in' -o -iname '*.cl' -o -iname '*.cuh' -o -iname '*.cu' -o -iname '*.inc' | xargs -n 1 -P 16 -I{} -t sh -c 'clang-format-10 -i -style=file {}'
-git status --porcelain | awk '$1 != "D" && (match($2, "\\.cpp|hpp|inc")) {print $2}' | xargs -n 1 -P 16 -I{} -t sh -c 'clang-format-10 -i -style=file {}'
+#find . -name deps -prune -o -name build -prune -o -iname '*.h' -o -iname '*.hpp' -o -iname '*.cpp' -o -iname '*.h.in' -o -iname '*.hpp.in' -o -iname '*.cpp.in' -o -iname '*.cl' -o -iname '*.cuh' -o -iname '*.cu' -o -iname '*.inc' | xargs -n 1 -P 16 -I{} -t sh -c 'clang-format-12 -i -style=file {}'
+git status --porcelain | awk '$1 != "D" && (match($2, "\\.cpp|hpp|inc")) {print $2}' | xargs -n 1 -P 16 -I{} -t sh -c 'clang-format-12 -i -style=file {}'

script/install_precommit.sh

@@ -11,7 +11,7 @@ run_and_check() {
 }
 
 echo "I: Installing tools required for pre-commit checks..."
-run_and_check apt install clang-format-10
+run_and_check apt install clang-format-12
 
 echo "I: Installing pre-commit itself..."
 run_and_check pip3 install pre-commit

test/batched_gemm_multi_d/test_batched_gemm_multi_d.cpp

@@ -71,6 +71,6 @@ TYPED_TEST_SUITE(TestBatchedGemmMultiD, KernelTypes);
 #ifdef __fp16
 TYPED_TEST(TestBatchedGemmMultiD, f16) { this->template Run<F16>(); }
 #endif
-#ifdef __int8__
+#ifdef CK_ENABLE_INT8
 TYPED_TEST(TestBatchedGemmMultiD, int8) { this->template Run<int8_t>(); }
 #endif

test/pool_fwd/test_avg_pool3d_fwd.cpp

@@ -43,7 +43,7 @@ class TestAvgPool3dFwd : public ::testing::Test
         }
     }
 };
-#ifdef __fp16__
+#ifdef CK_ENABLE_FP16
 using KernelTypes =
     ::testing::Types<std::tuple<F16, F16, F32, I32>, std::tuple<F32, F32, F32, I32>>;
 #else

test/pool_fwd/test_max_pool3d_fwd.cpp

@@ -66,7 +66,7 @@ class TestMaxPool3dFwd : public ::testing::Test
     }
 };
 
-#ifdef __fp16__
+#ifdef CK_ENABLE_FP16
 using KernelTypes =
     ::testing::Types<std::tuple<F16, F16, F32, I32>, std::tuple<F32, F32, F32, I32>>;
 #else

test/softmax/test_softmax_rank3.cpp

@@ -10,7 +10,7 @@
 
 template <ck::index_t N>
 using I = ck::Number<N>;
-#ifdef __fp16__
+#ifdef CK_ENABLE_FP16
 using F16 = ck::half_t;
 #endif
 using F32 = float;
@@ -23,7 +23,7 @@ class TestSoftmax : public ck::TestSoftmax<Tuple>
 // clang-format off
 using KernelTypes = ::testing::Types<
     //         InDataType, AccDataType, OutDataType, Rank
-#ifdef __fp16__
+#ifdef CK_ENABLE_FP16
     std::tuple<       F16,         F32,         F16,    I<3>>,
 #endif
     std::tuple<       F32,         F32,         F32,    I<3>>

test/softmax/test_softmax_rank4.cpp

@@ -10,7 +10,7 @@
 
 template <ck::index_t N>
 using I = ck::Number<N>;
-#ifdef __fp16__
+#ifdef CK_ENABLE_FP16
 using F16 = ck::half_t;
 #endif
 using F32 = float;
@@ -23,7 +23,7 @@ class TestSoftmax : public ck::TestSoftmax<Tuple>
 // clang-format off
 using KernelTypes = ::testing::Types<
     //         InDataType, AccDataType, OutDataType, Rank
-#ifdef __fp16__
+#ifdef CK_ENABLE_FP16
     std::tuple<       F16,         F32,         F16,    I<4>>,
 #endif
     std::tuple<       F32,         F32,         F32,    I<4>>