bf16A_Int8B with fastgelu/bias (#1264)

* changed the copy function to v7r2

* adding multi_abd

* in-progress

* add post-load oob check

* debugging

* adjust instances

* add run_lds

* add elemntwise_op

* replace multi_abd_device with v3

* clean up

* clean

* clean

* Added LDSType

* profiling

* adjust oobcheck

* add missing file

* refactor

* clean

* add examples

include/ck/utility/type.hpp

@@ -40,23 +40,10 @@ inline constexpr bool is_pointer_v = std::is_pointer<T>::value;
 template <typename Y, typename X, typename enable_if<sizeof(X) == sizeof(Y), bool>::type = false>
 __host__ __device__ constexpr Y bit_cast(const X& x)
 {
-#if CK_EXPERIMENTAL_USE_MEMCPY_FOR_BIT_CAST
-    Y y;
-
-    // auto t = reinterpret_cast<const Y*>(&x);
-    // y      = *t;
-    __builtin_memcpy(&y, &x, sizeof(X));
-
-    return y;
-#else
-    union AsType
-    {
-        X x;
-        Y y;
-    };
-
-    return AsType{x}.y;
-#endif
+    static_assert(__has_builtin(__builtin_bit_cast), "");
+    static_assert(sizeof(X) == sizeof(Y), "Do not support cast between different size of type");
+
+    return __builtin_bit_cast(Y, x);
 }
 
 } // namespace ck

library/include/ck/library/tensor_operation_instance/device_operation_instance_factory.hpp

@@ -91,23 +91,26 @@ using GK_Tuple    = ck::Tuple<G_K>;
 using GK_GK_Tuple = ck::Tuple<G_K, G_K>;
 
 // pointwise functor
-using PassThrough    = ck::tensor_operation::element_wise::PassThrough;
-using Relu           = ck::tensor_operation::element_wise::Relu;
-using TanH           = ck::tensor_operation::element_wise::TanH;
-using Scale          = ck::tensor_operation::element_wise::Scale;
-using Bilinear       = ck::tensor_operation::element_wise::Bilinear;
-using AddAddFastGelu = ck::tensor_operation::element_wise::AddAddFastGelu;
-using AddFastGelu    = ck::tensor_operation::element_wise::AddFastGelu;
-using AddRelu        = ck::tensor_operation::element_wise::AddRelu;
-using AddSilu        = ck::tensor_operation::element_wise::AddSilu;
-using AddReluAdd     = ck::tensor_operation::element_wise::AddReluAdd;
-using FastGelu       = ck::tensor_operation::element_wise::FastGelu;
-using AddMultiply    = ck::tensor_operation::element_wise::AddMultiply;
-using MultiplyAdd    = ck::tensor_operation::element_wise::MultiplyAdd;
-using ScaleAdd       = ck::tensor_operation::element_wise::ScaleAdd;
-using Gelu           = ck::tensor_operation::element_wise::Gelu;
-using Swish          = ck::tensor_operation::element_wise::Swish;
-using Add            = ck::tensor_operation::element_wise::Add;
+using PassThrough         = ck::tensor_operation::element_wise::PassThrough;
+using Relu                = ck::tensor_operation::element_wise::Relu;
+using TanH                = ck::tensor_operation::element_wise::TanH;
+using Scale               = ck::tensor_operation::element_wise::Scale;
+using Bilinear            = ck::tensor_operation::element_wise::Bilinear;
+using AddAddFastGelu      = ck::tensor_operation::element_wise::AddAddFastGelu;
+using AddFastGelu         = ck::tensor_operation::element_wise::AddFastGelu;
+using MultiplyAddFastGelu = ck::tensor_operation::element_wise::MultiplyAddFastGelu;
+using AddRelu             = ck::tensor_operation::element_wise::AddRelu;
+using AddSilu             = ck::tensor_operation::element_wise::AddSilu;
+using AddReluAdd          = ck::tensor_operation::element_wise::AddReluAdd;
+using FastGelu            = ck::tensor_operation::element_wise::FastGelu;
+using MultiplyFastGelu    = ck::tensor_operation::element_wise::MultiplyFastGelu;
+using AddMultiply         = ck::tensor_operation::element_wise::AddMultiply;
+using MultiplyAdd         = ck::tensor_operation::element_wise::MultiplyAdd;
+using ScaleAdd            = ck::tensor_operation::element_wise::ScaleAdd;
+using Gelu                = ck::tensor_operation::element_wise::Gelu;
+using Swish               = ck::tensor_operation::element_wise::Swish;
+using Add                 = ck::tensor_operation::element_wise::Add;
+using Multiply            = ck::tensor_operation::element_wise::Multiply;
 
 template <typename Activation>
 using Activation_Mul_Clamp = ck::tensor_operation::element_wise::Activation_Mul_Clamp<Activation>;

library/src/tensor_operation_instance/gpu/gemm_multi_abd/CMakeLists.txt

@@ -4,7 +4,8 @@ set(GEMM_MULTI_ABD_INSTANCES)
 list(APPEND GEMM_MULTI_ABD_INSTANCES 
 	device_gemm_xdl_multi_abd_bias_gelu_bf16_i8_bf16_mk_kn_mn_v1_instance.cpp
 	device_gemm_xdl_multi_abd_bias_gelu_bf16_i8_bf16_mk_nk_mn_v1_instance.cpp
-	device_gemm_xdl_multi_abd_bias_gelu_bf16_i8_bf16_km_kn_mn_v1_instance.cpp
+
+	device_gemm_xdl_multi_abd_multiply_bias_gelu_bf16_i8_bf16_mk_kn_mn_v1_instance.cpp
 	)
 
 add_instance_library(device_gemm_multi_abd_instance ${GEMM_MULTI_ABD_INSTANCES})

library/src/tensor_operation_instance/gpu/grouped_gemm_fixed_nk_multi_abd/device_grouped_gemm_xdl_fixed_nk_bf16_i8_bf16_km_kn_mn_common.hpp

@@ -47,14 +47,14 @@ using D0Layout = Row;
 // using DsLayout = ck::Tuple<Row>;
 using ELayout = Row;
 
-using Scales      = ck::tensor_operation::element_wise::Scales;
+using Multiply    = ck::tensor_operation::element_wise::Multiply;
 using PassThrough = ck::tensor_operation::element_wise::PassThrough;
 using AddFastGelu = ck::tensor_operation::element_wise::AddFastGelu;
 using Add         = ck::tensor_operation::element_wise::Add;
 using FastGelu    = ck::tensor_operation::element_wise::FastGelu;
 
 using AElementOp = PassThrough;
-using BElementOp = Scales;
+using BElementOp = Multiply;
 // using CDEElementOp = AddFastGelu;
 
 static constexpr auto GemmDefault    = ck::tensor_operation::device::GemmSpecialization::Default;