Merge branch 'develop' of https://github.com/ROCmSoftwarePlatform/composable_kernel into wmma_op

library/include/ck/library/reference_tensor_operation/cpu/reference_softmax.hpp

@@ -86,8 +86,8 @@ struct ReferenceSoftmax : public device::BaseOperator
             };
 
             arg.in_.ForEach([&](auto& self, auto idx) {
-                reduce_max(to_sm_scalar_idx(idx)) = std::max(reduce_max(to_sm_scalar_idx(idx)),
-                                                             static_cast<AccDataType>(self(idx)));
+                reduce_max(to_sm_scalar_idx(idx)) = std::max(
+                    reduce_max(to_sm_scalar_idx(idx)), ck::type_convert<AccDataType>(self(idx)));
             });
 
             // LogRangeAsType<float>(std::cout << "reduce_max: ", reduce_max.mData, ",") <<
@@ -96,7 +96,7 @@ struct ReferenceSoftmax : public device::BaseOperator
             Tensor<AccDataType> in_stable(arg.in_.mDesc);
             in_stable.ForEach([&](auto& self, auto idx) {
                 // numerator = exp(x - max(x))
-                self(idx) = std::exp(static_cast<AccDataType>(arg.in_(idx)) -
+                self(idx) = std::exp(ck::type_convert<AccDataType>(arg.in_(idx)) -
                                      reduce_max(to_sm_scalar_idx(idx)));
             });
 
@@ -111,8 +111,10 @@ struct ReferenceSoftmax : public device::BaseOperator
             // std::endl;
 
             arg.out_.ForEach([&](auto& self, auto idx) {
-                self(idx) = arg.alpha_ * in_stable(idx) / reduce_sum(to_sm_scalar_idx(idx)) +
-                            arg.beta_ * self(idx);
+                AccDataType temp_result =
+                    arg.alpha_ * in_stable(idx) / reduce_sum(to_sm_scalar_idx(idx)) +
+                    arg.beta_ * self(idx);
+                self(idx) = ck::type_convert<OutDataType>(temp_result);
             });
 
             // LogRangeAsType<float>(std::cout << "out: ", arg.out_.mData, ",") << std::endl;

library/include/ck/library/tensor_operation_instance/device_operation_instance_factory.hpp

@@ -87,6 +87,8 @@ using Relu           = ck::tensor_operation::element_wise::Relu;
 using Scale          = ck::tensor_operation::element_wise::Scale;
 using Bilinear       = ck::tensor_operation::element_wise::Bilinear;
 using AddAddFastGelu = ck::tensor_operation::element_wise::AddAddFastGelu;
+using AddFastGelu    = ck::tensor_operation::element_wise::AddFastGelu;
+using FastGelu       = ck::tensor_operation::element_wise::FastGelu;
 
 template <typename Activation>
 using Activation_Mul_Clamp = ck::tensor_operation::element_wise::Activation_Mul_Clamp<Activation>;

library/src/tensor_operation_instance/gpu/batched_gemm_softmax_gemm_permute/CMakeLists.txt

 add_instance_library(device_batched_gemm_softmax_gemm_permute_instance
     device_batched_gemm_softmax_gemm_permute_xdl_cshuffle_f16_f16_f16_f16_gmk_gnk_gno_gmo_instance.cpp
+    device_batched_gemm_softmax_gemm_permute_xdl_cshuffle_bf16_bf16_bf16_bf16_gmk_gnk_gno_gmo_instance.cpp
 )
 

library/src/tensor_operation_instance/gpu/batchnorm/CMakeLists.txt

+add_instance_library(device_batchnorm_instance
+    device_batchnorm_forward_f16_instance.cpp
+    device_batchnorm_forward_f32_instance.cpp
+    device_batchnorm_forward_bf16_instance.cpp
+    device_batchnorm_forward_f64_instance.cpp
+)

library/src/tensor_operation_instance/gpu/conv2d_bwd_data/CMakeLists.txt

@@ -3,4 +3,8 @@ add_instance_library(device_conv2d_bwd_data_instance
    device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_f16_instance.cpp
    device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_bf16_instance.cpp
    device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_int8_instance.cpp
+
+   device_conv2d_bwd_data_dl_nhwc_kyxc_nhwk_f16_instance.cpp
+   device_conv2d_bwd_data_dl_nhwc_kyxc_nhwk_f32_instance.cpp
+   device_conv2d_bwd_data_dl_nhwc_kyxc_nhwk_int8_instance.cpp
 )