Merge branch 'develop' into add_int8_wmma_example_instance

library/src/tensor_operation_instance/gpu/grouped_conv3d_fwd/xdl/device_grouped_conv3d_fwd_xdl_ndhwgc_gkzyxc_ndhwgk_f16_comp_fp8_instance.cpp

+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved.
+
+#include "ck/library/tensor_operation_instance/gpu/grouped_conv_fwd/device_grouped_conv_fwd_xdl_instance.hpp"
+#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp"
+
+namespace ck {
+namespace tensor_operation {
+namespace device {
+namespace instance {
+
+void add_device_grouped_conv3d_fwd_xdl_ndhwgc_gkzyxc_ndhwgk_f16_comp_f8_instances(
+    std::vector<std::unique_ptr<DeviceGroupedConvFwdMultipleD<3,
+                                                              NDHWGC,
+                                                              GKZYXC,
+                                                              Empty_Tuple,
+                                                              NDHWGK,
+                                                              F16,
+                                                              F16,
+                                                              Empty_Tuple,
+                                                              F16,
+                                                              PassThrough,
+                                                              PassThrough,
+                                                              PassThrough,
+                                                              F8>>>& instances)
+{
+    add_device_operation_instances(
+        instances,
+        device_grouped_conv_fwd_xdl_f16_comp_f8_instances<3,
+                                                          NDHWGC,
+                                                          GKZYXC,
+                                                          Empty_Tuple,
+                                                          NDHWGK,
+                                                          ConvFwdDefault>{});
+    add_device_operation_instances(
+        instances,
+        device_grouped_conv_fwd_xdl_f16_comp_f8_instances<3,
+                                                          NDHWGC,
+                                                          GKZYXC,
+                                                          Empty_Tuple,
+                                                          NDHWGK,
+                                                          ConvFwd1x1P0>{});
+    add_device_operation_instances(
+        instances,
+        device_grouped_conv_fwd_xdl_f16_comp_f8_instances<3,
+                                                          NDHWGC,
+                                                          GKZYXC,
+                                                          Empty_Tuple,
+                                                          NDHWGK,
+                                                          ConvFwd1x1S1P0>{});
+}
+
+} // namespace instance
+} // namespace device
+} // namespace tensor_operation
+} // namespace ck

library/src/tensor_operation_instance/gpu/grouped_gemm/CMakeLists.txt

-if(DTYPES MATCHES "fp16" OR NOT DEFINED DTYPES)
 add_instance_library(device_grouped_gemm_instance
    device_grouped_gemm_xdl_f16_f16_f16_mk_kn_mn_instance.cpp
    device_grouped_gemm_xdl_f16_f16_f16_mk_nk_mn_instance.cpp
@@ -9,4 +8,3 @@ add_instance_library(device_grouped_gemm_instance
    device_grouped_gemm_xdl_splitk_f16_f16_f16_mk_kn_mn_irregular_instance.cpp
    device_grouped_gemm_xdl_splitk_f16_f16_f16_mk_nk_mn_irregular_instance.cpp
 )
-endif()

library/src/tensor_operation_instance/gpu/grouped_gemm_fastgelu/CMakeLists.txt

-if(DTYPES MATCHES "fp16" OR NOT DEFINED DTYPES)
 add_instance_library(device_grouped_gemm_fastgelu_instance
    device_grouped_gemm_fastgelu_xdl_f16_f16_f16_mk_kn_mn_instance.cpp
    device_grouped_gemm_fastgelu_xdl_f16_f16_f16_mk_nk_mn_instance.cpp
    device_grouped_gemm_fastgelu_xdl_f16_f16_f16_km_kn_mn_instance.cpp
    device_grouped_gemm_fastgelu_xdl_f16_f16_f16_km_nk_mn_instance.cpp
 )
-endif()

library/src/tensor_operation_instance/gpu/grouped_gemm_fixed_nk/CMakeLists.txt

-add_instance_library(device_grouped_gemm_fixed_nk_instance
-   device_grouped_gemm_xdl_fixed_nk_f16_f16_f16_mk_kn_mn_instance.cpp
-   device_grouped_gemm_xdl_fixed_nk_f16_f16_f16_mk_nk_mn_instance.cpp
+set(GROUPED_GEMM_FIXED_NK_INSTANCES)
 
-   device_grouped_gemm_xdl_fixed_nk_f16_f8_f16_mk_kn_mn_instance.cpp
-   device_grouped_gemm_xdl_fixed_nk_f16_f8_f16_mk_nk_mn_instance.cpp
+list(APPEND GROUPED_GEMM_FIXED_NK_INSTANCES device_grouped_gemm_xdl_fixed_nk_f16_f16_f16_mk_kn_mn_instance.cpp
+                                            device_grouped_gemm_xdl_fixed_nk_f16_f16_f16_mk_nk_mn_instance.cpp
+                                            device_grouped_gemm_xdl_fixed_nk_f16_fp8_f16_mk_kn_mn_instance.cpp
+                                            device_grouped_gemm_xdl_fixed_nk_f16_fp8_f16_mk_nk_mn_instance.cpp
+                                            device_grouped_gemm_xdl_fixed_nk_f16_i8_f16_mk_kn_mn_instance.cpp
+                                            device_grouped_gemm_xdl_fixed_nk_f16_i8_f16_mk_nk_mn_instance.cpp)
 
-   device_grouped_gemm_xdl_fixed_nk_f16_i8_f16_mk_kn_mn_instance.cpp
-   device_grouped_gemm_xdl_fixed_nk_f16_i8_f16_mk_nk_mn_instance.cpp
-)
+add_instance_library(device_grouped_gemm_fixed_nk_instance ${GROUPED_GEMM_FIXED_NK_INSTANCES})

library/src/tensor_operation_instance/gpu/image_to_column/device_image_to_column_nhwc_1d_instance.cpp

 // SPDX-License-Identifier: MIT
 // Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved.
 
-#include "ck/library/tensor_operation_instance/gpu/image_to_column/device_image_to_column_instance.hpp"
+#include "ck/library/tensor_operation_instance/gpu/conv_tensor_rearrange/device_image_to_column_instance.hpp"
 #include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp"
 
 namespace ck {
@@ -9,28 +9,50 @@ namespace tensor_operation {
 namespace device {
 namespace instance {
 
-void add_device_image_to_column_nhwc_1d_bf16_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<1, GNWC, BF16, BF16>>>& instances)
+using namespace ck::conv_tensor_rearrange_op;
+
+void add_device_image_to_column_nwc_1d_bf16_instances(
+    std::vector<std::unique_ptr<DeviceConvTensorRearrange<1, GNWC, BF16, BF16, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_BF16
     add_device_operation_instances(instances, device_image_to_column_bf16_instances<1, GNWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
-void add_device_image_to_column_nhwc_1d_f16_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<1, GNWC, F16, F16>>>& instances)
+void add_device_image_to_column_nwc_1d_f16_instances(
+    std::vector<std::unique_ptr<DeviceConvTensorRearrange<1, GNWC, F16, F16, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_FP16
     add_device_operation_instances(instances, device_image_to_column_f16_instances<1, GNWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
-void add_device_image_to_column_nhwc_1d_f32_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<1, GNWC, F32, F32>>>& instances)
+void add_device_image_to_column_nwc_1d_f32_instances(
+    std::vector<std::unique_ptr<DeviceConvTensorRearrange<1, GNWC, F32, F32, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_FP32
     add_device_operation_instances(instances, device_image_to_column_f32_instances<1, GNWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
-void add_device_image_to_column_nhwc_1d_i8_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<1, GNWC, int8_t, int8_t>>>& instances)
+void add_device_image_to_column_nwc_1d_i8_instances(
+    std::vector<std::unique_ptr<DeviceConvTensorRearrange<1, GNWC, int8_t, int8_t, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_INT8
     add_device_operation_instances(instances, device_image_to_column_i8_instances<1, GNWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
 } // namespace instance

library/src/tensor_operation_instance/gpu/image_to_column/device_image_to_column_nhwc_2d_instance.cpp

 // SPDX-License-Identifier: MIT
 // Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved.
 
-#include "ck/library/tensor_operation_instance/gpu/image_to_column/device_image_to_column_instance.hpp"
+#include "ck/library/tensor_operation_instance/gpu/conv_tensor_rearrange/device_image_to_column_instance.hpp"
 #include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp"
 
 namespace ck {
@@ -9,28 +9,51 @@ namespace tensor_operation {
 namespace device {
 namespace instance {
 
+using namespace ck::conv_tensor_rearrange_op;
+
 void add_device_image_to_column_nhwc_2d_bf16_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<2, GNHWC, BF16, BF16>>>& instances)
+    std::vector<std::unique_ptr<DeviceConvTensorRearrange<2, GNHWC, BF16, BF16, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_BF16
     add_device_operation_instances(instances, device_image_to_column_bf16_instances<2, GNHWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
 void add_device_image_to_column_nhwc_2d_f16_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<2, GNHWC, F16, F16>>>& instances)
+    std::vector<std::unique_ptr<DeviceConvTensorRearrange<2, GNHWC, F16, F16, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_FP16
     add_device_operation_instances(instances, device_image_to_column_f16_instances<2, GNHWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
 void add_device_image_to_column_nhwc_2d_f32_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<2, GNHWC, F32, F32>>>& instances)
+    std::vector<std::unique_ptr<DeviceConvTensorRearrange<2, GNHWC, F32, F32, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_FP32
     add_device_operation_instances(instances, device_image_to_column_f32_instances<2, GNHWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
 void add_device_image_to_column_nhwc_2d_i8_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<2, GNHWC, int8_t, int8_t>>>& instances)
+    std::vector<
+        std::unique_ptr<DeviceConvTensorRearrange<2, GNHWC, int8_t, int8_t, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_INT8
     add_device_operation_instances(instances, device_image_to_column_i8_instances<2, GNHWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
 } // namespace instance

library/src/tensor_operation_instance/gpu/image_to_column/device_image_to_column_nhwc_3d_instance.cpp

 // SPDX-License-Identifier: MIT
 // Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved.
 
-#include "ck/library/tensor_operation_instance/gpu/image_to_column/device_image_to_column_instance.hpp"
+#include "ck/library/tensor_operation_instance/gpu/conv_tensor_rearrange/device_image_to_column_instance.hpp"
 #include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp"
 
 namespace ck {
@@ -9,28 +9,51 @@ namespace tensor_operation {
 namespace device {
 namespace instance {
 
-void add_device_image_to_column_nhwc_3d_bf16_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<3, GNDHWC, BF16, BF16>>>& instances)
+using namespace ck::conv_tensor_rearrange_op;
+
+void add_device_image_to_column_ndhwc_3d_bf16_instances(
+    std::vector<std::unique_ptr<DeviceConvTensorRearrange<3, GNDHWC, BF16, BF16, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_BF16
     add_device_operation_instances(instances, device_image_to_column_bf16_instances<3, GNDHWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
-void add_device_image_to_column_nhwc_3d_f16_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<3, GNDHWC, F16, F16>>>& instances)
+void add_device_image_to_column_ndhwc_3d_f16_instances(
+    std::vector<std::unique_ptr<DeviceConvTensorRearrange<3, GNDHWC, F16, F16, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_FP16
     add_device_operation_instances(instances, device_image_to_column_f16_instances<3, GNDHWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
-void add_device_image_to_column_nhwc_3d_f32_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<3, GNDHWC, F32, F32>>>& instances)
+void add_device_image_to_column_ndhwc_3d_f32_instances(
+    std::vector<std::unique_ptr<DeviceConvTensorRearrange<3, GNDHWC, F32, F32, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_FP32
     add_device_operation_instances(instances, device_image_to_column_f32_instances<3, GNDHWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
-void add_device_image_to_column_nhwc_3d_i8_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<3, GNDHWC, int8_t, int8_t>>>& instances)
+void add_device_image_to_column_ndhwc_3d_i8_instances(
+    std::vector<
+        std::unique_ptr<DeviceConvTensorRearrange<3, GNDHWC, int8_t, int8_t, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_INT8
     add_device_operation_instances(instances, device_image_to_column_i8_instances<3, GNDHWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
 } // namespace instance

library/src/tensor_operation_instance/gpu/max_pool_bwd/CMakeLists.txt

 set(DEVICE_MAXPOOL_BWD_INSTANCES)
-if(DTYPES MATCHES "fp16" OR NOT DEFINED DTYPES)
-    list(APPEND DEVICE_MAXPOOL_BWD_INSTANCES device_max_pool_bwd_f16_instance.cpp)
-endif()
-if(DTYPES MATCHES "bf16" OR NOT DEFINED DTYPES)
-    list(APPEND DEVICE_MAXPOOL_BWD_INSTANCES device_max_pool_bwd_bf16_instance.cpp)
-endif()
-if(DTYPES MATCHES "fp32" OR NOT DEFINED DTYPES)
-    list(APPEND DEVICE_MAXPOOL_BWD_INSTANCES device_max_pool_bwd_f32_instance.cpp)
-endif()
+list(APPEND DEVICE_MAXPOOL_BWD_INSTANCES device_max_pool_bwd_f16_instance.cpp
+                                         device_max_pool_bwd_bf16_instance.cpp
+                                         device_max_pool_bwd_f32_instance.cpp)
 add_instance_library(device_max_pool_bwd_instance ${DEVICE_MAXPOOL_BWD_INSTANCES})

library/src/tensor_operation_instance/gpu/normalization/CMakeLists.txt

 set(DEVICE_NORMALIZATION_INSTANCES)
-if(DTYPES MATCHES "fp16" OR NOT DEFINED DTYPES)
-    list(APPEND DEVICE_NORMALIZATION_INSTANCES device_layernorm2d_f16_instance.cpp
+
+list(APPEND DEVICE_NORMALIZATION_INSTANCES 
+    device_layernorm2d_f16_instance.cpp
     device_layernorm4d_f16_instance.cpp
     device_groupnorm_f16_instance.cpp
     device_groupnorm_swish_f16_instance.cpp
-    device_groupnorm_swish_f16_f32_f32_f16_instance.cpp)
-endif()
-if(DTYPES MATCHES "fp32" OR NOT DEFINED DTYPES)
-    list(APPEND DEVICE_NORMALIZATION_INSTANCES device_layernorm2d_f32_instance.cpp
+    device_groupnorm_swish_f16_f32_f32_f16_instance.cpp
+    device_layernorm2d_f32_instance.cpp
     device_layernorm4d_f32_instance.cpp
     device_groupnorm_f32_instance.cpp
     device_groupnorm_swish_f32_instance.cpp)
-endif()
+
 add_instance_library(device_normalization_instance ${DEVICE_NORMALIZATION_INSTANCES})

library/src/tensor_operation_instance/gpu/normalization/device_groupnorm_f16_instance.cpp

@@ -11,7 +11,7 @@ namespace instance {
 using Pass = ck::tensor_operation::element_wise::PassThrough;
 
 void add_device_normalization_rank_5_3_f16_instances(
-    std::vector<std::unique_ptr<DeviceNormalization<F16, F16, F16, F32, F16, Pass, 5, 3>>>&
+    std::vector<std::unique_ptr<DeviceNormalization<F16, F16, F16, F16, F32, Pass, 5, 3>>>&
         instances)
 {
     add_device_operation_instances(instances,

library/src/tensor_operation_instance/gpu/normalization/device_groupnorm_swish_f16_f32_f32_f16_instance.cpp

@@ -11,7 +11,7 @@ namespace instance {
 using Swish = ck::tensor_operation::element_wise::Swish;
 
 void add_device_normalization_rank_5_3_swish_f16_f32_f32_f16_instances(
-    std::vector<std::unique_ptr<DeviceNormalization<F16, F32, F32, F32, F16, Swish, 5, 3>>>&
+    std::vector<std::unique_ptr<DeviceNormalization<F16, F32, F32, F16, F32, Swish, 5, 3>>>&
         instances)
 {
     add_device_operation_instances(