Merge branch 'develop' into transpose_5d

library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/CMakeLists.txt

@@ -4,7 +4,8 @@ set(GROUPED_CONV3D_BWD_WEIGHT
     device_grouped_conv3d_bwd_weight_xdl_gndhwc_gkzyxc_gndhwk_bf16_instance.cpp
     device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f16_instance.cpp
     device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_instance.cpp
-    device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_bf16_instance.cpp)
+    device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_bf16_instance.cpp
+    device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f16_comp_bf8_fp8_instance.cpp)
 
 if(DL_KERNELS)
     list(APPEND GROUPED_CONV3D_BWD_WEIGHT

library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f16_comp_bf8_fp8_instance.cpp

+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved.
+
+#include "ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_weight/device_grouped_conv_bwd_weight_xdl_instance.hpp"
+#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp"
+
+namespace ck {
+namespace tensor_operation {
+namespace device {
+namespace instance {
+
+void add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f16_comp_bf8_f8_instances(
+    std::vector<std::unique_ptr<DeviceGroupedConvBwdWeight<3,
+                                                           NDHWGC,
+                                                           GKZYXC,
+                                                           NDHWGK,
+                                                           F16,
+                                                           F16,
+                                                           F16,
+                                                           PassThrough,
+                                                           PassThrough,
+                                                           PassThrough,
+                                                           BF8,
+                                                           F8>>>& instances)
+{
+    // 1. Default
+    add_device_operation_instances(
+        instances,
+        device_grouped_conv_bwd_weight_xdl_c_shuffle_f16_comp_bf8_f8_instances<
+            3,
+            NDHWGC,
+            GKZYXC,
+            NDHWGK,
+            ConvBwdWeightDefault>{});
+    // 2. Filter1x1Stride1Pad0
+    add_device_operation_instances(
+        instances,
+        device_grouped_conv_bwd_weight_xdl_c_shuffle_f16_comp_bf8_f8_instances<
+            3,
+            NDHWGC,
+            GKZYXC,
+            NDHWGK,
+            ConvBwdWeightFilter1x1Stride1Pad0>{});
+}
+
+} // namespace instance
+} // namespace device
+} // namespace tensor_operation
+} // namespace ck

library/src/tensor_operation_instance/gpu/grouped_conv3d_fwd/CMakeLists.txt

@@ -4,10 +4,12 @@ add_instance_library(device_grouped_conv3d_fwd_instance
    xdl/device_grouped_conv3d_fwd_xdl_gndhwc_gkzyxc_gndhwk_f32_instance.cpp
    xdl/device_grouped_conv3d_fwd_xdl_gndhwc_gkzyxc_gndhwk_int8_instance.cpp
 
+
    xdl/device_grouped_conv3d_fwd_xdl_ndhwgc_gkzyxc_ndhwgk_bf16_instance.cpp
    xdl/device_grouped_conv3d_fwd_xdl_ndhwgc_gkzyxc_ndhwgk_f16_instance.cpp
    xdl/device_grouped_conv3d_fwd_xdl_ndhwgc_gkzyxc_ndhwgk_f32_instance.cpp
    xdl/device_grouped_conv3d_fwd_xdl_ndhwgc_gkzyxc_ndhwgk_int8_instance.cpp
+   xdl/device_grouped_conv3d_fwd_xdl_ndhwgc_gkzyxc_ndhwgk_f16_comp_fp8_instance.cpp
 
    wmma/device_grouped_conv3d_fwd_wmma_gndhwc_gkzyxc_gndhwk_f16_instance.cpp
    wmma/device_grouped_conv3d_fwd_wmma_gndhwc_gkzyxc_gndhwk_i8_instance.cpp

library/src/tensor_operation_instance/gpu/grouped_conv3d_fwd/xdl/device_grouped_conv3d_fwd_xdl_ndhwgc_gkzyxc_ndhwgk_f16_comp_fp8_instance.cpp

+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved.
+
+#include "ck/library/tensor_operation_instance/gpu/grouped_conv_fwd/device_grouped_conv_fwd_xdl_instance.hpp"
+#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp"
+
+namespace ck {
+namespace tensor_operation {
+namespace device {
+namespace instance {
+
+void add_device_grouped_conv3d_fwd_xdl_ndhwgc_gkzyxc_ndhwgk_f16_comp_f8_instances(
+    std::vector<std::unique_ptr<DeviceGroupedConvFwdMultipleD<3,
+                                                              NDHWGC,
+                                                              GKZYXC,
+                                                              Empty_Tuple,
+                                                              NDHWGK,
+                                                              F16,
+                                                              F16,
+                                                              Empty_Tuple,
+                                                              F16,
+                                                              PassThrough,
+                                                              PassThrough,
+                                                              PassThrough,
+                                                              F8>>>& instances)
+{
+    add_device_operation_instances(
+        instances,
+        device_grouped_conv_fwd_xdl_f16_comp_f8_instances<3,
+                                                          NDHWGC,
+                                                          GKZYXC,
+                                                          Empty_Tuple,
+                                                          NDHWGK,
+                                                          ConvFwdDefault>{});
+    add_device_operation_instances(
+        instances,
+        device_grouped_conv_fwd_xdl_f16_comp_f8_instances<3,
+                                                          NDHWGC,
+                                                          GKZYXC,
+                                                          Empty_Tuple,
+                                                          NDHWGK,
+                                                          ConvFwd1x1P0>{});
+    add_device_operation_instances(
+        instances,
+        device_grouped_conv_fwd_xdl_f16_comp_f8_instances<3,
+                                                          NDHWGC,
+                                                          GKZYXC,
+                                                          Empty_Tuple,
+                                                          NDHWGK,
+                                                          ConvFwd1x1S1P0>{});
+}
+
+} // namespace instance
+} // namespace device
+} // namespace tensor_operation
+} // namespace ck

library/src/tensor_operation_instance/gpu/grouped_gemm_fixed_nk/CMakeLists.txt

-add_instance_library(device_grouped_gemm_fixed_nk_instance
-   device_grouped_gemm_xdl_fixed_nk_f16_f16_f16_mk_kn_mn_instance.cpp
-   device_grouped_gemm_xdl_fixed_nk_f16_f16_f16_mk_nk_mn_instance.cpp
+set(GROUPED_GEMM_FIXED_NK_INSTANCES)
 
-   device_grouped_gemm_xdl_fixed_nk_f16_f8_f16_mk_kn_mn_instance.cpp
-   device_grouped_gemm_xdl_fixed_nk_f16_f8_f16_mk_nk_mn_instance.cpp
+if(DTYPES MATCHES "fp16" OR NOT DEFINED DTYPES)
+  list(APPEND GROUPED_GEMM_FIXED_NK_INSTANCES device_grouped_gemm_xdl_fixed_nk_f16_f16_f16_mk_kn_mn_instance.cpp)
+  list(APPEND GROUPED_GEMM_FIXED_NK_INSTANCES device_grouped_gemm_xdl_fixed_nk_f16_f16_f16_mk_nk_mn_instance.cpp)
+endif()
 
-   device_grouped_gemm_xdl_fixed_nk_f16_i8_f16_mk_kn_mn_instance.cpp
-   device_grouped_gemm_xdl_fixed_nk_f16_i8_f16_mk_nk_mn_instance.cpp
-)
+if((DTYPES MATCHES "fp8" AND DTYPES MATCHES "fp16") OR NOT DEFINED DTYPES)
+  list(APPEND GROUPED_GEMM_FIXED_NK_INSTANCES device_grouped_gemm_xdl_fixed_nk_f16_fp8_f16_mk_kn_mn_instance.cpp)
+  list(APPEND GROUPED_GEMM_FIXED_NK_INSTANCES device_grouped_gemm_xdl_fixed_nk_f16_fp8_f16_mk_nk_mn_instance.cpp)
+endif()
+
+if((DTYPES MATCHES "int8" AND DTYPES MATCHES "fp16") OR NOT DEFINED DTYPES)
+  list(APPEND GROUPED_GEMM_FIXED_NK_INSTANCES device_grouped_gemm_xdl_fixed_nk_f16_i8_f16_mk_kn_mn_instance.cpp)
+  list(APPEND GROUPED_GEMM_FIXED_NK_INSTANCES device_grouped_gemm_xdl_fixed_nk_f16_i8_f16_mk_nk_mn_instance.cpp)
+endif()
+
+add_instance_library(device_grouped_gemm_fixed_nk_instance ${GROUPED_GEMM_FIXED_NK_INSTANCES})

library/src/tensor_operation_instance/gpu/image_to_column/device_image_to_column_nhwc_1d_instance.cpp

 // SPDX-License-Identifier: MIT
 // Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved.
 
-#include "ck/library/tensor_operation_instance/gpu/image_to_column/device_image_to_column_instance.hpp"
+#include "ck/library/tensor_operation_instance/gpu/conv_tensor_rearrange/device_image_to_column_instance.hpp"
 #include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp"
 
 namespace ck {
@@ -9,28 +9,50 @@ namespace tensor_operation {
 namespace device {
 namespace instance {
 
-void add_device_image_to_column_nhwc_1d_bf16_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<1, GNWC, BF16, BF16>>>& instances)
+using namespace ck::conv_tensor_rearrange_op;
+
+void add_device_image_to_column_nwc_1d_bf16_instances(
+    std::vector<std::unique_ptr<DeviceConvTensorRearrange<1, GNWC, BF16, BF16, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_BF16
     add_device_operation_instances(instances, device_image_to_column_bf16_instances<1, GNWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
-void add_device_image_to_column_nhwc_1d_f16_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<1, GNWC, F16, F16>>>& instances)
+void add_device_image_to_column_nwc_1d_f16_instances(
+    std::vector<std::unique_ptr<DeviceConvTensorRearrange<1, GNWC, F16, F16, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_FP16
     add_device_operation_instances(instances, device_image_to_column_f16_instances<1, GNWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
-void add_device_image_to_column_nhwc_1d_f32_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<1, GNWC, F32, F32>>>& instances)
+void add_device_image_to_column_nwc_1d_f32_instances(
+    std::vector<std::unique_ptr<DeviceConvTensorRearrange<1, GNWC, F32, F32, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_FP32
     add_device_operation_instances(instances, device_image_to_column_f32_instances<1, GNWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
-void add_device_image_to_column_nhwc_1d_i8_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<1, GNWC, int8_t, int8_t>>>& instances)
+void add_device_image_to_column_nwc_1d_i8_instances(
+    std::vector<std::unique_ptr<DeviceConvTensorRearrange<1, GNWC, int8_t, int8_t, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_INT8
     add_device_operation_instances(instances, device_image_to_column_i8_instances<1, GNWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
 } // namespace instance

library/src/tensor_operation_instance/gpu/image_to_column/device_image_to_column_nhwc_2d_instance.cpp

 // SPDX-License-Identifier: MIT
 // Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved.
 
-#include "ck/library/tensor_operation_instance/gpu/image_to_column/device_image_to_column_instance.hpp"
+#include "ck/library/tensor_operation_instance/gpu/conv_tensor_rearrange/device_image_to_column_instance.hpp"
 #include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp"
 
 namespace ck {
@@ -9,28 +9,51 @@ namespace tensor_operation {
 namespace device {
 namespace instance {
 
+using namespace ck::conv_tensor_rearrange_op;
+
 void add_device_image_to_column_nhwc_2d_bf16_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<2, GNHWC, BF16, BF16>>>& instances)
+    std::vector<std::unique_ptr<DeviceConvTensorRearrange<2, GNHWC, BF16, BF16, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_BF16
     add_device_operation_instances(instances, device_image_to_column_bf16_instances<2, GNHWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
 void add_device_image_to_column_nhwc_2d_f16_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<2, GNHWC, F16, F16>>>& instances)
+    std::vector<std::unique_ptr<DeviceConvTensorRearrange<2, GNHWC, F16, F16, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_FP16
     add_device_operation_instances(instances, device_image_to_column_f16_instances<2, GNHWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
 void add_device_image_to_column_nhwc_2d_f32_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<2, GNHWC, F32, F32>>>& instances)
+    std::vector<std::unique_ptr<DeviceConvTensorRearrange<2, GNHWC, F32, F32, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_FP32
     add_device_operation_instances(instances, device_image_to_column_f32_instances<2, GNHWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
 void add_device_image_to_column_nhwc_2d_i8_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<2, GNHWC, int8_t, int8_t>>>& instances)
+    std::vector<
+        std::unique_ptr<DeviceConvTensorRearrange<2, GNHWC, int8_t, int8_t, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_INT8
     add_device_operation_instances(instances, device_image_to_column_i8_instances<2, GNHWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
 } // namespace instance

library/src/tensor_operation_instance/gpu/image_to_column/device_image_to_column_nhwc_3d_instance.cpp

 // SPDX-License-Identifier: MIT
 // Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved.
 
-#include "ck/library/tensor_operation_instance/gpu/image_to_column/device_image_to_column_instance.hpp"
+#include "ck/library/tensor_operation_instance/gpu/conv_tensor_rearrange/device_image_to_column_instance.hpp"
 #include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp"
 
 namespace ck {
@@ -9,28 +9,51 @@ namespace tensor_operation {
 namespace device {
 namespace instance {
 
-void add_device_image_to_column_nhwc_3d_bf16_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<3, GNDHWC, BF16, BF16>>>& instances)
+using namespace ck::conv_tensor_rearrange_op;
+
+void add_device_image_to_column_ndhwc_3d_bf16_instances(
+    std::vector<std::unique_ptr<DeviceConvTensorRearrange<3, GNDHWC, BF16, BF16, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_BF16
     add_device_operation_instances(instances, device_image_to_column_bf16_instances<3, GNDHWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
-void add_device_image_to_column_nhwc_3d_f16_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<3, GNDHWC, F16, F16>>>& instances)
+void add_device_image_to_column_ndhwc_3d_f16_instances(
+    std::vector<std::unique_ptr<DeviceConvTensorRearrange<3, GNDHWC, F16, F16, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_FP16
     add_device_operation_instances(instances, device_image_to_column_f16_instances<3, GNDHWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
-void add_device_image_to_column_nhwc_3d_f32_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<3, GNDHWC, F32, F32>>>& instances)
+void add_device_image_to_column_ndhwc_3d_f32_instances(
+    std::vector<std::unique_ptr<DeviceConvTensorRearrange<3, GNDHWC, F32, F32, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_FP32
     add_device_operation_instances(instances, device_image_to_column_f32_instances<3, GNDHWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
-void add_device_image_to_column_nhwc_3d_i8_instances(
-    std::vector<std::unique_ptr<DeviceImageToColumn<3, GNDHWC, int8_t, int8_t>>>& instances)
+void add_device_image_to_column_ndhwc_3d_i8_instances(
+    std::vector<
+        std::unique_ptr<DeviceConvTensorRearrange<3, GNDHWC, int8_t, int8_t, ImageToColumn>>>&
+        instances)
 {
+#ifdef CK_ENABLE_INT8
     add_device_operation_instances(instances, device_image_to_column_i8_instances<3, GNDHWC>{});
+#else
+    ignore = instances;
+#endif
 }
 
 } // namespace instance

profiler/README.md

@@ -185,7 +185,7 @@ GB/s: 69.2301
 ```
 Note: This kernel use atomic add, this will cause output buffer to be accumulated multiple times, causing verification failure. To work around it, do not use CK's own timer and do verification at the same time.
 
-## Profile image to column kernels
+## Profile image to column/column to image kernels
 ```bash
 # arg1: tensor operation (" OP_NAME ": " OP_DESC ")
 # arg2: data type (0: Input fp32, Weight fp32, Output fp32
@@ -197,6 +197,7 @@ Note: This kernel use atomic add, this will cause output buffer to be accumulate
 # arg5: initialization (0: no init, 1: integer value, 2: decimal value)
 # arg6: print tensor value (0: no; 1: yes)
 # arg7: time kernel (0: no, 1: yes)
+# arg8: operation type (0: ImageToColumn, 1: ColumnToImage)
 # Following arguments (depending on number of spatial dims):
 #  Number of spatial dimensions (1=Conv1d, 2=Conv2d, 3=Conv3d)
 #  G, N, K, C, 
@@ -207,8 +208,8 @@ Note: This kernel use atomic add, this will cause output buffer to be accumulate
 #  <left padding>, (ie LeftPy, LeftPx for 2D)
 #  <right padding>, (ie RightPy, RightPx for 2D)
 
- ################             op   datatype  layout  verify  init  log  time  Ndims  G   N   K   C  Y  X  Hi  Wi  Sy  Sx  Dy  Dx  LeftPy  LeftPx  RightPy  RightPx
-./bin/ckProfiler image_to_column          0       0       1     1    0     1      2  1 256   1 512  3  3   28  28   1   1   1   1        0       0       0        0
+ ################                   op   datatype  layout  verify  init  log  time opType Ndims  G   N   K   C  Y  X  Hi  Wi  Sy  Sx  Dy  Dx  LeftPy  LeftPx  RightPy  RightPx
+./bin/ckProfiler conv_tensor_rearrange          0       0       0     1    0     1      0     2  1 256   1 512  3  3   28  28   1   1   1   1        0       0       0        0
 
  ```
 
@@ -222,3 +223,4 @@ name: DeviceImageToColumn<128, 32, 64, 4>
 avg_time: 3.12326
 GB/s: 2042.59
 ```
+Note: Column to image kernel adds to the output memory, this will cause output buffer to be accumulated multiple times, causing verification failure. To work around it, do not use CK's own timer and do verification at the same time.

profiler/include/profiler/profile_image_to_column_impl.hpp → profiler/include/profiler/profile_conv_tensor_rearrange_impl.hpp

@@ -9,9 +9,11 @@
 #include <limits>
 #include "ck/ck.hpp"
 #include "ck/tensor_operation/gpu/device/tensor_layout.hpp"
-#include "ck/tensor_operation/gpu/device/device_image_to_column.hpp"
+#include "ck/tensor_operation/gpu/device/device_conv_tensor_rearrange.hpp"
+#include "ck/tensor_operation/gpu/device/conv_tensor_rearrange_op.hpp"
 #include "ck/tensor_operation/gpu/device/impl/device_image_to_column_impl.hpp"
-#include "ck/library/tensor_operation_instance/gpu/image_to_column.hpp"
+#include "ck/tensor_operation/gpu/device/impl/device_column_to_image_impl.hpp"
+#include "ck/library/tensor_operation_instance/gpu/conv_tensor_rearrange.hpp"
 #include "ck/library/utility/check_err.hpp"
 #include "ck/library/utility/device_memory.hpp"
 #include "ck/library/utility/host_tensor.hpp"
@@ -19,22 +21,88 @@
 #include "ck/library/utility/convolution_parameter.hpp"
 #include "ck/library/utility/convolution_host_tensor_descriptor_helper.hpp"
 #include "ck/library/reference_tensor_operation/cpu/reference_image_to_column.hpp"
+#include "ck/library/reference_tensor_operation/cpu/reference_column_to_image.hpp"
 
 namespace ck {
 namespace profiler {
 
 template <ck::index_t... Is>
 using S = ck::Sequence<Is...>;
+using namespace conv_tensor_rearrange_op;
+
+template <typename InputDataType, typename ConvTensorRearrangeOp>
+Tensor<InputDataType> create_input(const HostTensorDescriptor& image_desc,
+                                   const HostTensorDescriptor& gemm_desc)
+{
+    if constexpr(std::is_same_v<ConvTensorRearrangeOp, ImageToColumn>)
+    {
+        Tensor<InputDataType> input(image_desc);
+        return input;
+    }
+    else if constexpr(std::is_same_v<ConvTensorRearrangeOp, ColumnToImage>)
+    {
+        Tensor<InputDataType> input(gemm_desc);
+        return input;
+    }
+    else
+    {
+        throw std::runtime_error("Unsupported op!");
+    }
+}
+
+template <typename OutputDataType, typename ConvTensorRearrangeOp>
+Tensor<OutputDataType> create_output(const HostTensorDescriptor& image_desc,
+                                     const HostTensorDescriptor& gemm_desc)
+{
+    if constexpr(std::is_same_v<ConvTensorRearrangeOp, ImageToColumn>)
+    {
+        Tensor<OutputDataType> output(gemm_desc);
+        return output;
+    }
+    else if constexpr(std::is_same_v<ConvTensorRearrangeOp, ColumnToImage>)
+    {
+        Tensor<OutputDataType> output(image_desc);
+        return output;
+    }
+    else
+    {
+        throw std::runtime_error("Unsupported op!");
+    }
+}
+
+template <index_t NDimSpatial,
+          typename InputLayout,
+          typename InputDataType,
+          typename OutputDataType,
+          typename ConvTensorRearrangeOp>
+static auto make_ref_op()
+{
+    if constexpr(std::is_same_v<ConvTensorRearrangeOp, ImageToColumn>)
+    {
+        return ck::tensor_operation::host::
+            ReferenceImageToColumn<NDimSpatial, InputLayout, InputDataType, OutputDataType>{};
+    }
+    else if constexpr(std::is_same_v<ConvTensorRearrangeOp, ColumnToImage>)
+    {
+        return ck::tensor_operation::host::
+            ReferenceColumnToImage<NDimSpatial, InputLayout, InputDataType, OutputDataType>{};
+    }
+    else
+    {
+        throw std::runtime_error("Unsupported op!");
+    }
+}
 
 template <index_t NDimSpatial,
           typename InputLayout,
           typename InputDataType,
-          typename OutputDataType>
-bool profile_image_to_column_impl(int do_verification,
-                                  int init_method,
-                                  bool do_log,
-                                  bool time_kernel,
-                                  const ck::utils::conv::ConvParam& conv_param)
+          typename OutputDataType,
+          typename ConvTensorRearrangeOp>
+bool profile_conv_tensor_rearrange_impl(int do_verification,
+                                        int init_method,
+                                        bool do_log,
+                                        bool time_kernel,
+                                        const ck::utils::conv::ConvParam& conv_param)
 {
     const ck::index_t NDoHoWo =
         conv_param.N_ *
@@ -45,16 +113,16 @@ bool profile_image_to_column_impl(int do_verification,
         ck::accumulate_n<ck::index_t>(
             conv_param.filter_spatial_lengths_.begin(), NDimSpatial, 1, std::multiplies<>());
 
-    const auto in_desc =
+    const auto image_desc =
         ck::utils::conv::make_input_host_tensor_descriptor_g_n_c_wis_packed<InputLayout>(
             conv_param);
-    const auto out_desc = HostTensorDescriptor({NDoHoWo, CZYX});
+    const auto gemm_desc = HostTensorDescriptor({NDoHoWo, CZYX});
 
     std::array<ck::index_t, NDimSpatial> input_spatial_lengths{};
     std::array<ck::index_t, NDimSpatial> filter_spatial_lengths{};
     std::array<ck::index_t, NDimSpatial> output_spatial_lengths{};
-    std::array<ck::index_t, NDimSpatial + 3> input_g_n_c_wis_strides{};
-    std::array<ck::index_t, 2> output_m_k_strides{};
+    std::array<ck::index_t, NDimSpatial + 3> image_g_n_c_wis_strides{};
+    std::array<ck::index_t, 2> gemm_m_k_strides{};
     std::array<ck::index_t, NDimSpatial> conv_filter_strides{};
     std::array<ck::index_t, NDimSpatial> conv_filter_dilations{};
     std::array<ck::index_t, NDimSpatial> input_left_pads{};
@@ -65,16 +133,19 @@ bool profile_image_to_column_impl(int do_verification,
     copy(conv_param.input_spatial_lengths_, input_spatial_lengths);
     copy(conv_param.filter_spatial_lengths_, filter_spatial_lengths);
     copy(conv_param.output_spatial_lengths_, output_spatial_lengths);
-    copy(in_desc.GetStrides(), input_g_n_c_wis_strides);
-    copy(out_desc.GetStrides(), output_m_k_strides);
+    copy(image_desc.GetStrides(), image_g_n_c_wis_strides);
+    copy(gemm_desc.GetStrides(), gemm_m_k_strides);
     copy(conv_param.conv_filter_strides_, conv_filter_strides);
     copy(conv_param.conv_filter_dilations_, conv_filter_dilations);
     copy(conv_param.input_left_pads_, input_left_pads);
     copy(conv_param.input_right_pads_, input_right_pads);
 
-    Tensor<InputDataType> input(in_desc);
-    Tensor<OutputDataType> host_output(out_desc);
-    Tensor<OutputDataType> device_output(out_desc);
+    Tensor<InputDataType> input =
+        create_input<InputDataType, ConvTensorRearrangeOp>(image_desc, gemm_desc);
+    Tensor<OutputDataType> device_output =
+        create_output<OutputDataType, ConvTensorRearrangeOp>(image_desc, gemm_desc);
+    Tensor<OutputDataType> host_output =
+        create_output<OutputDataType, ConvTensorRearrangeOp>(image_desc, gemm_desc);
 
     std::cout << "input: " << input.mDesc << std::endl;
     std::cout << "output: " << host_output.mDesc << std::endl;
@@ -94,17 +165,21 @@ bool profile_image_to_column_impl(int do_verification,
     // run reference op
     if(do_verification)
     {
-        auto ref_image_to_column = ck::tensor_operation::host::
-            ReferenceImageToColumn<NDimSpatial, InputLayout, InputDataType, OutputDataType>{};
+        auto ref_conv_tensor_rearrange = make_ref_op<NDimSpatial,
+                                                     InputLayout,
+                                                     InputDataType,
+                                                     OutputDataType,
+                                                     ConvTensorRearrangeOp>();
 
-        auto ref_invoker  = ref_image_to_column.MakeInvoker();
-        auto ref_argument = ref_image_to_column.MakeArgument(input,
-                                                             host_output,
-                                                             conv_param.filter_spatial_lengths_,
-                                                             conv_param.conv_filter_strides_,
-                                                             conv_param.conv_filter_dilations_,
-                                                             conv_param.input_left_pads_,
-                                                             conv_param.input_right_pads_);
+        auto ref_invoker = ref_conv_tensor_rearrange.MakeInvoker();
+        auto ref_argument =
+            ref_conv_tensor_rearrange.MakeArgument(input,
+                                                   host_output,
+                                                   conv_param.filter_spatial_lengths_,
+                                                   conv_param.conv_filter_strides_,
+                                                   conv_param.conv_filter_dilations_,
+                                                   conv_param.input_left_pads_,
+                                                   conv_param.input_right_pads_);
 
         // init host output to zero
         host_output.SetZero();
@@ -112,8 +187,11 @@ bool profile_image_to_column_impl(int do_verification,
         ref_invoker.Run(ref_argument);
     }
 
-    using DeviceOp = ck::tensor_operation::device::
-        DeviceImageToColumn<NDimSpatial, InputLayout, InputDataType, OutputDataType>;
+    using DeviceOp = ck::tensor_operation::device::DeviceConvTensorRearrange<NDimSpatial,
+                                                                             InputLayout,
+                                                                             InputDataType,
+                                                                             OutputDataType,
+                                                                             ConvTensorRearrangeOp>;
 
     // get device op instances
     const auto op_ptrs = ck::tensor_operation::device::instance::DeviceOperationInstanceFactory<
@@ -139,8 +217,8 @@ bool profile_image_to_column_impl(int do_verification,
             input_spatial_lengths,
             filter_spatial_lengths,
             output_spatial_lengths,
-            input_g_n_c_wis_strides,
-            output_m_k_strides,
+            image_g_n_c_wis_strides,
+            gemm_m_k_strides,
             conv_filter_strides,
             conv_filter_dilations,
             input_left_pads,

profiler/include/profiler/profile_gemm_impl.hpp

@@ -223,6 +223,12 @@ int profile_gemm_impl(int do_verification,
     {
         std::cout << "Best Perf for datatype = int8";
     }
+#if defined CK_ENABLE_FP8
+    else if constexpr(is_same<CDataType, f8_t>::value)
+    {
+        std::cout << "Best Perf for datatype = fp8";
+    }
+#endif
 
     if constexpr(is_same<ALayout, tensor_layout::gemm::RowMajor>::value)
     {

profiler/include/profiler/profile_grouped_conv_bwd_weight_impl.hpp

@@ -33,7 +33,9 @@ template <ck::index_t NDimSpatial,
           typename OutLayout,
           typename InDataType,
           typename WeiDataType,
-          typename OutDataType>
+          typename OutDataType,
+          typename ComputeTypeA = InDataType,
+          typename ComputeTypeB = ComputeTypeA>
 bool profile_grouped_conv_bwd_weight_impl(int do_verification,
                                           int init_method,
                                           bool do_log,
@@ -120,7 +122,9 @@ bool profile_grouped_conv_bwd_weight_impl(int do_verification,
                                                                               OutDataType,
                                                                               InElementOp,
                                                                               WeiElementOp,
-                                                                              OutElementOp>;
+                                                                              OutElementOp,
+                                                                              ComputeTypeA,
+                                                                              ComputeTypeB>;
 
     // get device op instances
     const auto op_ptrs = ck::tensor_operation::device::instance::DeviceOperationInstanceFactory<

profiler/src/CMakeLists.txt

@@ -28,7 +28,7 @@ set(PROFILER_SOURCES
     profile_contraction_bilinear.cpp
     profile_contraction_scale.cpp
     profile_grouped_conv_bwd_data.cpp
-    profile_image_to_column.cpp
+    profile_conv_tensor_rearrange.cpp
 )
 if(DL_KERNELS)
   list(APPEND PROFILER_SOURCES profile_batched_gemm_multi_d.cpp)
@@ -84,6 +84,7 @@ target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_max_pool_bwd_instanc
 target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_grouped_conv2d_bwd_data_instance)
 target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_grouped_conv3d_bwd_data_instance)
 target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_image_to_column_instance)
+target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_column_to_image_instance)
 if(DL_KERNELS)
   target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_batched_gemm_multi_d_instance)
 endif()

profiler/src/profile_image_to_column.cpp → profiler/src/profile_conv_tensor_rearrange.cpp

 // SPDX-License-Identifier: MIT
-// Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved.
+// Copyright (c) 2023, Advanced Micro Devices, Inc. All rights reserved.
 
 #include <iostream>
 #include <numeric>
 #include <initializer_list>
 #include <cstdlib>
 
-#include "profiler/profile_image_to_column_impl.hpp"
+#include "profiler/profile_conv_tensor_rearrange_impl.hpp"
 #include "profiler_operation_registry.hpp"
 
 namespace {
 
+enum struct RearrangeOp
+{
+    ImageToColumn, // 0
+    ColumnToImage, // 1
+};
+
 enum struct ConvLayout
 {
     NHWC, // 0
@@ -24,8 +30,8 @@ enum struct DataType
     INT8_INT8, // 3
 };
 
-#define OP_NAME "image_to_column"
-#define OP_DESC "Image To Column"
+#define OP_NAME "conv_tensor_rearrange"
+#define OP_DESC "Conv Tensor Rearrange"
 
 static void print_helper_msg()
 {
@@ -41,16 +47,17 @@ static void print_helper_msg()
         << "arg5: initialization (0: no init, 1: integer value, 2: decimal value)\n"
         << "arg6: print tensor value (0: no; 1: yes)\n"
         << "arg7: time kernel (0: no, 1: yes)\n"
+        << "arg8: operation type (0: ImageToColumn, 1: ColumnToImage)\n"
         << ck::utils::conv::get_conv_param_parser_helper_msg() << std::endl;
     // clang-format on
 }
 
 } // namespace
 
-int profile_image_to_column(int argc, char* argv[])
+int profile_conv_tensor_rearrange(int argc, char* argv[])
 {
-    // 8 for control, 1 for num_dim_spatial
-    if(argc < 9)
+    // 9 for control, 1 for num_dim_spatial
+    if(argc < 10)
     {
         print_helper_msg();
         return 1;
@@ -62,16 +69,17 @@ int profile_image_to_column(int argc, char* argv[])
     const int init_method      = std::stoi(argv[5]);
     const bool do_log          = std::stoi(argv[6]);
     const bool time_kernel     = std::stoi(argv[7]);
-    const int num_dim_spatial  = std::stoi(argv[8]);
+    const auto rearrange_op    = static_cast<RearrangeOp>(std::stoi(argv[8]));
+    const int num_dim_spatial  = std::stoi(argv[9]);
 
-    // 8 for control, 1 for num_dim_spatial, 4 for G/N/K/C, and 6 * num_dim_spatial
-    if(argc != 8 + 1 + 4 + 6 * num_dim_spatial)
+    // 9 for control, 1 for num_dim_spatial, 4 for G/N/K/C, and 6 * num_dim_spatial
+    if(argc != 9 + 1 + 4 + 6 * num_dim_spatial)
     {
         print_helper_msg();
         return 1;
     }
 
-    const auto params = ck::utils::conv::parse_conv_param(num_dim_spatial, 9, argv);
+    const auto params = ck::utils::conv::parse_conv_param(num_dim_spatial, 10, argv);
 
     using F32  = float;
     using F16  = ck::half_t;
@@ -79,12 +87,17 @@ int profile_image_to_column(int argc, char* argv[])
     using INT8 = int8_t;
 
     using namespace ck::tensor_layout::convolution;
+    using namespace ck::conv_tensor_rearrange_op;
 
     constexpr auto I1 = ck::Number<1>{};
     constexpr auto I2 = ck::Number<2>{};
     constexpr auto I3 = ck::Number<3>{};
 
-    auto profile = [&](auto num_dim_spatial_tmp, auto in_layout, auto in_type, auto out_type) {
+    auto profile = [&](auto num_dim_spatial_tmp,
+                       auto in_layout,
+                       auto in_type,
+                       auto out_type,
+                       auto rearrange_op_type) {
         constexpr ck::index_t NDimSpatial = num_dim_spatial_tmp.value;
 
         using InLayout = decltype(in_layout);
@@ -92,78 +105,147 @@ int profile_image_to_column(int argc, char* argv[])
         using InDataType  = decltype(in_type);
         using OutDataType = decltype(out_type);
 
+        using Op = decltype(rearrange_op_type);
+
         bool pass = ck::profiler::
-            profile_image_to_column_impl<NDimSpatial, InLayout, InDataType, OutDataType>(
+            profile_conv_tensor_rearrange_impl<NDimSpatial, InLayout, InDataType, OutDataType, Op>(
                 do_verification, init_method, do_log, time_kernel, params);
 
         return pass ? 0 : 1;
     };
 
-    // NHWC
-    if(layout == ConvLayout::NHWC)
+    // Image To Column
+    if(rearrange_op == RearrangeOp::ImageToColumn)
     {
-        if(num_dim_spatial == 1)
+        // NHWC
+        if(layout == ConvLayout::NHWC)
         {
-            if(data_type == DataType::F32_F32)
-            {
-                return profile(I1, GNWC{}, F32{}, F32{});
-            }
-            else if(data_type == DataType::F16_F16)
+            if(num_dim_spatial == 1)
             {
-                return profile(I1, GNWC{}, F16{}, F16{});
+                if(data_type == DataType::F32_F32)
+                {
+                    return profile(I1, GNWC{}, F32{}, F32{}, ImageToColumn{});
+                }
+                else if(data_type == DataType::F16_F16)
+                {
+                    return profile(I1, GNWC{}, F16{}, F16{}, ImageToColumn{});
+                }
+                else if(data_type == DataType::BF16_BF16)
+                {
+                    return profile(I1, GNWC{}, BF16{}, BF16{}, ImageToColumn{});
+                }
+                else if(data_type == DataType::INT8_INT8)
+                {
+                    return profile(I1, GNWC{}, INT8{}, INT8{}, ImageToColumn{});
+                }
             }
-            else if(data_type == DataType::BF16_BF16)
+            else if(num_dim_spatial == 2)
             {
-                return profile(I1, GNWC{}, BF16{}, BF16{});
+                if(data_type == DataType::F32_F32)
+                {
+                    return profile(I2, GNHWC{}, F32{}, F32{}, ImageToColumn{});
+                }
+                else if(data_type == DataType::F16_F16)
+                {
+                    return profile(I2, GNHWC{}, F16{}, F16{}, ImageToColumn{});
+                }
+                else if(data_type == DataType::BF16_BF16)
+                {
+                    return profile(I2, GNHWC{}, BF16{}, BF16{}, ImageToColumn{});
+                }
+                else if(data_type == DataType::INT8_INT8)
+                {
+                    return profile(I2, GNHWC{}, INT8{}, INT8{}, ImageToColumn{});
+                }
             }
-            else if(data_type == DataType::INT8_INT8)
+            else if(num_dim_spatial == 3)
             {
-                return profile(I1, GNWC{}, INT8{}, INT8{});
+                if(data_type == DataType::F32_F32)
+                {
+                    return profile(I3, GNDHWC{}, F32{}, F32{}, ImageToColumn{});
+                }
+                else if(data_type == DataType::F16_F16)
+                {
+                    return profile(I3, GNDHWC{}, F16{}, F16{}, ImageToColumn{});
+                }
+                else if(data_type == DataType::BF16_BF16)
+                {
+                    return profile(I3, GNDHWC{}, BF16{}, BF16{}, ImageToColumn{});
+                }
+                else if(data_type == DataType::INT8_INT8)
+                {
+                    return profile(I3, GNDHWC{}, INT8{}, INT8{}, ImageToColumn{});
+                }
             }
         }
-        else if(num_dim_spatial == 2)
-        {
-            if(data_type == DataType::F32_F32)
-            {
-                return profile(I2, GNHWC{}, F32{}, F32{});
-            }
-            else if(data_type == DataType::F16_F16)
-            {
-                return profile(I2, GNHWC{}, F16{}, F16{});
-            }
-            else if(data_type == DataType::BF16_BF16)
-            {
-                return profile(I2, GNHWC{}, BF16{}, BF16{});
-            }
-            else if(data_type == DataType::INT8_INT8)
-            {
-                return profile(I2, GNHWC{}, INT8{}, INT8{});
-            }
-        }
-        else if(num_dim_spatial == 3)
+    }
+    else if(rearrange_op == RearrangeOp::ColumnToImage)
+    {
+        // NHWC
+        if(layout == ConvLayout::NHWC)
         {
-            if(data_type == DataType::F32_F32)
+            if(num_dim_spatial == 1)
             {
-                return profile(I3, GNDHWC{}, F32{}, F32{});
+                if(data_type == DataType::F32_F32)
+                {
+                    return profile(I1, GNWC{}, F32{}, F32{}, ColumnToImage{});
+                }
+                else if(data_type == DataType::F16_F16)
+                {
+                    return profile(I1, GNWC{}, F16{}, F16{}, ColumnToImage{});
+                }
+                else if(data_type == DataType::BF16_BF16)
+                {
+                    return profile(I1, GNWC{}, BF16{}, BF16{}, ColumnToImage{});
+                }
+                else if(data_type == DataType::INT8_INT8)
+                {
+                    return profile(I1, GNWC{}, INT8{}, INT8{}, ColumnToImage{});
+                }
             }
-            else if(data_type == DataType::F16_F16)
+            else if(num_dim_spatial == 2)
             {
-                return profile(I3, GNDHWC{}, F16{}, F16{});
+                if(data_type == DataType::F32_F32)
+                {
+                    return profile(I2, GNHWC{}, F32{}, F32{}, ColumnToImage{});
+                }
+                else if(data_type == DataType::F16_F16)
+                {
+                    return profile(I2, GNHWC{}, F16{}, F16{}, ColumnToImage{});
+                }
+                else if(data_type == DataType::BF16_BF16)
+                {
+                    return profile(I2, GNHWC{}, BF16{}, BF16{}, ColumnToImage{});
+                }
+                else if(data_type == DataType::INT8_INT8)
+                {
+                    return profile(I2, GNHWC{}, INT8{}, INT8{}, ColumnToImage{});
+                }
             }
-            else if(data_type == DataType::BF16_BF16)
+            else if(num_dim_spatial == 3)
             {
-                return profile(I3, GNDHWC{}, BF16{}, BF16{});
-            }
-            else if(data_type == DataType::INT8_INT8)
-            {
-                return profile(I3, GNDHWC{}, INT8{}, INT8{});
+                if(data_type == DataType::F32_F32)
+                {
+                    return profile(I3, GNDHWC{}, F32{}, F32{}, ColumnToImage{});
+                }
+                else if(data_type == DataType::F16_F16)
+                {
+                    return profile(I3, GNDHWC{}, F16{}, F16{}, ColumnToImage{});
+                }
+                else if(data_type == DataType::BF16_BF16)
+                {
+                    return profile(I3, GNDHWC{}, BF16{}, BF16{}, ColumnToImage{});
+                }
+                else if(data_type == DataType::INT8_INT8)
+                {
+                    return profile(I3, GNDHWC{}, INT8{}, INT8{}, ColumnToImage{});
+                }
             }
         }
     }
 
     std::cout << "this data_type & layout is not implemented" << std::endl;
-
     return 1;
 }
 
-REGISTER_PROFILER_OPERATION(OP_NAME, OP_DESC, profile_image_to_column);
+REGISTER_PROFILER_OPERATION(OP_NAME, OP_DESC, profile_conv_tensor_rearrange);

profiler/src/profile_gemm.cpp

@@ -23,6 +23,7 @@ enum struct GemmDataType
     F16_F16_F16,    // 1
     BF16_BF16_BF16, // 2
     INT8_INT8_INT8, // 3
+    F8_F8_F8,       // 4
 };
 
 #define OP_NAME "gemm"
@@ -31,7 +32,7 @@ enum struct GemmDataType
 static void print_helper_msg()
 {
     std::cout << "arg1: tensor operation (" OP_NAME ": " OP_DESC ")\n"
-              << "arg2: data type (0: fp32; 1: fp16; 2: bf16; 3: int8)\n"
+              << "arg2: data type (0: fp32; 1: fp16; 2: bf16; 3: int8; 4: fp8)\n"
               << "arg3: matrix layout (0: A[m, k] * B[k, n] = C[m, n];\n"
               << "                     1: A[m, k] * B[n, k] = C[m, n];\n"
               << "                     2: A[k, m] * B[k, n] = C[m, n];\n"
@@ -76,6 +77,9 @@ int profile_gemm(int argc, char* argv[])
     using INT8  = int8_t;
     using INT32 = int32_t;
 #endif
+#ifdef CK_ENABLE_FP8
+    using F8 = ck::f8_t;
+#endif
 
     using Row = ck::tensor_layout::gemm::RowMajor;
     using Col = ck::tensor_layout::gemm::ColumnMajor;
@@ -194,6 +198,24 @@ int profile_gemm(int argc, char* argv[])
     {
         return profile(Col{}, Col{}, Row{}, INT8{}, INT8{}, INT32{}, INT8{});
     }
+#endif
+#ifdef CK_ENABLE_FP8
+    else if(data_type == GemmDataType::F8_F8_F8 && layout == GemmMatrixLayout::MK_KN_MN)
+    {
+        return profile(Row{}, Row{}, Row{}, F8{}, F8{}, F32{}, F8{});
+    }
+    else if(data_type == GemmDataType::F8_F8_F8 && layout == GemmMatrixLayout::MK_NK_MN)
+    {
+        return profile(Row{}, Col{}, Row{}, F8{}, F8{}, F32{}, F8{});
+    }
+    else if(data_type == GemmDataType::F8_F8_F8 && layout == GemmMatrixLayout::KM_KN_MN)
+    {
+        return profile(Col{}, Row{}, Row{}, F8{}, F8{}, F32{}, F8{});
+    }
+    else if(data_type == GemmDataType::F8_F8_F8 && layout == GemmMatrixLayout::KM_NK_MN)
+    {
+        return profile(Col{}, Col{}, Row{}, F8{}, F8{}, F32{}, F8{});
+    }
 #endif
     else
     {

profiler/src/profile_grouped_conv_bwd_weight.cpp

@@ -20,9 +20,10 @@ enum struct ConvLayout
 
 enum struct ConvDataType
 {
-    F32_F32_F32,   // 0
-    F16_F16_F16,   // 1
-    BF16_F32_BF16, // 2
+    F32_F32_F32,       // 0
+    F16_F16_F16,       // 1
+    BF16_F32_BF16,     // 2
+    F16_F16_F16_BF8_F8 // 3
 };
 
 #define OP_NAME "grouped_conv_bwd_weight"
@@ -33,7 +34,8 @@ static void print_helper_msg()
     std::cout << "arg1: tensor operation (" OP_NAME ": " OP_DESC ")\n"
               << "arg2: data type (0: Input fp32, Weight fp32, Output fp32\n"
               << "                 1: Input fp16, Weight fp16, Output fp16\n"
-              << "                 2: Input bf16, Weight fp32, Output bf16)\n"
+              << "                 2: Input bf16, Weight fp32, Output bf16\n"
+              << "                 3: Input fp16, Weight fp16, Output fp16, Gemm bf8@fp8)\n"
               << "arg3: tensor layout (0: Input[G, N, C, Hi, Wi], Weight[G, K, C, Y, X], Output[G, "
                  "N, K, Ho, Wo]\n"
               << "                     1: Input[G, N, Hi, Wi, C], Weight[G, K, Y, X, C], Output[G, "
@@ -82,6 +84,12 @@ int profile_grouped_conv_bwd_weight(int argc, char* argv[])
     using F32  = float;
     using F16  = ck::half_t;
     using BF16 = ck::bhalf_t;
+#ifdef CK_ENABLE_FP8
+    using F8 = ck::f8_t;
+#endif
+#ifdef CK_ENABLE_BF8
+    using BF8 = ck::bf8_t;
+#endif
 
     using namespace ck::tensor_layout::convolution;
 
@@ -95,7 +103,9 @@ int profile_grouped_conv_bwd_weight(int argc, char* argv[])
                        auto out_layout,
                        auto in_type,
                        auto wei_type,
-                       auto out_type) {
+                       auto out_type,
+                       auto compute_type_a,
+                       auto compute_type_b) {
         constexpr ck::index_t NDimSpatial = num_dim_spatial_tmp.value;
 
         using InLayout  = decltype(in_layout);
@@ -106,13 +116,18 @@ int profile_grouped_conv_bwd_weight(int argc, char* argv[])
         using WeiDataType = decltype(wei_type);
         using OutDataType = decltype(out_type);
 
+        using ComputeTypeA = decltype(compute_type_a);
+        using ComputeTypeB = decltype(compute_type_b);
+
         bool pass = ck::profiler::profile_grouped_conv_bwd_weight_impl<NDimSpatial,
                                                                        InLayout,
                                                                        WeiLayout,
                                                                        OutLayout,
                                                                        InDataType,
                                                                        WeiDataType,
-                                                                       OutDataType>(
+                                                                       OutDataType,
+                                                                       ComputeTypeA,
+                                                                       ComputeTypeB>(
             do_verification, init_method, do_log, time_kernel, params, split_k);
 
         return pass ? 0 : 1;
@@ -122,80 +137,84 @@ int profile_grouped_conv_bwd_weight(int argc, char* argv[])
     {
         if(data_type == ConvDataType::F32_F32_F32)
         {
-            return profile(I1, GNWC{}, GKXC{}, GNWK{}, F32{}, F32{}, F32{});
+            return profile(I1, GNWC{}, GKXC{}, GNWK{}, F32{}, F32{}, F32{}, F32{}, F32{});
         }
         else if(data_type == ConvDataType::F16_F16_F16)
         {
-            return profile(I1, GNWC{}, GKXC{}, GNWK{}, F16{}, F16{}, F16{});
+            return profile(I1, GNWC{}, GKXC{}, GNWK{}, F16{}, F16{}, F16{}, F16{}, F16{});
         }
         else if(data_type == ConvDataType::BF16_F32_BF16)
         {
             // fp32 atomic add is used for weight tensor in bf16 kernel
-            return profile(I1, GNWC{}, GKXC{}, GNWK{}, BF16{}, F32{}, BF16{});
+            return profile(I1, GNWC{}, GKXC{}, GNWK{}, BF16{}, F32{}, BF16{}, BF16{}, BF16{});
         }
     }
     else if(num_dim_spatial == 2 && layout == ConvLayout::GNHWC_GKYXC_GNHWK)
     {
         if(data_type == ConvDataType::F32_F32_F32)
         {
-            return profile(I2, GNHWC{}, GKYXC{}, GNHWK{}, F32{}, F32{}, F32{});
+            return profile(I2, GNHWC{}, GKYXC{}, GNHWK{}, F32{}, F32{}, F32{}, F32{}, F32{});
         }
         else if(data_type == ConvDataType::F16_F16_F16)
         {
-            return profile(I2, GNHWC{}, GKYXC{}, GNHWK{}, F16{}, F16{}, F16{});
+            return profile(I2, GNHWC{}, GKYXC{}, GNHWK{}, F16{}, F16{}, F16{}, F16{}, F16{});
         }
         else if(data_type == ConvDataType::BF16_F32_BF16)
         {
             // fp32 atomic add is used for weight tensor in bf16 kernel
-            return profile(I2, GNHWC{}, GKYXC{}, GNHWK{}, BF16{}, F32{}, BF16{});
+            return profile(I2, GNHWC{}, GKYXC{}, GNHWK{}, BF16{}, F32{}, BF16{}, BF16{}, BF16{});
         }
     }
     else if(num_dim_spatial == 2 && layout == ConvLayout::NHWGC_GKYXC_NHWGK)
     {
         if(data_type == ConvDataType::F32_F32_F32)
         {
-            return profile(I2, NHWGC{}, GKYXC{}, NHWGK{}, F32{}, F32{}, F32{});
+            return profile(I2, NHWGC{}, GKYXC{}, NHWGK{}, F32{}, F32{}, F32{}, F32{}, F32{});
         }
         else if(data_type == ConvDataType::F16_F16_F16)
         {
-            return profile(I2, NHWGC{}, GKYXC{}, NHWGK{}, F16{}, F16{}, F16{});
+            return profile(I2, NHWGC{}, GKYXC{}, NHWGK{}, F16{}, F16{}, F16{}, F16{}, F16{});
         }
         else if(data_type == ConvDataType::BF16_F32_BF16)
         {
             // fp32 atomic add is used for weight tensor in bf16 kernel
-            return profile(I2, NHWGC{}, GKYXC{}, NHWGK{}, BF16{}, F32{}, BF16{});
+            return profile(I2, NHWGC{}, GKYXC{}, NHWGK{}, BF16{}, F32{}, BF16{}, BF16{}, BF16{});
         }
     }
     else if(num_dim_spatial == 3 && layout == ConvLayout::GNHWC_GKYXC_GNHWK)
     {
         if(data_type == ConvDataType::F32_F32_F32)
         {
-            return profile(I3, GNDHWC{}, GKZYXC{}, GNDHWK{}, F32{}, F32{}, F32{});
+            return profile(I3, GNDHWC{}, GKZYXC{}, GNDHWK{}, F32{}, F32{}, F32{}, F32{}, F32{});
         }
         else if(data_type == ConvDataType::F16_F16_F16)
         {
-            return profile(I3, GNDHWC{}, GKZYXC{}, GNDHWK{}, F16{}, F16{}, F16{});
+            return profile(I3, GNDHWC{}, GKZYXC{}, GNDHWK{}, F16{}, F16{}, F16{}, F16{}, F16{});
         }
         else if(data_type == ConvDataType::BF16_F32_BF16)
         {
             // fp32 atomic add is used for weight tensor in bf16 kernel
-            return profile(I3, GNDHWC{}, GKZYXC{}, GNDHWK{}, BF16{}, F32{}, BF16{});
+            return profile(I3, GNDHWC{}, GKZYXC{}, GNDHWK{}, BF16{}, F32{}, BF16{}, BF16{}, BF16{});
         }
     }
     else if(num_dim_spatial == 3 && layout == ConvLayout::NHWGC_GKYXC_NHWGK)
     {
         if(data_type == ConvDataType::F32_F32_F32)
         {
-            return profile(I3, NDHWGC{}, GKZYXC{}, NDHWGK{}, F32{}, F32{}, F32{});
+            return profile(I3, NDHWGC{}, GKZYXC{}, NDHWGK{}, F32{}, F32{}, F32{}, F32{}, F32{});
         }
         else if(data_type == ConvDataType::F16_F16_F16)
         {
-            return profile(I3, NDHWGC{}, GKZYXC{}, NDHWGK{}, F16{}, F16{}, F16{});
+            return profile(I3, NDHWGC{}, GKZYXC{}, NDHWGK{}, F16{}, F16{}, F16{}, F16{}, F16{});
         }
         else if(data_type == ConvDataType::BF16_F32_BF16)
         {
             // fp32 atomic add is used for weight tensor in bf16 kernel
-            return profile(I3, NDHWGC{}, GKZYXC{}, NDHWGK{}, BF16{}, F32{}, BF16{});
+            return profile(I3, NDHWGC{}, GKZYXC{}, NDHWGK{}, BF16{}, F32{}, BF16{}, BF16{}, BF16{});
+        }
+        else if(data_type == ConvDataType::F16_F16_F16_BF8_F8)
+        {
+            return profile(I3, NDHWGC{}, GKZYXC{}, NDHWGK{}, F16{}, F16{}, F16{}, BF8{}, F8{});
         }
     }
 

test/CMakeLists.txt

@@ -32,7 +32,7 @@ function(add_test_executable TEST_NAME)
                 set(test 0)
                 break()
             elseif((source MATCHES "fp8" OR source MATCHES "fp32" OR source MATCHES "fp64" OR source MATCHES "bf16" OR source MATCHES "int8" OR source MATCHES "fp16" OR
-                source MATCHES "_f8" OR source MATCHES "_f32" OR source MATCHES "_f64" OR source MATCHES "_i8" OR source MATCHES "_f16" OR source MATCHES "_b16") AND 
+                source MATCHES "_f8" OR source MATCHES "_f32" OR source MATCHES "_f64" OR source MATCHES "_i8" OR source MATCHES "_f16" OR source MATCHES "_b16") AND
                 NOT(source MATCHES type OR source MATCHES type1))
                     #if filename contains a type which doesn't match any selected type, mark it for removal
                     set(test 1)
@@ -61,7 +61,7 @@ function(add_test_executable TEST_NAME)
         set(result 0)
     endif()
     #message("add_test returns ${result}")
-    return(PROPAGATE result)
+    set(result ${result} PARENT_SCOPE)
 endfunction(add_test_executable TEST_NAME)
 
 include(GoogleTest)
@@ -91,7 +91,7 @@ function(add_gtest_executable TEST_NAME)
                 set(test 0)
                 break()
             elseif((source MATCHES "fp8" OR source MATCHES "fp32" OR source MATCHES "fp64" OR source MATCHES "bf16" OR source MATCHES "int8" OR source MATCHES "fp16" OR
-                source MATCHES "_f8" OR source MATCHES "_f32" OR source MATCHES "_f64" OR source MATCHES "_i8" OR source MATCHES "_f16" OR source MATCHES "_b16") AND 
+                source MATCHES "_f8" OR source MATCHES "_f32" OR source MATCHES "_f64" OR source MATCHES "_i8" OR source MATCHES "_f16" OR source MATCHES "_b16") AND
                 NOT(source MATCHES type OR source MATCHES type1))
                     #if filename contains a type which doesn't match any selected type, mark it for removal
                     set(test 1)
@@ -123,7 +123,7 @@ function(add_gtest_executable TEST_NAME)
         set(result 0)
     endif()
     #message("add_gtest returns ${result}")
-    return(PROPAGATE result)
+    set(result ${result} PARENT_SCOPE)
 endfunction(add_gtest_executable TEST_NAME)
 
 add_subdirectory(magic_number_division)
@@ -155,7 +155,7 @@ add_subdirectory(contraction)
 add_subdirectory(pool)
 add_subdirectory(batched_gemm_multi_d)
 add_subdirectory(grouped_convnd_bwd_data)
-add_subdirectory(image_to_column)
+add_subdirectory(conv_tensor_rearrange)
 if(GPU_TARGETS MATCHES "gfx11")
     add_subdirectory(wmma_op)
 endif()

test/batchnorm/batchnorm_bwd_rank_4.cpp

@@ -70,10 +70,23 @@ class TestBatchNormBwdRank4 : public ::testing::Test
     }
 };
 
-using KernelTypes = ::testing::Types<std::tuple<F16, F32, F32, F32, F16, F32, F32>,
-                                     std::tuple<F32, F32, F32, F32, F32, F32, F32>,
-                                     std::tuple<BF16, F32, F32, F32, BF16, F32, F32>,
-                                     std::tuple<F64, F64, F64, F64, F64, F64, F64>>;
+using KernelTypes = ::testing::Types<
+#ifdef CK_ENABLE_FP16
+    std::tuple<F16, F32, F32, F32, F16, F32, F32>
+#endif
+#ifdef CK_ENABLE_FP32
+    ,
+    std::tuple<F32, F32, F32, F32, F32, F32, F32>
+#endif
+#ifdef CK_ENABLE_BF16
+    ,
+    std::tuple<BF16, F32, F32, F32, BF16, F32, F32>
+#endif
+#ifdef CK_ENABLE_FP64
+    ,
+    std::tuple<F64, F64, F64, F64, F64, F64, F64>
+#endif
+    >;
 
 TYPED_TEST_SUITE(TestBatchNormBwdRank4, KernelTypes);