update instance

include/ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp

@@ -58,7 +58,16 @@ template <
     typename CBlockTransferClusterLengths_MBlock_MXdlPerWave_MWaveMPerXdl_NBlock_NXdlPerWave_NWaveNPerXdl,
     index_t CBlockTransferScalarPerVector_NWaveNPerXdl>
 struct DeviceConv2dFwdXdl_C_Shuffle_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K
-    : public DeviceConvFwd<InElementwiseOperation, WeiElementwiseOperation, OutElementwiseOperation>
+    : public DeviceConvFwd<2,
+                           ck::tensor_layout::convolution::NHWC,
+                           ck::tensor_layout::convolution::KYXC,
+                           ck::tensor_layout::convolution::NHWK,
+                           InDataType,
+                           WeiDataType,
+                           OutDataType,
+                           InElementwiseOperation,
+                           WeiElementwiseOperation,
+                           OutElementwiseOperation>
 {
     using DeviceOp = DeviceConv2dFwdXdl_C_Shuffle_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K;
 

include/ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk.hpp

@@ -55,7 +55,16 @@ template <typename InDataType,
           ck::index_t CThreadTransferSrcDstVectorDim,
           ck::index_t CThreadTransferDstScalarPerVector>
 struct DeviceConv2dFwdXdl_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K
-    : public DeviceConvFwd<InElementwiseOperation, WeiElementwiseOperation, OutElementwiseOperation>
+    : public DeviceConvFwd<2,
+                           ck::tensor_layout::convolution::NHWC,
+                           ck::tensor_layout::convolution::KYXC,
+                           ck::tensor_layout::convolution::NHWK,
+                           InDataType,
+                           WeiDataType,
+                           OutDataType,
+                           InElementwiseOperation,
+                           WeiElementwiseOperation,
+                           OutElementwiseOperation>
 {
     using DeviceOp = DeviceConv2dFwdXdl_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K;
 

library/include/ck/library/tensor_operation_instance/gpu/convolution_forward.hpp

+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved.
+
+#pragma once
+
+#include <cstdlib>
+
+#include "ck/ck.hpp"
+#include "ck/tensor_operation/gpu/device/tensor_layout.hpp"
+#include "ck/tensor_operation/gpu/device/device_conv_fwd.hpp"
+#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp"
+
+#include "ck/library/tensor_operation_instance/device_operation_instance_factory.hpp"
+
+namespace ck {
+namespace tensor_operation {
+namespace device {
+namespace instance {
+
+void add_device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk_f16_instances(
+    std::vector<std::unique_ptr<
+        DeviceConvFwd<2, NHWC, KYXC, NHWK, F16, F16, F16, PassThrough, PassThrough, PassThrough>>>&
+        instances);
+
+void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instances(
+    std::vector<std::unique_ptr<DeviceConvFwd<2,
+                                              NHWC,
+                                              KYXC,
+                                              NHWK,
+                                              BF16,
+                                              BF16,
+                                              BF16,
+                                              PassThrough,
+                                              PassThrough,
+                                              PassThrough>>>& instances);
+
+void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instances(
+    std::vector<std::unique_ptr<
+        DeviceConvFwd<2, NHWC, KYXC, NHWK, F16, F16, F16, PassThrough, PassThrough, PassThrough>>>&
+        instances);
+
+void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instances(
+    std::vector<std::unique_ptr<
+        DeviceConvFwd<2, NHWC, KYXC, NHWK, F32, F32, F32, PassThrough, PassThrough, PassThrough>>>&
+        instances);
+
+void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instances(
+    std::vector<std::unique_ptr<DeviceConvFwd<2,
+                                              NHWC,
+                                              KYXC,
+                                              NHWK,
+                                              int8_t,
+                                              int8_t,
+                                              int8_t,
+                                              PassThrough,
+                                              PassThrough,
+                                              PassThrough>>>& instances);
+
+void add_device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instances(
+    std::vector<std::unique_ptr<DeviceConvFwd<2,
+                                              NHWC,
+                                              KYXC,
+                                              NHWK,
+                                              BF16,
+                                              BF16,
+                                              BF16,
+                                              PassThrough,
+                                              PassThrough,
+                                              PassThrough>>>& instances);
+
+void add_device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instances(
+    std::vector<std::unique_ptr<
+        DeviceConvFwd<2, NHWC, KYXC, NHWK, F16, F16, F16, PassThrough, PassThrough, PassThrough>>>&
+        instances);
+
+void add_device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instances(
+    std::vector<std::unique_ptr<
+        DeviceConvFwd<2, NHWC, KYXC, NHWK, F16, F16, F16, PassThrough, PassThrough, PassThrough>>>&
+        instances);
+
+void add_device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instances(
+    std::vector<std::unique_ptr<DeviceConvFwd<2,
+                                              NHWC,
+                                              KYXC,
+                                              NHWK,
+                                              int8_t,
+                                              int8_t,
+                                              int8_t,
+                                              PassThrough,
+                                              PassThrough,
+                                              PassThrough>>>& instances);
+
+template <ck::index_t NumDimSpatial,
+          typename InLayout,
+          typename WeiLayout,
+          typename OutLayout,
+          typename InDataType,
+          typename WeiDataType,
+          typename OutDataType,
+          struct DeviceOperationInstanceFactory<ck::tensor_operation::device::DeviceConvFwd<
+              NumDimSpatial,
+              InLayout,
+              WeiLayout,
+              OutLayout,
+              InDataType,
+              WeiDataType,
+              OutDataType,
+              ck::tensor_operation::element_wise::PassThrough,
+              ck::tensor_operation::element_wise::PassThrough,
+              ck::tensor_operation::element_wise::PassThrough>>
+{
+    using DeviceOp = DeviceConvFwd<NumDimSpatial,
+                                   InLayout,
+                                   WeiLayout,
+                                   OutLayout,
+                                   InDataType,
+                                   WeiDataType,
+                                   OutDataType,
+                                   ck::tensor_operation::element_wise::PassThrough,
+                                   ck::tensor_operation::element_wise::PassThrough,
+                                   ck::tensor_operation::element_wise::PassThrough>;
+
+    static auto GetInstances()
+    {
+        std::vector<std::unique_ptr<DeviceOp>> op_ptrs;
+
+        if constexpr(NumDimSpatial == 2 && is_same_v<InLayout, NHWC> &&
+                     is_same_v<WeiLayout, KYXC> && is_same_v<OutLayout, NHWK>)
+        {
+            if constexpr(is_same_v<InDataType, float> && is_same_v<WeiDataType, float> &&
+                         is_same_v<OutDataType, float>)
+            {
+                add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instances(op_ptrs);
+                add_device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instances(op_ptrs);
+            }
+            else if constexpr(is_same_v<InDataType, half_t> && is_same_v<WeiDataType, half_t> &&
+                              is_same_v<OutDataType, half_t>)
+            {
+                add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instances(op_ptrs);
+                add_device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instances(op_ptrs);
+            }
+            else if constexpr(is_same_v<InDataType, ck::bhalf_t> &&
+                              is_same_v<WeiDataType, ck::bhalf_t> &&
+                              is_same_v<OutDataType, ck::bhalf_t>)
+            {
+                add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instances(op_ptrs);
+                add_device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instances(op_ptrs);
+            }
+            else if constexpr(is_same_v<InDataType, int8_t> && is_same_v<WeiDataType, int8_t> &&
+                              is_same_v<OutDataType, int8_t>)
+            {
+                add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instances(op_ptrs);
+                add_device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instances(op_ptrs);
+            }
+        }
+
+        return op_ptrs;
+    }
+};
+
+} // namespace instance
+} // namespace device
+} // namespace tensor_operation
+} // namespace ck

library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk_f16_instance.cpp

@@ -20,6 +20,10 @@ using F32 = float;
 template <ck::index_t... Is>
 using S = ck::Sequence<Is...>;
 
+using NHWC = ck::tensor_layout::convolution::NHWC;
+using KYXC = ck::tensor_layout::convolution::KYXC;
+using NHWK = ck::tensor_layout::convolution::NHWK;
+
 using PassThrough = ck::tensor_operation::element_wise::PassThrough;
 
 static constexpr auto ConvFwdDefault =
@@ -131,7 +135,9 @@ using device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk_odd_c_f16_instances = std::
     >;
 
 void add_device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk_f16_instances(
-    std::vector<DeviceConvFwdPtr<PassThrough, PassThrough, PassThrough>>& instances)
+    std::vector<std::unique_ptr<
+        DeviceConvFwd<2, NHWC, KYXC, NHWK, F16, F16, F16, PassThrough, PassThrough, PassThrough>>>&
+        instances)
 {
     add_device_operation_instances(instances,
                                    device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk_f16_instances{});

library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instance.cpp

@@ -20,6 +20,10 @@ using F32  = float;
 template <ck::index_t... Is>
 using S = ck::Sequence<Is...>;
 
+using NHWC = ck::tensor_layout::convolution::NHWC;
+using KYXC = ck::tensor_layout::convolution::KYXC;
+using NHWK = ck::tensor_layout::convolution::NHWK;
+
 using PassThrough = ck::tensor_operation::element_wise::PassThrough;
 
 static constexpr auto ConvFwdDefault =
@@ -99,7 +103,16 @@ using device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_1x1_s1_p0_bf16_instances = std::tuple
     >;
 
 void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instances(
-    std::vector<DeviceConvFwdPtr<PassThrough, PassThrough, PassThrough>>& instances)
+    std::vector<std::unique_ptr<DeviceConvFwd<2,
+                                              NHWC,
+                                              KYXC,
+                                              NHWK,
+                                              BF16,
+                                              BF16,
+                                              BF16,
+                                              PassThrough,
+                                              PassThrough,
+                                              PassThrough>>>& instances)
 {
     add_device_operation_instances(instances,
                                    device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instances{});

library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instance.cpp

@@ -20,6 +20,10 @@ using F32 = float;
 template <ck::index_t... Is>
 using S = ck::Sequence<Is...>;
 
+using NHWC = ck::tensor_layout::convolution::NHWC;
+using KYXC = ck::tensor_layout::convolution::KYXC;
+using NHWK = ck::tensor_layout::convolution::NHWK;
+
 using PassThrough = ck::tensor_operation::element_wise::PassThrough;
 
 static constexpr auto ConvFwdDefault =
@@ -99,7 +103,9 @@ using device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_1x1_s1_p0_f16_instances = std::tuple<
     >;
 
 void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instances(
-    std::vector<DeviceConvFwdPtr<PassThrough, PassThrough, PassThrough>>& instances)
+    std::vector<std::unique_ptr<
+        DeviceConvFwd<2, NHWC, KYXC, NHWK, F16, F16, F16, PassThrough, PassThrough, PassThrough>>>&
+        instances)
 {
     add_device_operation_instances(instances, device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instances{});
     add_device_operation_instances(instances,

library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instance.cpp

@@ -19,6 +19,10 @@ using F32 = float;
 template <ck::index_t... Is>
 using S = ck::Sequence<Is...>;
 
+using NHWC = ck::tensor_layout::convolution::NHWC;
+using KYXC = ck::tensor_layout::convolution::KYXC;
+using NHWK = ck::tensor_layout::convolution::NHWK;
+
 using PassThrough = ck::tensor_operation::element_wise::PassThrough;
 
 static constexpr auto ConvFwdDefault =
@@ -98,7 +102,9 @@ using device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_1x1_s1_p0_f32_instances = std::tuple<
     >;
 
 void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instances(
-    std::vector<DeviceConvFwdPtr<PassThrough, PassThrough, PassThrough>>& instances)
+    std::vector<std::unique_ptr<
+        DeviceConvFwd<2, NHWC, KYXC, NHWK, F32, F32, F32, PassThrough, PassThrough, PassThrough>>>&
+        instances)
 {
     add_device_operation_instances(instances, device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instances{});
     add_device_operation_instances(instances,

library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instance.cpp

@@ -14,11 +14,13 @@ namespace tensor_operation {
 namespace device {
 namespace instance {
 
-using F32 = float;
-
 template <ck::index_t... Is>
 using S = ck::Sequence<Is...>;
 
+using NHWC = ck::tensor_layout::convolution::NHWC;
+using KYXC = ck::tensor_layout::convolution::KYXC;
+using NHWK = ck::tensor_layout::convolution::NHWK;
+
 using PassThrough = ck::tensor_operation::element_wise::PassThrough;
 
 static constexpr auto ConvFwdDefault =
@@ -98,7 +100,16 @@ using device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_1x1_s1_p0_int8_instances = std::tuple
     >;
 
 void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instances(
-    std::vector<DeviceConvFwdPtr<PassThrough, PassThrough, PassThrough>>& instances)
+    std::vector<std::unique_ptr<DeviceConvFwd<2,
+                                              NHWC,
+                                              KYXC,
+                                              NHWK,
+                                              int8_t,
+                                              int8_t,
+                                              int8_t,
+                                              PassThrough,
+                                              PassThrough,
+                                              PassThrough>>>& instances)
 {
     add_device_operation_instances(instances,
                                    device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instances{});

library/src/tensor_operation_instance/gpu/conv2d_fwd/device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instance.cpp

@@ -20,6 +20,10 @@ using F32  = float;
 template <ck::index_t... Is>
 using S = ck::Sequence<Is...>;
 
+using NHWC = ck::tensor_layout::convolution::NHWC;
+using KYXC = ck::tensor_layout::convolution::KYXC;
+using NHWK = ck::tensor_layout::convolution::NHWK;
+
 using PassThrough = ck::tensor_operation::element_wise::PassThrough;
 
 static constexpr auto ConvFwdDefault =
@@ -102,7 +106,16 @@ using device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_1x1_s1_p0_bf16_instances =
         >;
 
 void add_device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instances(
-    std::vector<DeviceConvFwdPtr<PassThrough, PassThrough, PassThrough>>& instances)
+    std::vector<std::unique_ptr<DeviceConvFwd<2,
+                                              NHWC,
+                                              KYXC,
+                                              NHWK,
+                                              BF16,
+                                              BF16,
+                                              BF16,
+                                              PassThrough,
+                                              PassThrough,
+                                              PassThrough>>>& instances)
 {
     add_device_operation_instances(instances,
                                    device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instances{});

library/src/tensor_operation_instance/gpu/conv2d_fwd/device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instance.cpp

@@ -20,6 +20,10 @@ using F32 = float;
 template <ck::index_t... Is>
 using S = ck::Sequence<Is...>;
 
+using NHWC = ck::tensor_layout::convolution::NHWC;
+using KYXC = ck::tensor_layout::convolution::KYXC;
+using NHWK = ck::tensor_layout::convolution::NHWK;
+
 using PassThrough = ck::tensor_operation::element_wise::PassThrough;
 
 static constexpr auto ConvFwdDefault =
@@ -102,7 +106,9 @@ using device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_1x1_s1_p0_f16_instances =
         >;
 
 void add_device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instances(
-    std::vector<DeviceConvFwdPtr<PassThrough, PassThrough, PassThrough>>& instances)
+    std::vector<std::unique_ptr<
+        DeviceConvFwd<2, NHWC, KYXC, NHWK, F16, F16, F16, PassThrough, PassThrough, PassThrough>>>&
+        instances)
 {
     add_device_operation_instances(instances, device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instances{});
     add_device_operation_instances(instances,

library/src/tensor_operation_instance/gpu/conv2d_fwd/device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instance.cpp

@@ -19,6 +19,10 @@ using F32 = float;
 template <ck::index_t... Is>
 using S = ck::Sequence<Is...>;
 
+using NHWC = ck::tensor_layout::convolution::NHWC;
+using KYXC = ck::tensor_layout::convolution::KYXC;
+using NHWK = ck::tensor_layout::convolution::NHWK;
+
 using PassThrough = ck::tensor_operation::element_wise::PassThrough;
 
 static constexpr auto ConvFwdDefault =
@@ -101,7 +105,9 @@ using device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_1x1_s1_p0_f32_instances =
         >;
 
 void add_device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instances(
-    std::vector<DeviceConvFwdPtr<PassThrough, PassThrough, PassThrough>>& instances)
+    std::vector<std::unique_ptr<
+        DeviceConvFwd<2, NHWC, KYXC, NHWK, F16, F16, F16, PassThrough, PassThrough, PassThrough>>>&
+        instances)
 {
     add_device_operation_instances(instances, device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instances{});
     add_device_operation_instances(instances,

library/src/tensor_operation_instance/gpu/conv2d_fwd/device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instance.cpp

@@ -19,6 +19,10 @@ using F32 = float;
 template <ck::index_t... Is>
 using S = ck::Sequence<Is...>;
 
+using NHWC = ck::tensor_layout::convolution::NHWC;
+using KYXC = ck::tensor_layout::convolution::KYXC;
+using NHWK = ck::tensor_layout::convolution::NHWK;
+
 using PassThrough = ck::tensor_operation::element_wise::PassThrough;
 
 static constexpr auto ConvFwdDefault =
@@ -101,7 +105,16 @@ using device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_1x1_s1_p0_int8_instances =
         >;
 
 void add_device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instances(
-    std::vector<DeviceConvFwdPtr<PassThrough, PassThrough, PassThrough>>& instances)
+    std::vector<std::unique_ptr<DeviceConvFwd<2,
+                                              NHWC,
+                                              KYXC,
+                                              NHWK,
+                                              int8_t,
+                                              int8_t,
+                                              int8_t,
+                                              PassThrough,
+                                              PassThrough,
+                                              PassThrough>>>& instances)
 {
     add_device_operation_instances(instances,
                                    device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instances{});