[MIOpen Downstream] Initial MIOpen integration (#52)

* update online kernel wrapper bundle all descriptors in a tuple

* change __CONSTANT__ to CONSTANT

* rename

* adding tuning

* added IsValidCompileParameter

* reorginze

* adding tunable for fp16 and int8

* fix kernel compile warning and bug fixes

* suppress warning about cast CONSTANT (address space 4) pointer

* fix building issue

composable_kernel/include/problem_transform/transform_forward_convolution_into_gemm_v6r1_nchw_kcyx_nkhw.hpp

@@ -7,9 +7,12 @@
 
 namespace ck {
 
-// GemmM = K
-// GemmN = N * Ho * Wo
-// GemmK = C * Y * X
+// GemmM0 = 1
+// GemmM1 = K
+// GemmN0 = N0
+// GemmN1 = (N / N0) * Ho * Wo
+// GemmK0 = (C / C0) * Y * X
+// GemmK1 = C0
 template <typename... Wei,
           typename... In,
           typename... Out,

composable_kernel/include/tensor_description/dynamic_multi_index_transform.hpp

@@ -46,7 +46,7 @@ struct DynamicPassThrough
     __host__ __device__ static void UpdateLowerIndex(LowIdxDiff& idx_diff_low,
                                                      const UpIdxDiff& idx_diff_up,
                                                      LowIdx& idx_low,
-                                                     const UpIdx& idx_up_new,
+                                                     const UpIdx&,
                                                      Number<Hack>)
     {
         static_assert(LowIdxDiff::Size() == 1 && UpIdxDiff::Size() == 1 && LowIdx::Size() == 1 &&
@@ -136,7 +136,7 @@ struct DynamicPad
     __host__ __device__ static void UpdateLowerIndex(LowIdxDiff& idx_diff_low,
                                                      const UpIdxDiff& idx_diff_up,
                                                      LowIdx& idx_low,
-                                                     const UpIdx& idx_up_new,
+                                                     const UpIdx&,
                                                      Number<Hack>)
     {
         static_assert(LowIdxDiff::Size() == 1 && UpIdxDiff::Size() == 1 && LowIdx::Size() == 1 &&
@@ -227,7 +227,7 @@ struct DynamicLeftPad
     __host__ __device__ static void UpdateLowerIndex(LowIdxDiff& idx_diff_low,
                                                      const UpIdxDiff& idx_diff_up,
                                                      LowIdx& idx_low,
-                                                     const UpIdx& idx_up_new,
+                                                     const UpIdx&,
                                                      Number<Hack>)
     {
         static_assert(LowIdxDiff::Size() == 1 && UpIdxDiff::Size() == 1 && LowIdx::Size() == 1 &&
@@ -318,7 +318,7 @@ struct DynamicRightPad
     __host__ __device__ static void UpdateLowerIndex(LowIdxDiff& idx_diff_low,
                                                      const UpIdxDiff& idx_diff_up,
                                                      LowIdx& idx_low,
-                                                     const UpIdx& idx_up_new,
+                                                     const UpIdx&,
                                                      Number<Hack>)
     {
         static_assert(LowIdxDiff::Size() == 1 && UpIdxDiff::Size() == 1 && LowIdx::Size() == 1 &&
@@ -420,7 +420,7 @@ struct DynamicEmbed
     __host__ __device__ void UpdateLowerIndex(LowIdxDiff& idx_diff_low,
                                               const UpIdxDiff& idx_diff_up,
                                               LowIdx& idx_low,
-                                              const UpIdx& idx_up_new,
+                                              const UpIdx&,
                                               Number<Hack>) const
     {
         static_assert(LowIdxDiff::Size() == 1 && UpIdxDiff::Size() == NDimUp &&
@@ -1096,7 +1096,7 @@ struct DynamicMerge_v2_magic_division
               typename UpIdx,
               index_t Hack>
     __host__ __device__ void UpdateLowerIndex(LowIdxDiff& idx_diff_low,
-                                              const UpIdxDiff& idx_diff_up,
+                                              const UpIdxDiff&,
                                               LowIdx& idx_low,
                                               const UpIdx& idx_up_new,
                                               Number<Hack>) const
@@ -1254,7 +1254,7 @@ struct DynamicMerge_v2r2_magic_division
               typename UpIdx,
               index_t Hack>
     __host__ __device__ void UpdateLowerIndex(LowIdxDiff& idx_diff_low,
-                                              const UpIdxDiff& idx_diff_up,
+                                              const UpIdxDiff&,
                                               LowIdx& idx_low,
                                               const UpIdx& idx_up_new,
                                               Number<Hack>) const
@@ -1383,7 +1383,7 @@ struct DynamicUnMerge
     __host__ __device__ void UpdateLowerIndex(LowIdxDiff& idx_diff_low,
                                               const UpIdxDiff& idx_diff_up,
                                               LowIdx& idx_low,
-                                              const UpIdx& idx_up_new,
+                                              const UpIdx&,
                                               Number<Hack>) const
     {
         CalculateLowerIndex(idx_diff_low, idx_diff_up);
@@ -1597,7 +1597,7 @@ struct DynamicVectorize
     __host__ __device__ void UpdateLowerIndex(LowIdxDiff& idx_diff_low,
                                               const UpIdxDiff& idx_diff_up,
                                               LowIdx& idx_low,
-                                              const UpIdx& idx_up_new,
+                                              const UpIdx&,
                                               Number<Hack>) const
     {
         static_assert(LowIdxDiff::Size() == 1 && UpIdxDiff::Size() == 1 && LowIdx::Size() == 1 &&
@@ -1654,7 +1654,7 @@ struct DynamicSlice
 
     __host__ __device__ constexpr DynamicSlice() = default;
 
-    __host__ __device__ constexpr DynamicSlice(const LowLength& low_length,
+    __host__ __device__ constexpr DynamicSlice(const LowLength&,
                                                const SliceBegin& slice_begin,
                                                const SliceEnd& slice_end)
         : up_lengths_{make_tuple(slice_end - slice_begin)},
@@ -1687,7 +1687,7 @@ struct DynamicSlice
     __host__ __device__ static void UpdateLowerIndex(LowIdxDiff& idx_diff_low,
                                                      const UpIdxDiff& idx_diff_up,
                                                      LowIdx& idx_low,
-                                                     const UpIdx& idx_up_new,
+                                                     const UpIdx&,
                                                      Number<Hack>)
     {
         static_assert(LowIdxDiff::Size() == 1 && UpIdxDiff::Size() == 1 && LowIdx::Size() == 1 &&
@@ -1709,8 +1709,7 @@ struct DynamicSlice
     }
 
     template <typename UpIdx>
-    __host__ __device__ constexpr bool
-    IsValidUpperIndexMappedToValidLowerIndex(const UpIdx& idx_up) const
+    __host__ __device__ constexpr bool IsValidUpperIndexMappedToValidLowerIndex(const UpIdx&) const
     {
         return true;
     }

composable_kernel/include/tensor_description/dynamic_tensor_descriptor.hpp

@@ -317,7 +317,7 @@ transform_dynamic_tensor_descriptor(const OldTensorDescriptor& old_tensor_desc,
                                                 NewUpperDimensionNewVisibleIdss{});
 
         static_assert(is_valid_sequence_map<decltype(all_old_top_ids)>::value &&
-                          is_valid_sequence_map<decltype(all_old_top_ids)>::value,
+                          is_valid_sequence_map<decltype(all_new_top_ids)>::value,
                       "wrong!");
     }
 
@@ -395,7 +395,6 @@ __host__ __device__ constexpr auto make_dynamic_tensor_coordinate(const TensorDe
 
     constexpr index_t ntransform   = TensorDesc::GetNumOfTransform();
     constexpr index_t ndim_hidden  = TensorDesc::GetNumOfHiddenDimension();
-    constexpr index_t ndim_visible = TensorDesc::GetNumOfVisibleDimension();
     constexpr auto visible_dim_ids = TensorDesc::GetVisibleDimensionIds();
 
     MultiIndex<ndim_hidden> idx_hidden;
@@ -491,11 +490,8 @@ template <typename TensorDesc, typename TensorCoord, typename TensorCoordIterato
 __host__ __device__ constexpr void move_dynamic_tensor_coordinate(
     const TensorDesc& tensor_desc, TensorCoord& coord, const TensorCoordIterator& coord_iterator)
 {
-    constexpr index_t ndim_hidden  = TensorDesc::GetNumOfHiddenDimension();
-    constexpr index_t ndim_visible = TensorDesc::GetNumOfVisibleDimension();
-    constexpr index_t ntransform   = TensorDesc::GetNumOfTransform();
-
-    using HiddenIndex = MultiIndex<ndim_hidden>;
+    constexpr index_t ndim_hidden = TensorDesc::GetNumOfHiddenDimension();
+    constexpr index_t ntransform  = TensorDesc::GetNumOfTransform();
 
     // this is what needs to be calculated
     auto idx_diff_hidden = make_zero_multi_index<ndim_hidden>();

composable_kernel/include/tensor_description/tensor_adaptor.hpp

@@ -236,15 +236,15 @@ __host__ __device__ constexpr auto chain_tensor_adaptors(const TensorAdaptor0& a
 
     // shift
     constexpr index_t adaptor0_max_hidden_id = [&]() {
-        index_t adaptor0_max_hidden_id = NumericLimits<index_t>::Min();
+        index_t adaptor0_max_hidden_id_ = NumericLimits<index_t>::Min();
 
         static_for<0, TensorAdaptor0::GetNumOfTransform(), 1>{}([&](auto itran) {
             constexpr index_t ndim_low =
                 TensorAdaptor0{}.GetTransforms()[itran].GetNumOfLowerDimension();
 
             static_for<0, ndim_low, 1>{}([&](auto idim_low) {
-                adaptor0_max_hidden_id =
-                    math::max(adaptor0_max_hidden_id,
+                adaptor0_max_hidden_id_ =
+                    math::max(adaptor0_max_hidden_id_,
                               TensorAdaptor0::GetLowerDimensionHiddenIdss()[itran][idim_low].value);
             });
 
@@ -252,17 +252,17 @@ __host__ __device__ constexpr auto chain_tensor_adaptors(const TensorAdaptor0& a
                 TensorAdaptor0{}.GetTransforms()[itran].GetNumOfUpperDimension();
 
             static_for<0, ndim_up, 1>{}([&](auto idim_up) {
-                adaptor0_max_hidden_id =
-                    math::max(adaptor0_max_hidden_id,
+                adaptor0_max_hidden_id_ =
+                    math::max(adaptor0_max_hidden_id_,
                               TensorAdaptor0::GetUpperDimensionHiddenIdss()[itran][idim_up].value);
             });
         });
 
-        return adaptor0_max_hidden_id;
+        return adaptor0_max_hidden_id_;
     }();
 
     constexpr index_t adaptor1_min_hidden_id = [&]() {
-        index_t adaptor1_min_hidden_id = NumericLimits<index_t>::Max();
+        index_t adaptor1_min_hidden_id_ = NumericLimits<index_t>::Max();
 
         static_for<0, TensorAdaptor1::GetNumOfTransform(), 1>{}([&](auto itran) {
             constexpr index_t ndim_low =
@@ -285,7 +285,7 @@ __host__ __device__ constexpr auto chain_tensor_adaptors(const TensorAdaptor0& a
 
                 if(!is_bottom_dim)
                 {
-                    adaptor1_min_hidden_id = math::min(adaptor1_min_hidden_id, low_dim_hidden_id);
+                    adaptor1_min_hidden_id_ = math::min(adaptor1_min_hidden_id_, low_dim_hidden_id);
                 }
             });
 
@@ -294,13 +294,13 @@ __host__ __device__ constexpr auto chain_tensor_adaptors(const TensorAdaptor0& a
 
             // get the min of all upper dimensions
             static_for<0, ndim_up, 1>{}([&](auto idim_up) {
-                adaptor1_min_hidden_id =
-                    math::min(adaptor1_min_hidden_id,
+                adaptor1_min_hidden_id_ =
+                    math::min(adaptor1_min_hidden_id_,
                               TensorAdaptor1::GetUpperDimensionHiddenIdss()[itran][idim_up].value);
             });
         });
 
-        return adaptor1_min_hidden_id;
+        return adaptor1_min_hidden_id_;
     }();
 
     constexpr index_t adaptor1_hidden_id_shift =
@@ -321,11 +321,11 @@ __host__ __device__ constexpr auto chain_tensor_adaptors(const TensorAdaptor0& a
             // sequence in, sequence out
             constexpr auto low_dim_hidden_ids_1_mod = [&]() constexpr
             {
-                auto low_dim_hidden_ids_1_mod = to_multi_index(low_dim_hidden_ids_1);
+                auto low_dim_hidden_ids_1_mod_ = to_multi_index(low_dim_hidden_ids_1);
 
                 // shift hidden id so every dim id is unique
                 static_for<0, ndim_low_1, 1>{}([&](auto idim_low_1) {
-                    low_dim_hidden_ids_1_mod(idim_low_1) += adaptor1_hidden_id_shift;
+                    low_dim_hidden_ids_1_mod_(idim_low_1) += adaptor1_hidden_id_shift;
                 });
 
                 // match hidden id
@@ -335,13 +335,13 @@ __host__ __device__ constexpr auto chain_tensor_adaptors(const TensorAdaptor0& a
                         if constexpr(low_dim_hidden_ids_1[idim_low_1] ==
                                      TensorAdaptor1::GetBottomDimensionHiddenIds()[idim_bottom_1])
                         {
-                            low_dim_hidden_ids_1_mod(idim_low_1) =
+                            low_dim_hidden_ids_1_mod_(idim_low_1) =
                                 TensorAdaptor0::GetTopDimensionHiddenIds()[idim_bottom_1];
                         }
                     });
                 });
 
-                return low_dim_hidden_ids_1_mod;
+                return low_dim_hidden_ids_1_mod_;
             }
             ();
 
@@ -367,14 +367,14 @@ __host__ __device__ constexpr auto chain_tensor_adaptors(const TensorAdaptor0& a
             // sequence in, constexpr tuple out
             constexpr auto up_dim_hidden_ids_1_mod = [&]() constexpr
             {
-                auto up_dim_hidden_ids_1_mod = to_multi_index(up_dim_hidden_ids_1);
+                auto up_dim_hidden_ids_1_mod_ = to_multi_index(up_dim_hidden_ids_1);
 
                 // shift hidden id
                 static_for<0, ndim_up_1, 1>{}([&](auto idim_up_1) {
-                    up_dim_hidden_ids_1_mod(idim_up_1) += adaptor1_hidden_id_shift;
+                    up_dim_hidden_ids_1_mod_(idim_up_1) += adaptor1_hidden_id_shift;
                 });
 
-                return up_dim_hidden_ids_1_mod;
+                return up_dim_hidden_ids_1_mod_;
             }
             ();
 

composable_kernel/include/tensor_operation/blockwise_dynamic_tensor_slice_transfer.hpp

@@ -14,7 +14,7 @@ namespace ck {
 // 2. ThreadwiseDynamicTensorSliceTransfer_v3 does not keep reference to tensor descriptor
 // 3. ThreadwiseDynamicTensorSliceTransfer_v3::Run() does not construct new tensor coordinate
 template <index_t BlockSize,
-          InMemoryDataOperation DstInMemOp,
+          InMemoryDataOperationEnum_t DstInMemOp,
           typename BlockSliceLengths,
           typename ThreadSliceLengths,
           typename ThreadClusterLengths,

composable_kernel/include/tensor_operation/blockwise_dynamic_tensor_slice_transfer_v2.hpp

@@ -14,7 +14,7 @@ namespace ck {
 // 2. ThreadwiseDynamicTensorSliceTransfer_v3 does not keep reference to tensor descriptor
 // 3. ThreadwiseDynamicTensorSliceTransfer_v3::Run() does not construct new tensor coordinate
 template <index_t BlockSize,
-          InMemoryDataOperation DstInMemOp,
+          InMemoryDataOperationEnum_t DstInMemOp,
           typename BlockSliceLengths,
           typename ThreadSliceLengths,
           typename ThreadClusterLengths,

composable_kernel/include/tensor_operation/blockwise_gemm_v2r2.hpp → composable_kernel/include/tensor_operation/blockwise_gemm_dlops_v2r2.hpp

-#ifndef CK_BLOCKWISE_GEMM_V2R2_HPP
-#define CK_BLOCKWISE_GEMM_V2R2_HPP
+#ifndef CK_BLOCKWISE_GEMM_DLOPS_V2R2_HPP
+#define CK_BLOCKWISE_GEMM_DLOPS_V2R2_HPP
 
 #include "common_header.hpp"
 #include "tensor_adaptor.hpp"
 #include "threadwise_dynamic_tensor_slice_transfer.hpp"
-#include "threadwise_contraction.hpp"
+#include "threadwise_contraction_dlops.hpp"
 
 namespace ck {
 
@@ -40,7 +40,7 @@ template <index_t BlockSize,
           typename std::enable_if<AKMBlockDesc::IsKnownAtCompileTime() &&
                                       BKNBlockDesc::IsKnownAtCompileTime(),
                                   bool>::type = false>
-struct BlockwiseGemm_km_kn_m0m1n0n1_v2r2_pipeline_2x2
+struct BlockwiseGemmDlops_km_kn_m0m1n0n1_v2r2_pipeline_2x2
 {
     using AIndex = MultiIndex<3>;
     using BIndex = MultiIndex<3>;
@@ -140,7 +140,7 @@ struct BlockwiseGemm_km_kn_m0m1n0n1_v2r2_pipeline_2x2
     static constexpr auto b_k_n0_n1_block_desc_ = MakeBKN0N1BlockDescriptor(BKNBlockDesc{});
 
     public:
-    __device__ BlockwiseGemm_km_kn_m0m1n0n1_v2r2_pipeline_2x2()
+    __device__ BlockwiseGemmDlops_km_kn_m0m1n0n1_v2r2_pipeline_2x2()
         : c_thread_origin_data_idx_{CalculateCM0M1N0N1ThreadOriginOnBlock(
               get_thread_local_1d_id())},
           a_thread_copy_{
@@ -183,7 +183,7 @@ struct BlockwiseGemm_km_kn_m0m1n0n1_v2r2_pipeline_2x2
 
         constexpr auto adaptor = chain_tensor_adaptors(adaptor0, adaptor1);
 
-        return adaptor.CalculateBottomIndex(make_multi_index(get_thread_local_1d_id(), 0, 0, 0, 0));
+        return adaptor.CalculateBottomIndex(make_multi_index(thread_id, 0, 0, 0, 0));
     }
 
     __host__ __device__ static constexpr index_t GetABlockAlignment() { return M1PerThreadM11; }
@@ -207,21 +207,21 @@ struct BlockwiseGemm_km_kn_m0m1n0n1_v2r2_pipeline_2x2
                           CM0M1N0N1ThreadDesc{}.GetLength(I2) == N0,
                       "wrong");
 
-        auto a_thread_buf = make_static_buffer<AddressSpace::Vgpr, FloatA>(
+        auto a_thread_buf = make_static_buffer<AddressSpaceEnum_t::Vgpr, FloatA>(
             a_k_m0_m1_thread_desc_.GetElementSpaceSize());
-        auto b_thread_buf = make_static_buffer<AddressSpace::Vgpr, FloatB>(
+        auto b_thread_buf = make_static_buffer<AddressSpaceEnum_t::Vgpr, FloatB>(
             b_k_n0_n1_thread_desc_.GetElementSpaceSize());
 
         constexpr auto threadwise_gemm =
-            ThreadwiseGemm_km0m1_kn0n1_m0m1n0n1<FloatA,
-                                                FloatB,
-                                                FloatC,
-                                                decltype(a_k_m0_m1_thread_desc_),
-                                                decltype(b_k_n0_n1_thread_desc_),
-                                                CM0M1N0N1ThreadDesc,
-                                                Sequence<KPerThread>,
-                                                Sequence<1, M1PerThreadM11>,
-                                                Sequence<1, N1PerThreadN11>>{};
+            ThreadwiseGemmDlops_km0m1_kn0n1_m0m1n0n1<FloatA,
+                                                     FloatB,
+                                                     FloatC,
+                                                     decltype(a_k_m0_m1_thread_desc_),
+                                                     decltype(b_k_n0_n1_thread_desc_),
+                                                     CM0M1N0N1ThreadDesc,
+                                                     Sequence<KPerThread>,
+                                                     Sequence<1, M1PerThreadM11>,
+                                                     Sequence<1, N1PerThreadN11>>{};
 
         // read A_sub_0
         a_thread_copy_.Run(a_k_m0_m1_block_desc_,

composable_kernel/include/tensor_operation/blockwise_gemm_v2r3.hpp → composable_kernel/include/tensor_operation/blockwise_gemm_dlops_v2r3.hpp

-#ifndef CK_BLOCKWISE_GEMM_V2R3_HPP
-#define CK_BLOCKWISE_GEMM_V2R3_HPP
+#ifndef CK_BLOCKWISE_GEMM_DLOPS_V2R3_HPP
+#define CK_BLOCKWISE_GEMM_DLOPS_V2R3_HPP
 
 #include "common_header.hpp"
 #include "tensor_adaptor.hpp"
 #include "threadwise_dynamic_tensor_slice_transfer_v2.hpp"
-#include "threadwise_contraction.hpp"
+#include "threadwise_contraction_dlops.hpp"
 
 namespace ck {
 
@@ -21,6 +21,7 @@ namespace ck {
 //     1. CThreadDesc_BM0_BM11_BN0_BN11 is known at compile-time
 //     2. CThreadBuffer is StaticBuffer
 // Also assume:
+//   BM10BN10ThreadClusterBM10Xs::Size() = BM10BN10ThreadClusterBN10Xs::Size() == 2
 //   BM0 = BN0 = 2. It will do 2x2 pipelined read and fma (ABBA optimization)
 template <index_t BlockSize,
           typename FloatA,
@@ -31,16 +32,16 @@ template <index_t BlockSize,
           index_t BM1PerThreadBM11,
           index_t BN1PerThreadBN11,
           index_t BK0PerThread,
-          index_t BM10BN10ThreadClusterBM100,
-          index_t BM10BN10ThreadClusterBN100,
-          index_t BM10BN10ThreadClusterBM101,
-          index_t BM10BN10ThreadClusterBN101,
+          typename BM10BN10ThreadClusterBM10Xs, // Sequence<BM10BN10ThreadClusterBM100,
+                                                //          BM10BN10ThreadClusterBM101, ...>
+          typename BM10BN10ThreadClusterBN10Xs, // Sequence<BM10BN10ThreadClusterBN100,
+                                                //          BM10BN10ThreadClusterBN101, ...>
           index_t AThreadCopyScalarPerVector_BM11,
           index_t BThreadCopyScalarPerVector_BN11,
           typename std::enable_if<ABlockDesc_BK0_BM_BK1::IsKnownAtCompileTime() &&
                                       BBlockDesc_BK0_BN_BK1::IsKnownAtCompileTime(),
                                   bool>::type = false>
-struct BlockwiseGemm_A_BK0_BM_BK1_B_BK0_BN_BK1_C_BM0_BM1_BN0_BN1_pipeline_BM0_2_BN0_2
+struct BlockwiseGemmDlops_A_BK0_BM_BK1_B_BK0_BN_BK1_C_BM0_BM1_BN0_BN1_pipeline_BM0_2_BN0_2
 {
     using AIndex = MultiIndex<3>;
     using BIndex = MultiIndex<3>;
@@ -56,19 +57,17 @@ struct BlockwiseGemm_A_BK0_BM_BK1_B_BK0_BN_BK1_C_BM0_BM1_BN0_BN1_pipeline_BM0_2_
     static constexpr index_t BM  = ABlockDesc_BK0_BM_BK1{}.GetLength(I1);
     static constexpr index_t BN  = BBlockDesc_BK0_BN_BK1{}.GetLength(I1);
 
-    static constexpr index_t BM100 = BM10BN10ThreadClusterBM100;
-    static constexpr index_t BN100 = BM10BN10ThreadClusterBN100;
+    static constexpr index_t BM100 = BM10BN10ThreadClusterBM10Xs{}[I0];
+    static constexpr index_t BN100 = BM10BN10ThreadClusterBN10Xs{}[I0];
 
-    static constexpr index_t BM101 = BM10BN10ThreadClusterBM101;
-    static constexpr index_t BN101 = BM10BN10ThreadClusterBN101;
+    static constexpr index_t BM101 = BM10BN10ThreadClusterBM10Xs{}[I1];
+    static constexpr index_t BN101 = BM10BN10ThreadClusterBN10Xs{}[I1];
 
     static constexpr index_t BM11 = BM1PerThreadBM11;
     static constexpr index_t BN11 = BN1PerThreadBN11;
 
-    static constexpr index_t BM1 =
-        BM10BN10ThreadClusterBM100 * BM10BN10ThreadClusterBM101 * BM1PerThreadBM11;
-    static constexpr index_t BN1 =
-        BM10BN10ThreadClusterBN100 * BM10BN10ThreadClusterBN101 * BN1PerThreadBN11;
+    static constexpr index_t BM1 = BM100 * BM101 * BM11;
+    static constexpr index_t BN1 = BN100 * BN101 * BN11;
 
     static constexpr index_t BM0 = BM / BM1;
     static constexpr index_t BN0 = BN / BN1;
@@ -149,7 +148,7 @@ struct BlockwiseGemm_A_BK0_BM_BK1_B_BK0_BN_BK1_C_BM0_BM1_BN0_BN1_pipeline_BM0_2_
         MakeBBlockDescriptor_BK0_BN0_BN1_BK1(BBlockDesc_BK0_BN_BK1{});
 
     public:
-    __device__ BlockwiseGemm_A_BK0_BM_BK1_B_BK0_BN_BK1_C_BM0_BM1_BN0_BN1_pipeline_BM0_2_BN0_2()
+    __device__ BlockwiseGemmDlops_A_BK0_BM_BK1_B_BK0_BN_BK1_C_BM0_BM1_BN0_BN1_pipeline_BM0_2_BN0_2()
         : c_thread_origin_data_idx_{CalculateCThreadOriginOnBlock_BM0_BM1_BN0_BN1(
               get_thread_local_1d_id())},
           a_thread_copy_{
@@ -170,6 +169,11 @@ struct BlockwiseGemm_A_BK0_BM_BK1_B_BK0_BN_BK1_C_BM0_BM1_BN0_BN1_pipeline_BM0_2_
                           BBlockDesc_BK0_BN_BK1{}.GetLength(I0),
                       "wrong! K dimension not consistent");
 
+        // TODO remove this restriction
+        static_assert(BM10BN10ThreadClusterBM10Xs::Size() == 2 &&
+                          BM10BN10ThreadClusterBN10Xs::Size() == 2,
+                      "wrong!");
+
         // TODO: remove this restriction
         static_assert(BM0 == 2 && BN0 == 2, "wrong");
     }
@@ -195,14 +199,14 @@ struct BlockwiseGemm_A_BK0_BM_BK1_B_BK0_BN_BK1_C_BM0_BM1_BN0_BN1_pipeline_BM0_2_
 
         constexpr auto adaptor = chain_tensor_adaptors(adaptor0, adaptor1);
 
-        return adaptor.CalculateBottomIndex(make_multi_index(get_thread_local_1d_id(), 0, 0, 0, 0));
+        return adaptor.CalculateBottomIndex(make_multi_index(thread_id, 0, 0, 0, 0));
     }
 
     template <typename CThreadDesc_BM0_BM11_BN0_BN11,
               typename ABlockBuffer,
               typename BBlockBuffer,
               typename CThreadBuffer>
-    __device__ void Run(const CThreadDesc_BM0_BM11_BN0_BN11& c_m0_m1_n0_n1_thread_desc,
+    __device__ void Run(const CThreadDesc_BM0_BM11_BN0_BN11&,
                         const ABlockBuffer& a_block_buf,
                         const BBlockBuffer& b_block_buf,
                         CThreadBuffer& c_thread_buf) const
@@ -216,13 +220,13 @@ struct BlockwiseGemm_A_BK0_BM_BK1_B_BK0_BN_BK1_C_BM0_BM1_BN0_BN1_pipeline_BM0_2_
                           CThreadDesc_BM0_BM11_BN0_BN11{}.GetLength(I2) == BN0,
                       "wrong");
 
-        auto a_thread_buf = make_static_buffer<AddressSpace::Vgpr, FloatA>(
+        auto a_thread_buf = make_static_buffer<AddressSpaceEnum_t::Vgpr, FloatA>(
             a_thread_desc_bk0_bm0_bm1_bk1_.GetElementSpaceSize());
-        auto b_thread_buf = make_static_buffer<AddressSpace::Vgpr, FloatB>(
+        auto b_thread_buf = make_static_buffer<AddressSpaceEnum_t::Vgpr, FloatB>(
             b_thread_desc_bk0_bn0_bn1_bk1_.GetElementSpaceSize());
 
         constexpr auto threadwise_contraction =
-            ThreadwiseContraction_A_TK0_TM0_TM1_TK1_B_TK0_TN0_TN1_TK1_C_TM0_TM1_TN0_TN1<
+            ThreadwiseContractionDlops_A_TK0_TM0_TM1_TK1_B_TK0_TN0_TN1_TK1_C_TM0_TM1_TN0_TN1<
                 FloatA,
                 FloatB,
                 FloatC,

composable_kernel/include/tensor_operation/blockwise_gemm_v3.hpp → composable_kernel/include/tensor_operation/blockwise_gemm_dlops_v3.hpp

-#ifndef CK_BLOCKWISE_GEMM_V3_HPP
-#define CK_BLOCKWISE_GEMM_V3_HPP
+#ifndef CK_BLOCKWISE_GEMM_DLOPS_V3_HPP
+#define CK_BLOCKWISE_GEMM_DLOPS_V3_HPP
 
 #include "common_header.hpp"
-#include "threadwise_gemm_v3.hpp"
+#include "threadwise_gemm_dlops_v3.hpp"
 
 namespace ck {
 
@@ -19,7 +19,7 @@ template <index_t BlockSize,
           index_t EPerThreadLoop,
           index_t ThreadGemmADataPerRead_K,
           index_t ThreadGemmBDataPerRead_W>
-struct BlockwiseGemm_km_kn_m0m1n0n1_v3
+struct BlockwiseGemmDlops_km_kn_m0m1n0n1_v3
 {
     struct MatrixIndex
     {
@@ -51,7 +51,7 @@ struct BlockwiseGemm_km_kn_m0m1n0n1_v3
                                                 ThreadGemmADataPerRead_K,
                                                 1>;
 
-    __device__ BlockwiseGemm_km_kn_m0m1n0n1_v3()
+    __device__ BlockwiseGemmDlops_km_kn_m0m1n0n1_v3()
         : c_thread_begin_mtx_idx_{GetBeginOfThreadMatrixC(get_thread_local_1d_id())},
           a_thread_copy_{make_tuple(0, c_thread_begin_mtx_idx_.k * KPerThread)}
     {
@@ -138,16 +138,17 @@ struct BlockwiseGemm_km_kn_m0m1n0n1_v3
         static_assert(WPerThread % WoPerThreadSubC == 0, "");
 
         // thread A buffer for GEMM
-        StaticBuffer<AddressSpace::Vgpr, FloatA, a_thread_mtx_.GetElementSpaceSize()> a_thread_buf;
-
-        constexpr auto threadwise_gemm = ThreadwiseGemm_km_kn_mn_v3<FloatA,
-                                                                    FloatB,
-                                                                    FloatC,
-                                                                    decltype(a_thread_mtx_),
-                                                                    decltype(b_thread_mtx_),
-                                                                    decltype(c_thread_mtx_),
-                                                                    HoPerThreadSubC,
-                                                                    WoPerThreadSubC>{};
+        StaticBuffer<AddressSpaceEnum_t::Vgpr, FloatA, a_thread_mtx_.GetElementSpaceSize()>
+            a_thread_buf;
+
+        constexpr auto threadwise_gemm = ThreadwiseGemmDlops_km_kn_mn_v3<FloatA,
+                                                                         FloatB,
+                                                                         FloatC,
+                                                                         decltype(a_thread_mtx_),
+                                                                         decltype(b_thread_mtx_),
+                                                                         decltype(c_thread_mtx_),
+                                                                         HoPerThreadSubC,
+                                                                         WoPerThreadSubC>{};
 
         static_for<0, EPerBlock, EPerThreadLoop>{}([&](auto e_begin) {
             static_for<0, KPerThread, KPerThreadSubC>{}([&](auto k_begin) {

composable_kernel/include/tensor_operation/blockwise_gemm_xdlops.hpp

@@ -138,10 +138,10 @@ struct BlockwiseGemmXdlops_km_kn_m0m1m2n_v1
                         const BBlockBuffer& b_block_buf,
                         CThreadBuffer& c_thread_buf) const
     {
-        auto a_thread_buf =
-            make_static_buffer<AddressSpace::Vgpr, FloatAB>(a_thread_desc_.GetElementSpaceSize());
-        auto b_thread_buf =
-            make_static_buffer<AddressSpace::Vgpr, FloatAB>(b_thread_desc_.GetElementSpaceSize());
+        auto a_thread_buf = make_static_buffer<AddressSpaceEnum_t::Vgpr, FloatAB>(
+            a_thread_desc_.GetElementSpaceSize());
+        auto b_thread_buf = make_static_buffer<AddressSpaceEnum_t::Vgpr, FloatAB>(
+            b_thread_desc_.GetElementSpaceSize());
 
         constexpr index_t KPerBlock = ABlockDesc{}.GetLength(I0);
 
@@ -358,10 +358,10 @@ struct BlockwiseGemmXdlops_km_kn_m0m1m2n_v1_2x2pipeline
                         const BBlockBuffer& b_block_buf,
                         CThreadBuffer& c_thread_buf) const
     {
-        auto a_thread_buf =
-            make_static_buffer<AddressSpace::Vgpr, FloatAB>(a_thread_desc_.GetElementSpaceSize());
-        auto b_thread_buf =
-            make_static_buffer<AddressSpace::Vgpr, FloatAB>(b_thread_desc_.GetElementSpaceSize());
+        auto a_thread_buf = make_static_buffer<AddressSpaceEnum_t::Vgpr, FloatAB>(
+            a_thread_desc_.GetElementSpaceSize());
+        auto b_thread_buf = make_static_buffer<AddressSpaceEnum_t::Vgpr, FloatAB>(
+            b_thread_desc_.GetElementSpaceSize());
 
         constexpr index_t KPerBlock = ABlockDesc{}.GetLength(I0);
 

composable_kernel/include/tensor_operation/gridwise_dynamic_contraction_v1r2.hpp → composable_kernel/include/tensor_operation/gridwise_dynamic_contraction_dlops_v1r2.hpp

-#ifndef CK_GRIDWISE_DYNAMIC_CONTRACTION_V1R2_HPP
-#define CK_GRIDWISE_DYNAMIC_CONTRACTION_V1R2_HPP
+#ifndef CK_GRIDWISE_DYNAMIC_CONTRACTION_DLOPS_V1R2_HPP
+#define CK_GRIDWISE_DYNAMIC_CONTRACTION_DLOPS_V1R2_HPP
 
 #include "common_header.hpp"
 #include "dynamic_multi_index_transform_helper.hpp"
 #include "dynamic_tensor_descriptor.hpp"
 #include "dynamic_tensor_descriptor_helper.hpp"
-#include "blockwise_gemm_v2r3.hpp"
+#include "blockwise_gemm_dlops_v2r3.hpp"
 #include "blockwise_dynamic_tensor_slice_transfer_v2.hpp"
 #include "threadwise_dynamic_tensor_slice_transfer.hpp"
 #include "threadwise_dynamic_tensor_slice_set.hpp"
@@ -25,7 +25,7 @@ __global__ void
 #if CK_USE_LAUNCH_BOUNDS
     __launch_bounds__(CK_MAX_THREAD_PER_BLOCK, CK_MIN_BLOCK_PER_CU)
 #endif
-        kernel_dynamic_contraction_v1r2(
+        kernel_dynamic_contraction_dlops_v1r2(
             const FloatAB* __restrict__ p_a_grid,
             const FloatAB* __restrict__ p_b_grid,
             FloatC* __restrict__ p_c_grid,
@@ -55,7 +55,7 @@ template <index_t BlockSize,
           typename FloatAB,
           typename FloatAcc,
           typename FloatC,
-          InMemoryDataOperation CGlobalMemoryDataOperation,
+          InMemoryDataOperationEnum_t CGlobalMemoryDataOperation,
           typename AGridDesc_GK0_GM0_GM1_GK1,
           typename BGridDesc_GK0_GN0_GN1_GK1,
           typename CGridDesc_GM0_GM1_GN0_GN1,
@@ -65,10 +65,8 @@ template <index_t BlockSize,
           index_t BM1PerThreadBM11,
           index_t BN1PerThreadBN11,
           index_t BK0PerThread,
-          index_t BM10BN10ThreadClusterBM100,
-          index_t BM10BN10ThreadClusterBN100,
-          index_t BM10BN10ThreadClusterBM101,
-          index_t BM10BN10ThreadClusterBN101,
+          typename BM10BN10ThreadClusterBM10Xs,
+          typename BM10BN10ThreadClusterBN10Xs,
           typename ABlockTransferThreadSliceLengths_GK0_GM0_GM10_GM11_GK1,
           typename ABlockTransferThreadClusterLengths_GK0_GM0_GM10_GM11_GK1,
           typename ABlockTransferThreadClusterArrangeOrder,
@@ -91,7 +89,7 @@ template <index_t BlockSize,
           typename CGridIteratorHacks,
           typename AGridMoveSliceWindowIteratorHacks,
           typename BGridMoveSliceWindowIteratorHacks>
-struct GridwiseDynamicContraction_A_GK0_GM0_GM1_GK1_B_GK0_GN0_GN1_GK1_C_GM0_GM1_GN0_GN1
+struct GridwiseDynamicContractionDlops_A_GK0_GM0_GM1_GK1_B_GK0_GN0_GN1_GK1_C_GM0_GM1_GN0_GN1
 {
     static constexpr auto I0 = Number<0>{};
     static constexpr auto I1 = Number<1>{};
@@ -252,9 +250,11 @@ struct GridwiseDynamicContraction_A_GK0_GM0_GM1_GK1_B_GK0_GN0_GN1_GK1_C_GM0_GM1_
         constexpr auto BN = GN0 * GN11;
 
         constexpr auto BM1 =
-            Number<BM10BN10ThreadClusterBM100 * BM10BN10ThreadClusterBM101 * BM1PerThreadBM11>{};
+            Number<container_reduce(BM10BN10ThreadClusterBM10Xs{}, math::multiplies_v2{}, I1) *
+                   BM1PerThreadBM11>{};
         constexpr auto BN1 =
-            Number<BM10BN10ThreadClusterBN100 * BM10BN10ThreadClusterBN101 * BN1PerThreadBN11>{};
+            Number<container_reduce(BM10BN10ThreadClusterBN10Xs{}, math::multiplies_v2{}, I1) *
+                   BN1PerThreadBN11>{};
 
         constexpr auto BM0 = BM / BM1;
         constexpr auto BN0 = BN / BN1;
@@ -331,11 +331,11 @@ struct GridwiseDynamicContraction_A_GK0_GM0_GM1_GK1_B_GK0_GN0_GN1_GK1_C_GM0_GM1_
         integral_constant<bool, HasMainKBlockLoop>,
         integral_constant<bool, HasDoubleTailKBlockLoop>)
     {
-        const auto a_global_buf = make_dynamic_buffer<AddressSpace::Global>(
+        const auto a_global_buf = make_dynamic_buffer<AddressSpaceEnum_t::Global>(
             p_a_grid, a_grid_desc_gk0_gm0_gm10_gm11_gk1.GetElementSpaceSize());
-        const auto b_global_buf = make_dynamic_buffer<AddressSpace::Global>(
+        const auto b_global_buf = make_dynamic_buffer<AddressSpaceEnum_t::Global>(
             p_b_grid, b_grid_desc_gk0_gn0_gn10_gn11_gk1.GetElementSpaceSize());
-        auto c_grid_buf = make_dynamic_buffer<AddressSpace::Global>(
+        auto c_grid_buf = make_dynamic_buffer<AddressSpaceEnum_t::Global>(
             p_c_grid, c_grid_desc_gm10_bm0_bm1_gn10_bn0_bn1.GetElementSpaceSize());
 
         const auto GK0 = a_grid_desc_gk0_gm0_gm10_gm11_gk1.GetLength(I0);
@@ -387,7 +387,7 @@ struct GridwiseDynamicContraction_A_GK0_GM0_GM1_GK1_B_GK0_GN0_GN1_GK1_C_GM0_GM1_
         // A matrix blockwise copy
         auto a_blockwise_copy = BlockwiseDynamicTensorSliceTransfer_v4r1<
             BlockSize,
-            InMemoryDataOperation::Set,
+            InMemoryDataOperationEnum_t::Set,
             Sequence<GK0PerBlock, GM0, 1, GM1PerBlockGM11, GK1.value>,
             ABlockTransferThreadSliceLengths_GK0_GM0_GM10_GM11_GK1,
             ABlockTransferThreadClusterLengths_GK0_GM0_GM10_GM11_GK1,
@@ -411,7 +411,7 @@ struct GridwiseDynamicContraction_A_GK0_GM0_GM1_GK1_B_GK0_GN0_GN1_GK1_C_GM0_GM1_
         // B matrix blockwise copy
         auto b_blockwise_copy = BlockwiseDynamicTensorSliceTransfer_v4r1<
             BlockSize,
-            InMemoryDataOperation::Set,
+            InMemoryDataOperationEnum_t::Set,
             Sequence<GK0PerBlock, GN0, 1, GN1PerBlockGN11, GK1.value>,
             BBlockTransferThreadSliceLengths_GK0_GN0_GN10_GN11_GK1,
             BBlockTransferThreadClusterLengths_GK0_GN0_GN10_GN11_GK1,
@@ -439,7 +439,7 @@ struct GridwiseDynamicContraction_A_GK0_GM0_GM1_GK1_B_GK0_GN0_GN1_GK1_C_GM0_GM1_
         //     c_mtx[GM1PerBlockGM11, GN1PerBlockGN11] is distributed among threads, and saved in
         //       register
         const auto blockwise_gemm =
-            BlockwiseGemm_A_BK0_BM_BK1_B_BK0_BN_BK1_C_BM0_BM1_BN0_BN1_pipeline_BM0_2_BN0_2<
+            BlockwiseGemmDlops_A_BK0_BM_BK1_B_BK0_BN_BK1_C_BM0_BM1_BN0_BN1_pipeline_BM0_2_BN0_2<
                 BlockSize,
                 FloatAB,
                 FloatAB,
@@ -449,10 +449,8 @@ struct GridwiseDynamicContraction_A_GK0_GM0_GM1_GK1_B_GK0_GN0_GN1_GK1_C_GM0_GM1_
                 BM1PerThreadBM11,
                 BN1PerThreadBN11,
                 BK0PerThread,
-                BM10BN10ThreadClusterBM100,
-                BM10BN10ThreadClusterBN100,
-                BM10BN10ThreadClusterBM101,
-                BM10BN10ThreadClusterBN101,
+                BM10BN10ThreadClusterBM10Xs,
+                BM10BN10ThreadClusterBN10Xs,
                 BM1PerThreadBM11,
                 BN1PerThreadBN11>{};
 
@@ -474,7 +472,7 @@ struct GridwiseDynamicContraction_A_GK0_GM0_GM1_GK1_B_GK0_GN0_GN1_GK1_C_GM0_GM1_
         FloatAB* p_b_block_double = p_shared_block + 2 * a_block_aligned_space_size;
 
         // register allocation for output
-        auto c_thread_buf = make_static_buffer<AddressSpace::Vgpr, FloatAcc>(
+        auto c_thread_buf = make_static_buffer<AddressSpaceEnum_t::Vgpr, FloatAcc>(
             c_thread_desc_bm0_bm1_bn0_bn1.GetElementSpaceSize());
 
         ThreadwiseDynamicTensorSliceSet_v1<FloatAcc,
@@ -488,15 +486,15 @@ struct GridwiseDynamicContraction_A_GK0_GM0_GM1_GK1_B_GK0_GN0_GN1_GK1_C_GM0_GM1_
         constexpr auto a_block_slice_copy_step = make_multi_index(GK0PerBlock, 0, 0, 0, 0);
         constexpr auto b_block_slice_copy_step = make_multi_index(GK0PerBlock, 0, 0, 0, 0);
 
-        auto a_block_even_buf = make_dynamic_buffer<AddressSpace::Lds>(
+        auto a_block_even_buf = make_dynamic_buffer<AddressSpaceEnum_t::Lds>(
             p_a_block_double, a_block_desc_gk0_gm0_gm10_gm11_gk1.GetElementSpaceSize());
-        auto b_block_even_buf = make_dynamic_buffer<AddressSpace::Lds>(
+        auto b_block_even_buf = make_dynamic_buffer<AddressSpaceEnum_t::Lds>(
             p_b_block_double, b_block_desc_gk0_gn0_gn10_gn11_gk1.GetElementSpaceSize());
 
-        auto a_block_odd_buf = make_dynamic_buffer<AddressSpace::Lds>(
+        auto a_block_odd_buf = make_dynamic_buffer<AddressSpaceEnum_t::Lds>(
             p_a_block_double + a_block_aligned_space_size,
             a_block_desc_gk0_gm0_gm10_gm11_gk1.GetElementSpaceSize());
-        auto b_block_odd_buf = make_dynamic_buffer<AddressSpace::Lds>(
+        auto b_block_odd_buf = make_dynamic_buffer<AddressSpaceEnum_t::Lds>(
             p_b_block_double + b_block_aligned_space_size,
             b_block_desc_gk0_gn0_gn10_gn11_gk1.GetElementSpaceSize());
 

composable_kernel/include/tensor_operation/gridwise_dynamic_gemm_v1r2.hpp → composable_kernel/include/tensor_operation/gridwise_dynamic_gemm_dlops_v1r2.hpp

-#ifndef CK_GRIDWISE_DYNAMIC_GEMM_V1R2_HPP
-#define CK_GRIDWISE_DYNAMIC_GEMM_V1R2_HPP
+#ifndef CK_GRIDWISE_DYNAMIC_GEMM_DLOPS_V1R2_HPP
+#define CK_GRIDWISE_DYNAMIC_GEMM_DLOPS_V1R2_HPP
 
 #include "common_header.hpp"
 #include "dynamic_multi_index_transform_helper.hpp"
 #include "dynamic_tensor_descriptor.hpp"
 #include "dynamic_tensor_descriptor_helper.hpp"
-#include "blockwise_gemm_v2r2.hpp"
+#include "blockwise_gemm_dlops_v2r2.hpp"
 #include "blockwise_dynamic_tensor_slice_transfer.hpp"
 #include "threadwise_dynamic_tensor_slice_transfer.hpp"
 #include "threadwise_dynamic_tensor_slice_set.hpp"
@@ -26,7 +26,7 @@ __global__ void
 #if CK_USE_LAUNCH_BOUNDS
     __launch_bounds__(CK_MAX_THREAD_PER_BLOCK, CK_MIN_BLOCK_PER_CU)
 #endif
-        kernel_dynamic_gemm_v1r2(
+        kernel_dynamic_gemm_dlops_v1r2(
             const FloatAB* __restrict__ p_a_grid,
             const FloatAB* __restrict__ p_b_grid,
             FloatC* __restrict__ p_c_grid,
@@ -52,8 +52,8 @@ __global__ void
                       integral_constant<bool, HasDoubleTailKBlockLoop>{});
 }
 #elif CK_EXPERIMENTAL_PASS_TENSOR_DESCRIPTOR_BY_VOID_POINTER
-// pass tensor descriptor by __CONSTANT__ void pointer
-// __CONSTANT__ is needed to inform compiler void pointers in the kernel signature are pointing to
+// pass tensor descriptor by CONSTANT void pointer
+// CONSTANT is needed to inform compiler void pointers in the kernel signature are pointing to
 // non-modifiable parameter address space, so compiler can enable corresponding optimization
 template <typename GridwiseGemm,
           typename FloatAB,
@@ -68,16 +68,16 @@ __global__ void
 #if CK_USE_LAUNCH_BOUNDS
     __launch_bounds__(CK_MAX_THREAD_PER_BLOCK, CK_MIN_BLOCK_PER_CU)
 #endif
-        kernel_dynamic_gemm_v1r2(
+        kernel_dynamic_gemm_dlops_v1r2(
             const FloatAB* __restrict__ p_a_grid,
             const FloatAB* __restrict__ p_b_grid,
             FloatC* __restrict__ p_c_grid,
-            const void __CONSTANT__* p_a_k_m0_m1_grid_desc,
-            const void __CONSTANT__* p_b_k_n0_n1_grid_desc,
-            const void __CONSTANT__* p_c_m0_m10_m11_n0_n10_n11_grid_desc,
-            const void __CONSTANT__* p_c_blockid_to_m0_n0_block_cluster_adaptor)
+            const void CONSTANT* p_a_k_m0_m1_grid_desc,
+            const void CONSTANT* p_b_k_n0_n1_grid_desc,
+            const void CONSTANT* p_c_m0_m10_m11_n0_n10_n11_grid_desc,
+            const void CONSTANT* p_c_blockid_to_m0_n0_block_cluster_adaptor)
 {
-    // first cast void __CONSTANT__ void* to void*
+    // first cast void CONSTANT void* to void*
     // second cast void* to Desc*
     // the copy constructor of tensor descriptor doesn't take address_space(4)
     const auto a_k_m0_m1_grid_desc =
@@ -113,7 +113,7 @@ template <index_t BlockSize,
           typename FloatAB,
           typename FloatAcc,
           typename FloatC,
-          InMemoryDataOperation CGlobalMemoryDataOperation,
+          InMemoryDataOperationEnum_t CGlobalMemoryDataOperation,
           typename AKMGridDesc,
           typename BKNGridDesc,
           typename CMNGridDesc,
@@ -151,7 +151,7 @@ template <index_t BlockSize,
           typename CGridIteratorHacks,
           typename AGridMoveSliceWindowIteratorHacks,
           typename BGridMoveSliceWindowIteratorHacks>
-struct GridwiseDynamicGemm_km_kn_mn_v1r2
+struct GridwiseDynamicGemmDlops_km_kn_mn_v1r2
 {
     static constexpr auto I0 = Number<0>{};
     static constexpr auto I1 = Number<1>{};
@@ -326,11 +326,11 @@ struct GridwiseDynamicGemm_km_kn_mn_v1r2
         integral_constant<bool, HasMainKBlockLoop>,
         integral_constant<bool, HasDoubleTailKBlockLoop>)
     {
-        const auto a_global_buf = make_dynamic_buffer<AddressSpace::Global>(
+        const auto a_global_buf = make_dynamic_buffer<AddressSpaceEnum_t::Global>(
             p_a_grid, a_k_m0_m1_grid_desc.GetElementSpaceSize());
-        const auto b_global_buf = make_dynamic_buffer<AddressSpace::Global>(
+        const auto b_global_buf = make_dynamic_buffer<AddressSpaceEnum_t::Global>(
             p_b_grid, b_k_n0_n1_grid_desc.GetElementSpaceSize());
-        auto c_grid_buf = make_dynamic_buffer<AddressSpace::Global>(
+        auto c_grid_buf = make_dynamic_buffer<AddressSpaceEnum_t::Global>(
             p_c_grid, c_m0_m10_m11_n0_n10_n11_grid_desc.GetElementSpaceSize());
 
         const auto K = a_k_m0_m1_grid_desc.GetLength(I0);
@@ -373,7 +373,7 @@ struct GridwiseDynamicGemm_km_kn_mn_v1r2
         // A matrix blockwise copy
         auto a_blockwise_copy =
             BlockwiseDynamicTensorSliceTransfer_v4<BlockSize,
-                                                   InMemoryDataOperation::Set,
+                                                   InMemoryDataOperationEnum_t::Set,
                                                    Sequence<KPerBlock, 1, MPerBlockM1>,
                                                    ABlockTransferThreadSliceLengths_K_M0_M1,
                                                    ABlockTransferThreadClusterLengths_K_M0_M1,
@@ -399,7 +399,7 @@ struct GridwiseDynamicGemm_km_kn_mn_v1r2
         // B matrix blockwise copy
         auto b_blockwise_copy =
             BlockwiseDynamicTensorSliceTransfer_v4<BlockSize,
-                                                   InMemoryDataOperation::Set,
+                                                   InMemoryDataOperationEnum_t::Set,
                                                    Sequence<KPerBlock, 1, NPerBlockN1>,
                                                    BBlockTransferThreadSliceLengths_K_N0_N1,
                                                    BBlockTransferThreadClusterLengths_K_N0_N1,
@@ -429,21 +429,21 @@ struct GridwiseDynamicGemm_km_kn_mn_v1r2
         //     c_mtx[MPerBlockM1, NPerBlockN1] is distributed among threads, and saved in
         //       register
         const auto blockwise_gemm =
-            BlockwiseGemm_km_kn_m0m1n0n1_v2r2_pipeline_2x2<BlockSize,
-                                                           FloatAB,
-                                                           FloatAB,
-                                                           FloatAcc,
-                                                           decltype(a_k_m_block_desc),
-                                                           decltype(b_k_n_block_desc),
-                                                           M1PerThreadM111,
-                                                           N1PerThreadN111,
-                                                           KPerThread,
-                                                           M11N11ThreadClusterM1100,
-                                                           M11N11ThreadClusterN1100,
-                                                           M11N11ThreadClusterM1101,
-                                                           M11N11ThreadClusterN1101,
-                                                           M1PerThreadM111,
-                                                           N1PerThreadN111>{};
+            BlockwiseGemmDlops_km_kn_m0m1n0n1_v2r2_pipeline_2x2<BlockSize,
+                                                                FloatAB,
+                                                                FloatAB,
+                                                                FloatAcc,
+                                                                decltype(a_k_m_block_desc),
+                                                                decltype(b_k_n_block_desc),
+                                                                M1PerThreadM111,
+                                                                N1PerThreadN111,
+                                                                KPerThread,
+                                                                M11N11ThreadClusterM1100,
+                                                                M11N11ThreadClusterN1100,
+                                                                M11N11ThreadClusterM1101,
+                                                                M11N11ThreadClusterN1101,
+                                                                M1PerThreadM111,
+                                                                N1PerThreadN111>{};
         constexpr auto c_m10_m11_n10_n11_thread_tensor_lengths =
             decltype(blockwise_gemm)::GetCM0M1N0N1ThreadTensorLengths();
 
@@ -462,7 +462,7 @@ struct GridwiseDynamicGemm_km_kn_mn_v1r2
         FloatAB* p_b_block_double = p_shared_block + 2 * a_block_aligned_space_size;
 
         // register allocation for output
-        auto c_thread_buf = make_static_buffer<AddressSpace::Vgpr, FloatAcc>(
+        auto c_thread_buf = make_static_buffer<AddressSpaceEnum_t::Vgpr, FloatAcc>(
             c_m10_m11_n10_n11_thread_desc.GetElementSpaceSize());
 
         ThreadwiseDynamicTensorSliceSet_v1<FloatAcc,
@@ -487,17 +487,17 @@ struct GridwiseDynamicGemm_km_kn_mn_v1r2
         constexpr auto b_k_n0_n1_global_move_slice_window_iterator_hack =
             BGridMoveSliceWindowIteratorHacks{};
 
-        auto a_block_even_buf = make_dynamic_buffer<AddressSpace::Lds>(
+        auto a_block_even_buf = make_dynamic_buffer<AddressSpaceEnum_t::Lds>(
             p_a_block_double, a_k_m0_m1_block_desc.GetElementSpaceSize());
-        auto b_block_even_buf = make_dynamic_buffer<AddressSpace::Lds>(
+        auto b_block_even_buf = make_dynamic_buffer<AddressSpaceEnum_t::Lds>(
             p_b_block_double, b_k_n0_n1_block_desc.GetElementSpaceSize());
 
-        auto a_block_odd_buf =
-            make_dynamic_buffer<AddressSpace::Lds>(p_a_block_double + a_block_aligned_space_size,
-                                                   a_k_m0_m1_block_desc.GetElementSpaceSize());
-        auto b_block_odd_buf =
-            make_dynamic_buffer<AddressSpace::Lds>(p_b_block_double + b_block_aligned_space_size,
-                                                   b_k_n0_n1_block_desc.GetElementSpaceSize());
+        auto a_block_odd_buf = make_dynamic_buffer<AddressSpaceEnum_t::Lds>(
+            p_a_block_double + a_block_aligned_space_size,
+            a_k_m0_m1_block_desc.GetElementSpaceSize());
+        auto b_block_odd_buf = make_dynamic_buffer<AddressSpaceEnum_t::Lds>(
+            p_b_block_double + b_block_aligned_space_size,
+            b_k_n0_n1_block_desc.GetElementSpaceSize());
 
         // LDS double buffer: preload data into LDS
         {

composable_kernel/include/tensor_operation/gridwise_dynamic_gemm_v1r3.hpp → composable_kernel/include/tensor_operation/gridwise_dynamic_gemm_dlops_v1r3.hpp

@@ -5,7 +5,7 @@
 #include "dynamic_multi_index_transform_helper.hpp"
 #include "dynamic_tensor_descriptor.hpp"
 #include "dynamic_tensor_descriptor_helper.hpp"
-#include "blockwise_gemm_v2r3.hpp"
+#include "blockwise_gemm_dlops_v2r3.hpp"
 #include "blockwise_dynamic_tensor_slice_transfer_v2.hpp"
 #include "threadwise_dynamic_tensor_slice_transfer_v2.hpp"
 #include "threadwise_dynamic_tensor_slice_set.hpp"
@@ -26,7 +26,7 @@ __global__ void
 #if CK_USE_LAUNCH_BOUNDS
     __launch_bounds__(CK_MAX_THREAD_PER_BLOCK, CK_MIN_BLOCK_PER_CU)
 #endif
-        kernel_dynamic_gemm_v1r3(
+        kernel_dynamic_gemm_dlops_v1r3(
             const FloatAB* __restrict__ p_a_grid,
             const FloatAB* __restrict__ p_b_grid,
             FloatC* __restrict__ p_c_grid,
@@ -52,8 +52,8 @@ __global__ void
                       integral_constant<bool, HasDoubleTailKBlockLoop>{});
 }
 #elif CK_EXPERIMENTAL_PASS_TENSOR_DESCRIPTOR_BY_VOID_POINTER
-// pass tensor descriptor by __CONSTANT__ void pointer
-// __CONSTANT__ is needed to inform compiler void pointers in the kernel signature are pointing to
+// pass tensor descriptor by CONSTANT void pointer
+// CONSTANT is needed to inform compiler void pointers in the kernel signature are pointing to
 // non-modifiable parameter address space, so compiler can enable corresponding optimization
 template <typename GridwiseGemm,
           typename FloatAB,
@@ -68,16 +68,16 @@ __global__ void
 #if CK_USE_LAUNCH_BOUNDS
     __launch_bounds__(CK_MAX_THREAD_PER_BLOCK, CK_MIN_BLOCK_PER_CU)
 #endif
-        kernel_dynamic_gemm_v1r3(
+        kernel_dynamic_gemm_dlops_v1r3(
             const FloatAB* __restrict__ p_a_grid,
             const FloatAB* __restrict__ p_b_grid,
             FloatC* __restrict__ p_c_grid,
-            const void __CONSTANT__* p_a_k0_m0_m1_k1_grid_desc,
-            const void __CONSTANT__* p_b_k0_n0_n1_k1_grid_desc,
-            const void __CONSTANT__* p_c_m0_m10_m11_n0_n10_n11_grid_desc,
-            const void __CONSTANT__* p_c_blockid_to_m0_n0_block_cluster_adaptor)
+            const void CONSTANT* p_a_k0_m0_m1_k1_grid_desc,
+            const void CONSTANT* p_b_k0_n0_n1_k1_grid_desc,
+            const void CONSTANT* p_c_m0_m10_m11_n0_n10_n11_grid_desc,
+            const void CONSTANT* p_c_blockid_to_m0_n0_block_cluster_adaptor)
 {
-    // first cast void __CONSTANT__ void* to void*
+    // first cast void CONSTANT void* to void*
     // second cast void* to Desc*
     // the copy constructor of tensor descriptor doesn't take address_space(4)
     const auto a_k0_m0_m1_k1_grid_desc =
@@ -113,7 +113,7 @@ template <index_t BlockSize,
           typename FloatAB,
           typename FloatAcc,
           typename FloatC,
-          InMemoryDataOperation CGlobalMemoryDataOperation,
+          InMemoryDataOperationEnum_t CGlobalMemoryDataOperation,
           typename AK0MK1GridDesc,
           typename BK0NK1GridDesc,
           typename CMNGridDesc,
@@ -123,10 +123,8 @@ template <index_t BlockSize,
           index_t M1PerThreadM111,
           index_t N1PerThreadN111,
           index_t KPerThread,
-          index_t M11N11ThreadClusterM1100,
-          index_t M11N11ThreadClusterN1100,
-          index_t M11N11ThreadClusterM1101,
-          index_t M11N11ThreadClusterN1101,
+          typename M11N11ThreadClusterM110Xs,
+          typename M11N11ThreadClusterN110Xs,
           typename ABlockTransferThreadSliceLengths_K0_M0_M1_K1,
           typename ABlockTransferThreadClusterLengths_K0_M0_M1_K1,
           typename ABlockTransferThreadClusterArrangeOrder,
@@ -149,7 +147,7 @@ template <index_t BlockSize,
           typename CGridIteratorHacks,
           typename AGridMoveSliceWindowIteratorHacks,
           typename BGridMoveSliceWindowIteratorHacks>
-struct GridwiseDynamicGemm_km_kn_mn_v1r3
+struct GridwiseDynamicGemmDlops_km_kn_mn_v1r3
 {
     static constexpr auto I0 = Number<0>{};
     static constexpr auto I1 = Number<1>{};
@@ -277,9 +275,11 @@ struct GridwiseDynamicGemm_km_kn_mn_v1r3
         const auto N0 = N / N1;
 
         constexpr auto M11 =
-            Number<M11N11ThreadClusterM1100 * M11N11ThreadClusterM1101 * M1PerThreadM111>{};
+            Number<container_reduce(M11N11ThreadClusterM110Xs{}, math::multiplies_v2{}, I1) *
+                   M1PerThreadM111>{};
         constexpr auto N11 =
-            Number<M11N11ThreadClusterN1100 * M11N11ThreadClusterN1101 * N1PerThreadN111>{};
+            Number<container_reduce(M11N11ThreadClusterN110Xs{}, math::multiplies_v2{}, I1) *
+                   N1PerThreadN111>{};
 
         constexpr auto M10 = M1 / M11;
         constexpr auto N10 = N1 / N11;
@@ -333,11 +333,11 @@ struct GridwiseDynamicGemm_km_kn_mn_v1r3
         integral_constant<bool, HasMainKBlockLoop>,
         integral_constant<bool, HasDoubleTailKBlockLoop>)
     {
-        const auto a_global_buf = make_dynamic_buffer<AddressSpace::Global>(
+        const auto a_global_buf = make_dynamic_buffer<AddressSpaceEnum_t::Global>(
             p_a_grid, a_k0_m0_m1_k1_grid_desc.GetElementSpaceSize());
-        const auto b_global_buf = make_dynamic_buffer<AddressSpace::Global>(
+        const auto b_global_buf = make_dynamic_buffer<AddressSpaceEnum_t::Global>(
             p_b_grid, b_k0_n0_n1_k1_grid_desc.GetElementSpaceSize());
-        auto c_grid_buf = make_dynamic_buffer<AddressSpace::Global>(
+        auto c_grid_buf = make_dynamic_buffer<AddressSpaceEnum_t::Global>(
             p_c_grid, c_m0_m10_m11_n0_n10_n11_grid_desc.GetElementSpaceSize());
 
         // divide block work by [M, N]
@@ -383,7 +383,7 @@ struct GridwiseDynamicGemm_km_kn_mn_v1r3
         // A matrix blockwise copy
         auto a_blockwise_copy = BlockwiseDynamicTensorSliceTransfer_v4r1<
             BlockSize,
-            InMemoryDataOperation::Set,
+            InMemoryDataOperationEnum_t::Set,
             Sequence<KPerBlock, 1, MPerBlockM1, K1.value>,
             ABlockTransferThreadSliceLengths_K0_M0_M1_K1,
             ABlockTransferThreadClusterLengths_K0_M0_M1_K1,
@@ -407,7 +407,7 @@ struct GridwiseDynamicGemm_km_kn_mn_v1r3
         // B matrix blockwise copy
         auto b_blockwise_copy = BlockwiseDynamicTensorSliceTransfer_v4r1<
             BlockSize,
-            InMemoryDataOperation::Set,
+            InMemoryDataOperationEnum_t::Set,
             Sequence<KPerBlock, 1, NPerBlockN1, K1.value>,
             BBlockTransferThreadSliceLengths_K0_N0_N1_K1,
             BBlockTransferThreadClusterLengths_K0_N0_N1_K1,
@@ -435,7 +435,7 @@ struct GridwiseDynamicGemm_km_kn_mn_v1r3
         //     c_mtx[MPerBlockM1, NPerBlockN1] is distributed among threads, and saved in
         //       register
         const auto blockwise_gemm =
-            BlockwiseGemm_A_BK0_BM_BK1_B_BK0_BN_BK1_C_BM0_BM1_BN0_BN1_pipeline_BM0_2_BN0_2<
+            BlockwiseGemmDlops_A_BK0_BM_BK1_B_BK0_BN_BK1_C_BM0_BM1_BN0_BN1_pipeline_BM0_2_BN0_2<
                 BlockSize,
                 FloatAB,
                 FloatAB,
@@ -445,15 +445,13 @@ struct GridwiseDynamicGemm_km_kn_mn_v1r3
                 M1PerThreadM111,
                 N1PerThreadN111,
                 KPerThread,
-                M11N11ThreadClusterM1100,
-                M11N11ThreadClusterN1100,
-                M11N11ThreadClusterM1101,
-                M11N11ThreadClusterN1101,
+                M11N11ThreadClusterM110Xs,
+                M11N11ThreadClusterN110Xs,
                 M1PerThreadM111,
                 N1PerThreadN111>{};
 
         constexpr auto c_m10_m11_n10_n11_thread_tensor_lengths =
-            decltype(blockwise_gemm)::GetCM0M1N0N1ThreadTensorLengths();
+            decltype(blockwise_gemm)::GetCThreadTensorLengths_BM0_BM1_BN0_BN1();
 
         constexpr auto c_m10_m11_n10_n11_thread_desc =
             make_dynamic_naive_tensor_descriptor_packed_v2(
@@ -470,7 +468,7 @@ struct GridwiseDynamicGemm_km_kn_mn_v1r3
         FloatAB* p_b_block_double = p_shared_block + 2 * a_block_aligned_space_size;
 
         // register allocation for output
-        auto c_thread_buf = make_static_buffer<AddressSpace::Vgpr, FloatAcc>(
+        auto c_thread_buf = make_static_buffer<AddressSpaceEnum_t::Vgpr, FloatAcc>(
             c_m10_m11_n10_n11_thread_desc.GetElementSpaceSize());
 
         ThreadwiseDynamicTensorSliceSet_v1<FloatAcc,
@@ -484,17 +482,17 @@ struct GridwiseDynamicGemm_km_kn_mn_v1r3
         constexpr auto a_block_slice_copy_step = make_multi_index(KPerBlock, 0, 0, 0);
         constexpr auto b_block_slice_copy_step = make_multi_index(KPerBlock, 0, 0, 0);
 
-        auto a_block_even_buf = make_dynamic_buffer<AddressSpace::Lds>(
+        auto a_block_even_buf = make_dynamic_buffer<AddressSpaceEnum_t::Lds>(
             p_a_block_double, a_k0_m0_m1_k1_block_desc.GetElementSpaceSize());
-        auto b_block_even_buf = make_dynamic_buffer<AddressSpace::Lds>(
+        auto b_block_even_buf = make_dynamic_buffer<AddressSpaceEnum_t::Lds>(
             p_b_block_double, b_k0_n0_n1_k1_block_desc.GetElementSpaceSize());
 
-        auto a_block_odd_buf =
-            make_dynamic_buffer<AddressSpace::Lds>(p_a_block_double + a_block_aligned_space_size,
-                                                   a_k0_m0_m1_k1_block_desc.GetElementSpaceSize());
-        auto b_block_odd_buf =
-            make_dynamic_buffer<AddressSpace::Lds>(p_b_block_double + b_block_aligned_space_size,
-                                                   b_k0_n0_n1_k1_block_desc.GetElementSpaceSize());
+        auto a_block_odd_buf = make_dynamic_buffer<AddressSpaceEnum_t::Lds>(
+            p_a_block_double + a_block_aligned_space_size,
+            a_k0_m0_m1_k1_block_desc.GetElementSpaceSize());
+        auto b_block_odd_buf = make_dynamic_buffer<AddressSpaceEnum_t::Lds>(
+            p_b_block_double + b_block_aligned_space_size,
+            b_k0_n0_n1_k1_block_desc.GetElementSpaceSize());
 
         // LDS double buffer: preload data into LDS
         {
@@ -610,10 +608,12 @@ struct GridwiseDynamicGemm_km_kn_mn_v1r3
 
         // output: register to global memory
         {
-            constexpr index_t M11 =
-                M1PerThreadM111 * M11N11ThreadClusterM1100 * M11N11ThreadClusterM1101;
-            constexpr index_t N11 =
-                N1PerThreadN111 * M11N11ThreadClusterN1100 * M11N11ThreadClusterN1101;
+            constexpr auto M11 =
+                Number<container_reduce(M11N11ThreadClusterM110Xs{}, math::multiplies_v2{}, I1) *
+                       M1PerThreadM111>{};
+            constexpr auto N11 =
+                Number<container_reduce(M11N11ThreadClusterN110Xs{}, math::multiplies_v2{}, I1) *
+                       N1PerThreadN111>{};
 
             constexpr index_t M10 = MPerBlockM1 / M11;
             constexpr index_t N10 = NPerBlockN1 / N11;
@@ -631,7 +631,8 @@ struct GridwiseDynamicGemm_km_kn_mn_v1r3
                                Number<c_m10_m11_n10_n11_thread_tensor_lengths[I3]>{}));
 
             const auto c_m10_m11_n10_n11_thread_origin_idx_on_block =
-                blockwise_gemm.CalculateCM0M1N0N1ThreadOriginOnBlock(get_thread_local_1d_id());
+                blockwise_gemm.CalculateCThreadOriginOnBlock_BM0_BM1_BN0_BN1(
+                    get_thread_local_1d_id());
 
             ThreadwiseDynamicTensorSliceTransfer_v1r3<
                 FloatAcc,

composable_kernel/include/tensor_operation/gridwise_dynamic_gemm_v2.hpp → composable_kernel/include/tensor_operation/gridwise_dynamic_gemm_dlops_v2.hpp

@@ -7,7 +7,7 @@
 #include "dynamic_tensor_descriptor_helper.hpp"
 #include "blockwise_dynamic_tensor_slice_transfer.hpp"
 #include "threadwise_dynamic_tensor_slice_transfer.hpp"
-#include "blockwise_gemm_v3.hpp"
+#include "blockwise_gemm_dlops_v3.hpp"
 
 namespace ck {
 
@@ -15,7 +15,7 @@ template <index_t BlockSize,
           typename FloatAB,
           typename FloatAcc,
           typename FloatC,
-          InMemoryDataOperation CGlobalMemoryDataOperation,
+          InMemoryDataOperationEnum_t CGlobalMemoryDataOperation,
           typename AGlobalDesc,
           typename BGlobalDesc,
           typename CGlobalDesc,
@@ -47,7 +47,7 @@ template <index_t BlockSize,
           typename CGlobalIteratorHacks,
           typename AGlobalMoveSliceWindowIteratorHacks,
           typename BGlobalMoveSliceWindowIteratorHacks>
-struct GridwiseDynamicGemm_km_kn_mn_v3
+struct GridwiseDynamicGemmDlops_km_kn_mn_v3
 {
     __host__ __device__ static constexpr index_t GetSharedMemoryNumberOfByte()
     {
@@ -84,11 +84,11 @@ struct GridwiseDynamicGemm_km_kn_mn_v3
         constexpr auto I2 = Number<2>{};
         constexpr auto I3 = Number<3>{};
 
-        const auto a_global_buf = make_dynamic_buffer<AddressSpace::Global>(
+        const auto a_global_buf = make_dynamic_buffer<AddressSpaceEnum_t::Global>(
             p_a_global, a_e_k_global_desc.GetElementSpaceSize());
-        const auto b_global_buf = make_dynamic_buffer<AddressSpace::Global>(
+        const auto b_global_buf = make_dynamic_buffer<AddressSpaceEnum_t::Global>(
             p_b_global, b_e_n_ho_wo_global_desc.GetElementSpaceSize());
-        auto c_global_buf = make_dynamic_buffer<AddressSpace::Global>(
+        auto c_global_buf = make_dynamic_buffer<AddressSpaceEnum_t::Global>(
             p_c_global, c_k_n_ho_wo_global_desc.GetElementSpaceSize());
 
         constexpr auto E = EPerBlock * 3 * 3;
@@ -100,7 +100,7 @@ struct GridwiseDynamicGemm_km_kn_mn_v3
         const auto Ho = b_e_n_ho_wo_global_desc.GetLength(I2);
         const auto Wo = b_e_n_ho_wo_global_desc.GetLength(I3);
 
-        // divide block work by [M, N]
+// divide block work by [M, N]
 #if 0
         const auto k_block_work_num   = K / Number<KPerBlock>{};
         const auto ho_block_work_num  = Ho / Number<HoPerBlock>{};
@@ -152,19 +152,20 @@ struct GridwiseDynamicGemm_km_kn_mn_v3
             make_dynamic_naive_tensor_descriptor_packed_v2(make_tuple(
                 Number<KPerThread>{}, Number<1>{}, Number<HoPerThread>{}, Number<WoPerThread>{}));
 
-        auto blockwise_gemm = BlockwiseGemm_km_kn_m0m1n0n1_v3<BlockSize,
-                                                              FloatAB,
-                                                              FloatAB,
-                                                              FloatAcc,
-                                                              decltype(a_e_k_block_desc),
-                                                              decltype(b_e_n_ho_wo_block_desc),
-                                                              decltype(c_k_n_ho_wo_thread_desc),
-                                                              KPerThread,
-                                                              HoPerThread,
-                                                              WoPerThread,
-                                                              EPerThread,
-                                                              ABlockTransferSrcScalarPerVector,
-                                                              ABlockTransferDstScalarPerVector_K>{};
+        auto blockwise_gemm =
+            BlockwiseGemmDlops_km_kn_m0m1n0n1_v3<BlockSize,
+                                                 FloatAB,
+                                                 FloatAB,
+                                                 FloatAcc,
+                                                 decltype(a_e_k_block_desc),
+                                                 decltype(b_e_n_ho_wo_block_desc),
+                                                 decltype(c_k_n_ho_wo_thread_desc),
+                                                 KPerThread,
+                                                 HoPerThread,
+                                                 WoPerThread,
+                                                 EPerThread,
+                                                 ABlockTransferSrcScalarPerVector,
+                                                 ABlockTransferDstScalarPerVector_K>{};
 
         auto c_thread_mtx_index = blockwise_gemm.GetBeginOfThreadMatrixC(get_thread_local_1d_id());
 
@@ -184,7 +185,7 @@ struct GridwiseDynamicGemm_km_kn_mn_v3
         // A matrix blockwise copy
         auto a_blockwise_copy =
             BlockwiseDynamicTensorSliceTransfer_v4<BlockSize,
-                                                   InMemoryDataOperation::Set,
+                                                   InMemoryDataOperationEnum_t::Set,
                                                    Sequence<E, KPerBlock>,
                                                    ABlockTransferThreadSliceLengths_E_K,
                                                    ABlockTransferThreadClusterLengths_E_K,
@@ -225,11 +226,13 @@ struct GridwiseDynamicGemm_km_kn_mn_v3
             true>(b_e_n_ho_wo_global_desc,
                   make_multi_index(0, 0, ho_thread_data_on_global, wo_thread_data_on_global));
 
-        auto a_block_buf = make_dynamic_buffer<AddressSpace::Lds>(p_shared_block,
-                                                                  a_e_k_desc.GetElementSpaceSize());
+        auto a_block_buf = make_dynamic_buffer<AddressSpaceEnum_t::Lds>(
+            p_shared_block, a_e_k_desc.GetElementSpaceSize());
 
         // register allocation for output
-        StaticBuffer<AddressSpace::Vgpr, FloatAcc, c_k_n_ho_wo_thread_desc.GetElementSpaceSize()>
+        StaticBuffer<AddressSpaceEnum_t::Vgpr,
+                     FloatAcc,
+                     c_k_n_ho_wo_thread_desc.GetElementSpaceSize()>
             c_thread_buf;
 
         // initialize output thread tensor
@@ -252,7 +255,9 @@ struct GridwiseDynamicGemm_km_kn_mn_v3
             BGlobalMoveSliceWindowIteratorHacks{};
 
         // double regsiter buffer for b
-        StaticBuffer<AddressSpace::Vgpr, FloatAB, b_e_n_ho_wo_thread_desc.GetElementSpaceSize()>
+        StaticBuffer<AddressSpaceEnum_t::Vgpr,
+                     FloatAB,
+                     b_e_n_ho_wo_thread_desc.GetElementSpaceSize()>
             b_thread_even_buf, b_thread_odd_buf;
 
         // LDS double buffer: preload data

composable_kernel/include/tensor_operation/gridwise_dynamic_gemm_xdlops_v2r3.hpp

@@ -61,10 +61,10 @@ __global__ void
         kernel_dynamic_gemm_xdlops_v2r3(const FloatAB* __restrict__ p_a_grid,
                                         const FloatAB* __restrict__ p_b_grid,
                                         FloatC* __restrict__ p_c_grid,
-                                        const void __CONSTANT__* p_a_k0_m_k1_grid_desc,
-                                        const void __CONSTANT__* p_b_k0_n_k1_grid_desc,
-                                        const void __CONSTANT__* p_c_m0_m1_m2_n_grid_desc,
-                                        const void __CONSTANT__* p_c_block_cluster_adaptor)
+                                        const void CONSTANT* p_a_k0_m_k1_grid_desc,
+                                        const void CONSTANT* p_b_k0_n_k1_grid_desc,
+                                        const void CONSTANT* p_c_m0_m1_m2_n_grid_desc,
+                                        const void CONSTANT* p_c_block_cluster_adaptor)
 {
     constexpr index_t shared_block_size =
         GridwiseGemm::GetSharedMemoryNumberOfByte() / sizeof(FloatAB);
@@ -95,7 +95,7 @@ template <index_t BlockSize,
           typename FloatAB,
           typename FloatAcc,
           typename FloatC,
-          InMemoryDataOperation CGlobalMemoryDataOperation,
+          InMemoryDataOperationEnum_t CGlobalMemoryDataOperation,
           typename AK0MK1GridDesc,
           typename BK0NK1GridDesc,
           typename CMNGridDesc,
@@ -274,11 +274,11 @@ struct GridwiseDynamicGemm_k0mk1_k0nk1_mn_xdlops_v2r3
         constexpr auto I2 = Number<2>{};
         constexpr auto I3 = Number<3>{};
 
-        const auto a_grid_buf = make_dynamic_buffer<AddressSpace::Global>(
+        const auto a_grid_buf = make_dynamic_buffer<AddressSpaceEnum_t::Global>(
             p_a_grid, a_k0_m_k1_grid_desc.GetElementSpaceSize());
-        const auto b_grid_buf = make_dynamic_buffer<AddressSpace::Global>(
+        const auto b_grid_buf = make_dynamic_buffer<AddressSpaceEnum_t::Global>(
             p_b_grid, b_k0_n_k1_grid_desc.GetElementSpaceSize());
-        auto c_grid_buf = make_dynamic_buffer<AddressSpace::Global>(
+        auto c_grid_buf = make_dynamic_buffer<AddressSpaceEnum_t::Global>(
             p_c_grid, c_m0_m1_m2_n_grid_desc.GetElementSpaceSize());
 
         const auto K0 = a_k0_m_k1_grid_desc.GetLength(I0);
@@ -312,7 +312,7 @@ struct GridwiseDynamicGemm_k0mk1_k0nk1_mn_xdlops_v2r3
         // A matrix blockwise copy
         auto a_blockwise_copy =
             BlockwiseDynamicTensorSliceTransfer_v4<BlockSize,
-                                                   InMemoryDataOperation::Set,
+                                                   InMemoryDataOperationEnum_t::Set,
                                                    Sequence<KPerBlock, MPerBlock, K1>,
                                                    ABlockTransferThreadSliceLengths_K0_M_K1,
                                                    ABlockTransferThreadClusterLengths_K0_M_K1,
@@ -339,7 +339,7 @@ struct GridwiseDynamicGemm_k0mk1_k0nk1_mn_xdlops_v2r3
         // B matrix blockwise copy
         auto b_blockwise_copy =
             BlockwiseDynamicTensorSliceTransfer_v4<BlockSize,
-                                                   InMemoryDataOperation::Set,
+                                                   InMemoryDataOperationEnum_t::Set,
                                                    Sequence<KPerBlock, NPerBlock, K1>,
                                                    BBlockTransferThreadSliceLengths_K0_N_K1,
                                                    BBlockTransferThreadClusterLengths_K0_N_K1,
@@ -413,7 +413,7 @@ struct GridwiseDynamicGemm_k0mk1_k0nk1_mn_xdlops_v2r3
         constexpr auto c_mr_nr_blk_desc = make_dynamic_naive_tensor_descriptor_packed_v2(
             make_tuple(Number<MRepeat>{}, Number<NRepeat>{}));
 
-        StaticBuffer<AddressSpace::Vgpr,
+        StaticBuffer<AddressSpaceEnum_t::Vgpr,
                      vector_type<FloatAcc, BlkSize>,
                      c_mr_nr_blk_desc.GetElementSpaceSize()>
             c_thread_buf;
@@ -442,9 +442,9 @@ struct GridwiseDynamicGemm_k0mk1_k0nk1_mn_xdlops_v2r3
         constexpr auto b_k0_n_k1_grid_move_slice_window_iterator_hack =
             BGridMoveSliceWindowIteratorHacks{};
 
-        auto a_block_buf = make_dynamic_buffer<AddressSpace::Lds>(
+        auto a_block_buf = make_dynamic_buffer<AddressSpaceEnum_t::Lds>(
             p_a_block, a_k0_m_k1_block_desc.GetElementSpaceSize());
-        auto b_block_buf = make_dynamic_buffer<AddressSpace::Lds>(
+        auto b_block_buf = make_dynamic_buffer<AddressSpaceEnum_t::Lds>(
             p_b_block, b_k0_n_k1_block_desc.GetElementSpaceSize());
 
         // preload data into LDS
@@ -515,7 +515,7 @@ struct GridwiseDynamicGemm_k0mk1_k0nk1_mn_xdlops_v2r3
                                                                           Number<M2>{},
                                                                           Number<1>{}));
 
-            StaticBuffer<AddressSpace::Vgpr, FloatC, c_m0_m1_m2_n_thread_desc.GetElementSpaceSize()>
+            StaticBuffer<AddressSpaceEnum_t::Vgpr, FloatC, c_m0_m1_m2_n_thread_desc.GetElementSpaceSize()>
                 c_blk_buf_;
 
             static_for<0, MRepeat, 1>{}([&](auto mr_i) {
@@ -585,7 +585,7 @@ struct GridwiseDynamicGemm_k0mk1_k0nk1_mn_xdlops_v2r3
                 make_dynamic_naive_tensor_descriptor_packed_v2(make_tuple(
                     I1, I1, I1, I1, Number<M0>{}, Number<1>{}, Number<M2>{}, Number<1>{}));
 
-            StaticBuffer<AddressSpace::Vgpr, FloatC, BlkSize> c_blk_buf_;
+            StaticBuffer<AddressSpaceEnum_t::Vgpr, FloatC, BlkSize> c_blk_buf_;
 
             // calculate origin of thread output tensor on global memory
             //     blockwise GEMM c matrix starting index

composable_kernel/include/tensor_operation/threadwise_contraction.hpp → composable_kernel/include/tensor_operation/threadwise_contraction_dlops.hpp

-#ifndef CK_THREADWISE_CONTRACTION_HPP
-#define CK_THREADWISE_CONTRACTION_HPP
+#ifndef CK_THREADWISE_CONTRACTION_DLOPS_HPP
+#define CK_THREADWISE_CONTRACTION_DLOPS_HPP
 
 #include "common_header.hpp"
 #include "math.hpp"
@@ -25,9 +25,9 @@ template <typename FloatA,
                                       BThreadDesc_TK0_TN0_TN1_TK1::IsKnownAtCompileTime() &&
                                       CThreadDesc_TM0_TM1_TN0_TN1::IsKnownAtCompileTime(),
                                   bool>::type = false>
-struct ThreadwiseGemm_km0m1_kn0n1_m0m1n0n1
+struct ThreadwiseGemmDlops_km0m1_kn0n1_m0m1n0n1
 {
-    __device__ constexpr ThreadwiseGemm_km0m1_kn0n1_m0m1n0n1()
+    __device__ constexpr ThreadwiseGemmDlops_km0m1_kn0n1_m0m1n0n1()
     {
         static_assert(AThreadDesc_TK0_TM0_TM1_TK1::IsKnownAtCompileTime() &&
                           BThreadDesc_TK0_TN0_TN1_TK1::IsKnownAtCompileTime() &&
@@ -71,8 +71,6 @@ struct ThreadwiseGemm_km0m1_kn0n1_m0m1n0n1
 
         constexpr auto I0 = Number<0>{};
         constexpr auto I1 = Number<1>{};
-        constexpr auto I2 = Number<2>{};
-        constexpr auto I3 = Number<3>{};
 
         constexpr auto TK  = TKLengths{}[I0];
         constexpr auto TM0 = TMLengths{}[I0];
@@ -131,9 +129,9 @@ template <typename FloatA,
                                       BThreadDesc_TK0_TN0_TN1_TK1::IsKnownAtCompileTime() &&
                                       CThreadDesc_TM0_TM1_TN0_TN1::IsKnownAtCompileTime(),
                                   bool>::type = false>
-struct ThreadwiseContraction_A_TK0_TM0_TM1_TK1_B_TK0_TN0_TN1_TK1_C_TM0_TM1_TN0_TN1
+struct ThreadwiseContractionDlops_A_TK0_TM0_TM1_TK1_B_TK0_TN0_TN1_TK1_C_TM0_TM1_TN0_TN1
 {
-    __device__ constexpr ThreadwiseContraction_A_TK0_TM0_TM1_TK1_B_TK0_TN0_TN1_TK1_C_TM0_TM1_TN0_TN1()
+    __device__ constexpr ThreadwiseContractionDlops_A_TK0_TM0_TM1_TK1_B_TK0_TN0_TN1_TK1_C_TM0_TM1_TN0_TN1()
     {
         static_assert(AThreadDesc_TK0_TM0_TM1_TK1::IsKnownAtCompileTime() &&
                           BThreadDesc_TK0_TN0_TN1_TK1::IsKnownAtCompileTime() &&
@@ -177,8 +175,6 @@ struct ThreadwiseContraction_A_TK0_TM0_TM1_TK1_B_TK0_TN0_TN1_TK1_C_TM0_TM1_TN0_T
 
         constexpr auto I0 = Number<0>{};
         constexpr auto I1 = Number<1>{};
-        constexpr auto I2 = Number<2>{};
-        constexpr auto I3 = Number<3>{};
 
         constexpr index_t TK0 = TKLengths{}[I0];
         constexpr index_t TK1 = TKLengths{}[I1];

composable_kernel/include/tensor_operation/threadwise_dynamic_tensor_slice_transfer_v2.hpp

@@ -13,7 +13,7 @@ namespace ck {
 //   3. src_slice_origin and dst_slice_origin are not known at compile-time,
 //   4. Use thread buffer
 template <typename SliceLengths,
-          InMemoryDataOperation DstInMemOp,
+          InMemoryDataOperationEnum_t DstInMemOp,
           typename SrcData,
           typename DstData,
           typename SrcDesc,
@@ -77,8 +77,8 @@ struct ThreadwiseDynamicTensorSliceTransfer_v3r1
                             const SrcBuffer& src_buf,
                             const SrcIteratorHacks& src_iterator_hacks)
     {
-        static_assert(SrcBuffer::GetAddressSpace() == AddressSpace::Global or
-                          SrcBuffer::GetAddressSpace() == AddressSpace::Lds,
+        static_assert(SrcBuffer::GetAddressSpace() == AddressSpaceEnum_t::Global or
+                          SrcBuffer::GetAddressSpace() == AddressSpaceEnum_t::Lds,
                       "wrong!");
 
         static_assert(is_same<remove_cv_t<remove_reference_t<typename SrcBuffer::type>>,
@@ -140,9 +140,9 @@ struct ThreadwiseDynamicTensorSliceTransfer_v3r1
         static_ford<decltype(ordered_src_access_lengths)>{}([&](auto ordered_src_access_idx) {
             // judge move forward or move backward
             constexpr auto forward_sweep = [&]() {
-                StaticallyIndexedArray<bool, nDim> forward_sweep;
+                StaticallyIndexedArray<bool, nDim> forward_sweep_;
 
-                forward_sweep(I0) = true;
+                forward_sweep_(I0) = true;
 
                 static_for<1, nDim, 1>{}([&](auto i) {
                     index_t tmp = ordered_src_access_idx[I0];
@@ -151,10 +151,10 @@ struct ThreadwiseDynamicTensorSliceTransfer_v3r1
                         tmp = tmp * ordered_src_access_lengths[j] + ordered_src_access_idx[j];
                     });
 
-                    forward_sweep(i) = tmp % 2 == 0;
+                    forward_sweep_(i) = tmp % 2 == 0;
                 });
 
-                return forward_sweep;
+                return forward_sweep_;
             }();
 
             // calculate src data index
@@ -167,11 +167,8 @@ struct ThreadwiseDynamicTensorSliceTransfer_v3r1
                                                             ordered_src_access_idx[i];
                 });
 
-                auto src_data_idx =
-                    container_reorder_given_old2new(ordered_idx, src_dim_access_order) *
-                    src_vector_tensor_lengths;
-
-                return src_data_idx;
+                return container_reorder_given_old2new(ordered_idx, src_dim_access_order) *
+                       src_vector_tensor_lengths;
             }();
 
             vector_type_maker_t<SrcData, src_vector_desc.GetElementSpaceSize()> src_vector;
@@ -201,18 +198,18 @@ struct ThreadwiseDynamicTensorSliceTransfer_v3r1
 
             constexpr auto move_on_dim = [&]() constexpr
             {
-                StaticallyIndexedArray<bool, nDim> move_on_dim;
+                StaticallyIndexedArray<bool, nDim> move_on_dim_;
 
                 static_for<0, nDim, 1>{}([&](auto i) {
-                    move_on_dim(i) = ordered_src_access_idx[i] < ordered_src_access_lengths[i] - 1;
+                    move_on_dim_(i) = ordered_src_access_idx[i] < ordered_src_access_lengths[i] - 1;
 
                     static_for<i + 1, nDim, 1>{}([&](auto j) {
-                        move_on_dim(i) &=
+                        move_on_dim_(i) &=
                             ordered_src_access_idx[j] == ordered_src_access_lengths[j] - 1;
                     });
                 });
 
-                return move_on_dim;
+                return move_on_dim_;
             }
             ();
 
@@ -249,8 +246,8 @@ struct ThreadwiseDynamicTensorSliceTransfer_v3r1
                              DstBuffer& dst_buf,
                              const DstIteratorHacks& dst_iterator_hacks)
     {
-        static_assert(DstBuffer::GetAddressSpace() == AddressSpace::Global or
-                          DstBuffer::GetAddressSpace() == AddressSpace::Lds,
+        static_assert(DstBuffer::GetAddressSpace() == AddressSpaceEnum_t::Global or
+                          DstBuffer::GetAddressSpace() == AddressSpaceEnum_t::Lds,
                       "wrong!");
 
         static_assert(is_same<remove_cv_t<remove_reference_t<typename DstBuffer::type>>,
@@ -316,9 +313,9 @@ struct ThreadwiseDynamicTensorSliceTransfer_v3r1
         static_ford<decltype(ordered_dst_access_lengths)>{}([&](auto ordered_dst_access_idx) {
             // judge move forward or move backward
             constexpr auto forward_sweep = [&]() {
-                StaticallyIndexedArray<bool, nDim> forward_sweep;
+                StaticallyIndexedArray<bool, nDim> forward_sweep_;
 
-                forward_sweep(I0) = true;
+                forward_sweep_(I0) = true;
 
                 static_for<1, nDim, 1>{}([&](auto i) {
                     index_t tmp = ordered_dst_access_idx[I0];
@@ -327,10 +324,10 @@ struct ThreadwiseDynamicTensorSliceTransfer_v3r1
                         tmp = tmp * ordered_dst_access_lengths[j] + ordered_dst_access_idx[j];
                     });
 
-                    forward_sweep(i) = tmp % 2 == 0;
+                    forward_sweep_(i) = tmp % 2 == 0;
                 });
 
-                return forward_sweep;
+                return forward_sweep_;
             }();
 
             // calculate dst data index
@@ -343,11 +340,8 @@ struct ThreadwiseDynamicTensorSliceTransfer_v3r1
                                                             ordered_dst_access_idx[i];
                 });
 
-                auto dst_data_idx =
-                    container_reorder_given_old2new(ordered_idx, dst_dim_access_order) *
-                    dst_vector_tensor_lengths;
-
-                return dst_data_idx;
+                return container_reorder_given_old2new(ordered_idx, dst_dim_access_order) *
+                       dst_vector_tensor_lengths;
             }();
 
             vector_type_maker_t<DstData, dst_vector_desc.GetElementSpaceSize()> dst_vector;
@@ -379,18 +373,18 @@ struct ThreadwiseDynamicTensorSliceTransfer_v3r1
 
             constexpr auto move_on_dim = [&]() constexpr
             {
-                StaticallyIndexedArray<bool, nDim> move_on_dim;
+                StaticallyIndexedArray<bool, nDim> move_on_dim_;
 
                 static_for<0, nDim, 1>{}([&](auto i) {
-                    move_on_dim(i) = ordered_dst_access_idx[i] < ordered_dst_access_lengths[i] - 1;
+                    move_on_dim_(i) = ordered_dst_access_idx[i] < ordered_dst_access_lengths[i] - 1;
 
                     static_for<i + 1, nDim, 1>{}([&](auto j) {
-                        move_on_dim(i) &=
+                        move_on_dim_(i) &=
                             ordered_dst_access_idx[j] == ordered_dst_access_lengths[j] - 1;
                     });
                 });
 
-                return move_on_dim;
+                return move_on_dim_;
             }
             ();
 
@@ -463,9 +457,9 @@ struct ThreadwiseDynamicTensorSliceTransfer_v3r1
 
         // judge move forward or move backward during the last iteration
         constexpr auto forward_sweep = [&]() {
-            StaticallyIndexedArray<bool, nDim> forward_sweep;
+            StaticallyIndexedArray<bool, nDim> forward_sweep_;
 
-            forward_sweep(I0) = true;
+            forward_sweep_(I0) = true;
 
             static_for<1, nDim, 1>{}([&](auto i) {
                 index_t tmp = ordered_src_access_lengths[I0] - 1;
@@ -474,10 +468,10 @@ struct ThreadwiseDynamicTensorSliceTransfer_v3r1
                     tmp = tmp * ordered_src_access_lengths[j] + ordered_src_access_lengths[j] - 1;
                 });
 
-                forward_sweep(i) = tmp % 2 == 0;
+                forward_sweep_(i) = tmp % 2 == 0;
             });
 
-            return forward_sweep;
+            return forward_sweep_;
         }();
 
         // calculate src data index after last iteration in RunRead(), if it has not being reset by
@@ -489,19 +483,17 @@ struct ThreadwiseDynamicTensorSliceTransfer_v3r1
                 ordered_idx(i) = forward_sweep[i] ? ordered_src_access_lengths[i] - 1 : 0;
             });
 
-            auto src_data_idx = container_reorder_given_old2new(ordered_idx, src_dim_access_order) *
-                                src_vector_tensor_lengths;
-
-            return src_data_idx;
+            return container_reorder_given_old2new(ordered_idx, src_dim_access_order) *
+                   src_vector_tensor_lengths;
         }();
 
         //
         constexpr auto reset_src_data_step = [&]() {
-            Index reset_src_data_step;
+            Index reset_src_data_step_;
 
-            static_for<0, nDim, 1>{}([&](auto i) { reset_src_data_step(i) = -src_data_idx[i]; });
+            static_for<0, nDim, 1>{}([&](auto i) { reset_src_data_step_(i) = -src_data_idx[i]; });
 
-            return reset_src_data_step;
+            return reset_src_data_step_;
         }();
 
         return reset_src_data_step;
@@ -520,9 +512,9 @@ struct ThreadwiseDynamicTensorSliceTransfer_v3r1
 
         // judge move forward or move backward during the last iteration
         constexpr auto forward_sweep = [&]() {
-            StaticallyIndexedArray<bool, nDim> forward_sweep;
+            StaticallyIndexedArray<bool, nDim> forward_sweep_;
 
-            forward_sweep(I0) = true;
+            forward_sweep_(I0) = true;
 
             static_for<1, nDim, 1>{}([&](auto i) {
                 index_t tmp = ordered_dst_access_lengths[I0] - 1;
@@ -531,10 +523,10 @@ struct ThreadwiseDynamicTensorSliceTransfer_v3r1
                     tmp = tmp * ordered_dst_access_lengths[j] + ordered_dst_access_lengths[j] - 1;
                 });
 
-                forward_sweep(i) = tmp % 2 == 0;
+                forward_sweep_(i) = tmp % 2 == 0;
             });
 
-            return forward_sweep;
+            return forward_sweep_;
         }();
 
         // calculate dst data index after last iteration in RunWrite(), if it has not being reset by
@@ -546,19 +538,17 @@ struct ThreadwiseDynamicTensorSliceTransfer_v3r1
                 ordered_idx(i) = forward_sweep[i] ? ordered_dst_access_lengths[i] - 1 : 0;
             });
 
-            auto dst_data_idx = container_reorder_given_old2new(ordered_idx, dst_dim_access_order) *
-                                dst_vector_tensor_lengths;
-
-            return dst_data_idx;
+            return container_reorder_given_old2new(ordered_idx, dst_dim_access_order) *
+                   dst_vector_tensor_lengths;
         }();
 
         //
         constexpr auto reset_dst_data_step = [&]() {
-            Index reset_dst_data_step;
+            Index reset_dst_data_step_;
 
-            static_for<0, nDim, 1>{}([&](auto i) { reset_dst_data_step(i) = -dst_data_idx[i]; });
+            static_for<0, nDim, 1>{}([&](auto i) { reset_dst_data_step_(i) = -dst_data_idx[i]; });
 
-            return reset_dst_data_step;
+            return reset_dst_data_step_;
         }();
 
         return reset_dst_data_step;
@@ -620,7 +610,7 @@ struct ThreadwiseDynamicTensorSliceTransfer_v3r1
 
     static constexpr auto buffer_size_ = buffer_desc_.GetElementSpaceSize();
 
-    StaticBuffer<AddressSpace::Vgpr, SrcData, buffer_size_> buffer_;
+    StaticBuffer<AddressSpaceEnum_t::Vgpr, SrcData, buffer_size_> buffer_;
 
     SrcCoord src_coord_;
     DstCoord dst_coord_;

composable_kernel/include/tensor_operation/threadwise_gemm_v3.hpp → composable_kernel/include/tensor_operation/threadwise_gemm_dlops_v3.hpp

-#ifndef CK_THREADWISE_GEMM_V3_HPP
-#define CK_THREADWISE_GEMM_V3_HPP
+#ifndef CK_THREADWISE_GEMM_DLOPS_V3_HPP
+#define CK_THREADWISE_GEMM_DLOPS_V3_HPP
 
 #include "common_header.hpp"
 #include "math.hpp"
@@ -22,7 +22,7 @@ template <typename FloatA,
           typename std::enable_if<ADesc::IsKnownAtCompileTime() && BDesc::IsKnownAtCompileTime() &&
                                       CDesc::IsKnownAtCompileTime(),
                                   bool>::type = false>
-struct ThreadwiseGemm_km_kn_mn_v3
+struct ThreadwiseGemmDlops_km_kn_mn_v3
 {
     template <typename ABuffer,
               typename AOriginIdx,