added intermediate class for qloop

include/ck/tensor_operation/gpu/device/device_batched_gemm_softmax_gemm_permute.hpp

@@ -122,6 +122,67 @@ struct DeviceBatchedMultiheadAttentionForward : public BaseOperator
     virtual std::unique_ptr<BaseInvoker> MakeInvokerPointer() = 0;
 };
 
+template <index_t NumDimG,
+          index_t NumDimM,
+          index_t NumDimN,
+          index_t NumDimK,
+          index_t NumDimO,
+          typename InputDataType,
+          typename OutputDataType,
+          typename ZDataType,
+          typename LSEDataType,
+          typename Acc0BiasDataType,
+          typename Acc1BiasDataType,
+          typename AElementwiseOperation,
+          typename BElementwiseOperation,
+          typename AccElementwiseOperation,
+          typename B1ElementwiseOperation,
+          typename CElementwiseOperation,
+          MaskingSpecialization MaskingSpec>
+struct DeviceBatchedMultiheadAttentionBackward : public BaseOperator
+{
+    static constexpr index_t NumAcc0Bias = Acc0BiasDataType::Size();
+    static constexpr index_t NumAcc1Bias = Acc1BiasDataType::Size();
+
+    virtual std::unique_ptr<BaseArgument> MakeArgumentPointer(
+        const void* p_a,
+        const void* p_b,
+        void* p_z,
+        const void* p_b1,
+        const void* p_c,
+        const void* p_lse,
+        const void* p_ygrad_grid,
+        void* p_qgrad_grid,
+        void* p_kgrad_grid,
+        void* p_vgrad_grid,
+        const std::array<void*, NumAcc0Bias> p_acc0_biases,
+        const std::array<void*, NumAcc1Bias> p_acc1_biases,
+        const std::vector<index_t>& a_gs_ms_ks_lengths,
+        const std::vector<index_t>& a_gs_ms_ks_strides,
+        const std::vector<index_t>& b_gs_ns_ks_lengths,
+        const std::vector<index_t>& b_gs_ns_ks_strides,
+        const std::vector<index_t>& z_gs_ms_ns_lengths,
+        const std::vector<index_t>& z_gs_ms_ns_strides,
+        const std::vector<index_t>& b1_gs_gemm1ns_gemm1ks_lengths, // b1_gs_os_ns_lengths
+        const std::vector<index_t>& b1_gs_gemm1ns_gemm1ks_strides, // b1_gs_os_ns_strides
+        const std::vector<index_t>& c_gs_ms_gemm1ns_lengths,       // c_gs_ms_os_lengths
+        const std::vector<index_t>& c_gs_ms_gemm1ns_strides,       // c_gs_ms_os_strides
+        const std::vector<index_t>& lse_gs_ms_lengths,             // lse_gs_ms_lengths
+        const std::array<std::vector<ck::index_t>, NumAcc0Bias> acc0_biases_gs_ms_ns_lengths,
+        const std::array<std::vector<ck::index_t>, NumAcc0Bias> acc0_biases_gs_ms_ns_strides,
+        const std::array<std::vector<ck::index_t>, NumAcc1Bias> acc1_biases_gs_ms_gemm1ns_lengths,
+        const std::array<std::vector<ck::index_t>, NumAcc1Bias> acc1_biases_gs_ms_gemm1ns_strides,
+        AElementwiseOperation a_element_op,
+        BElementwiseOperation b_element_op,
+        AccElementwiseOperation acc_element_op,
+        B1ElementwiseOperation b1_element_op,
+        CElementwiseOperation c_element_op,
+        float p_dropout,
+        std::tuple<unsigned long long, unsigned long long> seeds) = 0;
+
+    virtual std::unique_ptr<BaseInvoker> MakeInvokerPointer() = 0;
+};
+
 } // namespace device
 } // namespace tensor_operation
 } // namespace ck

include/ck/tensor_operation/gpu/device/impl/device_batched_mha_bwd_xdl_cshuffle_qloop_v1.hpp

@@ -12,6 +12,7 @@
 #include "ck/tensor_description/tensor_descriptor.hpp"
 #include "ck/tensor_description/tensor_descriptor_helper.hpp"
 #include "ck/tensor_operation/gpu/device/device_base.hpp"
+#include "ck/tensor_operation/gpu/device/device_batched_gemm_softmax_gemm_permute.hpp"
 #include "ck/tensor_operation/gpu/device/gemm_specialization.hpp"
 #include "ck/tensor_operation/gpu/device/masking_specialization.hpp"
 #include "ck/tensor_operation/gpu/device/matrix_padder.hpp"
@@ -287,7 +288,23 @@ template <index_t NumDimG,
           bool Deterministic,
           LoopScheduler LoopSched = LoopScheduler::Default>
 struct DeviceBatchedMultiheadAttentionBackward_Qloop_Xdl_CShuffle_V1
-    : public BaseOperator // TODO inherit atten bwd op once API stablizes
+    : public DeviceBatchedMultiheadAttentionBackward<NumDimG,
+                                                     NumDimM,
+                                                     NumDimN,
+                                                     NumDimK,
+                                                     NumDimO,
+                                                     InputDataType,
+                                                     OutputDataType,
+                                                     ZDataType,
+                                                     LSEDataType,
+                                                     Acc0BiasDataType,
+                                                     Acc1BiasDataType,
+                                                     AElementwiseOperation,
+                                                     BElementwiseOperation,
+                                                     AccElementwiseOperation,
+                                                     B1ElementwiseOperation,
+                                                     CElementwiseOperation,
+                                                     MaskingSpec>
 {
     static_assert(NumDimG > 0 && NumDimM > 0 && NumDimN > 0 && NumDimK > 0 && NumDimO > 0,
                   "Number of dimension must be greater than 0");
@@ -1223,7 +1240,7 @@ struct DeviceBatchedMultiheadAttentionBackward_Qloop_Xdl_CShuffle_V1
         B1ElementwiseOperation b1_element_op,
         CElementwiseOperation c_element_op,
         float p_drop,
-        std::tuple<unsigned long long, unsigned long long> seeds) // override
+        std::tuple<unsigned long long, unsigned long long> seeds) override
     {
         return std::make_unique<Argument>(static_cast<const InputDataType*>(p_a),
                                           static_cast<const InputDataType*>(p_b),
@@ -1262,7 +1279,7 @@ struct DeviceBatchedMultiheadAttentionBackward_Qloop_Xdl_CShuffle_V1
     }
 
     // polymorphic
-    std::unique_ptr<BaseInvoker> MakeInvokerPointer() // override
+    std::unique_ptr<BaseInvoker> MakeInvokerPointer() override
     {
         return std::make_unique<Invoker>(Invoker{});
     }

include/ck/tensor_operation/gpu/device/impl/device_batched_mha_bwd_xdl_cshuffle_qloop_v2.hpp

@@ -294,7 +294,23 @@ template <index_t NumDimG,
           bool Deterministic,
           LoopScheduler LoopSched = LoopScheduler::Default>
 struct DeviceBatchedMultiheadAttentionBackward_Qloop_Xdl_CShuffle_V2
-    : public BaseOperator // TODO inherit atten bwd op once API stablizes
+    : public DeviceBatchedMultiheadAttentionBackward<NumDimG,
+                                                     NumDimM,
+                                                     NumDimN,
+                                                     NumDimK,
+                                                     NumDimO,
+                                                     InputDataType,
+                                                     OutputDataType,
+                                                     ZDataType,
+                                                     LSEDataType,
+                                                     Acc0BiasDataType,
+                                                     Acc1BiasDataType,
+                                                     AElementwiseOperation,
+                                                     BElementwiseOperation,
+                                                     AccElementwiseOperation,
+                                                     B1ElementwiseOperation,
+                                                     CElementwiseOperation,
+                                                     MaskingSpec>
 {
     static_assert(NumDimG > 0 && NumDimM > 0 && NumDimN > 0 && NumDimK > 0 && NumDimO > 0,
                   "Number of dimension must be greater than 0");
@@ -1257,7 +1273,7 @@ struct DeviceBatchedMultiheadAttentionBackward_Qloop_Xdl_CShuffle_V2
         B1ElementwiseOperation b1_element_op,
         CElementwiseOperation c_element_op,
         float p_drop,
-        std::tuple<unsigned long long, unsigned long long> seeds) // override
+        std::tuple<unsigned long long, unsigned long long> seeds) override
     {
         return std::make_unique<Argument>(static_cast<const InputDataType*>(p_a),
                                           static_cast<const InputDataType*>(p_b),
@@ -1296,7 +1312,7 @@ struct DeviceBatchedMultiheadAttentionBackward_Qloop_Xdl_CShuffle_V2
     }
 
     // polymorphic
-    std::unique_ptr<BaseInvoker> MakeInvokerPointer() // override
+    std::unique_ptr<BaseInvoker> MakeInvokerPointer() override
     {
         return std::make_unique<Invoker>(Invoker{});
     }