clean up

example/31_batched_gemm_gemm/batched_gemm_gemm_xdl_fp16.cpp

@@ -216,21 +216,19 @@ int main(int argc, char* argv[])
     {
     case 0: break;
     case 1:
-        a_m_k.GenerateTensorValue(GeneratorTensor_3<ADataType>{-0.5, 0.5});
-        b0_k_n.GenerateTensorValue(GeneratorTensor_3<B0DataType>{-0.5, 0.5});
-        b1_n_o.GenerateTensorValue(GeneratorTensor_3<B1DataType>{-0.5, 0.5});
+        a_m_k.GenerateTensorValue(GeneratorTensor_2<ADataType>{-5, 5});
+        b0_k_n.GenerateTensorValue(GeneratorTensor_2<B0DataType>{-5, 5});
+        b1_n_o.GenerateTensorValue(GeneratorTensor_2<B1DataType>{-5, 5});
         break;
     case 2:
-        a_m_k.GenerateTensorValue(GeneratorTensor_1<ADataType>{1});
-        b0_k_n.GenerateTensorValue(GeneratorTensor_1<B0DataType>{1});
-        b1_n_o.GenerateTensorValue(GeneratorTensor_2<B1DataType>{-5, 5});
+        a_m_k.GenerateTensorValue(GeneratorTensor_3<ADataType>{0.0, 1.0});
+        b0_k_n.GenerateTensorValue(GeneratorTensor_3<B0DataType>{0.0, 1.0});
+        b1_n_o.GenerateTensorValue(GeneratorTensor_3<B1DataType>{-0.5, 0.5});
         break;
     default:
         a_m_k.GenerateTensorValue(GeneratorTensor_1<ADataType>{1});
-        // b0_k_n.GenerateTensorValue(GeneratorTensor_1<B0DataType>{1});
         b0_k_n.GenerateTensorValue(GeneratorTensor_Sequential<1>{});
         b1_n_o.GenerateTensorValue(GeneratorTensor_Diagonal<B1DataType>{});
-        // b1_n_o.GenerateTensorValue(GeneratorTensor_Sequential<0>{});
     }
 
     DeviceMem a_m_k_device_buf(sizeof(ADataType) * a_m_k.mDesc.GetElementSpace());
@@ -308,15 +306,6 @@ int main(int argc, char* argv[])
 
         ref_gemm1_invoker.Run(ref_gemm1_argument);
 
-        // LogRangeAsType<float>(std::cout << "a_m_k: ", a_m_k.mData, ",") << std::endl;
-        // LogRangeAsType<float>(std::cout << "b0_k_n : ", b0_k_n.mData, ",") << std::endl;
-        // LogRangeAsType<float>(std::cout << "b1_n_o : ", b1_n_o.mData, ",") << std::endl;
-        // LogRangeAsType<float>(std::cout << "c_m_o_device_result : ", c_m_o_device_result.mData, ",") << std::endl;
-
-        std::cout << "b0_k_n(0, 0) = " << (float)b0_k_n(0, 0) << ", b0_k_n(1, 0) = " << (float)b0_k_n(1, 0)
-                  << ", b0_k_n(0, 1) = " << (float)b0_k_n(0, 1) << ", b0_k_n(1, 1) = " << (float)b0_k_n(1, 1)
-                  << std::endl;
-
         return ck::utils::check_err(c_m_o_device_result.mData, c_m_o_host_result.mData) ? 0 : 1;
     }
 

include/ck/tensor_operation/gpu/block/blockwise_gemm_xdlops.hpp

@@ -158,22 +158,6 @@ struct BlockwiseGemmXdlops_k0mk1_k0nk1_m0n0m1n1m2m3m4n2_v1
         Tuple4 b_origin = CalculateBThreadOriginDataIndex())
         : a_thread_copy_(a_origin), b_thread_copy_(b_origin)
     {
-#if 0
-        if(!TransposeC && hipThreadIdx_x % 32 < 8)
-        {
-            printf("bid %zd tid %zd, a_mma = %d, %d, %d, %d, b_mma = %d, %d, %d, %d\n",
-                   hipBlockIdx_x,
-                   hipThreadIdx_x,
-                   a_origin[Number<0>{}],
-                   a_origin[Number<1>{}],
-                   a_origin[Number<2>{}],
-                   a_origin[Number<3>{}],
-                   b_origin[Number<0>{}],
-                   b_origin[Number<1>{}],
-                   b_origin[Number<2>{}],
-                   b_origin[Number<3>{}]);
-        }
-#endif
         static_assert(AMmaTileDesc::IsKnownAtCompileTime() && BMmaTileDesc::IsKnownAtCompileTime(),
                       "wrong! Desc should be known at compile-time");
 

include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v4r1.hpp

@@ -81,21 +81,7 @@ struct ThreadGroupTensorSliceTransfer_v4r1
                 make_multi_index(ThreadGroup::GetThreadId()));
 
             const auto thread_data_idx_begin = thread_cluster_idx * thread_slice_lengths;
-#if 0
-            if (std::is_same<Sequence<16,64,2>, BlockSliceLengths>::value)
-            {
-                auto s = src_block_slice_origin + thread_data_idx_begin;
-                auto d = dst_block_slice_origin + thread_data_idx_begin;
-                printf("bid %zd tid %zd, src origin %d %d %d, dst origin %d %d %d\n",
-                hipBlockIdx_x, hipThreadIdx_x,
-                s[Number<0>{}],
-                s[Number<1>{}],
-                s[Number<2>{}],
-                d[Number<0>{}],
-                d[Number<1>{}],
-                d[Number<2>{}]);
-            }
-#endif
+
             threadwise_transfer_.SetSrcSliceOrigin(src_desc,
                                                    src_block_slice_origin + thread_data_idx_begin);
             threadwise_transfer_.SetDstSliceOrigin(dst_desc,

include/ck/tensor_operation/gpu/device/device_batched_gemm_gemm_xdl_cshuffle.hpp

@@ -162,7 +162,7 @@ template <typename ALayout,
           index_t CShuffleNXdlPerWavePerShuffle,
           typename CShuffleBlockTransferClusterLengths_MBlock_MPerBlock_NBlock_NPerBlock,
           index_t CShuffleBlockTransferScalarPerVector_NPerBlock,
-          LoopScheduler LoopSched = make_default_loop_scheduler()>
+          LoopScheduler LoopSched = LoopScheduler::Default>
 struct DeviceGemmGemm_Xdl_CShuffle : public BaseOperator // TODO ANT: inherit from DeviceGemmGemm subtype
 {
     using DeviceOp = DeviceGemmGemm_Xdl_CShuffle;
@@ -553,7 +553,7 @@ struct DeviceGemmGemm_Xdl_CShuffle : public BaseOperator // TODO ANT: inherit fr
         ABlockTransferSrcVectorDim,
         ABlockTransferSrcScalarPerVector,
         ABlockTransferDstScalarPerVector_AK1,
-        false,
+        true,
         ABlockLdsExtraM,
         BBlockTransferThreadClusterLengths_BK0_N_BK1,
         BBlockTransferThreadClusterArrangeOrder,
@@ -561,7 +561,7 @@ struct DeviceGemmGemm_Xdl_CShuffle : public BaseOperator // TODO ANT: inherit fr
         BBlockTransferSrcVectorDim,
         BBlockTransferSrcScalarPerVector,
         BBlockTransferDstScalarPerVector_BK1,
-        false,
+        true,
         BBlockLdsExtraN,
         B1BlockTransferThreadClusterLengths_BK0_N_BK1,
         B1BlockTransferThreadClusterArrangeOrder,
@@ -655,24 +655,6 @@ struct DeviceGemmGemm_Xdl_CShuffle : public BaseOperator // TODO ANT: inherit fr
 
         float Run(const Argument& arg, const StreamConfig& stream_config = StreamConfig{})
         {
-#if 0
-            {
-                std::cout << "arg.a_grid_desc_ak0_m_ak1_{"
-                          << arg.a_grid_desc_ak0_m_ak1_.GetLength(I0) << ", "
-                          << arg.a_grid_desc_ak0_m_ak1_.GetLength(I1) << ", "
-                          << arg.a_grid_desc_ak0_m_ak1_.GetLength(I2) << "}" << std::endl;
-
-                std::cout << "arg.b_grid_desc_bk0_n_bk1_{"
-                          << arg.b_grid_desc_bk0_n_bk1_.GetLength(I0) << ", "
-                          << arg.b_grid_desc_bk0_n_bk1_.GetLength(I1) << ", "
-                          << arg.b_grid_desc_bk0_n_bk1_.GetLength(I2) << "}" << std::endl;
-
-                std::cout << "arg.c_grid_desc_m_n_{ " << arg.c_grid_desc_m_n_.GetLength(I0) << ", "
-                          << arg.c_grid_desc_m_n_.GetLength(I1) << "}" << std::endl;
-            }
-#endif
-
-            // TODO ANT: block id to ctilemap should infer acc0tile map
             if(!GridwiseGemm::CheckValidity(arg.a_grid_desc_ak0_m_ak1_,
                                             arg.b_grid_desc_bk0_n_bk1_,
                                             arg.b1_grid_desc_bk0_n_bk1_,
@@ -685,7 +667,7 @@ struct DeviceGemmGemm_Xdl_CShuffle : public BaseOperator // TODO ANT: inherit fr
             const index_t grid_size =
                 arg.block_2_ctile_map_.CalculateGridSize(arg.c_grid_desc_m_n_) * arg.batch_count_;
 
-            // TODO ANT: K for gemm1
+            // Gemm0_K
             const auto K =
                 arg.a_grid_desc_ak0_m_ak1_.GetLength(I0) * arg.a_grid_desc_ak0_m_ak1_.GetLength(I2);
 
@@ -728,7 +710,8 @@ struct DeviceGemmGemm_Xdl_CShuffle : public BaseOperator // TODO ANT: inherit fr
                                               arg.compute_base_ptr_of_batch_);
             };
 
-            // TODO ANT: handle tail loops for gemm0 & gemm1
+            // Gemm1_K is split into Gemm1_K0/K1 where K1 is known at compile time, so we only need
+            // to concern Gemm0's loop
             if(GridwiseGemm::CalculateHasMainKBlockLoop(K))
             {
                 ave_time = launch_kernel(integral_constant<bool, true>{});

include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp

@@ -1145,10 +1145,6 @@ struct ThreadwiseTensorSliceTransfer_v4
                 src_desc, src_data_coord);
 
             // copy data from src_buf into src_tmp_vector
-#if 0
-            src_tmp_vector.template AsType<src_vector_t>()(Number<0>{}) =
-                src_buf.template Get<src_vector_t>(src_data_coord.GetOffset(), is_src_valid);
-#else
             if constexpr(SrcBuffer::IsDynamicBuffer())
             {
                 src_tmp_vector.template AsType<src_vector_t>()(Number<0>{}) =
@@ -1164,33 +1160,7 @@ struct ThreadwiseTensorSliceTransfer_v4
                     src_tmp_vector.template AsType<SrcData>()(i) =
                         src_buf[Number<src_offset>{}];
                 });
-                // if constexpr(StaticBufferTupleOfVector)
-                // {
-                //     // constexpr auto offset_nd = SrcRefToOriginDisplacement{} + data_to_origin_disp_idx;
-                //     // // offset_nd.foo();
-                //     // constexpr auto offset = src_desc.CalculateOffset(offset_nd);
-                //     // src_tmp_vector.template AsType<src_vector_t>()(Number<0>{}) =
-                //     //     src_buf.template GetAsType<src_vector_t>(Number<offset>{});
-                //     static_for<0, SrcScalarPerVector, 1>{}([&](auto i) {
-                //         // constexpr auto src_offset_nd = src_ref_to_origin_disp_idx +
-                //         //                                data_to_origin_disp_idx + i * src_scalar_step_in_vector;
-                //         // constexpr auto src_offset = src_desc.CalculateOffset(src_offset_nd);
-                //         constexpr auto src_offset = src_desc.CalculateOffset(SrcRefToOriginDisplacement{});
-
-                //         // SrcData s = src_buf[Number<src_offset>{}];
-                //         SrcData s = src_buf[Number<0>{}];
-                //         // apply type convert
-                //         src_tmp_vector.template AsType<SrcData>()(i) = s;
-                //     });
-                // }
-                // else
-                // {
-                //     src_tmp_vector.template AsType<src_vector_t>()(Number<0>{}) =
-                //         src_buf.template Get<src_vector_t>(src_data_coord.GetOffset(),
-                //                                            is_src_valid);
-                // }
             }
-#endif
             // copy data from src_tmp_vector to dst_tmp_vector (data cast data from SrcData to
             // DstData)
             vector_type_maker_t<DstData, SrcScalarPerVector> dst_tmp_vector;
@@ -1236,16 +1206,14 @@ template <typename SrcData,
           typename DimAccessOrder,
           index_t DstVectorDim,
           index_t DstScalarPerVector,
-        //   InMemoryDataOperationEnum DstInMemOp,
-        //   index_t DstScalarStrideInVector,
           typename enable_if<SrcDesc::IsKnownAtCompileTime() && DstDesc::IsKnownAtCompileTime(), bool>::type = false>
-struct ThreadwiseTensorSliceTransfer_v1r3_Static
+struct ThreadwiseTensorSliceTransfer_StaticToStatic
 {
     static constexpr index_t nDim = SliceLengths::Size();
 
     using Index = MultiIndex<nDim>;
 
-    __device__ constexpr ThreadwiseTensorSliceTransfer_v1r3_Static()
+    __device__ constexpr ThreadwiseTensorSliceTransfer_StaticToStatic()
     {
         static_assert(SrcDesc::IsKnownAtCompileTime() && DstDesc::IsKnownAtCompileTime(),
                       "wrong! Desc need to known at compile-time");

include/ck/tensor_operation/gpu/warp/xdlops_gemm.hpp

@@ -30,17 +30,6 @@ enum struct MfmaInstr
     mfma_f64_16x16x4f64
 };
 
-// template <typename T, bool TransposeC>
-// struct mfma_base_type
-// {
-//     template <index_t MPerXdlops, index_t NPerXdlops, class FloatA, class FloatB, class FloatC>
-//     __device__ void run(const FloatA& a, const FloatB& b, FloatC& reg_c) const
-//     {
-//         if constexpr (!TransposeC) T::run(a, b, reg_c);
-//         else T::run(b, a, reg_c);
-//     }
-// };
-
 template <MfmaInstr instr>
 struct mfma_type;
 

include/ck/utility/static_buffer.hpp

@@ -72,6 +72,7 @@ struct StaticBufferTupleOfVector
     __host__ __device__ static constexpr bool IsDynamicBuffer() { return false; }
 
     __host__ __device__ static constexpr index_t Size() { return s_per_buf; };
+
     // Get S
     // i is offset of S
     template <index_t I>

include/ck/utility/tuple_helper.hpp

@@ -78,10 +78,4 @@ __host__ __device__ constexpr auto transform_tuples(F f, const X& x, const Y& y,
         f, x, y, z, typename arithmetic_sequence_gen<0, X::Size(), 1>::type{});
 }
 
-template <index_t... Is>
-__host__ __device__ constexpr Tuple<Number<Is>...> to_tuple(Sequence<Is...>)
-{
-    return Tuple<Number<Is>...>(Number<Is>{}...);
-}
-
 } // namespace ck

library/include/ck/library/utility/check_err.hpp

@@ -134,7 +134,7 @@ check_err(const std::vector<T>& out,
         {
             max_err = err > max_err ? err : max_err;
             err_count++;
-            if(err_count < 128)
+            if(err_count < 5)
             {
                 std::cout << msg << std::setw(12) << std::setprecision(7) << " out[" << i
                           << "] != ref[" << i << "]: " << o << " != " << r << std::endl;