"test/git@developer.sourcefind.cn:change/sglang.git" did not exist on "e47800e176b86d7d95309ab23d6cb3bd76d6c2be"
Commit 1566b317 authored by Chao Liu's avatar Chao Liu
Browse files

reorginzed files

parent c82b833d
......@@ -46,8 +46,19 @@ endif()
#
include_directories(BEFORE
include
${PROJECT_BINARY_DIR}/include
${PROJECT_SOURCE_DIR}/composable_kernel/include
${PROJECT_SOURCE_DIR}/composable_kernel/include/utility
${PROJECT_SOURCE_DIR}/composable_kernel/include/tensor_description
${PROJECT_SOURCE_DIR}/composable_kernel/include/tensor_operation
${PROJECT_SOURCE_DIR}/composable_kernel/include/kernel_algorithm
${PROJECT_SOURCE_DIR}/driver/include
${PROJECT_BINARY_DIR}/composable_kernel/include/utility
)
add_subdirectory(src)
if(DEVICE_BACKEND STREQUAL "AMD")
configure_file("${PROJECT_SOURCE_DIR}/composable_kernel/include/utility/config_amd.hpp.in" "${PROJECT_BINARY_DIR}/composable_kernel/include/utility/config.hpp")
elseif(DEVICE_BACKEND STREQUAL "NVIDIA")
configure_file("${PROJECT_SOURCE_DIR}/composable_kernel/include/utility/config_nvidia.hpp.in" "${PROJECT_BINARY_DIR}/composable_kernel/include/utility/config.hpp")
endif()
add_subdirectory(driver)
#ifndef CK_GRIDWISE_CONVOLUTION_DIRECT_V2_NCHW_KCYX_NKHW
#define CK_GRIDWISE_CONVOLUTION_DIRECT_V2_NCHW_KCYX_NKHW
#include "composable_kernel/utility/common.hpp"
#include "composable_kernel/tensor_description/ConstantTensorDescriptor.hpp"
#include "composable_kernel/tensor_operation/blockwise_2d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/blockwise_4d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/threadwise_tensor_slice_copy.hpp"
#include "composable_kernel/tensor_operation/threadwise_direct_convolution.hpp"
#include "common_header.hpp"
#include "ConstantTensorDescriptor.hpp"
#include "blockwise_2d_tensor_op.hpp"
#include "blockwise_4d_tensor_op.hpp"
#include "threadwise_tensor_slice_copy.hpp"
#include "threadwise_direct_convolution.hpp"
namespace ck {
......
#ifndef CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V1R1_CHWN_CYXK_KHWN
#define CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V1R1_CHWN_CYXK_KHWN
#include "composable_kernel/utility/common.hpp"
#include "composable_kernel/tensor_description/ConstantTensorDescriptor.hpp"
#include "composable_kernel/tensor_description/ConstantMatrixDescriptor.hpp"
#include "composable_kernel/tensor_operation/blockwise_4d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/blockwise_2d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/threadwise_tensor_slice_copy.hpp"
#include "composable_kernel/tensor_operation/threadwise_4d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/blockwise_batched_gemm.hpp"
#include "common_header.hpp"
#include "ConstantTensorDescriptor.hpp"
#include "ConstantMatrixDescriptor.hpp"
#include "blockwise_4d_tensor_op.hpp"
#include "blockwise_2d_tensor_op.hpp"
#include "threadwise_tensor_slice_copy.hpp"
#include "threadwise_4d_tensor_op.hpp"
#include "blockwise_batched_gemm.hpp"
namespace ck {
......
#ifndef CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V1R2_CHWN_CYXK_KHWN
#define CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V1R2_CHWN_CYXK_KHWN
#include "composable_kernel/utility/common.hpp"
#include "composable_kernel/tensor_description/ConstantTensorDescriptor.hpp"
#include "composable_kernel/tensor_description/ConstantMatrixDescriptor.hpp"
#include "composable_kernel/tensor_operation/blockwise_2d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/blockwise_3d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/blockwise_4d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/threadwise_tensor_slice_copy.hpp"
#include "composable_kernel/tensor_operation/threadwise_4d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/blockwise_batched_gemm.hpp"
#include "common_header.hpp"
#include "ConstantTensorDescriptor.hpp"
#include "ConstantMatrixDescriptor.hpp"
#include "blockwise_2d_tensor_op.hpp"
#include "blockwise_3d_tensor_op.hpp"
#include "blockwise_4d_tensor_op.hpp"
#include "threadwise_tensor_slice_copy.hpp"
#include "threadwise_4d_tensor_op.hpp"
#include "blockwise_batched_gemm.hpp"
namespace ck {
......
#ifndef CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V1R3_CHWN_CYXK_KHWN
#define CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V1R3_CHWN_CYXK_KHWN
#include "composable_kernel/utility/common.hpp"
#include "composable_kernel/tensor_description/ConstantTensorDescriptor.hpp"
#include "composable_kernel/tensor_description/ConstantMatrixDescriptor.hpp"
#include "composable_kernel/tensor_operation/blockwise_2d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/blockwise_4d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/threadwise_tensor_slice_copy.hpp"
#include "composable_kernel/tensor_operation/threadwise_4d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/blockwise_batched_gemm.hpp"
#include "common_header.hpp"
#include "ConstantTensorDescriptor.hpp"
#include "ConstantMatrixDescriptor.hpp"
#include "blockwise_2d_tensor_op.hpp"
#include "blockwise_4d_tensor_op.hpp"
#include "threadwise_tensor_slice_copy.hpp"
#include "threadwise_4d_tensor_op.hpp"
#include "blockwise_batched_gemm.hpp"
namespace ck {
......
#ifndef CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V1R3_CHWN_CYXK_KHWN_LDS_DOUBLE_BUFFER
#define CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V1R3_CHWN_CYXK_KHWN_LDS_DOUBLE_BUFFER
#include "composable_kernel/utility/common.hpp"
#include "composable_kernel/tensor_description/ConstantTensorDescriptor.hpp"
#include "composable_kernel/tensor_description/ConstantMatrixDescriptor.hpp"
#include "composable_kernel/tensor_operation/blockwise_2d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/blockwise_4d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/threadwise_tensor_slice_copy.hpp"
#include "composable_kernel/tensor_operation/threadwise_4d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/blockwise_batched_gemm.hpp"
#include "common_header.hpp"
#include "ConstantTensorDescriptor.hpp"
#include "ConstantMatrixDescriptor.hpp"
#include "blockwise_2d_tensor_op.hpp"
#include "blockwise_4d_tensor_op.hpp"
#include "threadwise_tensor_slice_copy.hpp"
#include "threadwise_4d_tensor_op.hpp"
#include "blockwise_batched_gemm.hpp"
namespace ck {
......
#ifndef CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V1R3_NCHW_CYXK_NKHW
#define CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V1R3_NCHW_CYXK_NKHW
#include "composable_kernel/utility/common.hpp"
#include "composable_kernel/tensor_description/ConstantTensorDescriptor.hpp"
#include "composable_kernel/tensor_description/ConstantMatrixDescriptor.hpp"
#include "composable_kernel/tensor_operation/blockwise_2d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/blockwise_tensor_slice_copy.hpp"
#include "composable_kernel/tensor_operation/threadwise_tensor_slice_copy.hpp"
#include "composable_kernel/tensor_operation/threadwise_generic_tensor_op.hpp"
#include "composable_kernel/tensor_operation/blockwise_batched_gemm.hpp"
#include "common_header.hpp"
#include "ConstantTensorDescriptor.hpp"
#include "ConstantMatrixDescriptor.hpp"
#include "blockwise_2d_tensor_op.hpp"
#include "blockwise_tensor_slice_copy.hpp"
#include "threadwise_tensor_slice_copy.hpp"
#include "threadwise_generic_tensor_op.hpp"
#include "blockwise_batched_gemm.hpp"
namespace ck {
......
#ifndef CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V1R3_NCHW_CYXK_NKHW_LDS_DOUBLE_BUFFER
#define CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V1R3_NCHW_CYXK_NKHW_LDS_DOUBLE_BUFFER
#include "composable_kernel/utility/common.hpp"
#include "composable_kernel/tensor_description/ConstantTensorDescriptor.hpp"
#include "composable_kernel/tensor_description/ConstantMatrixDescriptor.hpp"
#include "composable_kernel/tensor_operation/blockwise_2d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/blockwise_tensor_slice_copy.hpp"
#include "composable_kernel/tensor_operation/threadwise_tensor_slice_copy.hpp"
#include "composable_kernel/tensor_operation/threadwise_generic_tensor_op.hpp"
#include "composable_kernel/tensor_operation/blockwise_batched_gemm.hpp"
#include "common_header.hpp"
#include "ConstantTensorDescriptor.hpp"
#include "ConstantMatrixDescriptor.hpp"
#include "blockwise_2d_tensor_op.hpp"
#include "blockwise_tensor_slice_copy.hpp"
#include "threadwise_tensor_slice_copy.hpp"
#include "threadwise_generic_tensor_op.hpp"
#include "blockwise_batched_gemm.hpp"
namespace ck {
......
#ifndef CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V2_CHWN_CYXK_KHWN
#define CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V2_CHWN_CYXK_KHWN
#include "composable_kernel/utility/common.hpp"
#include "composable_kernel/tensor_description/ConstantTensorDescriptor.hpp"
#include "composable_kernel/tensor_description/ConstantMatrixDescriptor.hpp"
#include "composable_kernel/tensor_operation/blockwise_4d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/blockwise_2d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/blockwise_gemm.hpp"
#include "common_header.hpp"
#include "ConstantTensorDescriptor.hpp"
#include "ConstantMatrixDescriptor.hpp"
#include "blockwise_4d_tensor_op.hpp"
#include "blockwise_2d_tensor_op.hpp"
#include "blockwise_gemm.hpp"
namespace ck {
......
#ifndef CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V2_CHWN_CYXK_KHWN_LDS_DOUBLE_BUFFER
#define CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V2_CHWN_CYXK_KHWN_LDS_DOUBLE_BUFFER
#include "composable_kernel/utility/common.hpp"
#include "composable_kernel/tensor_description/ConstantTensorDescriptor.hpp"
#include "composable_kernel/tensor_description/ConstantMatrixDescriptor.hpp"
#include "composable_kernel/tensor_operation/blockwise_4d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/blockwise_2d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/threadwise_tensor_slice_copy.hpp"
#include "composable_kernel/tensor_operation/blockwise_gemm.hpp"
#include "common_header.hpp"
#include "ConstantTensorDescriptor.hpp"
#include "ConstantMatrixDescriptor.hpp"
#include "blockwise_4d_tensor_op.hpp"
#include "blockwise_2d_tensor_op.hpp"
#include "threadwise_tensor_slice_copy.hpp"
#include "blockwise_gemm.hpp"
namespace ck {
......
#ifndef CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V3_NCHW_CYXK_NKHW
#define CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V3_NCHW_CYXK_NKHW
#include "composable_kernel/utility/common.hpp"
#include "composable_kernel/tensor_description/ConstantTensorDescriptor.hpp"
#include "composable_kernel/tensor_description/ConstantMergedTensorDescriptor.hpp"
#include "composable_kernel/tensor_description/ConstantMatrixDescriptor.hpp"
#include "composable_kernel/tensor_operation/blockwise_generic_tensor_slice_copy.hpp"
#include "composable_kernel/tensor_operation/blockwise_gemm.hpp"
#include "common_header.hpp"
#include "ConstantTensorDescriptor.hpp"
#include "ConstantMergedTensorDescriptor.hpp"
#include "ConstantMatrixDescriptor.hpp"
#include "blockwise_generic_tensor_slice_copy.hpp"
#include "blockwise_gemm.hpp"
namespace ck {
......
#ifndef CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V3_NCHW_CYXK_NKHW_LDS_DOUBLE_BUFFER
#define CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V3_NCHW_CYXK_NKHW_LDS_DOUBLE_BUFFER
#include "composable_kernel/utility/common.hpp"
#include "composable_kernel/tensor_description/ConstantTensorDescriptor.hpp"
#include "composable_kernel/tensor_description/ConstantMergedTensorDescriptor.hpp"
#include "composable_kernel/tensor_description/ConstantMatrixDescriptor.hpp"
#include "composable_kernel/tensor_operation/blockwise_generic_tensor_slice_copy.hpp"
#include "composable_kernel/tensor_operation/blockwise_gemm.hpp"
#include "common_header.hpp"
#include "ConstantTensorDescriptor.hpp"
#include "ConstantMergedTensorDescriptor.hpp"
#include "ConstantMatrixDescriptor.hpp"
#include "blockwise_generic_tensor_slice_copy.hpp"
#include "blockwise_gemm.hpp"
namespace ck {
......
#ifndef CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V4_NCHW_KCYX_NKHW
#define CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V4_NCHW_KCYX_NKHW
#include "composable_kernel/utility/common.hpp"
#include "composable_kernel/tensor_description/ConstantTensorDescriptor.hpp"
#include "composable_kernel/tensor_description/ConstantMergedTensorDescriptor.hpp"
#include "composable_kernel/tensor_description/ConstantMatrixDescriptor.hpp"
#include "composable_kernel/tensor_operation/blockwise_generic_tensor_slice_copy.hpp"
#include "composable_kernel/tensor_operation/blockwise_gemm.hpp"
#include "composable_kernel/tensor_operation/threadwise_generic_tensor_slice_copy.hpp"
#include "common_header.hpp"
#include "ConstantTensorDescriptor.hpp"
#include "ConstantMergedTensorDescriptor.hpp"
#include "ConstantMatrixDescriptor.hpp"
#include "blockwise_generic_tensor_slice_copy.hpp"
#include "blockwise_gemm.hpp"
#include "threadwise_generic_tensor_slice_copy.hpp"
namespace ck {
......
#ifndef CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V4_NCHW_KCYX_NKHW_LDS_DOUBLE_BUFFER
#define CK_GRIDWISE_CONVOLUTION_IMPLICIT_GEMM_V4_NCHW_KCYX_NKHW_LDS_DOUBLE_BUFFER
#include "composable_kernel/utility/common.hpp"
#include "composable_kernel/tensor_description/ConstantTensorDescriptor.hpp"
#include "composable_kernel/tensor_description/ConstantMergedTensorDescriptor.hpp"
#include "composable_kernel/tensor_description/ConstantMatrixDescriptor.hpp"
#include "composable_kernel/tensor_operation/blockwise_generic_tensor_slice_copy.hpp"
#include "composable_kernel/tensor_operation/blockwise_gemm.hpp"
#include "composable_kernel/tensor_operation/threadwise_generic_tensor_slice_copy.hpp"
#include "common_header.hpp"
#include "ConstantTensorDescriptor.hpp"
#include "ConstantMergedTensorDescriptor.hpp"
#include "ConstantMatrixDescriptor.hpp"
#include "blockwise_generic_tensor_slice_copy.hpp"
#include "blockwise_gemm.hpp"
#include "threadwise_generic_tensor_slice_copy.hpp"
#ifndef CK_BLOCKWISE_GEMM_USE_AMD_INLINE_ASM
#define CK_BLOCKWISE_GEMM_USE_AMD_INLINE_ASM 1
#endif
namespace ck {
......@@ -233,10 +237,10 @@ struct GridwiseConvolutionImplicitGemm_v4_nchw_kcyx_nkhw_lds_double_buffer
// choose GEMM implementation here
const auto run_blockwise_gemm = [&](auto... Xs) {
#if 1
return blockwise_gemm.Run(Xs...);
#else
#if CK_USE_AMD_INLINE_ASM && CK_BLOCKWISE_GEMM_USE_AMD_INLINE_ASM
return blockwise_gemm.Run_asm(Xs...);
#else
return blockwise_gemm.Run(Xs...);
#endif
};
......
#pragma once
#include "composable_kernel/utility/common.hpp"
#include "composable_kernel/tensor_description/ConstantTensorDescriptor.hpp"
#include "composable_kernel/tensor_operation/blockwise_2d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/blockwise_4d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/blockwise_direct_convolution.hpp"
#include "composable_kernel/tensor_operation/threadwise_4d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/threadwise_direct_convolution.hpp"
#include "common_header.hpp"
#include "ConstantTensorDescriptor.hpp"
#include "blockwise_2d_tensor_op.hpp"
#include "blockwise_4d_tensor_op.hpp"
#include "blockwise_direct_convolution.hpp"
#include "threadwise_4d_tensor_op.hpp"
#include "threadwise_direct_convolution.hpp"
namespace ck {
......
#pragma once
#include "composable_kernel/utility/common.hpp"
#include "composable_kernel/tensor_description/ConstantTensorDescriptor.hpp"
#include "composable_kernel/tensor_description/ConstantMatrixDescriptor.hpp"
#include "composable_kernel/tensor_operation/blockwise_4d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/blockwise_2d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/threadwise_4d_tensor_op.hpp"
#include "composable_kernel/tensor_operation/blockwise_gemm.hpp"
#include "common_header.hpp"
#include "ConstantTensorDescriptor.hpp"
#include "ConstantMatrixDescriptor.hpp"
#include "blockwise_4d_tensor_op.hpp"
#include "blockwise_2d_tensor_op.hpp"
#include "threadwise_4d_tensor_op.hpp"
#include "blockwise_gemm.hpp"
namespace ck {
......
#ifndef CK_CONSTANT_MATRIX_DESCRIPTOR_HPP
#define CK_CONSTANT_MATRIX_DESCRIPTOR_HPP
#include "composable_kernel/utility/common.hpp"
#include "common_header.hpp"
namespace ck {
......
#ifndef CK_CONSTANT_MERGED_TENSOR_DESCRIPTOR_HPP
#define CK_CONSTANT_MERGED_TENSOR_DESCRIPTOR_HPP
#include "composable_kernel/utility/common.hpp"
#include "composable_kernel/tensor_description/ConstantTensorDescriptor.hpp"
#include "common_header.hpp"
#include "ConstantTensorDescriptor.hpp"
namespace ck {
......
#ifndef CK_CONSTANT_TENSOR_DESCRIPTOR_HPP
#define CK_CONSTANT_TENSOR_DESCRIPTOR_HPP
#include "composable_kernel/utility/common.hpp"
#include "common_header.hpp"
namespace ck {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment