2.5

992bec46 · “yuguo” · 0259837d · 992bec46 · 0259837d · 0259837d
Commit 992bec46 authored Oct 08, 2023 by “yuguo”
20 changed files
--- a/cmake/external/brpc.cmake
+++ b/cmake/external/brpc.cmake
@@ -36,6 +36,11 @@ set(BRPC_LIBRARIES

 include_directories(${BRPC_INCLUDE_DIR})

+# clone brpc to Paddle/third_party
+set(BRPC_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/brpc)
+set(BRPC_URL https://github.com/apache/brpc.git)
+set(BRPC_TAG 1.4.0)
+
 # Reference https://stackoverflow.com/questions/45414507/pass-a-list-of-prefix-paths-to-externalproject-add-in-cmake-args
 set(prefix_path
    "${THIRD_PARTY_PATH}/install/gflags|${THIRD_PARTY_PATH}/install/leveldb|${THIRD_PARTY_PATH}/install/snappy|${THIRD_PARTY_PATH}/install/gtest|${THIRD_PARTY_PATH}/install/protobuf|${THIRD_PARTY_PATH}/install/zlib|${THIRD_PARTY_PATH}/install/glog"
@@ -45,8 +50,7 @@ set(prefix_path
 ExternalProject_Add(
  extern_brpc
  ${EXTERNAL_PROJECT_LOG_ARGS}
-  GIT_REPOSITORY "https://github.com/apache/incubator-brpc"
-  GIT_TAG 1.2.0
+  SOURCE_DIR ${BRPC_SOURCE_DIR}
  PREFIX ${BRPC_PREFIX_DIR}
  UPDATE_COMMAND ""
  CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}

--- a/cmake/external/cinn.cmake
+++ b/cmake/external/cinn.cmake
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-if(NOT WITH_CINN)
-  return()
-endif()
-
-if(NOT CINN_GIT_TAG)
-  set(CINN_GIT_TAG release/v0.2)
-endif()
-
-message(STATUS "CINN version: " ${CINN_GIT_TAG})
-
-# TODO(zhhsplendid): CINN has lots of warnings during early development.
-# They will be treated as errors under paddle. We set no-error now and we will
-# clean the code in the future.
-add_definitions(-w)
-
-######################################
-# Build CINN from Git External Project
-######################################
-include(ExternalProject)
-set(CINN_PREFIX_DIR ${THIRD_PARTY_PATH}/CINN)
-set(CINN_OPTIONAL_ARGS
-    -DPY_VERSION=${PY_VERSION}
-    -DWITH_CUDA=${WITH_GPU}
-    -DWITH_CUDNN=${WITH_GPU}
-    -DWITH_MKL_CBLAS=${WITH_MKL}
-    -DWITH_MKLDNN=${WITH_MKL}
-    -DPUBLISH_LIBS=ON
-    -DWITH_TESTING=ON)
-set(CINN_BUILD_COMMAND $(MAKE) cinnapi -j)
-ExternalProject_Add(
-  external_cinn
-  ${EXTERNAL_PROJECT_LOG_ARGS}
-  GIT_REPOSITORY "${GIT_URL}/PaddlePaddle/CINN.git"
-  GIT_TAG ${CINN_GIT_TAG}
-  PREFIX ${CINN_PREFIX_DIR}
-  BUILD_COMMAND ${CINN_BUILD_COMMAND}
-  INSTALL_COMMAND ""
-  CMAKE_ARGS ${CINN_OPTIONAL_ARGS})
-
-ExternalProject_Get_Property(external_cinn BINARY_DIR)
-ExternalProject_Get_Property(external_cinn SOURCE_DIR)
-set(CINN_BINARY_DIR ${BINARY_DIR})
-set(CINN_SOURCE_DIR ${SOURCE_DIR})
-
-message(STATUS "CINN BINARY_DIR: ${CINN_BINARY_DIR}")
-message(STATUS "CINN SOURCE_DIR: ${CINN_SOURCE_DIR}")
-
-######################################
-# Add CINN's dependencies header files
-######################################
-
-# Add absl
-set(ABSL_INCLUDE_DIR "${CINN_BINARY_DIR}/dist/third_party/absl/include")
-include_directories(${ABSL_INCLUDE_DIR})
-
-# Add isl
-set(ISL_INCLUDE_DIR "${CINN_BINARY_DIR}/dist/third_party/isl/include")
-include_directories(${ISL_INCLUDE_DIR})
-
-# Add LLVM
-set(LLVM_INCLUDE_DIR "${CINN_BINARY_DIR}/dist/third_party/llvm/include")
-include_directories(${LLVM_INCLUDE_DIR})
-
-######################################################
-# Put external_cinn and dependencies together as a lib
-######################################################
-
-set(CINN_LIB_NAME "libcinnapi.so")
-set(CINN_LIB_LOCATION "${CINN_BINARY_DIR}/dist/cinn/lib")
-set(CINN_INCLUDE_DIR "${CINN_BINARY_DIR}/dist/cinn/include")
-
-add_library(cinn SHARED IMPORTED GLOBAL)
-set_target_properties(cinn PROPERTIES IMPORTED_LOCATION
-                                      "${CINN_LIB_LOCATION}/${CINN_LIB_NAME}")
-include_directories(${CINN_INCLUDE_DIR})
-add_dependencies(cinn external_cinn)
--- a/cmake/external/concurrentqueue.cmake
+++ b/cmake/external/concurrentqueue.cmake
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-include(ExternalProject)
-
-set(CONCURRENTQUEUE_PROJECT "extern_concurrentqueue")
-set(CONCURRENTQUEUE_VER "v1.0.3")
-set(CONCURRENTQUEUE_URL_MD5 118e5bb661b567634647312991e10222)
-set(CONCURRENTQUEUE_PREFIX_URL
-    "https://github.com/cameron314/concurrentqueue/archive/refs/tags")
-set(CONCURRENTQUEUE_URL
-    "${CONCURRENTQUEUE_PREFIX_URL}/${CONCURRENTQUEUE_VER}.tar.gz")
-
-message(
-  STATUS
-    "CONCURRENTQUEUE_VERSION: ${CONCURRENTQUEUE_VER}, CONCURRENTQUEUE_URL: ${CONCURRENTQUEUE_URL}"
-)
-
-set(CONCURRENTQUEUE_PREFIX_DIR ${THIRD_PARTY_PATH}/concurrentqueue)
-set(CONCURRENTQUEUE_SOURCE_DIR ${THIRD_PARTY_PATH}/concurrentqueue/src/)
-set(CONCURRENTQUEUE_INCLUDE_DIR
-    "${CONCURRENTQUEUE_SOURCE_DIR}/extern_concurrentqueue")
-
-ExternalProject_Add(
-  ${CONCURRENTQUEUE_PROJECT}
-  ${EXTERNAL_PROJECT_LOG_ARGS}
-  URL ${CONCURRENTQUEUE_URL}
-  URL_MD5 ${CONCURRENTQUEUE_URL_MD5}
-  PREFIX ${CONCURRENTQUEUE_PREFIX_DIR}
-  DOWNLOAD_NO_PROGRESS 1
-  CONFIGURE_COMMAND ""
-  BUILD_COMMAND ""
-  INSTALL_COMMAND ""
-  UPDATE_COMMAND "")
-
-include_directories(${CONCURRENTQUEUE_INCLUDE_DIR})
--- a/cmake/external/cub.cmake
+++ b/cmake/external/cub.cmake
@@ -22,26 +22,27 @@ set(CUB_PATH
    CACHE STRING "A path setting for external_cub path.")
 set(CUB_PREFIX_DIR ${CUB_PATH})

-set(CUB_REPOSITORY ${GIT_URL}/NVlabs/cub.git)
+set(CUB_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/cub)

 if(${CMAKE_CUDA_COMPILER_VERSION} GREATER_EQUAL 11.6)
  # cuda_11.6/11.7/11.8‘s own cub is 1.15.0, which will cause compiling error in windows.
  set(CUB_TAG 1.16.0)
+  execute_process(COMMAND git --git-dir=${CUB_SOURCE_DIR}/.git
+                          --work-tree=${CUB_SOURCE_DIR} checkout ${CUB_TAG})
  # cub 1.16.0 is not compitable with current thrust version
  add_definitions(-DTHRUST_IGNORE_CUB_VERSION_CHECK)
 else()
  set(CUB_TAG 1.8.0)
 endif()

-set(CUB_INCLUDE_DIR ${CUB_PREFIX_DIR}/src/extern_cub)
+set(CUB_INCLUDE_DIR ${CUB_SOURCE_DIR})
 message("CUB_INCLUDE_DIR is ${CUB_INCLUDE_DIR}")
 include_directories(${CUB_INCLUDE_DIR})

 ExternalProject_Add(
  extern_cub
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${CUB_REPOSITORY}
-  GIT_TAG ${CUB_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${CUB_SOURCE_DIR}
  PREFIX ${CUB_PREFIX_DIR}
  UPDATE_COMMAND ""
  CONFIGURE_COMMAND ""

--- a/cmake/external/cudnn-frontend.cmake
+++ b/cmake/external/cudnn-frontend.cmake
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include(ExternalProject)
+
+set(CUDNN_FRONTEND_CUDNN_MIN_VERSION 8000)
+
+if(NOT WITH_GPU)
+  message(FATAL_ERROR "Can't enable CUDNN Frontend API without CUDA.")
+endif()
+if(CUDNN_VERSION LESS 8000)
+  message(
+    FATAL_ERROR
+      "Minimum CUDNN version is ${CUDNN_FRONTEND_CUDNN_MIN_VERSION}. Current: ${CUDNN_VERSION}"
+  )
+endif()
+
+if((NOT DEFINED CUDNN_FRONTEND_NAME) OR (NOT DEFINED CUDNN_FRONTEND_URL))
+  set(CUDNN_FRONTEND_VER
+      "v0.9.1"
+      CACHE STRING "" FORCE)
+  set(CUDNN_FRONTEND_NAME
+      "cudnn-frontend"
+      CACHE STRING "" FORCE)
+  set(CUDNN_FRONTEND_URL
+      "https://github.com/NVIDIA/cudnn-frontend/archive/refs/tags/${CUDNN_FRONTEND_VER}.tar.gz"
+      CACHE STRING "" FORCE)
+endif()
+set(CUDNN_FRONTEND_CACHE_FILENAME "${CUDNN_FRONTEND_VER}.tar.gz")
+set(CUDNN_FRONTEND_URL_MD5 "da7cbad1305427f687dd4fd737178f80")
+
+message(
+  STATUS
+    "CUDNN_FRONTEND_NAME: ${CUDNN_FRONTEND_NAME}, CUDNN_FRONTEND_URL: ${CUDNN_FRONTEND_URL}"
+)
+set(CUDNN_FRONTEND_DOWNLOAD_DIR
+    "${PADDLE_SOURCE_DIR}/third_party/cudnn-frontend")
+set(CUDNN_FRONTEND_PREFIX_DIR ${THIRD_PARTY_PATH}/cudnn-frontend)
+set(CUDNN_FRONTEND_SOURCE_DIR
+    ${THIRD_PARTY_PATH}/cudnn-frontend/src/extern_cudnn_frontend/include)
+set(CUSPARSELT_INSTALL_DIR ${THIRD_PARTY_PATH}/install/cudnn-frontend)
+set(CUDNN_FRONTEND_INCLUDE_DIR ${CUDNN_FRONTEND_SOURCE_DIR})
+include_directories(${CUDNN_FRONTEND_INCLUDE_DIR})
+
+message(
+  STATUS
+    "Adding cudnn-frontend. Version: ${CUDNN_FRONTEND_VER}. Directory: ${CUDNN_FRONTEND_DOWNLOAD_DIR}"
+)
+
+function(download_cudnn_frontend)
+  message(
+    STATUS
+      "Downloading ${CUDNN_FRONTEND_URL} to ${CUDNN_FRONTEND_DOWNLOAD_DIR}/${CUDNN_FRONTEND_CACHE_FILENAME}"
+  )
+  # NOTE: If the version is updated, consider emptying the folder; maybe add timeout
+  file(
+    DOWNLOAD ${CUDNN_FRONTEND_URL}
+    ${CUDNN_FRONTEND_DOWNLOAD_DIR}/${CUDNN_FRONTEND_CACHE_FILENAME}
+    EXPECTED_MD5 ${CUDNN_FRONTEND_URL_MD5}
+    STATUS ERR)
+  if(ERR EQUAL 0)
+    message(STATUS "Download ${CUDNN_FRONTEND_CACHE_FILENAME} success")
+  else()
+    message(
+      FATAL_ERROR
+        "Download failed, error: ${ERR}\n You can try downloading ${CUDNN_FRONTEND_CACHE_FILENAME} again"
+    )
+  endif()
+endfunction()
+
+if(EXISTS ${CUDNN_FRONTEND_DOWNLOAD_DIR}/${CUDNN_FRONTEND_CACHE_FILENAME})
+  file(MD5 ${CUDNN_FRONTEND_DOWNLOAD_DIR}/${CUDNN_FRONTEND_CACHE_FILENAME}
+       CUDNN_FRONTEND_MD5)
+  if(NOT CUDNN_FRONTEND_MD5 STREQUAL CUDNN_FRONTEND_URL_MD5)
+    # clean build file
+    file(REMOVE_RECURSE ${CUDNN_FRONTEND_PREFIX_DIR})
+    download_cudnn_frontend()
+  endif()
+else()
+  download_cudnn_frontend()
+endif()
+
+ExternalProject_Add(
+  extern_cudnn_frontend
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  URL ${CUDNN_FRONTEND_DOWNLOAD_DIR}/${CUDNN_FRONTEND_CACHE_FILENAME}
+  PREFIX ${CUDNN_FRONTEND_PREFIX_DIR}
+  DOWNLOAD_DIR ${CUDNN_FRONTEND_DOWNLOAD_DIR}
+  DOWNLOAD_NO_PROGRESS 1
+  UPDATE_COMMAND ""
+  PATCH_COMMAND ""
+  CONFIGURE_COMMAND ""
+  BUILD_COMMAND ""
+  INSTALL_COMMAND ""
+  TEST_COMMAND "")
+
+add_library(cudnn-frontend INTERFACE)
+add_dependencies(cudnn-frontend extern_cudnn_frontend)
--- a/cmake/external/cusparselt.cmake
+++ b/cmake/external/cusparselt.cmake
@@ -22,7 +22,8 @@ if(WITH_ARM OR WIN32)
 endif()

 include(ExternalProject)
-
+set(CUSPARSELT_DOWNLOAD_DIR
+    ${PADDLE_SOURCE_DIR}/third_party/cusparselt/${CMAKE_SYSTEM_NAME})
 set(CUSPARSELT_PROJECT "extern_cusparselt")
 set(CUSPARSELT_P "https://developer.download.nvidia.com/compute")
 set(CUSPARSELT_F "libcusparse_lt-linux-x86_64-0.2.0.1.tar.gz")
@@ -37,22 +38,57 @@ set(CUSPARSELT_INC_DIR
 set(CUSPARSELT_LIB_DIR
    "${CUSPARSELT_INSTALL_DIR}/lib64"
    CACHE PATH "sparselt lib directory." FORCE)
+set(CUSPARSELT_CACHE_FILENAME "${CUSPARSELT_F}")
+set(CUSPARSELT_URL_MD5 "4f72f469e9cb1a85b09017fbace733d7")
+
 set_directory_properties(PROPERTIES CLEAN_NO_CUSTOM 1)
 include_directories(${CUSPARSELT_INC_DIR})

+function(download_cusparselt)
+  message(
+    STATUS
+      "Downloading ${CUSPARSELT_URL} to ${CUSPARSELT_DOWNLOAD_DIR}/${CUSPARSELT_CACHE_FILENAME}"
+  )
+  # NOTE: If the version is updated, consider emptying the folder; maybe add timeout
+  file(
+    DOWNLOAD ${CUSPARSELT_URL}
+    ${CUSPARSELT_DOWNLOAD_DIR}/${CUSPARSELT_CACHE_FILENAME}
+    EXPECTED_MD5 ${CUSPARSELT_URL_MD5}
+    STATUS ERR)
+  if(ERR EQUAL 0)
+    message(STATUS "Download ${CUSPARSELT_CACHE_FILENAME} success")
+  else()
+    message(
+      FATAL_ERROR
+        "Download failed, error: ${ERR}\n You can try downloading ${CUSPARSELT_CACHE_FILENAME} again"
+    )
+  endif()
+endfunction()
+
+if(EXISTS ${CUSPARSELT_DOWNLOAD_DIR}/${CUSPARSELT_CACHE_FILENAME})
+  file(MD5 ${CUSPARSELT_DOWNLOAD_DIR}/${CUSPARSELT_CACHE_FILENAME}
+       CUSPARSELT_MD5)
+  if(NOT CUSPARSELT_MD5 STREQUAL CUSPARSELT_URL_MD5)
+    # clean build file
+    file(REMOVE_RECURSE ${CUSPARSELT_PREFIX_DIR})
+    file(REMOVE_RECURSE ${CUSPARSELT_INSTALL_DIR})
+    download_cusparselt()
+  endif()
+else()
+  download_cusparselt()
+endif()
+
 ExternalProject_Add(
  ${CUSPARSELT_PROJECT}
  ${EXTERNAL_PROJECT_LOG_ARGS}
-  URL ${CUSPARSELT_URL}
+  URL ${CUSPARSELT_DOWNLOAD_DIR}/${CUSPARSELT_CACHE_FILENAME}
  PREFIX ${CUSPARSELT_PREFIX_DIR}
+  DOWNLOAD_DIR ${CUSPARSELT_DOWNLOAD_DIR}
  DOWNLOAD_NO_PROGRESS 1
+  SOURCE_DIR ${CUSPARSELT_INSTALL_DIR}
  CONFIGURE_COMMAND ""
  BUILD_COMMAND ""
-  INSTALL_COMMAND
-    ${CMAKE_COMMAND} -E copy_directory
-    ${CUSPARSELT_PREFIX_DIR}/src/extern_cusparselt/lib64 ${CUSPARSELT_LIB_DIR}
-    && ${CMAKE_COMMAND} -E copy_directory
-    ${CUSPARSELT_PREFIX_DIR}/src/extern_cusparselt/include ${CUSPARSELT_INC_DIR}
+  INSTALL_COMMAND ""
  UPDATE_COMMAND "")

 add_library(cusparselt INTERFACE)

--- a/cmake/external/cutlass.cmake
+++ b/cmake/external/cutlass.cmake
@@ -15,22 +15,24 @@
 include(ExternalProject)

 set(CUTLASS_PREFIX_DIR ${THIRD_PARTY_PATH}/cutlass)
+set(CUTLASS_TAG v2.11.0)
+set(CUTLASS_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/cutlass)

-set(CUTLASS_REPOSITORY https://github.com/NVIDIA/cutlass.git)
-set(CUTLASS_TAG v2.9.1)
-
-include_directories("${THIRD_PARTY_PATH}/cutlass/src/extern_cutlass/")
-include_directories("${THIRD_PARTY_PATH}/cutlass/src/extern_cutlass/include/")
-include_directories(
-  "${THIRD_PARTY_PATH}/cutlass/src/extern_cutlass/tools/util/include/")
+include_directories("${CUTLASS_SOURCE_DIR}/")
+include_directories("${CUTLASS_SOURCE_DIR}/include/")
+include_directories("${CUTLASS_SOURCE_DIR}/tools/util/include/")

 add_definitions("-DPADDLE_WITH_CUTLASS")
+add_definitions("-DSPCONV_WITH_CUTLASS=0")
+
+if(NOT PYTHON_EXECUTABLE)
+  find_package(PythonInterp REQUIRED)
+endif()

 ExternalProject_Add(
  extern_cutlass
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${CUTLASS_REPOSITORY}
-  GIT_TAG "${CUTLASS_TAG}"
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${CUTLASS_SOURCE_DIR}
  PREFIX ${CUTLASS_PREFIX_DIR}
  UPDATE_COMMAND ""
  CONFIGURE_COMMAND ""
@@ -38,6 +40,40 @@ ExternalProject_Add(
  INSTALL_COMMAND ""
  TEST_COMMAND "")

+set(tmp_gemm_operations_file
+    ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/generated/gemm/all_gemm_operations.h.tmp
+)
+set(tmp_configurations_file
+    ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/generated/gemm/configurations.h.tmp
+)
+set(gemm_operations_file
+    ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/all_gemm_operations.h
+)
+set(configurations_file
+    ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/configurations.h
+)
+
+add_custom_target(
+  cutlass_codegen
+  COMMAND
+    ${PYTHON_EXECUTABLE} -B
+    ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/gather_gemm_scatter_generator.py
+    "${CUTLASS_SOURCE_DIR}/tools/library/scripts/"
+    "${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator"
+    "${CMAKE_CUDA_COMPILER_VERSION}"
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_gemm_operations_file}
+          ${gemm_operations_file}
+  COMMAND
+    ${CMAKE_COMMAND} -E echo
+    "copy_if_different ${tmp_gemm_operations_file} to ${gemm_operations_file}"
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_configurations_file}
+          ${configurations_file}
+  COMMAND
+    ${CMAKE_COMMAND} -E echo
+    "copy_if_different ${tmp_configurations_file} to ${configurations_file}"
+  VERBATIM)
+
 add_library(cutlass INTERFACE)

+add_dependencies(cutlass_codegen extern_cutlass)
 add_dependencies(cutlass extern_cutlass)
--- a/cmake/external/dgc.cmake
+++ b/cmake/external/dgc.cmake
@@ -12,8 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+if(WIN32)
+  message(SEND_ERROR "The current dgc support linux only")
+  return()
+endif()
+
 include(ExternalProject)

+set(DGC_DOWNLOAD_DIR ${PADDLE_SOURCE_DIR}/third_party/dgc/${CMAKE_SYSTEM_NAME})
 set(DGC_PREFIX_DIR "${THIRD_PARTY_PATH}/dgc")
 set(DGC_SOURCES_DIR "${THIRD_PARTY_PATH}/dgc/src/extern_dgc")
 set(DGC_INSTALL_DIR "${THIRD_PARTY_PATH}/install/dgc")
@@ -23,17 +29,49 @@ set(DGC_INCLUDE_DIR
 set(DGC_LIBRARIES
    "${DGC_INSTALL_DIR}/lib/libdgc.a"
    CACHE FILEPATH "dgc library." FORCE)
-set(DGC_URL "https://fleet.bj.bcebos.com/dgc/collective_f66ef73.tgz")
+set(DGC_URL "https://fleet.bj.bcebos.com/dgc/collective_7369ff.tgz")
 include_directories(${DGC_INCLUDE_DIR})
+set(DGC_CACHE_FILENAME "collective_7369ff.tgz")
+set(DGC_URL_MD5 ede459281a0f979da8d84f81287369ff)
+
+function(download_dgc)
+  message(
+    STATUS "Downloading ${DGC_URL} to ${DGC_DOWNLOAD_DIR}/${DGC_CACHE_FILENAME}"
+  )
+  # NOTE: If the version is updated, consider emptying the folder; maybe add timeout
+  file(
+    DOWNLOAD ${DGC_URL} ${DGC_DOWNLOAD_DIR}/${DGC_CACHE_FILENAME}
+    EXPECTED_MD5 ${DGC_URL_MD5}
+    STATUS ERR)
+  if(ERR EQUAL 0)
+    message(STATUS "Download ${DGC_CACHE_FILENAME} success")
+  else()
+    message(
+      FATAL_ERROR
+        "Download failed, error: ${ERR}\n You can try downloading ${DGC_CACHE_FILENAME} again"
+    )
+  endif()
+endfunction()
+
+if(EXISTS ${DGC_DOWNLOAD_DIR}/${DGC_CACHE_FILENAME})
+  file(MD5 ${DGC_DOWNLOAD_DIR}/${DGC_CACHE_FILENAME} DGC_MD5)
+  if(NOT DGC_MD5 STREQUAL DGC_URL_MD5)
+    download_dgc()
+  endif()
+else()
+  download_dgc()
+endif()

 ExternalProject_Add(
  extern_dgc
  ${EXTERNAL_PROJECT_LOG_ARGS}
-  URL ${DGC_URL}
-  URL_MD5 "94e6fa1bc97169d0e1aad44570fe3251"
+  URL ${DGC_DOWNLOAD_DIR}/${DGC_CACHE_FILENAME}
+  URL_MD5 ${DGC_URL_MD5}
  PREFIX "${DGC_PREFIX_DIR}"
  CONFIGURE_COMMAND ""
  BUILD_COMMAND make -j${NPROC}
+  DOWNLOAD_DIR ${DGC_DOWNLOAD_DIR}
+  SOURCE_DIR ${DGC_SOURCES_DIR}
  INSTALL_COMMAND
    mkdir -p ${DGC_INSTALL_DIR}/lib/ ${DGC_INCLUDE_DIR}/dgc && cp
    ${DGC_SOURCES_DIR}/build/lib/libdgc.a ${DGC_LIBRARIES} && cp

--- a/cmake/external/dirent.cmake
+++ b/cmake/external/dirent.cmake
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,20 +17,65 @@

 include(ExternalProject)

+if((NOT DEFINED DIRENT_NAME) OR (NOT DEFINED DIRENT_URL))
+  set(DIRENT_VER
+      "1.23.2"
+      CACHE STRING "" FORCE)
+  set(DIRENT_NAME
+      "dirent"
+      CACHE STRING "" FORCE)
+  set(DIRENT_URL
+      "https://github.com/tronkko/dirent/archive/refs/tags/1.23.2.tar.gz"
+      CACHE STRING "" FORCE)
+  set(DIRENT_CACHE_FILENAME "1.23.2.tar.gz")
+endif()
+
+message(STATUS "DIRENT_NAME: ${DIRENT_NAME}, DIRENT_URL: ${DIRENT_URL}")
+set(DIRENT_DOWNLOAD_DIR "${PADDLE_SOURCE_DIR}/third_party/dirent")
 set(DIRENT_PREFIX_DIR ${THIRD_PARTY_PATH}/dirent)
 set(DIRENT_INCLUDE_DIR ${THIRD_PARTY_PATH}/dirent/src/extern_dirent/include)
+set(DIRENT_URL_MD5 "6bf6319ae71432ed6a4d90dc61e80131")

 include_directories(${DIRENT_INCLUDE_DIR})

-set(DIRENT_REPOSITORY ${GIT_URL}/tronkko/dirent)
-set(DIRENT_TAG 1.23.2)
+function(download_dirent)
+  message(
+    STATUS
+      "Downloading ${DIRENT_URL} to ${DIRENT_DOWNLOAD_DIR}/${DIRENT_CACHE_FILENAME}"
+  )
+  # NOTE: If the version is updated, consider emptying the folder; maybe add timeout
+  file(
+    DOWNLOAD ${DIRENT_URL} ${DIRENT_DOWNLOAD_DIR}/${DIRENT_CACHE_FILENAME}
+    EXPECTED_MD5 ${DIRENT_URL_MD5}
+    STATUS ERR)
+  if(ERR EQUAL 0)
+    message(STATUS "Download ${DIRENT_CACHE_FILENAME} success")
+  else()
+    message(
+      FATAL_ERROR
+        "Download failed, error: ${ERR}\n You can try downloading ${DIRENT_CACHE_FILENAME} again"
+    )
+  endif()
+endfunction()
+
+if(EXISTS ${DIRENT_DOWNLOAD_DIR}/${DIRENT_CACHE_FILENAME})
+  file(MD5 ${DIRENT_DOWNLOAD_DIR}/${DIRENT_CACHE_FILENAME} DIRENT_MD5)
+  if(NOT DIRENT_MD5 STREQUAL DIRENT_URL_MD5)
+    # clean build file
+    file(REMOVE_RECURSE ${DIRENT_PREFIX_DIR})
+    download_dirent()
+  endif()
+else()
+  download_dirent()
+endif()

 ExternalProject_Add(
  extern_dirent
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${DIRENT_REPOSITORY}
-  GIT_TAG ${DIRENT_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  URL ${DIRENT_DOWNLOAD_DIR}/${DIRENT_CACHE_FILENAME}
  PREFIX ${DIRENT_PREFIX_DIR}
+  DOWNLOAD_DIR ${DIRENT_DOWNLOAD_DIR}
+  DOWNLOAD_NO_PROGRESS 1
  UPDATE_COMMAND ""
  CONFIGURE_COMMAND ""
  BUILD_COMMAND ""

--- a/cmake/external/dlpack.cmake
+++ b/cmake/external/dlpack.cmake
@@ -15,18 +15,15 @@
 include(ExternalProject)

 set(DLPACK_PREFIX_DIR ${THIRD_PARTY_PATH}/dlpack)
-
-set(DLPACK_REPOSITORY ${GIT_URL}/dmlc/dlpack.git)
 set(DLPACK_TAG v0.4)
-
 set(DLPACK_INCLUDE_DIR ${THIRD_PARTY_PATH}/dlpack/src/extern_dlpack/include)
-include_directories(${DLPACK_INCLUDE_DIR})
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/dlpack)
+include_directories(${SOURCE_DIR}/include)

 ExternalProject_Add(
  extern_dlpack
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${DLPACK_REPOSITORY}
-  GIT_TAG ${DLPACK_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
  PREFIX ${DLPACK_PREFIX_DIR}
  UPDATE_COMMAND ""
  CONFIGURE_COMMAND ""

--- a/cmake/external/eigen.cmake
+++ b/cmake/external/eigen.cmake
@@ -17,8 +17,8 @@ include(ExternalProject)
 # update eigen to the commit id f612df27 on 03/16/2021
 set(EIGEN_PREFIX_DIR ${THIRD_PARTY_PATH}/eigen3)
 set(EIGEN_SOURCE_DIR ${THIRD_PARTY_PATH}/eigen3/src/extern_eigen3)
-set(EIGEN_REPOSITORY https://gitlab.com/libeigen/eigen.git)
 set(EIGEN_TAG f612df273689a19d25b45ca4f8269463207c4fee)
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/eigen3)

 if(WIN32)
  add_definitions(-DEIGEN_STRONG_INLINE=inline)
@@ -28,27 +28,47 @@ elseif(LINUX)
    # which will cause compiler error of using __host__ funciont
    # in __host__ __device__
    file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/Meta.h native_src)
-    file(TO_NATIVE_PATH ${EIGEN_SOURCE_DIR}/Eigen/src/Core/util/Meta.h
-         native_dst)
+    file(TO_NATIVE_PATH ${SOURCE_DIR}/Eigen/src/Core/util/Meta.h native_dst)
    file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/TensorReductionGpu.h
         native_src1)
-    file(
-      TO_NATIVE_PATH
-      ${EIGEN_SOURCE_DIR}/unsupported/Eigen/CXX11/src/Tensor/TensorReductionGpu.h
-      native_dst1)
+    file(TO_NATIVE_PATH
+         ${SOURCE_DIR}/unsupported/Eigen/CXX11/src/Tensor/TensorReductionGpu.h
+         native_dst1)
    set(EIGEN_PATCH_COMMAND cp ${native_src} ${native_dst} && cp ${native_src1}
                            ${native_dst1})
  endif()
 endif()

-set(EIGEN_INCLUDE_DIR ${EIGEN_SOURCE_DIR})
+if(CMAKE_COMPILER_IS_GNUCC)
+  file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/TensorRandom.h.patch
+       tensor_random_header)
+  # See: [Why calling some `git` commands before `patch`?]
+  set(EIGEN_PATCH_COMMAND
+      git checkout -- . && git checkout ${EIGEN_TAG} && patch -Nd
+      ${SOURCE_DIR}/unsupported/Eigen/CXX11/src/Tensor <
+      ${tensor_random_header})
+  execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpfullversion -dumpversion
+                  OUTPUT_VARIABLE GCC_VERSION)
+  string(REGEX MATCHALL "[0-9]+" GCC_VERSION_COMPONENTS ${GCC_VERSION})
+  list(GET GCC_VERSION_COMPONENTS 0 GCC_MAJOR)
+  list(GET GCC_VERSION_COMPONENTS 1 GCC_MINOR)
+  set(GCC_VERSION "${GCC_MAJOR}.${GCC_MINOR}")
+  if(GCC_VERSION GREATER_EQUAL "12.0")
+    file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/Complex.h.patch
+         complex_header)
+    set(EIGEN_PATCH_COMMAND
+        ${EIGEN_PATCH_COMMAND} && patch -Nd
+        ${SOURCE_DIR}/Eigen/src/Core/arch/SSE/ < ${complex_header})
+  endif()
+endif()
+
+set(EIGEN_INCLUDE_DIR ${SOURCE_DIR})
 include_directories(${EIGEN_INCLUDE_DIR})

 ExternalProject_Add(
  extern_eigen3
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${EIGEN_REPOSITORY}
-  GIT_TAG ${EIGEN_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
  PREFIX ${EIGEN_PREFIX_DIR}
  UPDATE_COMMAND ""
  PATCH_COMMAND ${EIGEN_PATCH_COMMAND}

--- a/cmake/external/flashattn.cmake
+++ b/cmake/external/flashattn.cmake
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include(ExternalProject)
+
+add_definitions(-DPADDLE_WITH_FLASHATTN)
+
+set(FLASHATTN_PREFIX_DIR ${THIRD_PARTY_PATH}/flashattn)
+set(FLASHATTN_SOURCE_SUBDIR csrc)
+set(FLASHATTN_INSTALL_DIR ${THIRD_PARTY_PATH}/install/flashattn)
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/flashattn)
+set(FLASHATTN_TAG 18106c1ba0ccee81b97ca947397c08a141815a47)
+
+set(FLASHATTN_INCLUDE_DIR
+    "${FLASHATTN_INSTALL_DIR}/include"
+    CACHE PATH "flash-attn Directory" FORCE)
+set(FLASHATTN_LIB_DIR
+    "${FLASHATTN_INSTALL_DIR}/lib"
+    CACHE PATH "flash-attn Library Directory" FORCE)
+
+if(WIN32)
+  set(FLASHATTN_LIBRARIES
+      "${FLASHATTN_INSTALL_DIR}/bin/flashattn${CMAKE_SHARED_LIBRARY_SUFFIX}"
+      CACHE FILEPATH "flash-attn Library" FORCE)
+else()
+  set(FLASHATTN_LIBRARIES
+      "${FLASHATTN_INSTALL_DIR}/lib/libflashattn${CMAKE_SHARED_LIBRARY_SUFFIX}"
+      CACHE FILEPATH "flash-attn Library" FORCE)
+endif()
+
+if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang"
+   OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang"
+   OR WIN32)
+  set(USE_OMP OFF)
+else()
+  set(USE_OMP ON)
+endif()
+
+if(WIN32)
+  set(FLASHATTN_C_FLAGS $<FILTER:${CMAKE_C_FLAGS},EXCLUDE,/Zc:inline>)
+  set(FLASHATTN_C_FLAGS_DEBUG
+      $<FILTER:${CMAKE_C_FLAGS_DEBUG},EXCLUDE,/Zc:inline>)
+  set(FLASHATTN_C_FLAGS_RELEASE
+      $<FILTER:${CMAKE_C_FLAGS_RELEASE},EXCLUDE,/Zc:inline>)
+  set(FLASHATTN_CXX_FLAGS $<FILTER:${CMAKE_CXX_FLAGS},EXCLUDE,/Zc:inline>)
+  set(FLASHATTN_CXX_FLAGS_RELEASE
+      $<FILTER:${CMAKE_CXX_FLAGS_RELEASE},EXCLUDE,/Zc:inline>)
+  set(FLASHATTN_CXX_FLAGS_DEBUG
+      $<FILTER:${CMAKE_CXX_FLAGS_DEBUG},EXCLUDE,/Zc:inline>)
+else()
+  set(FLASHATTN_C_FLAGS ${CMAKE_C_FLAGS})
+  set(FLASHATTN_C_FLAGS_DEBUG ${CMAKE_C_FLAGS_DEBUG})
+  set(FLASHATTN_C_FLAGS_RELEASE ${CMAKE_C_FLAGS_RELEASE})
+  set(FLASHATTN_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
+  set(FLASHATTN_CXX_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE})
+  set(FLASHATTN_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG})
+endif()
+
+ExternalProject_Add(
+  extern_flashattn
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
+  PREFIX ${FLASHATTN_PREFIX_DIR}
+  SOURCE_SUBDIR ${FLASHATTN_SOURCE_SUBDIR}
+  UPDATE_COMMAND ""
+  PATCH_COMMAND ""
+  #BUILD_ALWAYS    1
+  CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+             -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+             -DCMAKE_C_FLAGS=${FLASHATTN_C_FLAGS}
+             -DCMAKE_C_FLAGS_DEBUG=${FLASHATTN_C_FLAGS_DEBUG}
+             -DCMAKE_C_FLAGS_RELEASE=${FLASHATTN_C_FLAGS_RELEASE}
+             -DCMAKE_CXX_FLAGS=${FLASHATTN_CXX_FLAGS}
+             -DCMAKE_CXX_FLAGS_RELEASE=${FLASHATTN_CXX_FLAGS_RELEASE}
+             -DCMAKE_CXX_FLAGS_DEBUG=${FLASHATTN_CXX_FLAGS_DEBUG}
+             -DCMAKE_INSTALL_PREFIX=${FLASHATTN_INSTALL_DIR}
+             -DWITH_GPU=${WITH_GPU}
+             -DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}
+             -DWITH_ROCM=${WITH_ROCM}
+             -DWITH_OMP=${USE_OMP}
+             -DBUILD_SHARED=ON
+             -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+             -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
+             -DCMAKE_JOB_POOL_COMPILE:STRING=compile
+             -DCMAKE_JOB_POOLS:STRING=compile=4
+             ${EXTERNAL_OPTIONAL_ARGS}
+  CMAKE_CACHE_ARGS
+    -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
+    -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
+    -DCMAKE_INSTALL_PREFIX:PATH=${FLASHATTN_INSTALL_DIR}
+  BUILD_BYPRODUCTS ${FLASHATTN_LIBRARIES})
+
+message(STATUS "flash-attn library: ${FLASHATTN_LIBRARIES}")
+get_filename_component(FLASHATTN_LIBRARY_PATH ${FLASHATTN_LIBRARIES} DIRECTORY)
+include_directories(${FLASHATTN_INCLUDE_DIR})
+
+add_library(flashattn INTERFACE)
+#set_property(TARGET flashattn PROPERTY IMPORTED_LOCATION ${FLASHATTN_LIBRARIES})
+add_dependencies(flashattn extern_flashattn)
--- a/cmake/external/gflags.cmake
+++ b/cmake/external/gflags.cmake
@@ -14,13 +14,13 @@

 include(ExternalProject)

-set(GFLAGS_PREFIX_DIR ${THIRD_PARTY_PATH}/gflags)
 set(GFLAGS_INSTALL_DIR ${THIRD_PARTY_PATH}/install/gflags)
+set(GFLAGS_PREFIX_DIR ${THIRD_PARTY_PATH}/gflags)
 set(GFLAGS_INCLUDE_DIR
    "${GFLAGS_INSTALL_DIR}/include"
    CACHE PATH "gflags include directory." FORCE)
-set(GFLAGS_REPOSITORY ${GIT_URL}/gflags/gflags.git)
 set(GFLAGS_TAG "v2.2.2")
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/gflags)
 if(WIN32)
  set(GFLAGS_LIBRARIES
      "${GFLAGS_INSTALL_DIR}/lib/gflags_static.lib"
@@ -63,10 +63,9 @@ if(WITH_ARM_BRPC)
 else()
  ExternalProject_Add(
    extern_gflags
-    ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-    GIT_REPOSITORY ${GFLAGS_REPOSITORY}
-    GIT_TAG ${GFLAGS_TAG}
+    ${EXTERNAL_PROJECT_LOG_ARGS}
    PREFIX ${GFLAGS_PREFIX_DIR}
+    SOURCE_DIR ${SOURCE_DIR}
    UPDATE_COMMAND ""
    BUILD_COMMAND ${BUILD_COMMAND}
    INSTALL_COMMAND ${INSTALL_COMMAND}

--- a/cmake/external/glog.cmake
+++ b/cmake/external/glog.cmake
@@ -19,9 +19,8 @@ set(GLOG_INSTALL_DIR ${THIRD_PARTY_PATH}/install/glog)
 set(GLOG_INCLUDE_DIR
    "${GLOG_INSTALL_DIR}/include"
    CACHE PATH "glog include directory." FORCE)
-set(GLOG_REPOSITORY ${GIT_URL}/google/glog.git)
 set(GLOG_TAG v0.4.0)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/glog)
 if(WIN32)
  set(GLOG_LIBRARIES
      "${GLOG_INSTALL_DIR}/lib/glog.lib"
@@ -67,8 +66,7 @@ else()
  ExternalProject_Add(
    extern_glog
    ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-    GIT_REPOSITORY ${GLOG_REPOSITORY}
-    GIT_TAG ${GLOG_TAG}
+    SOURCE_DIR ${SOURCE_DIR}
    DEPENDS gflags
    PREFIX ${GLOG_PREFIX_DIR}
    UPDATE_COMMAND ""

--- a/cmake/external/gloo.cmake
+++ b/cmake/external/gloo.cmake
@@ -25,63 +25,61 @@ set(GLOO_LIBRARY_DIR
    "${GLOO_INSTALL_DIR}/lib"
    CACHE PATH "gloo library directory." FORCE)
 # As we add extra features for gloo, we use the non-official repo
-set(GLOO_REPOSITORY ${GIT_URL}/ziyoujiyi/gloo.git)
 set(GLOO_TAG v0.0.3)
 set(GLOO_LIBRARIES
    "${GLOO_INSTALL_DIR}/lib/libgloo.a"
    CACHE FILEPATH "gloo library." FORCE)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/gloo)
 set(GLOO_PATCH_COMMAND "")
 if(WITH_GPU)
  if(${CMAKE_CUDA_COMPILER_VERSION} LESS 12.0 AND ${CMAKE_CXX_COMPILER_VERSION}
                                                  VERSION_GREATER 12.0)
    file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/gloo/device.cc.patch
         native_dst)
-    set(GLOO_PATCH_COMMAND patch -d ${GLOO_SOURCE_DIR}/gloo/transport/tcp <
-                           ${native_dst})
+    set(GLOO_PATCH_COMMAND
+        git checkout -- . && git checkout ${GLOO_TAG} &&patch -Nd
+        ${SOURCE_DIR}/gloo/transport/tcp < ${native_dst})
  endif()
 endif()

+if(CMAKE_COMPILER_IS_GNUCC)
+  execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpfullversion -dumpversion
+                  OUTPUT_VARIABLE GCC_VERSION)
+  string(REGEX MATCHALL "[0-9]+" GCC_VERSION_COMPONENTS ${GCC_VERSION})
+  list(GET GCC_VERSION_COMPONENTS 0 GCC_MAJOR)
+  list(GET GCC_VERSION_COMPONENTS 1 GCC_MINOR)
+  set(GCC_VERSION "${GCC_MAJOR}.${GCC_MINOR}")
+  if(GCC_VERSION GREATER_EQUAL "12.0")
+    file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/gloo/device.cc.patch
+         native_dst)
+    file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/gloo/types.h.patch
+         types_header)
+    # See: [Why calling some `git` commands before `patch`?]
+    set(GLOO_PATCH_COMMAND
+        git checkout -- . && git checkout ${GLOO_TAG} && patch -Nd
+        ${SOURCE_DIR}/gloo/transport/tcp < ${native_dst} && patch -Nd
+        ${SOURCE_DIR}/gloo/ < ${types_header})
+  endif()
+endif()
 include_directories(${GLOO_INCLUDE_DIR})

-if(WITH_ASCEND OR WITH_ASCEND_CL)
-  ExternalProject_Add(
-    ${GLOO_PROJECT}
-    ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-    GIT_REPOSITORY ${GLOO_REPOSITORY}
-    GIT_TAG ${GLOO_TAG}
-    PREFIX "${GLOO_PREFIX_DIR}"
-    UPDATE_COMMAND ""
-    CONFIGURE_COMMAND ""
-    BUILD_COMMAND
-      mkdir -p ${GLOO_SOURCE_DIR}/build && cd ${GLOO_SOURCE_DIR}/build && cmake
-      .. -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} && make && mkdir -p
-      ${GLOO_LIBRARY_DIR} ${GLOO_INCLUDE_DIR}/gloo
-    INSTALL_COMMAND ${CMAKE_COMMAND} -E copy
-                    ${GLOO_SOURCE_DIR}/build/gloo/libgloo.a ${GLOO_LIBRARY_DIR}
-    COMMAND ${CMAKE_COMMAND} -E copy_directory "${GLOO_SOURCE_DIR}/gloo/"
-            "${GLOO_INCLUDE_DIR}/gloo"
-    BUILD_BYPRODUCTS ${GLOO_LIBRARIES})
-else()
-  ExternalProject_Add(
-    ${GLOO_PROJECT}
-    ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-    GIT_REPOSITORY ${GLOO_REPOSITORY}
-    GIT_TAG ${GLOO_TAG}
-    PREFIX "${GLOO_PREFIX_DIR}"
-    UPDATE_COMMAND ""
-    PATCH_COMMAND ${GLOO_PATCH_COMMAND}
-    CONFIGURE_COMMAND ""
-    BUILD_COMMAND
-      mkdir -p ${GLOO_SOURCE_DIR}/build && cd ${GLOO_SOURCE_DIR}/build && cmake
-      .. -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} && make && mkdir -p
-      ${GLOO_LIBRARY_DIR} ${GLOO_INCLUDE_DIR}/gloo
-    INSTALL_COMMAND ${CMAKE_COMMAND} -E copy
-                    ${GLOO_SOURCE_DIR}/build/gloo/libgloo.a ${GLOO_LIBRARY_DIR}
-    COMMAND ${CMAKE_COMMAND} -E copy_directory "${GLOO_SOURCE_DIR}/gloo/"
-            "${GLOO_INCLUDE_DIR}/gloo"
-    BUILD_BYPRODUCTS ${GLOO_LIBRARIES})
-endif()
+ExternalProject_Add(
+  ${GLOO_PROJECT}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
+  PREFIX "${GLOO_PREFIX_DIR}"
+  UPDATE_COMMAND ""
+  PATCH_COMMAND ${GLOO_PATCH_COMMAND}
+  CONFIGURE_COMMAND ""
+  BUILD_COMMAND
+    mkdir -p ${GLOO_SOURCE_DIR}/build && cd ${GLOO_SOURCE_DIR}/build && cmake
+    ${SOURCE_DIR} -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} && ${CMAKE_COMMAND}
+    --build . && mkdir -p ${GLOO_LIBRARY_DIR} ${GLOO_INCLUDE_DIR}/glo
+  INSTALL_COMMAND ${CMAKE_COMMAND} -E copy
+                  ${GLOO_SOURCE_DIR}/build/gloo/libgloo.a ${GLOO_LIBRARY_DIR}
+  COMMAND ${CMAKE_COMMAND} -E copy_directory "${SOURCE_DIR}/gloo/"
+          "${GLOO_INCLUDE_DIR}/gloo"
+  BUILD_BYPRODUCTS ${GLOO_LIBRARIES})

 add_library(gloo STATIC IMPORTED GLOBAL)
 set_property(TARGET gloo PROPERTY IMPORTED_LOCATION ${GLOO_LIBRARIES})

--- a/cmake/external/gtest.cmake
+++ b/cmake/external/gtest.cmake
@@ -26,9 +26,9 @@ set(GTEST_INSTALL_DIR ${THIRD_PARTY_PATH}/install/gtest)
 set(GTEST_INCLUDE_DIR
    "${GTEST_INSTALL_DIR}/include"
    CACHE PATH "gtest include directory." FORCE)
-set(GTEST_REPOSITORY ${GIT_URL}/google/googletest.git)
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/gtest)
 set(GTEST_TAG release-1.8.1)
-
+set(GTEST_SOURCE_DIR ${THIRD_PARTY_PATH}/gtest/src/extern_gtest)
 include_directories(${GTEST_INCLUDE_DIR})

 if(WIN32)
@@ -39,7 +39,7 @@ if(WIN32)
      "${GTEST_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/gtest_main.lib"
      CACHE FILEPATH "gtest main libraries." FORCE)
  set(GMOCK_LIBRARIES
-      "${GTEST_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libgmock.lib"
+      "${GTEST_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/gmock.lib"
      CACHE FILEPATH "gmock libraries." FORCE)
  string(REPLACE "/w " "" GTEST_CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
  string(REPLACE "/w " "" GTEST_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
@@ -63,37 +63,78 @@ if(WITH_MKLML)
  # wait for mklml downloading completed
  set(GTEST_DEPENDS ${MKLML_PROJECT})
 endif()
-
-ExternalProject_Add(
-  extern_gtest
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${GTEST_REPOSITORY}
-  GIT_TAG ${GTEST_TAG}
-  DEPENDS ${GTEST_DEPENDS}
-  PREFIX ${GTEST_PREFIX_DIR}
-  UPDATE_COMMAND ""
-  CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-             -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-             -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS}
-             -DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE}
-             -DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG}
-             -DCMAKE_C_FLAGS=${GTEST_CMAKE_C_FLAGS}
-             -DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG}
-             -DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE}
-             -DCMAKE_INSTALL_PREFIX=${GTEST_INSTALL_DIR}
-             -DCMAKE_POSITION_INDEPENDENT_CODE=ON
-             -DBUILD_GMOCK=ON
-             -Dgtest_disable_pthreads=ON
-             -Dgtest_force_shared_crt=ON
-             -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
-             ${EXTERNAL_OPTIONAL_ARGS}
-  CMAKE_CACHE_ARGS
-    -DCMAKE_INSTALL_PREFIX:PATH=${GTEST_INSTALL_DIR}
-    -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
-    -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
-  BUILD_BYPRODUCTS ${GTEST_LIBRARIES}
-  BUILD_BYPRODUCTS ${GTEST_MAIN_LIBRARIES}
-  BUILD_BYPRODUCTS ${GMOCK_LIBRARIES})
+if(NOT WIN32 AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER 12.0)
+  file(TO_NATIVE_PATH
+       ${PADDLE_SOURCE_DIR}/patches/gtest/gtest-death-test.cc.patch native_src)
+  # See: [Why calling some `git` commands before `patch`?]
+  set(GTEST_PATCH_COMMAND
+      git checkout -- . && git checkout ${GTEST_TAG} && patch -Nd
+      ${SOURCE_DIR}/googletest/src < ${native_src})
+endif()
+if(WIN32)
+  ExternalProject_Add(
+    extern_gtest
+    ${EXTERNAL_PROJECT_LOG_ARGS}
+    SOURCE_DIR ${SOURCE_DIR}
+    DEPENDS ${GTEST_DEPENDS}
+    PREFIX ${GTEST_PREFIX_DIR}
+    UPDATE_COMMAND ""
+    PATCH_COMMAND ${GTEST_PATCH_COMMAND}
+    CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+               -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+               -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS}
+               -DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE}
+               -DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG}
+               -DCMAKE_C_FLAGS=${GTEST_CMAKE_C_FLAGS}
+               -DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG}
+               -DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE}
+               -DCMAKE_INSTALL_PREFIX=${GTEST_INSTALL_DIR}
+               -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+               -DBUILD_GMOCK=ON
+               -Dgtest_disable_pthreads=ON
+               -Dgtest_force_shared_crt=ON
+               -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
+               ${EXTERNAL_OPTIONAL_ARGS}
+    CMAKE_CACHE_ARGS
+      -DCMAKE_INSTALL_PREFIX:PATH=${GTEST_INSTALL_DIR}
+      -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
+      -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
+    BUILD_BYPRODUCTS ${GTEST_LIBRARIES}
+    BUILD_BYPRODUCTS ${GTEST_MAIN_LIBRARIES}
+    BUILD_BYPRODUCTS ${GMOCK_LIBRARIES})
+else()
+  ExternalProject_Add(
+    extern_gtest
+    ${EXTERNAL_PROJECT_LOG_ARGS}
+    SOURCE_DIR ${SOURCE_DIR}
+    DEPENDS ${GTEST_DEPENDS}
+    PREFIX ${GTEST_PREFIX_DIR}
+    UPDATE_COMMAND ""
+    PATCH_COMMAND ${GTEST_PATCH_COMMAND}
+    CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+               -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+               -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS}
+               -DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE}
+               -DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG}
+               -DCMAKE_C_FLAGS=${GTEST_CMAKE_C_FLAGS}
+               -DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG}
+               -DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE}
+               -DCMAKE_INSTALL_PREFIX=${GTEST_INSTALL_DIR}
+               -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+               -DBUILD_GMOCK=ON
+               -Dgtest_disable_pthreads=ON
+               -Dgtest_force_shared_crt=ON
+               -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
+               ${EXTERNAL_OPTIONAL_ARGS}
+    CMAKE_CACHE_ARGS
+      -DCMAKE_INSTALL_PREFIX:PATH=${GTEST_INSTALL_DIR}
+      -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
+      -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
+    CMAKE_GENERATOR "Unix Makefiles"
+    BUILD_BYPRODUCTS ${GTEST_LIBRARIES}
+    BUILD_BYPRODUCTS ${GTEST_MAIN_LIBRARIES}
+    BUILD_BYPRODUCTS ${GMOCK_LIBRARIES})
+endif()

 add_library(gtest STATIC IMPORTED GLOBAL)
 set_property(TARGET gtest PROPERTY IMPORTED_LOCATION ${GTEST_LIBRARIES})

--- a/cmake/external/jemalloc.cmake
+++ b/cmake/external/jemalloc.cmake
+include(ExternalProject)
+
+set(JEMALLOC_DOWNLOAD_DIR
+    ${PADDLE_SOURCE_DIR}/third_party/jemalloc/${CMAKE_SYSTEM_NAME})
+set(JEMALLOC_PROJECT "extern_jemalloc")
+set(JEMALLOC_BUILD ${THIRD_PARTY_PATH}/jemalloc/src/extern_jemalloc)
+set(JEMALLOC_PREFIX_DIR ${THIRD_PARTY_PATH}/jemalloc)
+set(JEMALLOC_URL
+    https://github.com/jemalloc/jemalloc/releases/download/5.1.0/jemalloc-5.1.0.tar.bz2
+)
+set(JEMALLOC_INSTALL ${THIRD_PARTY_PATH}/install/jemalloc)
+set(JEMALLOC_INCLUDE_DIR ${JEMALLOC_INSTALL}/include)
+
+set(JEMALLOC_STATIC_LIBRARIES
+    ${THIRD_PARTY_PATH}/install/jemalloc/lib/libjemalloc_pic.a)
+set(JEMALLOC_LIBRARIES
+    ${THIRD_PARTY_PATH}/install/jemalloc/lib/libjemalloc_pic.a)
+set(JEMALLOC_CACHE_FILENAME "jemalloc-5.1.0.tar.bz2")
+set(JEMALLOC_URL_MD5 1f47a5aff2d323c317dfa4cf23be1ce4)
+
+function(download_jemalloc)
+  message(
+    STATUS
+      "Downloading ${JEMALLOC_URL} to ${JEMALLOC_DOWNLOAD_DIR}/${JEMALLOC_CACHE_FILENAME}"
+  )
+  # NOTE: If the version is updated, consider emptying the folder; maybe add timeout
+  file(
+    DOWNLOAD ${JEMALLOC_URL} ${JEMALLOC_DOWNLOAD_DIR}/${JEMALLOC_CACHE_FILENAME}
+    EXPECTED_MD5 ${JEMALLOC_URL_MD5}
+    STATUS ERR)
+  if(ERR EQUAL 0)
+    message(STATUS "Download ${JEMALLOC_CACHE_FILENAME} success")
+  else()
+    message(
+      FATAL_ERROR
+        "Download failed, error: ${ERR}\n You can try downloading ${JEMALLOC_CACHE_FILENAME} again"
+    )
+  endif()
+endfunction()
+
+if(EXISTS ${JEMALLOC_DOWNLOAD_DIR}/${JEMALLOC_CACHE_FILENAME})
+  file(MD5 ${JEMALLOC_DOWNLOAD_DIR}/${JEMALLOC_CACHE_FILENAME} JEMALLOC_MD5)
+  if(NOT JEMALLOC_MD5 STREQUAL JEMALLOC_URL_MD5)
+    download_jemalloc()
+  endif()
+else()
+  download_jemalloc()
+endif()
+
+ExternalProject_Add(
+  extern_jemalloc
+  PREFIX ${JEMALLOC_PREFIX_DIR}
+  URL ${JEMALLOC_DOWNLOAD_DIR}/${JEMALLOC_CACHE_FILENAME}
+  URL_MD5 ${JEMALLOC_URL_MD5}
+  #INSTALL_DIR ${JEMALLOC_INSTALL}
+  DOWNLOAD_DIR ${JEMALLOC_DOWNLOAD_DIR}
+  SOURCE_DIR ${JEMALLOC_BUILD}
+  BUILD_IN_SOURCE 1
+  BUILD_COMMAND make
+  INSTALL_COMMAND make install
+  CONFIGURE_COMMAND "${JEMALLOC_BUILD}/configure" --prefix=${JEMALLOC_INSTALL}
+                    --disable-initial-exec-tls)
+
+add_library(jemalloc STATIC IMPORTED GLOBAL)
+set_property(TARGET jemalloc PROPERTY IMPORTED_LOCATION
+                                      ${JEMALLOC_STATIC_LIBRARIES})
+
+include_directories(${JEMALLOC_INCLUDE_DIR})
+add_dependencies(jemalloc extern_jemalloc)
--- a/cmake/external/lapack.cmake
+++ b/cmake/external/lapack.cmake
--- a/cmake/external/leveldb.cmake
+++ b/cmake/external/leveldb.cmake
@@ -14,6 +14,8 @@

 include(ExternalProject)

+set(LEVELDB_TAG v1.18)
+set(LEVELDB_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/leveldb)
 set(LEVELDB_PREFIX_DIR ${THIRD_PARTY_PATH}/leveldb)
 set(LEVELDB_INSTALL_DIR ${THIRD_PARTY_PATH}/install/leveldb)
 set(LEVELDB_INCLUDE_DIR
@@ -23,24 +25,26 @@ set(LEVELDB_LIBRARIES
    "${LEVELDB_INSTALL_DIR}/lib/libleveldb.a"
    CACHE FILEPATH "leveldb library." FORCE)
 include_directories(${LEVELDB_INCLUDE_DIR})
+set(LEVELDN_CXXFLAGS "-fPIC")
+if(WITH_HETERPS AND WITH_PSLIB)
+  set(LEVELDN_CXXFLAGS "${LEVELDN_CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
+endif()

 ExternalProject_Add(
  extern_leveldb
  ${EXTERNAL_PROJECT_LOG_ARGS}
  PREFIX ${LEVELDB_PREFIX_DIR}
-  GIT_REPOSITORY "https://github.com/google/leveldb"
-  GIT_TAG v1.18
+  SOURCE_DIR ${LEVELDB_SOURCE_DIR}
  UPDATE_COMMAND ""
  CONFIGURE_COMMAND ""
-  BUILD_COMMAND CXXFLAGS=-fPIC make -j ${NUM_OF_PROCESSOR} libleveldb.a
+  BUILD_COMMAND export "CXXFLAGS=${LEVELDN_CXXFLAGS}" && make -j
+                ${NUM_OF_PROCESSOR} libleveldb.a
  INSTALL_COMMAND
    mkdir -p ${LEVELDB_INSTALL_DIR}/lib/ && cp
-    ${LEVELDB_PREFIX_DIR}/src/extern_leveldb/libleveldb.a ${LEVELDB_LIBRARIES}
-    && cp -r ${LEVELDB_PREFIX_DIR}/src/extern_leveldb/include
-    ${LEVELDB_INSTALL_DIR}/
+    ${LEVELDB_SOURCE_DIR}/libleveldb.a ${LEVELDB_LIBRARIES} && cp -r
+    ${LEVELDB_SOURCE_DIR}/include ${LEVELDB_INSTALL_DIR}/
  BUILD_IN_SOURCE 1
  BUILD_BYPRODUCTS ${LEVELDB_LIBRARIES})
-
 add_dependencies(extern_leveldb snappy)

 add_library(leveldb STATIC IMPORTED GLOBAL)

--- a/cmake/external/libmct.cmake
+++ b/cmake/external/libmct.cmake