CMakeLists.txt 7.82 KB
Newer Older
Li Zhang's avatar
Li Zhang committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#  * Neither the name of NVIDIA CORPORATION nor the names of its
#    contributors may be used to endorse or promote products derived
#    from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

cmake_minimum_required (VERSION 3.18)

29
project(tritonturbomindbackend LANGUAGES C CXX)
Li Zhang's avatar
Li Zhang committed
30

Chen Xin's avatar
Chen Xin committed
31
32
33
34
35
36
37
38
39
40
41
42
add_library(TransformerTritonBackend STATIC transformer_triton_backend.cpp)
target_link_libraries(TransformerTritonBackend PUBLIC nccl_utils)
set_property(TARGET TransformerTritonBackend PROPERTY POSITION_INDEPENDENT_CODE ON)
install(TARGETS TransformerTritonBackend DESTINATION ${CMAKE_INSTALL_LIBDIR})

add_subdirectory(llama)

# Needn't build triton backend on windows
if (MSVC)
  return ()
endif()

Li Zhang's avatar
Li Zhang committed
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
#
# Options
#
option(TRITON_ENABLE_GPU "Enable GPU support in backend" ON)
option(TRITON_ENABLE_STATS "Include statistics collections in backend" ON)

set(TRITON_PYTORCH_INCLUDE_PATHS "" CACHE PATH "Paths to Torch includes")
set(TRITON_PYTORCH_LIB_PATHS "" CACHE PATH "Paths to Torch libraries")

set(TRITON_BACKEND_REPO_TAG "r22.12" CACHE STRING "Tag for triton-inference-server/backend repo")
set(TRITON_CORE_REPO_TAG "r22.12" CACHE STRING "Tag for triton-inference-server/core repo")
set(TRITON_COMMON_REPO_TAG "r22.12" CACHE STRING "Tag for triton-inference-server/common repo")

if(NOT CMAKE_BUILD_TYPE)
  set(CMAKE_BUILD_TYPE Release)
endif()

set(USE_TRITONSERVER_DATATYPE "ON")
message("-- Enable USE_TRITONSERVER_DATATYPE")

#
# Dependencies
#
# FetchContent's composability isn't very good. We must include the
# transitive closure of all repos so that we can override the tag.
#
include(FetchContent)

FetchContent_Declare(
  repo-common
  GIT_REPOSITORY https://github.com/triton-inference-server/common.git
  GIT_TAG ${TRITON_COMMON_REPO_TAG}
  GIT_SHALLOW ON
)
FetchContent_Declare(
  repo-core
  GIT_REPOSITORY https://github.com/triton-inference-server/core.git
  GIT_TAG ${TRITON_CORE_REPO_TAG}
  GIT_SHALLOW ON
)
FetchContent_Declare(
  repo-backend
  GIT_REPOSITORY https://github.com/triton-inference-server/backend.git
  GIT_TAG ${TRITON_BACKEND_REPO_TAG}
  GIT_SHALLOW ON
)
FetchContent_MakeAvailable(repo-common repo-core repo-backend)

#
# CUDA
#
if(${TRITON_ENABLE_GPU})
  find_package(CUDAToolkit REQUIRED)
endif() # TRITON_ENABLE_GPU

#
# Shared library implementing the Triton Backend API
#
configure_file(libtriton_fastertransformer.ldscript libtriton_fastertransformer.ldscript COPYONLY)

add_library(
104
  triton-turbomind-backend SHARED
Li Zhang's avatar
Li Zhang committed
105
106
107
108
  libfastertransformer.cc
)

add_library(
109
  TritonTurboMindBackend::triton-turbomind-backend ALIAS triton-turbomind-backend
Li Zhang's avatar
Li Zhang committed
110
111
112
113
114
115
116
117
118
119
120
)

find_package(CUDAToolkit REQUIRED)
find_package(CUDA 10.1 REQUIRED)
if (${CUDA_VERSION} GREATER_EQUAL 11.0)
  message(STATUS "Add DCUDA11_MODE")
  add_definitions("-DCUDA11_MODE")
endif()

set(CUDA_PATH ${CUDA_TOOLKIT_ROOT_DIR})

121
target_compile_definitions(triton-turbomind-backend
Li Zhang's avatar
Li Zhang committed
122
123
124
125
126
  PUBLIC
  USE_TRITONSERVER_DATATYPE
  BUILD_MULTI_GPU)

target_include_directories(
127
  triton-turbomind-backend
Li Zhang's avatar
Li Zhang committed
128
129
130
131
132
133
134
135
136
137
  PRIVATE
  ${CMAKE_CURRENT_SOURCE_DIR}/src
  ${TRITON_PYTORCH_INCLUDE_PATHS}
  ${Python3_INCLUDE_DIRS}
  ${repo-ft_SOURCE_DIR}
  ${repo-ft_SOURCE_DIR}/3rdparty/cutlass/include
  ${repo-core_SOURCE_DIR}/include
  )

target_link_directories(
138
  triton-turbomind-backend
Li Zhang's avatar
Li Zhang committed
139
140
141
142
  PRIVATE
  ${CUDA_PATH}/lib64
  )

143
target_compile_features(triton-turbomind-backend PRIVATE cxx_std_14)
Li Zhang's avatar
Li Zhang committed
144
145

target_compile_options(
146
  triton-turbomind-backend PRIVATE
Li Zhang's avatar
Li Zhang committed
147
148
149
150
151
152
  $<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>:
    -Wall -Wextra -Wno-unused-parameter -Wno-type-limits >#-Werror>
)

if(${TRITON_ENABLE_GPU})
  target_compile_definitions(
153
    triton-turbomind-backend
Li Zhang's avatar
Li Zhang committed
154
155
156
157
158
    PRIVATE TRITON_ENABLE_GPU=1
  )
endif() # TRITON_ENABLE_GPU

set_target_properties(
159
  triton-turbomind-backend
Li Zhang's avatar
Li Zhang committed
160
161
  PROPERTIES
    POSITION_INDEPENDENT_CODE ON
162
    OUTPUT_NAME triton_turbomind
Li Zhang's avatar
Li Zhang committed
163
164
165
166
167
    SKIP_BUILD_RPATH TRUE
    BUILD_WITH_INSTALL_RPATH TRUE
    INSTALL_RPATH_USE_LINK_PATH FALSE
    INSTALL_RPATH "$\{ORIGIN\}"
    LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_fastertransformer.ldscript
q.yao's avatar
q.yao committed
168
    LINK_FLAGS "-Wl,--no-as-needed,--version-script ${CMAKE_CURRENT_BINARY_DIR}/libtriton_fastertransformer.ldscript"
Li Zhang's avatar
Li Zhang committed
169
170
171
172
173
)

# Need to turn off unused-but-set-variable due to Torchvision
# Need to turn off unknown-pragmas due to ATen OpenMP
set_target_properties(
174
  triton-turbomind-backend
Li Zhang's avatar
Li Zhang committed
175
176
177
178
179
180
181
182
183
184
  PROPERTIES COMPILE_FLAGS
    "-Wno-unknown-pragmas -Wno-unused-but-set-variable"
)

set(TRITON_PYTORCH_LDFLAGS "")
FOREACH(p ${TRITON_PYTORCH_LIB_PATHS})
  set(TRITON_PYTORCH_LDFLAGS ${TRITON_PYTORCH_LDFLAGS} "-L${p}")
ENDFOREACH(p)

target_link_libraries(
185
  triton-turbomind-backend
Li Zhang's avatar
Li Zhang committed
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
  PRIVATE
    triton-core-serverapi  # from repo-core
    triton-core-backendapi # from repo-core
    triton-core-serverstub # from repo-core
    triton-backend-utils   # from repo-backend
    transformer-shared     # from repo-ft
    ${TRITON_PYTORCH_LDFLAGS}
    -lcublas
    -lcublasLt
    -lcudart
    -lcurand
)

if (BUILD_MULTI_GPU)
  target_compile_definitions(
201
    triton-turbomind-backend
Li Zhang's avatar
Li Zhang committed
202
203
204
205
    PUBLIC
      BUILD_MULTI_GPU
  )
  target_include_directories(
206
    triton-turbomind-backend
Li Zhang's avatar
Li Zhang committed
207
208
209
210
    PRIVATE
      ${MPI_INCLUDE_PATH}
  )
  target_link_directories(
211
    triton-turbomind-backend
Li Zhang's avatar
Li Zhang committed
212
213
214
215
216
    PRIVATE
      ${MPI_Libraries}
      /usr/local/mpi/lib
  )
  target_link_libraries(
217
    triton-turbomind-backend
Li Zhang's avatar
Li Zhang committed
218
219
220
221
222
223
224
225
    PRIVATE
      ${NCCL_LIBRARIES}
      ${MPI_LIBRARIES}
  )
endif()

if(${TRITON_ENABLE_GPU})
  target_link_libraries(
226
    triton-turbomind-backend
Li Zhang's avatar
Li Zhang committed
227
228
229
230
231
232
233
234
235
    PRIVATE
      CUDA::cudart
  )
endif() # TRITON_ENABLE_GPU

#
# Install
#
include(GNUInstallDirs)
236
set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/TurboMindBackend)
Li Zhang's avatar
Li Zhang committed
237
238
239

install(
  TARGETS
240
    triton-turbomind-backend
Li Zhang's avatar
Li Zhang committed
241
  EXPORT
242
243
244
    triton-turbomind-backend-targets
  LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/turbomind
  ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/turbomind
Li Zhang's avatar
Li Zhang committed
245
246
247
248
)

install(
  EXPORT
249
    triton-turbomind-backend-targets
Li Zhang's avatar
Li Zhang committed
250
  FILE
251
    TritonTurboMindBackendTargets.cmake
Li Zhang's avatar
Li Zhang committed
252
  NAMESPACE
253
    TritonTurboMindBackend::
Li Zhang's avatar
Li Zhang committed
254
255
256
257
258
259
  DESTINATION
    ${INSTALL_CONFIGDIR}
)

include(CMakePackageConfigHelpers)
configure_package_config_file(
260
261
  ${CMAKE_SOURCE_DIR}/cmake/TritonTurboMindBackendConfig.cmake.in
  ${CMAKE_CURRENT_BINARY_DIR}/TritonTurboMindBackendConfig.cmake
Li Zhang's avatar
Li Zhang committed
262
263
264
265
266
  INSTALL_DESTINATION ${INSTALL_CONFIGDIR}
)

install(
  FILES
267
  ${CMAKE_CURRENT_BINARY_DIR}/TritonTurboMindBackendConfig.cmake
Li Zhang's avatar
Li Zhang committed
268
269
270
271
272
273
274
  DESTINATION ${INSTALL_CONFIGDIR}
)

#
# Export from build tree
#
export(
275
276
277
  EXPORT triton-turbomind-backend-targets
  FILE ${CMAKE_CURRENT_BINARY_DIR}/TritonTurboMindBackendTargets.cmake
  NAMESPACE TritonTurboMindBackend::
Li Zhang's avatar
Li Zhang committed
278
279
)

280
export(PACKAGE TritonTurboMindBackend)