CMakeLists.txt 23.7 KB
Newer Older
1
cmake_minimum_required(VERSION 3.26)
bnellnm's avatar
bnellnm committed
2

3
4
5
6
7
8
9
10
11
12
13
# When building directly using CMake, make sure you run the install step
# (it places the .so files in the correct location).
#
# Example:
# mkdir build && cd build
# cmake -G Ninja -DVLLM_PYTHON_EXECUTABLE=`which python3` -DCMAKE_INSTALL_PREFIX=.. ..
# cmake --build . --target install
#
# If you want to only build one target, make sure to install it manually:
# cmake --build . --target _C
# cmake --install . --component _C
bnellnm's avatar
bnellnm committed
14
15
project(vllm_extensions LANGUAGES CXX)

16
17
# CUDA by default, can be overridden by using -DVLLM_TARGET_DEVICE=... (used by setup.py)
set(VLLM_TARGET_DEVICE "cuda" CACHE STRING "Target device backend for vLLM")
18

bnellnm's avatar
bnellnm committed
19
message(STATUS "Build type: ${CMAKE_BUILD_TYPE}")
20
message(STATUS "Target device: ${VLLM_TARGET_DEVICE}")
bnellnm's avatar
bnellnm committed
21
22
23

include(${CMAKE_CURRENT_LIST_DIR}/cmake/utils.cmake)

24
25
26
# Suppress potential warnings about unused manually-specified variables
set(ignoreMe "${VLLM_PYTHON_PATH}")

bnellnm's avatar
bnellnm committed
27
28
29
30
#
# Supported python versions.  These versions will be searched in order, the
# first match will be selected.  These should be kept in sync with setup.py.
#
31
set(PYTHON_SUPPORTED_VERSIONS "3.9" "3.10" "3.11" "3.12")
bnellnm's avatar
bnellnm committed
32
33

# Supported NVIDIA architectures.
34
set(CUDA_SUPPORTED_ARCHS "7.0;7.2;7.5;8.0;8.6;8.7;8.9;9.0")
bnellnm's avatar
bnellnm committed
35
36

# Supported AMD GPU architectures.
37
set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101")
bnellnm's avatar
bnellnm committed
38
39
40
41
42
43
44
45
46
47
48

#
# Supported/expected torch versions for CUDA/ROCm.
#
# Currently, having an incorrect pytorch version results in a warning
# rather than an error.
#
# Note: the CUDA torch version is derived from pyproject.toml and various
# requirements.txt files and should be kept consistent.  The ROCm torch
# versions are derived from Dockerfile.rocm
#
49
50
set(TORCH_SUPPORTED_VERSION_CUDA "2.5.1")
set(TORCH_SUPPORTED_VERSION_ROCM "2.5.1")
bnellnm's avatar
bnellnm committed
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

#
# Try to find python package with an executable that exactly matches
# `VLLM_PYTHON_EXECUTABLE` and is one of the supported versions.
#
if (VLLM_PYTHON_EXECUTABLE)
  find_python_from_executable(${VLLM_PYTHON_EXECUTABLE} "${PYTHON_SUPPORTED_VERSIONS}")
else()
  message(FATAL_ERROR
    "Please set VLLM_PYTHON_EXECUTABLE to the path of the desired python version"
    " before running cmake configure.")
endif()

#
# Update cmake's `CMAKE_PREFIX_PATH` with torch location.
#
append_cmake_prefix_path("torch" "torch.utils.cmake_prefix_path")

69
70
# Ensure the 'nvcc' command is in the PATH
find_program(NVCC_EXECUTABLE nvcc)
71
if (CUDA_FOUND AND NOT NVCC_EXECUTABLE)
72
73
74
    message(FATAL_ERROR "nvcc not found")
endif()

bnellnm's avatar
bnellnm committed
75
76
77
78
79
80
81
82
#
# Import torch cmake configuration.
# Torch also imports CUDA (and partially HIP) languages with some customizations,
# so there is no need to do this explicitly with check_language/enable_language,
# etc.
#
find_package(Torch REQUIRED)

83
84
85
86
87
88
89
90
#
# Forward the non-CUDA device extensions to external CMake scripts.
#
if (NOT VLLM_TARGET_DEVICE STREQUAL "cuda" AND
    NOT VLLM_TARGET_DEVICE STREQUAL "rocm")
    if (VLLM_TARGET_DEVICE STREQUAL "cpu")
        include(${CMAKE_CURRENT_LIST_DIR}/cmake/cpu_extension.cmake)
    else()
91
        return()
92
93
94
95
    endif()
    return()
endif()

bnellnm's avatar
bnellnm committed
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
#
# Set up GPU language and check the torch version and warn if it isn't
# what is expected.
#
if (NOT HIP_FOUND AND CUDA_FOUND)
  set(VLLM_GPU_LANG "CUDA")

  if (NOT Torch_VERSION VERSION_EQUAL ${TORCH_SUPPORTED_VERSION_CUDA})
    message(WARNING "Pytorch version ${TORCH_SUPPORTED_VERSION_CUDA} "
      "expected for CUDA build, saw ${Torch_VERSION} instead.")
  endif()
elseif(HIP_FOUND)
  set(VLLM_GPU_LANG "HIP")

  # Importing torch recognizes and sets up some HIP/ROCm configuration but does
  # not let cmake recognize .hip files. In order to get cmake to understand the
  # .hip extension automatically, HIP must be enabled explicitly.
  enable_language(HIP)

115
116
117
  # ROCm 5.X and 6.X
  if (ROCM_VERSION_DEV_MAJOR GREATER_EQUAL 5 AND
      NOT Torch_VERSION VERSION_EQUAL ${TORCH_SUPPORTED_VERSION_ROCM})
118
    message(WARNING "Pytorch version >= ${TORCH_SUPPORTED_VERSION_ROCM} "
119
      "expected for ROCm build, saw ${Torch_VERSION} instead.")
bnellnm's avatar
bnellnm committed
120
121
122
123
124
  endif()
else()
  message(FATAL_ERROR "Can't find CUDA or HIP installation.")
endif()

125
126

if(VLLM_GPU_LANG STREQUAL "CUDA")
127
  #
128
  # For cuda we want to be able to control which architectures we compile for on
129
  # a per-file basis in order to cut down on compile time. So here we extract
130
  # the set of architectures we want to compile for and remove the from the
131
132
  # CMAKE_CUDA_FLAGS so that they are not applied globally.
  #
133
134
135
  clear_cuda_arches(CUDA_ARCH_FLAGS)
  extract_unique_cuda_archs_ascending(CUDA_ARCHS "${CUDA_ARCH_FLAGS}")
  message(STATUS "CUDA target architectures: ${CUDA_ARCHS}")
136
137
  # Filter the target architectures by the supported supported archs
  # since for some files we will build for all CUDA_ARCHS.
138
  cuda_archs_loose_intersection(CUDA_ARCHS
139
140
141
142
143
144
145
146
147
148
149
    "${CUDA_SUPPORTED_ARCHS}" "${CUDA_ARCHS}")
  message(STATUS "CUDA supported target architectures: ${CUDA_ARCHS}")
else()
  #
  # For other GPU targets override the GPU architectures detected by cmake/torch
  # and filter them by the supported versions for the current language.
  # The final set of arches is stored in `VLLM_GPU_ARCHES`.
  #
  override_gpu_arches(VLLM_GPU_ARCHES
    ${VLLM_GPU_LANG}
    "${${VLLM_GPU_LANG}_SUPPORTED_ARCHS}")
150
151
endif()

bnellnm's avatar
bnellnm committed
152
153
154
155
156
157
158
159
160
161
162
163
164
165
#
# Query torch for additional GPU compilation flags for the given
# `VLLM_GPU_LANG`.
# The final set of arches is stored in `VLLM_GPU_FLAGS`.
#
get_torch_gpu_compiler_flags(VLLM_GPU_FLAGS ${VLLM_GPU_LANG})

#
# Set nvcc parallelism.
#
if(NVCC_THREADS AND VLLM_GPU_LANG STREQUAL "CUDA")
  list(APPEND VLLM_GPU_FLAGS "--threads=${NVCC_THREADS}")
endif()

166
167
168

#
# Use FetchContent for C++ dependencies that are compiled as part of vLLM's build process.
169
170
171
# setup.py will override FETCHCONTENT_BASE_DIR to play nicely with sccache.
# Each dependency that produces build artifacts should override its BINARY_DIR to avoid
# conflicts between build types. It should instead be set to ${CMAKE_BINARY_DIR}/<dependency>.
172
#
173
include(FetchContent)
174
file(MAKE_DIRECTORY ${FETCHCONTENT_BASE_DIR}) # Ensure the directory exists
175
message(STATUS "FetchContent base directory: ${FETCHCONTENT_BASE_DIR}")
176

177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
#
# Set rocm version dev int.
#
if(VLLM_GPU_LANG STREQUAL "HIP")
  #
  # Overriding the default -O set up by cmake, adding ggdb3 for the most verbose devug info
  #
  set(CMAKE_${VLLM_GPU_LANG}_FLAGS_DEBUG "${CMAKE_${VLLM_GPU_LANG}_FLAGS_DEBUG} -O0 -ggdb3")
  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -ggdb3")


  #
  # Certain HIP functions are marked as [[nodiscard]], yet vllm ignores the result which generates
  # a lot of warnings that always mask real issues. Suppressing until this is properly addressed.
  #
  set(CMAKE_${VLLM_GPU_LANG}_FLAGS "${CMAKE_${VLLM_GPU_LANG}_FLAGS} -Wno-unused-result")
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-result")
endif()

bnellnm's avatar
bnellnm committed
196
#
197
# Define other extension targets
bnellnm's avatar
bnellnm committed
198
199
#

200
201
202
203
204
205
206
207
208
209
210
211
212
213
#
# cumem_allocator extension
#

set(VLLM_CUMEM_EXT_SRC
  "csrc/cumem_allocator.cpp")

set_gencode_flags_for_srcs(
  SRCS "${VLLM_CUMEM_EXT_SRC}"
  CUDA_ARCHS "${CUDA_ARCHS}")

if(VLLM_GPU_LANG STREQUAL "CUDA")
  message(STATUS "Enabling cumem allocator extension.")
  # link against cuda driver library
214
  list(APPEND CUMEM_LIBS CUDA::cuda_driver)
215
216
217
218
219
220
221
222
223
224
  define_gpu_extension_target(
    cumem_allocator
    DESTINATION vllm
    LANGUAGE CXX
    SOURCES ${VLLM_CUMEM_EXT_SRC}
    LIBRARIES ${CUMEM_LIBS}
    USE_SABI 3.8
    WITH_SOABI)
endif()

bnellnm's avatar
bnellnm committed
225
226
227
228
229
230
#
# _C extension
#

set(VLLM_EXT_SRC
  "csrc/cache_kernels.cu"
231
232
  "csrc/attention/paged_attention_v1.cu"
  "csrc/attention/paged_attention_v2.cu"
bnellnm's avatar
bnellnm committed
233
234
235
  "csrc/pos_encoding_kernels.cu"
  "csrc/activation_kernels.cu"
  "csrc/layernorm_kernels.cu"
236
  "csrc/layernorm_quant_kernels.cu"
bnellnm's avatar
bnellnm committed
237
  "csrc/quantization/gptq/q_gemm.cu"
238
  "csrc/quantization/compressed_tensors/int8_quant_kernels.cu"
239
  "csrc/quantization/fp8/common.cu"
240
  "csrc/quantization/fused_kernels/fused_layernorm_dynamic_per_token_quant.cu"
241
  "csrc/quantization/gguf/gguf_kernel.cu"
bnellnm's avatar
bnellnm committed
242
  "csrc/cuda_utils_kernels.cu"
243
  "csrc/prepare_inputs/advance_step.cu"
244
  "csrc/torch_bindings.cpp")
bnellnm's avatar
bnellnm committed
245
246

if(VLLM_GPU_LANG STREQUAL "CUDA")
247
  SET(CUTLASS_ENABLE_HEADERS_ONLY ON CACHE BOOL "Enable only the header library")
248
249

  # Set CUTLASS_REVISION manually -- its revision detection doesn't work in this case.
250
  # Please keep this in sync with FetchContent_Declare line below.
251
  set(CUTLASS_REVISION "v3.8.0" CACHE STRING "CUTLASS revision to use")
252

253
254
255
256
257
258
259
260
261
262
263
264
265
  # Use the specified CUTLASS source directory for compilation if VLLM_CUTLASS_SRC_DIR is provided
  if (DEFINED ENV{VLLM_CUTLASS_SRC_DIR})
    set(VLLM_CUTLASS_SRC_DIR $ENV{VLLM_CUTLASS_SRC_DIR})
  endif()

  if(VLLM_CUTLASS_SRC_DIR)
    if(NOT IS_ABSOLUTE VLLM_CUTLASS_SRC_DIR)
      get_filename_component(VLLM_CUTLASS_SRC_DIR "${VLLM_CUTLASS_SRC_DIR}" ABSOLUTE)
    endif()
    message(STATUS "The VLLM_CUTLASS_SRC_DIR is set, using ${VLLM_CUTLASS_SRC_DIR} for compilation")
    FetchContent_Declare(cutlass SOURCE_DIR ${VLLM_CUTLASS_SRC_DIR})
  else()
    FetchContent_Declare(
266
        cutlass
267
        GIT_REPOSITORY https://github.com/nvidia/cutlass.git
268
        # Please keep this in sync with CUTLASS_REVISION line above.
269
        GIT_TAG v3.8.0
270
        GIT_PROGRESS TRUE
271
272
273
274

        # Speed up CUTLASS download by retrieving only the specified GIT_TAG instead of the history.
        # Important: If GIT_SHALLOW is enabled then GIT_TAG works only with branch names and tags.
        # So if the GIT_TAG above is updated to a commit hash, GIT_SHALLOW must be set to FALSE
275
        GIT_SHALLOW TRUE
276
277
    )
  endif()
278
279
  FetchContent_MakeAvailable(cutlass)

bnellnm's avatar
bnellnm committed
280
  list(APPEND VLLM_EXT_SRC
281
282
    "csrc/mamba/mamba_ssm/selective_scan_fwd.cu"
    "csrc/mamba/causal_conv1d/causal_conv1d.cu"
James Fleming's avatar
James Fleming committed
283
    "csrc/quantization/aqlm/gemm_kernels.cu"
bnellnm's avatar
bnellnm committed
284
    "csrc/quantization/awq/gemm_kernels.cu"
285
    "csrc/custom_all_reduce.cu"
286
    "csrc/permute_cols.cu"
287
    "csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu"
288
    "csrc/quantization/fp4/nvfp4_quant_entry.cu"
289
    "csrc/quantization/fp4/nvfp4_scaled_mm_entry.cu"
290
291
    "csrc/sparse/cutlass/sparse_scaled_mm_entry.cu"
    "csrc/cutlass_extensions/common.cpp")
292
293
294
295
296
297
298
299

  set_gencode_flags_for_srcs(
    SRCS "${VLLM_EXT_SRC}"
    CUDA_ARCHS "${CUDA_ARCHS}")

  # Only build Marlin kernels if we are building for at least some compatible archs.
  # Keep building Marlin for 9.0 as there are some group sizes and shapes that
  # are not supported by Machete yet.
300
  cuda_archs_loose_intersection(MARLIN_ARCHS "8.0;8.6;8.7;8.9;9.0" "${CUDA_ARCHS}")
301
  if (MARLIN_ARCHS)
302
    set(MARLIN_SRCS
303
304
305
306
307
308
309
310
311
312
313
314
315
316
       "csrc/quantization/fp8/fp8_marlin.cu"
       "csrc/quantization/marlin/dense/marlin_cuda_kernel.cu"
       "csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu"
       "csrc/quantization/marlin/qqq/marlin_qqq_gemm_kernel.cu"
       "csrc/quantization/gptq_marlin/gptq_marlin.cu"
       "csrc/quantization/gptq_marlin/gptq_marlin_repack.cu"
       "csrc/quantization/gptq_marlin/awq_marlin_repack.cu")
    set_gencode_flags_for_srcs(
      SRCS "${MARLIN_SRCS}"
      CUDA_ARCHS "${MARLIN_ARCHS}")
    list(APPEND VLLM_EXT_SRC "${MARLIN_SRCS}")
    message(STATUS "Building Marlin kernels for archs: ${MARLIN_ARCHS}")
  else()
    message(STATUS "Not building Marlin kernels as no compatible archs found"
317
                   " in CUDA target architectures")
318
319
  endif()

320
  # The cutlass_scaled_mm kernels for Hopper (c3x, i.e. CUTLASS 3.x) require
321
322
  # CUDA 12.0 or later (and only work on Hopper, 9.0a for now).
  cuda_archs_loose_intersection(SCALED_MM_3X_ARCHS "9.0a" "${CUDA_ARCHS}")
323
  if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND SCALED_MM_3X_ARCHS)
324
    set(SRCS
325
326
327
328
329
       "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu"
       "csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_fp8.cu"
       "csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_int8.cu"
       "csrc/quantization/cutlass_w8a8/c3x/scaled_mm_azp_sm90_int8.cu"
       "csrc/quantization/cutlass_w8a8/c3x/scaled_mm_blockwise_sm90_fp8.cu")
330
331
332
333
334
335
336
337
    set_gencode_flags_for_srcs(
      SRCS "${SRCS}"
      CUDA_ARCHS "${SCALED_MM_3X_ARCHS}")
    list(APPEND VLLM_EXT_SRC "${SRCS}")
    list(APPEND VLLM_GPU_FLAGS "-DENABLE_SCALED_MM_C3X=1")
    message(STATUS "Building scaled_mm_c3x for archs: ${SCALED_MM_3X_ARCHS}")
  else()
    if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND SCALED_MM_3X_ARCHS)
338
      message(STATUS "Not building scaled_mm_c3x as CUDA Compiler version is "
339
                     "not >= 12.0, we recommend upgrading to CUDA 12.0 or "
340
                     "later if you intend on running FP8 quantized models on "
341
342
                     "Hopper.")
    else()
343
      message(STATUS "Not building scaled_mm_c3x as no compatible archs found "
344
345
                     "in CUDA target architectures")
    endif()
346

347
    # clear SCALED_MM_3X_ARCHS so the scaled_mm_c2x kernels know we didn't
348
349
    # build any 3x kernels
    set(SCALED_MM_3X_ARCHS)
350
  endif()
351
352

  #
353
354
  # For the cutlass_scaled_mm kernels we want to build the c2x (CUTLASS 2.x)
  # kernels for the remaining archs that are not already built for 3x.
355
  cuda_archs_loose_intersection(SCALED_MM_2X_ARCHS
356
    "7.5;8.0;8.6;8.7;8.9;9.0" "${CUDA_ARCHS}")
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
  # subtract out the archs that are already built for 3x
  list(REMOVE_ITEM SCALED_MM_2X_ARCHS ${SCALED_MM_3X_ARCHS})
  if (SCALED_MM_2X_ARCHS)
    set(SRCS "csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu")
    set_gencode_flags_for_srcs(
      SRCS "${SRCS}"
      CUDA_ARCHS "${SCALED_MM_2X_ARCHS}")
    list(APPEND VLLM_EXT_SRC "${SRCS}")
    list(APPEND VLLM_GPU_FLAGS "-DENABLE_SCALED_MM_C2X=1")
    message(STATUS "Building scaled_mm_c2x for archs: ${SCALED_MM_2X_ARCHS}")
  else()
    if (SCALED_MM_3X_ARCHS)
      message(STATUS "Not building scaled_mm_c2x as all archs are already built"
                     " for and covered by scaled_mm_c3x")
    else()
      message(STATUS "Not building scaled_mm_c2x as no compatible archs found "
                    "in CUDA target architectures")
374
375
376
377
378
379
380
    endif()
  endif()

  #
  # 2:4 Sparse Kernels

  # The 2:4 sparse kernels cutlass_scaled_sparse_mm and cutlass_compressor
381
  # require CUDA 12.2 or later (and only work on Hopper, 9.0a for now).
382
  if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.2 AND SCALED_MM_3X_ARCHS)
383
    set(SRCS "csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu")
384
385
386
387
388
389
390
391
392
393
394
395
396
397
    set_gencode_flags_for_srcs(
      SRCS "${SRCS}"
      CUDA_ARCHS "${SCALED_MM_3X_ARCHS}")
    list(APPEND VLLM_EXT_SRC "${SRCS}")
    list(APPEND VLLM_GPU_FLAGS "-DENABLE_SPARSE_SCALED_MM_C3X=1")
    message(STATUS "Building sparse_scaled_mm_c3x for archs: ${SCALED_MM_3X_ARCHS}")
  else()
    if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.2 AND SCALED_MM_3X_ARCHS)
      message(STATUS "Not building sparse_scaled_mm_c3x kernels as CUDA Compiler version is "
                     "not >= 12.2, we recommend upgrading to CUDA 12.2 or later "
                     "if you intend on running FP8 sparse quantized models on Hopper.")
    else()
      message(STATUS "Not building sparse_scaled_mm_c3x as no compatible archs found "
                     "in CUDA target architectures")
398
    endif()
399
  endif()
400

401
402
403
  # FP4 Archs and flags
  cuda_archs_loose_intersection(FP4_ARCHS "10.0a" "${CUDA_ARCHS}")
  if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.8 AND FP4_ARCHS)
404
    set(SRCS
405
      "csrc/quantization/fp4/nvfp4_quant_kernels.cu"
406
      "csrc/quantization/fp4/nvfp4_scaled_mm_kernels.cu"
407
408
409
410
411
412
413
414
415
416
417
418
    )
    set_gencode_flags_for_srcs(
      SRCS "${SRCS}"
      CUDA_ARCHS "${FP4_ARCHS}")
    list(APPEND VLLM_EXT_SRC "${SRCS}")
    list(APPEND VLLM_GPU_FLAGS "-DENABLE_NVFP4=1")
    message(STATUS "Building NVFP4 for archs: ${FP4_ARCHS}")
  else()
    message(STATUS "Not building NVFP4 as no compatible archs were found.")
    # clear FP4_ARCHS
    set(FP4_ARCHS)
  endif()
419

420
  #
421
  # Machete kernels
422

423
  # The machete kernels only work on hopper and require CUDA 12.0 or later.
424
425
426
  # Only build Machete kernels if we are building for something compatible with sm90a
  cuda_archs_loose_intersection(MACHETE_ARCHS "9.0a" "${CUDA_ARCHS}")
  if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND MACHETE_ARCHS)
427
    #
428
    # For the Machete kernels we automatically generate sources for various
429
430
    # preselected input type pairs and schedules.
    # Generate sources:
431
    set(MACHETE_GEN_SCRIPT
432
433
434
435
436
437
438
439
440
      ${CMAKE_CURRENT_SOURCE_DIR}/csrc/quantization/machete/generate.py)
    file(MD5 ${MACHETE_GEN_SCRIPT} MACHETE_GEN_SCRIPT_HASH)

    message(STATUS "Machete generation script hash: ${MACHETE_GEN_SCRIPT_HASH}")
    message(STATUS "Last run machete generate script hash: $CACHE{MACHETE_GEN_SCRIPT_HASH}")

    if (NOT DEFINED CACHE{MACHETE_GEN_SCRIPT_HASH}
        OR NOT $CACHE{MACHETE_GEN_SCRIPT_HASH} STREQUAL ${MACHETE_GEN_SCRIPT_HASH})
      execute_process(
441
442
        COMMAND ${CMAKE_COMMAND} -E env
        PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/csrc/cutlass_extensions/:${CUTLASS_DIR}/python/:${VLLM_PYTHON_PATH}:$PYTHONPATH
443
444
445
446
447
448
449
450
451
          ${Python_EXECUTABLE} ${MACHETE_GEN_SCRIPT}
        RESULT_VARIABLE machete_generation_result
        OUTPUT_VARIABLE machete_generation_output
        OUTPUT_FILE ${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log
        ERROR_FILE ${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log
      )

      if (NOT machete_generation_result EQUAL 0)
        message(FATAL_ERROR "Machete generation failed."
452
                            " Result: \"${machete_generation_result}\""
453
454
455
                            "\nCheck the log for details: "
                            "${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log")
      else()
456
        set(MACHETE_GEN_SCRIPT_HASH ${MACHETE_GEN_SCRIPT_HASH}
457
458
459
            CACHE STRING "Last run machete generate script hash" FORCE)
        message(STATUS "Machete generation completed successfully.")
      endif()
460
    else()
461
      message(STATUS "Machete generation script has not changed, skipping generation.")
462
    endif()
463

464
465
466
    # Add machete generated sources
    file(GLOB MACHETE_GEN_SOURCES "csrc/quantization/machete/generated/*.cu")
    list(APPEND VLLM_EXT_SRC ${MACHETE_GEN_SOURCES})
467

468
469
470
471
472
473
474
475
476
477
    # forward compatible
    set_gencode_flags_for_srcs(
      SRCS "${MACHETE_GEN_SOURCES}"
      CUDA_ARCHS "${MACHETE_ARCHS}")

    list(APPEND VLLM_EXT_SRC
      csrc/quantization/machete/machete_pytorch.cu)

    message(STATUS "Building Machete kernels for archs: ${MACHETE_ARCHS}")
  else()
478
    if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0
479
480
481
482
483
484
485
486
487
        AND MACHETE_ARCHS)
      message(STATUS "Not building Machete kernels as CUDA Compiler version is "
                     "not >= 12.0, we recommend upgrading to CUDA 12.0 or "
                     "later if you intend on running w4a16 quantized models on "
                     "Hopper.")
    else()
      message(STATUS "Not building Machete kernels as no compatible archs "
                     "found in CUDA target architectures")
    endif()
488
  endif()
489
# if CUDA endif
bnellnm's avatar
bnellnm committed
490
491
endif()

492
message(STATUS "Enabling C extension.")
bnellnm's avatar
bnellnm committed
493
494
495
496
497
498
499
define_gpu_extension_target(
  _C
  DESTINATION vllm
  LANGUAGE ${VLLM_GPU_LANG}
  SOURCES ${VLLM_EXT_SRC}
  COMPILE_FLAGS ${VLLM_GPU_FLAGS}
  ARCHITECTURES ${VLLM_GPU_ARCHES}
500
  INCLUDE_DIRECTORIES ${CUTLASS_INCLUDE_DIR}
501
  USE_SABI 3
bnellnm's avatar
bnellnm committed
502
503
  WITH_SOABI)

504
505
# If CUTLASS is compiled on NVCC >= 12.5, it by default uses
# cudaGetDriverEntryPointByVersion as a wrapper to avoid directly calling the
506
507
508
509
# driver API. This causes problems when linking with earlier versions of CUDA.
# Setting this variable sidesteps the issue by calling the driver directly.
target_compile_definitions(_C PRIVATE CUTLASS_ENABLE_DIRECT_CUDA_DRIVER_CALL=1)

bnellnm's avatar
bnellnm committed
510
511
512
513
514
#
# _moe_C extension
#

set(VLLM_MOE_EXT_SRC
515
  "csrc/moe/torch_bindings.cpp"
516
  "csrc/moe/moe_align_sum_kernels.cu"
517
  "csrc/moe/topk_softmax_kernels.cu")
bnellnm's avatar
bnellnm committed
518

519
520
521
522
set_gencode_flags_for_srcs(
  SRCS "${VLLM_MOE_EXT_SRC}"
  CUDA_ARCHS "${CUDA_ARCHS}")

523
if(VLLM_GPU_LANG STREQUAL "CUDA")
524
  cuda_archs_loose_intersection(MARLIN_MOE_ARCHS "8.0;8.6;8.7;8.9;9.0" "${CUDA_ARCHS}")
525
526
527
528
529
530
531
  if (MARLIN_MOE_ARCHS)
    set(MARLIN_MOE_SRC
        "csrc/moe/marlin_kernels/marlin_moe_kernel.h"
        "csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.h"
        "csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.cu"
        "csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.h"
        "csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.cu"
532
533
        "csrc/moe/marlin_kernels/marlin_moe_kernel_ku4.h"
        "csrc/moe/marlin_kernels/marlin_moe_kernel_ku4.cu"
534
535
536
537
538
539
540
541
542
543
        "csrc/moe/marlin_moe_ops.cu")

    set_gencode_flags_for_srcs(
      SRCS "${MARLIN_MOE_SRC}"
      CUDA_ARCHS "${MARLIN_MOE_ARCHS}")

    list(APPEND VLLM_MOE_EXT_SRC "${MARLIN_MOE_SRC}")
    message(STATUS "Building Marlin MOE kernels for archs: ${MARLIN_MOE_ARCHS}")
  else()
    message(STATUS "Not building Marlin MOE kernels as no compatible archs found"
544
                   " in CUDA target architectures")
545
  endif()
546
547
endif()

548
message(STATUS "Enabling moe extension.")
bnellnm's avatar
bnellnm committed
549
550
551
552
553
554
555
define_gpu_extension_target(
  _moe_C
  DESTINATION vllm
  LANGUAGE ${VLLM_GPU_LANG}
  SOURCES ${VLLM_MOE_EXT_SRC}
  COMPILE_FLAGS ${VLLM_GPU_FLAGS}
  ARCHITECTURES ${VLLM_GPU_ARCHES}
556
  USE_SABI 3
bnellnm's avatar
bnellnm committed
557
558
  WITH_SOABI)

559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
if(VLLM_GPU_LANG STREQUAL "HIP")
  #
  # _rocm_C extension
  #
  set(VLLM_ROCM_EXT_SRC
    "csrc/rocm/torch_bindings.cpp"
    "csrc/rocm/attention.cu")

  define_gpu_extension_target(
    _rocm_C
    DESTINATION vllm
    LANGUAGE ${VLLM_GPU_LANG}
    SOURCES ${VLLM_ROCM_EXT_SRC}
    COMPILE_FLAGS ${VLLM_GPU_FLAGS}
    ARCHITECTURES ${VLLM_GPU_ARCHES}
    USE_SABI 3
    WITH_SOABI)
endif()

578
# vllm-flash-attn currently only supported on CUDA
579
if (NOT VLLM_GPU_LANG STREQUAL "CUDA")
580
581
  return()
endif ()
bnellnm's avatar
bnellnm committed
582

583
584
585
# vLLM flash attention requires VLLM_GPU_ARCHES to contain the set of target
# arches in the CMake syntax (75-real, 89-virtual, etc), since we clear the
# arches in the CUDA case (and instead set the gencodes on a per file basis)
586
587
588
589
590
591
592
593
# we need to manually set VLLM_GPU_ARCHES here.
if(VLLM_GPU_LANG STREQUAL "CUDA")
  foreach(_ARCH ${CUDA_ARCHS})
    string(REPLACE "." "" _ARCH "${_ARCH}")
    list(APPEND VLLM_GPU_ARCHES "${_ARCH}-real")
  endforeach()
endif()

594
595
596
597
598
599
600
601
#
# Build vLLM flash attention from source
#
# IMPORTANT: This has to be the last thing we do, because vllm-flash-attn uses the same macros/functions as vLLM.
# Because functions all belong to the global scope, vllm-flash-attn's functions overwrite vLLMs.
# They should be identical but if they aren't, this is a massive footgun.
#
# The vllm-flash-attn install rules are nested under vllm to make sure the library gets installed in the correct place.
602
# To only install vllm-flash-attn, use --component _vllm_fa2_C (for FA2) or --component _vllm_fa3_C (for FA3).
603
# If no component is specified, vllm-flash-attn is still installed.
bnellnm's avatar
bnellnm committed
604

605
606
607
608
609
610
# If VLLM_FLASH_ATTN_SRC_DIR is set, vllm-flash-attn is installed from that directory instead of downloading.
# This is to enable local development of vllm-flash-attn within vLLM.
# It can be set as an environment variable or passed as a cmake argument.
# The environment variable takes precedence.
if (DEFINED ENV{VLLM_FLASH_ATTN_SRC_DIR})
  set(VLLM_FLASH_ATTN_SRC_DIR $ENV{VLLM_FLASH_ATTN_SRC_DIR})
611
endif()
612

613
if(VLLM_FLASH_ATTN_SRC_DIR)
614
  FetchContent_Declare(
615
          vllm-flash-attn SOURCE_DIR
616
617
618
          ${VLLM_FLASH_ATTN_SRC_DIR}
          BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn
  )
619
620
621
622
else()
  FetchContent_Declare(
          vllm-flash-attn
          GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git
623
          GIT_TAG 720c94869cf2e0ff5a706e9c7f1dce0939686ade
624
          GIT_PROGRESS TRUE
625
626
          # Don't share the vllm-flash-attn build between build types
          BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn
627
  )
bnellnm's avatar
bnellnm committed
628
endif()
629
630
631
632
633
634


# Fetch the vllm-flash-attn library
FetchContent_MakeAvailable(vllm-flash-attn)
message(STATUS "vllm-flash-attn is available at ${vllm-flash-attn_SOURCE_DIR}")

635
636
637
638
639
640
641
642
# Copy over the vllm-flash-attn python files (duplicated for fa2 and fa3, in
# case only one is built, in the case both are built redundant work is done)
install(
  DIRECTORY ${vllm-flash-attn_SOURCE_DIR}/vllm_flash_attn/
  DESTINATION vllm_flash_attn
  COMPONENT _vllm_fa2_C
  FILES_MATCHING PATTERN "*.py"
)
643
644

install(
645
646
647
648
  DIRECTORY ${vllm-flash-attn_SOURCE_DIR}/vllm_flash_attn/
  DESTINATION vllm_flash_attn
  COMPONENT _vllm_fa3_C
  FILES_MATCHING PATTERN "*.py"
649
650
651
)

# Nothing after vllm-flash-attn, see comment about macros above