setup.py 24.8 KB
Newer Older
1
2
# SPDX-License-Identifier: Apache-2.0

3
import ctypes
4
import importlib.util
5
import json
6
import logging
7
8
import os
import re
9
import subprocess
bnellnm's avatar
bnellnm committed
10
import sys
11
from pathlib import Path
12
from shutil import which
13

Woosuk Kwon's avatar
Woosuk Kwon committed
14
import torch
15
from packaging.version import Version, parse
16
from setuptools import Extension, setup
17
from setuptools.command.build_ext import build_ext
18
from setuptools_scm import get_version
19
from torch.utils.cpp_extension import CUDA_HOME, ROCM_HOME
20

21
22
23
24
25
26
27
28
29

def load_module_from_path(module_name, path):
    spec = importlib.util.spec_from_file_location(module_name, path)
    module = importlib.util.module_from_spec(spec)
    sys.modules[module_name] = module
    spec.loader.exec_module(module)
    return module


30
ROOT_DIR = Path(__file__).parent
31
logger = logging.getLogger(__name__)
32
33
34
35
36
37

# cannot import envs directly because it depends on vllm,
#  which is not installed yet
envs = load_module_from_path('envs', os.path.join(ROOT_DIR, 'vllm', 'envs.py'))

VLLM_TARGET_DEVICE = envs.VLLM_TARGET_DEVICE
38

39
if sys.platform.startswith("darwin") and VLLM_TARGET_DEVICE != "cpu":
40
    logger.warning(
41
42
43
44
45
46
        "VLLM_TARGET_DEVICE automatically set to `cpu` due to macOS")
    VLLM_TARGET_DEVICE = "cpu"
elif not (sys.platform.startswith("linux")
          or sys.platform.startswith("darwin")):
    logger.warning(
        "vLLM only supports Linux platform (including WSL) and MacOS."
47
48
49
        "Building on %s, "
        "so vLLM may not be able to run correctly", sys.platform)
    VLLM_TARGET_DEVICE = "empty"
50
elif (sys.platform.startswith("linux") and torch.version.cuda is None
51
52
53
      and os.getenv("VLLM_TARGET_DEVICE") is None
      and torch.version.hip is None):
    # if cuda or hip is not available and VLLM_TARGET_DEVICE is not set,
54
55
    # fallback to cpu
    VLLM_TARGET_DEVICE = "cpu"
56

57
MAIN_CUDA_VERSION = "12.4"
58

bnellnm's avatar
bnellnm committed
59
60
61
62
63
64
65
66
67
68
69
70
71

def is_sccache_available() -> bool:
    return which("sccache") is not None


def is_ccache_available() -> bool:
    return which("ccache") is not None


def is_ninja_available() -> bool:
    return which("ninja") is not None


72
73
74
75
76
77
78
79
80
81
82
83
def is_url_available(url: str) -> bool:
    from urllib.request import urlopen

    status = None
    try:
        with urlopen(url) as f:
            status = f.status
    except Exception:
        return False
    return status == 200


bnellnm's avatar
bnellnm committed
84
85
86
class CMakeExtension(Extension):

    def __init__(self, name: str, cmake_lists_dir: str = '.', **kwa) -> None:
87
        super().__init__(name, sources=[], py_limited_api=True, **kwa)
bnellnm's avatar
bnellnm committed
88
89
90
91
92
        self.cmake_lists_dir = os.path.abspath(cmake_lists_dir)


class cmake_build_ext(build_ext):
    # A dict of extension directories that have been configured.
93
    did_config: dict[str, bool] = {}
bnellnm's avatar
bnellnm committed
94
95
96
97
98

    #
    # Determine number of compilation jobs and optionally nvcc compile threads.
    #
    def compute_num_jobs(self):
99
100
        # `num_jobs` is either the value of the MAX_JOBS environment variable
        # (if defined) or the number of CPUs available.
101
        num_jobs = envs.MAX_JOBS
102
103
        if num_jobs is not None:
            num_jobs = int(num_jobs)
104
            logger.info("Using MAX_JOBS=%d as the number of jobs.", num_jobs)
105
106
107
108
109
110
111
        else:
            try:
                # os.sched_getaffinity() isn't universally available, so fall
                #  back to os.cpu_count() if we get an error here.
                num_jobs = len(os.sched_getaffinity(0))
            except AttributeError:
                num_jobs = os.cpu_count()
bnellnm's avatar
bnellnm committed
112

113
        nvcc_threads = None
114
115
116
117
118
        if _is_cuda() and get_nvcc_cuda_version() >= Version("11.2"):
            # `nvcc_threads` is either the value of the NVCC_THREADS
            # environment variable (if defined) or 1.
            # when it is set, we reduce `num_jobs` to avoid
            # overloading the system.
119
            nvcc_threads = envs.NVCC_THREADS
120
121
            if nvcc_threads is not None:
                nvcc_threads = int(nvcc_threads)
122
123
124
                logger.info(
                    "Using NVCC_THREADS=%d as the number of nvcc threads.",
                    nvcc_threads)
125
126
127
            else:
                nvcc_threads = 1
            num_jobs = max(1, num_jobs // nvcc_threads)
bnellnm's avatar
bnellnm committed
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144

        return num_jobs, nvcc_threads

    #
    # Perform cmake configuration for a single extension.
    #
    def configure(self, ext: CMakeExtension) -> None:
        # If we've already configured using the CMakeLists.txt for
        # this extension, exit early.
        if ext.cmake_lists_dir in cmake_build_ext.did_config:
            return

        cmake_build_ext.did_config[ext.cmake_lists_dir] = True

        # Select the build type.
        # Note: optimization level + debug info are set by the build type
        default_cfg = "Debug" if self.debug else "RelWithDebInfo"
145
        cfg = envs.CMAKE_BUILD_TYPE or default_cfg
bnellnm's avatar
bnellnm committed
146
147
148

        cmake_args = [
            '-DCMAKE_BUILD_TYPE={}'.format(cfg),
149
            '-DVLLM_TARGET_DEVICE={}'.format(VLLM_TARGET_DEVICE),
bnellnm's avatar
bnellnm committed
150
151
        ]

152
        verbose = envs.VERBOSE
bnellnm's avatar
bnellnm committed
153
154
155
156
157
        if verbose:
            cmake_args += ['-DCMAKE_VERBOSE_MAKEFILE=ON']

        if is_sccache_available():
            cmake_args += [
158
                '-DCMAKE_C_COMPILER_LAUNCHER=sccache',
bnellnm's avatar
bnellnm committed
159
160
                '-DCMAKE_CXX_COMPILER_LAUNCHER=sccache',
                '-DCMAKE_CUDA_COMPILER_LAUNCHER=sccache',
161
                '-DCMAKE_HIP_COMPILER_LAUNCHER=sccache',
bnellnm's avatar
bnellnm committed
162
163
164
            ]
        elif is_ccache_available():
            cmake_args += [
165
                '-DCMAKE_C_COMPILER_LAUNCHER=ccache',
bnellnm's avatar
bnellnm committed
166
167
                '-DCMAKE_CXX_COMPILER_LAUNCHER=ccache',
                '-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache',
168
                '-DCMAKE_HIP_COMPILER_LAUNCHER=ccache',
bnellnm's avatar
bnellnm committed
169
170
171
172
173
174
            ]

        # Pass the python executable to cmake so it can find an exact
        # match.
        cmake_args += ['-DVLLM_PYTHON_EXECUTABLE={}'.format(sys.executable)]

175
176
177
178
        # Pass the python path to cmake so it can reuse the build dependencies
        # on subsequent calls to python.
        cmake_args += ['-DVLLM_PYTHON_PATH={}'.format(":".join(sys.path))]

179
180
181
182
183
184
185
186
        # Override the base directory for FetchContent downloads to $ROOT/.deps
        # This allows sharing dependencies between profiles,
        # and plays more nicely with sccache.
        # To override this, set the FETCHCONTENT_BASE_DIR environment variable.
        fc_base_dir = os.path.join(ROOT_DIR, ".deps")
        fc_base_dir = os.environ.get("FETCHCONTENT_BASE_DIR", fc_base_dir)
        cmake_args += ['-DFETCHCONTENT_BASE_DIR={}'.format(fc_base_dir)]

bnellnm's avatar
bnellnm committed
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
        #
        # Setup parallelism and build tool
        #
        num_jobs, nvcc_threads = self.compute_num_jobs()

        if nvcc_threads:
            cmake_args += ['-DNVCC_THREADS={}'.format(nvcc_threads)]

        if is_ninja_available():
            build_tool = ['-G', 'Ninja']
            cmake_args += [
                '-DCMAKE_JOB_POOL_COMPILE:STRING=compile',
                '-DCMAKE_JOB_POOLS:STRING=compile={}'.format(num_jobs),
            ]
        else:
            # Default build tool to whatever cmake picks.
            build_tool = []
        subprocess.check_call(
            ['cmake', ext.cmake_lists_dir, *build_tool, *cmake_args],
            cwd=self.build_temp)

    def build_extensions(self) -> None:
        # Ensure that CMake is present and working
        try:
            subprocess.check_output(['cmake', '--version'])
        except OSError as e:
            raise RuntimeError('Cannot find CMake executable') from e

        # Create build directory if it does not exist.
        if not os.path.exists(self.build_temp):
            os.makedirs(self.build_temp)

219
        targets = []
220
221
222
223

        def target_name(s: str) -> str:
            return s.removeprefix("vllm.").removeprefix("vllm_flash_attn.")

bnellnm's avatar
bnellnm committed
224
225
226
        # Build all the extensions
        for ext in self.extensions:
            self.configure(ext)
227
            targets.append(target_name(ext.name))
bnellnm's avatar
bnellnm committed
228

229
        num_jobs, _ = self.compute_num_jobs()
bnellnm's avatar
bnellnm committed
230

231
232
233
234
235
236
        build_args = [
            "--build",
            ".",
            f"-j={num_jobs}",
            *[f"--target={name}" for name in targets],
        ]
bnellnm's avatar
bnellnm committed
237

238
        subprocess.check_call(["cmake", *build_args], cwd=self.build_temp)
239

240
241
242
243
244
245
246
247
248
249
250
        # Install the libraries
        for ext in self.extensions:
            # Install the extension into the proper location
            outdir = Path(self.get_ext_fullpath(ext.name)).parent.absolute()

            # Skip if the install directory is the same as the build directory
            if outdir == self.build_temp:
                continue

            # CMake appends the extension prefix to the install path,
            # and outdir already contains that prefix, so we need to remove it.
251
252
253
            # We assume only the final component of extension prefix is added by
            # CMake, this is currently true for current extensions but may not
            # always be the case.
254
            prefix = outdir
255
            if '.' in ext.name:
256
257
258
259
260
261
262
263
264
                prefix = prefix.parent

            # prefix here should actually be the same for all components
            install_args = [
                "cmake", "--install", ".", "--prefix", prefix, "--component",
                target_name(ext.name)
            ]
            subprocess.check_call(install_args, cwd=self.build_temp)

265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
    def run(self):
        # First, run the standard build_ext command to compile the extensions
        super().run()

        # copy vllm/vllm_flash_attn/*.py from self.build_lib to current
        # directory so that they can be included in the editable build
        import glob
        files = glob.glob(
            os.path.join(self.build_lib, "vllm", "vllm_flash_attn", "*.py"))
        for file in files:
            dst_file = os.path.join("vllm/vllm_flash_attn",
                                    os.path.basename(file))
            print(f"Copying {file} to {dst_file}")
            self.copy_file(file, dst_file)

280

281
282
283
class repackage_wheel(build_ext):
    """Extracts libraries and other files from an existing wheel."""

284
    def get_base_commit_in_main_branch(self) -> str:
285
286
287
        # Force to use the nightly wheel. This is mainly used for CI testing.
        if envs.VLLM_TEST_USE_PRECOMPILED_NIGHTLY_WHEEL:
            return "nightly"
288
289

        try:
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
            # Get the latest commit hash of the upstream main branch.
            resp_json = subprocess.check_output([
                "curl", "-s",
                "https://api.github.com/repos/vllm-project/vllm/commits/main"
            ]).decode("utf-8")
            upstream_main_commit = json.loads(resp_json)["sha"]

            # Check if the local main branch is up-to-date. This is to ensure
            # the base commit we found is the most recent commit on the main
            # branch.
            local_main_commit = subprocess.check_output(
                ["git", "rev-parse", "main"]).decode("utf-8").strip()
            if local_main_commit != upstream_main_commit:
                raise ValueError(
                    f"Local main branch ({local_main_commit}) is not "
                    "up-to-date with upstream main branch "
                    f"({upstream_main_commit}). Please pull the latest "
                    "changes from upstream main branch first.")

            # Then get the commit hash of the current branch that is the same as
            # the upstream main commit.
311
312
313
314
315
316
317
            current_branch = subprocess.check_output(
                ["git", "branch", "--show-current"]).decode("utf-8").strip()

            base_commit = subprocess.check_output(
                ["git", "merge-base", "main",
                 current_branch]).decode("utf-8").strip()
            return base_commit
318
319
        except ValueError as err:
            raise ValueError(err) from None
320
321
322
323
324
325
        except Exception as err:
            logger.warning(
                "Failed to get the base commit in the main branch. "
                "Using the nightly wheel. The libraries in this "
                "wheel may not be compatible with your dev branch: %s", err)
            return "nightly"
326

327
    def run(self) -> None:
328
329
330
        assert _is_cuda(
        ), "VLLM_USE_PRECOMPILED is only supported for CUDA builds"

331
332
333
334
        wheel_location = os.getenv("VLLM_PRECOMPILED_WHEEL_LOCATION", None)
        if wheel_location is None:
            base_commit = self.get_base_commit_in_main_branch()
            wheel_location = f"https://wheels.vllm.ai/{base_commit}/vllm-1.0.0.dev-cp38-abi3-manylinux1_x86_64.whl"
335
336
337
338
            # Fallback to nightly wheel if latest commit wheel is unavailable,
            # in this rare case, the nightly release CI hasn't finished on main.
            if not is_url_available(wheel_location):
                wheel_location = "https://wheels.vllm.ai/nightly/vllm-1.0.0.dev-cp38-abi3-manylinux1_x86_64.whl"
339

340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
        import zipfile

        if os.path.isfile(wheel_location):
            wheel_path = wheel_location
            print(f"Using existing wheel={wheel_path}")
        else:
            # Download the wheel from a given URL, assume
            # the filename is the last part of the URL
            wheel_filename = wheel_location.split("/")[-1]

            import tempfile

            # create a temporary directory to store the wheel
            temp_dir = tempfile.mkdtemp(prefix="vllm-wheels")
            wheel_path = os.path.join(temp_dir, wheel_filename)

            print(f"Downloading wheel from {wheel_location} to {wheel_path}")

            from urllib.request import urlretrieve

            try:
                urlretrieve(wheel_location, filename=wheel_path)
            except Exception as e:
                from setuptools.errors import SetupError

                raise SetupError(
                    f"Failed to get vLLM wheel from {wheel_location}") from e

        with zipfile.ZipFile(wheel_path) as wheel:
            files_to_copy = [
                "vllm/_C.abi3.so",
                "vllm/_moe_C.abi3.so",
372
                "vllm/_flashmla_C.abi3.so",
373
374
                "vllm/vllm_flash_attn/_vllm_fa2_C.abi3.so",
                "vllm/vllm_flash_attn/_vllm_fa3_C.abi3.so",
375
376
                "vllm/vllm_flash_attn/flash_attn_interface.py",
                "vllm/vllm_flash_attn/__init__.py",
377
                "vllm/cumem_allocator.abi3.so",
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
                # "vllm/_version.py", # not available in nightly wheels yet
            ]
            file_members = filter(lambda x: x.filename in files_to_copy,
                                  wheel.filelist)

            for file in file_members:
                print(f"Extracting and including {file.filename} "
                      "from existing wheel")
                package_name = os.path.dirname(file.filename).replace("/", ".")
                file_name = os.path.basename(file.filename)

                if package_name not in package_data:
                    package_data[package_name] = []

                wheel.extract(file)
                if file_name.endswith(".py"):
                    # python files shouldn't be added to package_data
                    continue

                package_data[package_name].append(file_name)


400
def _is_hpu() -> bool:
401
402
403
404
405
406
407
    # if VLLM_TARGET_DEVICE env var was set explicitly, skip HPU autodetection
    if os.getenv("VLLM_TARGET_DEVICE", None) == VLLM_TARGET_DEVICE:
        return VLLM_TARGET_DEVICE == "hpu"

    # if VLLM_TARGET_DEVICE was not set explicitly, check if hl-smi succeeds,
    # and if it doesn't, check if habanalabs driver is loaded
    is_hpu_available = False
408
    try:
409
410
        out = subprocess.run(["hl-smi"], capture_output=True, check=True)
        is_hpu_available = out.returncode == 0
411
    except (FileNotFoundError, PermissionError, subprocess.CalledProcessError):
412
        if sys.platform.startswith("linux"):
413
414
415
416
417
418
            try:
                output = subprocess.check_output(
                    'lsmod | grep habanalabs | wc -l', shell=True)
                is_hpu_available = int(output) > 0
            except (ValueError, FileNotFoundError, PermissionError,
                    subprocess.CalledProcessError):
419
420
                pass
    return is_hpu_available
421
422


423
424
425
426
def _no_device() -> bool:
    return VLLM_TARGET_DEVICE == "empty"


427
def _is_cuda() -> bool:
428
429
    has_cuda = torch.version.cuda is not None
    return (VLLM_TARGET_DEVICE == "cuda" and has_cuda
430
            and not (_is_neuron() or _is_tpu() or _is_hpu()))
431
432


433
def _is_hip() -> bool:
434
435
    return (VLLM_TARGET_DEVICE == "cuda"
            or VLLM_TARGET_DEVICE == "rocm") and torch.version.hip is not None
436
437


438
def _is_neuron() -> bool:
439
    return VLLM_TARGET_DEVICE == "neuron"
440
441


442
443
444
445
def _is_tpu() -> bool:
    return VLLM_TARGET_DEVICE == "tpu"


446
447
448
449
def _is_cpu() -> bool:
    return VLLM_TARGET_DEVICE == "cpu"


450
451
452
453
def _is_openvino() -> bool:
    return VLLM_TARGET_DEVICE == "openvino"


454
455
456
457
def _is_xpu() -> bool:
    return VLLM_TARGET_DEVICE == "xpu"


458
459
460
461
def _build_custom_ops() -> bool:
    return _is_cuda() or _is_hip() or _is_cpu()


462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
def get_rocm_version():
    # Get the Rocm version from the ROCM_HOME/bin/librocm-core.so
    # see https://github.com/ROCm/rocm-core/blob/d11f5c20d500f729c393680a01fa902ebf92094b/rocm_version.cpp#L21
    try:
        librocm_core_file = Path(ROCM_HOME) / "lib" / "librocm-core.so"
        if not librocm_core_file.is_file():
            return None
        librocm_core = ctypes.CDLL(librocm_core_file)
        VerErrors = ctypes.c_uint32
        get_rocm_core_version = librocm_core.getROCmVersion
        get_rocm_core_version.restype = VerErrors
        get_rocm_core_version.argtypes = [
            ctypes.POINTER(ctypes.c_uint32),
            ctypes.POINTER(ctypes.c_uint32),
            ctypes.POINTER(ctypes.c_uint32),
        ]
        major = ctypes.c_uint32()
        minor = ctypes.c_uint32()
        patch = ctypes.c_uint32()
481

482
483
        if (get_rocm_core_version(ctypes.byref(major), ctypes.byref(minor),
                                  ctypes.byref(patch)) == 0):
484
            return f"{major.value}.{minor.value}.{patch.value}"
485
        return None
486
    except Exception:
487
        return None
Woosuk Kwon's avatar
Woosuk Kwon committed
488

489

490
491
492
def get_neuronxcc_version():
    import sysconfig
    site_dir = sysconfig.get_paths()["purelib"]
493
494
    version_file = os.path.join(site_dir, "neuronxcc", "version",
                                "__init__.py")
495
496

    # Check if the command was executed successfully
497
    with open(version_file) as fp:
498
499
500
501
502
503
504
505
        content = fp.read()

    # Extract the version using a regular expression
    match = re.search(r"__version__ = '(\S+)'", content)
    if match:
        # Return the version string
        return match.group(1)
    else:
506
        raise RuntimeError("Could not find Neuron version in the output")
507
508


bnellnm's avatar
bnellnm committed
509
def get_nvcc_cuda_version() -> Version:
510
511
512
513
    """Get the CUDA version from nvcc.

    Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py
    """
514
    assert CUDA_HOME is not None, "CUDA_HOME is not set"
bnellnm's avatar
bnellnm committed
515
    nvcc_output = subprocess.check_output([CUDA_HOME + "/bin/nvcc", "-V"],
516
517
518
519
520
521
522
                                          universal_newlines=True)
    output = nvcc_output.split()
    release_idx = output.index("release") + 1
    nvcc_cuda_version = parse(output[release_idx].split(",")[0])
    return nvcc_cuda_version


523
524
525
526
527
528
529
530
def get_gaudi_sw_version():
    """
    Returns the driver version.
    """
    # Enable console printing for `hl-smi` check
    output = subprocess.run("hl-smi",
                            shell=True,
                            text=True,
531
                            capture_output=True,
532
533
534
535
536
537
538
                            env={"ENABLE_CONSOLE": "true"})
    if output.returncode == 0 and output.stdout:
        return output.stdout.split("\n")[2].replace(
            " ", "").split(":")[1][:-1].split("-")[0]
    return "0.0.0"  # when hl-smi is not available


539
def get_vllm_version() -> str:
540
    version = get_version(write_to="vllm/_version.py")
541
    sep = "+" if "+" not in version else "."  # dev versions might contain +
542

543
    if _no_device():
544
        if envs.VLLM_TARGET_DEVICE == "empty":
545
            version += f"{sep}empty"
546
    elif _is_cuda():
547
        if envs.VLLM_USE_PRECOMPILED:
548
            version += f"{sep}precompiled"
549
550
551
552
553
554
555
        else:
            cuda_version = str(get_nvcc_cuda_version())
            if cuda_version != MAIN_CUDA_VERSION:
                cuda_version_str = cuda_version.replace(".", "")[:3]
                # skip this for source tarball, required for pypi
                if "sdist" not in sys.argv:
                    version += f"{sep}cu{cuda_version_str}"
556
    elif _is_hip():
557
558
559
560
        # Get the Rocm Version
        rocm_version = get_rocm_version() or torch.version.hip
        if rocm_version and rocm_version != MAIN_CUDA_VERSION:
            version += f"{sep}rocm{rocm_version.replace('.', '')[:3]}"
561
562
    elif _is_neuron():
        # Get the Neuron version
bnellnm's avatar
bnellnm committed
563
        neuron_version = str(get_neuronxcc_version())
564
565
        if neuron_version != MAIN_CUDA_VERSION:
            neuron_version_str = neuron_version.replace(".", "")[:3]
566
            version += f"{sep}neuron{neuron_version_str}"
567
568
569
570
571
572
    elif _is_hpu():
        # Get the Intel Gaudi Software Suite version
        gaudi_sw_version = str(get_gaudi_sw_version())
        if gaudi_sw_version != MAIN_CUDA_VERSION:
            gaudi_sw_version = gaudi_sw_version.replace(".", "")[:3]
            version += f"{sep}gaudi{gaudi_sw_version}"
573
    elif _is_openvino():
574
        version += f"{sep}openvino"
575
    elif _is_tpu():
576
        version += f"{sep}tpu"
577
    elif _is_cpu():
578
579
        if envs.VLLM_TARGET_DEVICE == "cpu":
            version += f"{sep}cpu"
580
    elif _is_xpu():
581
        version += f"{sep}xpu"
582
    else:
583
        raise RuntimeError("Unknown runtime environment")
584

585
586
587
    return version


588
def get_requirements() -> list[str]:
589
    """Get Python package dependencies from requirements.txt."""
590
    requirements_dir = ROOT_DIR / "requirements"
591

592
    def _read_requirements(filename: str) -> list[str]:
593
        with open(requirements_dir / filename) as f:
594
            requirements = f.read().strip().split("\n")
595
596
597
598
        resolved_requirements = []
        for line in requirements:
            if line.startswith("-r "):
                resolved_requirements += _read_requirements(line.split()[1])
599
600
            elif line.startswith("--"):
                continue
601
602
603
604
            else:
                resolved_requirements.append(line)
        return resolved_requirements

605
    if _no_device():
606
        requirements = _read_requirements("common.txt")
607
    elif _is_cuda():
608
        requirements = _read_requirements("cuda.txt")
609
        cuda_major, cuda_minor = torch.version.cuda.split(".")
610
611
        modified_requirements = []
        for req in requirements:
612
613
            if ("vllm-flash-attn" in req and cuda_major != "12"):
                # vllm-flash-attn is built only for CUDA 12.x.
614
615
616
                # Skip for other versions.
                continue
            modified_requirements.append(req)
617
        requirements = modified_requirements
618
    elif _is_hip():
619
        requirements = _read_requirements("rocm.txt")
620
    elif _is_neuron():
621
        requirements = _read_requirements("neuron.txt")
622
    elif _is_hpu():
623
        requirements = _read_requirements("hpu.txt")
624
    elif _is_openvino():
625
        requirements = _read_requirements("openvino.txt")
626
    elif _is_tpu():
627
        requirements = _read_requirements("tpu.txt")
628
    elif _is_cpu():
629
        requirements = _read_requirements("cpu.txt")
630
    elif _is_xpu():
631
        requirements = _read_requirements("xpu.txt")
632
633
    else:
        raise ValueError(
634
            "Unsupported platform, please use CUDA, ROCm, Neuron, HPU, "
635
            "OpenVINO, or CPU.")
636
637
638
    return requirements


bnellnm's avatar
bnellnm committed
639
640
ext_modules = []

641
if _is_cuda() or _is_hip():
bnellnm's avatar
bnellnm committed
642
643
    ext_modules.append(CMakeExtension(name="vllm._moe_C"))

644
645
646
if _is_hip():
    ext_modules.append(CMakeExtension(name="vllm._rocm_C"))

647
if _is_cuda():
648
    ext_modules.append(CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa2_C"))
649
650
651
652
    if envs.VLLM_USE_PRECOMPILED or get_nvcc_cuda_version() >= Version("12.0"):
        # FA3 requires CUDA 12.0 or later
        ext_modules.append(
            CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa3_C"))
653
654
655
656
657
    if envs.VLLM_USE_PRECOMPILED or get_nvcc_cuda_version() >= Version("12.3"):
        # Optional since this doesn't get built (produce an .so file) when
        # not targeting a hopper system
        ext_modules.append(
            CMakeExtension(name="vllm._flashmla_C", optional=True))
658
    ext_modules.append(CMakeExtension(name="vllm.cumem_allocator"))
659

660
if _build_custom_ops():
bnellnm's avatar
bnellnm committed
661
662
    ext_modules.append(CMakeExtension(name="vllm._C"))

663
package_data = {
664
665
666
667
668
    "vllm": [
        "py.typed",
        "model_executor/layers/fused_moe/configs/*.json",
        "model_executor/layers/quantization/utils/configs/*.json",
    ]
669
}
Simon Mo's avatar
Simon Mo committed
670

671
672
673
if _no_device():
    ext_modules = []

674
675
676
677
678
679
680
681
if not ext_modules:
    cmdclass = {}
else:
    cmdclass = {
        "build_ext":
        repackage_wheel if envs.VLLM_USE_PRECOMPILED else cmake_build_ext
    }

bnellnm's avatar
bnellnm committed
682
setup(
683
    # static metadata should rather go in pyproject.toml
684
    version=get_vllm_version(),
Woosuk Kwon's avatar
Woosuk Kwon committed
685
    ext_modules=ext_modules,
686
    install_requires=get_requirements(),
687
    extras_require={
688
        "tensorizer": ["tensorizer>=2.9.0"],
689
        "runai": ["runai-model-streamer", "runai-model-streamer-s3", "boto3"],
690
691
        "audio": ["librosa", "soundfile"],  # Required for audio processing
        "video": ["decord"]  # Required for video processing
692
    },
693
    cmdclass=cmdclass,
Simon Mo's avatar
Simon Mo committed
694
    package_data=package_data,
Woosuk Kwon's avatar
Woosuk Kwon committed
695
)