setup.py 23.4 KB
Newer Older
1
import ctypes
2
import importlib.util
3
import logging
4
5
import os
import re
6
import subprocess
bnellnm's avatar
bnellnm committed
7
import sys
8
from pathlib import Path
9
from shutil import which
10
from typing import Dict, List
11

Woosuk Kwon's avatar
Woosuk Kwon committed
12
import torch
13
14
15
from packaging.version import Version, parse
from setuptools import Extension, find_packages, setup
from setuptools.command.build_ext import build_ext
16
from setuptools_scm import get_version
17
from torch.utils.cpp_extension import CUDA_HOME, ROCM_HOME
18

19
20
21
22
23
24
25
26
27

def load_module_from_path(module_name, path):
    spec = importlib.util.spec_from_file_location(module_name, path)
    module = importlib.util.module_from_spec(spec)
    sys.modules[module_name] = module
    spec.loader.exec_module(module)
    return module


28
ROOT_DIR = os.path.dirname(__file__)
29
logger = logging.getLogger(__name__)
30
31
32
33
34
35

# cannot import envs directly because it depends on vllm,
#  which is not installed yet
envs = load_module_from_path('envs', os.path.join(ROOT_DIR, 'vllm', 'envs.py'))

VLLM_TARGET_DEVICE = envs.VLLM_TARGET_DEVICE
36

37
if sys.platform.startswith("darwin") and VLLM_TARGET_DEVICE != "cpu":
38
    logger.warning(
39
40
41
42
43
44
        "VLLM_TARGET_DEVICE automatically set to `cpu` due to macOS")
    VLLM_TARGET_DEVICE = "cpu"
elif not (sys.platform.startswith("linux")
          or sys.platform.startswith("darwin")):
    logger.warning(
        "vLLM only supports Linux platform (including WSL) and MacOS."
45
46
47
        "Building on %s, "
        "so vLLM may not be able to run correctly", sys.platform)
    VLLM_TARGET_DEVICE = "empty"
48

49
50
MAIN_CUDA_VERSION = "12.1"

bnellnm's avatar
bnellnm committed
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

def is_sccache_available() -> bool:
    return which("sccache") is not None


def is_ccache_available() -> bool:
    return which("ccache") is not None


def is_ninja_available() -> bool:
    return which("ninja") is not None


class CMakeExtension(Extension):

    def __init__(self, name: str, cmake_lists_dir: str = '.', **kwa) -> None:
67
        super().__init__(name, sources=[], py_limited_api=True, **kwa)
bnellnm's avatar
bnellnm committed
68
69
70
71
72
        self.cmake_lists_dir = os.path.abspath(cmake_lists_dir)


class cmake_build_ext(build_ext):
    # A dict of extension directories that have been configured.
73
    did_config: Dict[str, bool] = {}
bnellnm's avatar
bnellnm committed
74
75
76
77
78

    #
    # Determine number of compilation jobs and optionally nvcc compile threads.
    #
    def compute_num_jobs(self):
79
80
        # `num_jobs` is either the value of the MAX_JOBS environment variable
        # (if defined) or the number of CPUs available.
81
        num_jobs = envs.MAX_JOBS
82
83
        if num_jobs is not None:
            num_jobs = int(num_jobs)
84
            logger.info("Using MAX_JOBS=%d as the number of jobs.", num_jobs)
85
86
87
88
89
90
91
        else:
            try:
                # os.sched_getaffinity() isn't universally available, so fall
                #  back to os.cpu_count() if we get an error here.
                num_jobs = len(os.sched_getaffinity(0))
            except AttributeError:
                num_jobs = os.cpu_count()
bnellnm's avatar
bnellnm committed
92

93
        nvcc_threads = None
94
95
96
97
98
        if _is_cuda() and get_nvcc_cuda_version() >= Version("11.2"):
            # `nvcc_threads` is either the value of the NVCC_THREADS
            # environment variable (if defined) or 1.
            # when it is set, we reduce `num_jobs` to avoid
            # overloading the system.
99
            nvcc_threads = envs.NVCC_THREADS
100
101
            if nvcc_threads is not None:
                nvcc_threads = int(nvcc_threads)
102
103
104
                logger.info(
                    "Using NVCC_THREADS=%d as the number of nvcc threads.",
                    nvcc_threads)
105
106
107
            else:
                nvcc_threads = 1
            num_jobs = max(1, num_jobs // nvcc_threads)
bnellnm's avatar
bnellnm committed
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124

        return num_jobs, nvcc_threads

    #
    # Perform cmake configuration for a single extension.
    #
    def configure(self, ext: CMakeExtension) -> None:
        # If we've already configured using the CMakeLists.txt for
        # this extension, exit early.
        if ext.cmake_lists_dir in cmake_build_ext.did_config:
            return

        cmake_build_ext.did_config[ext.cmake_lists_dir] = True

        # Select the build type.
        # Note: optimization level + debug info are set by the build type
        default_cfg = "Debug" if self.debug else "RelWithDebInfo"
125
        cfg = envs.CMAKE_BUILD_TYPE or default_cfg
bnellnm's avatar
bnellnm committed
126
127
128

        cmake_args = [
            '-DCMAKE_BUILD_TYPE={}'.format(cfg),
129
            '-DVLLM_TARGET_DEVICE={}'.format(VLLM_TARGET_DEVICE),
bnellnm's avatar
bnellnm committed
130
131
        ]

132
        verbose = envs.VERBOSE
bnellnm's avatar
bnellnm committed
133
134
135
136
137
        if verbose:
            cmake_args += ['-DCMAKE_VERBOSE_MAKEFILE=ON']

        if is_sccache_available():
            cmake_args += [
138
                '-DCMAKE_C_COMPILER_LAUNCHER=sccache',
bnellnm's avatar
bnellnm committed
139
140
                '-DCMAKE_CXX_COMPILER_LAUNCHER=sccache',
                '-DCMAKE_CUDA_COMPILER_LAUNCHER=sccache',
141
                '-DCMAKE_HIP_COMPILER_LAUNCHER=sccache',
bnellnm's avatar
bnellnm committed
142
143
144
            ]
        elif is_ccache_available():
            cmake_args += [
145
                '-DCMAKE_C_COMPILER_LAUNCHER=ccache',
bnellnm's avatar
bnellnm committed
146
147
                '-DCMAKE_CXX_COMPILER_LAUNCHER=ccache',
                '-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache',
148
                '-DCMAKE_HIP_COMPILER_LAUNCHER=ccache',
bnellnm's avatar
bnellnm committed
149
150
151
152
153
154
            ]

        # Pass the python executable to cmake so it can find an exact
        # match.
        cmake_args += ['-DVLLM_PYTHON_EXECUTABLE={}'.format(sys.executable)]

155
156
157
158
        # Pass the python path to cmake so it can reuse the build dependencies
        # on subsequent calls to python.
        cmake_args += ['-DVLLM_PYTHON_PATH={}'.format(":".join(sys.path))]

159
160
161
162
163
164
165
166
        # Override the base directory for FetchContent downloads to $ROOT/.deps
        # This allows sharing dependencies between profiles,
        # and plays more nicely with sccache.
        # To override this, set the FETCHCONTENT_BASE_DIR environment variable.
        fc_base_dir = os.path.join(ROOT_DIR, ".deps")
        fc_base_dir = os.environ.get("FETCHCONTENT_BASE_DIR", fc_base_dir)
        cmake_args += ['-DFETCHCONTENT_BASE_DIR={}'.format(fc_base_dir)]

bnellnm's avatar
bnellnm committed
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
        #
        # Setup parallelism and build tool
        #
        num_jobs, nvcc_threads = self.compute_num_jobs()

        if nvcc_threads:
            cmake_args += ['-DNVCC_THREADS={}'.format(nvcc_threads)]

        if is_ninja_available():
            build_tool = ['-G', 'Ninja']
            cmake_args += [
                '-DCMAKE_JOB_POOL_COMPILE:STRING=compile',
                '-DCMAKE_JOB_POOLS:STRING=compile={}'.format(num_jobs),
            ]
        else:
            # Default build tool to whatever cmake picks.
            build_tool = []
        subprocess.check_call(
            ['cmake', ext.cmake_lists_dir, *build_tool, *cmake_args],
            cwd=self.build_temp)

    def build_extensions(self) -> None:
        # Ensure that CMake is present and working
        try:
            subprocess.check_output(['cmake', '--version'])
        except OSError as e:
            raise RuntimeError('Cannot find CMake executable') from e

        # Create build directory if it does not exist.
        if not os.path.exists(self.build_temp):
            os.makedirs(self.build_temp)

199
        targets = []
200
201
202
203

        def target_name(s: str) -> str:
            return s.removeprefix("vllm.").removeprefix("vllm_flash_attn.")

bnellnm's avatar
bnellnm committed
204
205
206
        # Build all the extensions
        for ext in self.extensions:
            self.configure(ext)
207
            targets.append(target_name(ext.name))
bnellnm's avatar
bnellnm committed
208

209
        num_jobs, _ = self.compute_num_jobs()
bnellnm's avatar
bnellnm committed
210

211
212
213
214
215
216
        build_args = [
            "--build",
            ".",
            f"-j={num_jobs}",
            *[f"--target={name}" for name in targets],
        ]
bnellnm's avatar
bnellnm committed
217

218
        subprocess.check_call(["cmake", *build_args], cwd=self.build_temp)
219

220
221
222
223
224
225
226
227
228
229
230
        # Install the libraries
        for ext in self.extensions:
            # Install the extension into the proper location
            outdir = Path(self.get_ext_fullpath(ext.name)).parent.absolute()

            # Skip if the install directory is the same as the build directory
            if outdir == self.build_temp:
                continue

            # CMake appends the extension prefix to the install path,
            # and outdir already contains that prefix, so we need to remove it.
231
232
233
            # We assume only the final component of extension prefix is added by
            # CMake, this is currently true for current extensions but may not
            # always be the case.
234
            prefix = outdir
235
            if '.' in ext.name:
236
237
238
239
240
241
242
243
244
                prefix = prefix.parent

            # prefix here should actually be the same for all components
            install_args = [
                "cmake", "--install", ".", "--prefix", prefix, "--component",
                target_name(ext.name)
            ]
            subprocess.check_call(install_args, cwd=self.build_temp)

245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
    def run(self):
        # First, run the standard build_ext command to compile the extensions
        super().run()

        # copy vllm/vllm_flash_attn/*.py from self.build_lib to current
        # directory so that they can be included in the editable build
        import glob
        files = glob.glob(
            os.path.join(self.build_lib, "vllm", "vllm_flash_attn", "*.py"))
        for file in files:
            dst_file = os.path.join("vllm/vllm_flash_attn",
                                    os.path.basename(file))
            print(f"Copying {file} to {dst_file}")
            self.copy_file(file, dst_file)

260

261
262
class repackage_wheel(build_ext):
    """Extracts libraries and other files from an existing wheel."""
youkaichao's avatar
youkaichao committed
263
    default_wheel = "https://wheels.vllm.ai/nightly/vllm-1.0.0.dev-cp38-abi3-manylinux1_x86_64.whl"
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303

    def run(self) -> None:
        wheel_location = os.getenv("VLLM_PRECOMPILED_WHEEL_LOCATION",
                                   self.default_wheel)

        assert _is_cuda(
        ), "VLLM_USE_PRECOMPILED is only supported for CUDA builds"

        import zipfile

        if os.path.isfile(wheel_location):
            wheel_path = wheel_location
            print(f"Using existing wheel={wheel_path}")
        else:
            # Download the wheel from a given URL, assume
            # the filename is the last part of the URL
            wheel_filename = wheel_location.split("/")[-1]

            import tempfile

            # create a temporary directory to store the wheel
            temp_dir = tempfile.mkdtemp(prefix="vllm-wheels")
            wheel_path = os.path.join(temp_dir, wheel_filename)

            print(f"Downloading wheel from {wheel_location} to {wheel_path}")

            from urllib.request import urlretrieve

            try:
                urlretrieve(wheel_location, filename=wheel_path)
            except Exception as e:
                from setuptools.errors import SetupError

                raise SetupError(
                    f"Failed to get vLLM wheel from {wheel_location}") from e

        with zipfile.ZipFile(wheel_path) as wheel:
            files_to_copy = [
                "vllm/_C.abi3.so",
                "vllm/_moe_C.abi3.so",
304
305
                "vllm/vllm_flash_attn/_vllm_fa2_C.abi3.so",
                "vllm/vllm_flash_attn/_vllm_fa3_C.abi3.so",
306
307
                "vllm/vllm_flash_attn/flash_attn_interface.py",
                "vllm/vllm_flash_attn/__init__.py",
308
                "vllm/cumem_allocator.abi3.so",
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
                # "vllm/_version.py", # not available in nightly wheels yet
            ]
            file_members = filter(lambda x: x.filename in files_to_copy,
                                  wheel.filelist)

            for file in file_members:
                print(f"Extracting and including {file.filename} "
                      "from existing wheel")
                package_name = os.path.dirname(file.filename).replace("/", ".")
                file_name = os.path.basename(file.filename)

                if package_name not in package_data:
                    package_data[package_name] = []

                wheel.extract(file)
                if file_name.endswith(".py"):
                    # python files shouldn't be added to package_data
                    continue

                package_data[package_name].append(file_name)


331
def _is_hpu() -> bool:
332
333
334
335
336
337
338
    # if VLLM_TARGET_DEVICE env var was set explicitly, skip HPU autodetection
    if os.getenv("VLLM_TARGET_DEVICE", None) == VLLM_TARGET_DEVICE:
        return VLLM_TARGET_DEVICE == "hpu"

    # if VLLM_TARGET_DEVICE was not set explicitly, check if hl-smi succeeds,
    # and if it doesn't, check if habanalabs driver is loaded
    is_hpu_available = False
339
    try:
340
341
        out = subprocess.run(["hl-smi"], capture_output=True, check=True)
        is_hpu_available = out.returncode == 0
342
    except (FileNotFoundError, PermissionError, subprocess.CalledProcessError):
343
        if sys.platform.startswith("linux"):
344
345
346
347
348
349
            try:
                output = subprocess.check_output(
                    'lsmod | grep habanalabs | wc -l', shell=True)
                is_hpu_available = int(output) > 0
            except (ValueError, FileNotFoundError, PermissionError,
                    subprocess.CalledProcessError):
350
351
                pass
    return is_hpu_available
352
353


354
355
356
357
def _no_device() -> bool:
    return VLLM_TARGET_DEVICE == "empty"


358
def _is_cuda() -> bool:
359
360
    has_cuda = torch.version.cuda is not None
    return (VLLM_TARGET_DEVICE == "cuda" and has_cuda
361
            and not (_is_neuron() or _is_tpu() or _is_hpu()))
362
363


364
def _is_hip() -> bool:
365
366
    return (VLLM_TARGET_DEVICE == "cuda"
            or VLLM_TARGET_DEVICE == "rocm") and torch.version.hip is not None
367
368


369
370
371
372
def _is_neuron() -> bool:
    torch_neuronx_installed = True
    try:
        subprocess.run(["neuron-ls"], capture_output=True, check=True)
373
    except (FileNotFoundError, PermissionError, subprocess.CalledProcessError):
374
        torch_neuronx_installed = False
375
    return torch_neuronx_installed or VLLM_TARGET_DEVICE == "neuron"
376
377


378
379
380
381
def _is_tpu() -> bool:
    return VLLM_TARGET_DEVICE == "tpu"


382
383
384
385
def _is_cpu() -> bool:
    return VLLM_TARGET_DEVICE == "cpu"


386
387
388
389
def _is_openvino() -> bool:
    return VLLM_TARGET_DEVICE == "openvino"


390
391
392
393
def _is_xpu() -> bool:
    return VLLM_TARGET_DEVICE == "xpu"


394
395
396
397
def _build_custom_ops() -> bool:
    return _is_cuda() or _is_hip() or _is_cpu()


398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
def get_rocm_version():
    # Get the Rocm version from the ROCM_HOME/bin/librocm-core.so
    # see https://github.com/ROCm/rocm-core/blob/d11f5c20d500f729c393680a01fa902ebf92094b/rocm_version.cpp#L21
    try:
        librocm_core_file = Path(ROCM_HOME) / "lib" / "librocm-core.so"
        if not librocm_core_file.is_file():
            return None
        librocm_core = ctypes.CDLL(librocm_core_file)
        VerErrors = ctypes.c_uint32
        get_rocm_core_version = librocm_core.getROCmVersion
        get_rocm_core_version.restype = VerErrors
        get_rocm_core_version.argtypes = [
            ctypes.POINTER(ctypes.c_uint32),
            ctypes.POINTER(ctypes.c_uint32),
            ctypes.POINTER(ctypes.c_uint32),
        ]
        major = ctypes.c_uint32()
        minor = ctypes.c_uint32()
        patch = ctypes.c_uint32()
417

418
419
        if (get_rocm_core_version(ctypes.byref(major), ctypes.byref(minor),
                                  ctypes.byref(patch)) == 0):
420
            return f"{major.value}.{minor.value}.{patch.value}"
421
        return None
422
    except Exception:
423
        return None
Woosuk Kwon's avatar
Woosuk Kwon committed
424

425

426
427
428
def get_neuronxcc_version():
    import sysconfig
    site_dir = sysconfig.get_paths()["purelib"]
429
430
    version_file = os.path.join(site_dir, "neuronxcc", "version",
                                "__init__.py")
431
432

    # Check if the command was executed successfully
433
    with open(version_file) as fp:
434
435
436
437
438
439
440
441
        content = fp.read()

    # Extract the version using a regular expression
    match = re.search(r"__version__ = '(\S+)'", content)
    if match:
        # Return the version string
        return match.group(1)
    else:
442
        raise RuntimeError("Could not find Neuron version in the output")
443
444


bnellnm's avatar
bnellnm committed
445
def get_nvcc_cuda_version() -> Version:
446
447
448
449
    """Get the CUDA version from nvcc.

    Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py
    """
450
    assert CUDA_HOME is not None, "CUDA_HOME is not set"
bnellnm's avatar
bnellnm committed
451
    nvcc_output = subprocess.check_output([CUDA_HOME + "/bin/nvcc", "-V"],
452
453
454
455
456
457
458
                                          universal_newlines=True)
    output = nvcc_output.split()
    release_idx = output.index("release") + 1
    nvcc_cuda_version = parse(output[release_idx].split(",")[0])
    return nvcc_cuda_version


459
460
461
462
def get_path(*filepath) -> str:
    return os.path.join(ROOT_DIR, *filepath)


463
464
465
466
467
468
469
470
def get_gaudi_sw_version():
    """
    Returns the driver version.
    """
    # Enable console printing for `hl-smi` check
    output = subprocess.run("hl-smi",
                            shell=True,
                            text=True,
471
                            capture_output=True,
472
473
474
475
476
477
478
                            env={"ENABLE_CONSOLE": "true"})
    if output.returncode == 0 and output.stdout:
        return output.stdout.split("\n")[2].replace(
            " ", "").split(":")[1][:-1].split("-")[0]
    return "0.0.0"  # when hl-smi is not available


479
def get_vllm_version() -> str:
480
481
482
    version = get_version(
        write_to="vllm/_version.py",  # TODO: move this to pyproject.toml
    )
483

484
    sep = "+" if "+" not in version else "."  # dev versions might contain +
485

486
    if _no_device():
487
        if envs.VLLM_TARGET_DEVICE == "empty":
488
            version += f"{sep}empty"
489
    elif _is_cuda():
490
        if envs.VLLM_USE_PRECOMPILED:
491
            version += f"{sep}precompiled"
492
493
494
495
496
497
498
        else:
            cuda_version = str(get_nvcc_cuda_version())
            if cuda_version != MAIN_CUDA_VERSION:
                cuda_version_str = cuda_version.replace(".", "")[:3]
                # skip this for source tarball, required for pypi
                if "sdist" not in sys.argv:
                    version += f"{sep}cu{cuda_version_str}"
499
    elif _is_hip():
500
501
502
503
        # Get the Rocm Version
        rocm_version = get_rocm_version() or torch.version.hip
        if rocm_version and rocm_version != MAIN_CUDA_VERSION:
            version += f"{sep}rocm{rocm_version.replace('.', '')[:3]}"
504
505
    elif _is_neuron():
        # Get the Neuron version
bnellnm's avatar
bnellnm committed
506
        neuron_version = str(get_neuronxcc_version())
507
508
        if neuron_version != MAIN_CUDA_VERSION:
            neuron_version_str = neuron_version.replace(".", "")[:3]
509
            version += f"{sep}neuron{neuron_version_str}"
510
511
512
513
514
515
    elif _is_hpu():
        # Get the Intel Gaudi Software Suite version
        gaudi_sw_version = str(get_gaudi_sw_version())
        if gaudi_sw_version != MAIN_CUDA_VERSION:
            gaudi_sw_version = gaudi_sw_version.replace(".", "")[:3]
            version += f"{sep}gaudi{gaudi_sw_version}"
516
    elif _is_openvino():
517
        version += f"{sep}openvino"
518
    elif _is_tpu():
519
        version += f"{sep}tpu"
520
    elif _is_cpu():
521
        version += f"{sep}cpu"
522
    elif _is_xpu():
523
        version += f"{sep}xpu"
524
    else:
525
        raise RuntimeError("Unknown runtime environment")
526

527
528
529
    return version


530
def read_readme() -> str:
Stephen Krider's avatar
Stephen Krider committed
531
532
533
    """Read the README file if present."""
    p = get_path("README.md")
    if os.path.isfile(p):
534
535
        with open(get_path("README.md"), encoding="utf-8") as f:
            return f.read()
Stephen Krider's avatar
Stephen Krider committed
536
537
    else:
        return ""
538
539


540
541
def get_requirements() -> List[str]:
    """Get Python package dependencies from requirements.txt."""
542
543
544

    def _read_requirements(filename: str) -> List[str]:
        with open(get_path(filename)) as f:
545
            requirements = f.read().strip().split("\n")
546
547
548
549
        resolved_requirements = []
        for line in requirements:
            if line.startswith("-r "):
                resolved_requirements += _read_requirements(line.split()[1])
550
551
            elif line.startswith("--"):
                continue
552
553
554
555
            else:
                resolved_requirements.append(line)
        return resolved_requirements

556
    if _no_device():
557
        requirements = _read_requirements("requirements-cpu.txt")
558
    elif _is_cuda():
559
        requirements = _read_requirements("requirements-cuda.txt")
560
        cuda_major, cuda_minor = torch.version.cuda.split(".")
561
562
        modified_requirements = []
        for req in requirements:
563
564
            if ("vllm-flash-attn" in req
                    and not (cuda_major == "12" and cuda_minor == "1")):
565
566
567
568
                # vllm-flash-attn is built only for CUDA 12.1.
                # Skip for other versions.
                continue
            modified_requirements.append(req)
569
        requirements = modified_requirements
570
    elif _is_hip():
571
        requirements = _read_requirements("requirements-rocm.txt")
572
    elif _is_neuron():
573
        requirements = _read_requirements("requirements-neuron.txt")
574
575
    elif _is_hpu():
        requirements = _read_requirements("requirements-hpu.txt")
576
577
    elif _is_openvino():
        requirements = _read_requirements("requirements-openvino.txt")
578
579
    elif _is_tpu():
        requirements = _read_requirements("requirements-tpu.txt")
580
    elif _is_cpu():
581
        requirements = _read_requirements("requirements-cpu.txt")
582
583
    elif _is_xpu():
        requirements = _read_requirements("requirements-xpu.txt")
584
585
    else:
        raise ValueError(
586
            "Unsupported platform, please use CUDA, ROCm, Neuron, HPU, "
587
            "OpenVINO, or CPU.")
588
589
590
    return requirements


bnellnm's avatar
bnellnm committed
591
592
ext_modules = []

593
if _is_cuda() or _is_hip():
bnellnm's avatar
bnellnm committed
594
595
    ext_modules.append(CMakeExtension(name="vllm._moe_C"))

596
597
598
if _is_hip():
    ext_modules.append(CMakeExtension(name="vllm._rocm_C"))

599
if _is_cuda():
600
    ext_modules.append(CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa2_C"))
601
602
603
604
    if envs.VLLM_USE_PRECOMPILED or get_nvcc_cuda_version() >= Version("12.0"):
        # FA3 requires CUDA 12.0 or later
        ext_modules.append(
            CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa3_C"))
605
    ext_modules.append(CMakeExtension(name="vllm.cumem_allocator"))
606

607
if _build_custom_ops():
bnellnm's avatar
bnellnm committed
608
609
    ext_modules.append(CMakeExtension(name="vllm._C"))

610
611
612
package_data = {
    "vllm": ["py.typed", "model_executor/layers/fused_moe/configs/*.json"]
}
Simon Mo's avatar
Simon Mo committed
613

614
615
616
if _no_device():
    ext_modules = []

617
618
619
620
621
622
623
624
if not ext_modules:
    cmdclass = {}
else:
    cmdclass = {
        "build_ext":
        repackage_wheel if envs.VLLM_USE_PRECOMPILED else cmake_build_ext
    }

bnellnm's avatar
bnellnm committed
625
setup(
Woosuk Kwon's avatar
Woosuk Kwon committed
626
    name="vllm",
627
    version=get_vllm_version(),
Woosuk Kwon's avatar
Woosuk Kwon committed
628
    author="vLLM Team",
629
    license="Apache 2.0",
Woosuk Kwon's avatar
Woosuk Kwon committed
630
631
    description=("A high-throughput and memory-efficient inference and "
                 "serving engine for LLMs"),
632
633
    long_description=read_readme(),
    long_description_content_type="text/markdown",
634
    url="https://github.com/vllm-project/vllm",
635
    project_urls={
636
637
        "Homepage": "https://github.com/vllm-project/vllm",
        "Documentation": "https://vllm.readthedocs.io/en/latest/",
638
639
640
641
    },
    classifiers=[
        "Programming Language :: Python :: 3.9",
        "Programming Language :: Python :: 3.10",
Woosuk Kwon's avatar
Woosuk Kwon committed
642
        "Programming Language :: Python :: 3.11",
643
        "Programming Language :: Python :: 3.12",
644
        "License :: OSI Approved :: Apache Software License",
645
646
647
        "Intended Audience :: Developers",
        "Intended Audience :: Information Technology",
        "Intended Audience :: Science/Research",
648
        "Topic :: Scientific/Engineering :: Artificial Intelligence",
649
        "Topic :: Scientific/Engineering :: Information Analysis",
650
    ],
bnellnm's avatar
bnellnm committed
651
    packages=find_packages(exclude=("benchmarks", "csrc", "docs", "examples",
652
                                    "tests*")),
653
    python_requires=">=3.9",
654
    install_requires=get_requirements(),
Woosuk Kwon's avatar
Woosuk Kwon committed
655
    ext_modules=ext_modules,
656
    extras_require={
657
        "tensorizer": ["tensorizer>=2.9.0"],
658
        "runai": ["runai-model-streamer", "runai-model-streamer-s3", "boto3"],
659
660
        "audio": ["librosa", "soundfile"],  # Required for audio processing
        "video": ["decord"]  # Required for video processing
661
    },
662
    cmdclass=cmdclass,
Simon Mo's avatar
Simon Mo committed
663
    package_data=package_data,
Ethan Xu's avatar
Ethan Xu committed
664
665
666
667
668
    entry_points={
        "console_scripts": [
            "vllm=vllm.scripts:main",
        ],
    },
Woosuk Kwon's avatar
Woosuk Kwon committed
669
)