setup.py 23.8 KB
Newer Older
1
2
# SPDX-License-Identifier: Apache-2.0

3
import ctypes
4
import importlib.util
5
import logging
6
7
import os
import re
8
import subprocess
bnellnm's avatar
bnellnm committed
9
import sys
10
from pathlib import Path
11
from shutil import which
12
from typing import Dict, List
13

Woosuk Kwon's avatar
Woosuk Kwon committed
14
import torch
15
16
17
from packaging.version import Version, parse
from setuptools import Extension, find_packages, setup
from setuptools.command.build_ext import build_ext
18
from setuptools_scm import get_version
19
from torch.utils.cpp_extension import CUDA_HOME, ROCM_HOME
20

21
22
23
24
25
26
27
28
29

def load_module_from_path(module_name, path):
    spec = importlib.util.spec_from_file_location(module_name, path)
    module = importlib.util.module_from_spec(spec)
    sys.modules[module_name] = module
    spec.loader.exec_module(module)
    return module


30
ROOT_DIR = os.path.dirname(__file__)
31
logger = logging.getLogger(__name__)
32
33
34
35
36
37

# cannot import envs directly because it depends on vllm,
#  which is not installed yet
envs = load_module_from_path('envs', os.path.join(ROOT_DIR, 'vllm', 'envs.py'))

VLLM_TARGET_DEVICE = envs.VLLM_TARGET_DEVICE
38

39
if sys.platform.startswith("darwin") and VLLM_TARGET_DEVICE != "cpu":
40
    logger.warning(
41
42
43
44
45
46
        "VLLM_TARGET_DEVICE automatically set to `cpu` due to macOS")
    VLLM_TARGET_DEVICE = "cpu"
elif not (sys.platform.startswith("linux")
          or sys.platform.startswith("darwin")):
    logger.warning(
        "vLLM only supports Linux platform (including WSL) and MacOS."
47
48
49
        "Building on %s, "
        "so vLLM may not be able to run correctly", sys.platform)
    VLLM_TARGET_DEVICE = "empty"
50
51
52
53
54
elif (sys.platform.startswith("linux") and torch.version.cuda is None
      and os.getenv("VLLM_TARGET_DEVICE") is None):
    # if cuda is not available and VLLM_TARGET_DEVICE is not set,
    # fallback to cpu
    VLLM_TARGET_DEVICE = "cpu"
55

56
57
MAIN_CUDA_VERSION = "12.1"

bnellnm's avatar
bnellnm committed
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

def is_sccache_available() -> bool:
    return which("sccache") is not None


def is_ccache_available() -> bool:
    return which("ccache") is not None


def is_ninja_available() -> bool:
    return which("ninja") is not None


class CMakeExtension(Extension):

    def __init__(self, name: str, cmake_lists_dir: str = '.', **kwa) -> None:
74
        super().__init__(name, sources=[], py_limited_api=True, **kwa)
bnellnm's avatar
bnellnm committed
75
76
77
78
79
        self.cmake_lists_dir = os.path.abspath(cmake_lists_dir)


class cmake_build_ext(build_ext):
    # A dict of extension directories that have been configured.
80
    did_config: Dict[str, bool] = {}
bnellnm's avatar
bnellnm committed
81
82
83
84
85

    #
    # Determine number of compilation jobs and optionally nvcc compile threads.
    #
    def compute_num_jobs(self):
86
87
        # `num_jobs` is either the value of the MAX_JOBS environment variable
        # (if defined) or the number of CPUs available.
88
        num_jobs = envs.MAX_JOBS
89
90
        if num_jobs is not None:
            num_jobs = int(num_jobs)
91
            logger.info("Using MAX_JOBS=%d as the number of jobs.", num_jobs)
92
93
94
95
96
97
98
        else:
            try:
                # os.sched_getaffinity() isn't universally available, so fall
                #  back to os.cpu_count() if we get an error here.
                num_jobs = len(os.sched_getaffinity(0))
            except AttributeError:
                num_jobs = os.cpu_count()
bnellnm's avatar
bnellnm committed
99

100
        nvcc_threads = None
101
102
103
104
105
        if _is_cuda() and get_nvcc_cuda_version() >= Version("11.2"):
            # `nvcc_threads` is either the value of the NVCC_THREADS
            # environment variable (if defined) or 1.
            # when it is set, we reduce `num_jobs` to avoid
            # overloading the system.
106
            nvcc_threads = envs.NVCC_THREADS
107
108
            if nvcc_threads is not None:
                nvcc_threads = int(nvcc_threads)
109
110
111
                logger.info(
                    "Using NVCC_THREADS=%d as the number of nvcc threads.",
                    nvcc_threads)
112
113
114
            else:
                nvcc_threads = 1
            num_jobs = max(1, num_jobs // nvcc_threads)
bnellnm's avatar
bnellnm committed
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131

        return num_jobs, nvcc_threads

    #
    # Perform cmake configuration for a single extension.
    #
    def configure(self, ext: CMakeExtension) -> None:
        # If we've already configured using the CMakeLists.txt for
        # this extension, exit early.
        if ext.cmake_lists_dir in cmake_build_ext.did_config:
            return

        cmake_build_ext.did_config[ext.cmake_lists_dir] = True

        # Select the build type.
        # Note: optimization level + debug info are set by the build type
        default_cfg = "Debug" if self.debug else "RelWithDebInfo"
132
        cfg = envs.CMAKE_BUILD_TYPE or default_cfg
bnellnm's avatar
bnellnm committed
133
134
135

        cmake_args = [
            '-DCMAKE_BUILD_TYPE={}'.format(cfg),
136
            '-DVLLM_TARGET_DEVICE={}'.format(VLLM_TARGET_DEVICE),
bnellnm's avatar
bnellnm committed
137
138
        ]

139
        verbose = envs.VERBOSE
bnellnm's avatar
bnellnm committed
140
141
142
143
144
        if verbose:
            cmake_args += ['-DCMAKE_VERBOSE_MAKEFILE=ON']

        if is_sccache_available():
            cmake_args += [
145
                '-DCMAKE_C_COMPILER_LAUNCHER=sccache',
bnellnm's avatar
bnellnm committed
146
147
                '-DCMAKE_CXX_COMPILER_LAUNCHER=sccache',
                '-DCMAKE_CUDA_COMPILER_LAUNCHER=sccache',
148
                '-DCMAKE_HIP_COMPILER_LAUNCHER=sccache',
bnellnm's avatar
bnellnm committed
149
150
151
            ]
        elif is_ccache_available():
            cmake_args += [
152
                '-DCMAKE_C_COMPILER_LAUNCHER=ccache',
bnellnm's avatar
bnellnm committed
153
154
                '-DCMAKE_CXX_COMPILER_LAUNCHER=ccache',
                '-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache',
155
                '-DCMAKE_HIP_COMPILER_LAUNCHER=ccache',
bnellnm's avatar
bnellnm committed
156
157
158
159
160
161
            ]

        # Pass the python executable to cmake so it can find an exact
        # match.
        cmake_args += ['-DVLLM_PYTHON_EXECUTABLE={}'.format(sys.executable)]

162
163
164
165
        # Pass the python path to cmake so it can reuse the build dependencies
        # on subsequent calls to python.
        cmake_args += ['-DVLLM_PYTHON_PATH={}'.format(":".join(sys.path))]

166
167
168
169
170
171
172
173
        # Override the base directory for FetchContent downloads to $ROOT/.deps
        # This allows sharing dependencies between profiles,
        # and plays more nicely with sccache.
        # To override this, set the FETCHCONTENT_BASE_DIR environment variable.
        fc_base_dir = os.path.join(ROOT_DIR, ".deps")
        fc_base_dir = os.environ.get("FETCHCONTENT_BASE_DIR", fc_base_dir)
        cmake_args += ['-DFETCHCONTENT_BASE_DIR={}'.format(fc_base_dir)]

bnellnm's avatar
bnellnm committed
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
        #
        # Setup parallelism and build tool
        #
        num_jobs, nvcc_threads = self.compute_num_jobs()

        if nvcc_threads:
            cmake_args += ['-DNVCC_THREADS={}'.format(nvcc_threads)]

        if is_ninja_available():
            build_tool = ['-G', 'Ninja']
            cmake_args += [
                '-DCMAKE_JOB_POOL_COMPILE:STRING=compile',
                '-DCMAKE_JOB_POOLS:STRING=compile={}'.format(num_jobs),
            ]
        else:
            # Default build tool to whatever cmake picks.
            build_tool = []
        subprocess.check_call(
            ['cmake', ext.cmake_lists_dir, *build_tool, *cmake_args],
            cwd=self.build_temp)

    def build_extensions(self) -> None:
        # Ensure that CMake is present and working
        try:
            subprocess.check_output(['cmake', '--version'])
        except OSError as e:
            raise RuntimeError('Cannot find CMake executable') from e

        # Create build directory if it does not exist.
        if not os.path.exists(self.build_temp):
            os.makedirs(self.build_temp)

206
        targets = []
207
208
209
210

        def target_name(s: str) -> str:
            return s.removeprefix("vllm.").removeprefix("vllm_flash_attn.")

bnellnm's avatar
bnellnm committed
211
212
213
        # Build all the extensions
        for ext in self.extensions:
            self.configure(ext)
214
            targets.append(target_name(ext.name))
bnellnm's avatar
bnellnm committed
215

216
        num_jobs, _ = self.compute_num_jobs()
bnellnm's avatar
bnellnm committed
217

218
219
220
221
222
223
        build_args = [
            "--build",
            ".",
            f"-j={num_jobs}",
            *[f"--target={name}" for name in targets],
        ]
bnellnm's avatar
bnellnm committed
224

225
        subprocess.check_call(["cmake", *build_args], cwd=self.build_temp)
226

227
228
229
230
231
232
233
234
235
236
237
        # Install the libraries
        for ext in self.extensions:
            # Install the extension into the proper location
            outdir = Path(self.get_ext_fullpath(ext.name)).parent.absolute()

            # Skip if the install directory is the same as the build directory
            if outdir == self.build_temp:
                continue

            # CMake appends the extension prefix to the install path,
            # and outdir already contains that prefix, so we need to remove it.
238
239
240
            # We assume only the final component of extension prefix is added by
            # CMake, this is currently true for current extensions but may not
            # always be the case.
241
            prefix = outdir
242
            if '.' in ext.name:
243
244
245
246
247
248
249
250
251
                prefix = prefix.parent

            # prefix here should actually be the same for all components
            install_args = [
                "cmake", "--install", ".", "--prefix", prefix, "--component",
                target_name(ext.name)
            ]
            subprocess.check_call(install_args, cwd=self.build_temp)

252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
    def run(self):
        # First, run the standard build_ext command to compile the extensions
        super().run()

        # copy vllm/vllm_flash_attn/*.py from self.build_lib to current
        # directory so that they can be included in the editable build
        import glob
        files = glob.glob(
            os.path.join(self.build_lib, "vllm", "vllm_flash_attn", "*.py"))
        for file in files:
            dst_file = os.path.join("vllm/vllm_flash_attn",
                                    os.path.basename(file))
            print(f"Copying {file} to {dst_file}")
            self.copy_file(file, dst_file)

267

268
269
class repackage_wheel(build_ext):
    """Extracts libraries and other files from an existing wheel."""
youkaichao's avatar
youkaichao committed
270
    default_wheel = "https://wheels.vllm.ai/nightly/vllm-1.0.0.dev-cp38-abi3-manylinux1_x86_64.whl"
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310

    def run(self) -> None:
        wheel_location = os.getenv("VLLM_PRECOMPILED_WHEEL_LOCATION",
                                   self.default_wheel)

        assert _is_cuda(
        ), "VLLM_USE_PRECOMPILED is only supported for CUDA builds"

        import zipfile

        if os.path.isfile(wheel_location):
            wheel_path = wheel_location
            print(f"Using existing wheel={wheel_path}")
        else:
            # Download the wheel from a given URL, assume
            # the filename is the last part of the URL
            wheel_filename = wheel_location.split("/")[-1]

            import tempfile

            # create a temporary directory to store the wheel
            temp_dir = tempfile.mkdtemp(prefix="vllm-wheels")
            wheel_path = os.path.join(temp_dir, wheel_filename)

            print(f"Downloading wheel from {wheel_location} to {wheel_path}")

            from urllib.request import urlretrieve

            try:
                urlretrieve(wheel_location, filename=wheel_path)
            except Exception as e:
                from setuptools.errors import SetupError

                raise SetupError(
                    f"Failed to get vLLM wheel from {wheel_location}") from e

        with zipfile.ZipFile(wheel_path) as wheel:
            files_to_copy = [
                "vllm/_C.abi3.so",
                "vllm/_moe_C.abi3.so",
311
312
                "vllm/vllm_flash_attn/_vllm_fa2_C.abi3.so",
                "vllm/vllm_flash_attn/_vllm_fa3_C.abi3.so",
313
314
                "vllm/vllm_flash_attn/flash_attn_interface.py",
                "vllm/vllm_flash_attn/__init__.py",
315
                "vllm/cumem_allocator.abi3.so",
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
                # "vllm/_version.py", # not available in nightly wheels yet
            ]
            file_members = filter(lambda x: x.filename in files_to_copy,
                                  wheel.filelist)

            for file in file_members:
                print(f"Extracting and including {file.filename} "
                      "from existing wheel")
                package_name = os.path.dirname(file.filename).replace("/", ".")
                file_name = os.path.basename(file.filename)

                if package_name not in package_data:
                    package_data[package_name] = []

                wheel.extract(file)
                if file_name.endswith(".py"):
                    # python files shouldn't be added to package_data
                    continue

                package_data[package_name].append(file_name)


338
def _is_hpu() -> bool:
339
340
341
342
343
344
345
    # if VLLM_TARGET_DEVICE env var was set explicitly, skip HPU autodetection
    if os.getenv("VLLM_TARGET_DEVICE", None) == VLLM_TARGET_DEVICE:
        return VLLM_TARGET_DEVICE == "hpu"

    # if VLLM_TARGET_DEVICE was not set explicitly, check if hl-smi succeeds,
    # and if it doesn't, check if habanalabs driver is loaded
    is_hpu_available = False
346
    try:
347
348
        out = subprocess.run(["hl-smi"], capture_output=True, check=True)
        is_hpu_available = out.returncode == 0
349
    except (FileNotFoundError, PermissionError, subprocess.CalledProcessError):
350
        if sys.platform.startswith("linux"):
351
352
353
354
355
356
            try:
                output = subprocess.check_output(
                    'lsmod | grep habanalabs | wc -l', shell=True)
                is_hpu_available = int(output) > 0
            except (ValueError, FileNotFoundError, PermissionError,
                    subprocess.CalledProcessError):
357
358
                pass
    return is_hpu_available
359
360


361
362
363
364
def _no_device() -> bool:
    return VLLM_TARGET_DEVICE == "empty"


365
def _is_cuda() -> bool:
366
367
    has_cuda = torch.version.cuda is not None
    return (VLLM_TARGET_DEVICE == "cuda" and has_cuda
368
            and not (_is_neuron() or _is_tpu() or _is_hpu()))
369
370


371
def _is_hip() -> bool:
372
373
    return (VLLM_TARGET_DEVICE == "cuda"
            or VLLM_TARGET_DEVICE == "rocm") and torch.version.hip is not None
374
375


376
377
378
379
def _is_neuron() -> bool:
    torch_neuronx_installed = True
    try:
        subprocess.run(["neuron-ls"], capture_output=True, check=True)
380
    except (FileNotFoundError, PermissionError, subprocess.CalledProcessError):
381
        torch_neuronx_installed = False
382
    return torch_neuronx_installed or VLLM_TARGET_DEVICE == "neuron"
383
384


385
386
387
388
def _is_tpu() -> bool:
    return VLLM_TARGET_DEVICE == "tpu"


389
390
391
392
def _is_cpu() -> bool:
    return VLLM_TARGET_DEVICE == "cpu"


393
394
395
396
def _is_openvino() -> bool:
    return VLLM_TARGET_DEVICE == "openvino"


397
398
399
400
def _is_xpu() -> bool:
    return VLLM_TARGET_DEVICE == "xpu"


401
402
403
404
def _build_custom_ops() -> bool:
    return _is_cuda() or _is_hip() or _is_cpu()


405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
def get_rocm_version():
    # Get the Rocm version from the ROCM_HOME/bin/librocm-core.so
    # see https://github.com/ROCm/rocm-core/blob/d11f5c20d500f729c393680a01fa902ebf92094b/rocm_version.cpp#L21
    try:
        librocm_core_file = Path(ROCM_HOME) / "lib" / "librocm-core.so"
        if not librocm_core_file.is_file():
            return None
        librocm_core = ctypes.CDLL(librocm_core_file)
        VerErrors = ctypes.c_uint32
        get_rocm_core_version = librocm_core.getROCmVersion
        get_rocm_core_version.restype = VerErrors
        get_rocm_core_version.argtypes = [
            ctypes.POINTER(ctypes.c_uint32),
            ctypes.POINTER(ctypes.c_uint32),
            ctypes.POINTER(ctypes.c_uint32),
        ]
        major = ctypes.c_uint32()
        minor = ctypes.c_uint32()
        patch = ctypes.c_uint32()
424

425
426
        if (get_rocm_core_version(ctypes.byref(major), ctypes.byref(minor),
                                  ctypes.byref(patch)) == 0):
427
            return f"{major.value}.{minor.value}.{patch.value}"
428
        return None
429
    except Exception:
430
        return None
Woosuk Kwon's avatar
Woosuk Kwon committed
431

432

433
434
435
def get_neuronxcc_version():
    import sysconfig
    site_dir = sysconfig.get_paths()["purelib"]
436
437
    version_file = os.path.join(site_dir, "neuronxcc", "version",
                                "__init__.py")
438
439

    # Check if the command was executed successfully
440
    with open(version_file) as fp:
441
442
443
444
445
446
447
448
        content = fp.read()

    # Extract the version using a regular expression
    match = re.search(r"__version__ = '(\S+)'", content)
    if match:
        # Return the version string
        return match.group(1)
    else:
449
        raise RuntimeError("Could not find Neuron version in the output")
450
451


bnellnm's avatar
bnellnm committed
452
def get_nvcc_cuda_version() -> Version:
453
454
455
456
    """Get the CUDA version from nvcc.

    Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py
    """
457
    assert CUDA_HOME is not None, "CUDA_HOME is not set"
bnellnm's avatar
bnellnm committed
458
    nvcc_output = subprocess.check_output([CUDA_HOME + "/bin/nvcc", "-V"],
459
460
461
462
463
464
465
                                          universal_newlines=True)
    output = nvcc_output.split()
    release_idx = output.index("release") + 1
    nvcc_cuda_version = parse(output[release_idx].split(",")[0])
    return nvcc_cuda_version


466
467
468
469
def get_path(*filepath) -> str:
    return os.path.join(ROOT_DIR, *filepath)


470
471
472
473
474
475
476
477
def get_gaudi_sw_version():
    """
    Returns the driver version.
    """
    # Enable console printing for `hl-smi` check
    output = subprocess.run("hl-smi",
                            shell=True,
                            text=True,
478
                            capture_output=True,
479
480
481
482
483
484
485
                            env={"ENABLE_CONSOLE": "true"})
    if output.returncode == 0 and output.stdout:
        return output.stdout.split("\n")[2].replace(
            " ", "").split(":")[1][:-1].split("-")[0]
    return "0.0.0"  # when hl-smi is not available


486
def get_vllm_version() -> str:
487
488
489
    version = get_version(
        write_to="vllm/_version.py",  # TODO: move this to pyproject.toml
    )
490
    sep = "+" if "+" not in version else "."  # dev versions might contain +
491

492
    if _no_device():
493
        if envs.VLLM_TARGET_DEVICE == "empty":
494
            version += f"{sep}empty"
495
    elif _is_cuda():
496
        if envs.VLLM_USE_PRECOMPILED:
497
            version += f"{sep}precompiled"
498
499
500
501
502
503
504
        else:
            cuda_version = str(get_nvcc_cuda_version())
            if cuda_version != MAIN_CUDA_VERSION:
                cuda_version_str = cuda_version.replace(".", "")[:3]
                # skip this for source tarball, required for pypi
                if "sdist" not in sys.argv:
                    version += f"{sep}cu{cuda_version_str}"
505
    elif _is_hip():
506
507
508
509
        # Get the Rocm Version
        rocm_version = get_rocm_version() or torch.version.hip
        if rocm_version and rocm_version != MAIN_CUDA_VERSION:
            version += f"{sep}rocm{rocm_version.replace('.', '')[:3]}"
510
511
    elif _is_neuron():
        # Get the Neuron version
bnellnm's avatar
bnellnm committed
512
        neuron_version = str(get_neuronxcc_version())
513
514
        if neuron_version != MAIN_CUDA_VERSION:
            neuron_version_str = neuron_version.replace(".", "")[:3]
515
            version += f"{sep}neuron{neuron_version_str}"
516
517
518
519
520
521
    elif _is_hpu():
        # Get the Intel Gaudi Software Suite version
        gaudi_sw_version = str(get_gaudi_sw_version())
        if gaudi_sw_version != MAIN_CUDA_VERSION:
            gaudi_sw_version = gaudi_sw_version.replace(".", "")[:3]
            version += f"{sep}gaudi{gaudi_sw_version}"
522
    elif _is_openvino():
523
        version += f"{sep}openvino"
524
    elif _is_tpu():
525
        version += f"{sep}tpu"
526
    elif _is_cpu():
527
528
        if envs.VLLM_TARGET_DEVICE == "cpu":
            version += f"{sep}cpu"
529
    elif _is_xpu():
530
        version += f"{sep}xpu"
531
    else:
532
        raise RuntimeError("Unknown runtime environment")
533

534
535
536
    return version


537
def read_readme() -> str:
Stephen Krider's avatar
Stephen Krider committed
538
539
540
    """Read the README file if present."""
    p = get_path("README.md")
    if os.path.isfile(p):
541
542
        with open(get_path("README.md"), encoding="utf-8") as f:
            return f.read()
Stephen Krider's avatar
Stephen Krider committed
543
544
    else:
        return ""
545
546


547
548
def get_requirements() -> List[str]:
    """Get Python package dependencies from requirements.txt."""
549
550
551

    def _read_requirements(filename: str) -> List[str]:
        with open(get_path(filename)) as f:
552
            requirements = f.read().strip().split("\n")
553
554
555
556
        resolved_requirements = []
        for line in requirements:
            if line.startswith("-r "):
                resolved_requirements += _read_requirements(line.split()[1])
557
558
            elif line.startswith("--"):
                continue
559
560
561
562
            else:
                resolved_requirements.append(line)
        return resolved_requirements

563
    if _no_device():
564
        requirements = _read_requirements("requirements-common.txt")
565
    elif _is_cuda():
566
        requirements = _read_requirements("requirements-cuda.txt")
567
        cuda_major, cuda_minor = torch.version.cuda.split(".")
568
569
        modified_requirements = []
        for req in requirements:
570
571
            if ("vllm-flash-attn" in req
                    and not (cuda_major == "12" and cuda_minor == "1")):
572
573
574
575
                # vllm-flash-attn is built only for CUDA 12.1.
                # Skip for other versions.
                continue
            modified_requirements.append(req)
576
        requirements = modified_requirements
577
    elif _is_hip():
578
        requirements = _read_requirements("requirements-rocm.txt")
579
    elif _is_neuron():
580
        requirements = _read_requirements("requirements-neuron.txt")
581
582
    elif _is_hpu():
        requirements = _read_requirements("requirements-hpu.txt")
583
584
    elif _is_openvino():
        requirements = _read_requirements("requirements-openvino.txt")
585
586
    elif _is_tpu():
        requirements = _read_requirements("requirements-tpu.txt")
587
    elif _is_cpu():
588
        requirements = _read_requirements("requirements-cpu.txt")
589
590
    elif _is_xpu():
        requirements = _read_requirements("requirements-xpu.txt")
591
592
    else:
        raise ValueError(
593
            "Unsupported platform, please use CUDA, ROCm, Neuron, HPU, "
594
            "OpenVINO, or CPU.")
595
596
597
    return requirements


bnellnm's avatar
bnellnm committed
598
599
ext_modules = []

600
if _is_cuda() or _is_hip():
bnellnm's avatar
bnellnm committed
601
602
    ext_modules.append(CMakeExtension(name="vllm._moe_C"))

603
604
605
if _is_hip():
    ext_modules.append(CMakeExtension(name="vllm._rocm_C"))

606
if _is_cuda():
607
    ext_modules.append(CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa2_C"))
608
609
610
611
    if envs.VLLM_USE_PRECOMPILED or get_nvcc_cuda_version() >= Version("12.0"):
        # FA3 requires CUDA 12.0 or later
        ext_modules.append(
            CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa3_C"))
612
    ext_modules.append(CMakeExtension(name="vllm.cumem_allocator"))
613

614
if _build_custom_ops():
bnellnm's avatar
bnellnm committed
615
616
    ext_modules.append(CMakeExtension(name="vllm._C"))

617
package_data = {
618
619
620
621
622
    "vllm": [
        "py.typed",
        "model_executor/layers/fused_moe/configs/*.json",
        "model_executor/layers/quantization/utils/configs/*.json",
    ]
623
}
Simon Mo's avatar
Simon Mo committed
624

625
626
627
if _no_device():
    ext_modules = []

628
629
630
631
632
633
634
635
if not ext_modules:
    cmdclass = {}
else:
    cmdclass = {
        "build_ext":
        repackage_wheel if envs.VLLM_USE_PRECOMPILED else cmake_build_ext
    }

bnellnm's avatar
bnellnm committed
636
setup(
Woosuk Kwon's avatar
Woosuk Kwon committed
637
    name="vllm",
638
    version=get_vllm_version(),
Woosuk Kwon's avatar
Woosuk Kwon committed
639
    author="vLLM Team",
640
    license="Apache 2.0",
Woosuk Kwon's avatar
Woosuk Kwon committed
641
642
    description=("A high-throughput and memory-efficient inference and "
                 "serving engine for LLMs"),
643
644
    long_description=read_readme(),
    long_description_content_type="text/markdown",
645
    url="https://github.com/vllm-project/vllm",
646
    project_urls={
647
648
        "Homepage": "https://github.com/vllm-project/vllm",
        "Documentation": "https://vllm.readthedocs.io/en/latest/",
649
650
651
652
    },
    classifiers=[
        "Programming Language :: Python :: 3.9",
        "Programming Language :: Python :: 3.10",
Woosuk Kwon's avatar
Woosuk Kwon committed
653
        "Programming Language :: Python :: 3.11",
654
        "Programming Language :: Python :: 3.12",
655
        "License :: OSI Approved :: Apache Software License",
656
657
658
        "Intended Audience :: Developers",
        "Intended Audience :: Information Technology",
        "Intended Audience :: Science/Research",
659
        "Topic :: Scientific/Engineering :: Artificial Intelligence",
660
        "Topic :: Scientific/Engineering :: Information Analysis",
661
    ],
bnellnm's avatar
bnellnm committed
662
    packages=find_packages(exclude=("benchmarks", "csrc", "docs", "examples",
663
                                    "tests*")),
664
    python_requires=">=3.9",
665
    install_requires=get_requirements(),
Woosuk Kwon's avatar
Woosuk Kwon committed
666
    ext_modules=ext_modules,
667
    extras_require={
668
        "tensorizer": ["tensorizer>=2.9.0"],
669
        "runai": ["runai-model-streamer", "runai-model-streamer-s3", "boto3"],
670
671
        "audio": ["librosa", "soundfile"],  # Required for audio processing
        "video": ["decord"]  # Required for video processing
672
    },
673
    cmdclass=cmdclass,
Simon Mo's avatar
Simon Mo committed
674
    package_data=package_data,
Ethan Xu's avatar
Ethan Xu committed
675
676
677
678
679
    entry_points={
        "console_scripts": [
            "vllm=vllm.scripts:main",
        ],
    },
Woosuk Kwon's avatar
Woosuk Kwon committed
680
)