setup.py 27.1 KB
Newer Older
1
2
# SPDX-License-Identifier: Apache-2.0

3
import ctypes
4
import importlib.util
5
import json
6
import logging
7
8
import os
import re
9
import subprocess
bnellnm's avatar
bnellnm committed
10
import sys
11
from pathlib import Path
12
from shutil import which
13

Woosuk Kwon's avatar
Woosuk Kwon committed
14
import torch
15
from packaging.version import Version, parse
16
from setuptools import Extension, setup
17
from setuptools.command.build_ext import build_ext
18
from torch.utils.cpp_extension import CUDA_HOME, ROCM_HOME
19

zhuwenwen's avatar
zhuwenwen committed
20
21
22
23
from typing import Optional, Union
import subprocess
from pathlib import Path

zhuwenwen's avatar
zhuwenwen committed
24
pwd = os.path.dirname(os.path.abspath(__file__))
gaoqiong's avatar
gaoqiong committed
25
add_git_version = False
26
27
if int(os.environ.get('ADD_GIT_VERSION', '0')) == 1:
    add_git_version = True
28
29
30
31
32
33
34
35
36

def load_module_from_path(module_name, path):
    spec = importlib.util.spec_from_file_location(module_name, path)
    module = importlib.util.module_from_spec(spec)
    sys.modules[module_name] = module
    spec.loader.exec_module(module)
    return module


37
ROOT_DIR = Path(__file__).parent
38
logger = logging.getLogger(__name__)
39
40
41
42
43
44

# cannot import envs directly because it depends on vllm,
#  which is not installed yet
envs = load_module_from_path('envs', os.path.join(ROOT_DIR, 'vllm', 'envs.py'))

VLLM_TARGET_DEVICE = envs.VLLM_TARGET_DEVICE
45

46
if sys.platform.startswith("darwin") and VLLM_TARGET_DEVICE != "cpu":
47
    logger.warning(
48
49
50
51
52
53
        "VLLM_TARGET_DEVICE automatically set to `cpu` due to macOS")
    VLLM_TARGET_DEVICE = "cpu"
elif not (sys.platform.startswith("linux")
          or sys.platform.startswith("darwin")):
    logger.warning(
        "vLLM only supports Linux platform (including WSL) and MacOS."
54
55
56
        "Building on %s, "
        "so vLLM may not be able to run correctly", sys.platform)
    VLLM_TARGET_DEVICE = "empty"
57
elif (sys.platform.startswith("linux") and torch.version.cuda is None
58
59
60
      and os.getenv("VLLM_TARGET_DEVICE") is None
      and torch.version.hip is None):
    # if cuda or hip is not available and VLLM_TARGET_DEVICE is not set,
61
62
    # fallback to cpu
    VLLM_TARGET_DEVICE = "cpu"
63

64
MAIN_CUDA_VERSION = "12.4"
65

bnellnm's avatar
bnellnm committed
66
67
68
69
70
71
72
73
74
75
76
77
78

def is_sccache_available() -> bool:
    return which("sccache") is not None


def is_ccache_available() -> bool:
    return which("ccache") is not None


def is_ninja_available() -> bool:
    return which("ninja") is not None


79
80
81
82
83
84
85
86
87
88
89
90
def is_url_available(url: str) -> bool:
    from urllib.request import urlopen

    status = None
    try:
        with urlopen(url) as f:
            status = f.status
    except Exception:
        return False
    return status == 200


bnellnm's avatar
bnellnm committed
91
92
93
class CMakeExtension(Extension):

    def __init__(self, name: str, cmake_lists_dir: str = '.', **kwa) -> None:
94
        super().__init__(name, sources=[], py_limited_api=True, **kwa)
bnellnm's avatar
bnellnm committed
95
96
97
98
99
        self.cmake_lists_dir = os.path.abspath(cmake_lists_dir)


class cmake_build_ext(build_ext):
    # A dict of extension directories that have been configured.
100
    did_config: dict[str, bool] = {}
bnellnm's avatar
bnellnm committed
101
102
103
104
105

    #
    # Determine number of compilation jobs and optionally nvcc compile threads.
    #
    def compute_num_jobs(self):
106
107
        # `num_jobs` is either the value of the MAX_JOBS environment variable
        # (if defined) or the number of CPUs available.
108
        num_jobs = envs.MAX_JOBS
109
110
        if num_jobs is not None:
            num_jobs = int(num_jobs)
111
            logger.info("Using MAX_JOBS=%d as the number of jobs.", num_jobs)
112
113
114
115
116
117
118
        else:
            try:
                # os.sched_getaffinity() isn't universally available, so fall
                #  back to os.cpu_count() if we get an error here.
                num_jobs = len(os.sched_getaffinity(0))
            except AttributeError:
                num_jobs = os.cpu_count()
bnellnm's avatar
bnellnm committed
119

120
        nvcc_threads = None
121
122
123
124
125
        if _is_cuda() and get_nvcc_cuda_version() >= Version("11.2"):
            # `nvcc_threads` is either the value of the NVCC_THREADS
            # environment variable (if defined) or 1.
            # when it is set, we reduce `num_jobs` to avoid
            # overloading the system.
126
            nvcc_threads = envs.NVCC_THREADS
127
128
            if nvcc_threads is not None:
                nvcc_threads = int(nvcc_threads)
129
130
131
                logger.info(
                    "Using NVCC_THREADS=%d as the number of nvcc threads.",
                    nvcc_threads)
132
133
134
            else:
                nvcc_threads = 1
            num_jobs = max(1, num_jobs // nvcc_threads)
bnellnm's avatar
bnellnm committed
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151

        return num_jobs, nvcc_threads

    #
    # Perform cmake configuration for a single extension.
    #
    def configure(self, ext: CMakeExtension) -> None:
        # If we've already configured using the CMakeLists.txt for
        # this extension, exit early.
        if ext.cmake_lists_dir in cmake_build_ext.did_config:
            return

        cmake_build_ext.did_config[ext.cmake_lists_dir] = True

        # Select the build type.
        # Note: optimization level + debug info are set by the build type
        default_cfg = "Debug" if self.debug else "RelWithDebInfo"
152
        cfg = envs.CMAKE_BUILD_TYPE or default_cfg
bnellnm's avatar
bnellnm committed
153
154
155

        cmake_args = [
            '-DCMAKE_BUILD_TYPE={}'.format(cfg),
156
            '-DVLLM_TARGET_DEVICE={}'.format(VLLM_TARGET_DEVICE),
bnellnm's avatar
bnellnm committed
157
158
        ]

159
        verbose = envs.VERBOSE
bnellnm's avatar
bnellnm committed
160
161
162
163
164
        if verbose:
            cmake_args += ['-DCMAKE_VERBOSE_MAKEFILE=ON']

        if is_sccache_available():
            cmake_args += [
165
                '-DCMAKE_C_COMPILER_LAUNCHER=sccache',
bnellnm's avatar
bnellnm committed
166
167
                '-DCMAKE_CXX_COMPILER_LAUNCHER=sccache',
                '-DCMAKE_CUDA_COMPILER_LAUNCHER=sccache',
168
                '-DCMAKE_HIP_COMPILER_LAUNCHER=sccache',
bnellnm's avatar
bnellnm committed
169
170
171
            ]
        elif is_ccache_available():
            cmake_args += [
172
                '-DCMAKE_C_COMPILER_LAUNCHER=ccache',
bnellnm's avatar
bnellnm committed
173
174
                '-DCMAKE_CXX_COMPILER_LAUNCHER=ccache',
                '-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache',
175
                '-DCMAKE_HIP_COMPILER_LAUNCHER=ccache',
bnellnm's avatar
bnellnm committed
176
177
178
179
180
181
            ]

        # Pass the python executable to cmake so it can find an exact
        # match.
        cmake_args += ['-DVLLM_PYTHON_EXECUTABLE={}'.format(sys.executable)]

182
183
184
185
        # Pass the python path to cmake so it can reuse the build dependencies
        # on subsequent calls to python.
        cmake_args += ['-DVLLM_PYTHON_PATH={}'.format(":".join(sys.path))]

186
187
188
189
190
191
192
193
        # Override the base directory for FetchContent downloads to $ROOT/.deps
        # This allows sharing dependencies between profiles,
        # and plays more nicely with sccache.
        # To override this, set the FETCHCONTENT_BASE_DIR environment variable.
        fc_base_dir = os.path.join(ROOT_DIR, ".deps")
        fc_base_dir = os.environ.get("FETCHCONTENT_BASE_DIR", fc_base_dir)
        cmake_args += ['-DFETCHCONTENT_BASE_DIR={}'.format(fc_base_dir)]

bnellnm's avatar
bnellnm committed
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
        #
        # Setup parallelism and build tool
        #
        num_jobs, nvcc_threads = self.compute_num_jobs()

        if nvcc_threads:
            cmake_args += ['-DNVCC_THREADS={}'.format(nvcc_threads)]

        if is_ninja_available():
            build_tool = ['-G', 'Ninja']
            cmake_args += [
                '-DCMAKE_JOB_POOL_COMPILE:STRING=compile',
                '-DCMAKE_JOB_POOLS:STRING=compile={}'.format(num_jobs),
            ]
        else:
            # Default build tool to whatever cmake picks.
            build_tool = []
        subprocess.check_call(
            ['cmake', ext.cmake_lists_dir, *build_tool, *cmake_args],
            cwd=self.build_temp)

    def build_extensions(self) -> None:
        # Ensure that CMake is present and working
        try:
            subprocess.check_output(['cmake', '--version'])
        except OSError as e:
            raise RuntimeError('Cannot find CMake executable') from e

        # Create build directory if it does not exist.
        if not os.path.exists(self.build_temp):
            os.makedirs(self.build_temp)

226
        targets = []
227
228
229
230

        def target_name(s: str) -> str:
            return s.removeprefix("vllm.").removeprefix("vllm_flash_attn.")

bnellnm's avatar
bnellnm committed
231
232
233
        # Build all the extensions
        for ext in self.extensions:
            self.configure(ext)
234
            targets.append(target_name(ext.name))
bnellnm's avatar
bnellnm committed
235

236
        num_jobs, _ = self.compute_num_jobs()
bnellnm's avatar
bnellnm committed
237

238
239
240
241
242
243
        build_args = [
            "--build",
            ".",
            f"-j={num_jobs}",
            *[f"--target={name}" for name in targets],
        ]
bnellnm's avatar
bnellnm committed
244

245
        subprocess.check_call(["cmake", *build_args], cwd=self.build_temp)
bnellnm's avatar
bnellnm committed
246

247
248
249
250
251
252
253
254
255
256
257
        # Install the libraries
        for ext in self.extensions:
            # Install the extension into the proper location
            outdir = Path(self.get_ext_fullpath(ext.name)).parent.absolute()

            # Skip if the install directory is the same as the build directory
            if outdir == self.build_temp:
                continue

            # CMake appends the extension prefix to the install path,
            # and outdir already contains that prefix, so we need to remove it.
258
259
260
            # We assume only the final component of extension prefix is added by
            # CMake, this is currently true for current extensions but may not
            # always be the case.
261
            prefix = outdir
262
            if '.' in ext.name:
263
264
265
266
267
268
269
270
271
                prefix = prefix.parent

            # prefix here should actually be the same for all components
            install_args = [
                "cmake", "--install", ".", "--prefix", prefix, "--component",
                target_name(ext.name)
            ]
            subprocess.check_call(install_args, cwd=self.build_temp)

272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
    def run(self):
        # First, run the standard build_ext command to compile the extensions
        super().run()

        # copy vllm/vllm_flash_attn/*.py from self.build_lib to current
        # directory so that they can be included in the editable build
        import glob
        files = glob.glob(
            os.path.join(self.build_lib, "vllm", "vllm_flash_attn", "*.py"))
        for file in files:
            dst_file = os.path.join("vllm/vllm_flash_attn",
                                    os.path.basename(file))
            print(f"Copying {file} to {dst_file}")
            self.copy_file(file, dst_file)

287

288
289
290
class repackage_wheel(build_ext):
    """Extracts libraries and other files from an existing wheel."""

291
    def get_base_commit_in_main_branch(self) -> str:
292
293
294
        # Force to use the nightly wheel. This is mainly used for CI testing.
        if envs.VLLM_TEST_USE_PRECOMPILED_NIGHTLY_WHEEL:
            return "nightly"
295
296

        try:
297
298
299
300
301
302
303
            # Get the latest commit hash of the upstream main branch.
            resp_json = subprocess.check_output([
                "curl", "-s",
                "https://api.github.com/repos/vllm-project/vllm/commits/main"
            ]).decode("utf-8")
            upstream_main_commit = json.loads(resp_json)["sha"]

304
305
306
307
308
309
310
311
312
313
314
315
316
            # Check if the upstream_main_commit exists in the local repo
            try:
                subprocess.check_output(
                    ["git", "cat-file", "-e", f"{upstream_main_commit}"])
            except subprocess.CalledProcessError:
                # If not present, fetch it from the remote repository.
                # Note that this does not update any local branches,
                # but ensures that this commit ref and its history are
                # available in our local repo.
                subprocess.check_call([
                    "git", "fetch", "https://github.com/vllm-project/vllm",
                    "main"
                ])
317
318
319

            # Then get the commit hash of the current branch that is the same as
            # the upstream main commit.
320
321
322
            current_branch = subprocess.check_output(
                ["git", "branch", "--show-current"]).decode("utf-8").strip()

323
324
325
            base_commit = subprocess.check_output([
                "git", "merge-base", f"{upstream_main_commit}", current_branch
            ]).decode("utf-8").strip()
326
            return base_commit
327
328
        except ValueError as err:
            raise ValueError(err) from None
329
330
331
332
333
334
        except Exception as err:
            logger.warning(
                "Failed to get the base commit in the main branch. "
                "Using the nightly wheel. The libraries in this "
                "wheel may not be compatible with your dev branch: %s", err)
            return "nightly"
335

336
    def run(self) -> None:
337
338
339
        assert _is_cuda(
        ), "VLLM_USE_PRECOMPILED is only supported for CUDA builds"

340
341
342
343
        wheel_location = os.getenv("VLLM_PRECOMPILED_WHEEL_LOCATION", None)
        if wheel_location is None:
            base_commit = self.get_base_commit_in_main_branch()
            wheel_location = f"https://wheels.vllm.ai/{base_commit}/vllm-1.0.0.dev-cp38-abi3-manylinux1_x86_64.whl"
344
345
346
347
            # Fallback to nightly wheel if latest commit wheel is unavailable,
            # in this rare case, the nightly release CI hasn't finished on main.
            if not is_url_available(wheel_location):
                wheel_location = "https://wheels.vllm.ai/nightly/vllm-1.0.0.dev-cp38-abi3-manylinux1_x86_64.whl"
348

349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
        import zipfile

        if os.path.isfile(wheel_location):
            wheel_path = wheel_location
            print(f"Using existing wheel={wheel_path}")
        else:
            # Download the wheel from a given URL, assume
            # the filename is the last part of the URL
            wheel_filename = wheel_location.split("/")[-1]

            import tempfile

            # create a temporary directory to store the wheel
            temp_dir = tempfile.mkdtemp(prefix="vllm-wheels")
            wheel_path = os.path.join(temp_dir, wheel_filename)

            print(f"Downloading wheel from {wheel_location} to {wheel_path}")

            from urllib.request import urlretrieve

            try:
                urlretrieve(wheel_location, filename=wheel_path)
            except Exception as e:
                from setuptools.errors import SetupError

                raise SetupError(
                    f"Failed to get vLLM wheel from {wheel_location}") from e

        with zipfile.ZipFile(wheel_path) as wheel:
            files_to_copy = [
                "vllm/_C.abi3.so",
                "vllm/_moe_C.abi3.so",
381
                "vllm/_flashmla_C.abi3.so",
382
383
                "vllm/vllm_flash_attn/_vllm_fa2_C.abi3.so",
                "vllm/vllm_flash_attn/_vllm_fa3_C.abi3.so",
384
385
                "vllm/vllm_flash_attn/flash_attn_interface.py",
                "vllm/vllm_flash_attn/__init__.py",
386
                "vllm/cumem_allocator.abi3.so",
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
                # "vllm/_version.py", # not available in nightly wheels yet
            ]
            file_members = filter(lambda x: x.filename in files_to_copy,
                                  wheel.filelist)

            for file in file_members:
                print(f"Extracting and including {file.filename} "
                      "from existing wheel")
                package_name = os.path.dirname(file.filename).replace("/", ".")
                file_name = os.path.basename(file.filename)

                if package_name not in package_data:
                    package_data[package_name] = []

                wheel.extract(file)
                if file_name.endswith(".py"):
                    # python files shouldn't be added to package_data
                    continue

                package_data[package_name].append(file_name)


409
def _is_hpu() -> bool:
410
411
412
413
414
415
416
    # if VLLM_TARGET_DEVICE env var was set explicitly, skip HPU autodetection
    if os.getenv("VLLM_TARGET_DEVICE", None) == VLLM_TARGET_DEVICE:
        return VLLM_TARGET_DEVICE == "hpu"

    # if VLLM_TARGET_DEVICE was not set explicitly, check if hl-smi succeeds,
    # and if it doesn't, check if habanalabs driver is loaded
    is_hpu_available = False
417
    try:
418
419
        out = subprocess.run(["hl-smi"], capture_output=True, check=True)
        is_hpu_available = out.returncode == 0
420
    except (FileNotFoundError, PermissionError, subprocess.CalledProcessError):
421
        if sys.platform.startswith("linux"):
422
423
424
425
426
427
            try:
                output = subprocess.check_output(
                    'lsmod | grep habanalabs | wc -l', shell=True)
                is_hpu_available = int(output) > 0
            except (ValueError, FileNotFoundError, PermissionError,
                    subprocess.CalledProcessError):
428
429
                pass
    return is_hpu_available
430
431


432
433
434
435
def _no_device() -> bool:
    return VLLM_TARGET_DEVICE == "empty"


436
def _is_cuda() -> bool:
437
438
    has_cuda = torch.version.cuda is not None
    return (VLLM_TARGET_DEVICE == "cuda" and has_cuda
439
            and not (_is_neuron() or _is_tpu() or _is_hpu()))
440
441
442


def _is_hip() -> bool:
443
444
    return (VLLM_TARGET_DEVICE == "cuda"
            or VLLM_TARGET_DEVICE == "rocm") and torch.version.hip is not None
445
446


447
def _is_neuron() -> bool:
448
    return VLLM_TARGET_DEVICE == "neuron"
449
450


451
452
453
454
def _is_tpu() -> bool:
    return VLLM_TARGET_DEVICE == "tpu"


455
456
def _is_cpu() -> bool:
    return VLLM_TARGET_DEVICE == "cpu"
457
458


459
460
461
462
def _is_xpu() -> bool:
    return VLLM_TARGET_DEVICE == "xpu"


463
464
465
466
def _build_custom_ops() -> bool:
    return _is_cuda() or _is_hip() or _is_cpu()


467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
def get_rocm_version():
    # Get the Rocm version from the ROCM_HOME/bin/librocm-core.so
    # see https://github.com/ROCm/rocm-core/blob/d11f5c20d500f729c393680a01fa902ebf92094b/rocm_version.cpp#L21
    try:
        librocm_core_file = Path(ROCM_HOME) / "lib" / "librocm-core.so"
        if not librocm_core_file.is_file():
            return None
        librocm_core = ctypes.CDLL(librocm_core_file)
        VerErrors = ctypes.c_uint32
        get_rocm_core_version = librocm_core.getROCmVersion
        get_rocm_core_version.restype = VerErrors
        get_rocm_core_version.argtypes = [
            ctypes.POINTER(ctypes.c_uint32),
            ctypes.POINTER(ctypes.c_uint32),
            ctypes.POINTER(ctypes.c_uint32),
        ]
        major = ctypes.c_uint32()
        minor = ctypes.c_uint32()
        patch = ctypes.c_uint32()
486

487
488
        if (get_rocm_core_version(ctypes.byref(major), ctypes.byref(minor),
                                  ctypes.byref(patch)) == 0):
489
            return f"{major.value}.{minor.value}.{patch.value}"
490
        return None
491
    except Exception:
492
        return None
Woosuk Kwon's avatar
Woosuk Kwon committed
493

494

495
496
497
def get_neuronxcc_version():
    import sysconfig
    site_dir = sysconfig.get_paths()["purelib"]
498
499
    version_file = os.path.join(site_dir, "neuronxcc", "version",
                                "__init__.py")
500
501

    # Check if the command was executed successfully
502
    with open(version_file) as fp:
503
504
505
506
507
508
509
510
        content = fp.read()

    # Extract the version using a regular expression
    match = re.search(r"__version__ = '(\S+)'", content)
    if match:
        # Return the version string
        return match.group(1)
    else:
511
        raise RuntimeError("Could not find Neuron version in the output")
512
513


bnellnm's avatar
bnellnm committed
514
def get_nvcc_cuda_version() -> Version:
515
516
517
518
    """Get the CUDA version from nvcc.

    Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py
    """
519
    assert CUDA_HOME is not None, "CUDA_HOME is not set"
bnellnm's avatar
bnellnm committed
520
    nvcc_output = subprocess.check_output([CUDA_HOME + "/bin/nvcc", "-V"],
521
522
523
524
525
526
527
                                          universal_newlines=True)
    output = nvcc_output.split()
    release_idx = output.index("release") + 1
    nvcc_cuda_version = parse(output[release_idx].split(",")[0])
    return nvcc_cuda_version


zhuwenwen's avatar
zhuwenwen committed
528
529
530
531
532
533
def get_sha(root: Union[str, Path]) -> str:
    try:
        return subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=root).decode('ascii').strip()
    except Exception:
        return 'Unknown'

534

zhuwenwen's avatar
zhuwenwen committed
535
def get_version_add(sha: Optional[str] = None) -> str:
zhuwenwen's avatar
zhuwenwen committed
536
537
    command = "git config --global --add safe.directory "+pwd  
    result = subprocess.run(command, shell=True, capture_output=False, text=True) 
zhuwenwen's avatar
zhuwenwen committed
538
539
    vllm_root = os.path.dirname(os.path.abspath(__file__))
    add_version_path = os.path.join(os.path.join(vllm_root, "vllm"), "version.py")
zhuwenwen's avatar
zhuwenwen committed
540
    major, minor, _ = torch.__version__.split('.')
541
542
543
544
    if add_git_version:
        if sha != 'Unknown':
            if sha is None:
                sha = get_sha(vllm_root)
zhuwenwen's avatar
zhuwenwen committed
545
            if (major, minor) == ('2', '4'):
546
                version = 'das.opt1.' + sha[:7]
547
    else:
zhuwenwen's avatar
zhuwenwen committed
548
        if (major, minor) == ('2', '4'):
549
            version = 'das.opt1'
550

zhuwenwen's avatar
zhuwenwen committed
551
552
553
554
555
556
557

    # dtk version
    if os.getenv("ROCM_PATH"):
        rocm_path = os.getenv('ROCM_PATH', "")
        rocm_version_path = os.path.join(rocm_path, '.info', "rocm_version")
        with open(rocm_version_path, 'r',encoding='utf-8') as file:
            lines = file.readlines()
zhuwenwen's avatar
zhuwenwen committed
558
        rocm_version=lines[0].replace(".", "")
zhuwenwen's avatar
zhuwenwen committed
559
        version += ".dtk" + rocm_version
zhuwenwen's avatar
zhuwenwen committed
560
    
561
    new_version_content = f"""
562
try:
zhuwenwen's avatar
zhuwenwen committed
563
564
565
    __version__ = "0.8.2"
    __version_tuple__ = (0, 8, 2)
    __hcu_version__ = f'0.8.2+{version}' 
zhuwenwen's avatar
zhuwenwen committed
566
    
zhuwenwen's avatar
zhuwenwen committed
567
    from vllm.version import __version__, __version_tuple__, __hcu_version__
568
except Exception as e:
zhuwenwen's avatar
zhuwenwen committed
569
570
    import warnings

zhuwenwen's avatar
zhuwenwen committed
571
    warnings.warn(f"Failed to read commit hash:\\n + str(e)",
572
573
                  RuntimeWarning,
                  stacklevel=2)
574
575
    __version__ = "dev"
    __version_tuple__ = (0, 0, __version__)
576
"""
zhuwenwen's avatar
zhuwenwen committed
577
    
zhuwenwen's avatar
zhuwenwen committed
578
    with open(add_version_path, encoding="utf-8",mode="w") as file:
zhuwenwen's avatar
zhuwenwen committed
579
        file.write(new_version_content)
zhuwenwen's avatar
zhuwenwen committed
580
581
582
583
584
585
586
587
    file.close()
    
    
def get_version():
    get_version_add()
    version_file = 'vllm/version.py'
    with open(version_file, encoding='utf-8') as f:
        exec(compile(f.read(), version_file, 'exec'))
zhuwenwen's avatar
zhuwenwen committed
588
    return locals()['__hcu_version__']
zhuwenwen's avatar
zhuwenwen committed
589
590


591
592
593
594
595
596
597
598
def get_gaudi_sw_version():
    """
    Returns the driver version.
    """
    # Enable console printing for `hl-smi` check
    output = subprocess.run("hl-smi",
                            shell=True,
                            text=True,
599
                            capture_output=True,
600
601
602
603
604
605
606
                            env={"ENABLE_CONSOLE": "true"})
    if output.returncode == 0 and output.stdout:
        return output.stdout.split("\n")[2].replace(
            " ", "").split(":")[1][:-1].split("-")[0]
    return "0.0.0"  # when hl-smi is not available


607
def get_vllm_version() -> str:
608
609
610
    # from setuptools_scm import get_version
    # version = get_version(write_to="vllm/_version.py")
    # sep = "+" if "+" not in version else "."  # dev versions might contain +
611

612
    if _no_device():
613
        if envs.VLLM_TARGET_DEVICE == "empty":
614
            version += f"{sep}empty"
615
    elif _is_cuda():
616
        if envs.VLLM_USE_PRECOMPILED:
617
            version += f"{sep}precompiled"
618
619
620
621
622
623
624
        else:
            cuda_version = str(get_nvcc_cuda_version())
            if cuda_version != MAIN_CUDA_VERSION:
                cuda_version_str = cuda_version.replace(".", "")[:3]
                # skip this for source tarball, required for pypi
                if "sdist" not in sys.argv:
                    version += f"{sep}cu{cuda_version_str}"
625
    elif _is_hip():
626
        # Get the Rocm Version
zhuwenwen's avatar
zhuwenwen committed
627
628
629
        # rocm_version = get_rocm_version() or torch.version.hip
        # if rocm_version and rocm_version != MAIN_CUDA_VERSION:
        #     version += f"{sep}rocm{rocm_version.replace('.', '')[:3]}"
zhuwenwen's avatar
zhuwenwen committed
630
        version = get_version()
631
632
    elif _is_neuron():
        # Get the Neuron version
bnellnm's avatar
bnellnm committed
633
        neuron_version = str(get_neuronxcc_version())
634
635
        if neuron_version != MAIN_CUDA_VERSION:
            neuron_version_str = neuron_version.replace(".", "")[:3]
636
            version += f"{sep}neuron{neuron_version_str}"
637
638
639
640
641
642
    elif _is_hpu():
        # Get the Intel Gaudi Software Suite version
        gaudi_sw_version = str(get_gaudi_sw_version())
        if gaudi_sw_version != MAIN_CUDA_VERSION:
            gaudi_sw_version = gaudi_sw_version.replace(".", "")[:3]
            version += f"{sep}gaudi{gaudi_sw_version}"
643
    elif _is_tpu():
644
        version += f"{sep}tpu"
645
    elif _is_cpu():
646
647
        if envs.VLLM_TARGET_DEVICE == "cpu":
            version += f"{sep}cpu"
648
    elif _is_xpu():
649
        version += f"{sep}xpu"
650
    else:
651
        raise RuntimeError("Unknown runtime environment")
652

653
654
655
    return version


656
def get_requirements() -> list[str]:
657
    """Get Python package dependencies from requirements.txt."""
658
    requirements_dir = ROOT_DIR / "requirements"
659

660
    def _read_requirements(filename: str) -> list[str]:
661
        with open(requirements_dir / filename) as f:
662
            requirements = f.read().strip().split("\n")
663
664
665
666
        resolved_requirements = []
        for line in requirements:
            if line.startswith("-r "):
                resolved_requirements += _read_requirements(line.split()[1])
667
668
            elif line.startswith("--"):
                continue
669
670
671
672
            else:
                resolved_requirements.append(line)
        return resolved_requirements

673
    if _no_device():
674
        requirements = _read_requirements("common.txt")
675
    elif _is_cuda():
676
        requirements = _read_requirements("cuda.txt")
677
        cuda_major, cuda_minor = torch.version.cuda.split(".")
678
679
        modified_requirements = []
        for req in requirements:
680
681
            if ("vllm-flash-attn" in req and cuda_major != "12"):
                # vllm-flash-attn is built only for CUDA 12.x.
682
683
684
                # Skip for other versions.
                continue
            modified_requirements.append(req)
685
        requirements = modified_requirements
686
    elif _is_hip():
687
        requirements = _read_requirements("rocm.txt")
688
    elif _is_neuron():
689
        requirements = _read_requirements("neuron.txt")
690
    elif _is_hpu():
691
        requirements = _read_requirements("hpu.txt")
692
    elif _is_tpu():
693
        requirements = _read_requirements("tpu.txt")
694
    elif _is_cpu():
695
        requirements = _read_requirements("cpu.txt")
696
    elif _is_xpu():
697
        requirements = _read_requirements("xpu.txt")
698
    else:
699
        raise ValueError(
700
            "Unsupported platform, please use CUDA, ROCm, Neuron, HPU, "
701
            "or CPU.")
702
703
704
    return requirements


bnellnm's avatar
bnellnm committed
705
706
ext_modules = []

707
if _is_cuda() or _is_hip():
bnellnm's avatar
bnellnm committed
708
709
    ext_modules.append(CMakeExtension(name="vllm._moe_C"))

710
711
# if _is_hip():
#     ext_modules.append(CMakeExtension(name="vllm._rocm_C"))
712

713
if _is_cuda():
714
    ext_modules.append(CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa2_C"))
715
716
717
718
    if envs.VLLM_USE_PRECOMPILED or get_nvcc_cuda_version() >= Version("12.0"):
        # FA3 requires CUDA 12.0 or later
        ext_modules.append(
            CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa3_C"))
719
720
721
722
723
    if envs.VLLM_USE_PRECOMPILED or get_nvcc_cuda_version() >= Version("12.3"):
        # Optional since this doesn't get built (produce an .so file) when
        # not targeting a hopper system
        ext_modules.append(
            CMakeExtension(name="vllm._flashmla_C", optional=True))
724
    ext_modules.append(CMakeExtension(name="vllm.cumem_allocator"))
725

726
if _build_custom_ops():
bnellnm's avatar
bnellnm committed
727
728
    ext_modules.append(CMakeExtension(name="vllm._C"))

729
package_data = {
730
731
732
733
    "vllm": [
        "py.typed",
        "model_executor/layers/fused_moe/configs/*.json",
        "model_executor/layers/quantization/utils/configs/*.json",
zhuwenwen's avatar
zhuwenwen committed
734
        "benchmarks/*.py",
735
        "attention/backends/configs/*.json",
736
        "model_executor/layers/quantization/configs/awq/*.json"
737
    ]
738
}
Simon Mo's avatar
Simon Mo committed
739

740
741
742
if _no_device():
    ext_modules = []

743
744
745
746
747
748
749
750
if not ext_modules:
    cmdclass = {}
else:
    cmdclass = {
        "build_ext":
        repackage_wheel if envs.VLLM_USE_PRECOMPILED else cmake_build_ext
    }

bnellnm's avatar
bnellnm committed
751
setup(
752
    # static metadata should rather go in pyproject.toml
753
    version=get_vllm_version(),
Woosuk Kwon's avatar
Woosuk Kwon committed
754
    ext_modules=ext_modules,
755
    install_requires=get_requirements(),
756
    extras_require={
757
        "tensorizer": ["tensorizer>=2.9.0"],
758
        "fastsafetensors": ["fastsafetensors >= 0.1.10"],
759
        "runai": ["runai-model-streamer", "runai-model-streamer-s3", "boto3"],
760
761
        "audio": ["librosa", "soundfile"],  # Required for audio processing
        "video": ["decord"]  # Required for video processing
762
    },
763
    cmdclass=cmdclass,
Simon Mo's avatar
Simon Mo committed
764
    package_data=package_data,
Woosuk Kwon's avatar
Woosuk Kwon committed
765
)