setup.py 33.5 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import ctypes
5
import importlib.util
6
import json
7
import logging
8
9
import os
import re
10
import shutil
11
import subprocess
bnellnm's avatar
bnellnm committed
12
import sys
13
import sysconfig
14
from pathlib import Path
15
from shutil import which
16

Woosuk Kwon's avatar
Woosuk Kwon committed
17
import torch
18
from packaging.version import Version, parse
19
from setuptools import Extension, setup
20
from setuptools.command.build_ext import build_ext
21
# from setuptools_scm import get_version
22
from torch.utils.cpp_extension import CUDA_HOME, ROCM_HOME
23

zhuwenwen's avatar
zhuwenwen committed
24
25
26
27
from typing import Optional, Union
import subprocess
from pathlib import Path

zhuwenwen's avatar
zhuwenwen committed
28
pwd = os.path.dirname(os.path.abspath(__file__))
gaoqiong's avatar
gaoqiong committed
29
add_git_version = False
30
31
if int(os.environ.get('ADD_GIT_VERSION', '0')) == 1:
    add_git_version = True
zhuwenwen's avatar
zhuwenwen committed
32
33
34
35
    
skip_vllm_build = False
if int(os.environ.get('SKIP_VLLM_BUILD', '0')) == 1:
    skip_vllm_build = True
36
37
38
39
40
41
42
43
44

def load_module_from_path(module_name, path):
    spec = importlib.util.spec_from_file_location(module_name, path)
    module = importlib.util.module_from_spec(spec)
    sys.modules[module_name] = module
    spec.loader.exec_module(module)
    return module


45
ROOT_DIR = Path(__file__).parent
46
logger = logging.getLogger(__name__)
47
48
49

# cannot import envs directly because it depends on vllm,
#  which is not installed yet
50
envs = load_module_from_path("envs", os.path.join(ROOT_DIR, "vllm", "envs.py"))
51
52

VLLM_TARGET_DEVICE = envs.VLLM_TARGET_DEVICE
53

54
if sys.platform.startswith("darwin") and VLLM_TARGET_DEVICE != "cpu":
55
    logger.warning("VLLM_TARGET_DEVICE automatically set to `cpu` due to macOS")
56
    VLLM_TARGET_DEVICE = "cpu"
57
elif not (sys.platform.startswith("linux") or sys.platform.startswith("darwin")):
58
59
    logger.warning(
        "vLLM only supports Linux platform (including WSL) and MacOS."
60
        "Building on %s, "
61
62
63
        "so vLLM may not be able to run correctly",
        sys.platform,
    )
64
    VLLM_TARGET_DEVICE = "empty"
65
66
67
68
69
70
elif (
    sys.platform.startswith("linux")
    and torch.version.cuda is None
    and os.getenv("VLLM_TARGET_DEVICE") is None
    and torch.version.hip is None
):
71
    # if cuda or hip is not available and VLLM_TARGET_DEVICE is not set,
72
73
    # fallback to cpu
    VLLM_TARGET_DEVICE = "cpu"
74

bnellnm's avatar
bnellnm committed
75
76

def is_sccache_available() -> bool:
77
78
79
    return which("sccache") is not None and not bool(
        int(os.getenv("VLLM_DISABLE_SCCACHE", "0"))
    )
bnellnm's avatar
bnellnm committed
80
81
82
83
84
85
86
87
88
89


def is_ccache_available() -> bool:
    return which("ccache") is not None


def is_ninja_available() -> bool:
    return which("ninja") is not None


90
91
def is_freethreaded():
    return bool(sysconfig.get_config_var("Py_GIL_DISABLED"))
92
93


bnellnm's avatar
bnellnm committed
94
class CMakeExtension(Extension):
95
    def __init__(self, name: str, cmake_lists_dir: str = ".", **kwa) -> None:
96
        super().__init__(name, sources=[], py_limited_api=not is_freethreaded(), **kwa)
bnellnm's avatar
bnellnm committed
97
98
99
100
101
        self.cmake_lists_dir = os.path.abspath(cmake_lists_dir)


class cmake_build_ext(build_ext):
    # A dict of extension directories that have been configured.
102
    did_config: dict[str, bool] = {}
bnellnm's avatar
bnellnm committed
103
104
105
106
107

    #
    # Determine number of compilation jobs and optionally nvcc compile threads.
    #
    def compute_num_jobs(self):
108
109
        # `num_jobs` is either the value of the MAX_JOBS environment variable
        # (if defined) or the number of CPUs available.
110
        num_jobs = envs.MAX_JOBS
111
112
        if num_jobs is not None:
            num_jobs = int(num_jobs)
113
            logger.info("Using MAX_JOBS=%d as the number of jobs.", num_jobs)
114
115
116
117
118
119
120
        else:
            try:
                # os.sched_getaffinity() isn't universally available, so fall
                #  back to os.cpu_count() if we get an error here.
                num_jobs = len(os.sched_getaffinity(0))
            except AttributeError:
                num_jobs = os.cpu_count()
bnellnm's avatar
bnellnm committed
121

122
        nvcc_threads = None
123
124
125
126
127
        if _is_cuda() and get_nvcc_cuda_version() >= Version("11.2"):
            # `nvcc_threads` is either the value of the NVCC_THREADS
            # environment variable (if defined) or 1.
            # when it is set, we reduce `num_jobs` to avoid
            # overloading the system.
128
            nvcc_threads = envs.NVCC_THREADS
129
130
            if nvcc_threads is not None:
                nvcc_threads = int(nvcc_threads)
131
                logger.info(
132
133
                    "Using NVCC_THREADS=%d as the number of nvcc threads.", nvcc_threads
                )
134
135
136
            else:
                nvcc_threads = 1
            num_jobs = max(1, num_jobs // nvcc_threads)
bnellnm's avatar
bnellnm committed
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153

        return num_jobs, nvcc_threads

    #
    # Perform cmake configuration for a single extension.
    #
    def configure(self, ext: CMakeExtension) -> None:
        # If we've already configured using the CMakeLists.txt for
        # this extension, exit early.
        if ext.cmake_lists_dir in cmake_build_ext.did_config:
            return

        cmake_build_ext.did_config[ext.cmake_lists_dir] = True

        # Select the build type.
        # Note: optimization level + debug info are set by the build type
        default_cfg = "Debug" if self.debug else "RelWithDebInfo"
154
        cfg = envs.CMAKE_BUILD_TYPE or default_cfg
bnellnm's avatar
bnellnm committed
155
156

        cmake_args = [
157
158
            "-DCMAKE_BUILD_TYPE={}".format(cfg),
            "-DVLLM_TARGET_DEVICE={}".format(VLLM_TARGET_DEVICE),
bnellnm's avatar
bnellnm committed
159
160
        ]

161
        verbose = envs.VERBOSE
bnellnm's avatar
bnellnm committed
162
        if verbose:
163
            cmake_args += ["-DCMAKE_VERBOSE_MAKEFILE=ON"]
bnellnm's avatar
bnellnm committed
164
165
166

        if is_sccache_available():
            cmake_args += [
167
168
169
170
                "-DCMAKE_C_COMPILER_LAUNCHER=sccache",
                "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache",
                "-DCMAKE_CUDA_COMPILER_LAUNCHER=sccache",
                "-DCMAKE_HIP_COMPILER_LAUNCHER=sccache",
bnellnm's avatar
bnellnm committed
171
172
173
            ]
        elif is_ccache_available():
            cmake_args += [
174
175
176
177
                "-DCMAKE_C_COMPILER_LAUNCHER=ccache",
                "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache",
                "-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache",
                "-DCMAKE_HIP_COMPILER_LAUNCHER=ccache",
bnellnm's avatar
bnellnm committed
178
179
180
181
            ]

        # Pass the python executable to cmake so it can find an exact
        # match.
182
        cmake_args += ["-DVLLM_PYTHON_EXECUTABLE={}".format(sys.executable)]
bnellnm's avatar
bnellnm committed
183

184
185
        # Pass the python path to cmake so it can reuse the build dependencies
        # on subsequent calls to python.
186
        cmake_args += ["-DVLLM_PYTHON_PATH={}".format(":".join(sys.path))]
187

188
189
190
191
192
193
        # Override the base directory for FetchContent downloads to $ROOT/.deps
        # This allows sharing dependencies between profiles,
        # and plays more nicely with sccache.
        # To override this, set the FETCHCONTENT_BASE_DIR environment variable.
        fc_base_dir = os.path.join(ROOT_DIR, ".deps")
        fc_base_dir = os.environ.get("FETCHCONTENT_BASE_DIR", fc_base_dir)
194
        cmake_args += ["-DFETCHCONTENT_BASE_DIR={}".format(fc_base_dir)]
195

bnellnm's avatar
bnellnm committed
196
197
198
199
200
201
        #
        # Setup parallelism and build tool
        #
        num_jobs, nvcc_threads = self.compute_num_jobs()

        if nvcc_threads:
202
            cmake_args += ["-DNVCC_THREADS={}".format(nvcc_threads)]
bnellnm's avatar
bnellnm committed
203
204

        if is_ninja_available():
205
            build_tool = ["-G", "Ninja"]
bnellnm's avatar
bnellnm committed
206
            cmake_args += [
207
208
                "-DCMAKE_JOB_POOL_COMPILE:STRING=compile",
                "-DCMAKE_JOB_POOLS:STRING=compile={}".format(num_jobs),
bnellnm's avatar
bnellnm committed
209
210
211
212
            ]
        else:
            # Default build tool to whatever cmake picks.
            build_tool = []
213
214
        # Make sure we use the nvcc from CUDA_HOME
        if _is_cuda():
215
            cmake_args += [f"-DCMAKE_CUDA_COMPILER={CUDA_HOME}/bin/nvcc"]
216
217
        elif _is_hip():
            cmake_args += [f"-DROCM_PATH={ROCM_HOME}"]
218
219
220
221
222

        other_cmake_args = os.environ.get("CMAKE_ARGS")
        if other_cmake_args:
            cmake_args += other_cmake_args.split()

bnellnm's avatar
bnellnm committed
223
        subprocess.check_call(
224
225
226
            ["cmake", ext.cmake_lists_dir, *build_tool, *cmake_args],
            cwd=self.build_temp,
        )
bnellnm's avatar
bnellnm committed
227
228
229
230

    def build_extensions(self) -> None:
        # Ensure that CMake is present and working
        try:
231
            subprocess.check_output(["cmake", "--version"])
bnellnm's avatar
bnellnm committed
232
        except OSError as e:
233
            raise RuntimeError("Cannot find CMake executable") from e
bnellnm's avatar
bnellnm committed
234
235
236
237
238

        # Create build directory if it does not exist.
        if not os.path.exists(self.build_temp):
            os.makedirs(self.build_temp)

239
        targets = []
240
241
242
243

        def target_name(s: str) -> str:
            return s.removeprefix("vllm.").removeprefix("vllm_flash_attn.")

bnellnm's avatar
bnellnm committed
244
245
246
        # Build all the extensions
        for ext in self.extensions:
            self.configure(ext)
247
            targets.append(target_name(ext.name))
bnellnm's avatar
bnellnm committed
248

249
        num_jobs, _ = self.compute_num_jobs()
bnellnm's avatar
bnellnm committed
250

251
252
253
254
255
256
        build_args = [
            "--build",
            ".",
            f"-j={num_jobs}",
            *[f"--target={name}" for name in targets],
        ]
bnellnm's avatar
bnellnm committed
257

258
        subprocess.check_call(["cmake", *build_args], cwd=self.build_temp)
bnellnm's avatar
bnellnm committed
259

260
261
262
263
264
265
266
267
268
269
270
271
        # Install the libraries
        for ext in self.extensions:
            # Install the extension into the proper location
            outdir = Path(self.get_ext_fullpath(ext.name)).parent.absolute()

            # Skip if the install directory is the same as the build directory
            if outdir == self.build_temp:
                continue

            # CMake appends the extension prefix to the install path,
            # and outdir already contains that prefix, so we need to remove it.
            prefix = outdir
272
            for _ in range(ext.name.count(".")):
273
274
275
276
                prefix = prefix.parent

            # prefix here should actually be the same for all components
            install_args = [
277
278
279
280
281
282
283
                "cmake",
                "--install",
                ".",
                "--prefix",
                prefix,
                "--component",
                target_name(ext.name),
284
285
286
            ]
            subprocess.check_call(install_args, cwd=self.build_temp)

287
288
289
290
    def run(self):
        # First, run the standard build_ext command to compile the extensions
        super().run()

291
        # copy vllm/vllm_flash_attn/**/*.py from self.build_lib to current
292
293
        # directory so that they can be included in the editable build
        import glob
294
295
296
297
298

        files = glob.glob(
            os.path.join(self.build_lib, "vllm", "vllm_flash_attn", "**", "*.py"),
            recursive=True,
        )
299
        for file in files:
300
301
302
            dst_file = os.path.join(
                "vllm/vllm_flash_attn", file.split("vllm/vllm_flash_attn/")[-1]
            )
303
            print(f"Copying {file} to {dst_file}")
304
            os.makedirs(os.path.dirname(dst_file), exist_ok=True)
305
306
            self.copy_file(file, dst_file)

307
        if _is_cuda():
308
309
310
311
312
313
314
315
316
317
318
319
320
            # copy vllm/third_party/triton_kernels/**/*.py from self.build_lib
            # to current directory so that they can be included in the editable
            # build
            print(
                f"Copying {self.build_lib}/vllm/third_party/triton_kernels "
                "to vllm/third_party/triton_kernels"
            )
            shutil.copytree(
                f"{self.build_lib}/vllm/third_party/triton_kernels",
                "vllm/third_party/triton_kernels",
                dirs_exist_ok=True,
            )

321

322
323
324
325
class precompiled_build_ext(build_ext):
    """Disables extension building when using precompiled binaries."""

    def run(self) -> None:
326
        return
327
328
329
330
331
332
333

    def build_extensions(self) -> None:
        print("Skipping build_ext: using precompiled extensions.")
        return


class precompiled_wheel_utils:
334
335
    """Extracts libraries and other files from an existing wheel."""

336
    @staticmethod
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
    def fetch_metadata_for_variant(
        commit: str, variant: str | None
    ) -> tuple[list[dict], str]:
        """
        Fetches metadata for a specific variant of the precompiled wheel.
        """
        variant_dir = f"{variant}/" if variant is not None else ""
        repo_url = f"https://wheels.vllm.ai/{commit}/{variant_dir}vllm/"
        meta_url = repo_url + "metadata.json"
        print(f"Trying to fetch nightly build metadata from {meta_url}")
        from urllib.request import urlopen

        with urlopen(meta_url) as resp:
            # urlopen raises HTTPError on unexpected status code
            wheels = json.loads(resp.read().decode("utf-8"))
        return wheels, repo_url

    @staticmethod
    def determine_wheel_url() -> tuple[str, str | None]:
        """
        Try to determine the precompiled wheel URL or path to use.
        The order of preference is:
        1. user-specified wheel location (can be either local or remote, via
           VLLM_PRECOMPILED_WHEEL_LOCATION)
        2. user-specified variant from nightly repo (current main commit via
           VLLM_PRECOMPILED_WHEEL_VARIANT)
        3. the variant corresponding to VLLM_MAIN_CUDA_VERSION from nightly repo
        4. the default variant from nightly repo (current main commit)
        """
        wheel_location = os.getenv("VLLM_PRECOMPILED_WHEEL_LOCATION", None)
        if wheel_location is not None:
            print(f"Using user-specified precompiled wheel location: {wheel_location}")
            return wheel_location, None
        else:
            import platform

            arch = platform.machine()
            # try to fetch the wheel metadata from the nightly wheel repo
            main_variant = "cu" + envs.VLLM_MAIN_CUDA_VERSION.replace(".", "")
            variant = os.getenv("VLLM_PRECOMPILED_WHEEL_VARIANT", main_variant)
            commit = os.getenv(
                "VLLM_PRECOMPILED_WHEEL_COMMIT",
                precompiled_wheel_utils.get_base_commit_in_main_branch(),
            )
            print(f"Using precompiled wheel commit {commit} with variant {variant}")
            try_default = False
            wheels, repo_url, download_filename = None, None, None
            try:
                wheels, repo_url = precompiled_wheel_utils.fetch_metadata_for_variant(
                    commit, variant
                )
            except Exception as e:
                logger.warning(
                    "Failed to fetch precompiled wheel metadata for variant %s: %s",
                    variant,
                    e,
                )
                try_default = True  # try outside handler to keep the stacktrace simple
            if try_default:
                print("Trying the default variant from remote")
                wheels, repo_url = precompiled_wheel_utils.fetch_metadata_for_variant(
                    commit, None
                )
                # if this also fails, then we have nothing more to try / cache
            assert wheels is not None and repo_url is not None, (
                "Failed to fetch precompiled wheel metadata"
            )
            # The metadata.json has the following format:
            # see .buildkite/scripts/generate-nightly-index.py for details
            """[{
    "package_name": "vllm",
    "version": "0.11.2.dev278+gdbc3d9991",
    "build_tag": null,
    "python_tag": "cp38",
    "abi_tag": "abi3",
    "platform_tag": "manylinux1_x86_64",
    "variant": null,
    "filename": "vllm-0.11.2.dev278+gdbc3d9991-cp38-abi3-manylinux1_x86_64.whl",
    "path": "../vllm-0.11.2.dev278%2Bgdbc3d9991-cp38-abi3-manylinux1_x86_64.whl"
    },
    ...]"""
            from urllib.parse import urljoin

            for wheel in wheels:
                # TODO: maybe check more compatibility later? (python_tag, abi_tag, etc)
                if wheel.get("package_name") == "vllm" and arch in wheel.get(
                    "platform_tag", ""
                ):
                    print(f"Found precompiled wheel metadata: {wheel}")
                    if "path" not in wheel:
                        raise ValueError(f"Wheel metadata missing path: {wheel}")
                    wheel_url = urljoin(repo_url, wheel["path"])
                    download_filename = wheel.get("filename")
                    print(f"Using precompiled wheel URL: {wheel_url}")
                    break
            else:
                raise ValueError(
                    f"No precompiled vllm wheel found for architecture {arch} "
                    f"from repo {repo_url}. All available wheels: {wheels}"
                )

        return wheel_url, download_filename

    @staticmethod
    def extract_precompiled_and_patch_package(
        wheel_url_or_path: str, download_filename: str | None
    ) -> dict:
444
445
446
447
448
449
        import tempfile
        import zipfile

        temp_dir = None
        try:
            if not os.path.isfile(wheel_url_or_path):
450
451
                # use provided filename first, then derive from URL
                wheel_filename = download_filename or wheel_url_or_path.split("/")[-1]
452
453
                temp_dir = tempfile.mkdtemp(prefix="vllm-wheels")
                wheel_path = os.path.join(temp_dir, wheel_filename)
454
                print(f"Downloading wheel from {wheel_url_or_path} to {wheel_path}")
455
                from urllib.request import urlretrieve
456

457
458
459
460
461
462
463
464
465
466
467
468
                urlretrieve(wheel_url_or_path, filename=wheel_path)
            else:
                wheel_path = wheel_url_or_path
                print(f"Using existing wheel at {wheel_path}")

            package_data_patch = {}

            with zipfile.ZipFile(wheel_path) as wheel:
                files_to_copy = [
                    "vllm/_C.abi3.so",
                    "vllm/_moe_C.abi3.so",
                    "vllm/_flashmla_C.abi3.so",
469
470
                    "vllm/_flashmla_extension_C.abi3.so",
                    "vllm/_sparse_flashmla_C.abi3.so",
471
472
473
474
475
476
                    "vllm/vllm_flash_attn/_vllm_fa2_C.abi3.so",
                    "vllm/vllm_flash_attn/_vllm_fa3_C.abi3.so",
                    "vllm/cumem_allocator.abi3.so",
                ]

                compiled_regex = re.compile(
477
478
                    r"vllm/vllm_flash_attn/(?:[^/.][^/]*/)*(?!\.)[^/]*\.py"
                )
479
                file_members = list(
480
481
                    filter(lambda x: x.filename in files_to_copy, wheel.filelist)
                )
482
                file_members += list(
483
484
                    filter(lambda x: compiled_regex.match(x.filename), wheel.filelist)
                )
485
486
487
488
489

                for file in file_members:
                    print(f"[extract] {file.filename}")
                    target_path = os.path.join(".", file.filename)
                    os.makedirs(os.path.dirname(target_path), exist_ok=True)
490
491
492
493
                    with (
                        wheel.open(file.filename) as src,
                        open(target_path, "wb") as dst,
                    ):
494
495
496
497
                        shutil.copyfileobj(src, dst)

                    pkg = os.path.dirname(file.filename).replace("/", ".")
                    package_data_patch.setdefault(pkg, []).append(
498
499
                        os.path.basename(file.filename)
                    )
500
501
502
503
504
505
506
507
508

            return package_data_patch
        finally:
            if temp_dir is not None:
                print(f"Removing temporary directory {temp_dir}")
                shutil.rmtree(temp_dir)

    @staticmethod
    def get_base_commit_in_main_branch() -> str:
509
510
511
        # Force to use the nightly wheel. This is mainly used for CI testing.
        if envs.VLLM_TEST_USE_PRECOMPILED_NIGHTLY_WHEEL:
            return "nightly"
512
513

        try:
514
            # Get the latest commit hash of the upstream main branch.
515
516
517
518
519
520
521
            resp_json = subprocess.check_output(
                [
                    "curl",
                    "-s",
                    "https://api.github.com/repos/vllm-project/vllm/commits/main",
                ]
            ).decode("utf-8")
522
523
            upstream_main_commit = json.loads(resp_json)["sha"]

524
525
526
527
            # In Docker build context, .git may be immutable or missing.
            if envs.VLLM_DOCKER_BUILD_CONTEXT:
                return upstream_main_commit

528
529
530
            # Check if the upstream_main_commit exists in the local repo
            try:
                subprocess.check_output(
531
532
                    ["git", "cat-file", "-e", f"{upstream_main_commit}"]
                )
533
534
535
536
537
            except subprocess.CalledProcessError:
                # If not present, fetch it from the remote repository.
                # Note that this does not update any local branches,
                # but ensures that this commit ref and its history are
                # available in our local repo.
538
539
540
                subprocess.check_call(
                    ["git", "fetch", "https://github.com/vllm-project/vllm", "main"]
                )
541
542
543

            # Then get the commit hash of the current branch that is the same as
            # the upstream main commit.
544
545
546
547
548
            current_branch = (
                subprocess.check_output(["git", "branch", "--show-current"])
                .decode("utf-8")
                .strip()
            )
549

550
551
552
553
554
555
556
            base_commit = (
                subprocess.check_output(
                    ["git", "merge-base", f"{upstream_main_commit}", current_branch]
                )
                .decode("utf-8")
                .strip()
            )
557
            return base_commit
558
559
        except ValueError as err:
            raise ValueError(err) from None
560
561
562
563
        except Exception as err:
            logger.warning(
                "Failed to get the base commit in the main branch. "
                "Using the nightly wheel. The libraries in this "
564
565
566
                "wheel may not be compatible with your dev branch: %s",
                err,
            )
567
            return "nightly"
568
569


570
571
572
573
def _no_device() -> bool:
    return VLLM_TARGET_DEVICE == "empty"


574
def _is_cuda() -> bool:
575
    has_cuda = torch.version.cuda is not None
576
    return VLLM_TARGET_DEVICE == "cuda" and has_cuda and not _is_tpu()
577
578
579


def _is_hip() -> bool:
580
581
582
    return (
        VLLM_TARGET_DEVICE == "cuda" or VLLM_TARGET_DEVICE == "rocm"
    ) and torch.version.hip is not None
583
584


585
586
587
588
def _is_tpu() -> bool:
    return VLLM_TARGET_DEVICE == "tpu"


589
590
def _is_cpu() -> bool:
    return VLLM_TARGET_DEVICE == "cpu"
591
592


593
594
595
596
def _is_xpu() -> bool:
    return VLLM_TARGET_DEVICE == "xpu"


597
def _build_custom_ops() -> bool:
zhuwenwen's avatar
zhuwenwen committed
598
599
    if not skip_vllm_build:
        return _is_cuda() or _is_hip() or _is_cpu()
600
601


602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
def get_rocm_version():
    # Get the Rocm version from the ROCM_HOME/bin/librocm-core.so
    # see https://github.com/ROCm/rocm-core/blob/d11f5c20d500f729c393680a01fa902ebf92094b/rocm_version.cpp#L21
    try:
        librocm_core_file = Path(ROCM_HOME) / "lib" / "librocm-core.so"
        if not librocm_core_file.is_file():
            return None
        librocm_core = ctypes.CDLL(librocm_core_file)
        VerErrors = ctypes.c_uint32
        get_rocm_core_version = librocm_core.getROCmVersion
        get_rocm_core_version.restype = VerErrors
        get_rocm_core_version.argtypes = [
            ctypes.POINTER(ctypes.c_uint32),
            ctypes.POINTER(ctypes.c_uint32),
            ctypes.POINTER(ctypes.c_uint32),
        ]
        major = ctypes.c_uint32()
        minor = ctypes.c_uint32()
        patch = ctypes.c_uint32()
621

622
623
624
625
626
627
        if (
            get_rocm_core_version(
                ctypes.byref(major), ctypes.byref(minor), ctypes.byref(patch)
            )
            == 0
        ):
628
            return f"{major.value}.{minor.value}.{patch.value}"
629
        return None
630
    except Exception:
631
        return None
Woosuk Kwon's avatar
Woosuk Kwon committed
632

633

bnellnm's avatar
bnellnm committed
634
def get_nvcc_cuda_version() -> Version:
635
636
637
638
    """Get the CUDA version from nvcc.

    Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py
    """
639
    assert CUDA_HOME is not None, "CUDA_HOME is not set"
640
641
642
    nvcc_output = subprocess.check_output(
        [CUDA_HOME + "/bin/nvcc", "-V"], universal_newlines=True
    )
643
644
645
646
647
648
    output = nvcc_output.split()
    release_idx = output.index("release") + 1
    nvcc_cuda_version = parse(output[release_idx].split(",")[0])
    return nvcc_cuda_version


zhuwenwen's avatar
zhuwenwen committed
649
650
651
652
653
654
def get_sha(root: Union[str, Path]) -> str:
    try:
        return subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=root).decode('ascii').strip()
    except Exception:
        return 'Unknown'

655

zhuwenwen's avatar
zhuwenwen committed
656
def get_version_add(sha: Optional[str] = None) -> str:
zhuwenwen's avatar
zhuwenwen committed
657
    command = "git config --global --add safe.directory "+pwd  
zhuwenwen's avatar
zhuwenwen committed
658
    subprocess.run(command, shell=True, capture_output=False, text=True) 
zhuwenwen's avatar
zhuwenwen committed
659
660
    vllm_root = os.path.dirname(os.path.abspath(__file__))
    add_version_path = os.path.join(os.path.join(vllm_root, "vllm"), "version.py")
zhuwenwen's avatar
zhuwenwen committed
661
    major, minor, _ = torch.__version__.split('.')
662
663
664
665
    if add_git_version:
        if sha != 'Unknown':
            if sha is None:
                sha = get_sha(vllm_root)
zhuwenwen's avatar
zhuwenwen committed
666
            version = 'das.opt1.alpha.' + sha[:7]
667
    else:
zhuwenwen's avatar
zhuwenwen committed
668
        version = 'das.opt1.alpha'
669

zhuwenwen's avatar
zhuwenwen committed
670
671
672
673
674
675
676

    # dtk version
    if os.getenv("ROCM_PATH"):
        rocm_path = os.getenv('ROCM_PATH', "")
        rocm_version_path = os.path.join(rocm_path, '.info', "rocm_version")
        with open(rocm_version_path, 'r',encoding='utf-8') as file:
            lines = file.readlines()
zhuwenwen's avatar
zhuwenwen committed
677
        rocm_version=lines[0].replace(".", "")
zhuwenwen's avatar
zhuwenwen committed
678
        version += ".dtk" + rocm_version
zhuwenwen's avatar
zhuwenwen committed
679
    
680
    new_version_content = f"""
681
try:
682
683
684
    __version__ = "0.12.0"
    __version_tuple__ = (0, 12, 0)
    __hcu_version__ = f'0.12.0+{version}' 
zhuwenwen's avatar
zhuwenwen committed
685
    
zhuwenwen's avatar
zhuwenwen committed
686
    from vllm.version import __version__, __version_tuple__, __hcu_version__
687
except Exception as e:
zhuwenwen's avatar
zhuwenwen committed
688
689
    import warnings

zhuwenwen's avatar
zhuwenwen committed
690
    warnings.warn(f"Failed to read commit hash:\\n + str(e)",
691
692
                  RuntimeWarning,
                  stacklevel=2)
693
694
    __version__ = "dev"
    __version_tuple__ = (0, 0, __version__)
zhuwenwen's avatar
zhuwenwen committed
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
    
    
def _prev_minor_version_was(version_str):
    '''Check whether a given version matches the previous minor version.

    Return True if version_str matches the previous minor version.

    For example - return True if the current version if 0.7.4 and the
    supplied version_str is '0.6'.

    Used for --show-hidden-metrics-for-version.
    '''
    # Match anything if this is a dev tree
    if __version_tuple__[0:2] == (0, 0):
        return True

    # Note - this won't do the right thing when we release 1.0!
712
    # assert __version_tuple__[0] == 0
zhuwenwen's avatar
zhuwenwen committed
713
714
715
716
717
718
719
720
721
    assert isinstance(__version_tuple__[1], int)
    return version_str == f"{{__version_tuple__[0]}}.{{__version_tuple__[1] - 1}}"


def _prev_minor_version():
    '''For the purpose of testing, return a previous minor version number.'''
    # In dev tree, this will return "0.-1", but that will work fine"
    assert isinstance(__version_tuple__[1], int)
    return f"{{__version_tuple__[0]}}.{{__version_tuple__[1] - 1}}"
722
"""
zhuwenwen's avatar
zhuwenwen committed
723
    
zhuwenwen's avatar
zhuwenwen committed
724
    with open(add_version_path, encoding="utf-8",mode="w") as file:
zhuwenwen's avatar
zhuwenwen committed
725
        file.write(new_version_content)
zhuwenwen's avatar
zhuwenwen committed
726
727
728
729
730
731
732
733
    file.close()
    
    
def get_version():
    get_version_add()
    version_file = 'vllm/version.py'
    with open(version_file, encoding='utf-8') as f:
        exec(compile(f.read(), version_file, 'exec'))
zhuwenwen's avatar
zhuwenwen committed
734
    return locals()['__hcu_version__']
zhuwenwen's avatar
zhuwenwen committed
735
736


737
738
739
740
741
742
743
744
def get_gaudi_sw_version():
    """
    Returns the driver version.
    """
    # Enable console printing for `hl-smi` check
    output = subprocess.run("hl-smi",
                            shell=True,
                            text=True,
745
                            capture_output=True,
746
747
748
749
750
751
752
                            env={"ENABLE_CONSOLE": "true"})
    if output.returncode == 0 and output.stdout:
        return output.stdout.split("\n")[2].replace(
            " ", "").split(":")[1][:-1].split("-")[0]
    return "0.0.0"  # when hl-smi is not available


753
def get_vllm_version() -> str:
754
755
756
    # Allow overriding the version. This is useful to build platform-specific
    # wheels (e.g. CPU, TPU) without modifying the source.
    if env_version := os.getenv("VLLM_VERSION_OVERRIDE"):
757
758
759
        print(f"Overriding VLLM version with {env_version} from VLLM_VERSION_OVERRIDE")
        os.environ["SETUPTOOLS_SCM_PRETEND_VERSION"] = env_version
        return get_version(write_to="vllm/_version.py")
760

761
762
    # version = get_version(write_to="vllm/_version.py")
    # sep = "+" if "+" not in version else "."  # dev versions might contain +
763

764
    if _no_device():
765
        if envs.VLLM_TARGET_DEVICE == "empty":
766
            version += f"{sep}empty"
767
    elif _is_cuda():
768
        if envs.VLLM_USE_PRECOMPILED:
769
            version += f"{sep}precompiled"
770
771
        else:
            cuda_version = str(get_nvcc_cuda_version())
772
            if cuda_version != envs.VLLM_MAIN_CUDA_VERSION:
773
774
775
776
                cuda_version_str = cuda_version.replace(".", "")[:3]
                # skip this for source tarball, required for pypi
                if "sdist" not in sys.argv:
                    version += f"{sep}cu{cuda_version_str}"
777
    elif _is_hip():
778
        # Get the Rocm Version
zhuwenwen's avatar
zhuwenwen committed
779
        # rocm_version = get_rocm_version() or torch.version.hip
780
        # if rocm_version and rocm_version != envs.VLLM_MAIN_CUDA_VERSION:
zhuwenwen's avatar
zhuwenwen committed
781
        #     version += f"{sep}rocm{rocm_version.replace('.', '')[:3]}"
zhuwenwen's avatar
zhuwenwen committed
782
        version = get_version()
783
    elif _is_tpu():
784
        version += f"{sep}tpu"
785
    elif _is_cpu():
786
787
        if envs.VLLM_TARGET_DEVICE == "cpu":
            version += f"{sep}cpu"
788
    elif _is_xpu():
789
        version += f"{sep}xpu"
790
    else:
791
        raise RuntimeError("Unknown runtime environment")
792

793
794
795
    return version


796
def get_requirements() -> list[str]:
797
    """Get Python package dependencies from requirements.txt."""
798
    requirements_dir = ROOT_DIR / "requirements"
799

800
    def _read_requirements(filename: str) -> list[str]:
801
        with open(requirements_dir / filename) as f:
802
            requirements = f.read().strip().split("\n")
803
804
805
806
        resolved_requirements = []
        for line in requirements:
            if line.startswith("-r "):
                resolved_requirements += _read_requirements(line.split()[1])
807
808
809
810
811
            elif (
                not line.startswith("--")
                and not line.startswith("#")
                and line.strip() != ""
            ):
812
813
814
                resolved_requirements.append(line)
        return resolved_requirements

815
    if _no_device():
816
        requirements = _read_requirements("common.txt")
817
    elif _is_cuda():
818
        requirements = _read_requirements("cuda.txt")
819
        cuda_major, cuda_minor = torch.version.cuda.split(".")
820
821
        modified_requirements = []
        for req in requirements:
822
            if "vllm-flash-attn" in req and cuda_major != "12":
823
                # vllm-flash-attn is built only for CUDA 12.x.
824
825
826
                # Skip for other versions.
                continue
            modified_requirements.append(req)
827
        requirements = modified_requirements
828
    elif _is_hip():
829
        requirements = _read_requirements("rocm.txt")
830
    elif _is_tpu():
831
        requirements = _read_requirements("tpu.txt")
832
    elif _is_cpu():
833
        requirements = _read_requirements("cpu.txt")
834
    elif _is_xpu():
835
        requirements = _read_requirements("xpu.txt")
836
    else:
837
        raise ValueError("Unsupported platform, please use CUDA, ROCm, or CPU.")
838
839
840
    return requirements


bnellnm's avatar
bnellnm committed
841
842
ext_modules = []

843
if _is_cuda() or _is_hip():
bnellnm's avatar
bnellnm committed
844
    ext_modules.append(CMakeExtension(name="vllm._moe_C"))
845
    ext_modules.append(CMakeExtension(name="vllm.cumem_allocator"))
846
847
    # Optional since this doesn't get built (produce an .so file). This is just
    # copying the relevant .py files from the source repository.
848
    # ext_modules.append(CMakeExtension(name="vllm.triton_kernels", optional=True))
bnellnm's avatar
bnellnm committed
849

850
851
# if _is_hip():
#     ext_modules.append(CMakeExtension(name="vllm._rocm_C"))
852

853
if _is_cuda():
854
    ext_modules.append(CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa2_C"))
855
856
    if envs.VLLM_USE_PRECOMPILED or get_nvcc_cuda_version() >= Version("12.3"):
        # FA3 requires CUDA 12.3 or later
857
        ext_modules.append(CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa3_C"))
858
859
        # Optional since this doesn't get built (produce an .so file) when
        # not targeting a hopper system
860
        ext_modules.append(CMakeExtension(name="vllm._flashmla_C", optional=True))
861
        ext_modules.append(
862
863
            CMakeExtension(name="vllm._flashmla_extension_C", optional=True)
        )
864

865
if _build_custom_ops():
bnellnm's avatar
bnellnm committed
866
867
    ext_modules.append(CMakeExtension(name="vllm._C"))

zhuwenwen's avatar
zhuwenwen committed
868
869
870
871
872
873
874
875
876
if skip_vllm_build:
    package_data = {
        "vllm": [
            "py.typed",
            "model_executor/layers/fused_moe/configs/*.json",
            "model_executor/layers/quantization/utils/configs/*.json",
            "perf/*.py",
            "attention/backends/configs/*.json",
            "model_executor/layers/quantization/configs/awq/*.json",
zhuwenwen's avatar
zhuwenwen committed
877
878
            "_C.abi3.so",
            "_moe_C.abi3.so",
zhuwenwen's avatar
zhuwenwen committed
879
880
881
882
883
884
885
886
887
888
889
890
891
892
        ]
    }
else:
    package_data = {
        "vllm": [
            "py.typed",
            "model_executor/layers/fused_moe/configs/*.json",
            "model_executor/layers/quantization/utils/configs/*.json",
            "perf/*.py",
            "attention/backends/configs/*.json",
            "model_executor/layers/quantization/configs/awq/*.json",
        ]
    }

893

894
895
# If using precompiled, extract and patch package_data (in advance of setup)
if envs.VLLM_USE_PRECOMPILED:
896
    wheel_url, download_filename = precompiled_wheel_utils.determine_wheel_url()
897
    patch = precompiled_wheel_utils.extract_precompiled_and_patch_package(
898
899
        wheel_url, download_filename
    )
900
901
902
    for pkg, files in patch.items():
        package_data.setdefault(pkg, []).extend(files)

zhuwenwen's avatar
zhuwenwen committed
903
if _no_device() or skip_vllm_build:
904
905
    ext_modules = []

906
907
908
909
if not ext_modules:
    cmdclass = {}
else:
    cmdclass = {
910
911
912
        "build_ext": precompiled_build_ext
        if envs.VLLM_USE_PRECOMPILED
        else cmake_build_ext
913
914
    }

bnellnm's avatar
bnellnm committed
915
setup(
916
    # static metadata should rather go in pyproject.toml
917
    version=get_vllm_version(),
Woosuk Kwon's avatar
Woosuk Kwon committed
918
    ext_modules=ext_modules,
919
    install_requires=get_requirements(),
920
    extras_require={
921
        "bench": ["pandas", "matplotlib", "seaborn", "datasets"],
922
        "tensorizer": ["tensorizer==2.10.1"],
923
        "fastsafetensors": ["fastsafetensors >= 0.1.10"],
924
        "runai": ["runai-model-streamer[s3,gcs] >= 0.15.0"],
925
926
927
928
929
        "audio": [
            "librosa",
            "soundfile",
            "mistral_common[audio]",
        ],  # Required for audio processing
930
        "video": [],  # Kept for backwards compatibility
931
        "flashinfer": [],  # Kept for backwards compatibility
932
933
        # Optional deps for AMD FP4 quantization support
        "petit-kernel": ["petit-kernel"],
934
    },
935
    cmdclass=cmdclass,
Simon Mo's avatar
Simon Mo committed
936
    package_data=package_data,
Woosuk Kwon's avatar
Woosuk Kwon committed
937
)