"vscode:/vscode.git/clone" did not exist on "4f02cb2eac20a1281784914acca80cd1fe2bfd15"
setup.py 40.6 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import ctypes
5
import importlib.util
6
import json
7
import logging
8
9
import os
import re
10
import shutil
11
import subprocess
bnellnm's avatar
bnellnm committed
12
import sys
13
import sysconfig
14
from pathlib import Path
15
from shutil import which
16

Woosuk Kwon's avatar
Woosuk Kwon committed
17
import torch
18
from packaging.version import Version, parse
19
from setuptools import Extension, setup
20
from setuptools.command.build_ext import build_ext
21
from setuptools.command.build_py import build_py
22
from setuptools.command.develop import develop
23
# from setuptools_scm import get_version
24
from torch.utils.cpp_extension import CUDA_HOME, ROCM_HOME
25

zhuwenwen's avatar
zhuwenwen committed
26
27
28
29
from typing import Optional, Union
import subprocess
from pathlib import Path

zhuwenwen's avatar
zhuwenwen committed
30
pwd = os.path.dirname(os.path.abspath(__file__))
gaoqiong's avatar
gaoqiong committed
31
add_git_version = False
32
33
if int(os.environ.get('ADD_GIT_VERSION', '0')) == 1:
    add_git_version = True
zhuwenwen's avatar
zhuwenwen committed
34
35
36
37
    
skip_vllm_build = False
if int(os.environ.get('SKIP_VLLM_BUILD', '0')) == 1:
    skip_vllm_build = True
38
39
40
41
42
43
44
45
46

def load_module_from_path(module_name, path):
    spec = importlib.util.spec_from_file_location(module_name, path)
    module = importlib.util.module_from_spec(spec)
    sys.modules[module_name] = module
    spec.loader.exec_module(module)
    return module


47
ROOT_DIR = Path(__file__).parent
48
logger = logging.getLogger(__name__)
49
50
51

# cannot import envs directly because it depends on vllm,
#  which is not installed yet
52
envs = load_module_from_path("envs", os.path.join(ROOT_DIR, "vllm", "envs.py"))
53
54

VLLM_TARGET_DEVICE = envs.VLLM_TARGET_DEVICE
55

56
if sys.platform.startswith("darwin") and VLLM_TARGET_DEVICE != "cpu":
57
    logger.warning("VLLM_TARGET_DEVICE automatically set to `cpu` due to macOS")
58
    VLLM_TARGET_DEVICE = "cpu"
59
elif not (sys.platform.startswith("linux") or sys.platform.startswith("darwin")):
60
61
    logger.warning(
        "vLLM only supports Linux platform (including WSL) and MacOS."
62
        "Building on %s, "
63
64
65
        "so vLLM may not be able to run correctly",
        sys.platform,
    )
66
    VLLM_TARGET_DEVICE = "empty"
67
68
69
70
71
72
73
74
75
elif sys.platform.startswith("linux") and os.getenv("VLLM_TARGET_DEVICE") is None:
    if torch.version.hip is not None:
        VLLM_TARGET_DEVICE = "rocm"
        logger.info("Auto-detected ROCm")
    elif torch.version.cuda is not None:
        VLLM_TARGET_DEVICE = "cuda"
        logger.info("Auto-detected CUDA")
    else:
        VLLM_TARGET_DEVICE = "cpu"
76

bnellnm's avatar
bnellnm committed
77
78

def is_sccache_available() -> bool:
79
80
81
    return which("sccache") is not None and not bool(
        int(os.getenv("VLLM_DISABLE_SCCACHE", "0"))
    )
bnellnm's avatar
bnellnm committed
82
83
84
85
86
87
88
89
90
91


def is_ccache_available() -> bool:
    return which("ccache") is not None


def is_ninja_available() -> bool:
    return which("ninja") is not None


92
93
def is_freethreaded():
    return bool(sysconfig.get_config_var("Py_GIL_DISABLED"))
94
95


96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
def compile_grpc_protos():
    """Compile gRPC protobuf definitions during build.

    This generates *_pb2.py, *_pb2_grpc.py, and *_pb2.pyi files from
    the vllm_engine.proto definition.
    """
    try:
        from grpc_tools import protoc
    except ImportError:
        logger.warning(
            "grpcio-tools not installed, skipping gRPC proto compilation. "
            "gRPC server functionality will not be available."
        )
        return False

    proto_file = ROOT_DIR / "vllm" / "grpc" / "vllm_engine.proto"
    if not proto_file.exists():
        logger.warning("Proto file not found at %s, skipping compilation", proto_file)
        return False

    logger.info("Compiling gRPC protobuf: %s", proto_file)

    result = protoc.main(
        [
            "grpc_tools.protoc",
            f"--proto_path={ROOT_DIR}",
            f"--python_out={ROOT_DIR}",
            f"--grpc_python_out={ROOT_DIR}",
            f"--pyi_out={ROOT_DIR}",
            str(proto_file),
        ]
    )

    if result != 0:
        logger.error("protoc failed with exit code %s", result)
        return False

    # Add SPDX headers and mypy ignore to generated files
    spdx_header = (
        "# SPDX-License-Identifier: Apache-2.0\n"
        "# SPDX-FileCopyrightText: Copyright contributors to the vLLM project\n"
        "# mypy: ignore-errors\n"
    )

    grpc_dir = ROOT_DIR / "vllm" / "grpc"
    for generated_file in [
        grpc_dir / "vllm_engine_pb2.py",
        grpc_dir / "vllm_engine_pb2_grpc.py",
        grpc_dir / "vllm_engine_pb2.pyi",
    ]:
        if generated_file.exists():
            content = generated_file.read_text()
            if not content.startswith("# SPDX-License-Identifier"):
                generated_file.write_text(spdx_header + content)

    logger.info("gRPC protobuf compilation successful")
    return True


class BuildPyAndGenerateGrpc(build_py):
    """Build Python modules and generate gRPC stubs from proto files."""

    def run(self):
        compile_grpc_protos()
        super().run()


163
164
165
166
167
168
169
170
class DevelopAndGenerateGrpc(develop):
    """Develop mode that also generates gRPC stubs from proto files."""

    def run(self):
        compile_grpc_protos()
        super().run()


bnellnm's avatar
bnellnm committed
171
class CMakeExtension(Extension):
172
    def __init__(self, name: str, cmake_lists_dir: str = ".", **kwa) -> None:
173
        super().__init__(name, sources=[], py_limited_api=not is_freethreaded(), **kwa)
bnellnm's avatar
bnellnm committed
174
175
176
177
178
        self.cmake_lists_dir = os.path.abspath(cmake_lists_dir)


class cmake_build_ext(build_ext):
    # A dict of extension directories that have been configured.
179
    did_config: dict[str, bool] = {}
bnellnm's avatar
bnellnm committed
180
181
182
183
184

    #
    # Determine number of compilation jobs and optionally nvcc compile threads.
    #
    def compute_num_jobs(self):
185
186
        # `num_jobs` is either the value of the MAX_JOBS environment variable
        # (if defined) or the number of CPUs available.
187
        num_jobs = envs.MAX_JOBS
188
189
        if num_jobs is not None:
            num_jobs = int(num_jobs)
190
            logger.info("Using MAX_JOBS=%d as the number of jobs.", num_jobs)
191
192
193
194
195
196
197
        else:
            try:
                # os.sched_getaffinity() isn't universally available, so fall
                #  back to os.cpu_count() if we get an error here.
                num_jobs = len(os.sched_getaffinity(0))
            except AttributeError:
                num_jobs = os.cpu_count()
bnellnm's avatar
bnellnm committed
198

199
        nvcc_threads = None
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
        if _is_cuda() and CUDA_HOME is not None:
            try:
                nvcc_version = get_nvcc_cuda_version()
                if nvcc_version >= Version("11.2"):
                    # `nvcc_threads` is either the value of the NVCC_THREADS
                    # environment variable (if defined) or 1.
                    # when it is set, we reduce `num_jobs` to avoid
                    # overloading the system.
                    nvcc_threads = envs.NVCC_THREADS
                    if nvcc_threads is not None:
                        nvcc_threads = int(nvcc_threads)
                        logger.info(
                            "Using NVCC_THREADS=%d as the number of nvcc threads.",
                            nvcc_threads,
                        )
                    else:
                        nvcc_threads = 1
                    num_jobs = max(1, num_jobs // nvcc_threads)
            except Exception as e:
                logger.warning("Failed to get NVCC version: %s", e)
bnellnm's avatar
bnellnm committed
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236

        return num_jobs, nvcc_threads

    #
    # Perform cmake configuration for a single extension.
    #
    def configure(self, ext: CMakeExtension) -> None:
        # If we've already configured using the CMakeLists.txt for
        # this extension, exit early.
        if ext.cmake_lists_dir in cmake_build_ext.did_config:
            return

        cmake_build_ext.did_config[ext.cmake_lists_dir] = True

        # Select the build type.
        # Note: optimization level + debug info are set by the build type
        default_cfg = "Debug" if self.debug else "RelWithDebInfo"
237
        cfg = envs.CMAKE_BUILD_TYPE or default_cfg
bnellnm's avatar
bnellnm committed
238
239

        cmake_args = [
240
241
            "-DCMAKE_BUILD_TYPE={}".format(cfg),
            "-DVLLM_TARGET_DEVICE={}".format(VLLM_TARGET_DEVICE),
bnellnm's avatar
bnellnm committed
242
243
        ]

244
        verbose = envs.VERBOSE
bnellnm's avatar
bnellnm committed
245
        if verbose:
246
            cmake_args += ["-DCMAKE_VERBOSE_MAKEFILE=ON"]
bnellnm's avatar
bnellnm committed
247
248
249

        if is_sccache_available():
            cmake_args += [
250
251
252
253
                "-DCMAKE_C_COMPILER_LAUNCHER=sccache",
                "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache",
                "-DCMAKE_CUDA_COMPILER_LAUNCHER=sccache",
                "-DCMAKE_HIP_COMPILER_LAUNCHER=sccache",
bnellnm's avatar
bnellnm committed
254
255
256
            ]
        elif is_ccache_available():
            cmake_args += [
257
258
259
260
                "-DCMAKE_C_COMPILER_LAUNCHER=ccache",
                "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache",
                "-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache",
                "-DCMAKE_HIP_COMPILER_LAUNCHER=ccache",
bnellnm's avatar
bnellnm committed
261
262
263
264
            ]

        # Pass the python executable to cmake so it can find an exact
        # match.
265
        cmake_args += ["-DVLLM_PYTHON_EXECUTABLE={}".format(sys.executable)]
bnellnm's avatar
bnellnm committed
266

267
268
        # Pass the python path to cmake so it can reuse the build dependencies
        # on subsequent calls to python.
269
        cmake_args += ["-DVLLM_PYTHON_PATH={}".format(":".join(sys.path))]
270

271
272
273
274
275
276
        # Override the base directory for FetchContent downloads to $ROOT/.deps
        # This allows sharing dependencies between profiles,
        # and plays more nicely with sccache.
        # To override this, set the FETCHCONTENT_BASE_DIR environment variable.
        fc_base_dir = os.path.join(ROOT_DIR, ".deps")
        fc_base_dir = os.environ.get("FETCHCONTENT_BASE_DIR", fc_base_dir)
277
        cmake_args += ["-DFETCHCONTENT_BASE_DIR={}".format(fc_base_dir)]
278

bnellnm's avatar
bnellnm committed
279
280
281
282
283
284
        #
        # Setup parallelism and build tool
        #
        num_jobs, nvcc_threads = self.compute_num_jobs()

        if nvcc_threads:
285
            cmake_args += ["-DNVCC_THREADS={}".format(nvcc_threads)]
bnellnm's avatar
bnellnm committed
286
287

        if is_ninja_available():
288
            build_tool = ["-G", "Ninja"]
bnellnm's avatar
bnellnm committed
289
            cmake_args += [
290
291
                "-DCMAKE_JOB_POOL_COMPILE:STRING=compile",
                "-DCMAKE_JOB_POOLS:STRING=compile={}".format(num_jobs),
bnellnm's avatar
bnellnm committed
292
293
294
295
            ]
        else:
            # Default build tool to whatever cmake picks.
            build_tool = []
296
        # Make sure we use the nvcc from CUDA_HOME
297
        if _is_cuda() and CUDA_HOME is not None:
298
            cmake_args += [f"-DCMAKE_CUDA_COMPILER={CUDA_HOME}/bin/nvcc"]
299
        elif _is_hip() and ROCM_HOME is not None:
300
            cmake_args += [f"-DROCM_PATH={ROCM_HOME}"]
301
302
303
304
305

        other_cmake_args = os.environ.get("CMAKE_ARGS")
        if other_cmake_args:
            cmake_args += other_cmake_args.split()

bnellnm's avatar
bnellnm committed
306
        subprocess.check_call(
307
308
309
            ["cmake", ext.cmake_lists_dir, *build_tool, *cmake_args],
            cwd=self.build_temp,
        )
bnellnm's avatar
bnellnm committed
310
311
312
313

    def build_extensions(self) -> None:
        # Ensure that CMake is present and working
        try:
314
            subprocess.check_output(["cmake", "--version"])
bnellnm's avatar
bnellnm committed
315
        except OSError as e:
316
            raise RuntimeError("Cannot find CMake executable") from e
bnellnm's avatar
bnellnm committed
317
318
319
320
321

        # Create build directory if it does not exist.
        if not os.path.exists(self.build_temp):
            os.makedirs(self.build_temp)

322
        targets = []
323
324
325
326

        def target_name(s: str) -> str:
            return s.removeprefix("vllm.").removeprefix("vllm_flash_attn.")

bnellnm's avatar
bnellnm committed
327
328
329
        # Build all the extensions
        for ext in self.extensions:
            self.configure(ext)
330
            targets.append(target_name(ext.name))
bnellnm's avatar
bnellnm committed
331

332
        num_jobs, _ = self.compute_num_jobs()
bnellnm's avatar
bnellnm committed
333

334
335
336
337
338
339
        build_args = [
            "--build",
            ".",
            f"-j={num_jobs}",
            *[f"--target={name}" for name in targets],
        ]
bnellnm's avatar
bnellnm committed
340

341
        subprocess.check_call(["cmake", *build_args], cwd=self.build_temp)
bnellnm's avatar
bnellnm committed
342

343
344
345
346
347
348
349
350
351
352
353
354
        # Install the libraries
        for ext in self.extensions:
            # Install the extension into the proper location
            outdir = Path(self.get_ext_fullpath(ext.name)).parent.absolute()

            # Skip if the install directory is the same as the build directory
            if outdir == self.build_temp:
                continue

            # CMake appends the extension prefix to the install path,
            # and outdir already contains that prefix, so we need to remove it.
            prefix = outdir
355
            for _ in range(ext.name.count(".")):
356
357
358
359
                prefix = prefix.parent

            # prefix here should actually be the same for all components
            install_args = [
360
361
362
363
364
365
366
                "cmake",
                "--install",
                ".",
                "--prefix",
                prefix,
                "--component",
                target_name(ext.name),
367
368
369
            ]
            subprocess.check_call(install_args, cwd=self.build_temp)

370
371
372
373
    def run(self):
        # First, run the standard build_ext command to compile the extensions
        super().run()

374
        # copy vllm/vllm_flash_attn/**/*.py from self.build_lib to current
375
376
        # directory so that they can be included in the editable build
        import glob
377
378
379
380
381

        files = glob.glob(
            os.path.join(self.build_lib, "vllm", "vllm_flash_attn", "**", "*.py"),
            recursive=True,
        )
382
        for file in files:
383
384
385
            dst_file = os.path.join(
                "vllm/vllm_flash_attn", file.split("vllm/vllm_flash_attn/")[-1]
            )
386
            print(f"Copying {file} to {dst_file}")
387
            os.makedirs(os.path.dirname(dst_file), exist_ok=True)
388
389
            self.copy_file(file, dst_file)

390
        if _is_cuda():
391
392
393
394
395
396
397
398
399
400
401
402
403
            # copy vllm/third_party/triton_kernels/**/*.py from self.build_lib
            # to current directory so that they can be included in the editable
            # build
            print(
                f"Copying {self.build_lib}/vllm/third_party/triton_kernels "
                "to vllm/third_party/triton_kernels"
            )
            shutil.copytree(
                f"{self.build_lib}/vllm/third_party/triton_kernels",
                "vllm/third_party/triton_kernels",
                dirs_exist_ok=True,
            )

404

405
406
407
408
class precompiled_build_ext(build_ext):
    """Disables extension building when using precompiled binaries."""

    def run(self) -> None:
409
        return
410
411
412
413
414
415
416

    def build_extensions(self) -> None:
        print("Skipping build_ext: using precompiled extensions.")
        return


class precompiled_wheel_utils:
417
418
    """Extracts libraries and other files from an existing wheel."""

419
    @staticmethod
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
    def fetch_metadata_for_variant(
        commit: str, variant: str | None
    ) -> tuple[list[dict], str]:
        """
        Fetches metadata for a specific variant of the precompiled wheel.
        """
        variant_dir = f"{variant}/" if variant is not None else ""
        repo_url = f"https://wheels.vllm.ai/{commit}/{variant_dir}vllm/"
        meta_url = repo_url + "metadata.json"
        print(f"Trying to fetch nightly build metadata from {meta_url}")
        from urllib.request import urlopen

        with urlopen(meta_url) as resp:
            # urlopen raises HTTPError on unexpected status code
            wheels = json.loads(resp.read().decode("utf-8"))
        return wheels, repo_url

437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
    @staticmethod
    def is_rocm_system() -> bool:
        """Detect ROCm without relying on torch (for build environment)."""
        if os.getenv("ROCM_PATH"):
            return True
        if os.path.isdir("/opt/rocm"):
            return True
        if which("rocminfo") is not None:
            return True
        try:
            import torch

            return torch.version.hip is not None
        except ImportError:
            return False

    @staticmethod
    def find_local_rocm_wheel() -> str | None:
        """Search for a local vllm wheel in common locations."""
        import glob

        for pattern in ["/vllm-workspace/dist/vllm-*.whl", "./dist/vllm-*.whl"]:
            wheels = glob.glob(pattern)
            if wheels:
                return sorted(wheels)[-1]
        return None

    @staticmethod
    def fetch_wheel_from_pypi_index(index_url: str, package: str = "vllm") -> str:
        """Fetch the latest wheel URL from a PyPI-style simple index."""
        import platform
        from html.parser import HTMLParser
        from urllib.parse import urljoin
        from urllib.request import urlopen

        arch = platform.machine()

        class WheelLinkParser(HTMLParser):
            def __init__(self):
                super().__init__()
                self.wheels = []

            def handle_starttag(self, tag, attrs):
                if tag == "a":
                    for name, value in attrs:
                        if name == "href" and value.endswith(".whl"):
                            self.wheels.append(value)

        simple_url = f"{index_url.rstrip('/')}/{package}/"
        print(f"Fetching wheel list from {simple_url}")
        with urlopen(simple_url) as resp:
            html = resp.read().decode("utf-8")

        parser = WheelLinkParser()
        parser.feed(html)

        for wheel in reversed(parser.wheels):
            if arch in wheel:
                if wheel.startswith("http"):
                    return wheel
                return urljoin(simple_url, wheel)

        raise ValueError(f"No compatible wheel found for {arch} at {simple_url}")

    @staticmethod
    def determine_wheel_url_rocm() -> tuple[str, str | None]:
        """Determine the precompiled wheel for ROCm."""
        # Search for local wheel first
        local_wheel = precompiled_wheel_utils.find_local_rocm_wheel()
        if local_wheel is not None:
            print(f"Found local ROCm wheel: {local_wheel}")
            return local_wheel, None

        # Fall back to AMD's PyPI index
        index_url = os.getenv(
            "VLLM_ROCM_WHEEL_INDEX", "https://pypi.amd.com/vllm-rocm/simple"
        )
        print(f"Fetching ROCm precompiled wheel from {index_url}")
        wheel_url = precompiled_wheel_utils.fetch_wheel_from_pypi_index(index_url)
        download_filename = wheel_url.split("/")[-1].split("#")[0]
        print(f"Using ROCm precompiled wheel: {wheel_url}")
        return wheel_url, download_filename

520
521
522
523
524
525
526
    @staticmethod
    def determine_wheel_url() -> tuple[str, str | None]:
        """
        Try to determine the precompiled wheel URL or path to use.
        The order of preference is:
        1. user-specified wheel location (can be either local or remote, via
           VLLM_PRECOMPILED_WHEEL_LOCATION)
527
        2. user-specified variant (VLLM_PRECOMPILED_WHEEL_VARIANT) from nightly repo
528
        3. the variant corresponding to VLLM_MAIN_CUDA_VERSION from nightly repo
529
530
531
532
533
        4. the default variant from nightly repo

        If downloading from the nightly repo, the commit can be specified via
        VLLM_PRECOMPILED_WHEEL_COMMIT; otherwise, the head commit in the main branch
        is used.
534
535
536
537
538
539
        """
        wheel_location = os.getenv("VLLM_PRECOMPILED_WHEEL_LOCATION", None)
        if wheel_location is not None:
            print(f"Using user-specified precompiled wheel location: {wheel_location}")
            return wheel_location, None
        else:
540
541
542
543
544
            # ROCm: use local wheel or AMD's PyPI index
            # TODO: When we have ROCm nightly wheels, we can update this logic.
            if precompiled_wheel_utils.is_rocm_system():
                return precompiled_wheel_utils.determine_wheel_url_rocm()

545
546
547
548
549
550
            import platform

            arch = platform.machine()
            # try to fetch the wheel metadata from the nightly wheel repo
            main_variant = "cu" + envs.VLLM_MAIN_CUDA_VERSION.replace(".", "")
            variant = os.getenv("VLLM_PRECOMPILED_WHEEL_VARIANT", main_variant)
551
552
553
554
555
556
557
            commit = os.getenv("VLLM_PRECOMPILED_WHEEL_COMMIT", "").lower()
            if not commit or len(commit) != 40:
                print(
                    f"VLLM_PRECOMPILED_WHEEL_COMMIT not valid: {commit}"
                    ", trying to fetch base commit in main branch"
                )
                commit = precompiled_wheel_utils.get_base_commit_in_main_branch()
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
            print(f"Using precompiled wheel commit {commit} with variant {variant}")
            try_default = False
            wheels, repo_url, download_filename = None, None, None
            try:
                wheels, repo_url = precompiled_wheel_utils.fetch_metadata_for_variant(
                    commit, variant
                )
            except Exception as e:
                logger.warning(
                    "Failed to fetch precompiled wheel metadata for variant %s: %s",
                    variant,
                    e,
                )
                try_default = True  # try outside handler to keep the stacktrace simple
            if try_default:
                print("Trying the default variant from remote")
                wheels, repo_url = precompiled_wheel_utils.fetch_metadata_for_variant(
                    commit, None
                )
                # if this also fails, then we have nothing more to try / cache
            assert wheels is not None and repo_url is not None, (
                "Failed to fetch precompiled wheel metadata"
            )
            # The metadata.json has the following format:
            # see .buildkite/scripts/generate-nightly-index.py for details
            """[{
    "package_name": "vllm",
    "version": "0.11.2.dev278+gdbc3d9991",
    "build_tag": null,
    "python_tag": "cp38",
    "abi_tag": "abi3",
    "platform_tag": "manylinux1_x86_64",
    "variant": null,
    "filename": "vllm-0.11.2.dev278+gdbc3d9991-cp38-abi3-manylinux1_x86_64.whl",
    "path": "../vllm-0.11.2.dev278%2Bgdbc3d9991-cp38-abi3-manylinux1_x86_64.whl"
    },
    ...]"""
            from urllib.parse import urljoin

            for wheel in wheels:
                # TODO: maybe check more compatibility later? (python_tag, abi_tag, etc)
                if wheel.get("package_name") == "vllm" and arch in wheel.get(
                    "platform_tag", ""
                ):
                    print(f"Found precompiled wheel metadata: {wheel}")
                    if "path" not in wheel:
                        raise ValueError(f"Wheel metadata missing path: {wheel}")
                    wheel_url = urljoin(repo_url, wheel["path"])
                    download_filename = wheel.get("filename")
                    print(f"Using precompiled wheel URL: {wheel_url}")
                    break
            else:
                raise ValueError(
                    f"No precompiled vllm wheel found for architecture {arch} "
                    f"from repo {repo_url}. All available wheels: {wheels}"
                )

        return wheel_url, download_filename

    @staticmethod
    def extract_precompiled_and_patch_package(
        wheel_url_or_path: str, download_filename: str | None
    ) -> dict:
621
622
623
624
625
626
        import tempfile
        import zipfile

        temp_dir = None
        try:
            if not os.path.isfile(wheel_url_or_path):
627
628
                # use provided filename first, then derive from URL
                wheel_filename = download_filename or wheel_url_or_path.split("/")[-1]
629
630
                temp_dir = tempfile.mkdtemp(prefix="vllm-wheels")
                wheel_path = os.path.join(temp_dir, wheel_filename)
631
                print(f"Downloading wheel from {wheel_url_or_path} to {wheel_path}")
632
                from urllib.request import urlretrieve
633

634
635
636
637
638
639
640
641
642
643
644
645
                urlretrieve(wheel_url_or_path, filename=wheel_path)
            else:
                wheel_path = wheel_url_or_path
                print(f"Using existing wheel at {wheel_path}")

            package_data_patch = {}

            with zipfile.ZipFile(wheel_path) as wheel:
                files_to_copy = [
                    "vllm/_C.abi3.so",
                    "vllm/_moe_C.abi3.so",
                    "vllm/_flashmla_C.abi3.so",
646
647
                    "vllm/_flashmla_extension_C.abi3.so",
                    "vllm/_sparse_flashmla_C.abi3.so",
648
649
650
                    "vllm/vllm_flash_attn/_vllm_fa2_C.abi3.so",
                    "vllm/vllm_flash_attn/_vllm_fa3_C.abi3.so",
                    "vllm/cumem_allocator.abi3.so",
651
652
                    # ROCm-specific libraries
                    "vllm/_rocm_C.abi3.so",
653
654
                ]

655
                flash_attn_regex = re.compile(
656
657
                    r"vllm/vllm_flash_attn/(?:[^/.][^/]*/)*(?!\.)[^/]*\.py"
                )
658
659
660
                triton_kernels_regex = re.compile(
                    r"vllm/third_party/triton_kernels/(?:[^/.][^/]*/)*(?!\.)[^/]*\.py"
                )
661
                file_members = list(
662
663
                    filter(lambda x: x.filename in files_to_copy, wheel.filelist)
                )
664
                file_members += list(
665
666
667
668
669
670
                    filter(lambda x: flash_attn_regex.match(x.filename), wheel.filelist)
                )
                file_members += list(
                    filter(
                        lambda x: triton_kernels_regex.match(x.filename), wheel.filelist
                    )
671
                )
672
673
674
675
676

                for file in file_members:
                    print(f"[extract] {file.filename}")
                    target_path = os.path.join(".", file.filename)
                    os.makedirs(os.path.dirname(target_path), exist_ok=True)
677
678
679
680
                    with (
                        wheel.open(file.filename) as src,
                        open(target_path, "wb") as dst,
                    ):
681
682
683
684
                        shutil.copyfileobj(src, dst)

                    pkg = os.path.dirname(file.filename).replace("/", ".")
                    package_data_patch.setdefault(pkg, []).append(
685
686
                        os.path.basename(file.filename)
                    )
687
688
689
690
691
692
693
694
695

            return package_data_patch
        finally:
            if temp_dir is not None:
                print(f"Removing temporary directory {temp_dir}")
                shutil.rmtree(temp_dir)

    @staticmethod
    def get_base_commit_in_main_branch() -> str:
696
        try:
697
            # Get the latest commit hash of the upstream main branch.
698
699
700
701
702
703
704
            resp_json = subprocess.check_output(
                [
                    "curl",
                    "-s",
                    "https://api.github.com/repos/vllm-project/vllm/commits/main",
                ]
            ).decode("utf-8")
705
            upstream_main_commit = json.loads(resp_json)["sha"]
706
            print(f"Upstream main branch latest commit: {upstream_main_commit}")
707

708
709
710
711
            # In Docker build context, .git may be immutable or missing.
            if envs.VLLM_DOCKER_BUILD_CONTEXT:
                return upstream_main_commit

712
713
714
            # Check if the upstream_main_commit exists in the local repo
            try:
                subprocess.check_output(
715
716
                    ["git", "cat-file", "-e", f"{upstream_main_commit}"]
                )
717
718
719
720
721
            except subprocess.CalledProcessError:
                # If not present, fetch it from the remote repository.
                # Note that this does not update any local branches,
                # but ensures that this commit ref and its history are
                # available in our local repo.
722
723
724
                subprocess.check_call(
                    ["git", "fetch", "https://github.com/vllm-project/vllm", "main"]
                )
725
726
727

            # Then get the commit hash of the current branch that is the same as
            # the upstream main commit.
728
729
730
731
732
            current_branch = (
                subprocess.check_output(["git", "branch", "--show-current"])
                .decode("utf-8")
                .strip()
            )
733

734
735
736
737
738
739
740
            base_commit = (
                subprocess.check_output(
                    ["git", "merge-base", f"{upstream_main_commit}", current_branch]
                )
                .decode("utf-8")
                .strip()
            )
741
            return base_commit
742
743
        except ValueError as err:
            raise ValueError(err) from None
744
745
746
747
        except Exception as err:
            logger.warning(
                "Failed to get the base commit in the main branch. "
                "Using the nightly wheel. The libraries in this "
748
749
750
                "wheel may not be compatible with your dev branch: %s",
                err,
            )
751
            return "nightly"
752
753


754
755
756
757
def _no_device() -> bool:
    return VLLM_TARGET_DEVICE == "empty"


758
def _is_cuda() -> bool:
759
    has_cuda = torch.version.cuda is not None
760
    return VLLM_TARGET_DEVICE == "cuda" and has_cuda and not _is_tpu()
761
762
763


def _is_hip() -> bool:
764
765
766
    return (
        VLLM_TARGET_DEVICE == "cuda" or VLLM_TARGET_DEVICE == "rocm"
    ) and torch.version.hip is not None
767
768


769
770
771
772
def _is_tpu() -> bool:
    return VLLM_TARGET_DEVICE == "tpu"


773
774
def _is_cpu() -> bool:
    return VLLM_TARGET_DEVICE == "cpu"
775
776


777
778
779
780
def _is_xpu() -> bool:
    return VLLM_TARGET_DEVICE == "xpu"


781
def _build_custom_ops() -> bool:
zhuwenwen's avatar
zhuwenwen committed
782
783
    if not skip_vllm_build:
        return _is_cuda() or _is_hip() or _is_cpu()
784
785


786
787
788
789
def get_rocm_version():
    # Get the Rocm version from the ROCM_HOME/bin/librocm-core.so
    # see https://github.com/ROCm/rocm-core/blob/d11f5c20d500f729c393680a01fa902ebf92094b/rocm_version.cpp#L21
    try:
790
791
        if ROCM_HOME is None:
            return None
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
        librocm_core_file = Path(ROCM_HOME) / "lib" / "librocm-core.so"
        if not librocm_core_file.is_file():
            return None
        librocm_core = ctypes.CDLL(librocm_core_file)
        VerErrors = ctypes.c_uint32
        get_rocm_core_version = librocm_core.getROCmVersion
        get_rocm_core_version.restype = VerErrors
        get_rocm_core_version.argtypes = [
            ctypes.POINTER(ctypes.c_uint32),
            ctypes.POINTER(ctypes.c_uint32),
            ctypes.POINTER(ctypes.c_uint32),
        ]
        major = ctypes.c_uint32()
        minor = ctypes.c_uint32()
        patch = ctypes.c_uint32()
807

808
809
810
811
812
813
        if (
            get_rocm_core_version(
                ctypes.byref(major), ctypes.byref(minor), ctypes.byref(patch)
            )
            == 0
        ):
814
            return f"{major.value}.{minor.value}.{patch.value}"
815
        return None
816
    except Exception:
817
        return None
Woosuk Kwon's avatar
Woosuk Kwon committed
818

819

bnellnm's avatar
bnellnm committed
820
def get_nvcc_cuda_version() -> Version:
821
822
823
824
    """Get the CUDA version from nvcc.

    Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py
    """
825
    assert CUDA_HOME is not None, "CUDA_HOME is not set"
826
827
828
    nvcc_output = subprocess.check_output(
        [CUDA_HOME + "/bin/nvcc", "-V"], universal_newlines=True
    )
829
830
831
832
833
834
    output = nvcc_output.split()
    release_idx = output.index("release") + 1
    nvcc_cuda_version = parse(output[release_idx].split(",")[0])
    return nvcc_cuda_version


zhuwenwen's avatar
zhuwenwen committed
835
836
837
838
839
840
def get_sha(root: Union[str, Path]) -> str:
    try:
        return subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=root).decode('ascii').strip()
    except Exception:
        return 'Unknown'

841

zhuwenwen's avatar
zhuwenwen committed
842
def get_version_add(sha: Optional[str] = None) -> str:
zhuwenwen's avatar
zhuwenwen committed
843
    command = "git config --global --add safe.directory "+pwd  
zhuwenwen's avatar
zhuwenwen committed
844
    subprocess.run(command, shell=True, capture_output=False, text=True) 
zhuwenwen's avatar
zhuwenwen committed
845
846
    vllm_root = os.path.dirname(os.path.abspath(__file__))
    add_version_path = os.path.join(os.path.join(vllm_root, "vllm"), "version.py")
zhuwenwen's avatar
zhuwenwen committed
847
    major, minor, _ = torch.__version__.split('.')
848
849
850
851
    if add_git_version:
        if sha != 'Unknown':
            if sha is None:
                sha = get_sha(vllm_root)
zhuwenwen's avatar
zhuwenwen committed
852
            version = 'das.opt1.alpha.' + sha[:7]
853
    else:
zhuwenwen's avatar
zhuwenwen committed
854
        version = 'das.opt1.alpha'
855

zhuwenwen's avatar
zhuwenwen committed
856
857
858
859
860
861
862

    # dtk version
    if os.getenv("ROCM_PATH"):
        rocm_path = os.getenv('ROCM_PATH', "")
        rocm_version_path = os.path.join(rocm_path, '.info', "rocm_version")
        with open(rocm_version_path, 'r',encoding='utf-8') as file:
            lines = file.readlines()
zhuwenwen's avatar
zhuwenwen committed
863
        rocm_version=lines[0].replace(".", "")
zhuwenwen's avatar
zhuwenwen committed
864
        version += ".dtk" + rocm_version
zhuwenwen's avatar
zhuwenwen committed
865
    
866
    new_version_content = f"""
867
try:
zhuwenwen's avatar
zhuwenwen committed
868
869
    __version__ = "0.14.1"
    __version_tuple__ = (0, 14, 1)
zhuwenwen's avatar
zhuwenwen committed
870
    __hcu_version__ = f'0.14.1+{version}' 
zhuwenwen's avatar
zhuwenwen committed
871
    
zhuwenwen's avatar
zhuwenwen committed
872
    from vllm.version import __version__, __version_tuple__, __hcu_version__
873
except Exception as e:
zhuwenwen's avatar
zhuwenwen committed
874
875
    import warnings

zhuwenwen's avatar
zhuwenwen committed
876
    warnings.warn(f"Failed to read commit hash:\\n + str(e)",
877
878
                  RuntimeWarning,
                  stacklevel=2)
879
880
    __version__ = "dev"
    __version_tuple__ = (0, 0, __version__)
zhuwenwen's avatar
zhuwenwen committed
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
    
    
def _prev_minor_version_was(version_str):
    '''Check whether a given version matches the previous minor version.

    Return True if version_str matches the previous minor version.

    For example - return True if the current version if 0.7.4 and the
    supplied version_str is '0.6'.

    Used for --show-hidden-metrics-for-version.
    '''
    # Match anything if this is a dev tree
    if __version_tuple__[0:2] == (0, 0):
        return True

    # Note - this won't do the right thing when we release 1.0!
898
    # assert __version_tuple__[0] == 0
zhuwenwen's avatar
zhuwenwen committed
899
900
901
902
903
904
905
906
907
    assert isinstance(__version_tuple__[1], int)
    return version_str == f"{{__version_tuple__[0]}}.{{__version_tuple__[1] - 1}}"


def _prev_minor_version():
    '''For the purpose of testing, return a previous minor version number.'''
    # In dev tree, this will return "0.-1", but that will work fine"
    assert isinstance(__version_tuple__[1], int)
    return f"{{__version_tuple__[0]}}.{{__version_tuple__[1] - 1}}"
908
"""
zhuwenwen's avatar
zhuwenwen committed
909
    
zhuwenwen's avatar
zhuwenwen committed
910
    with open(add_version_path, encoding="utf-8",mode="w") as file:
zhuwenwen's avatar
zhuwenwen committed
911
        file.write(new_version_content)
zhuwenwen's avatar
zhuwenwen committed
912
913
914
915
916
917
918
919
    file.close()
    
    
def get_version():
    get_version_add()
    version_file = 'vllm/version.py'
    with open(version_file, encoding='utf-8') as f:
        exec(compile(f.read(), version_file, 'exec'))
zhuwenwen's avatar
zhuwenwen committed
920
    return locals()['__hcu_version__']
zhuwenwen's avatar
zhuwenwen committed
921
922


923
924
925
926
927
928
929
930
def get_gaudi_sw_version():
    """
    Returns the driver version.
    """
    # Enable console printing for `hl-smi` check
    output = subprocess.run("hl-smi",
                            shell=True,
                            text=True,
931
                            capture_output=True,
932
933
934
935
936
937
938
                            env={"ENABLE_CONSOLE": "true"})
    if output.returncode == 0 and output.stdout:
        return output.stdout.split("\n")[2].replace(
            " ", "").split(":")[1][:-1].split("-")[0]
    return "0.0.0"  # when hl-smi is not available


939
def get_vllm_version() -> str:
940
941
942
    # Allow overriding the version. This is useful to build platform-specific
    # wheels (e.g. CPU, TPU) without modifying the source.
    if env_version := os.getenv("VLLM_VERSION_OVERRIDE"):
943
944
945
        print(f"Overriding VLLM version with {env_version} from VLLM_VERSION_OVERRIDE")
        os.environ["SETUPTOOLS_SCM_PRETEND_VERSION"] = env_version
        return get_version(write_to="vllm/_version.py")
946

947
948
    # version = get_version(write_to="vllm/_version.py")
    # sep = "+" if "+" not in version else "."  # dev versions might contain +
949

950
    if _no_device():
951
        if envs.VLLM_TARGET_DEVICE == "empty":
952
            version += f"{sep}empty"
953
    elif _is_cuda():
954
        if envs.VLLM_USE_PRECOMPILED and not envs.VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX:
955
            version += f"{sep}precompiled"
956
957
        else:
            cuda_version = str(get_nvcc_cuda_version())
958
            if cuda_version != envs.VLLM_MAIN_CUDA_VERSION:
959
960
961
962
                cuda_version_str = cuda_version.replace(".", "")[:3]
                # skip this for source tarball, required for pypi
                if "sdist" not in sys.argv:
                    version += f"{sep}cu{cuda_version_str}"
963
    elif _is_hip():
964
        # Get the Rocm Version
zhuwenwen's avatar
zhuwenwen committed
965
        # rocm_version = get_rocm_version() or torch.version.hip
966
        # if rocm_version and rocm_version != envs.VLLM_MAIN_CUDA_VERSION:
zhuwenwen's avatar
zhuwenwen committed
967
        #     version += f"{sep}rocm{rocm_version.replace('.', '')[:3]}"
zhuwenwen's avatar
zhuwenwen committed
968
        version = get_version()
969
    elif _is_tpu():
970
        version += f"{sep}tpu"
971
    elif _is_cpu():
972
973
        if envs.VLLM_TARGET_DEVICE == "cpu":
            version += f"{sep}cpu"
974
    elif _is_xpu():
975
        version += f"{sep}xpu"
976
    else:
977
        raise RuntimeError("Unknown runtime environment")
978

979
980
981
    return version


982
def get_requirements() -> list[str]:
983
    """Get Python package dependencies from requirements.txt."""
984
    requirements_dir = ROOT_DIR / "requirements"
985

986
    def _read_requirements(filename: str) -> list[str]:
987
        with open(requirements_dir / filename) as f:
988
            requirements = f.read().strip().split("\n")
989
990
991
992
        resolved_requirements = []
        for line in requirements:
            if line.startswith("-r "):
                resolved_requirements += _read_requirements(line.split()[1])
993
994
995
996
997
            elif (
                not line.startswith("--")
                and not line.startswith("#")
                and line.strip() != ""
            ):
998
999
1000
                resolved_requirements.append(line)
        return resolved_requirements

1001
    if _no_device():
1002
        requirements = _read_requirements("common.txt")
1003
    elif _is_cuda():
1004
        requirements = _read_requirements("cuda.txt")
1005
        cuda_major, cuda_minor = torch.version.cuda.split(".")
1006
1007
        modified_requirements = []
        for req in requirements:
1008
            if "vllm-flash-attn" in req and cuda_major != "12":
1009
                # vllm-flash-attn is built only for CUDA 12.x.
1010
1011
1012
                # Skip for other versions.
                continue
            modified_requirements.append(req)
1013
        requirements = modified_requirements
1014
    elif _is_hip():
1015
        requirements = _read_requirements("rocm.txt")
1016
    elif _is_tpu():
1017
        requirements = _read_requirements("tpu.txt")
1018
    elif _is_cpu():
1019
        requirements = _read_requirements("cpu.txt")
1020
    elif _is_xpu():
1021
        requirements = _read_requirements("xpu.txt")
1022
    else:
1023
        raise ValueError("Unsupported platform, please use CUDA, ROCm, or CPU.")
1024
1025
1026
    return requirements


bnellnm's avatar
bnellnm committed
1027
1028
ext_modules = []

1029
if _is_cuda() or _is_hip():
bnellnm's avatar
bnellnm committed
1030
    ext_modules.append(CMakeExtension(name="vllm._moe_C"))
1031
    ext_modules.append(CMakeExtension(name="vllm.cumem_allocator"))
1032
1033
    # Optional since this doesn't get built (produce an .so file). This is just
    # copying the relevant .py files from the source repository.
1034
    # ext_modules.append(CMakeExtension(name="vllm.triton_kernels", optional=True))
bnellnm's avatar
bnellnm committed
1035

1036
1037
# if _is_hip():
#     ext_modules.append(CMakeExtension(name="vllm._rocm_C"))
1038

1039
if _is_cuda():
1040
    ext_modules.append(CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa2_C"))
1041
1042
1043
    if envs.VLLM_USE_PRECOMPILED or (
        CUDA_HOME and get_nvcc_cuda_version() >= Version("12.3")
    ):
1044
        # FA3 requires CUDA 12.3 or later
1045
        ext_modules.append(CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa3_C"))
1046
1047
        # Optional since this doesn't get built (produce an .so file) when
        # not targeting a hopper system
1048
        ext_modules.append(CMakeExtension(name="vllm._flashmla_C", optional=True))
1049
        ext_modules.append(
1050
1051
            CMakeExtension(name="vllm._flashmla_extension_C", optional=True)
        )
1052

1053
if _build_custom_ops():
bnellnm's avatar
bnellnm committed
1054
1055
    ext_modules.append(CMakeExtension(name="vllm._C"))

zhuwenwen's avatar
zhuwenwen committed
1056
1057
1058
1059
1060
1061
if skip_vllm_build:
    package_data = {
        "vllm": [
            "py.typed",
            "model_executor/layers/fused_moe/configs/*.json",
            "model_executor/layers/quantization/utils/configs/*.json",
1062
1063
            "entrypoints/serve/instrumentator/static/*.js",
            "entrypoints/serve/instrumentator/static/*.css",
zhuwenwen's avatar
zhuwenwen committed
1064
            "model_executor/layers/quantization/configs/awq/*.json",
1065
            "attention/backends/configs/*.json",
zhuwenwen's avatar
zhuwenwen committed
1066
1067
            "_C.abi3.so",
            "_moe_C.abi3.so",
zhuwenwen's avatar
zhuwenwen committed
1068
1069
1070
1071
1072
1073
1074
1075
        ]
    }
else:
    package_data = {
        "vllm": [
            "py.typed",
            "model_executor/layers/fused_moe/configs/*.json",
            "model_executor/layers/quantization/utils/configs/*.json",
1076
1077
            "entrypoints/serve/instrumentator/static/*.js",
            "entrypoints/serve/instrumentator/static/*.css",
zhuwenwen's avatar
zhuwenwen committed
1078
1079
1080
1081
1082
            "attention/backends/configs/*.json",
            "model_executor/layers/quantization/configs/awq/*.json",
        ]
    }

1083

1084
1085
# If using precompiled, extract and patch package_data (in advance of setup)
if envs.VLLM_USE_PRECOMPILED:
1086
    wheel_url, download_filename = precompiled_wheel_utils.determine_wheel_url()
1087
    patch = precompiled_wheel_utils.extract_precompiled_and_patch_package(
1088
1089
        wheel_url, download_filename
    )
1090
1091
1092
    for pkg, files in patch.items():
        package_data.setdefault(pkg, []).extend(files)

zhuwenwen's avatar
zhuwenwen committed
1093
if _no_device() or skip_vllm_build:
1094
1095
    ext_modules = []

1096
if not ext_modules:
1097
1098
1099
1100
    cmdclass = {
        "build_py": BuildPyAndGenerateGrpc,
        "develop": DevelopAndGenerateGrpc,
    }
1101
1102
else:
    cmdclass = {
1103
1104
        "build_ext": precompiled_build_ext
        if envs.VLLM_USE_PRECOMPILED
1105
1106
        else cmake_build_ext,
        "build_py": BuildPyAndGenerateGrpc,
1107
        "develop": DevelopAndGenerateGrpc,
1108
1109
    }

bnellnm's avatar
bnellnm committed
1110
setup(
1111
    # static metadata should rather go in pyproject.toml
1112
    version=get_vllm_version(),
Woosuk Kwon's avatar
Woosuk Kwon committed
1113
    ext_modules=ext_modules,
1114
    install_requires=get_requirements(),
1115
    extras_require={
1116
        "bench": ["pandas", "matplotlib", "seaborn", "datasets", "scipy"],
1117
        "tensorizer": ["tensorizer==2.10.1"],
1118
        "fastsafetensors": ["fastsafetensors >= 0.1.10"],
1119
        "runai": ["runai-model-streamer[s3,gcs] >= 0.15.3"],
1120
1121
        "audio": [
            "librosa",
1122
            "scipy",
1123
1124
1125
            "soundfile",
            "mistral_common[audio]",
        ],  # Required for audio processing
1126
        "video": [],  # Kept for backwards compatibility
1127
        "flashinfer": [],  # Kept for backwards compatibility
1128
1129
        # Optional deps for AMD FP4 quantization support
        "petit-kernel": ["petit-kernel"],
1130
    },
1131
    cmdclass=cmdclass,
Simon Mo's avatar
Simon Mo committed
1132
    package_data=package_data,
Woosuk Kwon's avatar
Woosuk Kwon committed
1133
)