Dockerfile 34.9 KB
Newer Older
Simon Mo's avatar
Simon Mo committed
1
2
3
# The vLLM Dockerfile is used to construct vLLM image that can be directly used
# to run the OpenAI compatible server.

4
# Please update any changes made here to
5
6
# docs/contributing/dockerfile/dockerfile.md and
# docs/assets/contributing/dockerfile-stages-dependency.png
7

8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# =============================================================================
# VERSION MANAGEMENT
# =============================================================================
# ARG defaults in this Dockerfile are the source of truth for pinned versions.
# docker/versions.json is auto-generated for use with docker buildx bake.
#
# When updating versions:
# 1. Edit the ARG defaults below
# 2. Run: python tools/generate_versions_json.py
#
# To query versions programmatically:
#   jq -r '.variable.CUDA_VERSION.default' docker/versions.json
#
# To build with bake:
#   docker buildx bake -f docker/docker-bake.hcl -f docker/versions.json
# =============================================================================

25
ARG CUDA_VERSION=12.9.1
26
27
28
29
30
31
32
ARG PYTHON_VERSION=3.12

# By parameterizing the base images, we allow third-party to use their own
# base images. One use case is hermetic builds with base images stored in
# private registries that use a different repository naming conventions.
#
# Example:
33
34
# docker build --build-arg BUILD_BASE_IMAGE=registry.acme.org/mirror/nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04

35
# Important: We build with an old version of Ubuntu to maintain broad
36
37
38
39
# compatibility with other Linux OSes. The main reason for this is that the
# glibc version is baked into the distro, and binaries built with one glibc
# version are not backwards compatible with OSes that use an earlier version.
ARG BUILD_BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04
40
41
# Using cuda base image with minimal dependencies necessary for JIT compilation (FlashInfer, DeepGEMM, EP kernels)
ARG FINAL_BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-base-ubuntu22.04
42
43
44
45
46
47
48
49
50
51

# By parameterizing the Deadsnakes repository URL, we allow third-party to use
# their own mirror. When doing so, we don't benefit from the transparent
# installation of the GPG key of the PPA, as done by add-apt-repository, so we
# also need a URL for the GPG key.
ARG DEADSNAKES_MIRROR_URL
ARG DEADSNAKES_GPGKEY_URL

# The PyPA get-pip.py script is a self contained script+zip file, that provides
# both the installer script and the pip base85-encoded zip archive. This allows
52
# bootstrapping pip in environment where a distribution package does not exist.
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
#
# By parameterizing the URL for get-pip.py installation script, we allow
# third-party to use their own copy of the script stored in a private mirror.
# We set the default value to the PyPA owned get-pip.py script.
#
# Reference: https://pip.pypa.io/en/stable/installation/#get-pip-py
ARG GET_PIP_URL="https://bootstrap.pypa.io/get-pip.py"

# PIP supports fetching the packages from custom indexes, allowing third-party
# to host the packages in private mirrors. The PIP_INDEX_URL and
# PIP_EXTRA_INDEX_URL are standard PIP environment variables to override the
# default indexes. By letting them empty by default, PIP will use its default
# indexes if the build process doesn't override the indexes.
#
# Uv uses different variables. We set them by default to the same values as
# PIP, but they can be overridden.
ARG PIP_INDEX_URL
ARG PIP_EXTRA_INDEX_URL
ARG UV_INDEX_URL=${PIP_INDEX_URL}
ARG UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}

# PyTorch provides its own indexes for standard and nightly builds
ARG PYTORCH_CUDA_INDEX_BASE_URL=https://download.pytorch.org/whl

# PIP supports multiple authentication schemes, including keyring
# By parameterizing the PIP_KEYRING_PROVIDER variable and setting it to
# disabled by default, we allow third-party to use keyring authentication for
# their private Python indexes, while not changing the default behavior which
# is no authentication.
#
# Reference: https://pip.pypa.io/en/stable/topics/authentication/#keyring-support
ARG PIP_KEYRING_PROVIDER=disabled
ARG UV_KEYRING_PROVIDER=${PIP_KEYRING_PROVIDER}

87
# Flag enables built-in KV-connector dependency libs into docker images
88
89
ARG INSTALL_KV_CONNECTORS=false

Simon Mo's avatar
Simon Mo committed
90
#################### BASE BUILD IMAGE ####################
91
# prepare basic build environment
92
FROM ${BUILD_BASE_IMAGE} AS base
93

94
95
ARG CUDA_VERSION
ARG PYTHON_VERSION
96

97
ENV DEBIAN_FRONTEND=noninteractive
98

99
# Install system dependencies including build tools
100
RUN apt-get update -y \
101
102
103
104
105
106
107
108
109
110
111
112
113
114
    && apt-get install -y --no-install-recommends \
        ccache \
        software-properties-common \
        git \
        curl \
        sudo \
        python3-pip \
        libibverbs-dev \
        # Upgrade to GCC 10 to avoid https://gcc.gnu.org/bugzilla/show_bug.cgi?id=92519
        # as it was causing spam when compiling the CUTLASS kernels
        gcc-10 \
        g++-10 \
    && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 110 --slave /usr/bin/g++ g++ /usr/bin/g++-10 \
    && rm -rf /var/lib/apt/lists/* \
115
116
117
118
119
120
    && curl -LsSf https://astral.sh/uv/install.sh | sh \
    && $HOME/.local/bin/uv venv /opt/venv --python ${PYTHON_VERSION} \
    && rm -f /usr/bin/python3 /usr/bin/python3-config /usr/bin/pip \
    && ln -s /opt/venv/bin/python3 /usr/bin/python3 \
    && ln -s /opt/venv/bin/python3-config /usr/bin/python3-config \
    && ln -s /opt/venv/bin/pip /usr/bin/pip \
121
    && python3 --version && python3 -m pip --version
122

123
124
125
# Activate virtual environment and add uv to PATH
ENV PATH="/opt/venv/bin:/root/.local/bin:$PATH"
ENV VIRTUAL_ENV="/opt/venv"
Stephen Krider's avatar
Stephen Krider committed
126

127
# Environment for uv
128
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
129
ENV UV_INDEX_STRATEGY="unsafe-best-match"
130
ENV UV_LINK_MODE=copy
131

132
133
# Verify GCC version
RUN gcc --version
134

135
136
137
138
# Enable CUDA forward compatibility by setting '-e VLLM_ENABLE_CUDA_COMPATIBILITY=1'
# Only needed for datacenter/professional GPUs with older drivers.
# See: https://docs.nvidia.com/deploy/cuda-compatibility/
ENV VLLM_ENABLE_CUDA_COMPATIBILITY=0
139

140
141
142
143
144
145
146
147
148
# ============================================================
# SLOW-CHANGING DEPENDENCIES BELOW
# These are the expensive layers that we want to cache
# ============================================================

# Install PyTorch and core CUDA dependencies
# This is ~2GB and rarely changes
ARG PYTORCH_CUDA_INDEX_BASE_URL

Stephen Krider's avatar
Stephen Krider committed
149
150
WORKDIR /workspace

151
152
153
154
155
# We can specify the standard or nightly build of PyTorch
ARG PYTORCH_NIGHTLY

# Install build and runtime dependencies, including PyTorch
# Check whether to install torch nightly instead of release for this build
156
157
COPY requirements/common.txt requirements/common.txt
COPY requirements/cuda.txt requirements/cuda.txt
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
COPY use_existing_torch.py use_existing_torch.py
COPY pyproject.toml pyproject.toml
RUN --mount=type=cache,target=/root/.cache/uv \
    if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
        echo "Installing torch nightly..." \
        && uv pip install --python /opt/venv/bin/python3 torch torchaudio torchvision --pre \
        --index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
        && echo "Installing other requirements..." \
        && /opt/venv/bin/python3 use_existing_torch.py --prefix \
        && uv pip install --python /opt/venv/bin/python3 -r requirements/cuda.txt \
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \
    else \
        uv pip install --python /opt/venv/bin/python3 -r requirements/cuda.txt \
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \
    fi

# Track PyTorch lib versions used during build and match in downstream instances.
# We do this for both nightly and release so we can strip dependencies/*.txt as needed.
# Otherwise library dependencies can upgrade/downgrade torch incorrectly.
177
RUN --mount=type=cache,target=/root/.cache/uv \
178
179
180
    uv pip freeze | grep -i "^torch=\|^torchvision=\|^torchaudio=" > torch_lib_versions.txt \
    && TORCH_LIB_VERSIONS=$(cat torch_lib_versions.txt | xargs) \
    && echo "Installed torch libs: ${TORCH_LIB_VERSIONS}"
181

182
183
184
# CUDA arch list used by torch
# Explicitly set the list to avoid issues with torch 2.2
# See https://github.com/pytorch/pytorch/pull/123243
185
# From versions.json: .torch.cuda_arch_list
186
ARG torch_cuda_arch_list='7.0 7.5 8.0 8.9 9.0 10.0 12.0'
187
ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
188
#################### BUILD BASE IMAGE ####################
Simon Mo's avatar
Simon Mo committed
189

190
191
#################### CSRC BUILD IMAGE ####################
FROM base AS csrc-build
192
ARG TARGETPLATFORM
193

194
195
196
197
ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL

198
199
200
201
# We can specify the standard or nightly build of PyTorch
ARG PYTORCH_NIGHTLY

# Install build dependencies
202
COPY requirements/build.txt requirements/build.txt
203
204
COPY use_existing_torch.py use_existing_torch.py
COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt
205

206
207
208
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
209
ENV UV_INDEX_STRATEGY="unsafe-best-match"
210
211
# Use copy mode to avoid hardlink failures with Docker cache mounts
ENV UV_LINK_MODE=copy
212

213
RUN --mount=type=cache,target=/root/.cache/uv \
214
215
216
217
218
219
220
221
222
223
224
225
    if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
        echo "Installing build requirements without torch..." \
        && python3 use_existing_torch.py --prefix \
        && uv pip install --python /opt/venv/bin/python3 -r requirements/build.txt \
        && echo "Installing torch nightly..." \
        && uv pip install --python /opt/venv/bin/python3 $(cat torch_lib_versions.txt | grep -i "^torch=" | xargs) --pre \
        --index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \
    else \
        echo "Installing build requirements..." \
        && uv pip install --python /opt/venv/bin/python3 -r requirements/build.txt \
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \
    fi
226

227
228
229
230
231
232
233
WORKDIR /workspace

COPY pyproject.toml setup.py CMakeLists.txt ./
COPY cmake cmake/
COPY csrc csrc/
COPY vllm/envs.py vllm/envs.py
COPY vllm/__init__.py vllm/__init__.py
Stephen Krider's avatar
Stephen Krider committed
234
235

# max jobs used by Ninja to build extensions
236
237
ARG max_jobs=2
ENV MAX_JOBS=${max_jobs}
238
239
240
# number of threads used by nvcc
ARG nvcc_threads=8
ENV NVCC_THREADS=$nvcc_threads
241

242
ARG USE_SCCACHE
243
ARG SCCACHE_DOWNLOAD_URL
244
ARG SCCACHE_ENDPOINT
245
246
ARG SCCACHE_BUCKET_NAME=vllm-build-sccache
ARG SCCACHE_REGION_NAME=us-west-2
247
ARG SCCACHE_S3_NO_CREDENTIALS=0
248
249

# Flag to control whether to use pre-built vLLM wheels
250
ARG VLLM_USE_PRECOMPILED=""
251
ARG VLLM_MERGE_BASE_COMMIT=""
252
ARG VLLM_MAIN_CUDA_VERSION=""
253

254
255
256
# Use dummy version for csrc-build wheel (only .so files are extracted, version doesn't matter)
ENV SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+csrc.build"

257
258
259
260
261
262
263
# Use existing torch for nightly builds
RUN --mount=type=cache,target=/root/.cache/uv \
    if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
        python3 use_existing_torch.py --prefix; \
    fi

# Build the vLLM wheel
264
# if USE_SCCACHE is set, use sccache to speed up compilation
265
RUN --mount=type=cache,target=/root/.cache/uv \
266
267
    if [ "$USE_SCCACHE" = "1" ]; then \
        echo "Installing sccache..." \
268
269
270
271
272
273
        && case "${TARGETPLATFORM}" in \
          linux/arm64) SCCACHE_ARCH="aarch64" ;; \
          linux/amd64) SCCACHE_ARCH="x86_64" ;; \
          *) echo "Unsupported TARGETPLATFORM for sccache: ${TARGETPLATFORM}" >&2; exit 1 ;; \
        esac \
        && export SCCACHE_DOWNLOAD_URL="${SCCACHE_DOWNLOAD_URL:-https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl.tar.gz}" \
274
        && curl -L -o sccache.tar.gz ${SCCACHE_DOWNLOAD_URL} \
275
        && tar -xzf sccache.tar.gz \
276
277
        && sudo mv sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl/sccache /usr/bin/sccache \
        && rm -rf sccache.tar.gz sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl \
278
        && if [ ! -z ${SCCACHE_ENDPOINT} ] ; then export SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT} ; fi \
279
280
        && export SCCACHE_BUCKET=${SCCACHE_BUCKET_NAME} \
        && export SCCACHE_REGION=${SCCACHE_REGION_NAME} \
281
        && export SCCACHE_S3_NO_CREDENTIALS=${SCCACHE_S3_NO_CREDENTIALS} \
282
        && export SCCACHE_IDLE_TIMEOUT=0 \
283
        && export CMAKE_BUILD_TYPE=Release \
284
        && export VLLM_USE_PRECOMPILED="${VLLM_USE_PRECOMPILED}" \
285
        && export VLLM_PRECOMPILED_WHEEL_COMMIT="${VLLM_MERGE_BASE_COMMIT}" \
286
        && export VLLM_MAIN_CUDA_VERSION="${VLLM_MAIN_CUDA_VERSION}" \
287
        && export VLLM_DOCKER_BUILD_CONTEXT=1 \
288
        && sccache --show-stats \
289
        && python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38 \
290
291
292
        && sccache --show-stats; \
    fi

293
294
ARG vllm_target_device="cuda"
ENV VLLM_TARGET_DEVICE=${vllm_target_device}
295
296
ENV CCACHE_DIR=/root/.cache/ccache
RUN --mount=type=cache,target=/root/.cache/ccache \
297
    --mount=type=cache,target=/root/.cache/uv \
298
    if [ "$USE_SCCACHE" != "1" ]; then \
299
300
301
        # Clean any existing CMake artifacts
        rm -rf .deps && \
        mkdir -p .deps && \
302
        export VLLM_USE_PRECOMPILED="${VLLM_USE_PRECOMPILED}" && \
303
        export VLLM_PRECOMPILED_WHEEL_COMMIT="${VLLM_MERGE_BASE_COMMIT}" && \
304
        export VLLM_DOCKER_BUILD_CONTEXT=1 && \
305
        python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38; \
306
    fi
307

308
309
#################### CSRC BUILD IMAGE ####################

310
#################### EXTENSIONS BUILD IMAGE ####################
311
# Build DeepGEMM, DeepEP - runs in PARALLEL with csrc-build
312
313
314
315
316
317
318
319
320
321
322
323
# This stage is independent and doesn't affect csrc cache
FROM base AS extensions-build
ARG CUDA_VERSION

# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
ENV UV_HTTP_TIMEOUT=500
ENV UV_INDEX_STRATEGY="unsafe-best-match"
ENV UV_LINK_MODE=copy

WORKDIR /workspace

# Build DeepGEMM wheel
324
# Default moved here from tools/install_deepgemm.sh for centralized version management
325
ARG DEEPGEMM_GIT_REF=477618cd51baffca09c4b0b87e97c03fe827ef03
326
327
328
329
330
331
332
333
334
335
336
337
COPY tools/install_deepgemm.sh /tmp/install_deepgemm.sh
RUN --mount=type=cache,target=/root/.cache/uv \
    mkdir -p /tmp/deepgemm/dist && \
    VLLM_DOCKER_BUILD_CONTEXT=1 TORCH_CUDA_ARCH_LIST="9.0a 10.0a" /tmp/install_deepgemm.sh \
        --cuda-version "${CUDA_VERSION}" \
        ${DEEPGEMM_GIT_REF:+--ref "$DEEPGEMM_GIT_REF"} \
        --wheel-dir /tmp/deepgemm/dist || \
    echo "DeepGEMM build skipped (CUDA version requirement not met)"

# Ensure the wheel dir exists so COPY won't fail when DeepGEMM is skipped
RUN mkdir -p /tmp/deepgemm/dist && touch /tmp/deepgemm/dist/.deepgemm_skipped

338
# Build DeepEP wheels
339
COPY tools/ep_kernels/install_python_libraries.sh /tmp/install_python_libraries.sh
340
341
# Defaults moved here from tools/ep_kernels/install_python_libraries.sh for centralized version management
ARG DEEPEP_COMMIT_HASH=73b6ea4
342
ARG NVSHMEM_VER
343
344
345
346
347
348
RUN --mount=type=cache,target=/root/.cache/uv \
    mkdir -p /tmp/ep_kernels_workspace/dist && \
    export TORCH_CUDA_ARCH_LIST='9.0a 10.0a' && \
    /tmp/install_python_libraries.sh \
        --workspace /tmp/ep_kernels_workspace \
        --mode wheel \
349
350
        ${DEEPEP_COMMIT_HASH:+--deepep-ref "$DEEPEP_COMMIT_HASH"} \
        ${NVSHMEM_VER:+--nvshmem-ver "$NVSHMEM_VER"} && \
351
352
353
    find /tmp/ep_kernels_workspace/nvshmem -name '*.a' -delete
#################### EXTENSIONS BUILD IMAGE ####################

354
355
356
357
358
359
360
361
#################### WHEEL BUILD IMAGE ####################
FROM base AS build
ARG TARGETPLATFORM

ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL

362
363
364
365
# We can specify the standard or nightly build of PyTorch
ARG PYTORCH_NIGHTLY

# Install build dependencies
366
COPY requirements/build.txt requirements/build.txt
367
368
COPY use_existing_torch.py use_existing_torch.py
COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt
369
370
371
372
373
374
375
376
377

# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
ENV UV_INDEX_STRATEGY="unsafe-best-match"
# Use copy mode to avoid hardlink failures with Docker cache mounts
ENV UV_LINK_MODE=copy

RUN --mount=type=cache,target=/root/.cache/uv \
378
379
380
381
382
383
384
385
386
387
388
389
    if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
        echo "Installing build requirements without torch..." \
        && python3 use_existing_torch.py --prefix \
        && uv pip install --python /opt/venv/bin/python3 -r requirements/build.txt \
        && echo "Installing torch nightly..." \
        && uv pip install --python /opt/venv/bin/python3 $(cat torch_lib_versions.txt | grep -i "^torch=" | xargs) --pre \
        --index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \
    else \
        echo "Installing build requirements..." \
        && uv pip install --python /opt/venv/bin/python3 -r requirements/build.txt \
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \
    fi
390
391
392

WORKDIR /workspace

393
# Copy pre-built csrc wheel directly
394
395
396
397
398
399
400
401
402
403
404
405
406
COPY --from=csrc-build /workspace/dist /precompiled-wheels
COPY . .

ARG GIT_REPO_CHECK=0
RUN --mount=type=bind,source=.git,target=.git \
    if [ "$GIT_REPO_CHECK" != "0" ]; then bash tools/check_repo.sh ; fi

ARG vllm_target_device="cuda"
ENV VLLM_TARGET_DEVICE=${vllm_target_device}

# Skip adding +precompiled suffix to version (preserves git-derived version)
ENV VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX=1

407
408
409
410
411
412
413
# Use existing torch for nightly builds
RUN --mount=type=cache,target=/root/.cache/uv \
    if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
        python3 use_existing_torch.py --prefix; \
    fi

# Build the vLLM wheel
414
415
416
417
418
419
RUN --mount=type=cache,target=/root/.cache/uv \
    --mount=type=bind,source=.git,target=.git \
    if [ "${vllm_target_device}" = "cuda" ]; then \
        export VLLM_PRECOMPILED_WHEEL_LOCATION=$(ls /precompiled-wheels/*.whl); \
    fi && \
    python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38
420

421
422
423
# Copy extension wheels from extensions-build stage for later use
COPY --from=extensions-build /tmp/deepgemm/dist /tmp/deepgemm/dist
COPY --from=extensions-build /tmp/ep_kernels_workspace/dist /tmp/ep_kernels_workspace/dist
424

425
# Check the size of the wheel if RUN_WHEEL_CHECK is true
426
COPY .buildkite/check-wheel-size.py check-wheel-size.py
427
# sync the default value with .buildkite/check-wheel-size.py
428
ARG VLLM_MAX_SIZE_MB=500
429
430
431
432
433
434
435
ENV VLLM_MAX_SIZE_MB=$VLLM_MAX_SIZE_MB
ARG RUN_WHEEL_CHECK=true
RUN if [ "$RUN_WHEEL_CHECK" = "true" ]; then \
        python3 check-wheel-size.py dist; \
    else \
        echo "Skipping wheel size check."; \
    fi
436
437

#################### WHEEL BUILD IMAGE ####################
Stephen Krider's avatar
Stephen Krider committed
438

439
#################### DEV IMAGE ####################
440
FROM base AS dev
441

442
443
444
445
ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL

446
447
448
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
449
ENV UV_INDEX_STRATEGY="unsafe-best-match"
450
451
# Use copy mode to avoid hardlink failures with Docker cache mounts
ENV UV_LINK_MODE=copy
Huy Do's avatar
Huy Do committed
452

453
# Install libnuma-dev, required by fastsafetensors (fixes #20384)
454
RUN apt-get update && apt-get install -y --no-install-recommends libnuma-dev && rm -rf /var/lib/apt/lists/*
455
456
457
458
459
460


# We can specify the standard or nightly build of PyTorch
ARG PYTORCH_NIGHTLY

# Install development dependencies
461
COPY requirements/lint.txt requirements/lint.txt
462
COPY requirements/test.in requirements/test.in
463
464
COPY requirements/test.txt requirements/test.txt
COPY requirements/dev.txt requirements/dev.txt
465
466
COPY use_existing_torch.py use_existing_torch.py
COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt
467
RUN --mount=type=cache,target=/root/.cache/uv \
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
    if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
        echo "Installing dev requirements plus torch nightly..." \
        && python3 use_existing_torch.py --prefix \
        && cat torch_lib_versions.txt >> requirements/test.in \
        && uv pip compile requirements/test.in -o requirements/test.txt --index-strategy unsafe-best-match \
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
        && uv pip install --python /opt/venv/bin/python3 $(cat torch_lib_versions.txt | xargs) --pre \
        -r requirements/dev.txt \
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \
    else \
        echo "Installing dev requirements..." \
        && uv pip install --python /opt/venv/bin/python3 -r requirements/dev.txt \
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \
    fi

483
#################### DEV IMAGE ####################
484
485
#################### vLLM installation IMAGE ####################
# image with vLLM installed
486
FROM ${FINAL_BASE_IMAGE} AS vllm-base
487

488
489
490
491
492
493
ARG CUDA_VERSION
ARG PYTHON_VERSION
ARG DEADSNAKES_MIRROR_URL
ARG DEADSNAKES_GPGKEY_URL
ARG GET_PIP_URL

494
495
496
497
498
ENV DEBIAN_FRONTEND=noninteractive
WORKDIR /vllm-workspace


# Python version string for paths (e.g., "312" for 3.12)
499
500
RUN PYTHON_VERSION_STR=$(echo ${PYTHON_VERSION} | sed 's/\.//g') && \
    echo "export PYTHON_VERSION_STR=${PYTHON_VERSION_STR}" >> /etc/environment
501

502
# Install Python and system dependencies
503
RUN apt-get update -y \
504
505
506
507
508
509
510
511
512
    && apt-get install -y --no-install-recommends \
        software-properties-common \
        curl \
        sudo \
        python3-pip \
        ffmpeg \
        libsm6 \
        libxext6 \
        libgl1 \
513
514
515
516
517
518
519
520
521
522
523
524
525
    && if [ ! -z ${DEADSNAKES_MIRROR_URL} ] ; then \
        if [ ! -z "${DEADSNAKES_GPGKEY_URL}" ] ; then \
            mkdir -p -m 0755 /etc/apt/keyrings ; \
            curl -L ${DEADSNAKES_GPGKEY_URL} | gpg --dearmor > /etc/apt/keyrings/deadsnakes.gpg ; \
            sudo chmod 644 /etc/apt/keyrings/deadsnakes.gpg ; \
            echo "deb [signed-by=/etc/apt/keyrings/deadsnakes.gpg] ${DEADSNAKES_MIRROR_URL} $(lsb_release -cs) main" > /etc/apt/sources.list.d/deadsnakes.list ; \
        fi ; \
    else \
        for i in 1 2 3; do \
            add-apt-repository -y ppa:deadsnakes/ppa && break || \
            { echo "Attempt $i failed, retrying in 5s..."; sleep 5; }; \
        done ; \
    fi \
526
    && apt-get update -y \
527
528
529
530
531
532
    && apt-get install -y --no-install-recommends \
        python${PYTHON_VERSION} \
        python${PYTHON_VERSION}-dev \
        python${PYTHON_VERSION}-venv \
        libibverbs-dev \
    && rm -rf /var/lib/apt/lists/* \
533
534
535
    && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \
    && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \
    && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \
536
    && curl -sS ${GET_PIP_URL} | python${PYTHON_VERSION} \
537
    && python3 --version && python3 -m pip --version
538

539
# Install CUDA development tools for runtime JIT compilation
540
541
542
543
# (FlashInfer, DeepGEMM, EP kernels all require compilation at runtime)
RUN CUDA_VERSION_DASH=$(echo $CUDA_VERSION | cut -d. -f1,2 | tr '.' '-') && \
    apt-get update -y && \
    apt-get install -y --no-install-recommends \
544
545
546
547
548
549
550
551
552
        cuda-nvcc-${CUDA_VERSION_DASH} \
        cuda-cudart-${CUDA_VERSION_DASH} \
        cuda-nvrtc-${CUDA_VERSION_DASH} \
        cuda-cuobjdump-${CUDA_VERSION_DASH} \
        libcurand-dev-${CUDA_VERSION_DASH} \
        libcublas-${CUDA_VERSION_DASH} \
        # Fixes nccl_allocator requiring nccl.h at runtime
        # https://github.com/vllm-project/vllm/blob/1336a1ea244fa8bfd7e72751cabbdb5b68a0c11a/vllm/distributed/device_communicators/pynccl_allocator.py#L22
        libnccl-dev && \
553
554
    rm -rf /var/lib/apt/lists/*

555
# Install uv for faster pip installs
556
RUN python3 -m pip install uv
557

558
# Environment for uv
559
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
560
ENV UV_INDEX_STRATEGY="unsafe-best-match"
561
ENV UV_LINK_MODE=copy
562

563
564
565
566
# Enable CUDA forward compatibility by setting '-e VLLM_ENABLE_CUDA_COMPATIBILITY=1'
# Only needed for datacenter/professional GPUs with older drivers.
# See: https://docs.nvidia.com/deploy/cuda-compatibility/
ENV VLLM_ENABLE_CUDA_COMPATIBILITY=0
567

568
569
570
571
572
573
574
575
576
577
578
579
580
581
# ============================================================
# SLOW-CHANGING DEPENDENCIES BELOW
# These are the expensive layers that we want to cache
# ============================================================

# Install PyTorch and core CUDA dependencies
# This is ~2GB and rarely changes
ARG PYTORCH_CUDA_INDEX_BASE_URL
COPY requirements/common.txt /tmp/common.txt
COPY requirements/cuda.txt /tmp/requirements-cuda.txt
RUN --mount=type=cache,target=/root/.cache/uv \
    uv pip install --system -r /tmp/requirements-cuda.txt \
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') && \
    rm /tmp/requirements-cuda.txt /tmp/common.txt
Mor Zusman's avatar
Mor Zusman committed
582

583
# Install FlashInfer pre-compiled kernel cache and binaries
584
# This is ~1.1GB and only changes when FlashInfer version bumps
585
# https://docs.flashinfer.ai/installation.html
586
# From versions.json: .flashinfer.version
587
ARG FLASHINFER_VERSION=0.6.4
588
RUN --mount=type=cache,target=/root/.cache/uv \
589
590
    uv pip install --system flashinfer-cubin==${FLASHINFER_VERSION} \
    && uv pip install --system flashinfer-jit-cache==${FLASHINFER_VERSION} \
591
592
593
        --extra-index-url https://flashinfer.ai/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
    && flashinfer show-config

594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
# ============================================================
# OPENAI API SERVER DEPENDENCIES
# Pre-install these to avoid reinstalling on every vLLM wheel rebuild
# ============================================================

# Install gdrcopy (saves ~6s per build)
# TODO (huydhn): There is no prebuilt gdrcopy package on 12.9 at the moment
ARG GDRCOPY_CUDA_VERSION=12.8
ARG GDRCOPY_OS_VERSION=Ubuntu22_04
ARG TARGETPLATFORM
COPY tools/install_gdrcopy.sh /tmp/install_gdrcopy.sh
RUN set -eux; \
    case "${TARGETPLATFORM}" in \
      linux/arm64) UUARCH="aarch64" ;; \
      linux/amd64) UUARCH="x64" ;; \
      *) echo "Unsupported TARGETPLATFORM: ${TARGETPLATFORM}" >&2; exit 1 ;; \
    esac; \
    /tmp/install_gdrcopy.sh "${GDRCOPY_OS_VERSION}" "${GDRCOPY_CUDA_VERSION}" "${UUARCH}" && \
    rm /tmp/install_gdrcopy.sh

# Install vllm-openai dependencies (saves ~2.6s per build)
# These are stable packages that don't depend on vLLM itself
616
617
618
619
620
621
# From versions.json: .bitsandbytes.x86_64, .bitsandbytes.arm64
# From versions.json: .openai_server_extras.timm, .openai_server_extras.runai_model_streamer
ARG BITSANDBYTES_VERSION_X86=0.46.1
ARG BITSANDBYTES_VERSION_ARM64=0.42.0
ARG TIMM_VERSION=">=1.0.17"
ARG RUNAI_MODEL_STREAMER_VERSION=">=0.15.3"
622
623
RUN --mount=type=cache,target=/root/.cache/uv \
    if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
624
        BITSANDBYTES_VERSION="${BITSANDBYTES_VERSION_ARM64}"; \
625
    else \
626
        BITSANDBYTES_VERSION="${BITSANDBYTES_VERSION_X86}"; \
627
628
    fi; \
    uv pip install --system accelerate hf_transfer modelscope \
629
        "bitsandbytes>=${BITSANDBYTES_VERSION}" "timm${TIMM_VERSION}" "runai-model-streamer[s3,gcs]${RUNAI_MODEL_STREAMER_VERSION}"
630
631
632
633
634
635
636
637
638
639

# ============================================================
# VLLM INSTALLATION (depends on build stage)
# ============================================================

ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL
ARG PIP_KEYRING_PROVIDER UV_KEYRING_PROVIDER

640
641
642
643
644
645
# We can specify the standard or nightly build of PyTorch
ARG PYTORCH_NIGHTLY

# Install vLLM wheel first, so that torch etc will be installed.
# Check whether to install torch nightly instead of release for this build.
COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt
646
647
RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \
    --mount=type=cache,target=/root/.cache/uv \
648
649
650
651
652
653
654
655
656
657
658
659
    if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
        echo "Installing torch nightly..." \
        && uv pip install --system $(cat torch_lib_versions.txt | xargs) --pre \
        --index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
        && echo "Installing vLLM..." \
        && uv pip install --system dist/*.whl --verbose \
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \
    else \
        echo "Installing vLLM..." \
        && uv pip install --system dist/*.whl --verbose \
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \
    fi
660

Huy Do's avatar
Huy Do committed
661
662
663
664
RUN --mount=type=cache,target=/root/.cache/uv \
. /etc/environment && \
uv pip list

665
# Install deepgemm wheel that has been built in the `build` stage
666
RUN --mount=type=cache,target=/root/.cache/uv \
667
668
669
670
671
672
673
    --mount=type=bind,from=build,source=/tmp/deepgemm/dist,target=/tmp/deepgemm/dist,ro \
    sh -c 'if ls /tmp/deepgemm/dist/*.whl >/dev/null 2>&1; then \
              uv pip install --system /tmp/deepgemm/dist/*.whl; \
           else \
              echo "No DeepGEMM wheels to install; skipping."; \
           fi'

674
# Pytorch now installs NVSHMEM, setting LD_LIBRARY_PATH
675
676
ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH

677
# Install EP kernels wheels (DeepEP) that have been built in the `build` stage
678
679
680
RUN --mount=type=bind,from=build,src=/tmp/ep_kernels_workspace/dist,target=/vllm-workspace/ep_kernels/dist \
    --mount=type=cache,target=/root/.cache/uv \
    uv pip install --system ep_kernels/dist/*.whl --verbose \
681
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
682

683
684
685
686
687
688
# CUDA image changed from /usr/local/nvidia to /usr/local/cuda in 12.8 but will
# return to /usr/local/nvidia in 13.0 to allow container providers to mount drivers
# consistently from the host (see https://github.com/vllm-project/vllm/issues/18859).
# Until then, add /usr/local/nvidia/lib64 before the image cuda path to allow override.
ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}

689
690
691
692
# Copy examples and benchmarks at the end to minimize cache invalidation
COPY examples examples
COPY benchmarks benchmarks
COPY ./vllm/collect_env.py .
693
694
695
696
697
#################### vLLM installation IMAGE ####################
#################### TEST IMAGE ####################
# image to run unit testing suite
# note that this uses vllm installed by `pip`
FROM vllm-base AS test
Stephen Krider's avatar
Stephen Krider committed
698

699
ADD . /vllm-workspace/
Stephen Krider's avatar
Stephen Krider committed
700

701
702
703
704
ARG PYTHON_VERSION

ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
705
ARG PYTORCH_CUDA_INDEX_BASE_URL
706

707
708
709
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
710
ENV UV_INDEX_STRATEGY="unsafe-best-match"
711
712
# Use copy mode to avoid hardlink failures with Docker cache mounts
ENV UV_LINK_MODE=copy
713

714
RUN apt-get update -y \
715
716
    && apt-get install -y git

717
718
719
720
721
722
723
724
725
726
# We can specify the standard or nightly build of PyTorch
ARG PYTORCH_NIGHTLY

# Install development dependencies (for testing)
COPY requirements/lint.txt requirements/lint.txt
COPY requirements/test.in requirements/test.in
COPY requirements/test.txt requirements/test.txt
COPY requirements/dev.txt requirements/dev.txt
COPY use_existing_torch.py use_existing_torch.py
COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt
727
RUN --mount=type=cache,target=/root/.cache/uv \
728
729
    CUDA_MAJOR="${CUDA_VERSION%%.*}"; \
    if [ "$CUDA_MAJOR" -ge 12 ]; then \
730
731
732
733
734
735
736
737
738
739
740
741
742
743
        if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
            echo "Installing dev requirements plus torch nightly..." \
            && python3 use_existing_torch.py --prefix \
            && cat torch_lib_versions.txt >> requirements/test.in \
            && uv pip compile requirements/test.in -o requirements/test.txt --index-strategy unsafe-best-match \
            --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
            && uv pip install --system $(cat torch_lib_versions.txt | xargs) --pre \
            -r requirements/dev.txt \
            --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \
        else \
            echo "Installing dev requirements..." \
            && uv pip install --system -r requirements/dev.txt \
            --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \
        fi \
744
    fi
745

youkaichao's avatar
youkaichao committed
746
# install development dependencies (for testing)
747
RUN --mount=type=cache,target=/root/.cache/uv \
748
    uv pip install --system -e tests/vllm_test_utils
youkaichao's avatar
youkaichao committed
749

750
# enable fast downloads from hf (for testing)
751
RUN --mount=type=cache,target=/root/.cache/uv \
752
    uv pip install --system hf_transfer
753
754
ENV HF_HUB_ENABLE_HF_TRANSFER 1

Joe Runde's avatar
Joe Runde committed
755
# Copy in the v1 package for testing (it isn't distributed yet)
756
COPY vllm/v1 /usr/local/lib/python${PYTHON_VERSION}/dist-packages/vllm/v1
Joe Runde's avatar
Joe Runde committed
757

758
759
# Source code is used in the `python_only_compile.sh` test
# We hide it inside `src/` so that this source code
760
# will not be imported by other tests
761
762
RUN mkdir src
RUN mv vllm src/vllm
763
#################### TEST IMAGE ####################
Stephen Krider's avatar
Stephen Krider committed
764

Simon Mo's avatar
Simon Mo committed
765
#################### OPENAI API SERVER ####################
766
767
# base openai image with additional requirements, for any subsequent openai-style images
FROM vllm-base AS vllm-openai-base
768
ARG TARGETPLATFORM
769
ARG INSTALL_KV_CONNECTORS=false
770
ARG CUDA_VERSION
771

772
773
774
ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL

775
776
777
778
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500

779
# install kv_connectors if requested
780
781
ARG torch_cuda_arch_list='7.0 7.5 8.0 8.9 9.0 10.0 12.0'
ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
782
RUN --mount=type=cache,target=/root/.cache/uv \
783
    --mount=type=bind,source=requirements/kv_connectors.txt,target=/tmp/kv_connectors.txt,ro \
784
785
786
787
788
789
790
    CUDA_MAJOR="${CUDA_VERSION%%.*}"; \
    CUDA_VERSION_DASH=$(echo $CUDA_VERSION | cut -d. -f1,2 | tr '.' '-'); \
    CUDA_HOME=/usr/local/cuda; \
    # lmcache requires explicit specifying CUDA_HOME
    BUILD_PKGS="libcusparse-dev-${CUDA_VERSION_DASH} \
                libcublas-dev-${CUDA_VERSION_DASH} \
                libcusolver-dev-${CUDA_VERSION_DASH}"; \
791
    if [ "$INSTALL_KV_CONNECTORS" = "true" ]; then \
792
793
794
795
796
797
798
799
800
801
802
803
        if [ "$CUDA_MAJOR" -ge 13 ]; then \
            uv pip install --system nixl-cu13; \
        fi; \
        uv pip install --system -r /tmp/kv_connectors.txt --no-build || ( \
            # if the above fails, install from source
            apt-get update -y && \
            apt-get install -y --no-install-recommends ${BUILD_PKGS} && \
            uv pip install --system -r /tmp/kv_connectors.txt --no-build-isolation && \
            apt-get purge -y ${BUILD_PKGS} && \
            # clean up -dev packages, keep runtime libraries
            rm -rf /var/lib/apt/lists/* \
        ); \
804
    fi
805

yhu422's avatar
yhu422 committed
806
807
ENV VLLM_USAGE_SOURCE production-docker-image

808
809
810
# define sagemaker first, so it is not default from `docker build`
FROM vllm-openai-base AS vllm-sagemaker

811
COPY examples/online_serving/sagemaker-entrypoint.sh .
812
813
814
815
816
RUN chmod +x sagemaker-entrypoint.sh
ENTRYPOINT ["./sagemaker-entrypoint.sh"]

FROM vllm-openai-base AS vllm-openai

817
ENTRYPOINT ["vllm", "serve"]
Simon Mo's avatar
Simon Mo committed
818
#################### OPENAI API SERVER ####################