Dockerfile 24.6 KB
Newer Older
Simon Mo's avatar
Simon Mo committed
1
2
3
# The vLLM Dockerfile is used to construct vLLM image that can be directly used
# to run the OpenAI compatible server.

4
# Please update any changes made here to
5
6
# docs/contributing/dockerfile/dockerfile.md and
# docs/assets/contributing/dockerfile-stages-dependency.png
7

8
ARG CUDA_VERSION=12.9.1
9
10
11
12
13
14
15
ARG PYTHON_VERSION=3.12

# By parameterizing the base images, we allow third-party to use their own
# base images. One use case is hermetic builds with base images stored in
# private registries that use a different repository naming conventions.
#
# Example:
16
17
# docker build --build-arg BUILD_BASE_IMAGE=registry.acme.org/mirror/nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04

18
# Important: We build with an old version of Ubuntu to maintain broad
19
20
21
22
# compatibility with other Linux OSes. The main reason for this is that the
# glibc version is baked into the distro, and binaries built with one glibc
# version are not backwards compatible with OSes that use an earlier version.
ARG BUILD_BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04
23
24
# Using cuda base image with minimal dependencies necessary for JIT compilation (FlashInfer, DeepGEMM, EP kernels)
ARG FINAL_BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-base-ubuntu22.04
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69

# By parameterizing the Deadsnakes repository URL, we allow third-party to use
# their own mirror. When doing so, we don't benefit from the transparent
# installation of the GPG key of the PPA, as done by add-apt-repository, so we
# also need a URL for the GPG key.
ARG DEADSNAKES_MIRROR_URL
ARG DEADSNAKES_GPGKEY_URL

# The PyPA get-pip.py script is a self contained script+zip file, that provides
# both the installer script and the pip base85-encoded zip archive. This allows
# bootstrapping pip in environment where a dsitribution package does not exist.
#
# By parameterizing the URL for get-pip.py installation script, we allow
# third-party to use their own copy of the script stored in a private mirror.
# We set the default value to the PyPA owned get-pip.py script.
#
# Reference: https://pip.pypa.io/en/stable/installation/#get-pip-py
ARG GET_PIP_URL="https://bootstrap.pypa.io/get-pip.py"

# PIP supports fetching the packages from custom indexes, allowing third-party
# to host the packages in private mirrors. The PIP_INDEX_URL and
# PIP_EXTRA_INDEX_URL are standard PIP environment variables to override the
# default indexes. By letting them empty by default, PIP will use its default
# indexes if the build process doesn't override the indexes.
#
# Uv uses different variables. We set them by default to the same values as
# PIP, but they can be overridden.
ARG PIP_INDEX_URL
ARG PIP_EXTRA_INDEX_URL
ARG UV_INDEX_URL=${PIP_INDEX_URL}
ARG UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}

# PyTorch provides its own indexes for standard and nightly builds
ARG PYTORCH_CUDA_INDEX_BASE_URL=https://download.pytorch.org/whl

# PIP supports multiple authentication schemes, including keyring
# By parameterizing the PIP_KEYRING_PROVIDER variable and setting it to
# disabled by default, we allow third-party to use keyring authentication for
# their private Python indexes, while not changing the default behavior which
# is no authentication.
#
# Reference: https://pip.pypa.io/en/stable/topics/authentication/#keyring-support
ARG PIP_KEYRING_PROVIDER=disabled
ARG UV_KEYRING_PROVIDER=${PIP_KEYRING_PROVIDER}

70
# Flag enables built-in KV-connector dependency libs into docker images
71
72
ARG INSTALL_KV_CONNECTORS=false

Simon Mo's avatar
Simon Mo committed
73
#################### BASE BUILD IMAGE ####################
74
# prepare basic build environment
75
76
77
FROM ${BUILD_BASE_IMAGE} AS base
ARG CUDA_VERSION
ARG PYTHON_VERSION
78
ARG TARGETPLATFORM
79
ARG INSTALL_KV_CONNECTORS=false
80
81
ENV DEBIAN_FRONTEND=noninteractive

82
83
ARG GET_PIP_URL

84
# Install system dependencies and uv, then create Python virtual environment
85
86
87
RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
    && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \
    && apt-get update -y \
88
89
90
91
92
93
94
95
96
97
98
99
100
101
    && apt-get install -y --no-install-recommends \
        ccache \
        software-properties-common \
        git \
        curl \
        sudo \
        python3-pip \
        libibverbs-dev \
        # Upgrade to GCC 10 to avoid https://gcc.gnu.org/bugzilla/show_bug.cgi?id=92519
        # as it was causing spam when compiling the CUTLASS kernels
        gcc-10 \
        g++-10 \
    && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 110 --slave /usr/bin/g++ g++ /usr/bin/g++-10 \
    && rm -rf /var/lib/apt/lists/* \
102
103
104
105
106
107
    && curl -LsSf https://astral.sh/uv/install.sh | sh \
    && $HOME/.local/bin/uv venv /opt/venv --python ${PYTHON_VERSION} \
    && rm -f /usr/bin/python3 /usr/bin/python3-config /usr/bin/pip \
    && ln -s /opt/venv/bin/python3 /usr/bin/python3 \
    && ln -s /opt/venv/bin/python3-config /usr/bin/python3-config \
    && ln -s /opt/venv/bin/pip /usr/bin/pip \
108
    && python3 --version && python3 -m pip --version
109
110
111
112
113
114

ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL
ARG PIP_KEYRING_PROVIDER UV_KEYRING_PROVIDER

115
116
117
# Activate virtual environment and add uv to PATH
ENV PATH="/opt/venv/bin:/root/.local/bin:$PATH"
ENV VIRTUAL_ENV="/opt/venv"
Stephen Krider's avatar
Stephen Krider committed
118

119
120
121
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
122
ENV UV_INDEX_STRATEGY="unsafe-best-match"
123
124
# Use copy mode to avoid hardlink failures with Docker cache mounts
ENV UV_LINK_MODE=copy
125

126
127
128
129
RUN <<EOF
gcc --version
EOF

130
131
132
133
# Workaround for https://github.com/openai/triton/issues/2507 and
# https://github.com/pytorch/pytorch/issues/107960 -- hopefully
# this won't be needed for future versions of this docker image
# or future versions of triton.
134
RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/
135

Stephen Krider's avatar
Stephen Krider committed
136
137
WORKDIR /workspace

138
# install build and runtime dependencies
139
140
COPY requirements/common.txt requirements/common.txt
COPY requirements/cuda.txt requirements/cuda.txt
141
RUN --mount=type=cache,target=/root/.cache/uv \
142
    uv pip install --python /opt/venv/bin/python3 -r requirements/cuda.txt \
143
    --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
144

145
146
147
148
# cuda arch list used by torch
# can be useful for both `dev` and `test`
# explicitly set the list to avoid issues with torch 2.2
# see https://github.com/pytorch/pytorch/pull/123243
149
ARG torch_cuda_arch_list='7.0 7.5 8.0 8.9 9.0 10.0 12.0'
150
ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
Simon Mo's avatar
Simon Mo committed
151
152
#################### BASE BUILD IMAGE ####################

153
154
#################### CSRC BUILD IMAGE ####################
FROM base AS csrc-build
155
ARG TARGETPLATFORM
156

157
158
159
160
ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL

161
# install build dependencies
162
COPY requirements/build.txt requirements/build.txt
163

164
165
166
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
167
ENV UV_INDEX_STRATEGY="unsafe-best-match"
168
169
# Use copy mode to avoid hardlink failures with Docker cache mounts
ENV UV_LINK_MODE=copy
170

171
RUN --mount=type=cache,target=/root/.cache/uv \
172
    uv pip install --python /opt/venv/bin/python3 -r requirements/build.txt \
173
    --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
174

175
176
177
178
179
180
181
WORKDIR /workspace

COPY pyproject.toml setup.py CMakeLists.txt ./
COPY cmake cmake/
COPY csrc csrc/
COPY vllm/envs.py vllm/envs.py
COPY vllm/__init__.py vllm/__init__.py
Stephen Krider's avatar
Stephen Krider committed
182
183

# max jobs used by Ninja to build extensions
184
185
ARG max_jobs=2
ENV MAX_JOBS=${max_jobs}
186
187
188
# number of threads used by nvcc
ARG nvcc_threads=8
ENV NVCC_THREADS=$nvcc_threads
189

190
ARG USE_SCCACHE
191
192
ARG SCCACHE_DOWNLOAD_URL=https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-x86_64-unknown-linux-musl.tar.gz
ARG SCCACHE_ENDPOINT
193
194
ARG SCCACHE_BUCKET_NAME=vllm-build-sccache
ARG SCCACHE_REGION_NAME=us-west-2
195
ARG SCCACHE_S3_NO_CREDENTIALS=0
196
197

# Flag to control whether to use pre-built vLLM wheels
198
ARG VLLM_USE_PRECOMPILED=""
199
ARG VLLM_MERGE_BASE_COMMIT=""
200
ARG VLLM_MAIN_CUDA_VERSION=""
201

202
203
204
# Use dummy version for csrc-build wheel (only .so files are extracted, version doesn't matter)
ENV SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+csrc.build"

205
# if USE_SCCACHE is set, use sccache to speed up compilation
206
RUN --mount=type=cache,target=/root/.cache/uv \
207
208
    if [ "$USE_SCCACHE" = "1" ]; then \
        echo "Installing sccache..." \
209
        && curl -L -o sccache.tar.gz ${SCCACHE_DOWNLOAD_URL} \
210
211
212
        && tar -xzf sccache.tar.gz \
        && sudo mv sccache-v0.8.1-x86_64-unknown-linux-musl/sccache /usr/bin/sccache \
        && rm -rf sccache.tar.gz sccache-v0.8.1-x86_64-unknown-linux-musl \
213
        && if [ ! -z ${SCCACHE_ENDPOINT} ] ; then export SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT} ; fi \
214
215
        && export SCCACHE_BUCKET=${SCCACHE_BUCKET_NAME} \
        && export SCCACHE_REGION=${SCCACHE_REGION_NAME} \
216
        && export SCCACHE_S3_NO_CREDENTIALS=${SCCACHE_S3_NO_CREDENTIALS} \
217
        && export SCCACHE_IDLE_TIMEOUT=0 \
218
        && export CMAKE_BUILD_TYPE=Release \
219
        && export VLLM_USE_PRECOMPILED="${VLLM_USE_PRECOMPILED}" \
220
        && export VLLM_PRECOMPILED_WHEEL_COMMIT="${VLLM_MERGE_BASE_COMMIT}" \
221
        && export VLLM_MAIN_CUDA_VERSION="${VLLM_MAIN_CUDA_VERSION}" \
222
        && export VLLM_DOCKER_BUILD_CONTEXT=1 \
223
        && sccache --show-stats \
224
        && python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38 \
225
226
227
        && sccache --show-stats; \
    fi

228
229
ARG vllm_target_device="cuda"
ENV VLLM_TARGET_DEVICE=${vllm_target_device}
230
231
ENV CCACHE_DIR=/root/.cache/ccache
RUN --mount=type=cache,target=/root/.cache/ccache \
232
    --mount=type=cache,target=/root/.cache/uv \
233
    if [ "$USE_SCCACHE" != "1" ]; then \
234
235
236
        # Clean any existing CMake artifacts
        rm -rf .deps && \
        mkdir -p .deps && \
237
        export VLLM_USE_PRECOMPILED="${VLLM_USE_PRECOMPILED}" && \
238
        export VLLM_PRECOMPILED_WHEEL_COMMIT="${VLLM_MERGE_BASE_COMMIT}" && \
239
        export VLLM_DOCKER_BUILD_CONTEXT=1 && \
240
        python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38; \
241
    fi
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
#################### CSRC BUILD IMAGE ####################

#################### WHEEL BUILD IMAGE ####################
FROM base AS build
ARG TARGETPLATFORM

ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL

# install build dependencies
COPY requirements/build.txt requirements/build.txt

# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
ENV UV_INDEX_STRATEGY="unsafe-best-match"
# Use copy mode to avoid hardlink failures with Docker cache mounts
ENV UV_LINK_MODE=copy

RUN --mount=type=cache,target=/root/.cache/uv \
    uv pip install --python /opt/venv/bin/python3 -r requirements/build.txt \
    --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')

WORKDIR /workspace

COPY --from=csrc-build /workspace/dist /precompiled-wheels

COPY . .

ARG GIT_REPO_CHECK=0
RUN --mount=type=bind,source=.git,target=.git \
    if [ "$GIT_REPO_CHECK" != "0" ]; then bash tools/check_repo.sh ; fi

ARG vllm_target_device="cuda"
ENV VLLM_TARGET_DEVICE=${vllm_target_device}

# Skip adding +precompiled suffix to version (preserves git-derived version)
ENV VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX=1

RUN --mount=type=cache,target=/root/.cache/uv \
    --mount=type=bind,source=.git,target=.git \
    if [ "${vllm_target_device}" = "cuda" ]; then \
        export VLLM_PRECOMPILED_WHEEL_LOCATION=$(ls /precompiled-wheels/*.whl); \
    fi && \
    python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38
288

289
290
291
292
293
294
295
296
297
298
299
# Install DeepGEMM from source
ARG DEEPGEMM_GIT_REF
COPY tools/install_deepgemm.sh /tmp/install_deepgemm.sh
RUN --mount=type=cache,target=/root/.cache/uv \
    VLLM_DOCKER_BUILD_CONTEXT=1 TORCH_CUDA_ARCH_LIST="9.0a 10.0a" /tmp/install_deepgemm.sh --cuda-version "${CUDA_VERSION}" ${DEEPGEMM_GIT_REF:+--ref "$DEEPGEMM_GIT_REF"} --wheel-dir /tmp/deepgemm/dist

# Ensure the wheel dir exists so later-stage COPY won't fail when DeepGEMM is skipped
RUN mkdir -p /tmp/deepgemm/dist && touch /tmp/deepgemm/dist/.deepgemm_skipped

COPY tools/ep_kernels/install_python_libraries.sh /tmp/install_python_libraries.sh
# Install EP kernels(pplx-kernels and DeepEP)
300
301
ARG PPLX_COMMIT_HASH
ARG DEEPEP_COMMIT_HASH
302
303
RUN --mount=type=cache,target=/root/.cache/uv \
    export TORCH_CUDA_ARCH_LIST='9.0a 10.0a' && \
304
305
306
307
308
    /tmp/install_python_libraries.sh \
        --workspace /tmp/ep_kernels_workspace \
        --mode wheel \
        ${PPLX_COMMIT_HASH:+--pplx-ref "$PPLX_COMMIT_HASH"} \
        ${DEEPEP_COMMIT_HASH:+--deepep-ref "$DEEPEP_COMMIT_HASH"} && \
309
310
    find /tmp/ep_kernels_workspace/nvshmem -name '*.a' -delete

311
# Check the size of the wheel if RUN_WHEEL_CHECK is true
312
COPY .buildkite/check-wheel-size.py check-wheel-size.py
313
# sync the default value with .buildkite/check-wheel-size.py
314
ARG VLLM_MAX_SIZE_MB=500
315
316
317
318
319
320
321
ENV VLLM_MAX_SIZE_MB=$VLLM_MAX_SIZE_MB
ARG RUN_WHEEL_CHECK=true
RUN if [ "$RUN_WHEEL_CHECK" = "true" ]; then \
        python3 check-wheel-size.py dist; \
    else \
        echo "Skipping wheel size check."; \
    fi
Simon Mo's avatar
Simon Mo committed
322
#################### EXTENSION Build IMAGE ####################
Stephen Krider's avatar
Stephen Krider committed
323

324
#################### DEV IMAGE ####################
325
FROM base AS dev
326

327
328
329
330
ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL

331
332
333
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
334
ENV UV_INDEX_STRATEGY="unsafe-best-match"
335
336
# Use copy mode to avoid hardlink failures with Docker cache mounts
ENV UV_LINK_MODE=copy
Huy Do's avatar
Huy Do committed
337

338
# Install libnuma-dev, required by fastsafetensors (fixes #20384)
339
RUN apt-get update && apt-get install -y --no-install-recommends libnuma-dev && rm -rf /var/lib/apt/lists/*
340
341
342
COPY requirements/lint.txt requirements/lint.txt
COPY requirements/test.txt requirements/test.txt
COPY requirements/dev.txt requirements/dev.txt
343
RUN --mount=type=cache,target=/root/.cache/uv \
344
    uv pip install --python /opt/venv/bin/python3 -r requirements/dev.txt \
345
    --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
346
#################### DEV IMAGE ####################
347

348
349
#################### vLLM installation IMAGE ####################
# image with vLLM installed
350
351
352
FROM ${FINAL_BASE_IMAGE} AS vllm-base
ARG CUDA_VERSION
ARG PYTHON_VERSION
353
ARG INSTALL_KV_CONNECTORS=false
Simon Mo's avatar
Simon Mo committed
354
WORKDIR /vllm-workspace
355
ENV DEBIAN_FRONTEND=noninteractive
356
357
ARG TARGETPLATFORM

358
# TODO (huydhn): There is no prebuilt gdrcopy package on 12.9 at the moment
359
360
361
362
ARG GDRCOPY_CUDA_VERSION=12.8
# Keep in line with FINAL_BASE_IMAGE
ARG GDRCOPY_OS_VERSION=Ubuntu22_04

Kebe's avatar
Kebe committed
363
364
SHELL ["/bin/bash", "-c"]

365
366
367
368
ARG DEADSNAKES_MIRROR_URL
ARG DEADSNAKES_GPGKEY_URL
ARG GET_PIP_URL

369
370
RUN PYTHON_VERSION_STR=$(echo ${PYTHON_VERSION} | sed 's/\.//g') && \
    echo "export PYTHON_VERSION_STR=${PYTHON_VERSION_STR}" >> /etc/environment
371

372
373
374
375
# Install Python and other dependencies
RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
    && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \
    && apt-get update -y \
376
377
378
379
380
381
382
383
384
    && apt-get install -y --no-install-recommends \
        software-properties-common \
        curl \
        sudo \
        python3-pip \
        ffmpeg \
        libsm6 \
        libxext6 \
        libgl1 \
385
386
387
388
389
390
391
392
393
394
395
396
397
    && if [ ! -z ${DEADSNAKES_MIRROR_URL} ] ; then \
        if [ ! -z "${DEADSNAKES_GPGKEY_URL}" ] ; then \
            mkdir -p -m 0755 /etc/apt/keyrings ; \
            curl -L ${DEADSNAKES_GPGKEY_URL} | gpg --dearmor > /etc/apt/keyrings/deadsnakes.gpg ; \
            sudo chmod 644 /etc/apt/keyrings/deadsnakes.gpg ; \
            echo "deb [signed-by=/etc/apt/keyrings/deadsnakes.gpg] ${DEADSNAKES_MIRROR_URL} $(lsb_release -cs) main" > /etc/apt/sources.list.d/deadsnakes.list ; \
        fi ; \
    else \
        for i in 1 2 3; do \
            add-apt-repository -y ppa:deadsnakes/ppa && break || \
            { echo "Attempt $i failed, retrying in 5s..."; sleep 5; }; \
        done ; \
    fi \
398
    && apt-get update -y \
399
400
401
402
403
404
    && apt-get install -y --no-install-recommends \
        python${PYTHON_VERSION} \
        python${PYTHON_VERSION}-dev \
        python${PYTHON_VERSION}-venv \
        libibverbs-dev \
    && rm -rf /var/lib/apt/lists/* \
405
406
407
    && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \
    && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \
    && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \
408
    && curl -sS ${GET_PIP_URL} | python${PYTHON_VERSION} \
409
    && python3 --version && python3 -m pip --version
410

411
412
413
414
415
416
417
418
419
# Install CUDA development tools and build essentials for runtime JIT compilation
# (FlashInfer, DeepGEMM, EP kernels all require compilation at runtime)
RUN CUDA_VERSION_DASH=$(echo $CUDA_VERSION | cut -d. -f1,2 | tr '.' '-') && \
    apt-get update -y && \
    apt-get install -y --no-install-recommends \
    cuda-nvcc-${CUDA_VERSION_DASH} \
    cuda-cudart-${CUDA_VERSION_DASH} \
    cuda-nvrtc-${CUDA_VERSION_DASH} \
    cuda-cuobjdump-${CUDA_VERSION_DASH} \
420
421
422
423
424
425
    # https://github.com/vllm-project/vllm/issues/29590
    libcurand-dev-${CUDA_VERSION_DASH} \
    libcublas-${CUDA_VERSION_DASH} \
    # Fixes nccl_allocator requiring nccl.h at runtime
    # https://github.com/vllm-project/vllm/blob/1336a1ea244fa8bfd7e72751cabbdb5b68a0c11a/vllm/distributed/device_communicators/pynccl_allocator.py#L22
    libnccl-dev && \
426
427
    rm -rf /var/lib/apt/lists/*

428
429
430
431
432
ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL
ARG PIP_KEYRING_PROVIDER UV_KEYRING_PROVIDER

433
434
435
# Install uv for faster pip installs
RUN --mount=type=cache,target=/root/.cache/uv \
    python3 -m pip install uv
436

437
438
439
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
440
ENV UV_INDEX_STRATEGY="unsafe-best-match"
441
442
# Use copy mode to avoid hardlink failures with Docker cache mounts
ENV UV_LINK_MODE=copy
443

444
445
446
447
# Workaround for https://github.com/openai/triton/issues/2507 and
# https://github.com/pytorch/pytorch/issues/107960 -- hopefully
# this won't be needed for future versions of this docker image
# or future versions of triton.
448
RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/
449

450
# Install vllm wheel first, so that torch etc will be installed.
451
RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \
452
    --mount=type=cache,target=/root/.cache/uv \
453
    uv pip install --system dist/*.whl --verbose \
454
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
Mor Zusman's avatar
Mor Zusman committed
455

456
457
458
# Install FlashInfer pre-compiled kernel cache and binaries
# https://docs.flashinfer.ai/installation.html
RUN --mount=type=cache,target=/root/.cache/uv \
459
460
    uv pip install --system flashinfer-cubin==0.5.3 \
    && uv pip install --system flashinfer-jit-cache==0.5.3 \
461
462
463
        --extra-index-url https://flashinfer.ai/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
    && flashinfer show-config

464
COPY examples examples
465
COPY benchmarks benchmarks
466
COPY ./vllm/collect_env.py .
467

Huy Do's avatar
Huy Do committed
468
469
470
471
RUN --mount=type=cache,target=/root/.cache/uv \
. /etc/environment && \
uv pip list

472
# Install deepgemm wheel that has been built in the `build` stage
473
RUN --mount=type=cache,target=/root/.cache/uv \
474
475
476
477
478
479
480
481
482
483
484
485
486
487
    --mount=type=bind,from=build,source=/tmp/deepgemm/dist,target=/tmp/deepgemm/dist,ro \
    sh -c 'if ls /tmp/deepgemm/dist/*.whl >/dev/null 2>&1; then \
              uv pip install --system /tmp/deepgemm/dist/*.whl; \
           else \
              echo "No DeepGEMM wheels to install; skipping."; \
           fi'

# Pytorch now installs NVSHMEM, setting LD_LIBRARY_PATH (https://github.com/pytorch/pytorch/blob/d38164a545b4a4e4e0cf73ce67173f70574890b6/.ci/manywheel/build_cuda.sh#L141C14-L141C36)
ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH

# Install EP kernels wheels (pplx-kernels and DeepEP) that have been built in the `build` stage
RUN --mount=type=bind,from=build,src=/tmp/ep_kernels_workspace/dist,target=/vllm-workspace/ep_kernels/dist \
    --mount=type=cache,target=/root/.cache/uv \
    uv pip install --system ep_kernels/dist/*.whl --verbose \
488
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
489

490
491
RUN --mount=type=bind,source=tools/install_gdrcopy.sh,target=/tmp/install_gdrcopy.sh,ro \
    set -eux; \
492
493
494
495
496
    case "${TARGETPLATFORM}" in \
      linux/arm64) UUARCH="aarch64" ;; \
      linux/amd64) UUARCH="x64" ;; \
      *) echo "Unsupported TARGETPLATFORM: ${TARGETPLATFORM}" >&2; exit 1 ;; \
    esac; \
497
    /tmp/install_gdrcopy.sh "${GDRCOPY_OS_VERSION}" "${GDRCOPY_CUDA_VERSION}" "${UUARCH}"
498

499
500
501
502
503
504
# CUDA image changed from /usr/local/nvidia to /usr/local/cuda in 12.8 but will
# return to /usr/local/nvidia in 13.0 to allow container providers to mount drivers
# consistently from the host (see https://github.com/vllm-project/vllm/issues/18859).
# Until then, add /usr/local/nvidia/lib64 before the image cuda path to allow override.
ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}

505
#################### vLLM installation IMAGE ####################
Stephen Krider's avatar
Stephen Krider committed
506

507
508
509
510
#################### TEST IMAGE ####################
# image to run unit testing suite
# note that this uses vllm installed by `pip`
FROM vllm-base AS test
Stephen Krider's avatar
Stephen Krider committed
511

512
ADD . /vllm-workspace/
Stephen Krider's avatar
Stephen Krider committed
513

514
515
516
517
ARG PYTHON_VERSION

ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
518
ARG PYTORCH_CUDA_INDEX_BASE_URL
519

520
521
522
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
523
ENV UV_INDEX_STRATEGY="unsafe-best-match"
524
525
# Use copy mode to avoid hardlink failures with Docker cache mounts
ENV UV_LINK_MODE=copy
526

527
528
529
530
531
RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
    && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \
    && apt-get update -y \
    && apt-get install -y git

Huy Do's avatar
Huy Do committed
532
# install development dependencies (for testing)
533
RUN --mount=type=cache,target=/root/.cache/uv \
534
535
    CUDA_MAJOR="${CUDA_VERSION%%.*}"; \
    if [ "$CUDA_MAJOR" -ge 12 ]; then \
536
537
        uv pip install --system -r requirements/dev.txt \
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \
538
    fi
539

youkaichao's avatar
youkaichao committed
540
# install development dependencies (for testing)
541
RUN --mount=type=cache,target=/root/.cache/uv \
542
    uv pip install --system -e tests/vllm_test_utils
youkaichao's avatar
youkaichao committed
543

544
# enable fast downloads from hf (for testing)
545
RUN --mount=type=cache,target=/root/.cache/uv \
546
    uv pip install --system hf_transfer
547
548
ENV HF_HUB_ENABLE_HF_TRANSFER 1

Joe Runde's avatar
Joe Runde committed
549
# Copy in the v1 package for testing (it isn't distributed yet)
550
COPY vllm/v1 /usr/local/lib/python${PYTHON_VERSION}/dist-packages/vllm/v1
Joe Runde's avatar
Joe Runde committed
551

552
553
# Source code is used in the `python_only_compile.sh` test
# We hide it inside `src/` so that this source code
554
# will not be imported by other tests
555
556
RUN mkdir src
RUN mv vllm src/vllm
557
#################### TEST IMAGE ####################
Stephen Krider's avatar
Stephen Krider committed
558

Simon Mo's avatar
Simon Mo committed
559
#################### OPENAI API SERVER ####################
560
561
# base openai image with additional requirements, for any subsequent openai-style images
FROM vllm-base AS vllm-openai-base
562
ARG TARGETPLATFORM
563
ARG INSTALL_KV_CONNECTORS=false
564

565
566
567
ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL

568
569
570
571
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500

572
# install additional dependencies for openai api server
573
RUN --mount=type=cache,target=/root/.cache/uv \
574
    --mount=type=bind,source=requirements/kv_connectors.txt,target=/tmp/kv_connectors.txt,ro \
575
    if [ "$INSTALL_KV_CONNECTORS" = "true" ]; then \
576
        uv pip install --system -r /tmp/kv_connectors.txt; \
577
    fi; \
578
    if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
579
        BITSANDBYTES_VERSION="0.42.0"; \
580
    else \
581
582
        BITSANDBYTES_VERSION="0.46.1"; \
    fi; \
583
    uv pip install --system accelerate hf_transfer modelscope "bitsandbytes>=${BITSANDBYTES_VERSION}" 'timm>=1.0.17' 'runai-model-streamer[s3,gcs]>=0.15.3'
584

yhu422's avatar
yhu422 committed
585
586
ENV VLLM_USAGE_SOURCE production-docker-image

587
588
589
# define sagemaker first, so it is not default from `docker build`
FROM vllm-openai-base AS vllm-sagemaker

590
COPY examples/online_serving/sagemaker-entrypoint.sh .
591
592
593
594
595
RUN chmod +x sagemaker-entrypoint.sh
ENTRYPOINT ["./sagemaker-entrypoint.sh"]

FROM vllm-openai-base AS vllm-openai

596
ENTRYPOINT ["vllm", "serve"]
Simon Mo's avatar
Simon Mo committed
597
#################### OPENAI API SERVER ####################