Dockerfile 21.8 KB
Newer Older
Simon Mo's avatar
Simon Mo committed
1
2
3
# The vLLM Dockerfile is used to construct vLLM image that can be directly used
# to run the OpenAI compatible server.

4
# Please update any changes made here to
5
6
# docs/contributing/dockerfile/dockerfile.md and
# docs/assets/contributing/dockerfile-stages-dependency.png
7

8
ARG CUDA_VERSION=12.9.1
9
10
11
12
13
14
15
ARG PYTHON_VERSION=3.12

# By parameterizing the base images, we allow third-party to use their own
# base images. One use case is hermetic builds with base images stored in
# private registries that use a different repository naming conventions.
#
# Example:
16
17
# docker build --build-arg BUILD_BASE_IMAGE=registry.acme.org/mirror/nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04

18
# Important: We build with an old version of Ubuntu to maintain broad
19
20
21
22
# compatibility with other Linux OSes. The main reason for this is that the
# glibc version is baked into the distro, and binaries built with one glibc
# version are not backwards compatible with OSes that use an earlier version.
ARG BUILD_BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04
23
# TODO: Restore to base image after FlashInfer AOT wheel fixed
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
ARG FINAL_BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04

# By parameterizing the Deadsnakes repository URL, we allow third-party to use
# their own mirror. When doing so, we don't benefit from the transparent
# installation of the GPG key of the PPA, as done by add-apt-repository, so we
# also need a URL for the GPG key.
ARG DEADSNAKES_MIRROR_URL
ARG DEADSNAKES_GPGKEY_URL

# The PyPA get-pip.py script is a self contained script+zip file, that provides
# both the installer script and the pip base85-encoded zip archive. This allows
# bootstrapping pip in environment where a dsitribution package does not exist.
#
# By parameterizing the URL for get-pip.py installation script, we allow
# third-party to use their own copy of the script stored in a private mirror.
# We set the default value to the PyPA owned get-pip.py script.
#
# Reference: https://pip.pypa.io/en/stable/installation/#get-pip-py
ARG GET_PIP_URL="https://bootstrap.pypa.io/get-pip.py"

# PIP supports fetching the packages from custom indexes, allowing third-party
# to host the packages in private mirrors. The PIP_INDEX_URL and
# PIP_EXTRA_INDEX_URL are standard PIP environment variables to override the
# default indexes. By letting them empty by default, PIP will use its default
# indexes if the build process doesn't override the indexes.
#
# Uv uses different variables. We set them by default to the same values as
# PIP, but they can be overridden.
ARG PIP_INDEX_URL
ARG PIP_EXTRA_INDEX_URL
ARG UV_INDEX_URL=${PIP_INDEX_URL}
ARG UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}

# PyTorch provides its own indexes for standard and nightly builds
ARG PYTORCH_CUDA_INDEX_BASE_URL=https://download.pytorch.org/whl
ARG PYTORCH_CUDA_NIGHTLY_INDEX_BASE_URL=https://download.pytorch.org/whl/nightly

# PIP supports multiple authentication schemes, including keyring
# By parameterizing the PIP_KEYRING_PROVIDER variable and setting it to
# disabled by default, we allow third-party to use keyring authentication for
# their private Python indexes, while not changing the default behavior which
# is no authentication.
#
# Reference: https://pip.pypa.io/en/stable/topics/authentication/#keyring-support
ARG PIP_KEYRING_PROVIDER=disabled
ARG UV_KEYRING_PROVIDER=${PIP_KEYRING_PROVIDER}

71
# Flag enables built-in KV-connector dependency libs into docker images
72
73
ARG INSTALL_KV_CONNECTORS=false

Simon Mo's avatar
Simon Mo committed
74
#################### BASE BUILD IMAGE ####################
75
# prepare basic build environment
76
77
78
FROM ${BUILD_BASE_IMAGE} AS base
ARG CUDA_VERSION
ARG PYTHON_VERSION
79
ARG TARGETPLATFORM
80
ARG INSTALL_KV_CONNECTORS=false
81
82
ENV DEBIAN_FRONTEND=noninteractive

83
84
ARG GET_PIP_URL

85
# Install system dependencies and uv, then create Python virtual environment
86
87
88
RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
    && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \
    && apt-get update -y \
89
90
91
92
93
94
95
    && apt-get install -y ccache software-properties-common git curl sudo python3-pip \
    && curl -LsSf https://astral.sh/uv/install.sh | sh \
    && $HOME/.local/bin/uv venv /opt/venv --python ${PYTHON_VERSION} \
    && rm -f /usr/bin/python3 /usr/bin/python3-config /usr/bin/pip \
    && ln -s /opt/venv/bin/python3 /usr/bin/python3 \
    && ln -s /opt/venv/bin/python3-config /usr/bin/python3-config \
    && ln -s /opt/venv/bin/pip /usr/bin/pip \
96
    && python3 --version && python3 -m pip --version
97
98
99
100
101
102
103

ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL
ARG PYTORCH_CUDA_NIGHTLY_INDEX_BASE_URL
ARG PIP_KEYRING_PROVIDER UV_KEYRING_PROVIDER

104
105
106
# Activate virtual environment and add uv to PATH
ENV PATH="/opt/venv/bin:/root/.local/bin:$PATH"
ENV VIRTUAL_ENV="/opt/venv"
Stephen Krider's avatar
Stephen Krider committed
107

108
109
110
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
111
ENV UV_INDEX_STRATEGY="unsafe-best-match"
112
113
# Use copy mode to avoid hardlink failures with Docker cache mounts
ENV UV_LINK_MODE=copy
114

115
116
117
118
119
120
121
122
# Upgrade to GCC 10 to avoid https://gcc.gnu.org/bugzilla/show_bug.cgi?id=92519
# as it was causing spam when compiling the CUTLASS kernels
RUN apt-get install -y gcc-10 g++-10
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 110 --slave /usr/bin/g++ g++ /usr/bin/g++-10
RUN <<EOF
gcc --version
EOF

123
124
125
126
# Workaround for https://github.com/openai/triton/issues/2507 and
# https://github.com/pytorch/pytorch/issues/107960 -- hopefully
# this won't be needed for future versions of this docker image
# or future versions of triton.
127
RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/
128

Stephen Krider's avatar
Stephen Krider committed
129
130
WORKDIR /workspace

131
# install build and runtime dependencies
132
133
COPY requirements/common.txt requirements/common.txt
COPY requirements/cuda.txt requirements/cuda.txt
134
RUN --mount=type=cache,target=/root/.cache/uv \
135
136
137
    # TODO: remove apache-tvm-ffi once FlashInfer is fixed https://github.com/flashinfer-ai/flashinfer/issues/1962
    uv pip install --python /opt/venv/bin/python3 --pre apache-tvm-ffi==0.1.0b15 \
    && uv pip install --python /opt/venv/bin/python3 -r requirements/cuda.txt \
138
    --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
139

140
141
142
143
# cuda arch list used by torch
# can be useful for both `dev` and `test`
# explicitly set the list to avoid issues with torch 2.2
# see https://github.com/pytorch/pytorch/pull/123243
144
ARG torch_cuda_arch_list='7.0 7.5 8.0 8.9 9.0 10.0 12.0'
145
ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
Simon Mo's avatar
Simon Mo committed
146
147
#################### BASE BUILD IMAGE ####################

148
#################### WHEEL BUILD IMAGE ####################
149
FROM base AS build
150
ARG TARGETPLATFORM
151

152
153
154
155
ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL

156
# install build dependencies
157
COPY requirements/build.txt requirements/build.txt
158

159
160
161
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
162
ENV UV_INDEX_STRATEGY="unsafe-best-match"
163
164
# Use copy mode to avoid hardlink failures with Docker cache mounts
ENV UV_LINK_MODE=copy
165

166
RUN --mount=type=cache,target=/root/.cache/uv \
167
    uv pip install --python /opt/venv/bin/python3 -r requirements/build.txt \
168
    --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
169

170
COPY . .
171
172
ARG GIT_REPO_CHECK=0
RUN --mount=type=bind,source=.git,target=.git \
173
    if [ "$GIT_REPO_CHECK" != "0" ]; then bash tools/check_repo.sh ; fi
Stephen Krider's avatar
Stephen Krider committed
174
175

# max jobs used by Ninja to build extensions
176
177
ARG max_jobs=2
ENV MAX_JOBS=${max_jobs}
178
179
180
# number of threads used by nvcc
ARG nvcc_threads=8
ENV NVCC_THREADS=$nvcc_threads
181

182
ARG USE_SCCACHE
183
184
ARG SCCACHE_DOWNLOAD_URL=https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-x86_64-unknown-linux-musl.tar.gz
ARG SCCACHE_ENDPOINT
185
186
ARG SCCACHE_BUCKET_NAME=vllm-build-sccache
ARG SCCACHE_REGION_NAME=us-west-2
187
ARG SCCACHE_S3_NO_CREDENTIALS=0
188
189

# Flag to control whether to use pre-built vLLM wheels
190
ARG VLLM_USE_PRECOMPILED=""
191
ARG VLLM_MAIN_CUDA_VERSION=""
192

193
# if USE_SCCACHE is set, use sccache to speed up compilation
194
RUN --mount=type=cache,target=/root/.cache/uv \
195
    --mount=type=bind,source=.git,target=.git \
196
197
    if [ "$USE_SCCACHE" = "1" ]; then \
        echo "Installing sccache..." \
198
        && curl -L -o sccache.tar.gz ${SCCACHE_DOWNLOAD_URL} \
199
200
201
        && tar -xzf sccache.tar.gz \
        && sudo mv sccache-v0.8.1-x86_64-unknown-linux-musl/sccache /usr/bin/sccache \
        && rm -rf sccache.tar.gz sccache-v0.8.1-x86_64-unknown-linux-musl \
202
        && if [ ! -z ${SCCACHE_ENDPOINT} ] ; then export SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT} ; fi \
203
204
        && export SCCACHE_BUCKET=${SCCACHE_BUCKET_NAME} \
        && export SCCACHE_REGION=${SCCACHE_REGION_NAME} \
205
        && export SCCACHE_S3_NO_CREDENTIALS=${SCCACHE_S3_NO_CREDENTIALS} \
206
        && export SCCACHE_IDLE_TIMEOUT=0 \
207
        && export CMAKE_BUILD_TYPE=Release \
208
        && export VLLM_USE_PRECOMPILED="${VLLM_USE_PRECOMPILED}" \
209
        && export VLLM_MAIN_CUDA_VERSION="${VLLM_MAIN_CUDA_VERSION}" \
210
        && export VLLM_DOCKER_BUILD_CONTEXT=1 \
211
        && sccache --show-stats \
212
        && python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38 \
213
214
215
        && sccache --show-stats; \
    fi

216
217
ARG vllm_target_device="cuda"
ENV VLLM_TARGET_DEVICE=${vllm_target_device}
218
219
ENV CCACHE_DIR=/root/.cache/ccache
RUN --mount=type=cache,target=/root/.cache/ccache \
220
    --mount=type=cache,target=/root/.cache/uv \
221
    --mount=type=bind,source=.git,target=.git  \
222
    if [ "$USE_SCCACHE" != "1" ]; then \
223
224
225
        # Clean any existing CMake artifacts
        rm -rf .deps && \
        mkdir -p .deps && \
226
227
        export VLLM_USE_PRECOMPILED="${VLLM_USE_PRECOMPILED}" && \
        export VLLM_DOCKER_BUILD_CONTEXT=1 && \
228
        python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38; \
229
    fi
230

231
# Check the size of the wheel if RUN_WHEEL_CHECK is true
232
COPY .buildkite/check-wheel-size.py check-wheel-size.py
233
# sync the default value with .buildkite/check-wheel-size.py
234
ARG VLLM_MAX_SIZE_MB=500
235
236
237
238
239
240
241
ENV VLLM_MAX_SIZE_MB=$VLLM_MAX_SIZE_MB
ARG RUN_WHEEL_CHECK=true
RUN if [ "$RUN_WHEEL_CHECK" = "true" ]; then \
        python3 check-wheel-size.py dist; \
    else \
        echo "Skipping wheel size check."; \
    fi
Simon Mo's avatar
Simon Mo committed
242
#################### EXTENSION Build IMAGE ####################
Stephen Krider's avatar
Stephen Krider committed
243

244
#################### DEV IMAGE ####################
245
FROM base AS dev
246

247
248
249
250
ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL

251
252
253
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
254
ENV UV_INDEX_STRATEGY="unsafe-best-match"
255
256
# Use copy mode to avoid hardlink failures with Docker cache mounts
ENV UV_LINK_MODE=copy
Huy Do's avatar
Huy Do committed
257

258
259
# Install libnuma-dev, required by fastsafetensors (fixes #20384)
RUN apt-get update && apt-get install -y libnuma-dev && rm -rf /var/lib/apt/lists/*
260
261
262
COPY requirements/lint.txt requirements/lint.txt
COPY requirements/test.txt requirements/test.txt
COPY requirements/dev.txt requirements/dev.txt
263
RUN --mount=type=cache,target=/root/.cache/uv \
264
    uv pip install --python /opt/venv/bin/python3 -r requirements/dev.txt \
265
    --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
266
#################### DEV IMAGE ####################
267

268
269
#################### vLLM installation IMAGE ####################
# image with vLLM installed
270
271
272
FROM ${FINAL_BASE_IMAGE} AS vllm-base
ARG CUDA_VERSION
ARG PYTHON_VERSION
273
ARG INSTALL_KV_CONNECTORS=false
Simon Mo's avatar
Simon Mo committed
274
WORKDIR /vllm-workspace
275
ENV DEBIAN_FRONTEND=noninteractive
276
277
ARG TARGETPLATFORM

278
# TODO (huydhn): There is no prebuilt gdrcopy package on 12.9 at the moment
279
280
281
282
ARG GDRCOPY_CUDA_VERSION=12.8
# Keep in line with FINAL_BASE_IMAGE
ARG GDRCOPY_OS_VERSION=Ubuntu22_04

Kebe's avatar
Kebe committed
283
284
SHELL ["/bin/bash", "-c"]

285
286
287
288
ARG DEADSNAKES_MIRROR_URL
ARG DEADSNAKES_GPGKEY_URL
ARG GET_PIP_URL

289
290
RUN PYTHON_VERSION_STR=$(echo ${PYTHON_VERSION} | sed 's/\.//g') && \
    echo "export PYTHON_VERSION_STR=${PYTHON_VERSION_STR}" >> /etc/environment
291

292
293
294
295
296
297
# Install Python and other dependencies
RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
    && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \
    && apt-get update -y \
    && apt-get install -y ccache software-properties-common git curl wget sudo vim python3-pip \
    && apt-get install -y ffmpeg libsm6 libxext6 libgl1 \
298
299
300
301
302
303
304
305
306
307
308
309
310
    && if [ ! -z ${DEADSNAKES_MIRROR_URL} ] ; then \
        if [ ! -z "${DEADSNAKES_GPGKEY_URL}" ] ; then \
            mkdir -p -m 0755 /etc/apt/keyrings ; \
            curl -L ${DEADSNAKES_GPGKEY_URL} | gpg --dearmor > /etc/apt/keyrings/deadsnakes.gpg ; \
            sudo chmod 644 /etc/apt/keyrings/deadsnakes.gpg ; \
            echo "deb [signed-by=/etc/apt/keyrings/deadsnakes.gpg] ${DEADSNAKES_MIRROR_URL} $(lsb_release -cs) main" > /etc/apt/sources.list.d/deadsnakes.list ; \
        fi ; \
    else \
        for i in 1 2 3; do \
            add-apt-repository -y ppa:deadsnakes/ppa && break || \
            { echo "Attempt $i failed, retrying in 5s..."; sleep 5; }; \
        done ; \
    fi \
311
312
313
314
315
    && apt-get update -y \
    && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv libibverbs-dev \
    && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \
    && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \
    && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \
316
    && curl -sS ${GET_PIP_URL} | python${PYTHON_VERSION} \
317
    && python3 --version && python3 -m pip --version
318
319
320
321
322
323
324

ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL
ARG PYTORCH_CUDA_NIGHTLY_INDEX_BASE_URL
ARG PIP_KEYRING_PROVIDER UV_KEYRING_PROVIDER

325
326
327
# Install uv for faster pip installs
RUN --mount=type=cache,target=/root/.cache/uv \
    python3 -m pip install uv
328

329
330
331
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
332
ENV UV_INDEX_STRATEGY="unsafe-best-match"
333
334
# Use copy mode to avoid hardlink failures with Docker cache mounts
ENV UV_LINK_MODE=copy
335

336
337
338
339
# Workaround for https://github.com/openai/triton/issues/2507 and
# https://github.com/pytorch/pytorch/issues/107960 -- hopefully
# this won't be needed for future versions of this docker image
# or future versions of triton.
340
RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/
341

342
343
344
345
# arm64 (GH200) build follows the practice of "use existing pytorch" build,
# we need to install torch and torchvision from the nightly builds first,
# pytorch will not appear as a vLLM dependency in all of the following steps
# after this step
346
RUN --mount=type=cache,target=/root/.cache/uv \
347
    if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
348
349
350
351
352
353
        uv pip install --system \
            --index-url ${PYTORCH_CUDA_NIGHTLY_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
            "torch==2.8.0.dev20250318+cu128" "torchvision==0.22.0.dev20250319" ; \
        uv pip install --system \
            --index-url ${PYTORCH_CUDA_NIGHTLY_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
            --pre pytorch_triton==3.3.0+gitab727c40 ; \
354
355
    fi

356
# Install vllm wheel first, so that torch etc will be installed.
357
RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \
358
    --mount=type=cache,target=/root/.cache/uv \
359
360
361
    # TODO: remove apache-tvm-ffi once FlashInfer is fixed https://github.com/flashinfer-ai/flashinfer/issues/1962
    uv pip install --system --pre apache-tvm-ffi==0.1.0b15 \
    && uv pip install --system dist/*.whl --verbose \
362
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
Mor Zusman's avatar
Mor Zusman committed
363

364
365
366
367
368
369
370
# TODO (huydhn): Remove this once xformers is released for 2.9.0
RUN --mount=type=cache,target=/root/.cache/uv bash - <<'BASH'
    . /etc/environment
    export TORCH_CUDA_ARCH_LIST='7.5 8.0+PTX 9.0a'
    uv pip install --system --no-build-isolation "git+https://github.com/facebookresearch/xformers@v0.0.32.post2"
BASH

371
372
373
# Install FlashInfer pre-compiled kernel cache and binaries
# https://docs.flashinfer.ai/installation.html
RUN --mount=type=cache,target=/root/.cache/uv \
374
375
    uv pip install --system flashinfer-cubin==0.4.1 \
    && uv pip install --system flashinfer-jit-cache==0.4.1 \
376
377
378
        --extra-index-url https://flashinfer.ai/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
    && flashinfer show-config

379
COPY examples examples
380
COPY benchmarks benchmarks
381
COPY ./vllm/collect_env.py .
382

Huy Do's avatar
Huy Do committed
383
384
385
386
RUN --mount=type=cache,target=/root/.cache/uv \
. /etc/environment && \
uv pip list

387
# Even when we build Flashinfer with AOT mode, there's still
388
389
390
# some issues w.r.t. JIT compilation. Therefore we need to
# install build dependencies for JIT compilation.
# TODO: Remove this once FlashInfer AOT wheel is fixed
391
COPY requirements/build.txt requirements/build.txt
392
RUN --mount=type=cache,target=/root/.cache/uv \
Huy Do's avatar
Huy Do committed
393
    uv pip install --system -r requirements/build.txt \
394
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
395

396
# Install DeepGEMM from source
397
ARG DEEPGEMM_GIT_REF
398
399
COPY tools/install_deepgemm.sh /tmp/install_deepgemm.sh
RUN --mount=type=cache,target=/root/.cache/uv \
400
    VLLM_DOCKER_BUILD_CONTEXT=1 TORCH_CUDA_ARCH_LIST="9.0a 10.0a" /tmp/install_deepgemm.sh --cuda-version "${CUDA_VERSION}" ${DEEPGEMM_GIT_REF:+--ref "$DEEPGEMM_GIT_REF"}
401

402
403
404
405
406
407
408
409
410
411
412
COPY tools/install_gdrcopy.sh install_gdrcopy.sh
RUN set -eux; \
    case "${TARGETPLATFORM}" in \
      linux/arm64) UUARCH="aarch64" ;; \
      linux/amd64) UUARCH="x64" ;; \
      *) echo "Unsupported TARGETPLATFORM: ${TARGETPLATFORM}" >&2; exit 1 ;; \
    esac; \
    ./install_gdrcopy.sh "${GDRCOPY_OS_VERSION}" "${GDRCOPY_CUDA_VERSION}" "${UUARCH}"; \
    rm ./install_gdrcopy.sh

# Install EP kernels(pplx-kernels and DeepEP)
413
414
COPY tools/ep_kernels/install_python_libraries.sh install_python_libraries.sh
ENV CUDA_HOME=/usr/local/cuda
415
RUN export TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST:-9.0a 10.0a+PTX}" \
416
    && bash install_python_libraries.sh
417

418
419
420
421
422
423
# CUDA image changed from /usr/local/nvidia to /usr/local/cuda in 12.8 but will
# return to /usr/local/nvidia in 13.0 to allow container providers to mount drivers
# consistently from the host (see https://github.com/vllm-project/vllm/issues/18859).
# Until then, add /usr/local/nvidia/lib64 before the image cuda path to allow override.
ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}

424
#################### vLLM installation IMAGE ####################
Stephen Krider's avatar
Stephen Krider committed
425

426
427
428
429
#################### TEST IMAGE ####################
# image to run unit testing suite
# note that this uses vllm installed by `pip`
FROM vllm-base AS test
Stephen Krider's avatar
Stephen Krider committed
430

431
ADD . /vllm-workspace/
Stephen Krider's avatar
Stephen Krider committed
432

433
434
435
436
ARG PYTHON_VERSION

ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
437
ARG PYTORCH_CUDA_INDEX_BASE_URL
438

439
440
441
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
442
ENV UV_INDEX_STRATEGY="unsafe-best-match"
443
444
# Use copy mode to avoid hardlink failures with Docker cache mounts
ENV UV_LINK_MODE=copy
445

Huy Do's avatar
Huy Do committed
446
# install development dependencies (for testing)
447
RUN --mount=type=cache,target=/root/.cache/uv \
448
449
    CUDA_MAJOR="${CUDA_VERSION%%.*}"; \
    if [ "$CUDA_MAJOR" -ge 12 ]; then \
450
451
        uv pip install --system -r requirements/dev.txt \
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \
452
    fi
453

youkaichao's avatar
youkaichao committed
454
# install development dependencies (for testing)
455
RUN --mount=type=cache,target=/root/.cache/uv \
456
    uv pip install --system -e tests/vllm_test_utils
youkaichao's avatar
youkaichao committed
457

458
# enable fast downloads from hf (for testing)
459
RUN --mount=type=cache,target=/root/.cache/uv \
460
    uv pip install --system hf_transfer
461
462
ENV HF_HUB_ENABLE_HF_TRANSFER 1

Joe Runde's avatar
Joe Runde committed
463
# Copy in the v1 package for testing (it isn't distributed yet)
464
COPY vllm/v1 /usr/local/lib/python${PYTHON_VERSION}/dist-packages/vllm/v1
Joe Runde's avatar
Joe Runde committed
465

466
467
# Source code is used in the `python_only_compile.sh` test
# We hide it inside `src/` so that this source code
468
# will not be imported by other tests
469
470
RUN mkdir src
RUN mv vllm src/vllm
471
#################### TEST IMAGE ####################
Stephen Krider's avatar
Stephen Krider committed
472

Simon Mo's avatar
Simon Mo committed
473
#################### OPENAI API SERVER ####################
474
475
# base openai image with additional requirements, for any subsequent openai-style images
FROM vllm-base AS vllm-openai-base
476
ARG TARGETPLATFORM
477
ARG INSTALL_KV_CONNECTORS=false
478

479
480
481
ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL

482
483
484
485
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500

486
487
COPY requirements/kv_connectors.txt requirements/kv_connectors.txt

488
# install additional dependencies for openai api server
489
RUN --mount=type=cache,target=/root/.cache/uv \
490
491
492
    if [ "$INSTALL_KV_CONNECTORS" = "true" ]; then \
        uv pip install --system -r requirements/kv_connectors.txt; \
    fi; \
493
    if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
494
        BITSANDBYTES_VERSION="0.42.0"; \
495
    else \
496
497
        BITSANDBYTES_VERSION="0.46.1"; \
    fi; \
498
    uv pip install --system accelerate hf_transfer modelscope "bitsandbytes>=${BITSANDBYTES_VERSION}" 'timm>=1.0.17' 'runai-model-streamer[s3,gcs]>=0.14.0'
499

yhu422's avatar
yhu422 committed
500
501
ENV VLLM_USAGE_SOURCE production-docker-image

502
503
504
# define sagemaker first, so it is not default from `docker build`
FROM vllm-openai-base AS vllm-sagemaker

505
COPY examples/online_serving/sagemaker-entrypoint.sh .
506
507
508
509
510
RUN chmod +x sagemaker-entrypoint.sh
ENTRYPOINT ["./sagemaker-entrypoint.sh"]

FROM vllm-openai-base AS vllm-openai

511
ENTRYPOINT ["vllm", "serve"]
Simon Mo's avatar
Simon Mo committed
512
#################### OPENAI API SERVER ####################