Dockerfile 22.8 KB
Newer Older
Simon Mo's avatar
Simon Mo committed
1
2
3
# The vLLM Dockerfile is used to construct vLLM image that can be directly used
# to run the OpenAI compatible server.

4
# Please update any changes made here to
5
6
# docs/contributing/dockerfile/dockerfile.md and
# docs/assets/contributing/dockerfile-stages-dependency.png
7

Huy Do's avatar
Huy Do committed
8
ARG CUDA_VERSION=12.8.1
9
10
11
12
13
14
15
16
17
ARG PYTHON_VERSION=3.12

# By parameterizing the base images, we allow third-party to use their own
# base images. One use case is hermetic builds with base images stored in
# private registries that use a different repository naming conventions.
#
# Example:
# docker build --build-arg BUILD_BASE_IMAGE=registry.acme.org/mirror/nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04
ARG BUILD_BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04
18
# TODO: Restore to base image after FlashInfer AOT wheel fixed
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
ARG FINAL_BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04

# By parameterizing the Deadsnakes repository URL, we allow third-party to use
# their own mirror. When doing so, we don't benefit from the transparent
# installation of the GPG key of the PPA, as done by add-apt-repository, so we
# also need a URL for the GPG key.
ARG DEADSNAKES_MIRROR_URL
ARG DEADSNAKES_GPGKEY_URL

# The PyPA get-pip.py script is a self contained script+zip file, that provides
# both the installer script and the pip base85-encoded zip archive. This allows
# bootstrapping pip in environment where a dsitribution package does not exist.
#
# By parameterizing the URL for get-pip.py installation script, we allow
# third-party to use their own copy of the script stored in a private mirror.
# We set the default value to the PyPA owned get-pip.py script.
#
# Reference: https://pip.pypa.io/en/stable/installation/#get-pip-py
ARG GET_PIP_URL="https://bootstrap.pypa.io/get-pip.py"

# PIP supports fetching the packages from custom indexes, allowing third-party
# to host the packages in private mirrors. The PIP_INDEX_URL and
# PIP_EXTRA_INDEX_URL are standard PIP environment variables to override the
# default indexes. By letting them empty by default, PIP will use its default
# indexes if the build process doesn't override the indexes.
#
# Uv uses different variables. We set them by default to the same values as
# PIP, but they can be overridden.
ARG PIP_INDEX_URL
ARG PIP_EXTRA_INDEX_URL
ARG UV_INDEX_URL=${PIP_INDEX_URL}
ARG UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}

# PyTorch provides its own indexes for standard and nightly builds
ARG PYTORCH_CUDA_INDEX_BASE_URL=https://download.pytorch.org/whl
ARG PYTORCH_CUDA_NIGHTLY_INDEX_BASE_URL=https://download.pytorch.org/whl/nightly

# PIP supports multiple authentication schemes, including keyring
# By parameterizing the PIP_KEYRING_PROVIDER variable and setting it to
# disabled by default, we allow third-party to use keyring authentication for
# their private Python indexes, while not changing the default behavior which
# is no authentication.
#
# Reference: https://pip.pypa.io/en/stable/topics/authentication/#keyring-support
ARG PIP_KEYRING_PROVIDER=disabled
ARG UV_KEYRING_PROVIDER=${PIP_KEYRING_PROVIDER}

66
# Flag enables built-in KV-connector dependency libs into docker images
67
68
ARG INSTALL_KV_CONNECTORS=false

Simon Mo's avatar
Simon Mo committed
69
#################### BASE BUILD IMAGE ####################
70
# prepare basic build environment
71
72
73
FROM ${BUILD_BASE_IMAGE} AS base
ARG CUDA_VERSION
ARG PYTHON_VERSION
74
ARG TARGETPLATFORM
75
ARG INSTALL_KV_CONNECTORS=false
76
77
ENV DEBIAN_FRONTEND=noninteractive

78
79
80
81
ARG DEADSNAKES_MIRROR_URL
ARG DEADSNAKES_GPGKEY_URL
ARG GET_PIP_URL

82
83
84
85
86
# Install Python and other dependencies
RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
    && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \
    && apt-get update -y \
    && apt-get install -y ccache software-properties-common git curl sudo \
87
88
89
90
91
92
93
94
95
96
97
98
99
    && if [ ! -z ${DEADSNAKES_MIRROR_URL} ] ; then \
        if [ ! -z "${DEADSNAKES_GPGKEY_URL}" ] ; then \
            mkdir -p -m 0755 /etc/apt/keyrings ; \
            curl -L ${DEADSNAKES_GPGKEY_URL} | gpg --dearmor > /etc/apt/keyrings/deadsnakes.gpg ; \
            sudo chmod 644 /etc/apt/keyrings/deadsnakes.gpg ; \
            echo "deb [signed-by=/etc/apt/keyrings/deadsnakes.gpg] ${DEADSNAKES_MIRROR_URL} $(lsb_release -cs) main" > /etc/apt/sources.list.d/deadsnakes.list ; \
        fi ; \
    else \
        for i in 1 2 3; do \
            add-apt-repository -y ppa:deadsnakes/ppa && break || \
            { echo "Attempt $i failed, retrying in 5s..."; sleep 5; }; \
        done ; \
    fi \
100
101
102
103
104
    && apt-get update -y \
    && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv \
    && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \
    && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \
    && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \
105
    && curl -sS ${GET_PIP_URL} | python${PYTHON_VERSION} \
106
    && python3 --version && python3 -m pip --version
107
108
109
110
111
112
113

ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL
ARG PYTORCH_CUDA_NIGHTLY_INDEX_BASE_URL
ARG PIP_KEYRING_PROVIDER UV_KEYRING_PROVIDER

114
115
116
# Install uv for faster pip installs
RUN --mount=type=cache,target=/root/.cache/uv \
    python3 -m pip install uv
Stephen Krider's avatar
Stephen Krider committed
117

118
119
120
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
121
ENV UV_INDEX_STRATEGY="unsafe-best-match"
122

123
124
125
126
127
128
129
130
# Upgrade to GCC 10 to avoid https://gcc.gnu.org/bugzilla/show_bug.cgi?id=92519
# as it was causing spam when compiling the CUTLASS kernels
RUN apt-get install -y gcc-10 g++-10
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 110 --slave /usr/bin/g++ g++ /usr/bin/g++-10
RUN <<EOF
gcc --version
EOF

131
132
133
134
# Workaround for https://github.com/openai/triton/issues/2507 and
# https://github.com/pytorch/pytorch/issues/107960 -- hopefully
# this won't be needed for future versions of this docker image
# or future versions of triton.
135
RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/
136

Stephen Krider's avatar
Stephen Krider committed
137
138
WORKDIR /workspace

139
140
# install build and runtime dependencies

141
142
143
144
# arm64 (GH200) build follows the practice of "use existing pytorch" build,
# we need to install torch and torchvision from the nightly builds first,
# pytorch will not appear as a vLLM dependency in all of the following steps
# after this step
145
RUN --mount=type=cache,target=/root/.cache/uv \
146
    if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
147
148
149
150
151
152
        uv pip install --system \
            --index-url ${PYTORCH_CUDA_NIGHTLY_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
            "torch==2.8.0.dev20250318+cu128" "torchvision==0.22.0.dev20250319";  \
        uv pip install --system \
            --index-url ${PYTORCH_CUDA_NIGHTLY_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
            --pre pytorch_triton==3.3.0+gitab727c40; \
153
    fi
Mor Zusman's avatar
Mor Zusman committed
154

155
156
COPY requirements/common.txt requirements/common.txt
COPY requirements/cuda.txt requirements/cuda.txt
157
RUN --mount=type=cache,target=/root/.cache/uv \
Huy Do's avatar
Huy Do committed
158
    uv pip install --system -r requirements/cuda.txt \
159
    --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
160

161
162
163
164
# cuda arch list used by torch
# can be useful for both `dev` and `test`
# explicitly set the list to avoid issues with torch 2.2
# see https://github.com/pytorch/pytorch/pull/123243
165
ARG torch_cuda_arch_list='7.0 7.5 8.0 8.9 9.0 10.0 12.0'
166
ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
Simon Mo's avatar
Simon Mo committed
167
168
#################### BASE BUILD IMAGE ####################

169
#################### WHEEL BUILD IMAGE ####################
170
FROM base AS build
171
ARG TARGETPLATFORM
172

173
174
175
176
ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL

177
# install build dependencies
178
COPY requirements/build.txt requirements/build.txt
179

180
181
182
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
183
ENV UV_INDEX_STRATEGY="unsafe-best-match"
184

185
RUN --mount=type=cache,target=/root/.cache/uv \
Huy Do's avatar
Huy Do committed
186
    uv pip install --system -r requirements/build.txt \
187
    --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
188

189
COPY . .
190
191
ARG GIT_REPO_CHECK=0
RUN --mount=type=bind,source=.git,target=.git \
192
    if [ "$GIT_REPO_CHECK" != "0" ]; then bash tools/check_repo.sh ; fi
Stephen Krider's avatar
Stephen Krider committed
193
194

# max jobs used by Ninja to build extensions
195
196
ARG max_jobs=2
ENV MAX_JOBS=${max_jobs}
197
198
199
# number of threads used by nvcc
ARG nvcc_threads=8
ENV NVCC_THREADS=$nvcc_threads
200

201
ARG USE_SCCACHE
202
203
ARG SCCACHE_DOWNLOAD_URL=https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-x86_64-unknown-linux-musl.tar.gz
ARG SCCACHE_ENDPOINT
204
205
ARG SCCACHE_BUCKET_NAME=vllm-build-sccache
ARG SCCACHE_REGION_NAME=us-west-2
206
ARG SCCACHE_S3_NO_CREDENTIALS=0
207
208

# Flag to control whether to use pre-built vLLM wheels
209
210
211
212
213
214
215
216
217
218
ARG VLLM_USE_PRECOMPILED
# TODO: in setup.py VLLM_USE_PRECOMPILED is sensitive to truthiness, it will take =0 as "true", this should be fixed
ENV VLLM_USE_PRECOMPILED=""
RUN if [ "${VLLM_USE_PRECOMPILED}" = "1" ]; then \
        export VLLM_USE_PRECOMPILED=1 && \
        echo "Using precompiled wheels"; \
    else \
        unset VLLM_USE_PRECOMPILED && \
        echo "Leaving VLLM_USE_PRECOMPILED unset to build wheels from source"; \
    fi
219

220
# if USE_SCCACHE is set, use sccache to speed up compilation
221
RUN --mount=type=cache,target=/root/.cache/uv \
222
    --mount=type=bind,source=.git,target=.git \
223
224
    if [ "$USE_SCCACHE" = "1" ]; then \
        echo "Installing sccache..." \
225
        && curl -L -o sccache.tar.gz ${SCCACHE_DOWNLOAD_URL} \
226
227
228
        && tar -xzf sccache.tar.gz \
        && sudo mv sccache-v0.8.1-x86_64-unknown-linux-musl/sccache /usr/bin/sccache \
        && rm -rf sccache.tar.gz sccache-v0.8.1-x86_64-unknown-linux-musl \
229
        && if [ ! -z ${SCCACHE_ENDPOINT} ] ; then export SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT} ; fi \
230
231
        && export SCCACHE_BUCKET=${SCCACHE_BUCKET_NAME} \
        && export SCCACHE_REGION=${SCCACHE_REGION_NAME} \
232
        && export SCCACHE_S3_NO_CREDENTIALS=${SCCACHE_S3_NO_CREDENTIALS} \
233
        && export SCCACHE_IDLE_TIMEOUT=0 \
234
        && export CMAKE_BUILD_TYPE=Release \
235
        && sccache --show-stats \
236
        && python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38 \
237
238
239
        && sccache --show-stats; \
    fi

240
241
ENV CCACHE_DIR=/root/.cache/ccache
RUN --mount=type=cache,target=/root/.cache/ccache \
242
    --mount=type=cache,target=/root/.cache/uv \
243
    --mount=type=bind,source=.git,target=.git  \
244
    if [ "$USE_SCCACHE" != "1" ]; then \
245
246
247
        # Clean any existing CMake artifacts
        rm -rf .deps && \
        mkdir -p .deps && \
248
        python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38; \
249
    fi
250

251
# Check the size of the wheel if RUN_WHEEL_CHECK is true
252
COPY .buildkite/check-wheel-size.py check-wheel-size.py
253
# sync the default value with .buildkite/check-wheel-size.py
254
ARG VLLM_MAX_SIZE_MB=400
255
256
257
258
259
260
261
ENV VLLM_MAX_SIZE_MB=$VLLM_MAX_SIZE_MB
ARG RUN_WHEEL_CHECK=true
RUN if [ "$RUN_WHEEL_CHECK" = "true" ]; then \
        python3 check-wheel-size.py dist; \
    else \
        echo "Skipping wheel size check."; \
    fi
Simon Mo's avatar
Simon Mo committed
262
#################### EXTENSION Build IMAGE ####################
Stephen Krider's avatar
Stephen Krider committed
263

264
#################### DEV IMAGE ####################
265
FROM base AS dev
266

267
268
269
270
ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL

271
272
273
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
274
275
ENV UV_INDEX_STRATEGY="unsafe-best-match"

276
277
278
COPY requirements/lint.txt requirements/lint.txt
COPY requirements/test.txt requirements/test.txt
COPY requirements/dev.txt requirements/dev.txt
279
RUN --mount=type=cache,target=/root/.cache/uv \
Huy Do's avatar
Huy Do committed
280
    uv pip install --system -r requirements/dev.txt \
281
    --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
282
#################### DEV IMAGE ####################
283

284
285
#################### vLLM installation IMAGE ####################
# image with vLLM installed
286
287
288
FROM ${FINAL_BASE_IMAGE} AS vllm-base
ARG CUDA_VERSION
ARG PYTHON_VERSION
289
ARG INSTALL_KV_CONNECTORS=false
Simon Mo's avatar
Simon Mo committed
290
WORKDIR /vllm-workspace
291
ENV DEBIAN_FRONTEND=noninteractive
292
293
ARG TARGETPLATFORM

Kebe's avatar
Kebe committed
294
295
SHELL ["/bin/bash", "-c"]

296
297
298
299
ARG DEADSNAKES_MIRROR_URL
ARG DEADSNAKES_GPGKEY_URL
ARG GET_PIP_URL

300
301
RUN PYTHON_VERSION_STR=$(echo ${PYTHON_VERSION} | sed 's/\.//g') && \
    echo "export PYTHON_VERSION_STR=${PYTHON_VERSION_STR}" >> /etc/environment
302

303
304
305
306
307
308
# Install Python and other dependencies
RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
    && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \
    && apt-get update -y \
    && apt-get install -y ccache software-properties-common git curl wget sudo vim python3-pip \
    && apt-get install -y ffmpeg libsm6 libxext6 libgl1 \
309
310
311
312
313
314
315
316
317
318
319
320
321
    && if [ ! -z ${DEADSNAKES_MIRROR_URL} ] ; then \
        if [ ! -z "${DEADSNAKES_GPGKEY_URL}" ] ; then \
            mkdir -p -m 0755 /etc/apt/keyrings ; \
            curl -L ${DEADSNAKES_GPGKEY_URL} | gpg --dearmor > /etc/apt/keyrings/deadsnakes.gpg ; \
            sudo chmod 644 /etc/apt/keyrings/deadsnakes.gpg ; \
            echo "deb [signed-by=/etc/apt/keyrings/deadsnakes.gpg] ${DEADSNAKES_MIRROR_URL} $(lsb_release -cs) main" > /etc/apt/sources.list.d/deadsnakes.list ; \
        fi ; \
    else \
        for i in 1 2 3; do \
            add-apt-repository -y ppa:deadsnakes/ppa && break || \
            { echo "Attempt $i failed, retrying in 5s..."; sleep 5; }; \
        done ; \
    fi \
322
323
324
325
326
    && apt-get update -y \
    && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv libibverbs-dev \
    && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \
    && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \
    && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \
327
    && curl -sS ${GET_PIP_URL} | python${PYTHON_VERSION} \
328
    && python3 --version && python3 -m pip --version
329
330
331
332
333
334
335

ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL
ARG PYTORCH_CUDA_NIGHTLY_INDEX_BASE_URL
ARG PIP_KEYRING_PROVIDER UV_KEYRING_PROVIDER

336
337
338
# Install uv for faster pip installs
RUN --mount=type=cache,target=/root/.cache/uv \
    python3 -m pip install uv
339

340
341
342
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
343
ENV UV_INDEX_STRATEGY="unsafe-best-match"
344

345
346
347
348
# Workaround for https://github.com/openai/triton/issues/2507 and
# https://github.com/pytorch/pytorch/issues/107960 -- hopefully
# this won't be needed for future versions of this docker image
# or future versions of triton.
349
RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/
350

351
352
353
354
# arm64 (GH200) build follows the practice of "use existing pytorch" build,
# we need to install torch and torchvision from the nightly builds first,
# pytorch will not appear as a vLLM dependency in all of the following steps
# after this step
355
RUN --mount=type=cache,target=/root/.cache/uv \
356
    if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
357
358
359
360
361
362
        uv pip install --system \
            --index-url ${PYTORCH_CUDA_NIGHTLY_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
            "torch==2.8.0.dev20250318+cu128" "torchvision==0.22.0.dev20250319" ; \
        uv pip install --system \
            --index-url ${PYTORCH_CUDA_NIGHTLY_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
            --pre pytorch_triton==3.3.0+gitab727c40 ; \
363
364
    fi

365
# Install vllm wheel first, so that torch etc will be installed.
366
RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \
367
    --mount=type=cache,target=/root/.cache/uv \
Huy Do's avatar
Huy Do committed
368
    uv pip install --system dist/*.whl --verbose \
369
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
Mor Zusman's avatar
Mor Zusman committed
370

371
# If we need to build FlashInfer wheel before its release:
372
# $ # Note we remove 7.0 from the arch list compared to the list below, since FlashInfer only supports sm75+
373
# $ export TORCH_CUDA_ARCH_LIST='7.5 8.0 8.9 9.0a 10.0a 12.0'
374
375
# $ git clone https://github.com/flashinfer-ai/flashinfer.git --recursive
# $ cd flashinfer
376
377
378
379
380
381
# $ git checkout v0.2.6.post1
# $ python -m flashinfer.aot
# $ python -m build --no-isolation --wheel
# $ ls -la dist
# -rw-rw-r-- 1 mgoin mgoin 205M Jun  9 18:03 flashinfer_python-0.2.6.post1-cp39-abi3-linux_x86_64.whl
# $ # upload the wheel to a public location, e.g. https://wheels.vllm.ai/flashinfer/v0.2.6.post1/flashinfer_python-0.2.6.post1-cp39-abi3-linux_x86_64.whl
382

383
# Install FlashInfer from source
384
ARG FLASHINFER_GIT_REPO="https://github.com/flashinfer-ai/flashinfer.git"
385
386
# Keep this in sync with https://github.com/vllm-project/vllm/blob/main/requirements/cuda.txt
# We use `--force-reinstall --no-deps` to avoid issues with the existing FlashInfer wheel.
387
ARG FLASHINFER_GIT_REF="v0.2.9rc2"
388
389
RUN --mount=type=cache,target=/root/.cache/uv bash - <<'BASH'
  . /etc/environment
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
    git clone --depth 1 --recursive --shallow-submodules \
        --branch ${FLASHINFER_GIT_REF} \
        ${FLASHINFER_GIT_REPO} flashinfer
    # Exclude CUDA arches for older versions (11.x and 12.0-12.7)
    # TODO: Update this to allow setting TORCH_CUDA_ARCH_LIST as a build arg.
    if [[ "${CUDA_VERSION}" == 11.* ]]; then
        FI_TORCH_CUDA_ARCH_LIST="7.5 8.0 8.9"
    elif [[ "${CUDA_VERSION}" == 12.[0-7]* ]]; then
        FI_TORCH_CUDA_ARCH_LIST="7.5 8.0 8.9 9.0a"
    else
        # CUDA 12.8+ supports 10.0a and 12.0
        FI_TORCH_CUDA_ARCH_LIST="7.5 8.0 8.9 9.0a 10.0a 12.0"
    fi
    echo "🏗️  Building FlashInfer for arches: ${FI_TORCH_CUDA_ARCH_LIST}"
    # Needed to build AOT kernels
    pushd flashinfer
        TORCH_CUDA_ARCH_LIST="${FI_TORCH_CUDA_ARCH_LIST}" \
            python3 -m flashinfer.aot
        TORCH_CUDA_ARCH_LIST="${FI_TORCH_CUDA_ARCH_LIST}" \
409
            uv pip install --system --no-build-isolation --force-reinstall --no-deps .
410
411
    popd
    rm -rf flashinfer
412
BASH
413
COPY examples examples
414
COPY benchmarks benchmarks
415
COPY ./vllm/collect_env.py .
416

Huy Do's avatar
Huy Do committed
417
418
419
420
RUN --mount=type=cache,target=/root/.cache/uv \
. /etc/environment && \
uv pip list

421
# Even when we build Flashinfer with AOT mode, there's still
422
423
424
# some issues w.r.t. JIT compilation. Therefore we need to
# install build dependencies for JIT compilation.
# TODO: Remove this once FlashInfer AOT wheel is fixed
425
COPY requirements/build.txt requirements/build.txt
426
RUN --mount=type=cache,target=/root/.cache/uv \
Huy Do's avatar
Huy Do committed
427
    uv pip install --system -r requirements/build.txt \
428
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
429

430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
# Install DeepGEMM from source
ARG DEEPGEMM_GIT_REPO="https://github.com/deepseek-ai/DeepGEMM.git"
ARG DEEPGEMM_GIT_REF="187656694f7f69e3e7975617a68bc3387680a7e1"
RUN --mount=type=cache,target=/root/.cache/uv bash - <<'BASH'
  . /etc/environment
    CUDA_MAJOR="${CUDA_VERSION%%.*}"
    CUDA_MINOR="${CUDA_VERSION#${CUDA_MAJOR}.}"
    CUDA_MINOR="${CUDA_MINOR%%.*}"
    if [ "$CUDA_MAJOR" -ge 12 ] && [ "$CUDA_MINOR" -ge 8 ]; then
        git clone --recursive --shallow-submodules \
            ${DEEPGEMM_GIT_REPO} deepgemm
        echo "🏗️  Building DeepGEMM"
        pushd deepgemm
            git checkout ${DEEPGEMM_GIT_REF}
            # Build DeepGEMM
            # (Based on https://github.com/deepseek-ai/DeepGEMM/blob/main/install.sh)
            rm -rf build dist
            rm -rf *.egg-info
            python3 setup.py bdist_wheel
            uv pip install --system dist/*.whl
        popd
        rm -rf deepgemm
    else
        echo "Skipping DeepGEMM installation (requires CUDA 12.8+ but got ${CUDA_VERSION})"
    fi
BASH

457
#################### vLLM installation IMAGE ####################
Stephen Krider's avatar
Stephen Krider committed
458

459
460
461
462
#################### TEST IMAGE ####################
# image to run unit testing suite
# note that this uses vllm installed by `pip`
FROM vllm-base AS test
Stephen Krider's avatar
Stephen Krider committed
463

464
ADD . /vllm-workspace/
Stephen Krider's avatar
Stephen Krider committed
465

466
467
468
469
470
ARG PYTHON_VERSION

ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL

471
472
473
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
474
ENV UV_INDEX_STRATEGY="unsafe-best-match"
475

Huy Do's avatar
Huy Do committed
476
# install development dependencies (for testing)
477
RUN --mount=type=cache,target=/root/.cache/uv \
478
479
480
481
    CUDA_MAJOR="${CUDA_VERSION%%.*}"; \
    if [ "$CUDA_MAJOR" -ge 12 ]; then \
        uv pip install --system -r requirements/dev.txt; \
    fi
482

youkaichao's avatar
youkaichao committed
483
# install development dependencies (for testing)
484
RUN --mount=type=cache,target=/root/.cache/uv \
485
    uv pip install --system -e tests/vllm_test_utils
youkaichao's avatar
youkaichao committed
486

487
# enable fast downloads from hf (for testing)
488
RUN --mount=type=cache,target=/root/.cache/uv \
489
    uv pip install --system hf_transfer
490
491
ENV HF_HUB_ENABLE_HF_TRANSFER 1

Joe Runde's avatar
Joe Runde committed
492
# Copy in the v1 package for testing (it isn't distributed yet)
493
COPY vllm/v1 /usr/local/lib/python${PYTHON_VERSION}/dist-packages/vllm/v1
Joe Runde's avatar
Joe Runde committed
494

495
496
497
498
499
# doc requires source code
# we hide them inside `test_docs/` , so that this source code
# will not be imported by other tests
RUN mkdir test_docs
RUN mv docs test_docs/
500
RUN cp -r examples test_docs/
501
RUN mv vllm test_docs/
502
RUN mv mkdocs.yaml test_docs/
503
#################### TEST IMAGE ####################
Stephen Krider's avatar
Stephen Krider committed
504

Simon Mo's avatar
Simon Mo committed
505
#################### OPENAI API SERVER ####################
506
507
# base openai image with additional requirements, for any subsequent openai-style images
FROM vllm-base AS vllm-openai-base
508
ARG TARGETPLATFORM
509
ARG INSTALL_KV_CONNECTORS=false
510

511
512
513
ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL

514
515
516
517
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500

518
519
COPY requirements/kv_connectors.txt requirements/kv_connectors.txt

520
# install additional dependencies for openai api server
521
RUN --mount=type=cache,target=/root/.cache/uv \
522
523
524
    if [ "$INSTALL_KV_CONNECTORS" = "true" ]; then \
        uv pip install --system -r requirements/kv_connectors.txt; \
    fi; \
525
    if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
526
        BITSANDBYTES_VERSION="0.42.0"; \
527
    else \
528
529
        BITSANDBYTES_VERSION="0.46.1"; \
    fi; \
530
    uv pip install --system accelerate hf_transfer modelscope "bitsandbytes>=${BITSANDBYTES_VERSION}" 'timm==0.9.10' boto3 runai-model-streamer runai-model-streamer[s3]
531

yhu422's avatar
yhu422 committed
532
533
ENV VLLM_USAGE_SOURCE production-docker-image

534
535
536
# define sagemaker first, so it is not default from `docker build`
FROM vllm-openai-base AS vllm-sagemaker

537
COPY examples/online_serving/sagemaker-entrypoint.sh .
538
539
540
541
542
RUN chmod +x sagemaker-entrypoint.sh
ENTRYPOINT ["./sagemaker-entrypoint.sh"]

FROM vllm-openai-base AS vllm-openai

Stephen Krider's avatar
Stephen Krider committed
543
ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
Simon Mo's avatar
Simon Mo committed
544
#################### OPENAI API SERVER ####################