Dockerfile 22.2 KB
Newer Older
1

Simon Mo's avatar
Simon Mo committed
2
3
4
# The vLLM Dockerfile is used to construct vLLM image that can be directly used
# to run the OpenAI compatible server.

5
# Please update any changes made here to
6
7
# docs/contributing/dockerfile/dockerfile.md and
# docs/assets/contributing/dockerfile-stages-dependency.png
8

Huy Do's avatar
Huy Do committed
9
ARG CUDA_VERSION=12.8.1
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
ARG PYTHON_VERSION=3.12

# By parameterizing the base images, we allow third-party to use their own
# base images. One use case is hermetic builds with base images stored in
# private registries that use a different repository naming conventions.
#
# Example:
# docker build --build-arg BUILD_BASE_IMAGE=registry.acme.org/mirror/nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04
ARG BUILD_BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04
ARG FINAL_BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04

# By parameterizing the Deadsnakes repository URL, we allow third-party to use
# their own mirror. When doing so, we don't benefit from the transparent
# installation of the GPG key of the PPA, as done by add-apt-repository, so we
# also need a URL for the GPG key.
ARG DEADSNAKES_MIRROR_URL
ARG DEADSNAKES_GPGKEY_URL

# The PyPA get-pip.py script is a self contained script+zip file, that provides
# both the installer script and the pip base85-encoded zip archive. This allows
# bootstrapping pip in environment where a dsitribution package does not exist.
#
# By parameterizing the URL for get-pip.py installation script, we allow
# third-party to use their own copy of the script stored in a private mirror.
# We set the default value to the PyPA owned get-pip.py script.
#
# Reference: https://pip.pypa.io/en/stable/installation/#get-pip-py
ARG GET_PIP_URL="https://bootstrap.pypa.io/get-pip.py"

# PIP supports fetching the packages from custom indexes, allowing third-party
# to host the packages in private mirrors. The PIP_INDEX_URL and
# PIP_EXTRA_INDEX_URL are standard PIP environment variables to override the
# default indexes. By letting them empty by default, PIP will use its default
# indexes if the build process doesn't override the indexes.
#
# Uv uses different variables. We set them by default to the same values as
# PIP, but they can be overridden.
ARG PIP_INDEX_URL
ARG PIP_EXTRA_INDEX_URL
ARG UV_INDEX_URL=${PIP_INDEX_URL}
ARG UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}

# PyTorch provides its own indexes for standard and nightly builds
ARG PYTORCH_CUDA_INDEX_BASE_URL=https://download.pytorch.org/whl
ARG PYTORCH_CUDA_NIGHTLY_INDEX_BASE_URL=https://download.pytorch.org/whl/nightly

# PIP supports multiple authentication schemes, including keyring
# By parameterizing the PIP_KEYRING_PROVIDER variable and setting it to
# disabled by default, we allow third-party to use keyring authentication for
# their private Python indexes, while not changing the default behavior which
# is no authentication.
#
# Reference: https://pip.pypa.io/en/stable/topics/authentication/#keyring-support
ARG PIP_KEYRING_PROVIDER=disabled
ARG UV_KEYRING_PROVIDER=${PIP_KEYRING_PROVIDER}

66
# Flag enables built-in KV-connector dependency libs into docker images
67
68
ARG INSTALL_KV_CONNECTORS=false

Simon Mo's avatar
Simon Mo committed
69
#################### BASE BUILD IMAGE ####################
70
# prepare basic build environment
71
72
73
FROM ${BUILD_BASE_IMAGE} AS base
ARG CUDA_VERSION
ARG PYTHON_VERSION
74
ARG TARGETPLATFORM
75
ARG INSTALL_KV_CONNECTORS=false
76
77
ENV DEBIAN_FRONTEND=noninteractive

78
79
80
81
ARG DEADSNAKES_MIRROR_URL
ARG DEADSNAKES_GPGKEY_URL
ARG GET_PIP_URL

82
83
84
85
86
# Install Python and other dependencies
RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
    && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \
    && apt-get update -y \
    && apt-get install -y ccache software-properties-common git curl sudo \
87
88
89
90
91
92
93
94
95
96
97
98
99
    && if [ ! -z ${DEADSNAKES_MIRROR_URL} ] ; then \
        if [ ! -z "${DEADSNAKES_GPGKEY_URL}" ] ; then \
            mkdir -p -m 0755 /etc/apt/keyrings ; \
            curl -L ${DEADSNAKES_GPGKEY_URL} | gpg --dearmor > /etc/apt/keyrings/deadsnakes.gpg ; \
            sudo chmod 644 /etc/apt/keyrings/deadsnakes.gpg ; \
            echo "deb [signed-by=/etc/apt/keyrings/deadsnakes.gpg] ${DEADSNAKES_MIRROR_URL} $(lsb_release -cs) main" > /etc/apt/sources.list.d/deadsnakes.list ; \
        fi ; \
    else \
        for i in 1 2 3; do \
            add-apt-repository -y ppa:deadsnakes/ppa && break || \
            { echo "Attempt $i failed, retrying in 5s..."; sleep 5; }; \
        done ; \
    fi \
100
101
102
103
104
    && apt-get update -y \
    && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv \
    && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \
    && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \
    && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \
105
    && curl -sS ${GET_PIP_URL} | python${PYTHON_VERSION} \
106
    && python3 --version && python3 -m pip --version
107
108
109
110
111
112
113

ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL
ARG PYTORCH_CUDA_NIGHTLY_INDEX_BASE_URL
ARG PIP_KEYRING_PROVIDER UV_KEYRING_PROVIDER

114
115
116
# Install uv for faster pip installs
RUN --mount=type=cache,target=/root/.cache/uv \
    python3 -m pip install uv
Stephen Krider's avatar
Stephen Krider committed
117

118
119
120
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
121
ENV UV_INDEX_STRATEGY="unsafe-best-match"
122

123
124
125
126
127
128
129
130
# Upgrade to GCC 10 to avoid https://gcc.gnu.org/bugzilla/show_bug.cgi?id=92519
# as it was causing spam when compiling the CUTLASS kernels
RUN apt-get install -y gcc-10 g++-10
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 110 --slave /usr/bin/g++ g++ /usr/bin/g++-10
RUN <<EOF
gcc --version
EOF

131
132
133
134
# Workaround for https://github.com/openai/triton/issues/2507 and
# https://github.com/pytorch/pytorch/issues/107960 -- hopefully
# this won't be needed for future versions of this docker image
# or future versions of triton.
135
RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/
136

Stephen Krider's avatar
Stephen Krider committed
137
138
WORKDIR /workspace

139
140
# install build and runtime dependencies

141
142
143
144
# arm64 (GH200) build follows the practice of "use existing pytorch" build,
# we need to install torch and torchvision from the nightly builds first,
# pytorch will not appear as a vLLM dependency in all of the following steps
# after this step
145
RUN --mount=type=cache,target=/root/.cache/uv \
146
    if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
147
148
149
150
151
152
        uv pip install --system \
            --index-url ${PYTORCH_CUDA_NIGHTLY_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
            "torch==2.8.0.dev20250318+cu128" "torchvision==0.22.0.dev20250319";  \
        uv pip install --system \
            --index-url ${PYTORCH_CUDA_NIGHTLY_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
            --pre pytorch_triton==3.3.0+gitab727c40; \
153
    fi
Mor Zusman's avatar
Mor Zusman committed
154

155
156
COPY requirements/common.txt requirements/common.txt
COPY requirements/cuda.txt requirements/cuda.txt
157
RUN --mount=type=cache,target=/root/.cache/uv \
Huy Do's avatar
Huy Do committed
158
    uv pip install --system -r requirements/cuda.txt \
159
    --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
160

161
162
163
164
# cuda arch list used by torch
# can be useful for both `dev` and `test`
# explicitly set the list to avoid issues with torch 2.2
# see https://github.com/pytorch/pytorch/pull/123243
165
ARG torch_cuda_arch_list='7.0 7.5 8.0 8.9 9.0 10.0 12.0'
166
ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
Simon Mo's avatar
Simon Mo committed
167
168
#################### BASE BUILD IMAGE ####################

169
#################### WHEEL BUILD IMAGE ####################
170
FROM base AS build
171
ARG TARGETPLATFORM
172

173
174
175
176
ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL

177
# install build dependencies
178
COPY requirements/build.txt requirements/build.txt
179

180
181
182
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
183
ENV UV_INDEX_STRATEGY="unsafe-best-match"
184

185
RUN --mount=type=cache,target=/root/.cache/uv \
Huy Do's avatar
Huy Do committed
186
    uv pip install --system -r requirements/build.txt \
187
    --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
188

189
COPY . .
190
191
ARG GIT_REPO_CHECK=0
RUN --mount=type=bind,source=.git,target=.git \
192
    if [ "$GIT_REPO_CHECK" != "0" ]; then bash tools/check_repo.sh ; fi
Stephen Krider's avatar
Stephen Krider committed
193
194

# max jobs used by Ninja to build extensions
195
196
ARG max_jobs=2
ENV MAX_JOBS=${max_jobs}
197
198
199
# number of threads used by nvcc
ARG nvcc_threads=8
ENV NVCC_THREADS=$nvcc_threads
200

201
ARG USE_SCCACHE
202
203
ARG SCCACHE_DOWNLOAD_URL=https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-x86_64-unknown-linux-musl.tar.gz
ARG SCCACHE_ENDPOINT
204
205
ARG SCCACHE_BUCKET_NAME=vllm-build-sccache
ARG SCCACHE_REGION_NAME=us-west-2
206
ARG SCCACHE_S3_NO_CREDENTIALS=0
207
208

# Flag to control whether to use pre-built vLLM wheels
209
ARG VLLM_USE_PRECOMPILED=""
210

211
# if USE_SCCACHE is set, use sccache to speed up compilation
212
RUN --mount=type=cache,target=/root/.cache/uv \
213
    --mount=type=bind,source=.git,target=.git \
214
215
    if [ "$USE_SCCACHE" = "1" ]; then \
        echo "Installing sccache..." \
216
        && curl -L -o sccache.tar.gz ${SCCACHE_DOWNLOAD_URL} \
217
218
219
        && tar -xzf sccache.tar.gz \
        && sudo mv sccache-v0.8.1-x86_64-unknown-linux-musl/sccache /usr/bin/sccache \
        && rm -rf sccache.tar.gz sccache-v0.8.1-x86_64-unknown-linux-musl \
220
        && if [ ! -z ${SCCACHE_ENDPOINT} ] ; then export SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT} ; fi \
221
222
        && export SCCACHE_BUCKET=${SCCACHE_BUCKET_NAME} \
        && export SCCACHE_REGION=${SCCACHE_REGION_NAME} \
223
        && export SCCACHE_S3_NO_CREDENTIALS=${SCCACHE_S3_NO_CREDENTIALS} \
224
        && export SCCACHE_IDLE_TIMEOUT=0 \
225
        && export CMAKE_BUILD_TYPE=Release \
226
227
        && export VLLM_USE_PRECOMPILED="${VLLM_USE_PRECOMPILED}" \
        && export VLLM_DOCKER_BUILD_CONTEXT=1 \
228
        && sccache --show-stats \
229
        && python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38 \
230
231
232
        && sccache --show-stats; \
    fi

233
234
ENV CCACHE_DIR=/root/.cache/ccache
RUN --mount=type=cache,target=/root/.cache/ccache \
235
    --mount=type=cache,target=/root/.cache/uv \
236
    --mount=type=bind,source=.git,target=.git  \
237
    if [ "$USE_SCCACHE" != "1" ]; then \
238
239
240
        # Clean any existing CMake artifacts
        rm -rf .deps && \
        mkdir -p .deps && \
241
242
        export VLLM_USE_PRECOMPILED="${VLLM_USE_PRECOMPILED}" && \
        export VLLM_DOCKER_BUILD_CONTEXT=1 && \
243
        python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38; \
244
    fi
245

246
247
248
249
250
251
252
253
254
255
256
# When using precompiled wheels, keep only the newest manylinux1 wheel and delete others
RUN if [ "$VLLM_USE_PRECOMPILED" = "1" ]; then \
        echo "Cleaning up extra wheels in dist/..." && \
        # Identify the most recent manylinux1_x86_64 wheel
        KEEP_WHEEL=$(ls -t dist/*manylinux1_x86_64.whl 2>/dev/null | head -n1) && \
        if [ -n "$KEEP_WHEEL" ]; then \
            echo "Keeping wheel: $KEEP_WHEEL"; \
            find dist/ -type f -name "*.whl" ! -path "${KEEP_WHEEL}" -delete; \
        fi; \
    fi

257
# Check the size of the wheel if RUN_WHEEL_CHECK is true
258
COPY .buildkite/check-wheel-size.py check-wheel-size.py
259
# sync the default value with .buildkite/check-wheel-size.py
260
ARG VLLM_MAX_SIZE_MB=400
261
262
263
264
265
266
267
ENV VLLM_MAX_SIZE_MB=$VLLM_MAX_SIZE_MB
ARG RUN_WHEEL_CHECK=true
RUN if [ "$RUN_WHEEL_CHECK" = "true" ]; then \
        python3 check-wheel-size.py dist; \
    else \
        echo "Skipping wheel size check."; \
    fi
Simon Mo's avatar
Simon Mo committed
268
#################### EXTENSION Build IMAGE ####################
Stephen Krider's avatar
Stephen Krider committed
269

270
#################### DEV IMAGE ####################
271
FROM base AS dev
272

273
274
275
276
ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL

277
278
279
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
280
281
ENV UV_INDEX_STRATEGY="unsafe-best-match"

282
283
284
COPY requirements/lint.txt requirements/lint.txt
COPY requirements/test.txt requirements/test.txt
COPY requirements/dev.txt requirements/dev.txt
285
RUN --mount=type=cache,target=/root/.cache/uv \
Huy Do's avatar
Huy Do committed
286
    uv pip install --system -r requirements/dev.txt \
287
    --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
288
#################### DEV IMAGE ####################
289

290
291
#################### vLLM installation IMAGE ####################
# image with vLLM installed
292
# TODO: Restore to base image after FlashInfer AOT wheel fixed
293
294
295
FROM ${FINAL_BASE_IMAGE} AS vllm-base
ARG CUDA_VERSION
ARG PYTHON_VERSION
296
ARG INSTALL_KV_CONNECTORS=false
Simon Mo's avatar
Simon Mo committed
297
WORKDIR /vllm-workspace
298
ENV DEBIAN_FRONTEND=noninteractive
299
300
ARG TARGETPLATFORM

Kebe's avatar
Kebe committed
301
302
SHELL ["/bin/bash", "-c"]

303
304
305
306
ARG DEADSNAKES_MIRROR_URL
ARG DEADSNAKES_GPGKEY_URL
ARG GET_PIP_URL

307
308
RUN PYTHON_VERSION_STR=$(echo ${PYTHON_VERSION} | sed 's/\.//g') && \
    echo "export PYTHON_VERSION_STR=${PYTHON_VERSION_STR}" >> /etc/environment
309

310
311
312
313
314
315
# Install Python and other dependencies
RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
    && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \
    && apt-get update -y \
    && apt-get install -y ccache software-properties-common git curl wget sudo vim python3-pip \
    && apt-get install -y ffmpeg libsm6 libxext6 libgl1 \
316
317
318
319
320
321
322
323
324
325
326
327
328
    && if [ ! -z ${DEADSNAKES_MIRROR_URL} ] ; then \
        if [ ! -z "${DEADSNAKES_GPGKEY_URL}" ] ; then \
            mkdir -p -m 0755 /etc/apt/keyrings ; \
            curl -L ${DEADSNAKES_GPGKEY_URL} | gpg --dearmor > /etc/apt/keyrings/deadsnakes.gpg ; \
            sudo chmod 644 /etc/apt/keyrings/deadsnakes.gpg ; \
            echo "deb [signed-by=/etc/apt/keyrings/deadsnakes.gpg] ${DEADSNAKES_MIRROR_URL} $(lsb_release -cs) main" > /etc/apt/sources.list.d/deadsnakes.list ; \
        fi ; \
    else \
        for i in 1 2 3; do \
            add-apt-repository -y ppa:deadsnakes/ppa && break || \
            { echo "Attempt $i failed, retrying in 5s..."; sleep 5; }; \
        done ; \
    fi \
329
330
331
332
333
    && apt-get update -y \
    && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv libibverbs-dev \
    && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \
    && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \
    && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \
334
    && curl -sS ${GET_PIP_URL} | python${PYTHON_VERSION} \
335
    && python3 --version && python3 -m pip --version
336
337
338
339
340
341
342

ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL
ARG PYTORCH_CUDA_NIGHTLY_INDEX_BASE_URL
ARG PIP_KEYRING_PROVIDER UV_KEYRING_PROVIDER

343
344
345
# Install uv for faster pip installs
RUN --mount=type=cache,target=/root/.cache/uv \
    python3 -m pip install uv
346

347
348
349
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
350
ENV UV_INDEX_STRATEGY="unsafe-best-match"
351

352
353
354
355
# Workaround for https://github.com/openai/triton/issues/2507 and
# https://github.com/pytorch/pytorch/issues/107960 -- hopefully
# this won't be needed for future versions of this docker image
# or future versions of triton.
356
RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/
357

358
359
360
361
# arm64 (GH200) build follows the practice of "use existing pytorch" build,
# we need to install torch and torchvision from the nightly builds first,
# pytorch will not appear as a vLLM dependency in all of the following steps
# after this step
362
RUN --mount=type=cache,target=/root/.cache/uv \
363
    if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
364
365
366
367
368
369
        uv pip install --system \
            --index-url ${PYTORCH_CUDA_NIGHTLY_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
            "torch==2.8.0.dev20250318+cu128" "torchvision==0.22.0.dev20250319" ; \
        uv pip install --system \
            --index-url ${PYTORCH_CUDA_NIGHTLY_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
            --pre pytorch_triton==3.3.0+gitab727c40 ; \
370
371
    fi

372
# Install vllm wheel first, so that torch etc will be installed.
373
RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \
374
    --mount=type=cache,target=/root/.cache/uv \
Huy Do's avatar
Huy Do committed
375
    uv pip install --system dist/*.whl --verbose \
376
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
Mor Zusman's avatar
Mor Zusman committed
377

378
# If we need to build FlashInfer wheel before its release:
379
# $ # Note we remove 7.0 from the arch list compared to the list below, since FlashInfer only supports sm75+
380
# $ export TORCH_CUDA_ARCH_LIST='7.5 8.0 8.9 9.0a 10.0a 12.0'
381
382
# $ git clone https://github.com/flashinfer-ai/flashinfer.git --recursive
# $ cd flashinfer
383
384
385
386
387
388
# $ git checkout v0.2.6.post1
# $ python -m flashinfer.aot
# $ python -m build --no-isolation --wheel
# $ ls -la dist
# -rw-rw-r-- 1 mgoin mgoin 205M Jun  9 18:03 flashinfer_python-0.2.6.post1-cp39-abi3-linux_x86_64.whl
# $ # upload the wheel to a public location, e.g. https://wheels.vllm.ai/flashinfer/v0.2.6.post1/flashinfer_python-0.2.6.post1-cp39-abi3-linux_x86_64.whl
389

390
# Install FlashInfer from source
391
ARG FLASHINFER_GIT_REPO="https://github.com/flashinfer-ai/flashinfer.git"
392
393
# Keep this in sync with https://github.com/vllm-project/vllm/blob/main/requirements/cuda.txt
# We use `--force-reinstall --no-deps` to avoid issues with the existing FlashInfer wheel.
394
ARG FLASHINFER_GIT_REF="v0.2.9rc2"
395
396
RUN --mount=type=cache,target=/root/.cache/uv bash - <<'BASH'
  . /etc/environment
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
    git clone --depth 1 --recursive --shallow-submodules \
        --branch ${FLASHINFER_GIT_REF} \
        ${FLASHINFER_GIT_REPO} flashinfer
    # Exclude CUDA arches for older versions (11.x and 12.0-12.7)
    # TODO: Update this to allow setting TORCH_CUDA_ARCH_LIST as a build arg.
    if [[ "${CUDA_VERSION}" == 11.* ]]; then
        FI_TORCH_CUDA_ARCH_LIST="7.5 8.0 8.9"
    elif [[ "${CUDA_VERSION}" == 12.[0-7]* ]]; then
        FI_TORCH_CUDA_ARCH_LIST="7.5 8.0 8.9 9.0a"
    else
        # CUDA 12.8+ supports 10.0a and 12.0
        FI_TORCH_CUDA_ARCH_LIST="7.5 8.0 8.9 9.0a 10.0a 12.0"
    fi
    echo "🏗️  Building FlashInfer for arches: ${FI_TORCH_CUDA_ARCH_LIST}"
    # Needed to build AOT kernels
    pushd flashinfer
        TORCH_CUDA_ARCH_LIST="${FI_TORCH_CUDA_ARCH_LIST}" \
            python3 -m flashinfer.aot
        TORCH_CUDA_ARCH_LIST="${FI_TORCH_CUDA_ARCH_LIST}" \
416
            uv pip install --system --no-build-isolation --force-reinstall --no-deps .
417
418
    popd
    rm -rf flashinfer
419
BASH
420
COPY examples examples
421
COPY benchmarks benchmarks
422
COPY ./vllm/collect_env.py .
423

Huy Do's avatar
Huy Do committed
424
425
426
427
RUN --mount=type=cache,target=/root/.cache/uv \
. /etc/environment && \
uv pip list

428
# Even when we build Flashinfer with AOT mode, there's still
429
430
431
# some issues w.r.t. JIT compilation. Therefore we need to
# install build dependencies for JIT compilation.
# TODO: Remove this once FlashInfer AOT wheel is fixed
432
COPY requirements/build.txt requirements/build.txt
433
RUN --mount=type=cache,target=/root/.cache/uv \
Huy Do's avatar
Huy Do committed
434
    uv pip install --system -r requirements/build.txt \
435
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
436

437
#################### vLLM installation IMAGE ####################
Stephen Krider's avatar
Stephen Krider committed
438

439
440
441
442
#################### TEST IMAGE ####################
# image to run unit testing suite
# note that this uses vllm installed by `pip`
FROM vllm-base AS test
Stephen Krider's avatar
Stephen Krider committed
443

444
ADD . /vllm-workspace/
Stephen Krider's avatar
Stephen Krider committed
445

446
447
448
449
450
ARG PYTHON_VERSION

ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL

451
452
453
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
454
ENV UV_INDEX_STRATEGY="unsafe-best-match"
455

Huy Do's avatar
Huy Do committed
456
# install development dependencies (for testing)
457
RUN --mount=type=cache,target=/root/.cache/uv \
458
459
460
461
    CUDA_MAJOR="${CUDA_VERSION%%.*}"; \
    if [ "$CUDA_MAJOR" -ge 12 ]; then \
        uv pip install --system -r requirements/dev.txt; \
    fi
462

youkaichao's avatar
youkaichao committed
463
# install development dependencies (for testing)
464
RUN --mount=type=cache,target=/root/.cache/uv \
465
    uv pip install --system -e tests/vllm_test_utils
youkaichao's avatar
youkaichao committed
466

467
# enable fast downloads from hf (for testing)
468
RUN --mount=type=cache,target=/root/.cache/uv \
469
    uv pip install --system hf_transfer
470
471
ENV HF_HUB_ENABLE_HF_TRANSFER 1

Joe Runde's avatar
Joe Runde committed
472
# Copy in the v1 package for testing (it isn't distributed yet)
473
COPY vllm/v1 /usr/local/lib/python${PYTHON_VERSION}/dist-packages/vllm/v1
Joe Runde's avatar
Joe Runde committed
474

475
476
477
478
479
# doc requires source code
# we hide them inside `test_docs/` , so that this source code
# will not be imported by other tests
RUN mkdir test_docs
RUN mv docs test_docs/
480
RUN cp -r examples test_docs/
481
RUN mv vllm test_docs/
482
RUN mv mkdocs.yaml test_docs/
483
#################### TEST IMAGE ####################
Stephen Krider's avatar
Stephen Krider committed
484

Simon Mo's avatar
Simon Mo committed
485
#################### OPENAI API SERVER ####################
486
487
# base openai image with additional requirements, for any subsequent openai-style images
FROM vllm-base AS vllm-openai-base
488
ARG TARGETPLATFORM
489
ARG INSTALL_KV_CONNECTORS=false
490

491
492
493
ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL

494
495
496
497
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500

498
499
COPY requirements/kv_connectors.txt requirements/kv_connectors.txt

500
# install additional dependencies for openai api server
501
RUN --mount=type=cache,target=/root/.cache/uv \
502
503
504
    if [ "$INSTALL_KV_CONNECTORS" = "true" ]; then \
        uv pip install --system -r requirements/kv_connectors.txt; \
    fi; \
505
    if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
506
        BITSANDBYTES_VERSION="0.42.0"; \
507
    else \
508
509
        BITSANDBYTES_VERSION="0.46.1"; \
    fi; \
510
    uv pip install --system accelerate hf_transfer modelscope "bitsandbytes>=${BITSANDBYTES_VERSION}" 'timm==0.9.10' boto3 runai-model-streamer runai-model-streamer[s3]
511

yhu422's avatar
yhu422 committed
512
513
ENV VLLM_USAGE_SOURCE production-docker-image

514
515
516
# define sagemaker first, so it is not default from `docker build`
FROM vllm-openai-base AS vllm-sagemaker

517
COPY examples/online_serving/sagemaker-entrypoint.sh .
518
519
520
521
522
RUN chmod +x sagemaker-entrypoint.sh
ENTRYPOINT ["./sagemaker-entrypoint.sh"]

FROM vllm-openai-base AS vllm-openai

Stephen Krider's avatar
Stephen Krider committed
523
ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
Simon Mo's avatar
Simon Mo committed
524
#################### OPENAI API SERVER ####################