Dockerfile 20.1 KB
Newer Older
Simon Mo's avatar
Simon Mo committed
1
2
3
# The vLLM Dockerfile is used to construct vLLM image that can be directly used
# to run the OpenAI compatible server.

4
# Please update any changes made here to
5
6
# docs/contributing/dockerfile/dockerfile.md and
# docs/assets/contributing/dockerfile-stages-dependency.png
7

8
ARG CUDA_VERSION=12.9.1
9
10
11
12
13
14
15
ARG PYTHON_VERSION=3.12

# By parameterizing the base images, we allow third-party to use their own
# base images. One use case is hermetic builds with base images stored in
# private registries that use a different repository naming conventions.
#
# Example:
16
17
# docker build --build-arg BUILD_BASE_IMAGE=registry.acme.org/mirror/nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04

18
# Important: We build with an old version of Ubuntu to maintain broad
19
20
21
22
# compatibility with other Linux OSes. The main reason for this is that the
# glibc version is baked into the distro, and binaries built with one glibc
# version are not backwards compatible with OSes that use an earlier version.
ARG BUILD_BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04
23
# TODO: Restore to base image after FlashInfer AOT wheel fixed
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
ARG FINAL_BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04

# By parameterizing the Deadsnakes repository URL, we allow third-party to use
# their own mirror. When doing so, we don't benefit from the transparent
# installation of the GPG key of the PPA, as done by add-apt-repository, so we
# also need a URL for the GPG key.
ARG DEADSNAKES_MIRROR_URL
ARG DEADSNAKES_GPGKEY_URL

# The PyPA get-pip.py script is a self contained script+zip file, that provides
# both the installer script and the pip base85-encoded zip archive. This allows
# bootstrapping pip in environment where a dsitribution package does not exist.
#
# By parameterizing the URL for get-pip.py installation script, we allow
# third-party to use their own copy of the script stored in a private mirror.
# We set the default value to the PyPA owned get-pip.py script.
#
# Reference: https://pip.pypa.io/en/stable/installation/#get-pip-py
ARG GET_PIP_URL="https://bootstrap.pypa.io/get-pip.py"

# PIP supports fetching the packages from custom indexes, allowing third-party
# to host the packages in private mirrors. The PIP_INDEX_URL and
# PIP_EXTRA_INDEX_URL are standard PIP environment variables to override the
# default indexes. By letting them empty by default, PIP will use its default
# indexes if the build process doesn't override the indexes.
#
# Uv uses different variables. We set them by default to the same values as
# PIP, but they can be overridden.
ARG PIP_INDEX_URL
ARG PIP_EXTRA_INDEX_URL
ARG UV_INDEX_URL=${PIP_INDEX_URL}
ARG UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}

# PyTorch provides its own indexes for standard and nightly builds
ARG PYTORCH_CUDA_INDEX_BASE_URL=https://download.pytorch.org/whl

# PIP supports multiple authentication schemes, including keyring
# By parameterizing the PIP_KEYRING_PROVIDER variable and setting it to
# disabled by default, we allow third-party to use keyring authentication for
# their private Python indexes, while not changing the default behavior which
# is no authentication.
#
# Reference: https://pip.pypa.io/en/stable/topics/authentication/#keyring-support
ARG PIP_KEYRING_PROVIDER=disabled
ARG UV_KEYRING_PROVIDER=${PIP_KEYRING_PROVIDER}

70
# Flag enables built-in KV-connector dependency libs into docker images
71
72
ARG INSTALL_KV_CONNECTORS=false

Simon Mo's avatar
Simon Mo committed
73
#################### BASE BUILD IMAGE ####################
74
# prepare basic build environment
75
76
77
FROM ${BUILD_BASE_IMAGE} AS base
ARG CUDA_VERSION
ARG PYTHON_VERSION
78
ARG TARGETPLATFORM
79
ARG INSTALL_KV_CONNECTORS=false
80
81
ENV DEBIAN_FRONTEND=noninteractive

82
83
ARG GET_PIP_URL

84
# Install system dependencies and uv, then create Python virtual environment
85
86
87
RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
    && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \
    && apt-get update -y \
88
89
90
91
92
93
94
    && apt-get install -y ccache software-properties-common git curl sudo python3-pip \
    && curl -LsSf https://astral.sh/uv/install.sh | sh \
    && $HOME/.local/bin/uv venv /opt/venv --python ${PYTHON_VERSION} \
    && rm -f /usr/bin/python3 /usr/bin/python3-config /usr/bin/pip \
    && ln -s /opt/venv/bin/python3 /usr/bin/python3 \
    && ln -s /opt/venv/bin/python3-config /usr/bin/python3-config \
    && ln -s /opt/venv/bin/pip /usr/bin/pip \
95
    && python3 --version && python3 -m pip --version
96
97
98
99
100
101

ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL
ARG PIP_KEYRING_PROVIDER UV_KEYRING_PROVIDER

102
103
104
# Activate virtual environment and add uv to PATH
ENV PATH="/opt/venv/bin:/root/.local/bin:$PATH"
ENV VIRTUAL_ENV="/opt/venv"
Stephen Krider's avatar
Stephen Krider committed
105

106
107
108
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
109
ENV UV_INDEX_STRATEGY="unsafe-best-match"
110
111
# Use copy mode to avoid hardlink failures with Docker cache mounts
ENV UV_LINK_MODE=copy
112

113
114
115
116
117
118
119
120
# Upgrade to GCC 10 to avoid https://gcc.gnu.org/bugzilla/show_bug.cgi?id=92519
# as it was causing spam when compiling the CUTLASS kernels
RUN apt-get install -y gcc-10 g++-10
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 110 --slave /usr/bin/g++ g++ /usr/bin/g++-10
RUN <<EOF
gcc --version
EOF

121
122
123
124
# Workaround for https://github.com/openai/triton/issues/2507 and
# https://github.com/pytorch/pytorch/issues/107960 -- hopefully
# this won't be needed for future versions of this docker image
# or future versions of triton.
125
RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/
126

Stephen Krider's avatar
Stephen Krider committed
127
128
WORKDIR /workspace

129
# install build and runtime dependencies
130
131
COPY requirements/common.txt requirements/common.txt
COPY requirements/cuda.txt requirements/cuda.txt
132
RUN --mount=type=cache,target=/root/.cache/uv \
133
    uv pip install --python /opt/venv/bin/python3 -r requirements/cuda.txt \
134
    --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
135

136
137
138
139
# cuda arch list used by torch
# can be useful for both `dev` and `test`
# explicitly set the list to avoid issues with torch 2.2
# see https://github.com/pytorch/pytorch/pull/123243
140
ARG torch_cuda_arch_list='7.0 7.5 8.0 8.9 9.0 10.0 12.0'
141
ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
Simon Mo's avatar
Simon Mo committed
142
143
#################### BASE BUILD IMAGE ####################

144
#################### WHEEL BUILD IMAGE ####################
145
FROM base AS build
146
ARG TARGETPLATFORM
147

148
149
150
151
ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL

152
# install build dependencies
153
COPY requirements/build.txt requirements/build.txt
154

155
156
157
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
158
ENV UV_INDEX_STRATEGY="unsafe-best-match"
159
160
# Use copy mode to avoid hardlink failures with Docker cache mounts
ENV UV_LINK_MODE=copy
161

162
RUN --mount=type=cache,target=/root/.cache/uv \
163
    uv pip install --python /opt/venv/bin/python3 -r requirements/build.txt \
164
    --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
165

166
COPY . .
167
168
ARG GIT_REPO_CHECK=0
RUN --mount=type=bind,source=.git,target=.git \
169
    if [ "$GIT_REPO_CHECK" != "0" ]; then bash tools/check_repo.sh ; fi
Stephen Krider's avatar
Stephen Krider committed
170
171

# max jobs used by Ninja to build extensions
172
173
ARG max_jobs=2
ENV MAX_JOBS=${max_jobs}
174
175
176
# number of threads used by nvcc
ARG nvcc_threads=8
ENV NVCC_THREADS=$nvcc_threads
177

178
ARG USE_SCCACHE
179
180
ARG SCCACHE_DOWNLOAD_URL=https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-x86_64-unknown-linux-musl.tar.gz
ARG SCCACHE_ENDPOINT
181
182
ARG SCCACHE_BUCKET_NAME=vllm-build-sccache
ARG SCCACHE_REGION_NAME=us-west-2
183
ARG SCCACHE_S3_NO_CREDENTIALS=0
184
185

# Flag to control whether to use pre-built vLLM wheels
186
ARG VLLM_USE_PRECOMPILED=""
187
ARG VLLM_MAIN_CUDA_VERSION=""
188

189
# if USE_SCCACHE is set, use sccache to speed up compilation
190
RUN --mount=type=cache,target=/root/.cache/uv \
191
    --mount=type=bind,source=.git,target=.git \
192
193
    if [ "$USE_SCCACHE" = "1" ]; then \
        echo "Installing sccache..." \
194
        && curl -L -o sccache.tar.gz ${SCCACHE_DOWNLOAD_URL} \
195
196
197
        && tar -xzf sccache.tar.gz \
        && sudo mv sccache-v0.8.1-x86_64-unknown-linux-musl/sccache /usr/bin/sccache \
        && rm -rf sccache.tar.gz sccache-v0.8.1-x86_64-unknown-linux-musl \
198
        && if [ ! -z ${SCCACHE_ENDPOINT} ] ; then export SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT} ; fi \
199
200
        && export SCCACHE_BUCKET=${SCCACHE_BUCKET_NAME} \
        && export SCCACHE_REGION=${SCCACHE_REGION_NAME} \
201
        && export SCCACHE_S3_NO_CREDENTIALS=${SCCACHE_S3_NO_CREDENTIALS} \
202
        && export SCCACHE_IDLE_TIMEOUT=0 \
203
        && export CMAKE_BUILD_TYPE=Release \
204
        && export VLLM_USE_PRECOMPILED="${VLLM_USE_PRECOMPILED}" \
205
        && export VLLM_MAIN_CUDA_VERSION="${VLLM_MAIN_CUDA_VERSION}" \
206
        && export VLLM_DOCKER_BUILD_CONTEXT=1 \
207
        && sccache --show-stats \
208
        && python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38 \
209
210
211
        && sccache --show-stats; \
    fi

212
213
ARG vllm_target_device="cuda"
ENV VLLM_TARGET_DEVICE=${vllm_target_device}
214
215
ENV CCACHE_DIR=/root/.cache/ccache
RUN --mount=type=cache,target=/root/.cache/ccache \
216
    --mount=type=cache,target=/root/.cache/uv \
217
    --mount=type=bind,source=.git,target=.git  \
218
    if [ "$USE_SCCACHE" != "1" ]; then \
219
220
221
        # Clean any existing CMake artifacts
        rm -rf .deps && \
        mkdir -p .deps && \
222
223
        export VLLM_USE_PRECOMPILED="${VLLM_USE_PRECOMPILED}" && \
        export VLLM_DOCKER_BUILD_CONTEXT=1 && \
224
        python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38; \
225
    fi
226

227
# Check the size of the wheel if RUN_WHEEL_CHECK is true
228
COPY .buildkite/check-wheel-size.py check-wheel-size.py
229
# sync the default value with .buildkite/check-wheel-size.py
230
ARG VLLM_MAX_SIZE_MB=500
231
232
233
234
235
236
237
ENV VLLM_MAX_SIZE_MB=$VLLM_MAX_SIZE_MB
ARG RUN_WHEEL_CHECK=true
RUN if [ "$RUN_WHEEL_CHECK" = "true" ]; then \
        python3 check-wheel-size.py dist; \
    else \
        echo "Skipping wheel size check."; \
    fi
Simon Mo's avatar
Simon Mo committed
238
#################### EXTENSION Build IMAGE ####################
Stephen Krider's avatar
Stephen Krider committed
239

240
#################### DEV IMAGE ####################
241
FROM base AS dev
242

243
244
245
246
ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL

247
248
249
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
250
ENV UV_INDEX_STRATEGY="unsafe-best-match"
251
252
# Use copy mode to avoid hardlink failures with Docker cache mounts
ENV UV_LINK_MODE=copy
Huy Do's avatar
Huy Do committed
253

254
255
# Install libnuma-dev, required by fastsafetensors (fixes #20384)
RUN apt-get update && apt-get install -y libnuma-dev && rm -rf /var/lib/apt/lists/*
256
257
258
COPY requirements/lint.txt requirements/lint.txt
COPY requirements/test.txt requirements/test.txt
COPY requirements/dev.txt requirements/dev.txt
259
RUN --mount=type=cache,target=/root/.cache/uv \
260
    uv pip install --python /opt/venv/bin/python3 -r requirements/dev.txt \
261
    --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
262
#################### DEV IMAGE ####################
263

264
265
#################### vLLM installation IMAGE ####################
# image with vLLM installed
266
267
268
FROM ${FINAL_BASE_IMAGE} AS vllm-base
ARG CUDA_VERSION
ARG PYTHON_VERSION
269
ARG INSTALL_KV_CONNECTORS=false
Simon Mo's avatar
Simon Mo committed
270
WORKDIR /vllm-workspace
271
ENV DEBIAN_FRONTEND=noninteractive
272
273
ARG TARGETPLATFORM

274
# TODO (huydhn): There is no prebuilt gdrcopy package on 12.9 at the moment
275
276
277
278
ARG GDRCOPY_CUDA_VERSION=12.8
# Keep in line with FINAL_BASE_IMAGE
ARG GDRCOPY_OS_VERSION=Ubuntu22_04

Kebe's avatar
Kebe committed
279
280
SHELL ["/bin/bash", "-c"]

281
282
283
284
ARG DEADSNAKES_MIRROR_URL
ARG DEADSNAKES_GPGKEY_URL
ARG GET_PIP_URL

285
286
RUN PYTHON_VERSION_STR=$(echo ${PYTHON_VERSION} | sed 's/\.//g') && \
    echo "export PYTHON_VERSION_STR=${PYTHON_VERSION_STR}" >> /etc/environment
287

288
289
290
291
292
293
# Install Python and other dependencies
RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
    && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \
    && apt-get update -y \
    && apt-get install -y ccache software-properties-common git curl wget sudo vim python3-pip \
    && apt-get install -y ffmpeg libsm6 libxext6 libgl1 \
294
295
296
297
298
299
300
301
302
303
304
305
306
    && if [ ! -z ${DEADSNAKES_MIRROR_URL} ] ; then \
        if [ ! -z "${DEADSNAKES_GPGKEY_URL}" ] ; then \
            mkdir -p -m 0755 /etc/apt/keyrings ; \
            curl -L ${DEADSNAKES_GPGKEY_URL} | gpg --dearmor > /etc/apt/keyrings/deadsnakes.gpg ; \
            sudo chmod 644 /etc/apt/keyrings/deadsnakes.gpg ; \
            echo "deb [signed-by=/etc/apt/keyrings/deadsnakes.gpg] ${DEADSNAKES_MIRROR_URL} $(lsb_release -cs) main" > /etc/apt/sources.list.d/deadsnakes.list ; \
        fi ; \
    else \
        for i in 1 2 3; do \
            add-apt-repository -y ppa:deadsnakes/ppa && break || \
            { echo "Attempt $i failed, retrying in 5s..."; sleep 5; }; \
        done ; \
    fi \
307
308
309
310
311
    && apt-get update -y \
    && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv libibverbs-dev \
    && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \
    && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \
    && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \
312
    && curl -sS ${GET_PIP_URL} | python${PYTHON_VERSION} \
313
    && python3 --version && python3 -m pip --version
314
315
316
317
318
319

ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL
ARG PIP_KEYRING_PROVIDER UV_KEYRING_PROVIDER

320
321
322
# Install uv for faster pip installs
RUN --mount=type=cache,target=/root/.cache/uv \
    python3 -m pip install uv
323

324
325
326
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
327
ENV UV_INDEX_STRATEGY="unsafe-best-match"
328
329
# Use copy mode to avoid hardlink failures with Docker cache mounts
ENV UV_LINK_MODE=copy
330

331
332
333
334
# Workaround for https://github.com/openai/triton/issues/2507 and
# https://github.com/pytorch/pytorch/issues/107960 -- hopefully
# this won't be needed for future versions of this docker image
# or future versions of triton.
335
RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/
336

337
# Install vllm wheel first, so that torch etc will be installed.
338
RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \
339
    --mount=type=cache,target=/root/.cache/uv \
340
    uv pip install --system dist/*.whl --verbose \
341
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
Mor Zusman's avatar
Mor Zusman committed
342

343
344
345
# Install FlashInfer pre-compiled kernel cache and binaries
# https://docs.flashinfer.ai/installation.html
RUN --mount=type=cache,target=/root/.cache/uv \
346
347
    uv pip install --system flashinfer-cubin==0.5.2 \
    && uv pip install --system flashinfer-jit-cache==0.5.2 \
348
349
350
        --extra-index-url https://flashinfer.ai/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
    && flashinfer show-config

351
COPY examples examples
352
COPY benchmarks benchmarks
353
COPY ./vllm/collect_env.py .
354

Huy Do's avatar
Huy Do committed
355
356
357
358
RUN --mount=type=cache,target=/root/.cache/uv \
. /etc/environment && \
uv pip list

359
# Even when we build Flashinfer with AOT mode, there's still
360
361
362
# some issues w.r.t. JIT compilation. Therefore we need to
# install build dependencies for JIT compilation.
# TODO: Remove this once FlashInfer AOT wheel is fixed
363
COPY requirements/build.txt requirements/build.txt
364
RUN --mount=type=cache,target=/root/.cache/uv \
Huy Do's avatar
Huy Do committed
365
    uv pip install --system -r requirements/build.txt \
366
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
367

368
# Install DeepGEMM from source
369
ARG DEEPGEMM_GIT_REF
370
371
COPY tools/install_deepgemm.sh /tmp/install_deepgemm.sh
RUN --mount=type=cache,target=/root/.cache/uv \
372
    VLLM_DOCKER_BUILD_CONTEXT=1 TORCH_CUDA_ARCH_LIST="9.0a 10.0a" /tmp/install_deepgemm.sh --cuda-version "${CUDA_VERSION}" ${DEEPGEMM_GIT_REF:+--ref "$DEEPGEMM_GIT_REF"}
373

374
375
376
377
378
379
380
381
382
383
384
COPY tools/install_gdrcopy.sh install_gdrcopy.sh
RUN set -eux; \
    case "${TARGETPLATFORM}" in \
      linux/arm64) UUARCH="aarch64" ;; \
      linux/amd64) UUARCH="x64" ;; \
      *) echo "Unsupported TARGETPLATFORM: ${TARGETPLATFORM}" >&2; exit 1 ;; \
    esac; \
    ./install_gdrcopy.sh "${GDRCOPY_OS_VERSION}" "${GDRCOPY_CUDA_VERSION}" "${UUARCH}"; \
    rm ./install_gdrcopy.sh

# Install EP kernels(pplx-kernels and DeepEP)
385
386
COPY tools/ep_kernels/install_python_libraries.sh install_python_libraries.sh
ENV CUDA_HOME=/usr/local/cuda
387
RUN export TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST:-9.0a 10.0a+PTX}" \
388
    && bash install_python_libraries.sh
389

390
391
392
393
394
395
# CUDA image changed from /usr/local/nvidia to /usr/local/cuda in 12.8 but will
# return to /usr/local/nvidia in 13.0 to allow container providers to mount drivers
# consistently from the host (see https://github.com/vllm-project/vllm/issues/18859).
# Until then, add /usr/local/nvidia/lib64 before the image cuda path to allow override.
ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}

396
#################### vLLM installation IMAGE ####################
Stephen Krider's avatar
Stephen Krider committed
397

398
399
400
401
#################### TEST IMAGE ####################
# image to run unit testing suite
# note that this uses vllm installed by `pip`
FROM vllm-base AS test
Stephen Krider's avatar
Stephen Krider committed
402

403
ADD . /vllm-workspace/
Stephen Krider's avatar
Stephen Krider committed
404

405
406
407
408
ARG PYTHON_VERSION

ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
409
ARG PYTORCH_CUDA_INDEX_BASE_URL
410

411
412
413
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500
Huy Do's avatar
Huy Do committed
414
ENV UV_INDEX_STRATEGY="unsafe-best-match"
415
416
# Use copy mode to avoid hardlink failures with Docker cache mounts
ENV UV_LINK_MODE=copy
417

Huy Do's avatar
Huy Do committed
418
# install development dependencies (for testing)
419
RUN --mount=type=cache,target=/root/.cache/uv \
420
421
    CUDA_MAJOR="${CUDA_VERSION%%.*}"; \
    if [ "$CUDA_MAJOR" -ge 12 ]; then \
422
423
        uv pip install --system -r requirements/dev.txt \
        --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \
424
    fi
425

youkaichao's avatar
youkaichao committed
426
# install development dependencies (for testing)
427
RUN --mount=type=cache,target=/root/.cache/uv \
428
    uv pip install --system -e tests/vllm_test_utils
youkaichao's avatar
youkaichao committed
429

430
# enable fast downloads from hf (for testing)
431
RUN --mount=type=cache,target=/root/.cache/uv \
432
    uv pip install --system hf_transfer
433
434
ENV HF_HUB_ENABLE_HF_TRANSFER 1

Joe Runde's avatar
Joe Runde committed
435
# Copy in the v1 package for testing (it isn't distributed yet)
436
COPY vllm/v1 /usr/local/lib/python${PYTHON_VERSION}/dist-packages/vllm/v1
Joe Runde's avatar
Joe Runde committed
437

438
439
# Source code is used in the `python_only_compile.sh` test
# We hide it inside `src/` so that this source code
440
# will not be imported by other tests
441
442
RUN mkdir src
RUN mv vllm src/vllm
443
#################### TEST IMAGE ####################
Stephen Krider's avatar
Stephen Krider committed
444

Simon Mo's avatar
Simon Mo committed
445
#################### OPENAI API SERVER ####################
446
447
# base openai image with additional requirements, for any subsequent openai-style images
FROM vllm-base AS vllm-openai-base
448
ARG TARGETPLATFORM
449
ARG INSTALL_KV_CONNECTORS=false
450

451
452
453
ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL

454
455
456
457
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500

458
459
COPY requirements/kv_connectors.txt requirements/kv_connectors.txt

460
# install additional dependencies for openai api server
461
RUN --mount=type=cache,target=/root/.cache/uv \
462
463
464
    if [ "$INSTALL_KV_CONNECTORS" = "true" ]; then \
        uv pip install --system -r requirements/kv_connectors.txt; \
    fi; \
465
    if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
466
        BITSANDBYTES_VERSION="0.42.0"; \
467
    else \
468
469
        BITSANDBYTES_VERSION="0.46.1"; \
    fi; \
470
    uv pip install --system accelerate hf_transfer modelscope "bitsandbytes>=${BITSANDBYTES_VERSION}" 'timm>=1.0.17' 'runai-model-streamer[s3,gcs]>=0.15.0'
471

yhu422's avatar
yhu422 committed
472
473
ENV VLLM_USAGE_SOURCE production-docker-image

474
475
476
# define sagemaker first, so it is not default from `docker build`
FROM vllm-openai-base AS vllm-sagemaker

477
COPY examples/online_serving/sagemaker-entrypoint.sh .
478
479
480
481
482
RUN chmod +x sagemaker-entrypoint.sh
ENTRYPOINT ["./sagemaker-entrypoint.sh"]

FROM vllm-openai-base AS vllm-openai

483
ENTRYPOINT ["vllm", "serve"]
Simon Mo's avatar
Simon Mo committed
484
#################### OPENAI API SERVER ####################