Dockerfile 14.4 KB
Newer Older
Simon Mo's avatar
Simon Mo committed
1
2
3
# The vLLM Dockerfile is used to construct vLLM image that can be directly used
# to run the OpenAI compatible server.

4
# Please update any changes made here to
5
6
# docs/source/contributing/dockerfile/dockerfile.md and
# docs/source/assets/contributing/dockerfile-stages-dependency.png
7

8
ARG CUDA_VERSION=12.4.1
Simon Mo's avatar
Simon Mo committed
9
#################### BASE BUILD IMAGE ####################
10
# prepare basic build environment
11
FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04 AS base
12
ARG CUDA_VERSION=12.4.1
13
ARG PYTHON_VERSION=3.12
14
ARG TARGETPLATFORM
15
16
ENV DEBIAN_FRONTEND=noninteractive

17
18
19
20
21
# Install Python and other dependencies
RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
    && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \
    && apt-get update -y \
    && apt-get install -y ccache software-properties-common git curl sudo \
22
23
24
25
    && for i in 1 2 3; do \
        add-apt-repository -y ppa:deadsnakes/ppa && break || \
        { echo "Attempt $i failed, retrying in 5s..."; sleep 5; }; \
    done \
26
27
28
29
30
31
32
33
34
35
    && apt-get update -y \
    && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv \
    && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \
    && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \
    && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \
    && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} \
    && python3 --version && python3 -m pip --version
# Install uv for faster pip installs
RUN --mount=type=cache,target=/root/.cache/uv \
    python3 -m pip install uv
Stephen Krider's avatar
Stephen Krider committed
36

37
38
39
40
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500

41
42
43
44
45
46
47
48
# Upgrade to GCC 10 to avoid https://gcc.gnu.org/bugzilla/show_bug.cgi?id=92519
# as it was causing spam when compiling the CUTLASS kernels
RUN apt-get install -y gcc-10 g++-10
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 110 --slave /usr/bin/g++ g++ /usr/bin/g++-10
RUN <<EOF
gcc --version
EOF

49
50
51
52
# Workaround for https://github.com/openai/triton/issues/2507 and
# https://github.com/pytorch/pytorch/issues/107960 -- hopefully
# this won't be needed for future versions of this docker image
# or future versions of triton.
53
RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/
54

Stephen Krider's avatar
Stephen Krider committed
55
56
WORKDIR /workspace

57
58
# install build and runtime dependencies

59
60
61
62
# arm64 (GH200) build follows the practice of "use existing pytorch" build,
# we need to install torch and torchvision from the nightly builds first,
# pytorch will not appear as a vLLM dependency in all of the following steps
# after this step
63
RUN --mount=type=cache,target=/root/.cache/uv \
64
    if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
65
66
        uv pip install --system --index-url https://download.pytorch.org/whl/nightly/cu128 "torch==2.8.0.dev20250318+cu128" "torchvision==0.22.0.dev20250319";  \
        uv pip install --system --index-url https://download.pytorch.org/whl/nightly/cu128 --pre pytorch_triton==3.3.0+gitab727c40; \
67
    fi
Mor Zusman's avatar
Mor Zusman committed
68

69
70
COPY requirements/common.txt requirements/common.txt
COPY requirements/cuda.txt requirements/cuda.txt
71
RUN --mount=type=cache,target=/root/.cache/uv \
72
    uv pip install --system -r requirements/cuda.txt
73

74
75
76
77
78
79
# cuda arch list used by torch
# can be useful for both `dev` and `test`
# explicitly set the list to avoid issues with torch 2.2
# see https://github.com/pytorch/pytorch/pull/123243
ARG torch_cuda_arch_list='7.0 7.5 8.0 8.6 8.9 9.0+PTX'
ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
80
81
82
# Override the arch list for flash-attn to reduce the binary size
ARG vllm_fa_cmake_gpu_arches='80-real;90-real'
ENV VLLM_FA_CMAKE_GPU_ARCHES=${vllm_fa_cmake_gpu_arches}
Simon Mo's avatar
Simon Mo committed
83
84
#################### BASE BUILD IMAGE ####################

85
#################### WHEEL BUILD IMAGE ####################
86
FROM base AS build
87
ARG TARGETPLATFORM
88

89
# install build dependencies
90
COPY requirements/build.txt requirements/build.txt
91

92
93
94
95
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500

96
RUN --mount=type=cache,target=/root/.cache/uv \
97
    uv pip install --system -r requirements/build.txt
98

99
COPY . .
100
101
ARG GIT_REPO_CHECK=0
RUN --mount=type=bind,source=.git,target=.git \
102
    if [ "$GIT_REPO_CHECK" != "0" ]; then bash tools/check_repo.sh ; fi
Stephen Krider's avatar
Stephen Krider committed
103
104

# max jobs used by Ninja to build extensions
105
106
ARG max_jobs=2
ENV MAX_JOBS=${max_jobs}
107
108
109
# number of threads used by nvcc
ARG nvcc_threads=8
ENV NVCC_THREADS=$nvcc_threads
110

111
ARG USE_SCCACHE
112
113
ARG SCCACHE_BUCKET_NAME=vllm-build-sccache
ARG SCCACHE_REGION_NAME=us-west-2
114
ARG SCCACHE_S3_NO_CREDENTIALS=0
115
# if USE_SCCACHE is set, use sccache to speed up compilation
116
RUN --mount=type=cache,target=/root/.cache/uv \
117
    --mount=type=bind,source=.git,target=.git \
118
119
120
121
122
123
    if [ "$USE_SCCACHE" = "1" ]; then \
        echo "Installing sccache..." \
        && curl -L -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-x86_64-unknown-linux-musl.tar.gz \
        && tar -xzf sccache.tar.gz \
        && sudo mv sccache-v0.8.1-x86_64-unknown-linux-musl/sccache /usr/bin/sccache \
        && rm -rf sccache.tar.gz sccache-v0.8.1-x86_64-unknown-linux-musl \
124
125
        && export SCCACHE_BUCKET=${SCCACHE_BUCKET_NAME} \
        && export SCCACHE_REGION=${SCCACHE_REGION_NAME} \
126
        && export SCCACHE_S3_NO_CREDENTIALS=${SCCACHE_S3_NO_CREDENTIALS} \
127
        && export SCCACHE_IDLE_TIMEOUT=0 \
128
        && export CMAKE_BUILD_TYPE=Release \
129
        && sccache --show-stats \
130
        && python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38 \
131
132
133
        && sccache --show-stats; \
    fi

134
135
ENV CCACHE_DIR=/root/.cache/ccache
RUN --mount=type=cache,target=/root/.cache/ccache \
136
    --mount=type=cache,target=/root/.cache/uv \
137
    --mount=type=bind,source=.git,target=.git  \
138
    if [ "$USE_SCCACHE" != "1" ]; then \
139
140
141
        # Clean any existing CMake artifacts
        rm -rf .deps && \
        mkdir -p .deps && \
142
        python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38; \
143
    fi
144

145
# Check the size of the wheel if RUN_WHEEL_CHECK is true
146
COPY .buildkite/check-wheel-size.py check-wheel-size.py
147
# sync the default value with .buildkite/check-wheel-size.py
148
ARG VLLM_MAX_SIZE_MB=400
149
150
151
152
153
154
155
ENV VLLM_MAX_SIZE_MB=$VLLM_MAX_SIZE_MB
ARG RUN_WHEEL_CHECK=true
RUN if [ "$RUN_WHEEL_CHECK" = "true" ]; then \
        python3 check-wheel-size.py dist; \
    else \
        echo "Skipping wheel size check."; \
    fi
Simon Mo's avatar
Simon Mo committed
156
#################### EXTENSION Build IMAGE ####################
Stephen Krider's avatar
Stephen Krider committed
157

158
159
160
#################### DEV IMAGE ####################
FROM base as dev

161
162
163
164
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500

165
166
167
COPY requirements/lint.txt requirements/lint.txt
COPY requirements/test.txt requirements/test.txt
COPY requirements/dev.txt requirements/dev.txt
168
169
170
# Workaround for #17068
RUN --mount=type=cache,target=/root/.cache/uv \
    uv pip install --system mamba-ssm==2.2.4 --no-build-isolation
171
RUN --mount=type=cache,target=/root/.cache/uv \
172
    uv pip install --system -r requirements/dev.txt
173
#################### DEV IMAGE ####################
174

175
176
#################### vLLM installation IMAGE ####################
# image with vLLM installed
177
178
# TODO: Restore to base image after FlashInfer AOT wheel fixed
FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04 AS vllm-base
179
ARG CUDA_VERSION=12.4.1
180
ARG PYTHON_VERSION=3.12
Simon Mo's avatar
Simon Mo committed
181
WORKDIR /vllm-workspace
182
ENV DEBIAN_FRONTEND=noninteractive
183
184
ARG TARGETPLATFORM

185
186
RUN PYTHON_VERSION_STR=$(echo ${PYTHON_VERSION} | sed 's/\.//g') && \
    echo "export PYTHON_VERSION_STR=${PYTHON_VERSION_STR}" >> /etc/environment
187

188
189
190
191
192
193
# Install Python and other dependencies
RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
    && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \
    && apt-get update -y \
    && apt-get install -y ccache software-properties-common git curl wget sudo vim python3-pip \
    && apt-get install -y ffmpeg libsm6 libxext6 libgl1 \
194
195
196
197
    && for i in 1 2 3; do \
        add-apt-repository -y ppa:deadsnakes/ppa && break || \
        { echo "Attempt $i failed, retrying in 5s..."; sleep 5; }; \
    done \
198
199
200
201
202
203
204
205
206
207
    && apt-get update -y \
    && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv libibverbs-dev \
    && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \
    && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \
    && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \
    && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} \
    && python3 --version && python3 -m pip --version
# Install uv for faster pip installs
RUN --mount=type=cache,target=/root/.cache/uv \
    python3 -m pip install uv
208

209
210
211
212
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500

213
214
215
216
# Workaround for https://github.com/openai/triton/issues/2507 and
# https://github.com/pytorch/pytorch/issues/107960 -- hopefully
# this won't be needed for future versions of this docker image
# or future versions of triton.
217
RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/
218

219
220
221
222
# arm64 (GH200) build follows the practice of "use existing pytorch" build,
# we need to install torch and torchvision from the nightly builds first,
# pytorch will not appear as a vLLM dependency in all of the following steps
# after this step
223
RUN --mount=type=cache,target=/root/.cache/uv \
224
    if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
225
226
        uv pip install --system --index-url https://download.pytorch.org/whl/nightly/cu128 "torch==2.8.0.dev20250318+cu128" "torchvision==0.22.0.dev20250319";  \
        uv pip install --system --index-url https://download.pytorch.org/whl/nightly/cu128 --pre pytorch_triton==3.3.0+gitab727c40; \
227
228
    fi

229
# Install vllm wheel first, so that torch etc will be installed.
230
RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \
231
    --mount=type=cache,target=/root/.cache/uv \
232
    uv pip install --system dist/*.whl --verbose
Mor Zusman's avatar
Mor Zusman committed
233

234
# If we need to build FlashInfer wheel before its release:
235
236
237
238
239
240
# $ export FLASHINFER_ENABLE_AOT=1
# $ # Note we remove 7.0 from the arch list compared to the list below, since FlashInfer only supports sm75+
# $ export TORCH_CUDA_ARCH_LIST='7.5 8.0 8.6 8.9 9.0+PTX'
# $ git clone https://github.com/flashinfer-ai/flashinfer.git --recursive
# $ cd flashinfer
# $ git checkout 524304395bd1d8cd7d07db083859523fcaa246a4
241
# $ rm -rf build
242
# $ python3 setup.py bdist_wheel --dist-dir=dist --verbose
243
244
# $ ls dist
# $ # upload the wheel to a public location, e.g. https://wheels.vllm.ai/flashinfer/524304395bd1d8cd7d07db083859523fcaa246a4/flashinfer_python-0.2.1.post1+cu124torch2.5-cp38-abi3-linux_x86_64.whl
245

246
RUN --mount=type=cache,target=/root/.cache/uv \
247
. /etc/environment && \
248
if [ "$TARGETPLATFORM" != "linux/arm64" ]; then \
249
    uv pip install --system https://github.com/flashinfer-ai/flashinfer/releases/download/v0.2.1.post2/flashinfer_python-0.2.1.post2+cu124torch2.6-cp38-abi3-linux_x86_64.whl ; \
250
fi
251
COPY examples examples
252
COPY benchmarks benchmarks
253
COPY ./vllm/collect_env.py .
254
255
256
257
258

# Although we build Flashinfer with AOT mode, there's still
# some issues w.r.t. JIT compilation. Therefore we need to
# install build dependencies for JIT compilation.
# TODO: Remove this once FlashInfer AOT wheel is fixed
259
COPY requirements/build.txt requirements/build.txt
260
RUN --mount=type=cache,target=/root/.cache/uv \
261
    uv pip install --system -r requirements/build.txt
262

263
#################### vLLM installation IMAGE ####################
Stephen Krider's avatar
Stephen Krider committed
264

265
266
267
268
#################### TEST IMAGE ####################
# image to run unit testing suite
# note that this uses vllm installed by `pip`
FROM vllm-base AS test
Stephen Krider's avatar
Stephen Krider committed
269

270
ADD . /vllm-workspace/
Stephen Krider's avatar
Stephen Krider committed
271

272
273
274
275
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500

276
# install development dependencies (for testing)
277
278
279
# Workaround for #17068
RUN --mount=type=cache,target=/root/.cache/uv \
    uv pip install --system mamba-ssm==2.2.4 --no-build-isolation
280
RUN --mount=type=cache,target=/root/.cache/uv \
281
    uv pip install --system -r requirements/dev.txt
282

youkaichao's avatar
youkaichao committed
283
# install development dependencies (for testing)
284
RUN --mount=type=cache,target=/root/.cache/uv \
285
    uv pip install --system -e tests/vllm_test_utils
youkaichao's avatar
youkaichao committed
286

287
# enable fast downloads from hf (for testing)
288
RUN --mount=type=cache,target=/root/.cache/uv \
289
    uv pip install --system hf_transfer
290
291
ENV HF_HUB_ENABLE_HF_TRANSFER 1

Joe Runde's avatar
Joe Runde committed
292
293
294
# Copy in the v1 package for testing (it isn't distributed yet)
COPY vllm/v1 /usr/local/lib/python3.12/dist-packages/vllm/v1

295
296
297
298
299
300
301
# doc requires source code
# we hide them inside `test_docs/` , so that this source code
# will not be imported by other tests
RUN mkdir test_docs
RUN mv docs test_docs/
RUN mv vllm test_docs/
#################### TEST IMAGE ####################
Stephen Krider's avatar
Stephen Krider committed
302

Simon Mo's avatar
Simon Mo committed
303
#################### OPENAI API SERVER ####################
304
305
# base openai image with additional requirements, for any subsequent openai-style images
FROM vllm-base AS vllm-openai-base
306
ARG TARGETPLATFORM
307

308
309
310
311
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500

312
# install additional dependencies for openai api server
313
RUN --mount=type=cache,target=/root/.cache/uv \
314
    if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
315
        uv pip install --system accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.42.0' 'timm==0.9.10' boto3 runai-model-streamer runai-model-streamer[s3]; \
316
    else \
317
        uv pip install --system accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.45.3' 'timm==0.9.10' boto3 runai-model-streamer runai-model-streamer[s3]; \
318
    fi
319

yhu422's avatar
yhu422 committed
320
321
ENV VLLM_USAGE_SOURCE production-docker-image

322
323
324
# define sagemaker first, so it is not default from `docker build`
FROM vllm-openai-base AS vllm-sagemaker

325
COPY examples/online_serving/sagemaker-entrypoint.sh .
326
327
328
329
330
RUN chmod +x sagemaker-entrypoint.sh
ENTRYPOINT ["./sagemaker-entrypoint.sh"]

FROM vllm-openai-base AS vllm-openai

Stephen Krider's avatar
Stephen Krider committed
331
ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
Simon Mo's avatar
Simon Mo committed
332
#################### OPENAI API SERVER ####################