Dockerfile 12.3 KB
Newer Older
Simon Mo's avatar
Simon Mo committed
1
2
3
# The vLLM Dockerfile is used to construct vLLM image that can be directly used
# to run the OpenAI compatible server.

4
# Please update any changes made here to
5
6
# docs/source/contributing/dockerfile/dockerfile.md and
# docs/source/assets/contributing/dockerfile-stages-dependency.png
7

8
ARG CUDA_VERSION=12.4.1
Simon Mo's avatar
Simon Mo committed
9
#################### BASE BUILD IMAGE ####################
10
# prepare basic build environment
11
FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04 AS base
12
ARG CUDA_VERSION=12.4.1
13
ARG PYTHON_VERSION=3.12
14
ARG TARGETPLATFORM
15
16
ENV DEBIAN_FRONTEND=noninteractive

17
# Install minimal dependencies and uv
18
RUN apt-get update -y \
19
20
21
22
23
24
25
26
27
    && apt-get install -y ccache git curl wget sudo \
    && curl -LsSf https://astral.sh/uv/install.sh | sh

# Add uv to PATH
ENV PATH="/root/.local/bin:$PATH"
# Create venv with specified Python and activate by placing at the front of path
ENV VIRTUAL_ENV="/opt/venv"
RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV}
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
Stephen Krider's avatar
Stephen Krider committed
28

29
30
31
32
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500

33
34
35
36
37
38
39
40
# Upgrade to GCC 10 to avoid https://gcc.gnu.org/bugzilla/show_bug.cgi?id=92519
# as it was causing spam when compiling the CUTLASS kernels
RUN apt-get install -y gcc-10 g++-10
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 110 --slave /usr/bin/g++ g++ /usr/bin/g++-10
RUN <<EOF
gcc --version
EOF

41
42
43
44
# Workaround for https://github.com/openai/triton/issues/2507 and
# https://github.com/pytorch/pytorch/issues/107960 -- hopefully
# this won't be needed for future versions of this docker image
# or future versions of triton.
45
RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/
46

Stephen Krider's avatar
Stephen Krider committed
47
48
WORKDIR /workspace

49
50
51
52
# arm64 (GH200) build follows the practice of "use existing pytorch" build,
# we need to install torch and torchvision from the nightly builds first,
# pytorch will not appear as a vLLM dependency in all of the following steps
# after this step
53
RUN --mount=type=cache,target=/root/.cache/uv \
54
    if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
55
        uv pip install --index-url https://download.pytorch.org/whl/nightly/cu126 "torch==2.7.0.dev20250121+cu126" "torchvision==0.22.0.dev20250121";  \
56
    fi
Mor Zusman's avatar
Mor Zusman committed
57

58
59
COPY requirements-common.txt requirements-common.txt
COPY requirements-cuda.txt requirements-cuda.txt
60
RUN --mount=type=cache,target=/root/.cache/uv \
61
    uv pip install -r requirements-cuda.txt
62

63
64
65
66
67
68
# cuda arch list used by torch
# can be useful for both `dev` and `test`
# explicitly set the list to avoid issues with torch 2.2
# see https://github.com/pytorch/pytorch/pull/123243
ARG torch_cuda_arch_list='7.0 7.5 8.0 8.6 8.9 9.0+PTX'
ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
69
70
71
# Override the arch list for flash-attn to reduce the binary size
ARG vllm_fa_cmake_gpu_arches='80-real;90-real'
ENV VLLM_FA_CMAKE_GPU_ARCHES=${vllm_fa_cmake_gpu_arches}
Simon Mo's avatar
Simon Mo committed
72
73
#################### BASE BUILD IMAGE ####################

74
#################### WHEEL BUILD IMAGE ####################
75
FROM base AS build
76
ARG TARGETPLATFORM
77

78
79
# install build dependencies
COPY requirements-build.txt requirements-build.txt
80

81
82
83
84
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500

85
RUN --mount=type=cache,target=/root/.cache/uv \
86
    uv pip install -r requirements-build.txt
87

88
COPY . .
89
90
ARG GIT_REPO_CHECK=0
RUN --mount=type=bind,source=.git,target=.git \
91
    if [ "$GIT_REPO_CHECK" != "0" ]; then bash tools/check_repo.sh ; fi
Stephen Krider's avatar
Stephen Krider committed
92
93

# max jobs used by Ninja to build extensions
94
95
ARG max_jobs=2
ENV MAX_JOBS=${max_jobs}
96
97
98
# number of threads used by nvcc
ARG nvcc_threads=8
ENV NVCC_THREADS=$nvcc_threads
99

100
ARG USE_SCCACHE
101
102
ARG SCCACHE_BUCKET_NAME=vllm-build-sccache
ARG SCCACHE_REGION_NAME=us-west-2
103
ARG SCCACHE_S3_NO_CREDENTIALS=0
104
# if USE_SCCACHE is set, use sccache to speed up compilation
105
RUN --mount=type=cache,target=/root/.cache/uv \
106
    --mount=type=bind,source=.git,target=.git \
107
108
109
110
111
112
    if [ "$USE_SCCACHE" = "1" ]; then \
        echo "Installing sccache..." \
        && curl -L -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-x86_64-unknown-linux-musl.tar.gz \
        && tar -xzf sccache.tar.gz \
        && sudo mv sccache-v0.8.1-x86_64-unknown-linux-musl/sccache /usr/bin/sccache \
        && rm -rf sccache.tar.gz sccache-v0.8.1-x86_64-unknown-linux-musl \
113
114
        && export SCCACHE_BUCKET=${SCCACHE_BUCKET_NAME} \
        && export SCCACHE_REGION=${SCCACHE_REGION_NAME} \
115
        && export SCCACHE_S3_NO_CREDENTIALS=${SCCACHE_S3_NO_CREDENTIALS} \
116
        && export SCCACHE_IDLE_TIMEOUT=0 \
117
        && export CMAKE_BUILD_TYPE=Release \
118
        && sccache --show-stats \
119
        && python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38 \
120
121
122
        && sccache --show-stats; \
    fi

123
124
ENV CCACHE_DIR=/root/.cache/ccache
RUN --mount=type=cache,target=/root/.cache/ccache \
125
    --mount=type=cache,target=/root/.cache/uv \
126
    --mount=type=bind,source=.git,target=.git  \
127
    if [ "$USE_SCCACHE" != "1" ]; then \
128
129
130
        # Clean any existing CMake artifacts
        rm -rf .deps && \
        mkdir -p .deps && \
131
        python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38; \
132
    fi
133

134
# Check the size of the wheel if RUN_WHEEL_CHECK is true
135
COPY .buildkite/check-wheel-size.py check-wheel-size.py
136
# sync the default value with .buildkite/check-wheel-size.py
137
ARG VLLM_MAX_SIZE_MB=400
138
139
140
141
142
143
144
ENV VLLM_MAX_SIZE_MB=$VLLM_MAX_SIZE_MB
ARG RUN_WHEEL_CHECK=true
RUN if [ "$RUN_WHEEL_CHECK" = "true" ]; then \
        python3 check-wheel-size.py dist; \
    else \
        echo "Skipping wheel size check."; \
    fi
Simon Mo's avatar
Simon Mo committed
145
#################### EXTENSION Build IMAGE ####################
Stephen Krider's avatar
Stephen Krider committed
146

147
148
149
#################### DEV IMAGE ####################
FROM base as dev

150
151
152
153
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500

154
155
156
COPY requirements-lint.txt requirements-lint.txt
COPY requirements-test.txt requirements-test.txt
COPY requirements-dev.txt requirements-dev.txt
157
RUN --mount=type=cache,target=/root/.cache/uv \
158
    uv pip install -r requirements-dev.txt
159
#################### DEV IMAGE ####################
160

161
162
#################### vLLM installation IMAGE ####################
# image with vLLM installed
163
164
# TODO: Restore to base image after FlashInfer AOT wheel fixed
FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04 AS vllm-base
165
ARG CUDA_VERSION=12.4.1
166
ARG PYTHON_VERSION=3.12
Simon Mo's avatar
Simon Mo committed
167
WORKDIR /vllm-workspace
168
ENV DEBIAN_FRONTEND=noninteractive
169
170
ARG TARGETPLATFORM

171
172
RUN PYTHON_VERSION_STR=$(echo ${PYTHON_VERSION} | sed 's/\.//g') && \
    echo "export PYTHON_VERSION_STR=${PYTHON_VERSION_STR}" >> /etc/environment
173

174
# Install minimal dependencies and uv
175
RUN apt-get update -y \
176
177
178
179
180
181
182
183
184
185
    && apt-get install -y ccache git curl wget sudo vim \
    && apt-get install -y ffmpeg libsm6 libxext6 libgl1 libibverbs-dev \
    && curl -LsSf https://astral.sh/uv/install.sh | sh

# Add uv to PATH
ENV PATH="/root/.local/bin:$PATH"
# Create venv with specified Python and activate by placing at the front of path
ENV VIRTUAL_ENV="/opt/venv"
RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV}
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
186

187
188
189
190
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500

191
192
193
194
# Workaround for https://github.com/openai/triton/issues/2507 and
# https://github.com/pytorch/pytorch/issues/107960 -- hopefully
# this won't be needed for future versions of this docker image
# or future versions of triton.
195
RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/
196

197
198
199
200
# arm64 (GH200) build follows the practice of "use existing pytorch" build,
# we need to install torch and torchvision from the nightly builds first,
# pytorch will not appear as a vLLM dependency in all of the following steps
# after this step
201
RUN --mount=type=cache,target=/root/.cache/uv \
202
    if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
203
        uv pip install --index-url https://download.pytorch.org/whl/nightly/cu124 "torch==2.6.0.dev20241210+cu124" "torchvision==0.22.0.dev20241215";  \
204
205
    fi

206
# Install vllm wheel first, so that torch etc will be installed.
207
RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \
208
    --mount=type=cache,target=/root/.cache/uv \
209
    uv pip install dist/*.whl --verbose
Mor Zusman's avatar
Mor Zusman committed
210

211
# If we need to build FlashInfer wheel before its release:
212
213
214
215
216
217
# $ export FLASHINFER_ENABLE_AOT=1
# $ # Note we remove 7.0 from the arch list compared to the list below, since FlashInfer only supports sm75+
# $ export TORCH_CUDA_ARCH_LIST='7.5 8.0 8.6 8.9 9.0+PTX'
# $ git clone https://github.com/flashinfer-ai/flashinfer.git --recursive
# $ cd flashinfer
# $ git checkout 524304395bd1d8cd7d07db083859523fcaa246a4
218
# $ rm -rf build
219
# $ python3 setup.py bdist_wheel --dist-dir=dist --verbose
220
221
# $ ls dist
# $ # upload the wheel to a public location, e.g. https://wheels.vllm.ai/flashinfer/524304395bd1d8cd7d07db083859523fcaa246a4/flashinfer_python-0.2.1.post1+cu124torch2.5-cp38-abi3-linux_x86_64.whl
222

223
RUN --mount=type=cache,target=/root/.cache/uv \
224
if [ "$TARGETPLATFORM" != "linux/arm64" ]; then \
225
    uv pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.2.1.post1/flashinfer_python-0.2.1.post1+cu124torch2.5-cp38-abi3-linux_x86_64.whl ; \
226
fi
227
COPY examples examples
228
229
230
231
232
233

# Although we build Flashinfer with AOT mode, there's still
# some issues w.r.t. JIT compilation. Therefore we need to
# install build dependencies for JIT compilation.
# TODO: Remove this once FlashInfer AOT wheel is fixed
COPY requirements-build.txt requirements-build.txt
234
RUN --mount=type=cache,target=/root/.cache/uv \
235
    uv pip install -r requirements-build.txt
236

237
#################### vLLM installation IMAGE ####################
Stephen Krider's avatar
Stephen Krider committed
238

239
240
241
242
#################### TEST IMAGE ####################
# image to run unit testing suite
# note that this uses vllm installed by `pip`
FROM vllm-base AS test
Stephen Krider's avatar
Stephen Krider committed
243

244
ADD . /vllm-workspace/
Stephen Krider's avatar
Stephen Krider committed
245

246
247
248
249
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500

250
# install development dependencies (for testing)
251
RUN --mount=type=cache,target=/root/.cache/uv \
252
    uv pip install -r requirements-dev.txt
253

youkaichao's avatar
youkaichao committed
254
# install development dependencies (for testing)
255
RUN --mount=type=cache,target=/root/.cache/uv \
256
    uv pip install -e tests/vllm_test_utils
youkaichao's avatar
youkaichao committed
257

258
# enable fast downloads from hf (for testing)
259
RUN --mount=type=cache,target=/root/.cache/uv \
260
    uv pip install hf_transfer
261
262
ENV HF_HUB_ENABLE_HF_TRANSFER 1

Joe Runde's avatar
Joe Runde committed
263
264
265
# Copy in the v1 package for testing (it isn't distributed yet)
COPY vllm/v1 /usr/local/lib/python3.12/dist-packages/vllm/v1

266
267
268
269
270
271
272
# doc requires source code
# we hide them inside `test_docs/` , so that this source code
# will not be imported by other tests
RUN mkdir test_docs
RUN mv docs test_docs/
RUN mv vllm test_docs/
#################### TEST IMAGE ####################
Stephen Krider's avatar
Stephen Krider committed
273

Simon Mo's avatar
Simon Mo committed
274
#################### OPENAI API SERVER ####################
275
276
# base openai image with additional requirements, for any subsequent openai-style images
FROM vllm-base AS vllm-openai-base
277

278
279
280
281
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
ENV UV_HTTP_TIMEOUT=500

282
# install additional dependencies for openai api server
283
RUN --mount=type=cache,target=/root/.cache/uv \
284
    if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
285
        uv pip install accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.42.0' 'timm==0.9.10' boto3 runai-model-streamer runai-model-streamer[s3]; \
286
    else \
287
        uv pip install accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.45.0' 'timm==0.9.10' boto3 runai-model-streamer runai-model-streamer[s3]; \
288
    fi
289

yhu422's avatar
yhu422 committed
290
291
ENV VLLM_USAGE_SOURCE production-docker-image

292
293
294
# define sagemaker first, so it is not default from `docker build`
FROM vllm-openai-base AS vllm-sagemaker

295
COPY examples/online_serving/sagemaker-entrypoint.sh .
296
297
298
299
300
RUN chmod +x sagemaker-entrypoint.sh
ENTRYPOINT ["./sagemaker-entrypoint.sh"]

FROM vllm-openai-base AS vllm-openai

Stephen Krider's avatar
Stephen Krider committed
301
ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
Simon Mo's avatar
Simon Mo committed
302
#################### OPENAI API SERVER ####################