Skip to content

GitLab

".devcontainer/sglang/devcontainer.json" did not exist on "8bdf18e5e70b6101b04fcbbd873b2d422c59c729"
Unverified Commit 7a3b15e6 authored by Dmitry Tokarev's avatar Dmitry Tokarev Committed by GitHub
Browse files

feat: VLLM + CUDA 13 (#4997) 


Signed-off-by: default avatarDmitry Tokarev <dtokarev@nvidia.com>
Co-authored-by: default avatarcoderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
parent aaed4f3b
Show whitespace changes
Inline Side-by-side
Showing with 88 additions and 33 deletions
container/Dockerfile.vllm
View file @ 7a3b15e6
......@@ -510,7 +510,15 @@ RUN --mount=type=bind,source=./container/deps/,target=/tmp/deps \
--mount=type=cache,target=/root/.cache/uv \
cp /tmp/deps/vllm/install_vllm.sh /tmp/install_vllm.sh && \
chmod +x /tmp/install_vllm.sh && \
/tmp/install_vllm.sh --vllm-ref $VLLM_REF --max-jobs $MAX_JOBS --arch $ARCH --installation-dir /opt ${DEEPGEMM_REF:+--deepgemm-ref "$DEEPGEMM_REF"} ${FLASHINF_REF:+--flashinf-ref "$FLASHINF_REF"} ${LMCACHE_REF:+--lmcache-ref "$LMCACHE_REF"} --cuda-version $CUDA_VERSION
/tmp/install_vllm.sh \
--vllm-ref $VLLM_REF \
--max-jobs $MAX_JOBS \
--arch $ARCH \
--installation-dir /opt \
${DEEPGEMM_REF:+--deepgemm-ref "$DEEPGEMM_REF"} \
${FLASHINF_REF:+--flashinf-ref "$FLASHINF_REF"} \
${LMCACHE_REF:+--lmcache-ref "$LMCACHE_REF"} \
--cuda-version $CUDA_VERSION
ENV LD_LIBRARY_PATH=\
/opt/vllm/tools/ep_kernels/ep_kernels_workspace/nvshmem_install/lib:\
......@@ -553,8 +561,9 @@ COPY --from=dynamo_base /usr/local/cuda/bin/fatbinary /usr/local/cuda/bin/fatbin
COPY --from=dynamo_base /usr/local/cuda/include/ /usr/local/cuda/include/
COPY --from=dynamo_base /usr/local/cuda/nvvm /usr/local/cuda/nvvm
COPY --from=dynamo_base /usr/local/cuda/lib64/libcudart.so* /usr/local/cuda/lib64/
RUN ln -s /usr/local/cuda/lib64/libcublas.so.12 /usr/local/cuda/lib64/libcublas.so
RUN ln -s /usr/local/cuda/lib64/libcublasLt.so.12 /usr/local/cuda/lib64/libcublasLt.so
RUN CUDA_VERSION_MAJOR="${CUDA_VERSION%%.*}" &&\
ln -s /usr/local/cuda/lib64/libcublas.so.${CUDA_VERSION_MAJOR} /usr/local/cuda/lib64/libcublas.so &&\
ln -s /usr/local/cuda/lib64/libcublasLt.so.${CUDA_VERSION_MAJOR} /usr/local/cuda/lib64/libcublasLt.so
# DeepGemm runs nvcc for JIT kernel compilation, however the CUDA include path
# is not properly set for complilation. Set CPATH to help nvcc find the headers.
......@@ -587,6 +596,8 @@ ARG PYTHON_VERSION
# Install Python, build-essential and python3-dev as apt dependencies
RUN apt-get update && \
CUDA_VERSION_MAJOR=${CUDA_VERSION%%.*} &&\
CUDA_VERSION_MINOR=$(echo "${CUDA_VERSION#*.}" | cut -d. -f1) && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
# Python runtime - CRITICAL for virtual environment to work
python${PYTHON_VERSION}-dev \
......@@ -605,7 +616,7 @@ RUN apt-get update && \
# prometheus dependencies
ca-certificates \
# DeepGemm uses 'cuobjdump' which does not come with CUDA image
cuda-command-line-tools-12-9 && \
cuda-command-line-tools-${CUDA_VERSION_MAJOR}-${CUDA_VERSION_MINOR} && \
rm -rf /var/lib/apt/lists/*
USER dynamo
......
container/build.sh
View file @ 7a3b15e6
......@@ -107,6 +107,10 @@ VLLM_BASE_IMAGE="nvcr.io/nvidia/cuda-dl-base"
# for details and reproducer to manually test if the image
# can be updated to later versions.
VLLM_BASE_IMAGE_TAG="25.04-cuda12.9-devel-ubuntu24.04"
VLLM_BASE_IMAGE_TAG_CU13="25.11-cuda13.0-devel-ubuntu24.04"
VLLM_RUNTIME_IMAGE="nvcr.io/nvidia/cuda"
VLLM_RUNTIME_IMAGE_TAG="12.9.0-runtime-ubuntu24.04"
VLLM_RUNTIME_IMAGE_TAG_CU13="13.0.2-runtime-ubuntu24.04"
NONE_BASE_IMAGE="nvcr.io/nvidia/cuda-dl-base"
NONE_BASE_IMAGE_TAG="25.01-cuda12.8-devel-ubuntu24.04"
......@@ -161,6 +165,16 @@ get_options() {
missing_requirement "$1"
fi
;;
--cuda-version)
if [ "$2" ]; then
echo "INFO: Setting CUDA_VERSION to $2"
CUDA_VERSION=$2
BUILD_ARGS+=" --build-arg CUDA_VERSION=$2 "
shift
else
missing_requirement "$1"
fi
;;
--tensorrtllm-pip-wheel-dir)
if [ "$2" ]; then
TENSORRTLLM_PIP_WHEEL_DIR=$2
......@@ -316,7 +330,6 @@ get_options() {
missing_requirement "$1"
fi
;;
--sccache-region)
if [ "$2" ]; then
SCCACHE_REGION=$2
......@@ -379,6 +392,7 @@ get_options() {
if [ -z "$BASE_IMAGE_TAG" ]; then
BASE_IMAGE_TAG=${FRAMEWORK}_BASE_IMAGE_TAG
BASE_IMAGE_TAG=${!BASE_IMAGE_TAG}
echo "INFO: Using default base image tag for $FRAMEWORK: $BASE_IMAGE_TAG"
fi
if [ -z "$BASE_IMAGE" ]; then
......@@ -386,6 +400,14 @@ get_options() {
BASE_IMAGE=${!BASE_IMAGE}
fi
if [[ $FRAMEWORK == "VLLM" ]] && [[ $CUDA_VERSION == "13."* ]]; then
BASE_IMAGE_TAG=$VLLM_BASE_IMAGE_TAG_CU13
BUILD_ARGS+=" --build-arg BASE_IMAGE_TAG=${VLLM_BASE_IMAGE_TAG_CU13} "
RUNTIME_IMAGE_TAG=$VLLM_RUNTIME_IMAGE_TAG_CU13
BUILD_ARGS+=" --build-arg RUNTIME_IMAGE_TAG=${VLLM_RUNTIME_IMAGE_TAG_CU13} "
echo "INFO: Overriding base image tag for vLLM with CUDA 13: $BASE_IMAGE_TAG AND RUNTIME_IMAGE_TAG: $RUNTIME_IMAGE_TAG"
fi
if [ -z "$BASE_IMAGE" ]; then
error "ERROR: Framework $FRAMEWORK without BASE_IMAGE"
fi
......@@ -521,17 +543,6 @@ if [[ $FRAMEWORK == "VLLM" ]] && [[ "$PLATFORM" == *"linux/arm64"* ]]; then
BUILD_ARGS+=" --build-arg RUNTIME_IMAGE_TAG=12.9.0-runtime-ubuntu24.04 "
echo "INFO: Automatically setting RUNTIME_IMAGE_TAG=12.9.0-runtime-ubuntu24.04 for vLLM ARM64"
fi
if [[ "$BUILD_ARGS" != *"CUDA_VERSION"* ]]; then
BUILD_ARGS+=" --build-arg CUDA_VERSION=129 "
echo "INFO: Automatically setting CUDA_VERSION=129 for vLLM ARM64"
fi
if [[ "$BUILD_ARGS" != *"TORCH_BACKEND"* ]]; then
BUILD_ARGS+=" --build-arg TORCH_BACKEND=cu129 "
echo "INFO: Automatically setting TORCH_BACKEND=cu129 for vLLM ARM64"
fi
fi
# Update DOCKERFILE if framework is VLLM
......
container/deps/vllm/install_vllm.sh
View file @ 7a3b15e6
......@@ -11,7 +11,8 @@
set -euo pipefail
VLLM_REF="v0.12.0"
VLLM_VER="0.12.0"
VLLM_REF="v${VLLM_VER}"
# Basic Configurations
ARCH=$(uname -m)
......@@ -19,7 +20,7 @@ MAX_JOBS=16
INSTALLATION_DIR=/tmp
# VLLM and Dependency Configurations
TORCH_CUDA_ARCH_LIST="9.0;10.0" # For EP Kernels
TORCH_CUDA_ARCH_LIST="9.0;10.0" # For EP Kernels -- TODO: check if we need to add 12.0+PTX
DEEPGEMM_REF=""
CUDA_VERSION="12.9"
FLASHINF_REF="v0.5.3"
......@@ -97,44 +98,76 @@ export CUDA_HOME=/usr/local/cuda
# Derive torch backend from CUDA version (e.g., "12.9" -> "cu129")
TORCH_BACKEND="cu$(echo $CUDA_VERSION | tr -d '.')"
CUDA_VERSION_MAJOR=${CUDA_VERSION%%.*}
echo "=== Installing prerequisites ==="
uv pip install pip cuda-python
echo "\n=== Configuration Summary ==="
echo " VLLM_REF=$VLLM_REF | ARCH=$ARCH | CUDA_VERSION=$CUDA_VERSION | TORCH_BACKEND=$TORCH_BACKEND"
echo " FLASHINF_REF=$FLASHINF_REF | LMCACHE_REF=$LMCACHE_REF | DEEPGEMM_REF=$DEEPGEMM_REF"
echo " TORCH_CUDA_ARCH_LIST=$TORCH_CUDA_ARCH_LIST | INSTALLATION_DIR=$INSTALLATION_DIR"
echo "\n=== Installing LMCache ==="
if [ "$ARCH" = "amd64" ]; then
if [[ "$CUDA_VERSION_MAJOR" == "12" ]]; then
echo " FLASHINF_REF=$FLASHINF_REF | LMCACHE_REF=$LMCACHE_REF | DEEPGEMM_REF=$DEEPGEMM_REF"
echo "\n=== Installing LMCache ==="
if [ "$ARCH" = "amd64" ]; then
# LMCache installation currently fails on arm64 due to CUDA dependency issues
# Install LMCache BEFORE vLLM so vLLM's dependencies take precedence
uv pip install lmcache==${LMCACHE_REF} --torch-backend=${TORCH_BACKEND}
echo "✓ LMCache ${LMCACHE_REF} installed"
else
else
echo "⚠ Skipping LMCache on ARM64 (compatibility issues)"
fi
else
echo " FLASHINF_REF=$FLASHINF_REF | LMCache will not be installed as it doesn't support CUDA 13 yet | DEEPGEMM_REF=$DEEPGEMM_REF"
fi
echo "\n=== Cloning vLLM repository ==="
# Clone needed for DeepGEMM and EP kernels install scripts
cd $INSTALLATION_DIR
git clone https://github.com/vllm-project/vllm.git vllm
cd vllm
git checkout $VLLM_REF
# TODO: remove this cherry-pick when vllm is upgraded to > 0.12.0 (when the fix is shipped)
git cherry-pick --no-commit 799804d140fc99ce3964648ba91aaa810cf28fef # nvshmem fix for CUDA 13.0
echo "✓ vLLM repository cloned"
echo "\n=== Installing vLLM & FlashInfer ==="
echo "Installing vLLM $VLLM_REF from PyPI..."
uv pip install vllm[flashinfer,runai]==$VLLM_REF --torch-backend=${TORCH_BACKEND}
uv pip install flashinfer-cubin==$FLASHINF_REF
uv pip install flashinfer-jit-cache==$FLASHINF_REF --extra-index-url https://flashinfer.ai/whl/${TORCH_BACKEND}
echo "\n=== Installing vLLM & FlashInfer ==="
if [[ "$CUDA_VERSION_MAJOR" == "12" ]]; then
echo "Installing vLLM $VLLM_REF from PyPI..."
uv pip install vllm[flashinfer,runai]==$VLLM_REF --torch-backend=${TORCH_BACKEND}
uv pip install flashinfer-cubin==$FLASHINF_REF
uv pip install flashinfer-jit-cache==$FLASHINF_REF --extra-index-url https://flashinfer.ai/whl/${TORCH_BACKEND}
elif [[ "$CUDA_VERSION_MAJOR" == "13" ]]; then
if [ "$ARCH" = "amd64" ]; then
echo "Installing vLLM $VLLM_REF from GitHub since CUDA 13 x86_64 wheel is only present on GitHub..."
uv pip install \
--index-strategy=unsafe-best-match \
--extra-index-url https://download.pytorch.org/whl/${TORCH_BACKEND} \
nixl[cu13]==0.7.1 \
https://github.com/vllm-project/vllm/releases/download/v${VLLM_VER}/vllm-${VLLM_VER}+${TORCH_BACKEND}-cp38-abi3-manylinux_2_31_x86_64.whl[flashinfer,runai] \
--torch-backend=${TORCH_BACKEND}
uv pip install flashinfer-cubin==$FLASHINF_REF
uv pip install flashinfer-jit-cache==$FLASHINF_REF --extra-index-url https://flashinfer.ai/whl/${TORCH_BACKEND}
echo "✓ vLLM installation completed"
else
echo "⚠ Skipping LMCache on ARM64 (compatibility issues, missing aarch64 wheels)"
echo "Building vLLM from source for ${ARCH} architecture..."
echo "Try to install specific PyTorch and other dependencies first"
uv pip install --index-strategy=unsafe-best-match --index https://download.pytorch.org/whl/ -r requirements/cuda.txt
uv pip install setuptools_scm # required to build vLLM from source
MAX_JOBS=${MAX_JOBS} uv pip install -v --no-build-isolation .
fi
else
echo "❌ Unsupported CUDA version for vLLM installation: ${CUDA_VERSION}"
exit 1
fi
echo "✓ vLLM installation completed"
echo "\n=== Installing DeepGEMM ==="
cd $INSTALLATION_DIR/vllm/tools
if [ -n "$DEEPGEMM_REF" ]; then
bash install_deepgemm.sh --cuda-version "${CUDA_VERSION}" --ref "$DEEPGEMM_REF"
else
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment