Add manylinux builder (#164)

* update builder

* remove root permission

* update readme

* update setup.py

* add install cuda 12.1 script

* use generate.sh

* add nccl to install_requires

* update README.md

* fix lint

* update setup.py

---------
Co-authored-by: chenxin <chenxin@pjlab.org.cn>

.gitignore

@@ -6,6 +6,7 @@ __pycache__/
 workspace/
 .cache
 *build*/
+!builder/
 lmdeploy/lib/
 dist/
 examples/cpp/llama/*.csv

builder/manywheel/Dockerfile_2014

+# syntax = docker/dockerfile:experimental
+FROM quay.io/pypa/manylinux2014_x86_64 as base
+ARG DEVTOOLSET_VERSION=9
+ARG BASE_CUDA_VERSION=11.8
+
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US.UTF-8
+
+# gcc/g++
+RUN yum install -y \
+    devtoolset-${DEVTOOLSET_VERSION}-gcc \
+    devtoolset-${DEVTOOLSET_VERSION}-gcc-c++ \
+    devtoolset-${DEVTOOLSET_VERSION}-gcc-gfortran \
+    devtoolset-${DEVTOOLSET_VERSION}-binutils
+ENV PATH=/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH
+ENV LD_LIBRARY_PATH=/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH
+
+RUN yum install -y wget rapidjson-devel glog-devel
+
+FROM base as cuda
+ADD manywheel/scripts/install_cuda.sh install_cuda.sh
+RUN bash ./install_cuda.sh ${BASE_CUDA_VERSION} && rm install_cuda.sh
+
+FROM base as conda
+ADD manywheel/scripts/install_conda.sh install_conda.sh
+RUN bash ./install_conda.sh && rm install_conda.sh
+RUN
+RUN /opt/conda/bin/conda create -n py38 python=3.8 -yq && \
+    /opt/conda/envs/py38/bin/pip install pybind11 && \
+    /opt/conda/bin/conda create -n py39 python=3.9 -yq && \
+    /opt/conda/envs/py39/bin/pip install pybind11 && \
+    /opt/conda/bin/conda create -n py310 python=3.10 -yq && \
+    /opt/conda/envs/py310/bin/pip install pybind11 && \
+    /opt/conda/bin/conda create -n py311 python=3.11 -yq && \
+    /opt/conda/envs/py311/bin/pip install pybind11
+
+FROM base as mpi
+ADD manywheel/scripts/install_openmpi.sh install_openmpi.sh
+RUN bash ./install_openmpi.sh && rm install_openmpi.sh
+
+FROM base as cuda_final
+COPY --from=cuda            /usr/local/cuda-${BASE_CUDA_VERSION}  /usr/local/cuda-${BASE_CUDA_VERSION}
+RUN ln -sf /usr/local/cuda-${BASE_CUDA_VERSION} /usr/local/cuda
+ENV PATH=/usr/local/cuda/bin:$PATH
+COPY --from=conda           /opt/conda                            /opt/conda
+RUN /opt/conda/bin/conda init bash
+COPY --from=mpi             /usr/local/mpi                        /usr/local/mpi
+ENV PATH=/usr/local/mpi/bin:$PATH

builder/manywheel/README.md

+# Build lmdeploy manylinux wheel
+
+## Prepare docker image
+
+To build all docker images you can use the convenient script:
+
+```bash
+./build_all_docker.sh
+# Build with pushing
+WITH_PUSH=true ./build_all_docker.sh
+```
+
+To build a docker image with specific cuda version or manylinux-docker version, you may use:
+
+```bash
+MANY_LINUX_VERSION=2014 GPU_ARCH_VERSION=11.8 ./build_docker.sh
+```
+
+## Build lmdeploy wheel
+
+```bash
+./build_all_wheel.sh
+```

builder/manywheel/build_all_docker.sh

+#!/usr/bin/env bash
+
+set -eou pipefail
+
+TOPDIR=$(git rev-parse --show-toplevel)/builder
+
+for cuda_version in 11.8; do
+    MANY_LINUX_VERSION=2014 GPU_ARCH_VERSION="${cuda_version}" "${TOPDIR}/manywheel/build_docker.sh"
+done

builder/manywheel/build_all_wheel.sh

+#!/usr/bin/env bash
+
+set -eou pipefail
+
+TOPDIR=$(git rev-parse --show-toplevel)/builder
+
+PLAT_NAME=manylinux2014_x86_64
+for cuver in 11.8; do
+    DOCKER_TAG=cuda${cuver}
+    OUTPUT_FOLDER=cuda${cuver}_dist
+    for pyver in py38 py39 py310 py311; do
+        bash ${TOPDIR}/manywheel/build_wheel.sh ${pyver} ${PLAT_NAME} ${DOCKER_TAG} ${OUTPUT_FOLDER} \
+            |& tee ${PLAT_NAME}.${pyver}.cuda${cuver}.log.txt
+    done
+done

builder/manywheel/build_docker.sh

+#!/usr/bin/env bash
+
+set -eou pipefail
+
+TOPDIR=$(git rev-parse --show-toplevel)/builder
+GPU_ARCH_VERSION=${GPU_ARCH_VERSION}
+WITH_PUSH=${WITH_PUSH:-}
+
+TARGET=cuda_final
+DOCKER_TAG=cuda${GPU_ARCH_VERSION}
+DOCKER_BUILD_ARG="--build-arg BASE_CUDA_VERSION=${GPU_ARCH_VERSION} --build-arg DEVTOOLSET_VERSION=9"
+DOCKER_TAG=cuda${GPU_ARCH_VERSION}
+
+DOCKER_IMAGE=openmmlab/lmdeploy-builder:${DOCKER_TAG}
+if [[ -n ${MANY_LINUX_VERSION} ]]; then
+    DOCKERFILE_SUFFIX=_${MANY_LINUX_VERSION}
+else
+    DOCKERFILE_SUFFIX=''
+fi
+
+(
+    set -x
+    DOCKER_BUILDKIT=1 docker build \
+        -t "${DOCKER_IMAGE}" \
+        ${DOCKER_BUILD_ARG} \
+        --target "${TARGET}" \
+        -f "${TOPDIR}/manywheel/Dockerfile${DOCKERFILE_SUFFIX}" \
+        "${TOPDIR}"
+)
+
+if [[ "${WITH_PUSH}" == true ]]; then
+    (
+        set -x
+        docker push "${DOCKER_IMAGE}"
+    )
+fi

builder/manywheel/build_wheel.sh

+#!/usr/bin/env bash
+set -eux
+
+PYTHON_VERSION="$1"
+PLAT_NAME="$2"
+DOCKER_TAG="$3"
+OUTPUT_DIR="$4"
+
+DOCKER_IMAGE="openmmlab/lmdeploy-builder:${DOCKER_TAG}"
+export USERID=$(id -u)
+export GROUPID=$(id -g)
+
+cd "$(dirname "$0")"  # move inside the script directory
+mkdir -p "${OUTPUT_DIR}"
+docker pull ${DOCKER_IMAGE}
+docker run --rm -it \
+    --env PYTHON_VERSION="${PYTHON_VERSION}" \
+    --env PLAT_NAME="${PLAT_NAME}" \
+    --env USERID="${USERID}" \
+    --env GROUPID="${GROUPID}" \
+    --volume "$(pwd)/${OUTPUT_DIR}:/lmdeploy_build" \
+    --volume "$(pwd)/entrypoint_build.sh:/entrypoint_build.sh" \
+    --entrypoint /entrypoint_build.sh \
+    ${DOCKER_IMAGE}

builder/manywheel/entrypoint_build.sh

+#!/usr/bin/env bash
+set -eux
+
+export PYTHON_VERSION=$PYTHON_VERSION
+export PLAT_NAME=$PLAT_NAME
+export USERID=${USERID}
+export GROUPID=${GROUPID}
+export CUDAVER=$(nvcc --version | sed -n 's/^.*release \([0-9]\+\).*$/\1/p')
+
+source /opt/conda/bin/activate
+conda activate $PYTHON_VERSION
+
+git clone https://github.com/InternLM/lmdeploy
+cd lmdeploy
+mkdir build && cd build
+bash ../generate.sh
+make -j$(nproc) && make install
+cd ..
+rm -rf build
+python setup.py bdist_wheel --cuda=${CUDAVER} --plat-name $PLAT_NAME -d /tmpbuild/
+chown ${USERID}:${GROUPID} /tmpbuild/*
+mv /tmpbuild/* /lmdeploy_build/

builder/manywheel/scripts/install_conda.sh

+#!/bin/bash
+
+set -ex
+
+wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+chmod +x  Miniconda3-latest-Linux-x86_64.sh
+bash ./Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda
+rm Miniconda3-latest-Linux-x86_64.sh

builder/manywheel/scripts/install_cuda.sh

+#!/bin/bash
+
+set -ex
+
+function install_118 {
+    echo "Installing CUDA 11.8 and cuDNN 8.7 and NCCL 2.15"
+    rm -rf /usr/local/cuda-11.8 /usr/local/cuda
+    # install CUDA 11.8.0 in the same container
+    wget -q https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda_11.8.0_520.61.05_linux.run
+    chmod +x cuda_11.8.0_520.61.05_linux.run
+    ./cuda_11.8.0_520.61.05_linux.run --toolkit --silent
+    rm -f cuda_11.8.0_520.61.05_linux.run
+    rm -f /usr/local/cuda && ln -s /usr/local/cuda-11.8 /usr/local/cuda
+
+    # cuDNN license: https://developer.nvidia.com/cudnn/license_agreement
+    mkdir tmp_cudnn && cd tmp_cudnn
+    wget -q https://developer.download.nvidia.com/compute/redist/cudnn/v8.7.0/local_installers/11.8/cudnn-linux-x86_64-8.7.0.84_cuda11-archive.tar.xz -O cudnn-linux-x86_64-8.7.0.84_cuda11-archive.tar.xz
+    tar xf cudnn-linux-x86_64-8.7.0.84_cuda11-archive.tar.xz
+    cp -a cudnn-linux-x86_64-8.7.0.84_cuda11-archive/include/* /usr/local/cuda/include/
+    cp -a cudnn-linux-x86_64-8.7.0.84_cuda11-archive/lib/* /usr/local/cuda/lib64/
+    cd ..
+    rm -rf tmp_cudnn
+    ldconfig
+
+    # NCCL license: https://docs.nvidia.com/deeplearning/nccl/#licenses
+    mkdir tmp_nccl && cd tmp_nccl
+    wget -q https://developer.download.nvidia.com/compute/redist/nccl/v2.15.5/nccl_2.15.5-1+cuda11.8_x86_64.txz
+    tar xf nccl_2.15.5-1+cuda11.8_x86_64.txz
+    cp -a nccl_2.15.5-1+cuda11.8_x86_64/include/* /usr/local/cuda/include/
+    cp -a nccl_2.15.5-1+cuda11.8_x86_64/lib/* /usr/local/cuda/lib64/
+    cd ..
+    rm -rf tmp_nccl
+    ldconfig
+}
+
+function install_121 {
+    echo "Installing CUDA 12.1 and cuDNN 8.9 and NCCL 2.18.1"
+    rm -rf /usr/local/cuda-12.1 /usr/local/cuda
+    # install CUDA 12.1.0 in the same container
+    wget -q https://developer.download.nvidia.com/compute/cuda/12.1.0/local_installers/cuda_12.1.0_530.30.02_linux.run
+    chmod +x cuda_12.1.0_530.30.02_linux.run
+    ./cuda_12.1.0_530.30.02_linux.run --toolkit --silent
+    rm -f cuda_12.1.0_530.30.02_linux.run
+    rm -f /usr/local/cuda && ln -s /usr/local/cuda-12.1 /usr/local/cuda
+
+    # cuDNN license: https://developer.nvidia.com/cudnn/license_agreement
+    mkdir tmp_cudnn && cd tmp_cudnn
+    wget -q https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.9.2.26_cuda12-archive.tar.xz -O cudnn-linux-x86_64-8.9.2.26_cuda12-archive.tar.xz
+    tar xf cudnn-linux-x86_64-8.9.2.26_cuda12-archive.tar.xz
+    cp -a cudnn-linux-x86_64-8.9.2.26_cuda12-archive/include/* /usr/local/cuda/include/
+    cp -a cudnn-linux-x86_64-8.9.2.26_cuda12-archive/lib/* /usr/local/cuda/lib64/
+    cd ..
+    rm -rf tmp_cudnn
+    ldconfig
+
+    # NCCL license: https://docs.nvidia.com/deeplearning/nccl/#licenses
+    mkdir tmp_nccl && cd tmp_nccl
+    wget -q https://developer.download.nvidia.com/compute/redist/nccl/v2.18.1/nccl_2.18.1-1+cuda12.1_x86_64.txz
+    tar xf nccl_2.18.1-1+cuda12.1_x86_64.txz
+    cp -a nccl_2.18.1-1+cuda12.1_x86_64/include/* /usr/local/cuda/include/
+    cp -a nccl_2.18.1-1+cuda12.1_x86_64/lib/* /usr/local/cuda/lib64/
+    cd ..
+    rm -rf tmp_nccl
+    ldconfig
+}
+
+if test $# -eq 0
+then
+    echo "doesn't provide cuda version"; exit 1;
+fi
+
+# idiomatic parameter and option handling in sh
+while test $# -gt 0
+do
+    case "$1" in
+    11.8) install_118
+	        ;;
+    12.1) install_121
+            ;;
+	*) echo "bad argument $1"; exit 1
+	   ;;
+    esac
+    shift
+done

builder/manywheel/scripts/install_openmpi.sh

+#!/bin/bash
+
+set -ex
+
+wget -q https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.5.tar.gz
+tar xf openmpi-4.1.5.tar.gz
+cd openmpi-4.1.5
+./configure --prefix=/usr/local/mpi
+make -j$(nproc)
+make install

setup.py

@@ -20,6 +20,12 @@ def get_version():
     return locals()['__version__']
 
 
+def check_ext_modules():
+    if os.path.exists(os.path.join(pwd, 'lmdeploy', 'lib')):
+        return True
+    return False
+
+
 def parse_requirements(fname='requirements.txt', with_version=True):
     """Parse the package dependencies listed in a file but strips specific
     versioning information.
@@ -36,6 +42,21 @@ def parse_requirements(fname='requirements.txt', with_version=True):
     """
     require_fpath = fname
 
+    def get_nccl_pkg():
+        arg_name = '--cuda='
+        arg_value = None
+        for arg in sys.argv[1:]:
+            if arg.startswith(arg_name):
+                arg_value = arg[len(arg_name):]
+                sys.argv.remove(arg)
+                break
+
+        if arg_value == '11':
+            return 'nvidia-nccl-cu11'
+        elif arg_value == '12':
+            return 'nvidia-nccl-cu12'
+        return None
+
     def parse_line(line):
         """Parse information from a line in a requirements text file."""
         if line.startswith('-r '):
@@ -92,6 +113,9 @@ def parse_requirements(fname='requirements.txt', with_version=True):
                 yield item
 
     packages = list(gen_packages_items())
+    nccl_pkg = get_nccl_pkg()
+    if nccl_pkg is not None:
+        packages += [nccl_pkg]
     return packages
 
 
@@ -106,10 +130,12 @@ if __name__ == '__main__':
           packages=find_packages(exclude=()),
           include_package_data=True,
           install_requires=parse_requirements('requirements.txt'),
+          has_ext_modules=check_ext_modules,
           classifiers=[
               'Programming Language :: Python :: 3.8',
               'Programming Language :: Python :: 3.9',
               'Programming Language :: Python :: 3.10',
+              'Programming Language :: Python :: 3.11',
               'Intended Audience :: Developers',
               'Intended Audience :: Education',
               'Intended Audience :: Science/Research',

src/turbomind/python/CMakeLists.txt

@@ -19,4 +19,4 @@ target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_14)
 
 set_target_properties(${PROJECT_NAME} PROPERTIES
         BUILD_RPATH "\$ORIGIN"
-        INSTALL_RPATH "\$ORIGIN")
+        INSTALL_RPATH "\$ORIGIN;../../nvidia/nccl/lib/")

src/turbomind/triton_backend/CMakeLists.txt

@@ -284,6 +284,9 @@ export(PACKAGE TritonTurboMindBackend)
 
 add_library(TransformerTritonBackend SHARED transformer_triton_backend.cpp)
 target_link_libraries(TransformerTritonBackend PRIVATE nccl_utils)
+set_target_properties(TransformerTritonBackend PROPERTIES
+  INSTALL_RPATH "../../nvidia/nccl/lib/"
+)
 install(TARGETS TransformerTritonBackend DESTINATION ${CMAKE_INSTALL_LIBDIR})
 
 add_subdirectory(llama)