Skip to content

GitLab

Commit 62485a34 authored by Aarni Koskela's avatar Aarni Koskela Committed by Titus
Browse files

Move build scripts to .github/scripts (from scripts/ and workflow YAML)

parent 958dfa99
Hide whitespace changes
Inline Side-by-side
Showing with 64 additions and 78 deletions
.github/scripts/build-cpu.sh 0 → 100644
View file @ 62485a34
#!/bin/bash
declare build_arch
declare build_os
set -xeuo pipefail
pip install cmake==3.28.3
if [ "${build_os:0:6}" == ubuntu ] && [ "${build_arch}" == aarch64 ]; then
# Allow cross-compile on aarch64
sudo apt-get update
sudo apt-get install -y gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu g++-aarch64-linux-gnu
cmake -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++ -DCOMPUTE_BACKEND=cpu .
elif [ "${build_os:0:5}" == macos ] && [ "${build_arch}" == aarch64 ]; then
cmake -DCMAKE_OSX_ARCHITECTURES=arm64 -DCOMPUTE_BACKEND=cpu .
else
cmake -DCOMPUTE_BACKEND=cpu .
fi
cmake --build . --config Release
output_dir="output/${build_os}/${build_arch}"
mkdir -p "${output_dir}"
(shopt -s nullglob && cp bitsandbytes/*.{so,dylib,dll} "${output_dir}")
.github/scripts/build-cuda.sh 0 → 100644
View file @ 62485a34
#!/bin/bash
declare build_arch
declare build_os
declare cuda_version
set -xeuo pipefail
build_capability="50;52;60;61;70;75;80;86;89;90"
[[ "${cuda_version}" == 11.7.* ]] && build_capability=${build_capability%??????}
[[ "${cuda_version}" == 11.8.* ]] && build_capability=${build_capability%???}
[[ "${build_os}" = windows-* ]] && python3 -m pip install ninja
for NO_CUBLASLT in ON OFF; do
if [ "${build_os:0:6}" == ubuntu ]; then
image=nvidia/cuda:${cuda_version}-devel-ubuntu22.04
echo "Using image $image"
docker run --platform "linux/$build_arch" -i -w /src -v "$PWD:/src" "$image" sh -c \
"apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends cmake \
&& cmake -DCOMPUTE_BACKEND=cuda -DCOMPUTE_CAPABILITY=\"${build_capability}\" -DNO_CUBLASLT=${NO_CUBLASLT} . \
&& cmake --build ."
else
pip install cmake==3.28.3
cmake -G Ninja -DCOMPUTE_BACKEND=cuda -DCOMPUTE_CAPABILITY="${build_capability}" -DNO_CUBLASLT=${NO_CUBLASLT} -DCMAKE_BUILD_TYPE=Release -S .
cmake --build . --config Release
fi
done
output_dir="output/${build_os}/${build_arch}"
mkdir -p "${output_dir}"
(shopt -s nullglob && cp bitsandbytes/*.{so,dylib,dll} "${output_dir}")
.github/workflows/python-package.yml
View file @ 62485a34
...@@ -37,37 +37,15 @@ jobs: ...@@ -37,37 +37,15 @@ jobs:
arch: aarch64 arch: aarch64
runs-on: ${{ matrix.os }} # One day, we could run them on native agents. Azure supports this now but it's planned only for Q3 2023 for hosted agents runs-on: ${{ matrix.os }} # One day, we could run them on native agents. Azure supports this now but it's planned only for Q3 2023 for hosted agents
steps: steps:
# Check out code
- uses: actions/checkout@v4 - uses: actions/checkout@v4
# On Linux we use CMake within Docker
- name: Setup cmake
uses: jwlawson/actions-setup-cmake@v1.14
with:
cmake-version: "3.26.x"
- name: Setup MSVC - name: Setup MSVC
if: startsWith(matrix.os, 'windows') if: startsWith(matrix.os, 'windows')
#uses: microsoft/setup-msbuild@v1.1 # to use msbuild
uses: ilammy/msvc-dev-cmd@v1.13.0 # to use cl uses: ilammy/msvc-dev-cmd@v1.13.0 # to use cl
# Compile C++ code
- name: Build C++ - name: Build C++
shell: bash run: bash .github/scripts/build-cpu.sh
run: | env:
set -ex build_os: ${{ matrix.os }}
build_os=${{ matrix.os }} build_arch: ${{ matrix.arch }}
build_arch=${{ matrix.arch }}
if [ ${build_os:0:6} == ubuntu -a ${build_arch} == aarch64 ]; then
# Allow cross-compile on aarch64
sudo apt-get update
sudo apt-get install -y gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu g++-aarch64-linux-gnu
cmake -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++ -DCOMPUTE_BACKEND=cpu .
elif [ ${build_os:0:5} == macos -a ${build_arch} == aarch64 ]; then
cmake -DCMAKE_OSX_ARCHITECTURES=arm64 -DCOMPUTE_BACKEND=cpu .
else
cmake -DCOMPUTE_BACKEND=cpu .
fi
cmake --build . --config Release
mkdir -p output/${{ matrix.os }}/${{ matrix.arch }}
( shopt -s nullglob && cp bitsandbytes/*.{so,dylib,dll} output/${{ matrix.os }}/${{ matrix.arch }}/ )
- name: Upload build artifact - name: Upload build artifact
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
...@@ -91,18 +69,11 @@ jobs: ...@@ -91,18 +69,11 @@ jobs:
arch: aarch64 arch: aarch64
runs-on: ${{ matrix.os }} # One day, we could run them on native agents. Azure supports this now but it's planned only for Q3 2023 for hosted agents runs-on: ${{ matrix.os }} # One day, we could run them on native agents. Azure supports this now but it's planned only for Q3 2023 for hosted agents
steps: steps:
# Check out code
- uses: actions/checkout@v4 - uses: actions/checkout@v4
# Linux: We use Docker to build cross platform Cuda (aarch64 is built in emulation) # Linux: We use Docker to build cross platform Cuda (aarch64 is built in emulation)
- name: Set up Docker multiarch - name: Set up Docker multiarch
if: startsWith(matrix.os, 'ubuntu') if: startsWith(matrix.os, 'ubuntu')
uses: docker/setup-qemu-action@v2 uses: docker/setup-qemu-action@v2
# On Linux we use CMake within Docker
- name: Setup cmake
if: ${{ !startsWith(matrix.os, 'linux') }}
uses: jwlawson/actions-setup-cmake@v1.14
with:
cmake-version: "3.26.x"
# Windows: We install Cuda on the agent (slow) # Windows: We install Cuda on the agent (slow)
- uses: Jimver/cuda-toolkit@v0.2.14 - uses: Jimver/cuda-toolkit@v0.2.14
if: startsWith(matrix.os, 'windows') if: startsWith(matrix.os, 'windows')
...@@ -115,35 +86,13 @@ jobs: ...@@ -115,35 +86,13 @@ jobs:
use-github-cache: false use-github-cache: false
- name: Setup MSVC - name: Setup MSVC
if: startsWith(matrix.os, 'windows') if: startsWith(matrix.os, 'windows')
#uses: microsoft/setup-msbuild@v1.1 # to use msbuild
uses: ilammy/msvc-dev-cmd@v1.13.0 # to use cl uses: ilammy/msvc-dev-cmd@v1.13.0 # to use cl
# Compile C++ code
- name: Build C++ - name: Build C++
shell: bash run: bash .github/scripts/build-cuda.sh
run: | env:
set -ex build_os: ${{ matrix.os }}
build_os=${{ matrix.os }} build_arch: ${{ matrix.arch }}
build_arch=${{ matrix.arch }} cuda_version: ${{ matrix.cuda_version }}
build_capability="50;52;60;61;70;75;80;86;89;90"
[[ "${{ matrix.cuda_version }}" == 11.7.* ]] && build_capability=${build_capability%??????}
[[ "${{ matrix.cuda_version }}" == 11.8.* ]] && build_capability=${build_capability%???}
[[ "${{ matrix.os }}" = windows-* ]] && python3 -m pip install ninja
for NO_CUBLASLT in ON OFF; do
if [ ${build_os:0:6} == ubuntu ]; then
image=nvidia/cuda:${{ matrix.cuda_version }}-devel-ubuntu22.04
echo "Using image $image"
docker run --platform linux/$build_arch -i -w /src -v $PWD:/src $image sh -c \
"apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends cmake \
&& cmake -DCOMPUTE_BACKEND=cuda -DCOMPUTE_CAPABILITY=\"${build_capability}\" -DNO_CUBLASLT=${NO_CUBLASLT} . \
&& cmake --build ."
else
cmake -G Ninja -DCOMPUTE_BACKEND=cuda -DCOMPUTE_CAPABILITY="${build_capability}" -DNO_CUBLASLT=${NO_CUBLASLT} -DCMAKE_BUILD_TYPE=Release -S .
cmake --build . --config Release
fi
done
mkdir -p output/${{ matrix.os }}/${{ matrix.arch }}
( shopt -s nullglob && cp bitsandbytes/*.{so,dylib,dll} output/${{ matrix.os }}/${{ matrix.arch }}/ )
- name: Upload build artifact - name: Upload build artifact
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
...@@ -167,9 +116,7 @@ jobs: ...@@ -167,9 +116,7 @@ jobs:
arch: aarch64 arch: aarch64
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
# Check out code
- uses: actions/checkout@v4 - uses: actions/checkout@v4
# Download shared libraries
- name: Download build artifact - name: Download build artifact
uses: actions/download-artifact@v4 uses: actions/download-artifact@v4
with: with:
...@@ -181,30 +128,17 @@ jobs: ...@@ -181,30 +128,17 @@ jobs:
run: | run: |
ls -lR output/ ls -lR output/
cp output/${{ matrix.os }}/${{ matrix.arch }}/* bitsandbytes/ cp output/${{ matrix.os }}/${{ matrix.arch }}/* bitsandbytes/
# Set up the Python version needed
- name: Set up Python ${{ matrix.python-version }} - name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: ${{ matrix.python-version }} python-version: ${{ matrix.python-version }}
cache: pip cache: pip
- name: Install build package - run: pip install build wheel
shell: bash - run: python -m build .
run: pip install build
- name: Install Python test dependencies
shell: bash
run: pip install -r requirements-ci.txt
# TODO: How to run CUDA tests on GitHub actions?
#- name: Run unit tests
# if: ${{ matrix.arch == 'x86_64' }} # Tests are too slow to run in emulation. Wait for real aarch64 agents
# run: |
# PYTHONPATH=. pytest --log-cli-level=DEBUG tests
- name: Build wheel
shell: bash
run: python -m build .
- name: Determine and Set Platform Tag, then Tag Wheel - name: Determine and Set Platform Tag, then Tag Wheel
shell: bash shell: bash
run: | run: |
PLATFORM_TAG=$(python scripts/set_platform_tag.py ${{ matrix.arch }}) PLATFORM_TAG=$(python .github/scripts/set_platform_tag.py "${{ matrix.arch }}")
echo "PLATFORM_TAG=$PLATFORM_TAG" echo "PLATFORM_TAG=$PLATFORM_TAG"
wheel tags --remove --abi-tag=none --python-tag=py3 --platform-tag=$PLATFORM_TAG dist/bitsandbytes-*.whl wheel tags --remove --abi-tag=none --python-tag=py3 --platform-tag=$PLATFORM_TAG dist/bitsandbytes-*.whl
- name: Upload build artifact - name: Upload build artifact
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment