Skip to content

GitLab

Unverified Commit cb1b9231 authored by Muyang Li's avatar Muyang Li Committed by GitHub
Browse files

chore: add linux nightly build workflows 

* update the manylinux2_28 builder

* add a workflow

* finished the nightly-build wheels
parent 67a20245
Hide whitespace changes
Inline Side-by-side
Showing with 110 additions and 126 deletions
.github/workflows/nightly-build.yaml
View file @ cb1b9231
#name: Nightly Build
#
#on:
# schedule:
# - cron: '0 8 * * *'
# workflow_dispatch:
#
#jobs:
# tag:
# name: Tag Dev Branch if dev version
# runs-on: ubuntu-latest
# outputs:
# is_dev: ${{ steps.check.outputs.is_dev }}
# tag_name: ${{ steps.tag.outputs.tag_name }}
# steps:
# - name: Checkout dev branch
# uses: actions/checkout@v4
# with:
# fetch-depth: 0
# ref: dev
#
# - name: Extract version from __version__.py
# id: version
# run: |
# version=$(grep '__version__' nunchaku/__version__.py | sed -E 's/.*"([^"]+)".*/\1/')
# echo "version=$version" >> "$GITHUB_OUTPUT"
# echo "Extracted version: $version"
#
# - name: Check if version contains 'dev'
# id: check
# run: |
# if [[ "${{ steps.version.outputs.version }}" == *dev* ]]; then
# echo "is_dev=true" >> "$GITHUB_OUTPUT"
# else
# echo "is_dev=false" >> "$GITHUB_OUTPUT"
# fi
#
# - name: Set tag name
# id: tag
# if: steps.check.outputs.is_dev == 'true'
# run: |
# today=$(date -u +"%Y%m%d")
# tag_name="v${{ steps.version.outputs.version }}-$today"
# echo "tag_name=$tag_name" >> "$GITHUB_OUTPUT"
#
# - name: Create and push tag
# if: steps.check.outputs.is_dev == 'true'
# run: |
# git config user.name "github-actions"
# git config user.email "github-actions@users.noreply.github.com"
# git tag ${{ steps.tag.outputs.tag_name }}
# git push origin ${{ steps.tag.outputs.tag_name }}
#
# - name: Skip tagging (version is not dev)
# if: steps.check.outputs.is_dev == 'false'
# run: echo "Version is not a dev version. Skipping tag."
#
# windows-build:
# name: Build the windows nightly wheels
# runs-on: self-hosted, windows-build
# needs: tag
# if: needs.tag.outputs.is_dev == 'true'
# steps:
# - name: Checkout to the tag
# uses: actions/checkout@v4
# with:
# fetch-depth: 0
# ref: ${{ needs.tag.outputs.tag_name }}
# submodules: true
# - name: Show current commit
# run: git log -1 --oneline
# - name: setup Conda
# run: |
# pwd
# - name: Build all wheels
# run: .\scripts\build_all_windows_wheels.cmd
# - name: Upload wheels
# run: |
# echo "Uploading wheels"
# mkdir -p wheels
# cp dist/*.whl wheels/
#
#
name: Nightly Build
on:
schedule:
- cron: '0 8 * * *' # UTC time
workflow_dispatch:
permissions:
contents: write
jobs:
tag:
name: Tag Dev Branch if dev version
runs-on: ubuntu-latest
if: github.repository == 'mit-han-lab/nunchaku'
outputs:
is_dev: ${{ steps.check.outputs.is_dev }}
tag_name: ${{ steps.tag.outputs.tag_name }}
steps:
- name: Checkout dev branch
uses: actions/checkout@v4
with:
fetch-depth: 0
ref: dev
- name: Extract version from __version__.py
id: version
run: |
version=$(grep '__version__' nunchaku/__version__.py | sed -E 's/.*"([^"]+)".*/\1/')
echo "Extracted version: $version"
echo "version=$version" >> "$GITHUB_OUTPUT"
- name: Check if version contains 'dev'
id: check
run: |
if [[ "${{ steps.version.outputs.version }}" == *dev* ]]; then
echo "is_dev=true" >> "$GITHUB_OUTPUT"
else
echo "is_dev=false" >> "$GITHUB_OUTPUT"
fi
- name: Set tag name
id: tag
if: steps.check.outputs.is_dev == 'true'
run: |
today=$(date -u +"%Y%m%d")
tag_name="v${{ steps.version.outputs.version }}-$today"
echo "tag_name=$tag_name"
echo "tag_name=$tag_name" >> "$GITHUB_OUTPUT"
- name: Create and push tag
if: steps.check.outputs.is_dev == 'true'
run: |
git config user.name "github-actions"
git config user.email "github-actions@users.noreply.github.com"
git tag ${{ steps.tag.outputs.tag_name }}
git push origin ${{ steps.tag.outputs.tag_name }}
- name: Skip tagging (version is not dev)
if: steps.check.outputs.is_dev == 'false'
run: echo "Version is not a dev version. Skipping tag."
linux-wheels:
name: Build the linux nightly wheels
runs-on: [self-hosted, linux-build]
needs: tag
if: needs.tag.outputs.is_dev == 'true'
steps:
- name: Checkout to the tag
uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ needs.tag.outputs.tag_name }}
submodules: true
- name: Show current commit
run: git log -1 --oneline
- name: Build wheels
run: |
bash scripts/build_linux_wheel.sh "3.10" "2.7" "12.8"
- name: Upload wheels to GitHub Release
uses: softprops/action-gh-release@v2
with:
files: dist/*.whl
name: Nunchaku Nightly ${{ needs.tag.outputs.tag_name }}
tag_name: ${{ needs.tag.outputs.tag_name }}
prerelease: true
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Clean up
run: bash scripts/linux_cleanup.sh
linux-clean-up:
if: always()
needs: [ linux-wheels ]
runs-on: [ self-hosted, linux-build ]
steps:
- name: Clean up
run: |
cd ..
rm -rf *nunchaku*
scripts/build_all_linux_wheels.sh
View file @ cb1b9231
......@@ -22,6 +22,6 @@ bash scripts/build_linux_wheel.sh "3.10" "2.7" "12.8"
bash scripts/build_linux_wheel.sh "3.11" "2.7" "12.8"
bash scripts/build_linux_wheel.sh "3.12" "2.7" "12.8"
bash scripts/build_linux_wheel_cu128.sh "3.10" "2.8" "12.8"
bash scripts/build_linux_wheel_cu128.sh "3.11" "2.8" "12.8"
bash scripts/build_linux_wheel_cu128.sh "3.12" "2.8" "12.8"
#bash scripts/build_linux_wheel_cu128.sh "3.10" "2.8" "12.8"
#bash scripts/build_linux_wheel_cu128.sh "3.11" "2.8" "12.8"
#bash scripts/build_linux_wheel_cu128.sh "3.12" "2.8" "12.8"
scripts/build_linux_wheel.sh
View file @ cb1b9231
......@@ -26,12 +26,10 @@ fi
docker run --rm \
-v "$(pwd)":/nunchaku \
pytorch/manylinux-builder:cuda${CUDA_VERSION} \
pytorch/manylinux2_28-builder:cuda${CUDA_VERSION} \
bash -c "
cd /nunchaku && \
rm -rf build && \
yum install -y devtoolset-11 && \
source scl_source enable devtoolset-11 && \
gcc --version && g++ --version && \
${PYTHON_ROOT_PATH}/bin/pip install --no-cache-dir torch==${TORCH_VERSION} torchvision==${TORCHVISION_VERSION} torchaudio==${TORCHAUDIO_VERSION} --index-url https://download.pytorch.org/whl/cu${CUDA_VERSION//.} && \
${PYTHON_ROOT_PATH}/bin/pip install build ninja wheel setuptools && \
......
scripts/build_linux_wheel_cu128.sh deleted 100644 → 0
View file @ 67a20245
#!/bin/bash
# Modified from https://github.com/sgl-project/sglang/blob/main/sgl-kernel/build.sh
set -ex
PYTHON_VERSION=$1
TORCH_VERSION=$2 # has no use for now
CUDA_VERSION=$3
MAX_JOBS=${4:-} # optional
PYTHON_ROOT_PATH=/opt/python/cp${PYTHON_VERSION//.}-cp${PYTHON_VERSION//.}
# Check if TORCH_VERSION is 2.5 or 2.6 and set the corresponding versions for TORCHVISION and TORCHAUDIO
#if [ "$TORCH_VERSION" == "2.5" ]; then
# TORCHVISION_VERSION="0.20"
# TORCHAUDIO_VERSION="2.5"
# echo "TORCH_VERSION is 2.5, setting TORCHVISION_VERSION to $TORCHVISION_VERSION and TORCHAUDIO_VERSION to $TORCHAUDIO_VERSION"
#elif [ "$TORCH_VERSION" == "2.6" ]; then
# TORCHVISION_VERSION="0.21"
# TORCHAUDIO_VERSION="2.6"
# echo "TORCH_VERSION is 2.6, setting TORCHVISION_VERSION to $TORCHVISION_VERSION and TORCHAUDIO_VERSION to $TORCHAUDIO_VERSION"
#else
# echo "TORCH_VERSION is not 2.5 or 2.6, no changes to versions."
#fi
docker run --rm \
-v "$(pwd)":/nunchaku \
pytorch/manylinux2_28-builder:cuda${CUDA_VERSION} \
bash -c "
cd /nunchaku && \
rm -rf build && \
gcc --version && g++ --version && \
${PYTHON_ROOT_PATH}/bin/pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu128 && \
${PYTHON_ROOT_PATH}/bin/pip install build ninja wheel setuptools && \
export NUNCHAKU_INSTALL_MODE=ALL && \
export NUNCHAKU_BUILD_WHEELS=1 && \
export MAX_JOBS=${MAX_JOBS} && \
${PYTHON_ROOT_PATH}/bin/python -m build --wheel --no-isolation
"
scripts/linux_cleanup.sh
View file @ cb1b9231
......@@ -3,5 +3,5 @@ set -ex
docker run --rm \
-v "$(pwd)":/nunchaku \
pytorch/manylinux-builder:cuda12.4 \
pytorch/manylinux2_28-builder:cuda12.4 \
bash -c "cd /nunchaku && rm -rf *"
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment