Skip to content

GitLab

Unverified Commit 84a9d0ea authored by Simo Lin's avatar Simo Lin Committed by GitHub
Browse files

[router] support arm, windows, mac, linux, reduce wheel size and number (#12285)

parent 737b58d6
Hide whitespace changes
Inline Side-by-side
Showing with 168 additions and 69 deletions
.github/workflows/nightly-release-router.yml
View file @ 84a9d0ea
......@@ -10,14 +10,40 @@ on:
jobs:
build:
name: Build on ${{ matrix.os }} (${{ matrix.target }})
name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }})
runs-on: ${{ matrix.os }}-latest
strategy:
fail-fast: false
matrix:
os: [ubuntu, macos, windows]
target: [x86_64, aarch64]
manylinux: [auto]
include:
- os: ubuntu
platform: linux
- os: windows
ls: dir
target: x86_64
python-architecture: x64
interpreter: 3.8 3.9 3.10 3.11 3.12 3.13
- os: macos
target: aarch64
interpreter: 3.8 3.9 3.10 3.11 3.12 3.13
- os: ubuntu
platform: linux
target: aarch64
# musllinux
- os: ubuntu
platform: linux
target: x86_64
manylinux: musllinux_1_1
- os: ubuntu
platform: linux
target: aarch64
manylinux: musllinux_1_1
exclude:
- os: windows
target: aarch64
steps:
- uses: actions/checkout@v4
......@@ -29,56 +55,79 @@ jobs:
mv sglang-repo/sgl-router/* .
rm -rf sglang-repo
ls -alt
shell: bash
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
python-version: "3.13"
architecture: ${{ matrix.python-architecture || 'x64' }}
- name: Modify version for nightly release
run: |
# Get current version from pyproject.toml
CURRENT_VERSION=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])" 2>/dev/null || python -c "import tomli; print(tomli.load(open('pyproject.toml', 'rb'))['project']['version'])")
# Create nightly version with date: e.g., 0.1.9.dev20250112
# Create nightly version with date: e.g., 0.2.1.dev20250128
NIGHTLY_VERSION="${CURRENT_VERSION}.dev$(date +%Y%m%d)"
echo "Nightly version: $NIGHTLY_VERSION"
# Update pyproject.toml with nightly version (temporary, not committed)
sed -i "s/version = \"${CURRENT_VERSION}\"/version = \"${NIGHTLY_VERSION}\"/" pyproject.toml
sed -i.bak "s/version = \"${CURRENT_VERSION}\"/version = \"${NIGHTLY_VERSION}\"/" pyproject.toml
# Verify the change
cat pyproject.toml | grep "^version"
shell: bash
- name: Install build dependencies
run: |
python -m pip install -U pip
python -m pip install build twine auditwheel tomli
- name: Install twine and tomli
run: pip install -U twine tomli
- name: Build package
uses: pypa/cibuildwheel@v2.21.3
env:
CIBW_BUILD: "cp38-manylinux_x86_64 cp39-manylinux_x86_64 cp310-manylinux_x86_64 cp311-manylinux_x86_64 cp312-manylinux_x86_64 cp313-manylinux_x86_64 cp314-manylinux_x86_64"
CIBW_BEFORE_ALL: |
yum update -y && yum install -y openssl-devel wget unzip && \
# Install latest protoc (v32.0) that supports proto3
cd /tmp && \
wget https://github.com/protocolbuffers/protobuf/releases/download/v32.0/protoc-32.0-linux-x86_64.zip && \
unzip protoc-32.0-linux-x86_64.zip -d /usr/local && \
rm protoc-32.0-linux-x86_64.zip && \
# Install Rust
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
CIBW_ENVIRONMENT: "PATH=$HOME/.cargo/bin:$PATH"
- name: Install protoc (macOS)
if: matrix.os == 'macos'
run: brew install protobuf
- name: Install protoc (Windows)
if: matrix.os == 'windows'
run: choco install protoc -y
- name: Build wheels
uses: PyO3/maturin-action@v1
with:
target: ${{ matrix.target }}
manylinux: ${{ matrix.manylinux || 'auto' }}
args: --release --out dist --interpreter ${{ matrix.interpreter || '3.8 3.9 3.10 3.11 3.12 3.13' }}
rust-toolchain: stable
docker-options: -e CI -e CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc -e CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++
before-script-linux: |
# Install build dependencies (perl/make for vendored OpenSSL, protoc for gRPC)
if command -v yum &> /dev/null; then
yum update -y && yum install -y wget unzip gcc gcc-c++ perl-core make
# Install cross-compilation toolchain for aarch64 if needed
if [ "${{ matrix.target }}" = "aarch64" ]; then
yum install -y gcc-aarch64-linux-gnu gcc-c++-aarch64-linux-gnu || true
fi
elif command -v apt-get &> /dev/null; then
apt-get update && apt-get install -y wget unzip gcc g++ perl make
# Install cross-compilation toolchain for aarch64 if needed
if [ "${{ matrix.target }}" = "aarch64" ]; then
apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu || true
fi
fi
(cd /tmp && \
wget https://github.com/protocolbuffers/protobuf/releases/download/v32.0/protoc-32.0-linux-x86_64.zip && \
unzip protoc-32.0-linux-x86_64.zip -d /usr/local && \
rm protoc-32.0-linux-x86_64.zip)
protoc --version
- name: List built packages
run: ls -lh wheelhouse/
run: ${{ matrix.ls || 'ls -lh' }} dist/
- name: Check packages
run: twine check --strict wheelhouse/*
run: twine check --strict dist/*
- uses: actions/upload-artifact@v4
with:
name: packages-${{ matrix.os }}-${{ matrix.target }}
path: wheelhouse/
name: packages-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.manylinux || 'auto' }}
path: dist/
build-sdist:
name: Build SDist
......@@ -98,28 +147,28 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
python-version: "3.13"
- name: Modify version for nightly release
run: |
# Get current version from pyproject.toml
CURRENT_VERSION=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])" 2>/dev/null || python -c "import tomli; print(tomli.load(open('pyproject.toml', 'rb'))['project']['version'])")
# Create nightly version with date: e.g., 0.1.9.dev20250112
# Create nightly version with date: e.g., 0.2.1.dev20250128
NIGHTLY_VERSION="${CURRENT_VERSION}.dev$(date +%Y%m%d)"
echo "Nightly version: $NIGHTLY_VERSION"
# Update pyproject.toml with nightly version (temporary, not committed)
sed -i "s/version = \"${CURRENT_VERSION}\"/version = \"${CURRENT_VERSION}\"/g" pyproject.toml
sed -i "0,/version = \"${CURRENT_VERSION}\"/s//version = \"${NIGHTLY_VERSION}\"/" pyproject.toml
sed -i "s/version = \"${CURRENT_VERSION}\"/version = \"${NIGHTLY_VERSION}\"/" pyproject.toml
# Verify the change
cat pyproject.toml | grep "^version"
- name: Build SDist
run: |
pip install build
python -m pip install -U packaging tomli
python -m build --sdist
uses: PyO3/maturin-action@v1
with:
command: sdist
args: --out dist
rust-toolchain: stable
- uses: actions/upload-artifact@v4
with:
......
.github/workflows/release-pypi-router.yml
View file @ 84a9d0ea
# Reference: https://github.com/openai/tiktoken/blob/63527649963def8c759b0f91f2eb69a40934e468/.github/workflows/build_wheels.yml#L1
name: Release SGLang Router to PyPI
name: Release SGLang Model Gateway to PyPI
on:
push:
......@@ -12,14 +10,40 @@ on:
jobs:
build:
name: Build on ${{ matrix.os }} (${{ matrix.target }})
name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }})
runs-on: ${{ matrix.os }}-latest
strategy:
fail-fast: false
matrix:
os: [ubuntu, macos, windows]
target: [x86_64, aarch64]
manylinux: [auto]
include:
- os: ubuntu
platform: linux
- os: windows
ls: dir
target: x86_64
python-architecture: x64
interpreter: 3.8 3.9 3.10 3.11 3.12 3.13
- os: macos
target: aarch64
interpreter: 3.8 3.9 3.10 3.11 3.12 3.13
- os: ubuntu
platform: linux
target: aarch64
# musllinux
- os: ubuntu
platform: linux
target: x86_64
manylinux: musllinux_1_1
- os: ubuntu
platform: linux
target: aarch64
manylinux: musllinux_1_1
exclude:
- os: windows
target: aarch64
steps:
- uses: actions/checkout@v4
......@@ -31,42 +55,64 @@ jobs:
mv sglang-repo/sgl-router/* .
rm -rf sglang-repo
ls -alt
shell: bash
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
python-version: "3.13"
architecture: ${{ matrix.python-architecture || 'x64' }}
- name: Install build dependencies
run: |
python -m pip install -U pip
python -m pip install build twine auditwheel
- name: Install twine
run: pip install -U twine
- name: Build package
uses: pypa/cibuildwheel@v2.21.3
env:
CIBW_BUILD: "cp38-manylinux_x86_64 cp39-manylinux_x86_64 cp310-manylinux_x86_64 cp311-manylinux_x86_64 cp312-manylinux_x86_64 cp313-manylinux_x86_64 cp314-manylinux_x86_64"
CIBW_BEFORE_ALL: |
yum update -y && yum install -y openssl-devel wget unzip && \
# Install latest protoc (v32.0) that supports proto3
cd /tmp && \
wget https://github.com/protocolbuffers/protobuf/releases/download/v32.0/protoc-32.0-linux-x86_64.zip && \
unzip protoc-32.0-linux-x86_64.zip -d /usr/local && \
rm protoc-32.0-linux-x86_64.zip && \
# Install Rust
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
CIBW_ENVIRONMENT: "PATH=$HOME/.cargo/bin:$PATH"
- name: Install protoc (macOS)
if: matrix.os == 'macos'
run: brew install protobuf
- name: Install protoc (Windows)
if: matrix.os == 'windows'
run: choco install protoc -y
- name: Build wheels
uses: PyO3/maturin-action@v1
with:
target: ${{ matrix.target }}
manylinux: ${{ matrix.manylinux || 'auto' }}
args: --release --out dist --interpreter ${{ matrix.interpreter || '3.8 3.9 3.10 3.11 3.12 3.13' }}
rust-toolchain: stable
docker-options: -e CI -e CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc -e CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++
before-script-linux: |
# Install build dependencies (perl/make for vendored OpenSSL, protoc for gRPC)
if command -v yum &> /dev/null; then
yum update -y && yum install -y wget unzip gcc gcc-c++ perl-core make
# Install cross-compilation toolchain for aarch64 if needed
if [ "${{ matrix.target }}" = "aarch64" ]; then
yum install -y gcc-aarch64-linux-gnu gcc-c++-aarch64-linux-gnu || true
fi
elif command -v apt-get &> /dev/null; then
apt-get update && apt-get install -y wget unzip gcc g++ perl make
# Install cross-compilation toolchain for aarch64 if needed
if [ "${{ matrix.target }}" = "aarch64" ]; then
apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu || true
fi
fi
(cd /tmp && \
wget https://github.com/protocolbuffers/protobuf/releases/download/v32.0/protoc-32.0-linux-x86_64.zip && \
unzip protoc-32.0-linux-x86_64.zip -d /usr/local && \
rm protoc-32.0-linux-x86_64.zip)
protoc --version
- name: List built packages
run: ls -lh wheelhouse/
run: ${{ matrix.ls || 'ls -lh' }} dist/
- name: Check packages
run: twine check --strict wheelhouse/*
run: twine check --strict dist/*
- uses: actions/upload-artifact@v4
with:
name: packages-${{ matrix.os }}-${{ matrix.target }}
path: wheelhouse/
name: packages-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.manylinux || 'auto' }}
path: dist/
build-sdist:
name: Build SDist
......@@ -86,13 +132,14 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
python-version: "3.13"
- name: Build SDist
run: |
pip install build
python -m pip install -U packaging
python -m build --sdist
uses: PyO3/maturin-action@v1
with:
command: sdist
args: --out dist
rust-toolchain: stable
- uses: actions/upload-artifact@v4
with:
......
sgl-router/Cargo.toml
View file @ 84a9d0ea
......@@ -36,7 +36,7 @@ rand = "0.9.2"
reqwest = { version = "0.12.8", features = ["stream", "blocking", "json", "rustls-tls"], default-features = false }
futures-util = "0.3"
futures = "0.3"
pyo3 = { version = "0.27.1", features = ["extension-module"] }
pyo3 = { version = "0.27.1", features = ["extension-module", "abi3-py38"] }
dashmap = "6.1.0"
blake3 = "1.5"
http = "1.1.0"
......@@ -66,6 +66,7 @@ tiktoken-rs = { version = "0.7.0" }
minijinja = { version = "2.0", features = ["unstable_machinery", "json", "builtins"] }
minijinja-contrib = { version = "2.0", features = ["pycompat"] }
rustls = { version = "0.23", default-features = false, features = ["ring", "std"] }
openssl = { version = "0.10.73", features = ["vendored"] }
hf-hub = { version = "0.4.3", features = ["tokio"] }
rmcp = { version = "0.8.3", features = ["client", "server",
"transport-child-process",
......@@ -119,8 +120,10 @@ harness = false
path = "benches/tool_parser_benchmark.rs"
[profile.release]
lto = "thin"
codegen-units = 1
opt-level = "z" # Optimize for size
lto = "fat" # Full LTO for smaller binaries
codegen-units = 1 # Better optimization, slower compile
strip = true # Strip debug symbols
[profile.dev]
opt-level = 0
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment