[Wheel] Provide a bare docker scripts to help build wheels for manylinux (#105)

* [Build] Improve build configuration and package distribution support

- Add `build` to requirements-build.txt for package building
- Update MANIFEST.in to include Cython wrapper source file
- Enhance setup.py to improve Cython file copying logic
- Update build scripts to support multi-Python version distribution

* [Build] Improve Cython file handling in setup.py and MANIFEST.in

- Remove Cython wrapper from MANIFEST.in
- Enhance setup.py to create target directory if it doesn't exist when copying Cython files
- Improve file copying logic for Cython source files during build process

* [Build] Remove Cython file copying logic in setup.py

- Comment out Cython file copying code in TileLangBuilPydCommand
- Simplify setup.py build process by removing redundant Cython file handling

* [Build] Enhance Docker distribution scripts for multi-Python version support

- Refactor local and PyPI distribution Docker scripts
- Replace hardcoded Python installation with Miniconda-based multi-version Python environment
- Improve Docker image setup with dynamic Python version creation
- Simplify build process by using Miniconda for Python environment management

* [Build] Separate lint requirements into a dedicated file

- Create new requirements-lint.txt for formatting and linting tools
- Update format.sh to use requirements-lint.txt instead of requirements-dev.txt
- Update requirements-dev.txt and requirements-test.txt to reference requirements-lint.txt
- Improve dependency management by isolating lint-specific requirements

* [Build] Restore Cython file copying logic in setup.py

- Re-add Cython file copying mechanism in TileLangBuilPydCommand
- Implement robust file search across multiple potential directories
- Add warning for cases where Cython source files cannot be found
- Improve build process reliability for Cython source files

* [Build] Refactor Cython file copying logic in setup.py

- Simplify Cython file copying mechanism in TileLangBuilPydCommand
- Improve directory creation and file copying for Cython source files
- Relocate potential directories list to a more logical position
- Enhance robustness of file and directory handling during build process

* [Build] Refine Cython file copying logic in setup.py

- Improve file existence check when copying Cython source files
- Use os.path.join to construct full path for more robust file checking
- Enhance file copying mechanism in TileLangBuilPydCommand

format.sh

@@ -35,9 +35,9 @@ tool_version_check() {
     fi
 }
 
-tool_version_check "yapf" $YAPF_VERSION "$(grep yapf requirements-dev.txt | cut -d'=' -f3)"
-tool_version_check "ruff" $RUFF_VERSION "$(grep "ruff==" requirements-dev.txt | cut -d'=' -f3)"
-tool_version_check "codespell" "$CODESPELL_VERSION" "$(grep codespell requirements-dev.txt | cut -d'=' -f3)"
+tool_version_check "yapf" $YAPF_VERSION "$(grep yapf requirements-lint.txt | cut -d'=' -f3)"
+tool_version_check "ruff" $RUFF_VERSION "$(grep "ruff==" requirements-lint.txt | cut -d'=' -f3)"
+tool_version_check "codespell" "$CODESPELL_VERSION" "$(grep codespell requirements-lint.txt | cut -d'=' -f3)"
 
 echo 'tile-lang yapf: Check Start'
 
@@ -196,7 +196,7 @@ echo 'tile-lang clang-format: Check Start'
 # If clang-format is available, run it; otherwise, skip
 if command -v clang-format &>/dev/null; then
     CLANG_FORMAT_VERSION=$(clang-format --version | awk '{print $3}')
-    tool_version_check "clang-format" "$CLANG_FORMAT_VERSION" "$(grep clang-format requirements-dev.txt | cut -d'=' -f3)"
+    tool_version_check "clang-format" "$CLANG_FORMAT_VERSION" "$(grep clang-format requirements-lint.txt | cut -d'=' -f3)"
 
     CLANG_FORMAT_FLAGS=("-i")
 

maint/scripts/build_docs.sh

 #!/bin/bash
 
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
 cd docs
 
 pip install -r requirements.txt
@@ -7,5 +10,3 @@ pip install -r requirements.txt
 make html
 
 cp CNAME _build/html/
-
-

maint/scripts/docker_local_distribute.sh

+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# Get the CUDA version from the command line
+IMAGE=nvidia/cuda:12.1.0-devel-ubuntu18.04
+
+docker pull ${IMAGE}
+
+apt_command="apt update && apt install -y software-properties-common && add-apt-repository -y ppa:deadsnakes/ppa && apt update && apt install -y wget curl libtinfo-dev zlib1g-dev libssl-dev build-essential libedit-dev libxml2-dev"
+
+install_python_env="curl -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && bash Miniconda3-latest-Linux-x86_64.sh -b -p ~/miniconda3 && export PATH=~/miniconda3/bin/:$PATH && conda create -n py38 python=3.8 -y && conda create -n py39 python=3.9 -y && conda create -n py310 python=3.10 -y && conda create -n py311 python=3.11 -y && conda create -n py312 python=3.12 -y && ln -s ~/miniconda3/envs/py38/bin/python3.8 /usr/bin/python3.8 && ln -s ~/miniconda3/envs/py39/bin/python3.9 /usr/bin/python3.9 && ln -s ~/miniconda3/envs/py310/bin/python3.10 /usr/bin/python3.10 && ln -s ~/miniconda3/envs/py311/bin/python3.11 /usr/bin/python3.11 && ln -s ~/miniconda3/envs/py312/bin/python3.12 /usr/bin/python3.12"
+
+install_cmake="wget https://github.com/Kitware/CMake/releases/download/v3.28.4/cmake-3.28.4-linux-x86_64.tar.gz && tar -xvzf cmake-*.tar.gz && rm cmake-*.tar.gz && cd cmake-* && cp bin/* /usr/local/bin/ && mv share/* /usr/local/share/ && mv man/* /usr/local/man/ && hash -r && cd /tilelang && export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH"
+
+install_pip="python3.8 -m pip install  --upgrade pip && python3.8 -m pip install -r requirements-build.txt"
+
+tox_command="python3.8 -m tox -e py38,py39,py310,py311,py312"
+
+docker run --rm -v $(pwd):/tilelang ${IMAGE} /bin/bash -c "$apt_command && $install_python_env && $install_cmake && $install_pip && $tox_command"

maint/scripts/docker_pypi_distribute.sh

+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# Get the CUDA version from the command line
+IMAGE=nvidia/cuda:12.1.0-devel-ubuntu18.04
+
+docker pull ${IMAGE}
+
+apt_command="apt update && apt install -y software-properties-common && add-apt-repository -y ppa:deadsnakes/ppa && apt update && apt install -y wget curl libtinfo-dev zlib1g-dev libssl-dev build-essential libedit-dev libxml2-dev"
+
+install_python_env="curl -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && bash Miniconda3-latest-Linux-x86_64.sh -b -p ~/miniconda3 && export PATH=~/miniconda3/bin/:$PATH && conda create -n py38 python=3.8 -y && conda create -n py39 python=3.9 -y && conda create -n py310 python=3.10 -y && conda create -n py311 python=3.11 -y && conda create -n py312 python=3.12 -y && ln -s ~/miniconda3/envs/py38/bin/python3.8 /usr/bin/python3.8 && ln -s ~/miniconda3/envs/py39/bin/python3.9 /usr/bin/python3.9 && ln -s ~/miniconda3/envs/py310/bin/python3.10 /usr/bin/python3.10 && ln -s ~/miniconda3/envs/py311/bin/python3.11 /usr/bin/python3.11 && ln -s ~/miniconda3/envs/py312/bin/python3.12 /usr/bin/python3.12"
+
+install_cmake="wget https://github.com/Kitware/CMake/releases/download/v3.28.4/cmake-3.28.4-linux-x86_64.tar.gz && tar -xvzf cmake-*.tar.gz && rm cmake-*.tar.gz && cd cmake-* && cp bin/* /usr/local/bin/ && mv share/* /usr/local/share/ && mv man/* /usr/local/man/ && hash -r && cd /tilelang && export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH"
+
+install_pip="python3.8 -m pip install --upgrade pip && python3.8 -m pip install -r requirements-build.txt"
+
+tox_command="python3.8 -m tox -e py38-pypi,py39-pypi,py310-pypi,py311-pypi,py312-pypi"
+
+docker run --rm -v $(pwd):/tilelang ${IMAGE} /bin/bash -c "$apt_command && $install_python_env && $install_cmake && $install_pip && $tox_command"

maint/scripts/local_distribution_tox.sh

@@ -3,7 +3,7 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
-multi_python_version=("3.8","3.9","3.10" "3.11", "3.12")
+multi_python_version=("3.8" "3.9" "3.10" "3.11" "3.12")
 for python_version in "${multi_python_version[@]}"; do
     echo "Installing Python ${python_version}..."
     apt-get install -y python${python_version}

maint/scripts/pypi_distribution_tox.sh

@@ -3,7 +3,7 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
-multi_python_version=("3.8","3.9","3.10" "3.11", "3.12")
+multi_python_version=("3.8" "3.9" "3.10" "3.11" "3.12")
 for python_version in "${multi_python_version[@]}"; do
     echo "Installing Python ${python_version}..."
     apt-get install -y python${python_version}

requirements-build.txt

 # Should be mirrored in pyproject.toml
+build
 cmake>=3.26
 packaging
 setuptools>=61

requirements-dev.txt

-# formatting
-yapf==0.40.2
-toml==0.10.2
-tomli==2.0.1
-ruff==0.6.5
-codespell==2.3.0
-clang-format==15.0.7
-
+# lint requirements
+-r requirements-lint.txt
 # build requirements
 cmake>=3.26
 # runtime requirements

requirements-lint.txt

+# formatting
+yapf==0.40.2
+toml==0.10.2
+tomli==2.0.1
+ruff==0.6.5
+codespell==2.3.0
+clang-format==15.0.7

requirements-test.txt

-# formatting
-yapf==0.40.2
-toml==0.10.2
-tomli==2.0.1
-ruff==0.6.5
-codespell==2.3.0
-clang-format==15.0.7
-
+# lint requirements
+-r requirements-lint.txt
 # build requirements
 cmake>=3.26
 # runtime requirements

setup.py

@@ -227,6 +227,7 @@ class TileLangBuilPydCommand(build_py):
         ext_output_dir = os.path.dirname(extdir)
         print(f"Extension output directory (parent): {ext_output_dir}")
         print(f"Build temp directory: {build_temp_dir}")
+
         # copy cython files
         CYTHON_SRC = [
             "tilelang/jit/adapter/cython/cython_wrapper.pyx",
@@ -241,7 +242,12 @@ class TileLangBuilPydCommand(build_py):
                 target_dir = os.path.dirname(target_dir)
                 if not os.path.exists(target_dir):
                     os.makedirs(target_dir)
+                if not os.path.exists(os.path.join(target_dir, os.path.basename(source_dir))):
+                    # if not exists, copy the file
+                    # as tox will copy the file to the build
+                    # directory based on manifest file
                     shutil.copy2(source_dir, target_dir)
+
         # copy the tl_templates
         TILELANG_SRC = [
             "src/tl_templates",
@@ -258,13 +264,6 @@ class TileLangBuilPydCommand(build_py):
                     os.makedirs(target_dir)
                 shutil.copy2(source_dir, target_dir)
 
-        potential_dirs = [
-            ext_output_dir,
-            self.build_lib,
-            build_temp_dir,
-            os.path.join(ROOT_DIR, "build"),
-        ]
-
         TVM_PREBUILD_ITEMS = [
             "libtvm_runtime.so",
             "libtvm.so",
@@ -272,6 +271,13 @@ class TileLangBuilPydCommand(build_py):
             "libtilelang_module.so",
         ]
 
+        potential_dirs = [
+            ext_output_dir,
+            self.build_lib,
+            build_temp_dir,
+            os.path.join(ROOT_DIR, "build"),
+        ]
+
         for item in TVM_PREBUILD_ITEMS:
             source_lib_file = None
             for dir in potential_dirs: