[python] Allow to build Python wheel package (#3594)

Co-authored-by: Мартынов Максим Сергеевич <msmarty5@mts.ru>

.ci/test.sh

@@ -126,8 +126,14 @@ if [[ $TASK == "gpu" ]]; then
         pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu --install-option="--opencl-include-dir=$AMDAPPSDK_PATH/include/" || exit -1
         pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
         exit 0
+    elif [[ $METHOD == "wheel" ]]; then
+        cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --gpu --opencl-include-dir="$AMDAPPSDK_PATH/include/" || exit -1
+        pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER*.whl -v || exit -1
+        pytest $BUILD_DIRECTORY/tests || exit -1
+        exit 0
+    elif [[ $METHOD == "source" ]]; then
+        cmake -DUSE_GPU=ON -DOpenCL_INCLUDE_DIR=$AMDAPPSDK_PATH/include/ ..
     fi
-    cmake -DUSE_GPU=ON -DOpenCL_INCLUDE_DIR=$AMDAPPSDK_PATH/include/ ..
 elif [[ $TASK == "cuda" ]]; then
     sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "cuda";/' $BUILD_DIRECTORY/include/LightGBM/config.h
     grep -q 'std::string device_type = "cuda"' $BUILD_DIRECTORY/include/LightGBM/config.h || exit -1  # make sure that changes were really done
@@ -136,16 +142,28 @@ elif [[ $TASK == "cuda" ]]; then
         pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--cuda || exit -1
         pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
         exit 0
+    elif [[ $METHOD == "wheel" ]]; then
+        cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --cuda || exit -1
+        pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER*.whl -v || exit -1
+        pytest $BUILD_DIRECTORY/tests || exit -1
+        exit 0
+    elif [[ $METHOD == "source" ]]; then
+        cmake -DUSE_CUDA=ON ..
     fi
-    cmake -DUSE_CUDA=ON ..
 elif [[ $TASK == "mpi" ]]; then
     if [[ $METHOD == "pip" ]]; then
         cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
         pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--mpi || exit -1
         pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
         exit 0
+    elif [[ $METHOD == "wheel" ]]; then
+        cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --mpi || exit -1
+        pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER*.whl -v || exit -1
+        pytest $BUILD_DIRECTORY/tests || exit -1
+        exit 0
+    elif [[ $METHOD == "source" ]]; then
+        cmake -DUSE_MPI=ON -DUSE_DEBUG=ON ..
     fi
-    cmake -DUSE_MPI=ON -DUSE_DEBUG=ON ..
 else
     cmake ..
 fi

.travis.yml

@@ -24,8 +24,10 @@ env:
     - TASK=check-docs
     - TASK=mpi METHOD=source
     - TASK=mpi METHOD=pip PYTHON_VERSION=3.7
+    - TASK=mpi METHOD=wheel PYTHON_VERSION=3.7
     - TASK=gpu METHOD=source
     - TASK=gpu METHOD=pip PYTHON_VERSION=3.6
+    - TASK=gpu METHOD=wheel PYTHON_VERSION=3.7
 
 matrix:
   exclude:
@@ -33,6 +35,8 @@ matrix:
       env: TASK=gpu METHOD=source
     - os: osx
       env: TASK=gpu METHOD=pip PYTHON_VERSION=3.6
+    - os: osx
+      env: TASK=gpu METHOD=wheel PYTHON_VERSION=3.7
     - os: osx
       env: TASK=lint
     - os: osx

python-package/README.rst

@@ -172,6 +172,12 @@ Run ``python setup.py install --bit32``, if you want to use 32-bit version. All
 
 If you get any errors during installation or due to any other reasons, you may want to build dynamic library from sources by any method you prefer (see `Installation Guide <https://github.com/microsoft/LightGBM/blob/master/docs/Installation-Guide.rst>`__) and then just run ``python setup.py install --precompile``.
 
+
+Build Wheel File
+****************
+
+You can use ``python setup.py bdist_wheel`` instead of ``python setup.py install`` to build wheel file and use it for installation later. This might be useful for systems with restricted or completely without network access.
+
 Troubleshooting
 ---------------
 

python-package/setup.py

@@ -5,7 +5,6 @@ from __future__ import absolute_import
 import io
 import logging
 import os
-import shutil
 import struct
 import subprocess
 import sys
@@ -15,8 +14,28 @@ from setuptools import find_packages, setup
 from setuptools.command.install import install
 from setuptools.command.install_lib import install_lib
 from setuptools.command.sdist import sdist
-from distutils.dir_util import copy_tree
+from distutils.dir_util import copy_tree, create_tree, remove_tree
 from distutils.file_util import copy_file
+from wheel.bdist_wheel import bdist_wheel
+
+
+LIGHTGBM_OPTIONS = [
+    ('mingw', 'm', 'Compile with MinGW'),
+    ('integrated-opencl', None, 'Compile integrated OpenCL version'),
+    ('gpu', 'g', 'Compile GPU version'),
+    ('cuda', None, 'Compile CUDA version'),
+    ('mpi', None, 'Compile MPI version'),
+    ('nomp', None, 'Compile version without OpenMP support'),
+    ('hdfs', 'h', 'Compile HDFS version'),
+    ('bit32', None, 'Compile 32-bit version'),
+    ('precompile', 'p', 'Use precompiled library'),
+    ('boost-root=', None, 'Boost preferred installation prefix'),
+    ('boost-dir=', None, 'Directory with Boost package configuration file'),
+    ('boost-include-dir=', None, 'Directory containing Boost headers'),
+    ('boost-librarydir=', None, 'Preferred Boost library directory'),
+    ('opencl-include-dir=', None, 'OpenCL include directory'),
+    ('opencl-library=', None, 'Path to OpenCL library')
+]
 
 
 def find_lib():
@@ -35,7 +54,13 @@ def copy_files(integrated_opencl=False, use_gpu=False):
         src = os.path.join(CURRENT_DIR, os.path.pardir, folder_name)
         if os.path.exists(src):
             dst = os.path.join(CURRENT_DIR, 'compile', folder_name)
-            shutil.rmtree(dst, ignore_errors=True)
+            if os.path.exists(dst):
+                if os.path.isdir:
+                    # see https://github.com/pypa/distutils/pull/21
+                    remove_tree(dst)
+                else:
+                    os.remove(dst)
+            create_tree(src, dst, verbose=0)
             copy_tree(src, dst, verbose=0)
         else:
             raise Exception('Cannot copy {0} folder'.format(src))
@@ -73,7 +98,7 @@ def clear_path(path):
             if os.path.isfile(file_path):
                 os.remove(file_path)
             else:
-                shutil.rmtree(file_path)
+                remove_tree(file_path)
 
 
 def silent_call(cmd, raise_error=False, error_msg=''):
@@ -98,7 +123,7 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_cuda=False, use_mpi=False,
                 nomp=False, bit32=False, integrated_opencl=False):
 
     if os.path.exists(os.path.join(CURRENT_DIR, "build_cpp")):
-        shutil.rmtree(os.path.join(CURRENT_DIR, "build_cpp"))
+        remove_tree(os.path.join(CURRENT_DIR, "build_cpp"))
     os.makedirs(os.path.join(CURRENT_DIR, "build_cpp"))
     os.chdir(os.path.join(CURRENT_DIR, "build_cpp"))
 
@@ -194,23 +219,7 @@ class CustomInstallLib(install_lib):
 
 class CustomInstall(install):
 
-    user_options = install.user_options + [
-        ('mingw', 'm', 'Compile with MinGW'),
-        ('integrated-opencl', None, 'Compile integrated OpenCL version'),
-        ('gpu', 'g', 'Compile GPU version'),
-        ('cuda', None, 'Compile CUDA version'),
-        ('mpi', None, 'Compile MPI version'),
-        ('nomp', None, 'Compile version without OpenMP support'),
-        ('hdfs', 'h', 'Compile HDFS version'),
-        ('bit32', None, 'Compile 32-bit version'),
-        ('precompile', 'p', 'Use precompiled library'),
-        ('boost-root=', None, 'Boost preferred installation prefix'),
-        ('boost-dir=', None, 'Directory with Boost package configuration file'),
-        ('boost-include-dir=', None, 'Directory containing Boost headers'),
-        ('boost-librarydir=', None, 'Preferred Boost library directory'),
-        ('opencl-include-dir=', None, 'OpenCL include directory'),
-        ('opencl-library=', None, 'Path to OpenCL library')
-    ]
+    user_options = install.user_options + LIGHTGBM_OPTIONS
 
     def initialize_options(self):
         install.initialize_options(self)
@@ -251,15 +260,59 @@ class CustomInstall(install):
             os.remove(LOG_PATH)
 
 
+class CustomBdistWheel(bdist_wheel):
+
+    user_options = bdist_wheel.user_options + LIGHTGBM_OPTIONS
+
+    def initialize_options(self):
+        bdist_wheel.initialize_options(self)
+        self.mingw = 0
+        self.integrated_opencl = 0
+        self.gpu = 0
+        self.cuda = 0
+        self.boost_root = None
+        self.boost_dir = None
+        self.boost_include_dir = None
+        self.boost_librarydir = None
+        self.opencl_include_dir = None
+        self.opencl_library = None
+        self.mpi = 0
+        self.hdfs = 0
+        self.precompile = 0
+        self.nomp = 0
+        self.bit32 = 0
+
+    def finalize_options(self):
+        bdist_wheel.finalize_options(self)
+
+        install = self.reinitialize_command('install')
+
+        install.mingw = self.mingw
+        install.integrated_opencl = self.integrated_opencl
+        install.gpu = self.gpu
+        install.cuda = self.cuda
+        install.boost_root = self.boost_root
+        install.boost_dir = self.boost_dir
+        install.boost_include_dir = self.boost_include_dir
+        install.boost_librarydir = self.boost_librarydir
+        install.opencl_include_dir = self.opencl_include_dir
+        install.opencl_library = self.opencl_library
+        install.mpi = self.mpi
+        install.hdfs = self.hdfs
+        install.precompile = self.precompile
+        install.nomp = self.nomp
+        install.bit32 = self.bit32
+
+
 class CustomSdist(sdist):
 
     def run(self):
         copy_files(integrated_opencl=True, use_gpu=True)
         open(os.path.join(CURRENT_DIR, '_IS_SOURCE_PACKAGE.txt'), 'w').close()
         if os.path.exists(os.path.join(CURRENT_DIR, 'lightgbm', 'Release')):
-            shutil.rmtree(os.path.join(CURRENT_DIR, 'lightgbm', 'Release'))
+            remove_tree(os.path.join(CURRENT_DIR, 'lightgbm', 'Release'))
         if os.path.exists(os.path.join(CURRENT_DIR, 'lightgbm', 'windows', 'x64')):
-            shutil.rmtree(os.path.join(CURRENT_DIR, 'lightgbm', 'windows', 'x64'))
+            remove_tree(os.path.join(CURRENT_DIR, 'lightgbm', 'windows', 'x64'))
         if os.path.isfile(os.path.join(CURRENT_DIR, 'lightgbm', 'lib_lightgbm.so')):
             os.remove(os.path.join(CURRENT_DIR, 'lightgbm', 'lib_lightgbm.so'))
         sdist.run(self)
@@ -288,6 +341,7 @@ if __name__ == "__main__":
           description='LightGBM Python Package',
           long_description=readme,
           install_requires=[
+              'wheel',
               'numpy',
               'scipy',
               'scikit-learn!=0.22.0'
@@ -298,6 +352,7 @@ if __name__ == "__main__":
           cmdclass={
               'install': CustomInstall,
               'install_lib': CustomInstallLib,
+              'bdist_wheel': CustomBdistWheel,
               'sdist': CustomSdist,
           },
           packages=find_packages(),