[ci] [python-package] replace 'python setup.py' with a shell script (#5837)

.appveyor.yml

@@ -23,7 +23,6 @@ clone_depth: 5
 
 install:
   - git submodule update --init --recursive  # get `external_libs` folder
-  - set PATH=%PATH:C:\Program Files\Git\usr\bin;=%  # delete sh.exe from PATH (mingw32-make fix)
   - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH%
   - set PYTHON_VERSION=%CONFIGURATION%
   - set CONDA_ENV="test-env"

.ci/test.sh

@@ -146,19 +146,21 @@ if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "clang" ]]; then
 fi
 
 if [[ $TASK == "sdist" ]]; then
-    cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
-    sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1
-    pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v || exit -1
+    cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit -1
+    sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
+    pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz -v || exit -1
     if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
-        cp $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz $BUILD_ARTIFACTSTAGINGDIRECTORY
+        cp $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz $BUILD_ARTIFACTSTAGINGDIRECTORY
     fi
     pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
     exit 0
 elif [[ $TASK == "bdist" ]]; then
     if [[ $OS_NAME == "macos" ]]; then
-        cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --plat-name=macosx --python-tag py3 || exit -1
-        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1
-        mv dist/lightgbm-$LGB_VER-py3-none-macosx.whl dist/lightgbm-$LGB_VER-py3-none-macosx_10_15_x86_64.macosx_11_6_x86_64.macosx_12_5_x86_64.whl
+        cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel || exit -1
+        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
+        mv \
+            dist/lightgbm-$LGB_VER-py3-none-macosx*.whl \
+            dist/lightgbm-$LGB_VER-py3-none-macosx_10_15_x86_64.macosx_11_6_x86_64.macosx_12_5_x86_64.whl
         if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
             cp dist/lightgbm-$LGB_VER-py3-none-macosx*.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
         fi
@@ -169,21 +171,22 @@ elif [[ $TASK == "bdist" ]]; then
         else
             PLATFORM="manylinux2014_$ARCH"
         fi
-        cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --integrated-opencl --plat-name=$PLATFORM --python-tag py3 || exit -1
-        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1
+        cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --integrated-opencl || exit -1
+        mv \
+            ./dist/*.whl \
+            ./dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl
+        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
         if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
             cp dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
         fi
         # Make sure we can do both CPU and GPU; see tests/python_package_test/test_dual.py
         export LIGHTGBM_TEST_DUAL_CPU_GPU=1
     fi
-    pip install --user $BUILD_DIRECTORY/python-package/dist/*.whl || exit -1
+    pip install --user $BUILD_DIRECTORY/dist/*.whl || exit -1
     pytest $BUILD_DIRECTORY/tests || exit -1
     exit 0
 fi
 
-mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build
-
 # temporarily pin pip to versions that support 'pip install --install-option'
 # ref: https://github.com/microsoft/LightGBM/issues/5061#issuecomment-1510642287
 if [[ $METHOD == "pip" ]]; then
@@ -194,18 +197,20 @@ if [[ $TASK == "gpu" ]]; then
     sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "gpu";/' $BUILD_DIRECTORY/include/LightGBM/config.h
     grep -q 'std::string device_type = "gpu"' $BUILD_DIRECTORY/include/LightGBM/config.h || exit -1  # make sure that changes were really done
     if [[ $METHOD == "pip" ]]; then
-        cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
-        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1
-        pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu || exit -1
+        cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit -1
+        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
+        pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu || exit -1
         pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
         exit 0
     elif [[ $METHOD == "wheel" ]]; then
-        cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --gpu || exit -1
-        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1
-        pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER*.whl -v || exit -1
+        cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --gpu || exit -1
+        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
+        pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER*.whl -v || exit -1
         pytest $BUILD_DIRECTORY/tests || exit -1
         exit 0
     elif [[ $METHOD == "source" ]]; then
+        mkdir $BUILD_DIRECTORY/build
+        cd $BUILD_DIRECTORY/build
         cmake -DUSE_GPU=ON ..
     fi
 elif [[ $TASK == "cuda" ]]; then
@@ -215,43 +220,49 @@ elif [[ $TASK == "cuda" ]]; then
     sed -i'.bak' 's/gpu_use_dp = false;/gpu_use_dp = true;/' $BUILD_DIRECTORY/include/LightGBM/config.h
     grep -q 'gpu_use_dp = true' $BUILD_DIRECTORY/include/LightGBM/config.h || exit -1  # make sure that changes were really done
     if [[ $METHOD == "pip" ]]; then
-        cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
-        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1
-        pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--cuda || exit -1
+        cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit -1
+        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
+        pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--cuda || exit -1
         pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
         exit 0
     elif [[ $METHOD == "wheel" ]]; then
-        cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --cuda || exit -1
-        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1
-        pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER*.whl -v || exit -1
+        cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --cuda || exit -1
+        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
+        pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER*.whl -v || exit -1
         pytest $BUILD_DIRECTORY/tests || exit -1
         exit 0
     elif [[ $METHOD == "source" ]]; then
+        mkdir $BUILD_DIRECTORY/build
+        cd $BUILD_DIRECTORY/build
         cmake -DUSE_CUDA=ON ..
     fi
 elif [[ $TASK == "mpi" ]]; then
     if [[ $METHOD == "pip" ]]; then
-        cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
-        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1
-        pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--mpi || exit -1
+        cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit -1
+        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
+        pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--mpi || exit -1
         pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
         exit 0
     elif [[ $METHOD == "wheel" ]]; then
-        cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --mpi || exit -1
-        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1
-        pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER*.whl -v || exit -1
+        cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --mpi || exit -1
+        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
+        pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER*.whl -v || exit -1
         pytest $BUILD_DIRECTORY/tests || exit -1
         exit 0
     elif [[ $METHOD == "source" ]]; then
+        mkdir $BUILD_DIRECTORY/build
+        cd $BUILD_DIRECTORY/build
         cmake -DUSE_MPI=ON -DUSE_DEBUG=ON ..
     fi
 else
+    mkdir $BUILD_DIRECTORY/build
+    cd $BUILD_DIRECTORY/build
     cmake ..
 fi
 
 make _lightgbm -j4 || exit -1
 
-cd $BUILD_DIRECTORY/python-package && python setup.py install --precompile --user || exit -1
+cd $BUILD_DIRECTORY && sh ./build-python.sh install --precompile --user || exit -1
 pytest $BUILD_DIRECTORY/tests || exit -1
 
 if [[ $TASK == "regular" ]]; then

.ci/test_windows.ps1

@@ -65,15 +65,15 @@ if ($env:TASK -ne "bdist") {
 if ($env:TASK -eq "regular") {
   mkdir $env:BUILD_SOURCESDIRECTORY/build; cd $env:BUILD_SOURCESDIRECTORY/build
   cmake -A x64 .. ; cmake --build . --target ALL_BUILD --config Release ; Check-Output $?
-  cd $env:BUILD_SOURCESDIRECTORY/python-package
-  python setup.py install --precompile ; Check-Output $?
+  cd $env:BUILD_SOURCESDIRECTORY
+  sh $env:BUILD_SOURCESDIRECTORY/build-python.sh install --precompile ; Check-Output $?
   cp $env:BUILD_SOURCESDIRECTORY/Release/lib_lightgbm.dll $env:BUILD_ARTIFACTSTAGINGDIRECTORY
   cp $env:BUILD_SOURCESDIRECTORY/Release/lightgbm.exe $env:BUILD_ARTIFACTSTAGINGDIRECTORY
 }
 elseif ($env:TASK -eq "sdist") {
-  cd $env:BUILD_SOURCESDIRECTORY/python-package
-  python setup.py sdist --formats gztar ; Check-Output $?
-  sh $env:BUILD_SOURCESDIRECTORY/.ci/check_python_dists.sh $env:BUILD_SOURCESDIRECTORY/python-package/dist ; Check-Output $?
+  cd $env:BUILD_SOURCESDIRECTORY
+  sh $env:BUILD_SOURCESDIRECTORY/build-python.sh sdist ; Check-Output $?
+  sh $env:BUILD_SOURCESDIRECTORY/.ci/check_python_dists.sh $env:BUILD_SOURCESDIRECTORY/dist ; Check-Output $?
   cd dist; pip install @(Get-ChildItem *.gz) -v ; Check-Output $?
 }
 elseif ($env:TASK -eq "bdist") {
@@ -87,17 +87,17 @@ elseif ($env:TASK -eq "bdist") {
   Get-ItemProperty -Path Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Khronos\OpenCL\Vendors
 
   conda activate $env:CONDA_ENV
-  cd $env:BUILD_SOURCESDIRECTORY/python-package
-  python setup.py bdist_wheel --integrated-opencl --plat-name=win-amd64 --python-tag py3 ; Check-Output $?
-  sh $env:BUILD_SOURCESDIRECTORY/.ci/check_python_dists.sh $env:BUILD_SOURCESDIRECTORY/python-package/dist ; Check-Output $?
+  cd $env:BUILD_SOURCESDIRECTORY
+  sh "build-python.sh" bdist_wheel --integrated-opencl ; Check-Output $?
+  sh $env:BUILD_SOURCESDIRECTORY/.ci/check_python_dists.sh $env:BUILD_SOURCESDIRECTORY/dist ; Check-Output $?
   cd dist; pip install --user @(Get-ChildItem *.whl) ; Check-Output $?
   cp @(Get-ChildItem *.whl) $env:BUILD_ARTIFACTSTAGINGDIRECTORY
 } elseif (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python")) {
-  cd $env:BUILD_SOURCESDIRECTORY\python-package
+  cd $env:BUILD_SOURCESDIRECTORY
   if ($env:COMPILER -eq "MINGW") {
-    python setup.py install --mingw ; Check-Output $?
+    sh $env:BUILD_SOURCESDIRECTORY/build-python.sh install --mingw ; Check-Output $?
   } else {
-    python setup.py install ; Check-Output $?
+    sh $env:BUILD_SOURCESDIRECTORY/build-python.sh install ; Check-Output $?
   }
 }
 

.gitignore

@@ -399,6 +399,7 @@ lightgbm.model
 /cmake-build-debug/
 
 # Files from local Python install
+lightgbm-python/
 python-package/LICENSE
 python-package/build_cpp/
 python-package/compile/

build-python.sh

+#!/bin/sh
+
+# [description]
+#
+#     Prepare a source distribution (sdist) or built distribution (wheel)
+#     of the Python package, and optionally install it.
+#
+# [usage]
+#
+#     # build sdist and put it in dist/
+#     sh ./build-python.sh sdist
+#
+#     # build wheel and put it in dist/
+#     sh ./build-python.sh bdist_wheel [OPTIONS]
+#
+#     # compile lib_lightgbm and install the Python package wrapping it
+#     sh ./build-python.sh install [OPTIONS]
+#
+#     # install the Python package using a pre-compiled lib_lightgbm
+#     # (assumes lib_lightgbm.{dll,so} is located at the root of the repo)
+#     sh ./build-python.sh install --precompile
+#
+# [options]
+#
+#     --boost-include-dir=FILEPATH
+#                                   Directory containing Boost headers.
+#     --boost-librarydir=FILEPATH
+#                                   Preferred Boost library directory.
+#     --boost-root=FILEPATH
+#                                   Boost preferred installation prefix.
+#     --opencl-include-dir=FILEPATH
+#                                   OpenCL include directory.
+#     --opencl-library=FILEPATH
+#                                   Path to OpenCL library.
+#     --bit32
+#                                   Compile 32-bit version.
+#     --cuda
+#                                   Compile CUDA version.
+#     --gpu
+#                                   Compile GPU version.
+#     --hdfs
+#                                   Compile HDFS version.
+#     --integrated-opencl
+#                                   Compile integrated OpenCL version.
+#     --mingw
+#                                   Compile with MinGW.
+#     --mpi
+#                                   Compile MPI version.
+#     --nomp
+#                                   Compile version without OpenMP support.
+#     --precompile
+#                                   Use precompiled library.
+#                                   Only used with 'install' command.
+#     --time-costs
+#                                   Output time costs for different internal routines.
+#     --user
+#                                   Install into user-specific instead of global site-packages directory.
+#                                   Only used with 'install' command.
+
+set -e -u
+
+echo "building lightgbm"
+
+# Default values of arguments
+INSTALL="false"
+BUILD_SDIST="false"
+BUILD_WHEEL="false"
+
+PIP_INSTALL_ARGS=""
+BUILD_ARGS=""
+PRECOMPILE="false"
+
+BOOST_INCLUDE_DIR=""
+BOOST_LIBRARY_DIR=""
+BOOST_ROOT=""
+OPENCL_INCLUDE_DIR=""
+OPENCL_LIBRARY=""
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    ############################
+    # sub-commands of setup.py #
+    ############################
+    install)
+      INSTALL="true"
+      ;;
+    sdist)
+      BUILD_SDIST="true"
+      ;;
+    bdist_wheel)
+      BUILD_WHEEL="true"
+      ;;
+    ############################
+    # customized library paths #
+    ############################
+    --boost-include-dir|--boost-include-dir=*)
+        if [[ "$1" != *=* ]];
+            then shift;
+        fi
+        BOOST_INCLUDE_DIR="${1#*=}"
+        BUILD_ARGS="${BUILD_ARGS} --boost-include-dir='${BOOST_INCLUDE_DIR}'"
+        ;;
+    --boost-librarydir|--boost-librarydir=*)
+        if [[ "$1" != *=* ]];
+            then shift;
+        fi
+        BOOST_LIBRARY_DIR="${1#*=}"
+        BUILD_ARGS="${BUILD_ARGS} --boost-librarydir='${BOOST_LIBRARY_DIR}'"
+        ;;
+    --boost-root|--boost-root=*)
+        if [[ "$1" != *=* ]];
+            then shift;
+        fi
+        BOOST_ROOT="${1#*=}"
+        BUILD_ARGS="${BUILD_ARGS} --boost-root='${BOOST_ROOT}'"
+        ;;
+    --opencl-include-dir|--opencl-include-dir=*)
+        if [[ "$1" != *=* ]];
+            then shift;
+        fi
+        OPENCL_INCLUDE_DIR="${1#*=}"
+        BUILD_ARGS="${BUILD_ARGS} --opencl-include-dir='${OPENCL_INCLUDE_DIR}'"
+        ;;
+    --opencl-library|--opencl-library=*)
+        if [[ "$1" != *=* ]];
+            then shift;
+        fi
+        OPENCL_LIBRARY="${1#*=}"
+        BUILD_ARGS="${BUILD_ARGS} --opencl-library='${OPENCL_LIBRARY}'"
+        ;;
+    #########
+    # flags #
+    #########
+    --bit32)
+        BUILD_ARGS="${BUILD_ARGS} --bit32"
+        ;;
+    --cuda)
+        BUILD_ARGS="${BUILD_ARGS} --cuda"
+        ;;
+    --gpu)
+        BUILD_ARGS="${BUILD_ARGS} --gpu"
+        ;;
+    --hdfs)
+        BUILD_ARGS="${BUILD_ARGS} --hdfs"
+        ;;
+    --integrated-opencl)
+        BUILD_ARGS="${BUILD_ARGS} --integrated-opencl"
+        ;;
+    --mingw)
+        BUILD_ARGS="${BUILD_ARGS} --mingw"
+        ;;
+    --mpi)
+        BUILD_ARGS="${BUILD_ARGS} --mpi"
+        ;;
+    --nomp)
+        BUILD_ARGS="${BUILD_ARGS} --nomp"
+        ;;
+    --precompile)
+        PRECOMPILE="true"
+        ;;
+    --time-costs)
+        BUILD_ARGS="${PIP_INSTALL_ARGS} --time-costs"
+        ;;
+    --user)
+        PIP_INSTALL_ARGS="${PIP_INSTALL_ARGS} --user"
+        ;;
+    *)
+        echo "invalid argument '${1}'"
+        exit -1
+        ;;
+  esac
+  shift
+done
+
+# create a new directory that just contains the files needed
+# to build the Python package
+create_isolated_source_dir() {
+    rm -rf \
+        ./lightgbm-python \
+        ./lightgbm \
+        ./python-package/build \
+        ./python-package/build_cpp \
+        ./python-package/compile \
+        ./python-package/dist \
+        ./python-package/lightgbm.egg-info
+
+    cp -R ./python-package ./lightgbm-python
+
+    cp LICENSE ./lightgbm-python/
+    cp VERSION.txt ./lightgbm-python/lightgbm/VERSION.txt
+
+    mkdir -p ./lightgbm-python/compile
+    cp -R ./cmake ./lightgbm-python/compile
+    cp CMakeLists.txt ./lightgbm-python/compile
+    cp -R ./include ./lightgbm-python/compile
+    cp -R ./src ./lightgbm-python/compile
+    cp -R ./swig ./lightgbm-python/compile
+    cp -R ./windows ./lightgbm-python/compile
+
+    # include only specific files from external_libs, to keep the package
+    # small and avoid redistributing code with licenses incompatible with
+    # LightGBM's license
+
+    ######################
+    # fast_double_parser #
+    ######################
+    mkdir -p ./lightgbm-python/compile/external_libs/fast_double_parser
+    cp \
+        external_libs/fast_double_parser/CMakeLists.txt \
+        ./lightgbm-python/compile/external_libs/fast_double_parser/CMakeLists.txt
+    cp \
+        external_libs/fast_double_parser/LICENSE* \
+        ./lightgbm-python/compile/external_libs/fast_double_parser/
+
+    mkdir -p ./lightgbm-python/compile/external_libs/fast_double_parser/include/
+    cp \
+        external_libs/fast_double_parser/include/fast_double_parser.h \
+        ./lightgbm-python/compile/external_libs/fast_double_parser/include/
+
+    #######
+    # fmt #
+    #######
+    mkdir -p ./lightgbm-python/compile/external_libs/fmt
+    cp \
+        external_libs/fast_double_parser/CMakeLists.txt \
+        ./lightgbm-python/compile/external_libs/fmt/CMakeLists.txt
+    cp \
+        external_libs/fmt/LICENSE* \
+        ./lightgbm-python/compile/external_libs/fmt/
+
+    mkdir -p ./lightgbm-python/compile/external_libs/fmt/include/fmt
+    cp \
+        external_libs/fmt/include/fmt/*.h \
+        ./lightgbm-python/compile/external_libs/fmt/include/fmt/
+
+    #########
+    # Eigen #
+    #########
+    mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen
+    cp \
+        external_libs/eigen/CMakeLists.txt \
+        ./lightgbm-python/compile/external_libs/eigen/CMakeLists.txt
+
+    modules="Cholesky Core Dense Eigenvalues Geometry Householder Jacobi LU QR SVD"
+    for eigen_module in ${modules}; do
+        cp \
+            external_libs/eigen/Eigen/${eigen_module} \
+            ./lightgbm-python/compile/external_libs/eigen/Eigen/${eigen_module}
+        if [ ${eigen_module} != "Dense" ]; then
+            mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen/src/${eigen_module}/
+            cp \
+                -R \
+                external_libs/eigen/Eigen/src/${eigen_module}/* \
+                ./lightgbm-python/compile/external_libs/eigen/Eigen/src/${eigen_module}/
+        fi
+    done
+
+    mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen/misc
+    cp \
+        -R \
+        external_libs/eigen/Eigen/src/misc \
+        ./lightgbm-python/compile/external_libs/eigen/Eigen/src/misc/
+
+    mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen/plugins
+    cp \
+        -R \
+        external_libs/eigen/Eigen/src/plugins \
+        ./lightgbm-python/compile/external_libs/eigen/Eigen/src/plugins/
+
+    ###################
+    # compute (Boost) #
+    ###################
+    mkdir -p ./lightgbm-python/compile/external_libs/compute
+    cp \
+        external_libs/compute/CMakeLists.txt \
+        ./lightgbm-python/compile/external_libs/compute/
+    cp \
+        -R \
+        external_libs/compute/cmake \
+        ./lightgbm-python/compile/external_libs/compute/cmake/
+    cp \
+        -R \
+        external_libs/compute/include \
+        ./lightgbm-python/compile/external_libs/compute/include/
+    cp \
+        -R \
+        external_libs/compute/meta \
+        ./lightgbm-python/compile/external_libs/compute/meta/
+}
+
+create_isolated_source_dir
+
+cd ./lightgbm-python
+
+# installation involves building the wheel + `pip install`-ing it
+if test "${INSTALL}" = true; then
+    if test "${PRECOMPILE}" = true; then
+        echo "--- installing lightgbm (from precompiled lib_lightgbm) ---"
+        python setup.py install ${PIP_INSTALL_ARGS} --precompile
+        exit 0
+    else
+        BUILD_SDIST="false"
+        BUILD_WHEEL="true"
+    fi
+fi
+
+if test "${BUILD_SDIST}" = true; then
+    echo "--- building sdist ---"
+    rm -f ../dist/*.tar.gz
+    python ./setup.py sdist \
+        --dist-dir ../dist
+fi
+
+if test "${BUILD_WHEEL}" = true; then
+    echo "--- building wheel ---"#
+    rm -f ../dist/*.whl || true
+    python setup.py bdist_wheel \
+        --dist-dir ../dist \
+        ${BUILD_ARGS}
+fi
+
+if test "${INSTALL}" = true; then
+    echo "--- installing lightgbm ---"
+    # ref for use of '--find-links': https://stackoverflow.com/a/52481267/3986677
+    cd ../dist
+    pip install \
+        ${PIP_INSTALL_ARGS} \
+        --find-links=. \
+        lightgbm
+    cd ../
+fi
+
+echo "cleaning up"
+rm -rf ./lightgbm-python

docker/dockerfile-python

@@ -26,7 +26,7 @@ RUN apt-get update && \
     # lightgbm
     conda install -q -y numpy scipy scikit-learn pandas && \
     git clone --recursive --branch stable --depth 1 https://github.com/Microsoft/LightGBM && \
-    cd LightGBM/python-package && python setup.py install && \
+    sh ./build-python.sh install && \
     # clean
     apt-get autoremove -y && apt-get clean && \
     conda clean -a -y && \

docker/gpu/dockerfile.gpu

@@ -88,7 +88,7 @@ RUN cd /usr/local/src && mkdir lightgbm && cd lightgbm && \
 
 ENV PATH /usr/local/src/lightgbm/LightGBM:${PATH}
 
-RUN /bin/bash -c "source activate py3 && cd /usr/local/src/lightgbm/LightGBM/python-package && python setup.py install --precompile && source deactivate"
+RUN /bin/bash -c "source activate py3 && cd /usr/local/src/lightgbm/LightGBM && sh ./build-python.sh install --precompile && source deactivate"
 
 #################################################################################################################
 #           System CleanUp

docs/FAQ.rst

@@ -277,6 +277,10 @@ Python-package
 1. ``Error: setup script specifies an absolute path`` when installing from GitHub using ``python setup.py install``.
 --------------------------------------------------------------------------------------------------------------------
 
+.. note::
+    As of v4.0.0, ``lightgbm`` does not support directly invoking ``setup.py``.
+    This answer refers only to versions of ``lightgbm`` prior to v4.0.0.
+
 .. code-block:: console
 
    error: Error: setup script specifies an absolute path:

docs/GPU-Tutorial.rst

@@ -80,9 +80,7 @@ If you want to use the Python interface of LightGBM, you can install it now (alo
 
     sudo apt-get -y install python-pip
     sudo -H pip install setuptools numpy scipy scikit-learn -U
-    cd python-package/
-    sudo python setup.py install --precompile
-    cd ..
+    sudo sh ./build-python.sh install --precompile
 
 You need to set an additional parameter ``"device" : "gpu"`` (along with your other options like ``learning_rate``, ``num_leaves``, etc) to use GPU in Python.
 

python-package/README.rst

@@ -193,34 +193,33 @@ For **Windows** users, if you get any errors during installation and there is th
 .. code:: sh
 
     git clone --recursive https://github.com/microsoft/LightGBM.git
-    cd LightGBM/python-package
     # export CXX=g++-7 CC=gcc-7  # macOS users, if you decided to compile with gcc, don't forget to specify compilers (replace "7" with version of gcc installed on your machine)
-    python setup.py install
+    sh ./build-python.sh install
 
 Note: ``sudo`` (or administrator rights in **Windows**) may be needed to perform the command.
 
-Run ``python setup.py install --nomp`` to disable **OpenMP** support. All requirements from `Build Threadless Version section <#build-threadless-version>`__ apply for this installation option as well.
+Run ``sh ./build-python.sh install --nomp`` to disable **OpenMP** support. All requirements from `Build Threadless Version section <#build-threadless-version>`__ apply for this installation option as well.
 
-Run ``python setup.py install --mpi`` to enable **MPI** support. All requirements from `Build MPI Version section <#build-mpi-version>`__ apply for this installation option as well.
+Run ``sh ./build-python.sh install --mpi`` to enable **MPI** support. All requirements from `Build MPI Version section <#build-mpi-version>`__ apply for this installation option as well.
 
-Run ``python setup.py install --mingw``, if you want to use **MinGW-w64** on **Windows** instead of **Visual Studio**. All requirements from `Build with MinGW-w64 on Windows section <#build-with-mingw-w64-on-windows>`__ apply for this installation option as well.
+Run ``sh ./build-python.sh install --mingw``, if you want to use **MinGW-w64** on **Windows** instead of **Visual Studio**. All requirements from `Build with MinGW-w64 on Windows section <#build-with-mingw-w64-on-windows>`__ apply for this installation option as well.
 
-Run ``python setup.py install --gpu`` to enable GPU support. All requirements from `Build GPU Version section <#build-gpu-version>`__ apply for this installation option as well. To pass additional options to **CMake** use the following syntax: ``python setup.py install --gpu --opencl-include-dir=/usr/local/cuda/include/``, see `Build GPU Version section <#build-gpu-version>`__ for the complete list of them.
+Run ``sh ./build-python.sh install --gpu`` to enable GPU support. All requirements from `Build GPU Version section <#build-gpu-version>`__ apply for this installation option as well. To pass additional options to **CMake** use the following syntax: ``sh ./build-python.sh install --gpu --opencl-include-dir="/usr/local/cuda/include/"``, see `Build GPU Version section <#build-gpu-version>`__ for the complete list of them.
 
-Run ``python setup.py install --cuda`` to enable CUDA support. All requirements from `Build CUDA Version section <#build-cuda-version>`__ apply for this installation option as well.
+Run ``sh ./build-python.sh install --cuda`` to enable CUDA support. All requirements from `Build CUDA Version section <#build-cuda-version>`__ apply for this installation option as well.
 
-Run ``python setup.py install --hdfs`` to enable HDFS support. All requirements from `Build HDFS Version section <#build-hdfs-version>`__ apply for this installation option as well.
+Run ``sh ./build-python.sh install --hdfs`` to enable HDFS support. All requirements from `Build HDFS Version section <#build-hdfs-version>`__ apply for this installation option as well.
 
-Run ``python setup.py install --bit32``, if you want to use 32-bit version. All requirements from `Build 32-bit Version with 32-bit Python section <#build-32-bit-version-with-32-bit-python>`__ apply for this installation option as well.
+Run ``sh ./build-python.sh install --bit32``, if you want to use 32-bit version. All requirements from `Build 32-bit Version with 32-bit Python section <#build-32-bit-version-with-32-bit-python>`__ apply for this installation option as well.
 
-Run ``python setup.py install --time-costs``, if you want to output time costs for different internal routines. All requirements from `Build with Time Costs Output section <#build-with-time-costs-output>`__ apply for this installation option as well.
+Run ``sh ./build-python.sh install --time-costs``, if you want to output time costs for different internal routines. All requirements from `Build with Time Costs Output section <#build-with-time-costs-output>`__ apply for this installation option as well.
 
-If you get any errors during installation or due to any other reasons, you may want to build dynamic library from sources by any method you prefer (see `Installation Guide <https://github.com/microsoft/LightGBM/blob/master/docs/Installation-Guide.rst>`__) and then just run ``python setup.py install --precompile``.
+If you get any errors during installation or due to any other reasons, you may want to build dynamic library from sources by any method you prefer (see `Installation Guide <https://github.com/microsoft/LightGBM/blob/master/docs/Installation-Guide.rst>`__) and then just run ``sh ./build-python.sh install --precompile``.
 
 Build Wheel File
 ****************
 
-You can use ``python setup.py bdist_wheel`` instead of ``python setup.py install`` to build wheel file and use it for installation later. This might be useful for systems with restricted or completely without network access.
+You can use ``sh ./build-python.sh install bdist_wheel`` instead of ``sh ./build-python.sh install`` to build wheel file and use it for installation later. This might be useful for systems with restricted or completely without network access.
 
 Install Dask-package
 ''''''''''''''''''''
@@ -235,7 +234,7 @@ To install all additional dependencies required for Dask-package, you can append
 
     pip install lightgbm[dask]
 
-Or replace ``python setup.py install`` with ``pip install -e .[dask]`` if you are installing the package from source files.
+Or replace ``sh ./build-python.sh install`` with ``pip install -e .[dask]`` if you are installing the package from source files.
 
 Troubleshooting
 ---------------

python-package/setup.py

@@ -7,8 +7,8 @@ import sys
 from os import chdir
 from pathlib import Path
 from platform import system
-from shutil import copyfile, copytree, rmtree
-from typing import List, Optional, Union
+from shutil import rmtree
+from typing import List, Optional
 
 from setuptools import find_packages, setup
 from setuptools.command.install import install
@@ -46,41 +46,6 @@ def find_lib() -> List[str]:
     return LIB_PATH
 
 
-def copy_files(integrated_opencl: bool = False, use_gpu: bool = False) -> None:
-
-    def copy_files_helper(folder_name: Union[str, Path]) -> None:
-        src = CURRENT_DIR.parent / folder_name
-        if src.is_dir():
-            dst = CURRENT_DIR / 'compile' / folder_name
-            if dst.is_dir():
-                rmtree(dst)
-            copytree(src, dst)
-        else:
-            raise Exception(f'Cannot copy {src} folder')
-
-    if not IS_SOURCE_FLAG_PATH.is_file():
-        copy_files_helper('include')
-        copy_files_helper('src')
-        for submodule in (CURRENT_DIR.parent / 'external_libs').iterdir():
-            submodule_stem = submodule.stem
-            if submodule_stem == 'compute' and not use_gpu:
-                continue
-            copy_files_helper(Path('external_libs') / submodule_stem)
-        (CURRENT_DIR / "compile" / "windows").mkdir(parents=True, exist_ok=True)
-        copyfile(CURRENT_DIR.parent / "windows" / "LightGBM.sln",
-                 CURRENT_DIR / "compile" / "windows" / "LightGBM.sln")
-        copyfile(CURRENT_DIR.parent / "windows" / "LightGBM.vcxproj",
-                 CURRENT_DIR / "compile" / "windows" / "LightGBM.vcxproj")
-        copyfile(CURRENT_DIR.parent / "LICENSE",
-                 CURRENT_DIR / "LICENSE")
-        copyfile(CURRENT_DIR.parent / "CMakeLists.txt",
-                 CURRENT_DIR / "compile" / "CMakeLists.txt")
-        if integrated_opencl:
-            (CURRENT_DIR / "compile" / "cmake").mkdir(parents=True, exist_ok=True)
-            copyfile(CURRENT_DIR.parent / "cmake" / "IntegratedOpenCL.cmake",
-                     CURRENT_DIR / "compile" / "cmake" / "IntegratedOpenCL.cmake")
-
-
 def clear_path(path: Path) -> None:
     if path.is_dir():
         for file_name in path.iterdir():
@@ -160,7 +125,8 @@ def compile_cpp(
             if use_mpi:
                 raise Exception('MPI version cannot be compiled by MinGW due to the miss of MPI library in it')
             logger.info("Starting to compile with CMake and MinGW.")
-            silent_call(cmake_cmd + ["-G", "MinGW Makefiles"], raise_error=True,
+            # ref: https://stackoverflow.com/a/45104058/3986677
+            silent_call(cmake_cmd + ["-G", "MinGW Makefiles", "-DCMAKE_SH=CMAKE_SH-NOTFOUND"], raise_error=True,
                         error_msg='Please install CMake and all required dependencies first')
             silent_call(["mingw32-make.exe", "_lightgbm", f"-I{build_dir}", "-j4"], raise_error=True,
                         error_msg='Please install MinGW first')
@@ -254,7 +220,6 @@ class CustomInstall(install):
                                 "please use 64-bit Python instead.")
         LOG_PATH.touch()
         if not self.precompile:
-            copy_files(integrated_opencl=self.integrated_opencl, use_gpu=self.gpu)
             compile_cpp(use_mingw=self.mingw, use_gpu=self.gpu, use_cuda=self.cuda, use_mpi=self.mpi,
                         use_hdfs=self.hdfs, boost_root=self.boost_root, boost_dir=self.boost_dir,
                         boost_include_dir=self.boost_include_dir, boost_librarydir=self.boost_librarydir,
@@ -315,7 +280,6 @@ class CustomBdistWheel(bdist_wheel):
 class CustomSdist(sdist):
 
     def run(self) -> None:
-        copy_files(integrated_opencl=True, use_gpu=True)
         IS_SOURCE_FLAG_PATH.touch()
         rmtree(CURRENT_DIR / 'lightgbm' / 'Release', ignore_errors=True)
         rmtree(CURRENT_DIR / 'lightgbm' / 'windows' / 'x64', ignore_errors=True)
@@ -332,11 +296,8 @@ if __name__ == "__main__":
     LOG_PATH = Path.home() / 'LightGBM_compilation.log'
     LOG_NOTICE = f"The full version of error log was saved into {LOG_PATH}"
     IS_SOURCE_FLAG_PATH = CURRENT_DIR / '_IS_SOURCE_PACKAGE.txt'
-    _version_src = CURRENT_DIR.parent / 'VERSION.txt'
-    _version_dst = CURRENT_DIR / 'lightgbm' / 'VERSION.txt'
-    if _version_src.is_file():
-        copyfile(_version_src, _version_dst)
-    version = _version_dst.read_text(encoding='utf-8').strip()
+    _version_file = CURRENT_DIR / 'lightgbm' / 'VERSION.txt'
+    version = _version_file.read_text(encoding='utf-8').strip()
     readme = (CURRENT_DIR / 'README.rst').read_text(encoding='utf-8')
 
     sys.path.insert(0, str(CURRENT_DIR))