Skip to content

GitLab

Unverified Commit 30942a32 authored by James Lamb's avatar James Lamb Committed by GitHub
Browse files

[python-package] [ci] switch to PEP 517 / 518 builds (remove `setup.py`) (fixes #5061) (#5759)

parent a323c758
Hide whitespace changes
Inline Side-by-side
Showing with 224 additions and 491 deletions
.ci/test.sh
View file @ 30942a32
......@@ -159,6 +159,9 @@ elif [[ $TASK == "bdist" ]]; then
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
mv \
./dist/*.whl \
./dist/tmp.whl || exit -1
mv \
./dist/tmp.whl \
dist/lightgbm-$LGB_VER-py3-none-macosx_10_15_x86_64.macosx_11_6_x86_64.macosx_12_5_x86_64.whl || exit -1
if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
cp dist/lightgbm-$LGB_VER-py3-none-macosx*.whl $BUILD_ARTIFACTSTAGINGDIRECTORY || exit -1
......@@ -173,6 +176,9 @@ elif [[ $TASK == "bdist" ]]; then
cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --integrated-opencl || exit -1
mv \
./dist/*.whl \
./dist/tmp.whl || exit -1
mv \
./dist/tmp.whl \
./dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl || exit -1
sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
......@@ -186,12 +192,6 @@ elif [[ $TASK == "bdist" ]]; then
exit 0
fi
# temporarily pin pip to versions that support 'pip install --install-option'
# ref: https://github.com/microsoft/LightGBM/issues/5061#issuecomment-1510642287
if [[ $METHOD == "pip" ]]; then
pip install 'pip<23.1'
fi
if [[ $TASK == "gpu" ]]; then
sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "gpu";/' $BUILD_DIRECTORY/include/LightGBM/config.h
grep -q 'std::string device_type = "gpu"' $BUILD_DIRECTORY/include/LightGBM/config.h || exit -1 # make sure that changes were really done
......@@ -201,7 +201,7 @@ if [[ $TASK == "gpu" ]]; then
pip install \
--user \
-v \
--install-option=--gpu \
--config-settings=cmake.define.USE_GPU=ON \
$BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz \
|| exit -1
pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
......@@ -229,7 +229,7 @@ elif [[ $TASK == "cuda" ]]; then
pip install \
--user \
-v \
--install-option=--cuda \
--config-settings=cmake.define.USE_CUDA=ON \
$BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz \
|| exit -1
pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
......@@ -252,7 +252,7 @@ elif [[ $TASK == "mpi" ]]; then
pip install \
--user \
-v \
--install-option=--mpi \
--config-settings=cmake.define.USE_MPI=ON \
$BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz \
|| exit -1
pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
......
.ci/test_windows.ps1
View file @ 30942a32
......@@ -96,8 +96,8 @@ elseif ($env:TASK -eq "bdist") {
cd $env:BUILD_SOURCESDIRECTORY
sh "build-python.sh" bdist_wheel --integrated-opencl ; Check-Output $?
sh $env:BUILD_SOURCESDIRECTORY/.ci/check_python_dists.sh $env:BUILD_SOURCESDIRECTORY/dist ; Check-Output $?
cd dist; pip install --user @(Get-ChildItem *.whl) ; Check-Output $?
cp @(Get-ChildItem *.whl) $env:BUILD_ARTIFACTSTAGINGDIRECTORY
cd dist; pip install --user @(Get-ChildItem *py3-none-win_amd64.whl) ; Check-Output $?
cp @(Get-ChildItem *py3-none-win_amd64.whl) $env:BUILD_ARTIFACTSTAGINGDIRECTORY
} elseif (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python")) {
cd $env:BUILD_SOURCESDIRECTORY
if ($env:COMPILER -eq "MINGW") {
......
CMakeLists.txt
View file @ 30942a32
......@@ -645,6 +645,10 @@ if(BUILD_CLI)
)
endif()
if(__BUILD_FOR_PYTHON)
set(CMAKE_INSTALL_PREFIX "lightgbm")
endif()
install(
TARGETS _lightgbm
RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin
......
build-python.sh
View file @ 30942a32
......@@ -94,75 +94,79 @@ while [ $# -gt 0 ]; do
then shift;
fi
BOOST_DIR="${1#*=}"
BUILD_ARGS="${BUILD_ARGS} --boost-dir='${BOOST_DIR}'"
BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.Boost_DIR='${BOOST_DIR}'"
;;
--boost-include-dir|--boost-include-dir=*)
if [[ "$1" != *=* ]];
then shift;
fi
BOOST_INCLUDE_DIR="${1#*=}"
BUILD_ARGS="${BUILD_ARGS} --boost-include-dir='${BOOST_INCLUDE_DIR}'"
BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.Boost_INCLUDE_DIR='${BOOST_INCLUDE_DIR}'"
;;
--boost-librarydir|--boost-librarydir=*)
if [[ "$1" != *=* ]];
then shift;
fi
BOOST_LIBRARY_DIR="${1#*=}"
BUILD_ARGS="${BUILD_ARGS} --boost-librarydir='${BOOST_LIBRARY_DIR}'"
BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.BOOST_LIBRARYDIR='${BOOST_LIBRARY_DIR}'"
;;
--boost-root|--boost-root=*)
if [[ "$1" != *=* ]];
then shift;
fi
BOOST_ROOT="${1#*=}"
BUILD_ARGS="${BUILD_ARGS} --boost-root='${BOOST_ROOT}'"
BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.Boost_ROOT='${BOOST_ROOT}'"
;;
--opencl-include-dir|--opencl-include-dir=*)
if [[ "$1" != *=* ]];
then shift;
fi
OPENCL_INCLUDE_DIR="${1#*=}"
BUILD_ARGS="${BUILD_ARGS} --opencl-include-dir='${OPENCL_INCLUDE_DIR}'"
BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.OpenCL_INCLUDE_DIR='${OPENCL_INCLUDE_DIR}'"
;;
--opencl-library|--opencl-library=*)
if [[ "$1" != *=* ]];
then shift;
fi
OPENCL_LIBRARY="${1#*=}"
BUILD_ARGS="${BUILD_ARGS} --opencl-library='${OPENCL_LIBRARY}'"
BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.OpenCL_LIBRARY='${OPENCL_LIBRARY}'"
;;
#########
# flags #
#########
--bit32)
BUILD_ARGS="${BUILD_ARGS} --bit32"
export CMAKE_GENERATOR="Visual Studio 17 2022"
export CMAKE_GENERATOR_PLATFORM="Win32"
echo "[INFO] Attempting to build 32-bit version of LightGBM, which is only supported on Windows with generator '${CMAKE_GENERATOR}'."
;;
--cuda)
BUILD_ARGS="${BUILD_ARGS} --cuda"
BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_CUDA=ON"
;;
--gpu)
BUILD_ARGS="${BUILD_ARGS} --gpu"
BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_GPU=ON"
;;
--hdfs)
BUILD_ARGS="${BUILD_ARGS} --hdfs"
BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_HDFS=ON"
;;
--integrated-opencl)
BUILD_ARGS="${BUILD_ARGS} --integrated-opencl"
BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.__INTEGRATE_OPENCL=ON"
;;
--mingw)
BUILD_ARGS="${BUILD_ARGS} --mingw"
export CMAKE_GENERATOR='MinGW Makefiles'
# ref: https://stackoverflow.com/a/45104058/3986677
BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.CMAKE_SH=CMAKE_SH-NOTFOUND"
;;
--mpi)
BUILD_ARGS="${BUILD_ARGS} --mpi"
BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_MPI=ON"
;;
--nomp)
BUILD_ARGS="${BUILD_ARGS} --nomp"
BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_OPENMP=OFF"
;;
--precompile)
PRECOMPILE="true"
;;
--time-costs)
BUILD_ARGS="${PIP_INSTALL_ARGS} --time-costs"
BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_TIMETAG=ON"
;;
--user)
PIP_INSTALL_ARGS="${PIP_INSTALL_ARGS} --user"
......@@ -175,6 +179,8 @@ while [ $# -gt 0 ]; do
shift
done
pip install --prefer-binary 'build>=0.10.0'
# create a new directory that just contains the files needed
# to build the Python package
create_isolated_source_dir() {
......@@ -189,21 +195,15 @@ create_isolated_source_dir() {
cp -R ./python-package ./lightgbm-python
# temporarily remove these files until
# https://github.com/microsoft/LightGBM/issues/5061 is done
rm ./lightgbm-python/pyproject.toml
rm ./lightgbm-python/setup.cfg
cp LICENSE ./lightgbm-python/
cp VERSION.txt ./lightgbm-python/lightgbm/VERSION.txt
mkdir -p ./lightgbm-python/compile
cp -R ./cmake ./lightgbm-python/compile
cp CMakeLists.txt ./lightgbm-python/compile
cp -R ./include ./lightgbm-python/compile
cp -R ./src ./lightgbm-python/compile
cp -R ./swig ./lightgbm-python/compile
cp -R ./windows ./lightgbm-python/compile
cp -R ./cmake ./lightgbm-python
cp CMakeLists.txt ./lightgbm-python
cp -R ./include ./lightgbm-python
cp -R ./src ./lightgbm-python
cp -R ./swig ./lightgbm-python
cp -R ./windows ./lightgbm-python
# include only specific files from external_libs, to keep the package
# small and avoid redistributing code with licenses incompatible with
......@@ -212,77 +212,77 @@ create_isolated_source_dir() {
######################
# fast_double_parser #
######################
mkdir -p ./lightgbm-python/compile/external_libs/fast_double_parser
mkdir -p ./lightgbm-python/external_libs/fast_double_parser
cp \
external_libs/fast_double_parser/CMakeLists.txt \
./lightgbm-python/compile/external_libs/fast_double_parser/CMakeLists.txt
./lightgbm-python/external_libs/fast_double_parser/CMakeLists.txt
cp \
external_libs/fast_double_parser/LICENSE* \
./lightgbm-python/compile/external_libs/fast_double_parser/
./lightgbm-python/external_libs/fast_double_parser/
mkdir -p ./lightgbm-python/compile/external_libs/fast_double_parser/include/
mkdir -p ./lightgbm-python/external_libs/fast_double_parser/include/
cp \
external_libs/fast_double_parser/include/fast_double_parser.h \
./lightgbm-python/compile/external_libs/fast_double_parser/include/
./lightgbm-python/external_libs/fast_double_parser/include/
#######
# fmt #
#######
mkdir -p ./lightgbm-python/compile/external_libs/fmt
mkdir -p ./lightgbm-python/external_libs/fmt
cp \
external_libs/fast_double_parser/CMakeLists.txt \
./lightgbm-python/compile/external_libs/fmt/CMakeLists.txt
./lightgbm-python/external_libs/fmt/CMakeLists.txt
cp \
external_libs/fmt/LICENSE* \
./lightgbm-python/compile/external_libs/fmt/
./lightgbm-python/external_libs/fmt/
mkdir -p ./lightgbm-python/compile/external_libs/fmt/include/fmt
mkdir -p ./lightgbm-python/external_libs/fmt/include/fmt
cp \
external_libs/fmt/include/fmt/*.h \
./lightgbm-python/compile/external_libs/fmt/include/fmt/
./lightgbm-python/external_libs/fmt/include/fmt/
#########
# Eigen #
#########
mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen
mkdir -p ./lightgbm-python/external_libs/eigen/Eigen
cp \
external_libs/eigen/CMakeLists.txt \
./lightgbm-python/compile/external_libs/eigen/CMakeLists.txt
./lightgbm-python/external_libs/eigen/CMakeLists.txt
modules="Cholesky Core Dense Eigenvalues Geometry Householder Jacobi LU QR SVD"
for eigen_module in ${modules}; do
cp \
external_libs/eigen/Eigen/${eigen_module} \
./lightgbm-python/compile/external_libs/eigen/Eigen/${eigen_module}
./lightgbm-python/external_libs/eigen/Eigen/${eigen_module}
if [ ${eigen_module} != "Dense" ]; then
mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen/src/${eigen_module}/
mkdir -p ./lightgbm-python/external_libs/eigen/Eigen/src/${eigen_module}/
cp \
-R \
external_libs/eigen/Eigen/src/${eigen_module}/* \
./lightgbm-python/compile/external_libs/eigen/Eigen/src/${eigen_module}/
./lightgbm-python/external_libs/eigen/Eigen/src/${eigen_module}/
fi
done
mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen/misc
mkdir -p ./lightgbm-python/external_libs/eigen/Eigen/misc
cp \
-R \
external_libs/eigen/Eigen/src/misc \
./lightgbm-python/compile/external_libs/eigen/Eigen/src/misc/
./lightgbm-python/external_libs/eigen/Eigen/src/misc/
mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen/plugins
mkdir -p ./lightgbm-python/external_libs/eigen/Eigen/plugins
cp \
-R \
external_libs/eigen/Eigen/src/plugins \
./lightgbm-python/compile/external_libs/eigen/Eigen/src/plugins/
./lightgbm-python/external_libs/eigen/Eigen/src/plugins/
###################
# compute (Boost) #
###################
mkdir -p ./lightgbm-python/compile/external_libs/compute
mkdir -p ./lightgbm-python/external_libs/compute
cp \
-R \
external_libs/compute/include \
./lightgbm-python/compile/external_libs/compute/include/
./lightgbm-python/external_libs/compute/include/
}
create_isolated_source_dir
......@@ -292,9 +292,39 @@ cd ./lightgbm-python
# installation involves building the wheel + `pip install`-ing it
if test "${INSTALL}" = true; then
if test "${PRECOMPILE}" = true; then
echo "--- installing lightgbm (from precompiled lib_lightgbm) ---"
python setup.py install ${PIP_INSTALL_ARGS} --precompile
exit 0
BUILD_SDIST=true
BUILD_WHEEL=false
BUILD_ARGS=""
rm -rf \
./cmake \
./CMakeLists.txt \
./external_libs \
./include \
./src \
./swig \
./windows
# use regular-old setuptools for these builds, to avoid
# trying to recompile the shared library
sed -i.bak -e '/start:build-system/,/end:build-system/d' pyproject.toml
echo '[build-system]' >> ./pyproject.toml
echo 'requires = ["setuptools"]' >> ./pyproject.toml
echo 'build-backend = "setuptools.build_meta"' >> ./pyproject.toml
echo "" >> ./pyproject.toml
echo "recursive-include lightgbm *.dll *.so" > ./MANIFEST.in
echo "" >> ./MANIFEST.in
mkdir -p ./lightgbm/lib
if test -f ../lib_lightgbm.so; then
echo "found pre-compiled lib_lightgbm.so"
cp ../lib_lightgbm.so ./lightgbm/lib/lib_lightgbm.so
elif test -f ../Release/lib_lightgbm.dll; then
echo "found pre-compiled Release/lib_lightgbm.dll"
cp ../Release/lib_lightgbm.dll ./lightgbm/lib/lib_lightgbm.dll
elif test -f ../windows/x64/DLL/lib_lightgbm.dll; then
echo "found pre-compiled windows/x64/DLL/lib_lightgbm.dll"
cp ../windows/x64/DLL/lib_lightgbm.dll ./lightgbm/lib/lib_lightgbm.dll
cp ../windows/x64/DLL/lib_lightgbm.lib ./lightgbm/lib/lib_lightgbm.lib
fi
rm -f ./*.bak
else
BUILD_SDIST="false"
BUILD_WHEEL="true"
......@@ -304,16 +334,20 @@ fi
if test "${BUILD_SDIST}" = true; then
echo "--- building sdist ---"
rm -f ../dist/*.tar.gz
python ./setup.py sdist \
--dist-dir ../dist
python -m build \
--sdist \
--outdir ../dist \
.
fi
if test "${BUILD_WHEEL}" = true; then
echo "--- building wheel ---"
rm -f ../dist/*.whl || true
python setup.py bdist_wheel \
--dist-dir ../dist \
${BUILD_ARGS}
python -m build \
--wheel \
--outdir ../dist \
${BUILD_ARGS} \
.
fi
if test "${INSTALL}" = true; then
......
python-package/MANIFEST.in deleted 100644 → 0
View file @ a323c758
prune build
include LICENSE
include *.rst *.txt
recursive-include lightgbm VERSION.txt py.typed *.py *.so
include compile/CMakeLists.txt
include compile/cmake/IntegratedOpenCL.cmake
recursive-include compile *.so
recursive-include compile/Release *.dll
include compile/external_libs/compute/CMakeLists.txt
recursive-include compile/external_libs/compute/cmake *
recursive-include compile/external_libs/compute/include *
recursive-include compile/external_libs/compute/meta *
include compile/external_libs/eigen/CMakeLists.txt
include compile/external_libs/eigen/Eigen/Cholesky
include compile/external_libs/eigen/Eigen/Core
include compile/external_libs/eigen/Eigen/Dense
include compile/external_libs/eigen/Eigen/Eigenvalues
include compile/external_libs/eigen/Eigen/Geometry
include compile/external_libs/eigen/Eigen/Householder
include compile/external_libs/eigen/Eigen/Jacobi
include compile/external_libs/eigen/Eigen/LU
include compile/external_libs/eigen/Eigen/QR
include compile/external_libs/eigen/Eigen/SVD
recursive-include compile/external_libs/eigen/Eigen/src/Cholesky *
recursive-include compile/external_libs/eigen/Eigen/src/Core *
recursive-include compile/external_libs/eigen/Eigen/src/Eigenvalues *
recursive-include compile/external_libs/eigen/Eigen/src/Geometry *
recursive-include compile/external_libs/eigen/Eigen/src/Householder *
recursive-include compile/external_libs/eigen/Eigen/src/Jacobi *
recursive-include compile/external_libs/eigen/Eigen/src/LU *
recursive-include compile/external_libs/eigen/Eigen/src/misc *
recursive-include compile/external_libs/eigen/Eigen/src/plugins *
recursive-include compile/external_libs/eigen/Eigen/src/QR *
recursive-include compile/external_libs/eigen/Eigen/src/SVD *
include compile/external_libs/fast_double_parser/CMakeLists.txt
include compile/external_libs/fast_double_parser/LICENSE
include compile/external_libs/fast_double_parser/LICENSE.BSL
recursive-include compile/external_libs/fast_double_parser/include *
include compile/external_libs/fmt/CMakeLists.txt
include compile/external_libs/fmt/LICENSE.rst
recursive-include compile/external_libs/fmt/include *
recursive-include compile/include *
recursive-include compile/src *
recursive-include compile/windows LightGBM.sln LightGBM.vcxproj
recursive-include compile/windows/x64/DLL *.dll
global-exclude *.py[co]
exclude compile/external_libs/compute/.git
python-package/README.rst
View file @ 30942a32
......@@ -60,7 +60,7 @@ Build Threadless Version
.. code:: sh
pip install lightgbm --install-option=--nomp
pip install lightgbm --config-settings=cmake.define.USE_OPENMP=OFF
All requirements, except the **OpenMP** requirement, from `Build from Sources section <#build-from-sources>`__ apply for this installation option as well.
......@@ -71,7 +71,7 @@ Build MPI Version
.. code:: sh
pip install lightgbm --install-option=--mpi
pip install lightgbm --config-settings=cmake.define.USE_MPI=ON
All requirements from `Build from Sources section <#build-from-sources>`__ apply for this installation option as well.
......@@ -84,7 +84,7 @@ Build GPU Version
.. code:: sh
pip install lightgbm --install-option=--gpu
pip install lightgbm --config-settings=cmake.define.USE_GPU=ON
All requirements from `Build from Sources section <#build-from-sources>`__ apply for this installation option as well.
......@@ -94,21 +94,24 @@ For **Windows** users, `CMake`_ (version 3.8 or higher) is strongly required.
.. code:: sh
pip install lightgbm --install-option=--gpu --install-option="--opencl-include-dir=/usr/local/cuda/include/" --install-option="--opencl-library=/usr/local/cuda/lib64/libOpenCL.so"
pip install lightgbm \
--config-settings=cmake.define.USE_GPU=ON \
--config-settings=cmake.define.OpenCL_INCLUDE_DIR="/usr/local/cuda/include/" \
--config-settings=cmake.define.OpenCL_LIBRARY="/usr/local/cuda/lib64/libOpenCL.so"
All available options:
All available options that can be passed via ``cmake.define.{option}``.
- boost-root
- Boost_ROOT
- boost-dir
- Boost_DIR
- boost-include-dir
- Boost_INCLUDE_DIR
- boost-librarydir
- BOOST_LIBRARYDIR
- opencl-include-dir
- OpenCL_INCLUDE_DIR
- opencl-library
- OpenCL_LIBRARY
For more details see `FindBoost <https://cmake.org/cmake/help/latest/module/FindBoost.html>`__ and `FindOpenCL <https://cmake.org/cmake/help/latest/module/FindOpenCL.html>`__.
......@@ -117,7 +120,7 @@ Build CUDA Version
.. code:: sh
pip install lightgbm --install-option=--cuda
pip install lightgbm --config-settings=cmake.define.USE_CUDA=ON
All requirements from `Build from Sources section <#build-from-sources>`__ apply for this installation option as well, and `CMake`_ (version 3.16 or higher) is strongly required.
......@@ -130,7 +133,7 @@ Build HDFS Version
.. code:: sh
pip install lightgbm --install-option=--hdfs
pip install lightgbm --config-settings=cmake.define.USE_HDFS=ON
All requirements from `Build from Sources section <#build-from-sources>`__ apply for this installation option as well.
......@@ -143,7 +146,9 @@ Build with MinGW-w64 on Windows
.. code:: sh
pip install lightgbm --install-option=--mingw
# in sh.exe, git bash, or other Unix-like shell
export CMAKE_GENERATOR='MinGW Makefiles'
pip install lightgbm --config-settings=cmake.define.CMAKE_SH=CMAKE_SH-NOTFOUND
`CMake`_ and `MinGW-w64 <https://www.mingw-w64.org/>`_ should be installed first.
......@@ -155,7 +160,10 @@ Build 32-bit Version with 32-bit Python
.. code:: sh
pip install lightgbm --install-option=--bit32
# in sh.exe, git bash, or other Unix-like shell
export CMAKE_GENERATOR='Visual Studio 17 2022'
export CMAKE_GENERATOR_PLATFORM='Win32'
pip install --no-binary lightgbm lightgbm
By default, installation in environment with 32-bit Python is prohibited. However, you can remove this prohibition on your own risk by passing ``bit32`` option.
......@@ -166,7 +174,7 @@ Build with Time Costs Output
.. code:: sh
pip install lightgbm --install-option=--time-costs
pip install lightgbm --config-settings=cmake.define.USE_TIMETAG=ON
Use this option to make LightGBM output time costs for different internal routines, to investigate and benchmark its performance.
......@@ -221,6 +229,21 @@ Build Wheel File
You can use ``sh ./build-python.sh install bdist_wheel`` instead of ``sh ./build-python.sh install`` to build wheel file and use it for installation later. This might be useful for systems with restricted or completely without network access.
Build With MSBuild
******************
To use ``MSBuild`` (Windows-only), first build ``lib_lightgbm.dll`` by running the following from the root of the repo.
.. code:: sh
MSBuild.exe windows/LightGBM.sln /p:Configuration=DLL /p:Platform=x64 /p:PlatformToolset=v143
Then install the Python package using that library.
.. code:: sh
sh ./build-python.sh install --precompile
Install Dask-package
''''''''''''''''''''
......
python-package/lightgbm/libpath.py
View file @ 30942a32
......@@ -15,15 +15,12 @@ def find_lib_path() -> List[str]:
lib_path: list of str
List of all found library paths to LightGBM.
"""
curr_path = Path(__file__).absolute().parent
curr_path = Path(__file__).absolute()
dll_path = [curr_path,
curr_path.parents[1],
curr_path / 'compile',
curr_path.parent / 'compile',
curr_path.parents[1] / 'lib']
curr_path.parents[0] / 'bin',
curr_path.parents[0] / 'lib']
if system() in ('Windows', 'Microsoft'):
dll_path.append(curr_path.parent / 'compile' / 'Release')
dll_path.append(curr_path.parent / 'compile' / 'windows' / 'x64' / 'DLL')
dll_path.append(curr_path.parents[1] / 'Release')
dll_path.append(curr_path.parents[1] / 'windows' / 'x64' / 'DLL')
dll_path = [p / 'lib_lightgbm.dll' for p in dll_path]
......
python-package/pyproject.toml
View file @ 30942a32
[project]
classifiers = [
"Development Status :: 5 - Production/Stable",
"Intended Audience :: Science/Research",
"License :: OSI Approved :: MIT License",
"Natural Language :: English",
"Operating System :: MacOS",
"Operating System :: Microsoft :: Windows",
"Operating System :: POSIX",
"Operating System :: Unix",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Topic :: Scientific/Engineering :: Artificial Intelligence"
]
description = "LightGBM Python Package"
license = {file = "LICENSE"}
maintainers = [
{name = "Yu Shi", email = "yushi@microsoft.com"}
]
name = "lightgbm"
readme = "README.rst"
requires-python = ">=3.6"
version = "3.3.5.99"
[project.urls]
homepage = "https://github.com/microsoft/LightGBM"
documentation = "https://lightgbm.readthedocs.io/en/latest/"
repository = "https://github.com/microsoft/LightGBM.git"
changelog = "https://github.com/microsoft/LightGBM/releases"
# start:build-system
[build-system]
requires = ["scikit-build-core>=0.4.4"]
build-backend = "scikit_build_core.build"
# based on https://github.com/scikit-build/scikit-build-core#configuration
[tool.scikit-build]
cmake.minimum-version = "3.15"
ninja.minimum-version = "1.11"
ninja.make-fallback = true
cmake.args = [
"-D__BUILD_FOR_PYTHON:BOOL=ON"
]
cmake.verbose = false
cmake.build-type = "Release"
logging.level = "INFO"
sdist.reproducible = true
wheel.py-api = "py3"
experimental = false
strict-config = true
minimum-version = "0.4.4"
# end:build-system
[tool.isort]
line_length = 120
skip_glob = [
......
python-package/setup.cfg
View file @ 30942a32
[options]
include_package_data = True
install_requires =
numpy
scikit-learn!=0.22.0
scipy
[options.extras_require]
dask =
dask[array]>=2.0.0
dask[dataframe]>=2.0.0
dask[distributed]>=2.0.0
pandas
[options.packages.find]
where = lightgbm
[flake8]
ignore =
# line too long
......
python-package/setup.py deleted 100644 → 0
View file @ a323c758
# coding: utf-8
"""Setup lightgbm package."""
import logging
import struct
import subprocess
import sys
from os import chdir
from pathlib import Path
from platform import system
from shutil import rmtree
from typing import List, Optional
from setuptools import find_packages, setup
from setuptools.command.install import install
from setuptools.command.install_lib import install_lib
from setuptools.command.sdist import sdist
from wheel.bdist_wheel import bdist_wheel
LIGHTGBM_OPTIONS = [
('mingw', 'm', 'Compile with MinGW'),
('integrated-opencl', None, 'Compile integrated OpenCL version'),
('gpu', 'g', 'Compile GPU version'),
('cuda', None, 'Compile CUDA version'),
('mpi', None, 'Compile MPI version'),
('nomp', None, 'Compile version without OpenMP support'),
('hdfs', 'h', 'Compile HDFS version'),
('bit32', None, 'Compile 32-bit version'),
('precompile', 'p', 'Use precompiled library'),
('time-costs', None, 'Output time costs for different internal routines'),
('boost-root=', None, 'Boost preferred installation prefix'),
('boost-dir=', None, 'Directory with Boost package configuration file'),
('boost-include-dir=', None, 'Directory containing Boost headers'),
('boost-librarydir=', None, 'Preferred Boost library directory'),
('opencl-include-dir=', None, 'OpenCL include directory'),
('opencl-library=', None, 'Path to OpenCL library')
]
def find_lib() -> List[str]:
libpath_py = CURRENT_DIR / 'lightgbm' / 'libpath.py'
libpath = {'__file__': libpath_py}
exec(compile(libpath_py.read_bytes(), libpath_py, 'exec'), libpath, libpath)
LIB_PATH = libpath['find_lib_path']() # type: ignore
logger.info(f"Installing lib_lightgbm from: {LIB_PATH}")
return LIB_PATH
def clear_path(path: Path) -> None:
if path.is_dir():
for file_name in path.iterdir():
if file_name.is_dir():
rmtree(file_name)
else:
file_name.unlink()
def silent_call(cmd: List[str], raise_error: bool = False, error_msg: str = '') -> int:
try:
with open(LOG_PATH, "ab") as log:
subprocess.check_call(cmd, stderr=log, stdout=log)
return 0
except Exception:
if raise_error:
raise Exception("\n".join((error_msg, LOG_NOTICE)))
return 1
def compile_cpp(
use_mingw: bool = False,
use_gpu: bool = False,
use_cuda: bool = False,
use_mpi: bool = False,
use_hdfs: bool = False,
boost_root: Optional[str] = None,
boost_dir: Optional[str] = None,
boost_include_dir: Optional[str] = None,
boost_librarydir: Optional[str] = None,
opencl_include_dir: Optional[str] = None,
opencl_library: Optional[str] = None,
nomp: bool = False,
bit32: bool = False,
integrated_opencl: bool = False,
time_costs: bool = False
) -> None:
build_dir = CURRENT_DIR / "build_cpp"
rmtree(build_dir, ignore_errors=True)
build_dir.mkdir(parents=True)
original_dir = Path.cwd()
chdir(build_dir)
logger.info("Starting to compile the library.")
cmake_cmd = ["cmake", str(CURRENT_DIR / "compile"), "-D__BUILD_FOR_PYTHON=ON"]
if integrated_opencl:
use_gpu = False
cmake_cmd.append("-D__INTEGRATE_OPENCL=ON")
if use_gpu:
cmake_cmd.append("-DUSE_GPU=ON")
if boost_root:
cmake_cmd.append(f"-DBOOST_ROOT={boost_root}")
if boost_dir:
cmake_cmd.append(f"-DBoost_DIR={boost_dir}")
if boost_include_dir:
cmake_cmd.append(f"-DBoost_INCLUDE_DIR={boost_include_dir}")
if boost_librarydir:
cmake_cmd.append(f"-DBOOST_LIBRARYDIR={boost_librarydir}")
if opencl_include_dir:
cmake_cmd.append(f"-DOpenCL_INCLUDE_DIR={opencl_include_dir}")
if opencl_library:
cmake_cmd.append(f"-DOpenCL_LIBRARY={opencl_library}")
elif use_cuda:
cmake_cmd.append("-DUSE_CUDA=ON")
if use_mpi:
cmake_cmd.append("-DUSE_MPI=ON")
if nomp:
cmake_cmd.append("-DUSE_OPENMP=OFF")
if use_hdfs:
cmake_cmd.append("-DUSE_HDFS=ON")
if time_costs:
cmake_cmd.append("-DUSE_TIMETAG=ON")
if system() in {'Windows', 'Microsoft'}:
if use_mingw:
if use_mpi:
raise Exception('MPI version cannot be compiled by MinGW due to the miss of MPI library in it')
logger.info("Starting to compile with CMake and MinGW.")
# ref: https://stackoverflow.com/a/45104058/3986677
silent_call(cmake_cmd + ["-G", "MinGW Makefiles", "-DCMAKE_SH=CMAKE_SH-NOTFOUND"], raise_error=True,
error_msg='Please install CMake and all required dependencies first')
silent_call(["mingw32-make.exe", "_lightgbm", f"-I{build_dir}", "-j4"], raise_error=True,
error_msg='Please install MinGW first')
else:
status = 1
lib_path = CURRENT_DIR / "compile" / "windows" / "x64" / "DLL" / "lib_lightgbm.dll"
if not any((use_gpu, use_cuda, use_mpi, use_hdfs, nomp, bit32, integrated_opencl)):
logger.info("Starting to compile with MSBuild from existing solution file.")
platform_toolsets = ("v143", "v142", "v141", "v140")
for pt in platform_toolsets:
status = silent_call(["MSBuild",
str(CURRENT_DIR / "compile" / "windows" / "LightGBM.sln"),
"/p:Configuration=DLL",
"/p:Platform=x64",
f"/p:PlatformToolset={pt}"])
if status == 0 and lib_path.is_file():
break
else:
clear_path(CURRENT_DIR / "compile" / "windows" / "x64")
if status != 0 or not lib_path.is_file():
logger.warning("Compilation with MSBuild from existing solution file failed.")
if status != 0 or not lib_path.is_file():
arch = "Win32" if bit32 else "x64"
vs_versions = (
"Visual Studio 17 2022",
"Visual Studio 16 2019",
"Visual Studio 15 2017",
"Visual Studio 14 2015"
)
for vs in vs_versions:
logger.info(f"Starting to compile with {vs} ({arch}).")
status = silent_call(cmake_cmd + ["-G", vs, "-A", arch])
if status == 0:
break
else:
clear_path(build_dir)
if status != 0:
raise Exception("\n".join(('Please install Visual Studio or MS Build and all required dependencies first',
LOG_NOTICE)))
silent_call(["cmake", "--build", str(build_dir), "--target", "_lightgbm", "--config", "Release"], raise_error=True,
error_msg='Please install CMake first')
else: # Linux, Darwin (macOS), etc.
logger.info("Starting to compile with CMake.")
silent_call(cmake_cmd, raise_error=True, error_msg='Please install CMake and all required dependencies first')
silent_call(["make", "_lightgbm", f"-I{build_dir}", "-j4"], raise_error=True,
error_msg='An error has occurred while building lightgbm library file')
chdir(original_dir)
class CustomInstallLib(install_lib):
def install(self) -> List[str]:
outfiles = install_lib.install(self)
src = find_lib()[0]
dst = Path(self.install_dir) / 'lightgbm'
dst, _ = self.copy_file(src, str(dst))
outfiles.append(dst)
return outfiles
class CustomInstall(install):
user_options = install.user_options + LIGHTGBM_OPTIONS
def initialize_options(self) -> None:
install.initialize_options(self)
self.mingw = False
self.integrated_opencl = False
self.gpu = False
self.cuda = False
self.boost_root = None
self.boost_dir = None
self.boost_include_dir = None
self.boost_librarydir = None
self.opencl_include_dir = None
self.opencl_library = None
self.mpi = False
self.hdfs = False
self.precompile = False
self.time_costs = False
self.nomp = False
self.bit32 = False
def run(self) -> None:
if (8 * struct.calcsize("P")) != 64:
if self.bit32:
logger.warning("You're installing 32-bit version. "
"This version is slow and untested, so use it on your own risk.")
else:
raise Exception("Cannot install LightGBM in 32-bit Python, "
"please use 64-bit Python instead.")
LOG_PATH.touch()
if not self.precompile:
compile_cpp(use_mingw=self.mingw, use_gpu=self.gpu, use_cuda=self.cuda, use_mpi=self.mpi,
use_hdfs=self.hdfs, boost_root=self.boost_root, boost_dir=self.boost_dir,
boost_include_dir=self.boost_include_dir, boost_librarydir=self.boost_librarydir,
opencl_include_dir=self.opencl_include_dir, opencl_library=self.opencl_library,
nomp=self.nomp, bit32=self.bit32, integrated_opencl=self.integrated_opencl,
time_costs=self.time_costs)
install.run(self)
if LOG_PATH.is_file():
LOG_PATH.unlink()
class CustomBdistWheel(bdist_wheel):
user_options = bdist_wheel.user_options + LIGHTGBM_OPTIONS
def initialize_options(self) -> None:
bdist_wheel.initialize_options(self)
self.mingw = False
self.integrated_opencl = False
self.gpu = False
self.cuda = False
self.boost_root = None
self.boost_dir = None
self.boost_include_dir = None
self.boost_librarydir = None
self.opencl_include_dir = None
self.opencl_library = None
self.mpi = False
self.hdfs = False
self.precompile = False
self.time_costs = False
self.nomp = False
self.bit32 = False
def finalize_options(self) -> None:
bdist_wheel.finalize_options(self)
install = self.reinitialize_command('install')
install.mingw = self.mingw
install.integrated_opencl = self.integrated_opencl
install.gpu = self.gpu
install.cuda = self.cuda
install.boost_root = self.boost_root
install.boost_dir = self.boost_dir
install.boost_include_dir = self.boost_include_dir
install.boost_librarydir = self.boost_librarydir
install.opencl_include_dir = self.opencl_include_dir
install.opencl_library = self.opencl_library
install.mpi = self.mpi
install.hdfs = self.hdfs
install.precompile = self.precompile
install.time_costs = self.time_costs
install.nomp = self.nomp
install.bit32 = self.bit32
class CustomSdist(sdist):
def run(self) -> None:
IS_SOURCE_FLAG_PATH.touch()
rmtree(CURRENT_DIR / 'lightgbm' / 'Release', ignore_errors=True)
rmtree(CURRENT_DIR / 'lightgbm' / 'windows' / 'x64', ignore_errors=True)
lib_file = CURRENT_DIR / 'lightgbm' / 'lib_lightgbm.so'
if lib_file.is_file():
lib_file.unlink()
sdist.run(self)
if IS_SOURCE_FLAG_PATH.is_file():
IS_SOURCE_FLAG_PATH.unlink()
if __name__ == "__main__":
CURRENT_DIR = Path(__file__).absolute().parent
LOG_PATH = Path.home() / 'LightGBM_compilation.log'
LOG_NOTICE = f"The full version of error log was saved into {LOG_PATH}"
IS_SOURCE_FLAG_PATH = CURRENT_DIR / '_IS_SOURCE_PACKAGE.txt'
_version_file = CURRENT_DIR / 'lightgbm' / 'VERSION.txt'
version = _version_file.read_text(encoding='utf-8').strip()
readme = (CURRENT_DIR / 'README.rst').read_text(encoding='utf-8')
sys.path.insert(0, str(CURRENT_DIR))
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('LightGBM')
setup(name='lightgbm',
version=version,
description='LightGBM Python Package',
long_description=readme,
long_description_content_type='text/x-rst',
python_requires='>=3.6',
install_requires=[
'wheel',
'numpy',
'scipy',
'scikit-learn!=0.22.0'
],
extras_require={
'dask': [
'dask[array]>=2.0.0',
'dask[dataframe]>=2.0.0',
'dask[distributed]>=2.0.0',
'pandas',
],
},
maintainer='Yu Shi',
maintainer_email='yushi2@microsoft.com',
zip_safe=False,
cmdclass={
'install': CustomInstall,
'install_lib': CustomInstallLib,
'bdist_wheel': CustomBdistWheel,
'sdist': CustomSdist,
},
packages=find_packages(),
include_package_data=True,
license='The MIT License (Microsoft)',
url='https://github.com/microsoft/LightGBM',
classifiers=['Development Status :: 5 - Production/Stable',
'Intended Audience :: Science/Research',
'License :: OSI Approved :: MIT License',
'Natural Language :: English',
'Operating System :: MacOS',
'Operating System :: Microsoft :: Windows',
'Operating System :: POSIX',
'Operating System :: Unix',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8',
'Programming Language :: Python :: 3.9',
'Programming Language :: Python :: 3.10',
'Topic :: Scientific/Engineering :: Artificial Intelligence',
'Typing :: Typed'])
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment