setup.py 16 KB
Newer Older
wxchan's avatar
wxchan committed
1
2
# coding: utf-8
"""Setup lightgbm package."""
3
import logging
4
import struct
5
import subprocess
6
import sys
7
8
from os import chdir
from pathlib import Path
9
from platform import system
10
11
from shutil import copyfile, copytree, rmtree
from typing import List, Optional, Union
12

13
from setuptools import find_packages, setup
14
15
16
from setuptools.command.install import install
from setuptools.command.install_lib import install_lib
from setuptools.command.sdist import sdist
17
18
19
20
21
22
23
from wheel.bdist_wheel import bdist_wheel

LIGHTGBM_OPTIONS = [
    ('mingw', 'm', 'Compile with MinGW'),
    ('integrated-opencl', None, 'Compile integrated OpenCL version'),
    ('gpu', 'g', 'Compile GPU version'),
    ('cuda', None, 'Compile CUDA version'),
24
    ('cuda-exp', None, 'Compile CUDA Experimental version'),
25
26
27
28
29
    ('mpi', None, 'Compile MPI version'),
    ('nomp', None, 'Compile version without OpenMP support'),
    ('hdfs', 'h', 'Compile HDFS version'),
    ('bit32', None, 'Compile 32-bit version'),
    ('precompile', 'p', 'Use precompiled library'),
30
    ('time-costs', None, 'Output time costs for different internal routines'),
31
32
33
34
35
36
37
    ('boost-root=', None, 'Boost preferred installation prefix'),
    ('boost-dir=', None, 'Directory with Boost package configuration file'),
    ('boost-include-dir=', None, 'Directory containing Boost headers'),
    ('boost-librarydir=', None, 'Preferred Boost library directory'),
    ('opencl-include-dir=', None, 'OpenCL include directory'),
    ('opencl-library=', None, 'Path to OpenCL library')
]
38

39

40
def find_lib() -> List[str]:
41
    libpath_py = CURRENT_DIR / 'lightgbm' / 'libpath.py'
42
    libpath = {'__file__': libpath_py}
43
    exec(compile(libpath_py.read_bytes(), libpath_py, 'exec'), libpath, libpath)
44

45
    LIB_PATH = libpath['find_lib_path']()  # type: ignore
46
    logger.info(f"Installing lib_lightgbm from: {LIB_PATH}")
47
48
49
    return LIB_PATH


50
def copy_files(integrated_opencl: bool = False, use_gpu: bool = False) -> None:
51

52
53
54
55
56
57
58
    def copy_files_helper(folder_name: Union[str, Path]) -> None:
        src = CURRENT_DIR.parent / folder_name
        if src.is_dir():
            dst = CURRENT_DIR / 'compile' / folder_name
            if dst.is_dir():
                rmtree(dst)
            copytree(src, dst)
59
        else:
60
            raise Exception(f'Cannot copy {src} folder')
61

62
    if not IS_SOURCE_FLAG_PATH.is_file():
63
64
        copy_files_helper('include')
        copy_files_helper('src')
65
        for submodule in (CURRENT_DIR.parent / 'external_libs').iterdir():
66
67
            submodule_stem = submodule.stem
            if submodule_stem == 'compute' and not use_gpu:
68
                continue
69
            copy_files_helper(Path('external_libs') / submodule_stem)
70
71
72
73
74
75
76
77
78
        (CURRENT_DIR / "compile" / "windows").mkdir(parents=True, exist_ok=True)
        copyfile(CURRENT_DIR.parent / "windows" / "LightGBM.sln",
                 CURRENT_DIR / "compile" / "windows" / "LightGBM.sln")
        copyfile(CURRENT_DIR.parent / "windows" / "LightGBM.vcxproj",
                 CURRENT_DIR / "compile" / "windows" / "LightGBM.vcxproj")
        copyfile(CURRENT_DIR.parent / "LICENSE",
                 CURRENT_DIR / "LICENSE")
        copyfile(CURRENT_DIR.parent / "CMakeLists.txt",
                 CURRENT_DIR / "compile" / "CMakeLists.txt")
79
        if integrated_opencl:
80
81
82
83
84
85
86
87
88
89
            (CURRENT_DIR / "compile" / "cmake").mkdir(parents=True, exist_ok=True)
            copyfile(CURRENT_DIR.parent / "cmake" / "IntegratedOpenCL.cmake",
                     CURRENT_DIR / "compile" / "cmake" / "IntegratedOpenCL.cmake")


def clear_path(path: Path) -> None:
    if path.is_dir():
        for file_name in path.iterdir():
            if file_name.is_dir():
                rmtree(file_name)
90
            else:
91
                file_name.unlink()
92
93


94
def silent_call(cmd: List[str], raise_error: bool = False, error_msg: str = '') -> int:
95
    try:
96
        with open(LOG_PATH, "ab") as log:
97
            subprocess.check_call(cmd, stderr=log, stdout=log)
98
99
        return 0
    except Exception as err:
100
        if raise_error:
101
            raise Exception("\n".join((error_msg, LOG_NOTICE)))
102
        return 1
103
104


105
106
107
108
def compile_cpp(
    use_mingw: bool = False,
    use_gpu: bool = False,
    use_cuda: bool = False,
109
    use_cuda_exp: bool = False,
110
111
112
113
114
115
116
117
118
119
    use_mpi: bool = False,
    use_hdfs: bool = False,
    boost_root: Optional[str] = None,
    boost_dir: Optional[str] = None,
    boost_include_dir: Optional[str] = None,
    boost_librarydir: Optional[str] = None,
    opencl_include_dir: Optional[str] = None,
    opencl_library: Optional[str] = None,
    nomp: bool = False,
    bit32: bool = False,
120
121
    integrated_opencl: bool = False,
    time_costs: bool = False
122
) -> None:
123
124
125
126
127
    build_dir = CURRENT_DIR / "build_cpp"
    rmtree(build_dir, ignore_errors=True)
    build_dir.mkdir(parents=True)
    original_dir = Path.cwd()
    chdir(build_dir)
128

129
130
    logger.info("Starting to compile the library.")

131
    cmake_cmd = ["cmake", str(CURRENT_DIR / "compile")]
132
133
134
    if integrated_opencl:
        use_gpu = False
        cmake_cmd.append("-D__INTEGRATE_OPENCL=ON")
135
    if use_gpu:
136
        cmake_cmd.append("-DUSE_GPU=ON")
137
        if boost_root:
138
            cmake_cmd.append(f"-DBOOST_ROOT={boost_root}")
139
        if boost_dir:
140
            cmake_cmd.append(f"-DBoost_DIR={boost_dir}")
141
        if boost_include_dir:
142
            cmake_cmd.append(f"-DBoost_INCLUDE_DIR={boost_include_dir}")
143
        if boost_librarydir:
144
            cmake_cmd.append(f"-DBOOST_LIBRARYDIR={boost_librarydir}")
145
        if opencl_include_dir:
146
            cmake_cmd.append(f"-DOpenCL_INCLUDE_DIR={opencl_include_dir}")
147
        if opencl_library:
148
            cmake_cmd.append(f"-DOpenCL_LIBRARY={opencl_library}")
149
150
    elif use_cuda:
        cmake_cmd.append("-DUSE_CUDA=ON")
151
152
    elif use_cuda_exp:
        cmake_cmd.append("-DUSE_CUDA_EXP=ON")
153
154
    if use_mpi:
        cmake_cmd.append("-DUSE_MPI=ON")
155
156
    if nomp:
        cmake_cmd.append("-DUSE_OPENMP=OFF")
157
158
    if use_hdfs:
        cmake_cmd.append("-DUSE_HDFS=ON")
159
160
    if time_costs:
        cmake_cmd.append("-DUSE_TIMETAG=ON")
161

162
    if system() in {'Windows', 'Microsoft'}:
163
        if use_mingw:
164
165
            if use_mpi:
                raise Exception('MPI version cannot be compiled by MinGW due to the miss of MPI library in it')
166
167
            logger.info("Starting to compile with CMake and MinGW.")
            silent_call(cmake_cmd + ["-G", "MinGW Makefiles"], raise_error=True,
168
                        error_msg='Please install CMake and all required dependencies first')
169
            silent_call(["mingw32-make.exe", "_lightgbm", f"-I{build_dir}", "-j4"], raise_error=True,
170
                        error_msg='Please install MinGW first')
171
        else:
172
            status = 1
173
            lib_path = CURRENT_DIR / "compile" / "windows" / "x64" / "DLL" / "lib_lightgbm.dll"
174
            if not any((use_gpu, use_cuda, use_cuda_exp, use_mpi, use_hdfs, nomp, bit32, integrated_opencl)):
175
                logger.info("Starting to compile with MSBuild from existing solution file.")
176
                platform_toolsets = ("v143", "v142", "v141", "v140")
177
                for pt in platform_toolsets:
178
                    status = silent_call(["MSBuild",
179
                                          str(CURRENT_DIR / "compile" / "windows" / "LightGBM.sln"),
180
181
                                          "/p:Configuration=DLL",
                                          "/p:Platform=x64",
182
                                          f"/p:PlatformToolset={pt}"])
183
                    if status == 0 and lib_path.is_file():
184
185
                        break
                    else:
186
187
                        clear_path(CURRENT_DIR / "compile" / "windows" / "x64")
                if status != 0 or not lib_path.is_file():
188
                    logger.warning("Compilation with MSBuild from existing solution file failed.")
189
            if status != 0 or not lib_path.is_file():
190
                arch = "Win32" if bit32 else "x64"
191
192
193
194
195
196
                vs_versions = (
                    "Visual Studio 17 2022",
                    "Visual Studio 16 2019",
                    "Visual Studio 15 2017",
                    "Visual Studio 14 2015"
                )
197
                for vs in vs_versions:
198
                    logger.info(f"Starting to compile with {vs} ({arch}).")
199
                    status = silent_call(cmake_cmd + ["-G", vs, "-A", arch])
200
201
202
                    if status == 0:
                        break
                    else:
203
                        clear_path(build_dir)
204
                if status != 0:
205
                    raise Exception("\n".join(('Please install Visual Studio or MS Build and all required dependencies first',
206
                                    LOG_NOTICE)))
207
                silent_call(["cmake", "--build", str(build_dir), "--target", "_lightgbm", "--config", "Release"], raise_error=True,
208
                            error_msg='Please install CMake first')
209
    else:  # Linux, Darwin (macOS), etc.
210
        logger.info("Starting to compile with CMake.")
211
        silent_call(cmake_cmd, raise_error=True, error_msg='Please install CMake and all required dependencies first')
212
        silent_call(["make", "_lightgbm", f"-I{build_dir}", "-j4"], raise_error=True,
213
                    error_msg='An error has occurred while building lightgbm library file')
214
    chdir(original_dir)
215
216
217
218


class CustomInstallLib(install_lib):

219
    def install(self) -> List[str]:
220
221
        outfiles = install_lib.install(self)
        src = find_lib()[0]
222
223
        dst = Path(self.install_dir) / 'lightgbm'
        dst, _ = self.copy_file(src, str(dst))
224
225
226
227
228
229
        outfiles.append(dst)
        return outfiles


class CustomInstall(install):

230
    user_options = install.user_options + LIGHTGBM_OPTIONS
231

232
    def initialize_options(self) -> None:
233
        install.initialize_options(self)
234
235
236
237
        self.mingw = False
        self.integrated_opencl = False
        self.gpu = False
        self.cuda = False
238
        self.cuda_exp = False
239
240
241
242
243
244
        self.boost_root = None
        self.boost_dir = None
        self.boost_include_dir = None
        self.boost_librarydir = None
        self.opencl_include_dir = None
        self.opencl_library = None
245
246
247
        self.mpi = False
        self.hdfs = False
        self.precompile = False
248
        self.time_costs = False
249
250
        self.nomp = False
        self.bit32 = False
251

252
    def run(self) -> None:
253
254
255
256
257
258
259
        if (8 * struct.calcsize("P")) != 64:
            if self.bit32:
                logger.warning("You're installing 32-bit version. "
                               "This version is slow and untested, so use it on your own risk.")
            else:
                raise Exception("Cannot install LightGBM in 32-bit Python, "
                                "please use 64-bit Python instead.")
260
        LOG_PATH.touch()
261
        if not self.precompile:
262
            copy_files(integrated_opencl=self.integrated_opencl, use_gpu=self.gpu)
263
            compile_cpp(use_mingw=self.mingw, use_gpu=self.gpu, use_cuda=self.cuda, use_cuda_exp=self.cuda_exp, use_mpi=self.mpi,
264
                        use_hdfs=self.hdfs, boost_root=self.boost_root, boost_dir=self.boost_dir,
265
                        boost_include_dir=self.boost_include_dir, boost_librarydir=self.boost_librarydir,
266
                        opencl_include_dir=self.opencl_include_dir, opencl_library=self.opencl_library,
267
268
                        nomp=self.nomp, bit32=self.bit32, integrated_opencl=self.integrated_opencl,
                        time_costs=self.time_costs)
269
        install.run(self)
270
271
        if LOG_PATH.is_file():
            LOG_PATH.unlink()
272
273


274
275
276
277
class CustomBdistWheel(bdist_wheel):

    user_options = bdist_wheel.user_options + LIGHTGBM_OPTIONS

278
    def initialize_options(self) -> None:
279
        bdist_wheel.initialize_options(self)
280
281
282
283
        self.mingw = False
        self.integrated_opencl = False
        self.gpu = False
        self.cuda = False
284
        self.cuda_exp = False
285
286
287
288
289
290
        self.boost_root = None
        self.boost_dir = None
        self.boost_include_dir = None
        self.boost_librarydir = None
        self.opencl_include_dir = None
        self.opencl_library = None
291
292
293
        self.mpi = False
        self.hdfs = False
        self.precompile = False
294
        self.time_costs = False
295
296
        self.nomp = False
        self.bit32 = False
297

298
    def finalize_options(self) -> None:
299
300
301
302
303
304
305
306
        bdist_wheel.finalize_options(self)

        install = self.reinitialize_command('install')

        install.mingw = self.mingw
        install.integrated_opencl = self.integrated_opencl
        install.gpu = self.gpu
        install.cuda = self.cuda
307
        install.cuda_exp = self.cuda_exp
308
309
310
311
312
313
314
315
316
        install.boost_root = self.boost_root
        install.boost_dir = self.boost_dir
        install.boost_include_dir = self.boost_include_dir
        install.boost_librarydir = self.boost_librarydir
        install.opencl_include_dir = self.opencl_include_dir
        install.opencl_library = self.opencl_library
        install.mpi = self.mpi
        install.hdfs = self.hdfs
        install.precompile = self.precompile
317
        install.time_costs = self.time_costs
318
319
320
321
        install.nomp = self.nomp
        install.bit32 = self.bit32


322
323
class CustomSdist(sdist):

324
    def run(self) -> None:
325
        copy_files(integrated_opencl=True, use_gpu=True)
326
327
328
329
330
331
        IS_SOURCE_FLAG_PATH.touch()
        rmtree(CURRENT_DIR / 'lightgbm' / 'Release', ignore_errors=True)
        rmtree(CURRENT_DIR / 'lightgbm' / 'windows' / 'x64', ignore_errors=True)
        lib_file = CURRENT_DIR / 'lightgbm' / 'lib_lightgbm.so'
        if lib_file.is_file():
            lib_file.unlink()
332
        sdist.run(self)
333
334
        if IS_SOURCE_FLAG_PATH.is_file():
            IS_SOURCE_FLAG_PATH.unlink()
335
336
337


if __name__ == "__main__":
338
339
    CURRENT_DIR = Path(__file__).absolute().parent
    LOG_PATH = Path.home() / 'LightGBM_compilation.log'
340
    LOG_NOTICE = f"The full version of error log was saved into {LOG_PATH}"
341
342
343
344
345
346
347
348
349
    IS_SOURCE_FLAG_PATH = CURRENT_DIR / '_IS_SOURCE_PACKAGE.txt'
    _version_src = CURRENT_DIR.parent / 'VERSION.txt'
    _version_dst = CURRENT_DIR / 'lightgbm' / 'VERSION.txt'
    if _version_src.is_file():
        copyfile(_version_src, _version_dst)
    version = _version_dst.read_text(encoding='utf-8').strip()
    readme = (CURRENT_DIR / 'README.rst').read_text(encoding='utf-8')

    sys.path.insert(0, str(CURRENT_DIR))
350

351
352
353
    logging.basicConfig(level=logging.INFO)
    logger = logging.getLogger('LightGBM')

Guolin Ke's avatar
Guolin Ke committed
354
355
356
    setup(name='lightgbm',
          version=version,
          description='LightGBM Python Package',
357
          long_description=readme,
358
          python_requires='>=3.6',
Guolin Ke's avatar
Guolin Ke committed
359
          install_requires=[
360
              'wheel',
Guolin Ke's avatar
Guolin Ke committed
361
362
              'numpy',
              'scipy',
363
              'scikit-learn!=0.22.0'
Guolin Ke's avatar
Guolin Ke committed
364
          ],
365
366
367
          extras_require={
              'dask': [
                  'dask[array]>=2.0.0',
368
                  'dask[dataframe]>=2.0.0',
369
370
371
372
                  'dask[distributed]>=2.0.0',
                  'pandas',
              ],
          },
373
374
          maintainer='Yu Shi',
          maintainer_email='yushi2@microsoft.com',
Guolin Ke's avatar
Guolin Ke committed
375
          zip_safe=False,
376
377
378
          cmdclass={
              'install': CustomInstall,
              'install_lib': CustomInstallLib,
379
              'bdist_wheel': CustomBdistWheel,
380
381
              'sdist': CustomSdist,
          },
Guolin Ke's avatar
Guolin Ke committed
382
383
          packages=find_packages(),
          include_package_data=True,
384
          license='The MIT License (Microsoft)',
385
          url='https://github.com/microsoft/LightGBM',
386
387
388
389
390
391
392
393
394
          classifiers=['Development Status :: 5 - Production/Stable',
                       'Intended Audience :: Science/Research',
                       'License :: OSI Approved :: MIT License',
                       'Natural Language :: English',
                       'Operating System :: MacOS',
                       'Operating System :: Microsoft :: Windows',
                       'Operating System :: POSIX',
                       'Operating System :: Unix',
                       'Programming Language :: Python :: 3',
395
                       'Programming Language :: Python :: 3.7',
396
                       'Programming Language :: Python :: 3.8',
397
                       'Programming Language :: Python :: 3.9',
398
                       'Programming Language :: Python :: 3.10',
399
                       'Topic :: Scientific/Engineering :: Artificial Intelligence'])