setup.py 14.2 KB
Newer Older
wxchan's avatar
wxchan committed
1
2
3
# coding: utf-8
"""Setup lightgbm package."""
from __future__ import absolute_import
4

5
import io
6
import logging
7
import os
Guolin Ke's avatar
Guolin Ke committed
8
import shutil
9
import struct
10
import subprocess
11
12
import sys

13
from platform import system
14
from setuptools import find_packages, setup
15
16
17
from setuptools.command.install import install
from setuptools.command.install_lib import install_lib
from setuptools.command.sdist import sdist
18
19
from distutils.dir_util import copy_tree
from distutils.file_util import copy_file
20

21
22

def find_lib():
23
    libpath_py = os.path.join(CURRENT_DIR, 'lightgbm', 'libpath.py')
24
25
26
27
    libpath = {'__file__': libpath_py}
    exec(compile(open(libpath_py, "rb").read(), libpath_py, 'exec'), libpath, libpath)

    LIB_PATH = [os.path.relpath(path, CURRENT_DIR) for path in libpath['find_lib_path']()]
28
    logger.info("Installing lib_lightgbm from: %s" % LIB_PATH)
29
30
31
    return LIB_PATH


32
def copy_files(integrated_opencl=False, use_gpu=False):
33
34

    def copy_files_helper(folder_name):
35
        src = os.path.join(CURRENT_DIR, os.path.pardir, folder_name)
36
        if os.path.exists(src):
37
            dst = os.path.join(CURRENT_DIR, 'compile', folder_name)
38
            shutil.rmtree(dst, ignore_errors=True)
39
            copy_tree(src, dst, verbose=0)
40
        else:
41
            raise Exception('Cannot copy {0} folder'.format(src))
42

43
    if not os.path.isfile(os.path.join(CURRENT_DIR, '_IS_SOURCE_PACKAGE.txt')):
44
45
        copy_files_helper('include')
        copy_files_helper('src')
46
47
        if not os.path.exists(os.path.join(CURRENT_DIR, "compile", "windows")):
            os.makedirs(os.path.join(CURRENT_DIR, "compile", "windows"))
48
49
50
51
52
53
54
55
56
        copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "windows", "LightGBM.sln"),
                  os.path.join(CURRENT_DIR, "compile", "windows", "LightGBM.sln"),
                  verbose=0)
        copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "windows", "LightGBM.vcxproj"),
                  os.path.join(CURRENT_DIR, "compile", "windows", "LightGBM.vcxproj"),
                  verbose=0)
        copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "LICENSE"),
                  os.path.join(CURRENT_DIR, "LICENSE"),
                  verbose=0)
57
58
59
60
61
62
63
64
65
        copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeLists.txt"),
                  os.path.join(CURRENT_DIR, "compile", "CMakeLists.txt"),
                  verbose=0)
        if integrated_opencl:
            copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeIntegratedOpenCL.cmake"),
                      os.path.join(CURRENT_DIR, "compile", "CMakeIntegratedOpenCL.cmake"),
                      verbose=0)
        if use_gpu:
            copy_files_helper('compute')
wxchan's avatar
wxchan committed
66
67


68
def clear_path(path):
69
70
71
72
73
74
75
76
77
78
79
80
    if os.path.isdir(path):
        contents = os.listdir(path)
        for file_name in contents:
            file_path = os.path.join(path, file_name)
            if os.path.isfile(file_path):
                os.remove(file_path)
            else:
                shutil.rmtree(file_path)


def silent_call(cmd, raise_error=False, error_msg=''):
    try:
81
        output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
82
        with open(LOG_PATH, "ab") as log:
83
84
85
86
            log.write(output)
        return 0
    except Exception as err:
        if isinstance(err, subprocess.CalledProcessError):
87
            with open(LOG_PATH, "ab") as log:
88
                log.write(err.output)
89
        if raise_error:
90
            raise Exception("\n".join((error_msg, LOG_NOTICE)))
91
        return 1
92
93


94
def compile_cpp(use_mingw=False, use_gpu=False, use_cuda=False, use_mpi=False,
95
96
                use_hdfs=False, boost_root=None, boost_dir=None,
                boost_include_dir=None, boost_librarydir=None,
97
                opencl_include_dir=None, opencl_library=None,
98
                nomp=False, bit32=False, integrated_opencl=False):
99

100
101
102
103
    if os.path.exists(os.path.join(CURRENT_DIR, "build_cpp")):
        shutil.rmtree(os.path.join(CURRENT_DIR, "build_cpp"))
    os.makedirs(os.path.join(CURRENT_DIR, "build_cpp"))
    os.chdir(os.path.join(CURRENT_DIR, "build_cpp"))
104

105
106
    logger.info("Starting to compile the library.")

107
    cmake_cmd = ["cmake", "../compile/"]
108
109
110
    if integrated_opencl:
        use_gpu = False
        cmake_cmd.append("-D__INTEGRATE_OPENCL=ON")
111
    if use_gpu:
112
        cmake_cmd.append("-DUSE_GPU=ON")
113
114
115
116
117
118
119
120
121
122
123
124
        if boost_root:
            cmake_cmd.append("-DBOOST_ROOT={0}".format(boost_root))
        if boost_dir:
            cmake_cmd.append("-DBoost_DIR={0}".format(boost_dir))
        if boost_include_dir:
            cmake_cmd.append("-DBoost_INCLUDE_DIR={0}".format(boost_include_dir))
        if boost_librarydir:
            cmake_cmd.append("-DBOOST_LIBRARYDIR={0}".format(boost_librarydir))
        if opencl_include_dir:
            cmake_cmd.append("-DOpenCL_INCLUDE_DIR={0}".format(opencl_include_dir))
        if opencl_library:
            cmake_cmd.append("-DOpenCL_LIBRARY={0}".format(opencl_library))
125
126
    elif use_cuda:
        cmake_cmd.append("-DUSE_CUDA=ON")
127
128
    if use_mpi:
        cmake_cmd.append("-DUSE_MPI=ON")
129
130
    if nomp:
        cmake_cmd.append("-DUSE_OPENMP=OFF")
131
132
    if use_hdfs:
        cmake_cmd.append("-DUSE_HDFS=ON")
133

134
    if system() in {'Windows', 'Microsoft'}:
135
        if use_mingw:
136
137
            if use_mpi:
                raise Exception('MPI version cannot be compiled by MinGW due to the miss of MPI library in it')
138
139
            logger.info("Starting to compile with CMake and MinGW.")
            silent_call(cmake_cmd + ["-G", "MinGW Makefiles"], raise_error=True,
140
                        error_msg='Please install CMake and all required dependencies first')
141
142
            silent_call(["mingw32-make.exe", "_lightgbm"], raise_error=True,
                        error_msg='Please install MinGW first')
143
        else:
144
            status = 1
145
            lib_path = os.path.join(CURRENT_DIR, "compile", "windows", "x64", "DLL", "lib_lightgbm.dll")
146
            if not any((use_gpu, use_mpi, use_hdfs, nomp, bit32, integrated_opencl)):
147
                logger.info("Starting to compile with MSBuild from existing solution file.")
148
                platform_toolsets = ("v142", "v141", "v140")
149
                for pt in platform_toolsets:
150
151
                    status = silent_call(["MSBuild",
                                          os.path.join(CURRENT_DIR, "compile", "windows", "LightGBM.sln"),
152
153
154
155
156
157
                                          "/p:Configuration=DLL",
                                          "/p:Platform=x64",
                                          "/p:PlatformToolset={0}".format(pt)])
                    if status == 0 and os.path.exists(lib_path):
                        break
                    else:
158
                        clear_path(os.path.join(CURRENT_DIR, "compile", "windows", "x64"))
159
160
161
                if status != 0 or not os.path.exists(lib_path):
                    logger.warning("Compilation with MSBuild from existing solution file failed.")
            if status != 0 or not os.path.exists(lib_path):
162
                arch = "Win32" if bit32 else "x64"
163
                vs_versions = ("Visual Studio 16 2019", "Visual Studio 15 2017", "Visual Studio 14 2015")
164
165
                for vs in vs_versions:
                    logger.info("Starting to compile with %s." % vs)
166
                    status = silent_call(cmake_cmd + ["-G", vs, "-A", arch])
167
168
169
                    if status == 0:
                        break
                    else:
170
                        clear_path(os.path.join(CURRENT_DIR, "build_cpp"))
171
                if status != 0:
172
                    raise Exception("\n".join(('Please install Visual Studio or MS Build and all required dependencies first',
173
                                    LOG_NOTICE)))
174
175
                silent_call(["cmake", "--build", ".", "--target", "_lightgbm", "--config", "Release"], raise_error=True,
                            error_msg='Please install CMake first')
176
    else:  # Linux, Darwin (macOS), etc.
177
        logger.info("Starting to compile with CMake.")
178
179
180
        silent_call(cmake_cmd, raise_error=True, error_msg='Please install CMake and all required dependencies first')
        silent_call(["make", "_lightgbm", "-j4"], raise_error=True,
                    error_msg='An error has occurred while building lightgbm library file')
181
    os.chdir(CURRENT_DIR)
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197


class CustomInstallLib(install_lib):

    def install(self):
        outfiles = install_lib.install(self)
        src = find_lib()[0]
        dst = os.path.join(self.install_dir, 'lightgbm')
        dst, _ = self.copy_file(src, dst)
        outfiles.append(dst)
        return outfiles


class CustomInstall(install):

    user_options = install.user_options + [
198
        ('mingw', 'm', 'Compile with MinGW'),
199
        ('integrated-opencl', None, 'Compile integrated OpenCL version'),
200
        ('gpu', 'g', 'Compile GPU version'),
201
        ('cuda', None, 'Compile CUDA version'),
202
        ('mpi', None, 'Compile MPI version'),
203
        ('nomp', None, 'Compile version without OpenMP support'),
204
        ('hdfs', 'h', 'Compile HDFS version'),
205
        ('bit32', None, 'Compile 32-bit version'),
206
207
208
209
210
211
        ('precompile', 'p', 'Use precompiled library'),
        ('boost-root=', None, 'Boost preferred installation prefix'),
        ('boost-dir=', None, 'Directory with Boost package configuration file'),
        ('boost-include-dir=', None, 'Directory containing Boost headers'),
        ('boost-librarydir=', None, 'Preferred Boost library directory'),
        ('opencl-include-dir=', None, 'OpenCL include directory'),
212
        ('opencl-library=', None, 'Path to OpenCL library')
213
214
215
216
217
    ]

    def initialize_options(self):
        install.initialize_options(self)
        self.mingw = 0
218
        self.integrated_opencl = 0
219
        self.gpu = 0
220
        self.cuda = 0
221
222
223
224
225
226
227
        self.boost_root = None
        self.boost_dir = None
        self.boost_include_dir = None
        self.boost_librarydir = None
        self.opencl_include_dir = None
        self.opencl_library = None
        self.mpi = 0
228
        self.hdfs = 0
229
        self.precompile = 0
230
        self.nomp = 0
231
        self.bit32 = 0
232
233

    def run(self):
234
235
236
237
238
239
240
        if (8 * struct.calcsize("P")) != 64:
            if self.bit32:
                logger.warning("You're installing 32-bit version. "
                               "This version is slow and untested, so use it on your own risk.")
            else:
                raise Exception("Cannot install LightGBM in 32-bit Python, "
                                "please use 64-bit Python instead.")
241
        open(LOG_PATH, 'wb').close()
242
        if not self.precompile:
243
            copy_files(integrated_opencl=self.integrated_opencl, use_gpu=self.gpu)
244
            compile_cpp(use_mingw=self.mingw, use_gpu=self.gpu, use_cuda=self.cuda, use_mpi=self.mpi,
245
                        use_hdfs=self.hdfs, boost_root=self.boost_root, boost_dir=self.boost_dir,
246
                        boost_include_dir=self.boost_include_dir, boost_librarydir=self.boost_librarydir,
247
                        opencl_include_dir=self.opencl_include_dir, opencl_library=self.opencl_library,
248
                        nomp=self.nomp, bit32=self.bit32, integrated_opencl=self.integrated_opencl)
249
        install.run(self)
250
251
        if os.path.isfile(LOG_PATH):
            os.remove(LOG_PATH)
252
253
254
255
256


class CustomSdist(sdist):

    def run(self):
257
        copy_files(integrated_opencl=True, use_gpu=True)
258
259
260
261
262
263
264
        open(os.path.join(CURRENT_DIR, '_IS_SOURCE_PACKAGE.txt'), 'w').close()
        if os.path.exists(os.path.join(CURRENT_DIR, 'lightgbm', 'Release')):
            shutil.rmtree(os.path.join(CURRENT_DIR, 'lightgbm', 'Release'))
        if os.path.exists(os.path.join(CURRENT_DIR, 'lightgbm', 'windows', 'x64')):
            shutil.rmtree(os.path.join(CURRENT_DIR, 'lightgbm', 'windows', 'x64'))
        if os.path.isfile(os.path.join(CURRENT_DIR, 'lightgbm', 'lib_lightgbm.so')):
            os.remove(os.path.join(CURRENT_DIR, 'lightgbm', 'lib_lightgbm.so'))
265
        sdist.run(self)
266
267
        if os.path.isfile(os.path.join(CURRENT_DIR, '_IS_SOURCE_PACKAGE.txt')):
            os.remove(os.path.join(CURRENT_DIR, '_IS_SOURCE_PACKAGE.txt'))
268
269
270


if __name__ == "__main__":
271
272
273
274
    CURRENT_DIR = os.path.abspath(os.path.dirname(__file__))
    LOG_PATH = os.path.join(os.path.expanduser('~'), 'LightGBM_compilation.log')
    LOG_NOTICE = "The full version of error log was saved into {0}".format(LOG_PATH)
    if os.path.isfile(os.path.join(CURRENT_DIR, os.path.pardir, 'VERSION.txt')):
275
276
277
        copy_file(os.path.join(CURRENT_DIR, os.path.pardir, 'VERSION.txt'),
                  os.path.join(CURRENT_DIR, 'lightgbm', 'VERSION.txt'),
                  verbose=0)
278
279
    version = io.open(os.path.join(CURRENT_DIR, 'lightgbm', 'VERSION.txt'), encoding='utf-8').read().strip()
    readme = io.open(os.path.join(CURRENT_DIR, 'README.rst'), encoding='utf-8').read()
280

281
    sys.path.insert(0, CURRENT_DIR)
282

283
284
285
    logging.basicConfig(level=logging.INFO)
    logger = logging.getLogger('LightGBM')

Guolin Ke's avatar
Guolin Ke committed
286
287
288
    setup(name='lightgbm',
          version=version,
          description='LightGBM Python Package',
289
          long_description=readme,
Guolin Ke's avatar
Guolin Ke committed
290
291
292
          install_requires=[
              'numpy',
              'scipy',
293
              'scikit-learn!=0.22.0'
Guolin Ke's avatar
Guolin Ke committed
294
295
296
297
          ],
          maintainer='Guolin Ke',
          maintainer_email='guolin.ke@microsoft.com',
          zip_safe=False,
298
299
300
301
302
          cmdclass={
              'install': CustomInstall,
              'install_lib': CustomInstallLib,
              'sdist': CustomSdist,
          },
Guolin Ke's avatar
Guolin Ke committed
303
304
          packages=find_packages(),
          include_package_data=True,
305
          license='The MIT License (Microsoft)',
306
          url='https://github.com/microsoft/LightGBM',
307
308
309
310
311
312
313
314
315
316
317
318
          classifiers=['Development Status :: 5 - Production/Stable',
                       'Intended Audience :: Science/Research',
                       'License :: OSI Approved :: MIT License',
                       'Natural Language :: English',
                       'Operating System :: MacOS',
                       'Operating System :: Microsoft :: Windows',
                       'Operating System :: POSIX',
                       'Operating System :: Unix',
                       'Programming Language :: Python :: 2',
                       'Programming Language :: Python :: 2.7',
                       'Programming Language :: Python :: 3',
                       'Programming Language :: Python :: 3.6',
319
                       'Programming Language :: Python :: 3.7',
320
                       'Programming Language :: Python :: 3.8',
321
                       'Topic :: Scientific/Engineering :: Artificial Intelligence'])