setup.py 15.8 KB
Newer Older
wxchan's avatar
wxchan committed
1
2
# coding: utf-8
"""Setup lightgbm package."""
3
import logging
4
5
import os
import struct
6
import subprocess
7
8
import sys

9
from platform import system
10
from setuptools import find_packages, setup
11
12
13
from setuptools.command.install import install
from setuptools.command.install_lib import install_lib
from setuptools.command.sdist import sdist
14
from distutils.dir_util import copy_tree, create_tree, remove_tree
15
from distutils.file_util import copy_file
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
from wheel.bdist_wheel import bdist_wheel


LIGHTGBM_OPTIONS = [
    ('mingw', 'm', 'Compile with MinGW'),
    ('integrated-opencl', None, 'Compile integrated OpenCL version'),
    ('gpu', 'g', 'Compile GPU version'),
    ('cuda', None, 'Compile CUDA version'),
    ('mpi', None, 'Compile MPI version'),
    ('nomp', None, 'Compile version without OpenMP support'),
    ('hdfs', 'h', 'Compile HDFS version'),
    ('bit32', None, 'Compile 32-bit version'),
    ('precompile', 'p', 'Use precompiled library'),
    ('boost-root=', None, 'Boost preferred installation prefix'),
    ('boost-dir=', None, 'Directory with Boost package configuration file'),
    ('boost-include-dir=', None, 'Directory containing Boost headers'),
    ('boost-librarydir=', None, 'Preferred Boost library directory'),
    ('opencl-include-dir=', None, 'OpenCL include directory'),
    ('opencl-library=', None, 'Path to OpenCL library')
]
36

37
38

def find_lib():
39
    libpath_py = os.path.join(CURRENT_DIR, 'lightgbm', 'libpath.py')
40
41
42
43
    libpath = {'__file__': libpath_py}
    exec(compile(open(libpath_py, "rb").read(), libpath_py, 'exec'), libpath, libpath)

    LIB_PATH = [os.path.relpath(path, CURRENT_DIR) for path in libpath['find_lib_path']()]
44
    logger.info("Installing lib_lightgbm from: %s" % LIB_PATH)
45
46
47
    return LIB_PATH


48
def copy_files(integrated_opencl=False, use_gpu=False):
49
50

    def copy_files_helper(folder_name):
51
        src = os.path.join(CURRENT_DIR, os.path.pardir, folder_name)
52
        if os.path.exists(src):
53
            dst = os.path.join(CURRENT_DIR, 'compile', folder_name)
54
55
56
57
58
59
60
            if os.path.exists(dst):
                if os.path.isdir:
                    # see https://github.com/pypa/distutils/pull/21
                    remove_tree(dst)
                else:
                    os.remove(dst)
            create_tree(src, dst, verbose=0)
61
            copy_tree(src, dst, verbose=0)
62
        else:
63
            raise Exception('Cannot copy {0} folder'.format(src))
64

65
    if not os.path.isfile(os.path.join(CURRENT_DIR, '_IS_SOURCE_PACKAGE.txt')):
66
67
        copy_files_helper('include')
        copy_files_helper('src')
68
        copy_files_helper('external_libs')
69
70
        if not os.path.exists(os.path.join(CURRENT_DIR, "compile", "windows")):
            os.makedirs(os.path.join(CURRENT_DIR, "compile", "windows"))
71
72
73
74
75
76
77
78
79
        copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "windows", "LightGBM.sln"),
                  os.path.join(CURRENT_DIR, "compile", "windows", "LightGBM.sln"),
                  verbose=0)
        copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "windows", "LightGBM.vcxproj"),
                  os.path.join(CURRENT_DIR, "compile", "windows", "LightGBM.vcxproj"),
                  verbose=0)
        copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "LICENSE"),
                  os.path.join(CURRENT_DIR, "LICENSE"),
                  verbose=0)
80
81
82
83
84
85
86
87
88
        copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeLists.txt"),
                  os.path.join(CURRENT_DIR, "compile", "CMakeLists.txt"),
                  verbose=0)
        if integrated_opencl:
            copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeIntegratedOpenCL.cmake"),
                      os.path.join(CURRENT_DIR, "compile", "CMakeIntegratedOpenCL.cmake"),
                      verbose=0)
        if use_gpu:
            copy_files_helper('compute')
wxchan's avatar
wxchan committed
89
90


91
def clear_path(path):
92
93
94
95
96
97
98
    if os.path.isdir(path):
        contents = os.listdir(path)
        for file_name in contents:
            file_path = os.path.join(path, file_name)
            if os.path.isfile(file_path):
                os.remove(file_path)
            else:
99
                remove_tree(file_path)
100
101
102
103


def silent_call(cmd, raise_error=False, error_msg=''):
    try:
104
        output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
105
        with open(LOG_PATH, "ab") as log:
106
107
108
109
            log.write(output)
        return 0
    except Exception as err:
        if isinstance(err, subprocess.CalledProcessError):
110
            with open(LOG_PATH, "ab") as log:
111
                log.write(err.output)
112
        if raise_error:
113
            raise Exception("\n".join((error_msg, LOG_NOTICE)))
114
        return 1
115
116


117
def compile_cpp(use_mingw=False, use_gpu=False, use_cuda=False, use_mpi=False,
118
119
                use_hdfs=False, boost_root=None, boost_dir=None,
                boost_include_dir=None, boost_librarydir=None,
120
                opencl_include_dir=None, opencl_library=None,
121
                nomp=False, bit32=False, integrated_opencl=False):
122

123
    if os.path.exists(os.path.join(CURRENT_DIR, "build_cpp")):
124
        remove_tree(os.path.join(CURRENT_DIR, "build_cpp"))
125
126
    os.makedirs(os.path.join(CURRENT_DIR, "build_cpp"))
    os.chdir(os.path.join(CURRENT_DIR, "build_cpp"))
127

128
129
    logger.info("Starting to compile the library.")

130
    cmake_cmd = ["cmake", "../compile/"]
131
132
133
    if integrated_opencl:
        use_gpu = False
        cmake_cmd.append("-D__INTEGRATE_OPENCL=ON")
134
    if use_gpu:
135
        cmake_cmd.append("-DUSE_GPU=ON")
136
137
138
139
140
141
142
143
144
145
146
147
        if boost_root:
            cmake_cmd.append("-DBOOST_ROOT={0}".format(boost_root))
        if boost_dir:
            cmake_cmd.append("-DBoost_DIR={0}".format(boost_dir))
        if boost_include_dir:
            cmake_cmd.append("-DBoost_INCLUDE_DIR={0}".format(boost_include_dir))
        if boost_librarydir:
            cmake_cmd.append("-DBOOST_LIBRARYDIR={0}".format(boost_librarydir))
        if opencl_include_dir:
            cmake_cmd.append("-DOpenCL_INCLUDE_DIR={0}".format(opencl_include_dir))
        if opencl_library:
            cmake_cmd.append("-DOpenCL_LIBRARY={0}".format(opencl_library))
148
149
    elif use_cuda:
        cmake_cmd.append("-DUSE_CUDA=ON")
150
151
    if use_mpi:
        cmake_cmd.append("-DUSE_MPI=ON")
152
153
    if nomp:
        cmake_cmd.append("-DUSE_OPENMP=OFF")
154
155
    if use_hdfs:
        cmake_cmd.append("-DUSE_HDFS=ON")
156

157
    if system() in {'Windows', 'Microsoft'}:
158
        if use_mingw:
159
160
            if use_mpi:
                raise Exception('MPI version cannot be compiled by MinGW due to the miss of MPI library in it')
161
162
            logger.info("Starting to compile with CMake and MinGW.")
            silent_call(cmake_cmd + ["-G", "MinGW Makefiles"], raise_error=True,
163
                        error_msg='Please install CMake and all required dependencies first')
164
165
            silent_call(["mingw32-make.exe", "_lightgbm"], raise_error=True,
                        error_msg='Please install MinGW first')
166
        else:
167
            status = 1
168
            lib_path = os.path.join(CURRENT_DIR, "compile", "windows", "x64", "DLL", "lib_lightgbm.dll")
169
            if not any((use_gpu, use_mpi, use_hdfs, nomp, bit32, integrated_opencl)):
170
                logger.info("Starting to compile with MSBuild from existing solution file.")
171
                platform_toolsets = ("v142", "v141", "v140")
172
                for pt in platform_toolsets:
173
174
                    status = silent_call(["MSBuild",
                                          os.path.join(CURRENT_DIR, "compile", "windows", "LightGBM.sln"),
175
176
177
178
179
180
                                          "/p:Configuration=DLL",
                                          "/p:Platform=x64",
                                          "/p:PlatformToolset={0}".format(pt)])
                    if status == 0 and os.path.exists(lib_path):
                        break
                    else:
181
                        clear_path(os.path.join(CURRENT_DIR, "compile", "windows", "x64"))
182
183
184
                if status != 0 or not os.path.exists(lib_path):
                    logger.warning("Compilation with MSBuild from existing solution file failed.")
            if status != 0 or not os.path.exists(lib_path):
185
                arch = "Win32" if bit32 else "x64"
186
                vs_versions = ("Visual Studio 16 2019", "Visual Studio 15 2017", "Visual Studio 14 2015")
187
188
                for vs in vs_versions:
                    logger.info("Starting to compile with %s." % vs)
189
                    status = silent_call(cmake_cmd + ["-G", vs, "-A", arch])
190
191
192
                    if status == 0:
                        break
                    else:
193
                        clear_path(os.path.join(CURRENT_DIR, "build_cpp"))
194
                if status != 0:
195
                    raise Exception("\n".join(('Please install Visual Studio or MS Build and all required dependencies first',
196
                                    LOG_NOTICE)))
197
198
                silent_call(["cmake", "--build", ".", "--target", "_lightgbm", "--config", "Release"], raise_error=True,
                            error_msg='Please install CMake first')
199
    else:  # Linux, Darwin (macOS), etc.
200
        logger.info("Starting to compile with CMake.")
201
202
203
        silent_call(cmake_cmd, raise_error=True, error_msg='Please install CMake and all required dependencies first')
        silent_call(["make", "_lightgbm", "-j4"], raise_error=True,
                    error_msg='An error has occurred while building lightgbm library file')
204
    os.chdir(CURRENT_DIR)
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219


class CustomInstallLib(install_lib):

    def install(self):
        outfiles = install_lib.install(self)
        src = find_lib()[0]
        dst = os.path.join(self.install_dir, 'lightgbm')
        dst, _ = self.copy_file(src, dst)
        outfiles.append(dst)
        return outfiles


class CustomInstall(install):

220
    user_options = install.user_options + LIGHTGBM_OPTIONS
221
222
223
224

    def initialize_options(self):
        install.initialize_options(self)
        self.mingw = 0
225
        self.integrated_opencl = 0
226
        self.gpu = 0
227
        self.cuda = 0
228
229
230
231
232
233
234
        self.boost_root = None
        self.boost_dir = None
        self.boost_include_dir = None
        self.boost_librarydir = None
        self.opencl_include_dir = None
        self.opencl_library = None
        self.mpi = 0
235
        self.hdfs = 0
236
        self.precompile = 0
237
        self.nomp = 0
238
        self.bit32 = 0
239
240

    def run(self):
241
242
243
244
245
246
247
        if (8 * struct.calcsize("P")) != 64:
            if self.bit32:
                logger.warning("You're installing 32-bit version. "
                               "This version is slow and untested, so use it on your own risk.")
            else:
                raise Exception("Cannot install LightGBM in 32-bit Python, "
                                "please use 64-bit Python instead.")
248
        open(LOG_PATH, 'wb').close()
249
        if not self.precompile:
250
            copy_files(integrated_opencl=self.integrated_opencl, use_gpu=self.gpu)
251
            compile_cpp(use_mingw=self.mingw, use_gpu=self.gpu, use_cuda=self.cuda, use_mpi=self.mpi,
252
                        use_hdfs=self.hdfs, boost_root=self.boost_root, boost_dir=self.boost_dir,
253
                        boost_include_dir=self.boost_include_dir, boost_librarydir=self.boost_librarydir,
254
                        opencl_include_dir=self.opencl_include_dir, opencl_library=self.opencl_library,
255
                        nomp=self.nomp, bit32=self.bit32, integrated_opencl=self.integrated_opencl)
256
        install.run(self)
257
258
        if os.path.isfile(LOG_PATH):
            os.remove(LOG_PATH)
259
260


261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
class CustomBdistWheel(bdist_wheel):

    user_options = bdist_wheel.user_options + LIGHTGBM_OPTIONS

    def initialize_options(self):
        bdist_wheel.initialize_options(self)
        self.mingw = 0
        self.integrated_opencl = 0
        self.gpu = 0
        self.cuda = 0
        self.boost_root = None
        self.boost_dir = None
        self.boost_include_dir = None
        self.boost_librarydir = None
        self.opencl_include_dir = None
        self.opencl_library = None
        self.mpi = 0
        self.hdfs = 0
        self.precompile = 0
        self.nomp = 0
        self.bit32 = 0

    def finalize_options(self):
        bdist_wheel.finalize_options(self)

        install = self.reinitialize_command('install')

        install.mingw = self.mingw
        install.integrated_opencl = self.integrated_opencl
        install.gpu = self.gpu
        install.cuda = self.cuda
        install.boost_root = self.boost_root
        install.boost_dir = self.boost_dir
        install.boost_include_dir = self.boost_include_dir
        install.boost_librarydir = self.boost_librarydir
        install.opencl_include_dir = self.opencl_include_dir
        install.opencl_library = self.opencl_library
        install.mpi = self.mpi
        install.hdfs = self.hdfs
        install.precompile = self.precompile
        install.nomp = self.nomp
        install.bit32 = self.bit32


305
306
307
class CustomSdist(sdist):

    def run(self):
308
        copy_files(integrated_opencl=True, use_gpu=True)
309
310
        open(os.path.join(CURRENT_DIR, '_IS_SOURCE_PACKAGE.txt'), 'w').close()
        if os.path.exists(os.path.join(CURRENT_DIR, 'lightgbm', 'Release')):
311
            remove_tree(os.path.join(CURRENT_DIR, 'lightgbm', 'Release'))
312
        if os.path.exists(os.path.join(CURRENT_DIR, 'lightgbm', 'windows', 'x64')):
313
            remove_tree(os.path.join(CURRENT_DIR, 'lightgbm', 'windows', 'x64'))
314
315
        if os.path.isfile(os.path.join(CURRENT_DIR, 'lightgbm', 'lib_lightgbm.so')):
            os.remove(os.path.join(CURRENT_DIR, 'lightgbm', 'lib_lightgbm.so'))
316
        sdist.run(self)
317
318
        if os.path.isfile(os.path.join(CURRENT_DIR, '_IS_SOURCE_PACKAGE.txt')):
            os.remove(os.path.join(CURRENT_DIR, '_IS_SOURCE_PACKAGE.txt'))
319
320
321


if __name__ == "__main__":
322
323
324
325
    CURRENT_DIR = os.path.abspath(os.path.dirname(__file__))
    LOG_PATH = os.path.join(os.path.expanduser('~'), 'LightGBM_compilation.log')
    LOG_NOTICE = "The full version of error log was saved into {0}".format(LOG_PATH)
    if os.path.isfile(os.path.join(CURRENT_DIR, os.path.pardir, 'VERSION.txt')):
326
327
328
        copy_file(os.path.join(CURRENT_DIR, os.path.pardir, 'VERSION.txt'),
                  os.path.join(CURRENT_DIR, 'lightgbm', 'VERSION.txt'),
                  verbose=0)
329
330
    version = open(os.path.join(CURRENT_DIR, 'lightgbm', 'VERSION.txt'), encoding='utf-8').read().strip()
    readme = open(os.path.join(CURRENT_DIR, 'README.rst'), encoding='utf-8').read()
331

332
    sys.path.insert(0, CURRENT_DIR)
333

334
335
336
    logging.basicConfig(level=logging.INFO)
    logger = logging.getLogger('LightGBM')

Guolin Ke's avatar
Guolin Ke committed
337
338
339
    setup(name='lightgbm',
          version=version,
          description='LightGBM Python Package',
340
          long_description=readme,
Guolin Ke's avatar
Guolin Ke committed
341
          install_requires=[
342
              'wheel',
Guolin Ke's avatar
Guolin Ke committed
343
344
              'numpy',
              'scipy',
345
              'scikit-learn!=0.22.0'
Guolin Ke's avatar
Guolin Ke committed
346
347
348
349
          ],
          maintainer='Guolin Ke',
          maintainer_email='guolin.ke@microsoft.com',
          zip_safe=False,
350
351
352
          cmdclass={
              'install': CustomInstall,
              'install_lib': CustomInstallLib,
353
              'bdist_wheel': CustomBdistWheel,
354
355
              'sdist': CustomSdist,
          },
Guolin Ke's avatar
Guolin Ke committed
356
357
          packages=find_packages(),
          include_package_data=True,
358
          license='The MIT License (Microsoft)',
359
          url='https://github.com/microsoft/LightGBM',
360
361
362
363
364
365
366
367
368
369
          classifiers=['Development Status :: 5 - Production/Stable',
                       'Intended Audience :: Science/Research',
                       'License :: OSI Approved :: MIT License',
                       'Natural Language :: English',
                       'Operating System :: MacOS',
                       'Operating System :: Microsoft :: Windows',
                       'Operating System :: POSIX',
                       'Operating System :: Unix',
                       'Programming Language :: Python :: 3',
                       'Programming Language :: Python :: 3.6',
370
                       'Programming Language :: Python :: 3.7',
371
                       'Programming Language :: Python :: 3.8',
372
                       'Topic :: Scientific/Engineering :: Artificial Intelligence'])