setup.py 11.6 KB
Newer Older
wxchan's avatar
wxchan committed
1
# coding: utf-8
2
# pylint: disable=invalid-name, exec-used, C0111
wxchan's avatar
wxchan committed
3
4
"""Setup lightgbm package."""
from __future__ import absolute_import
5

Guolin Ke's avatar
Guolin Ke committed
6
import distutils
7
import logging
8
import os
Guolin Ke's avatar
Guolin Ke committed
9
import shutil
10
import struct
11
import subprocess
12
13
import sys

14
from setuptools import find_packages, setup
15
16
17
from setuptools.command.install import install
from setuptools.command.install_lib import install_lib
from setuptools.command.sdist import sdist
18

19
20
21
22
23
24
25
26

def find_lib():
    CURRENT_DIR = os.path.dirname(__file__)
    libpath_py = os.path.join(CURRENT_DIR, 'lightgbm/libpath.py')
    libpath = {'__file__': libpath_py}
    exec(compile(open(libpath_py, "rb").read(), libpath_py, 'exec'), libpath, libpath)

    LIB_PATH = [os.path.relpath(path, CURRENT_DIR) for path in libpath['find_lib_path']()]
27
    logging.info("Installing lib_lightgbm from: %s" % LIB_PATH)
28
29
30
31
32
33
34
35
    return LIB_PATH


def copy_files(use_gpu=False):

    def copy_files_helper(folder_name):
        src = os.path.join('..', folder_name)
        if os.path.exists(src):
36
            dst = os.path.join('./compile', folder_name)
37
38
39
40
41
42
43
44
            shutil.rmtree(dst, ignore_errors=True)
            distutils.dir_util.copy_tree(src, dst)
        else:
            raise Exception('Cannot copy {} folder'.format(src))

    if not os.path.isfile('./_IS_SOURCE_PACKAGE.txt'):
        copy_files_helper('include')
        copy_files_helper('src')
45
46
47
48
        if not os.path.exists("./compile/windows/"):
            os.makedirs("./compile/windows/")
        distutils.file_util.copy_file("../windows/LightGBM.sln", "./compile/windows/LightGBM.sln")
        distutils.file_util.copy_file("../windows/LightGBM.vcxproj", "./compile/windows/LightGBM.vcxproj")
Guolin Ke's avatar
Guolin Ke committed
49
        if use_gpu:
50
            copy_files_helper('compute')
51
        distutils.file_util.copy_file("../CMakeLists.txt", "./compile/")
52
        distutils.file_util.copy_file("../LICENSE", "./")
wxchan's avatar
wxchan committed
53
54


55
def clear_path(path):
56
57
58
59
60
61
62
63
64
65
66
67
    if os.path.isdir(path):
        contents = os.listdir(path)
        for file_name in contents:
            file_path = os.path.join(path, file_name)
            if os.path.isfile(file_path):
                os.remove(file_path)
            else:
                shutil.rmtree(file_path)


def silent_call(cmd, raise_error=False, error_msg=''):
    try:
68
69
70
71
72
73
74
75
        output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
        with open(path_log, "ab") as log:
            log.write(output)
        return 0
    except Exception as err:
        if isinstance(err, subprocess.CalledProcessError):
            with open(path_log, "ab") as log:
                log.write(err.output)
76
        if raise_error:
77
            raise Exception("\n".join((error_msg, log_notice)))
78
        return 1
79
80


81
def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False, use_hdfs=False,
82
83
84
                boost_root=None, boost_dir=None, boost_include_dir=None,
                boost_librarydir=None, opencl_include_dir=None,
                opencl_library=None):
85

Guolin Ke's avatar
Guolin Ke committed
86
87
88
89
    if os.path.exists("build_cpp"):
        shutil.rmtree("build_cpp")
    os.makedirs("build_cpp")
    os.chdir("build_cpp")
90

91
92
    logger.info("Starting to compile the library.")

93
    cmake_cmd = ["cmake", "../compile/"]
94
    if use_gpu:
95
        cmake_cmd.append("-DUSE_GPU=ON")
96
97
98
99
100
101
102
103
104
105
106
107
108
109
        if boost_root:
            cmake_cmd.append("-DBOOST_ROOT={0}".format(boost_root))
        if boost_dir:
            cmake_cmd.append("-DBoost_DIR={0}".format(boost_dir))
        if boost_include_dir:
            cmake_cmd.append("-DBoost_INCLUDE_DIR={0}".format(boost_include_dir))
        if boost_librarydir:
            cmake_cmd.append("-DBOOST_LIBRARYDIR={0}".format(boost_librarydir))
        if opencl_include_dir:
            cmake_cmd.append("-DOpenCL_INCLUDE_DIR={0}".format(opencl_include_dir))
        if opencl_library:
            cmake_cmd.append("-DOpenCL_LIBRARY={0}".format(opencl_library))
    if use_mpi:
        cmake_cmd.append("-DUSE_MPI=ON")
110
111
    if use_hdfs:
        cmake_cmd.append("-DUSE_HDFS=ON")
112
113
    if os.name == "nt":
        if use_mingw:
114
115
            if use_mpi:
                raise Exception('MPI version cannot be compiled by MinGW due to the miss of MPI library in it')
116
117
            logger.info("Starting to compile with CMake and MinGW.")
            silent_call(cmake_cmd + ["-G", "MinGW Makefiles"], raise_error=True,
118
                        error_msg='Please install CMake and all required dependencies first')
119
120
            silent_call(["mingw32-make.exe", "_lightgbm"], raise_error=True,
                        error_msg='Please install MinGW first')
121
        else:
122
            status = 1
123
            lib_path = "../compile/windows/x64/DLL/lib_lightgbm.dll"
124
            if not use_gpu and not use_hdfs:
125
                logger.info("Starting to compile with MSBuild from existing solution file.")
126
                platform_toolsets = ("v141", "v140")
127
                for pt in platform_toolsets:
128
                    status = silent_call(["MSBuild", "../compile/windows/LightGBM.sln",
129
130
131
132
133
134
                                          "/p:Configuration=DLL",
                                          "/p:Platform=x64",
                                          "/p:PlatformToolset={0}".format(pt)])
                    if status == 0 and os.path.exists(lib_path):
                        break
                    else:
135
                        clear_path("../compile/windows/x64")
136
137
138
                if status != 0 or not os.path.exists(lib_path):
                    logger.warning("Compilation with MSBuild from existing solution file failed.")
            if status != 0 or not os.path.exists(lib_path):
139
                vs_versions = ("Visual Studio 15 2017 Win64", "Visual Studio 14 2015 Win64")
140
141
142
143
144
145
146
147
                for vs in vs_versions:
                    logger.info("Starting to compile with %s." % vs)
                    status = silent_call(cmake_cmd + ["-G", vs])
                    if status == 0:
                        break
                    else:
                        clear_path("./")
                if status != 0:
148
149
                    raise Exception("\n".join(('Please install Visual Studio or MS Build and all required dependencies first',
                                    log_notice)))
150
151
                silent_call(["cmake", "--build", ".", "--target", "_lightgbm", "--config", "Release"], raise_error=True,
                            error_msg='Please install CMake first')
152
    else:  # Linux, Darwin (macOS), etc.
153
        logger.info("Starting to compile with CMake.")
154
        silent_call(cmake_cmd, raise_error=True, error_msg='Please install CMake and all required dependencies first')
155
156
        silent_call(["make", "_lightgbm"], raise_error=True,
                    error_msg='An error has occurred while building lightgbm library file')
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
    os.chdir("..")


class CustomInstallLib(install_lib):

    def install(self):
        outfiles = install_lib.install(self)
        src = find_lib()[0]
        dst = os.path.join(self.install_dir, 'lightgbm')
        dst, _ = self.copy_file(src, dst)
        outfiles.append(dst)
        return outfiles


class CustomInstall(install):

    user_options = install.user_options + [
174
175
176
        ('mingw', 'm', 'Compile with MinGW'),
        ('gpu', 'g', 'Compile GPU version'),
        ('mpi', None, 'Compile MPI version'),
177
        ('hdfs', 'h', 'Compile HDFS version'),
178
179
180
181
182
183
184
        ('precompile', 'p', 'Use precompiled library'),
        ('boost-root=', None, 'Boost preferred installation prefix'),
        ('boost-dir=', None, 'Directory with Boost package configuration file'),
        ('boost-include-dir=', None, 'Directory containing Boost headers'),
        ('boost-librarydir=', None, 'Preferred Boost library directory'),
        ('opencl-include-dir=', None, 'OpenCL include directory'),
        ('opencl-library=', None, 'Path to OpenCL library')
185
186
187
188
189
190
    ]

    def initialize_options(self):
        install.initialize_options(self)
        self.mingw = 0
        self.gpu = 0
191
192
193
194
195
196
197
        self.boost_root = None
        self.boost_dir = None
        self.boost_include_dir = None
        self.boost_librarydir = None
        self.opencl_include_dir = None
        self.opencl_library = None
        self.mpi = 0
198
        self.hdfs = 0
199
200
201
        self.precompile = 0

    def run(self):
202
        open(path_log, 'wb').close()
203
204
        if not self.precompile:
            copy_files(use_gpu=self.gpu)
205
            compile_cpp(use_mingw=self.mingw, use_gpu=self.gpu, use_mpi=self.mpi, use_hdfs=self.hdfs,
206
207
208
                        boost_root=self.boost_root, boost_dir=self.boost_dir,
                        boost_include_dir=self.boost_include_dir, boost_librarydir=self.boost_librarydir,
                        opencl_include_dir=self.opencl_include_dir, opencl_library=self.opencl_library)
209
        install.run(self)
210
211
        if os.path.isfile(path_log):
            os.remove(path_log)
212
213
214
215
216


class CustomSdist(sdist):

    def run(self):
Guolin Ke's avatar
Guolin Ke committed
217
        copy_files(use_gpu=True)
218
        open("./_IS_SOURCE_PACKAGE.txt", 'w').close()
Guolin Ke's avatar
Guolin Ke committed
219
220
        if os.path.exists("./lightgbm/Release/"):
            shutil.rmtree('./lightgbm/Release/')
221
222
        if os.path.exists("./lightgbm/windows/x64/"):
            shutil.rmtree('./lightgbm/windows/x64/')
Guolin Ke's avatar
Guolin Ke committed
223
224
        if os.path.isfile('./lightgbm/lib_lightgbm.so'):
            os.remove('./lightgbm/lib_lightgbm.so')
225
226
227
228
229
230
231
        sdist.run(self)
        if os.path.isfile('./_IS_SOURCE_PACKAGE.txt'):
            os.remove('./_IS_SOURCE_PACKAGE.txt')


if __name__ == "__main__":
    if (8 * struct.calcsize("P")) != 64:
232
        raise Exception('Cannot install LightGBM in 32-bit Python, please use 64-bit python instead.')
233
234

    dir_path = os.path.dirname(os.path.realpath(__file__))
235
236
    path_log = os.path.join(os.path.expanduser('~'), 'LightGBM_compilation.log')
    log_notice = "The full version of error log was saved into {0}".format(path_log)
237
    if os.path.isfile(os.path.join('..', 'VERSION.txt')):
238
239
        distutils.file_util.copy_file(os.path.join('..', 'VERSION.txt'),
                                      os.path.join('.', 'lightgbm'))
240
    version = open(os.path.join(dir_path, 'lightgbm', 'VERSION.txt')).read().strip()
241

242
    sys.path.insert(0, '.')
243

244
245
246
    logging.basicConfig(level=logging.INFO)
    logger = logging.getLogger('LightGBM')

Guolin Ke's avatar
Guolin Ke committed
247
248
249
    setup(name='lightgbm',
          version=version,
          description='LightGBM Python Package',
Guolin Ke's avatar
Guolin Ke committed
250
          long_description=open('README.rst').read(),
Guolin Ke's avatar
Guolin Ke committed
251
252
253
254
255
256
257
258
          install_requires=[
              'numpy',
              'scipy',
              'scikit-learn'
          ],
          maintainer='Guolin Ke',
          maintainer_email='guolin.ke@microsoft.com',
          zip_safe=False,
259
260
261
262
263
          cmdclass={
              'install': CustomInstall,
              'install_lib': CustomInstallLib,
              'sdist': CustomSdist,
          },
Guolin Ke's avatar
Guolin Ke committed
264
265
          packages=find_packages(),
          include_package_data=True,
266
          license='The MIT License (Microsoft)',
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
          url='https://github.com/Microsoft/LightGBM',
          classifiers=['Development Status :: 5 - Production/Stable',
                       'Intended Audience :: Science/Research',
                       'License :: OSI Approved :: MIT License',
                       'Natural Language :: English',
                       'Operating System :: MacOS',
                       'Operating System :: Microsoft :: Windows',
                       'Operating System :: POSIX',
                       'Operating System :: Unix',
                       'Programming Language :: Python :: 2',
                       'Programming Language :: Python :: 2.7',
                       'Programming Language :: Python :: 3',
                       'Programming Language :: Python :: 3.4',
                       'Programming Language :: Python :: 3.5',
                       'Programming Language :: Python :: 3.6',
                       'Topic :: Scientific/Engineering :: Artificial Intelligence'])