[python] Submit to PyPI (#635)

* add make command to the python package.

* Update README.rst

* Update README.rst

* Update README.rst

* fix tests.

* fix unix build

* update readme

* fix setup.py

* update travis

* Update .travis.yml

* Update test.py

* some fixes.

* check the 64-bit python

* fix build.

* refine MANIFEST.in

* update Manifest.in

* add more build options.

* Add fatal in cmake

* fix a endif.

* fix bugs.

* fix pep8

* add test for the pip package build

* add test pip install in travis.

* fix version with pre-compile dll

* fix readme.rst

* update readme

.travis.yml

@@ -36,24 +36,30 @@ install:
 
 script:
 - cd $TRAVIS_BUILD_DIR
-- mkdir build && cd build && cmake -DUSE_MPI=ON ..&& make
-- cd $TRAVIS_BUILD_DIR/python-package && python setup.py install
+- mkdir build && cd build && cmake -DUSE_MPI=ON -DBUILD_EXE=OFF -DBUILD_LIB=ON ..&& make
+- cd $TRAVIS_BUILD_DIR/python-package && python setup.py install --precompile
 - cd $TRAVIS_BUILD_DIR && pytest tests/c_api_test/test.py
 - cd $TRAVIS_BUILD_DIR && pytest tests/python_package_test
 - cd $TRAVIS_BUILD_DIR && pep8 --ignore=E501 --exclude=./compute,./docs .
 - cd $TRAVIS_BUILD_DIR
-- rm -rf build && mkdir build && cd build && cmake -DUSE_GPU=ON -DBOOST_ROOT="$HOME/miniconda/" -DOpenCL_INCLUDE_DIR=$AMDAPPSDK/include/ ..
+- rm -rf build && mkdir build && cd build && cmake -DBUILD_EXE=OFF -DBUILD_LIB=ON -DUSE_GPU=ON -DBOOST_ROOT="$HOME/miniconda/" -DOpenCL_INCLUDE_DIR=$AMDAPPSDK/include/ ..
 - sed -i 's/std::string device_type = "cpu";/std::string device_type = "gpu";/' ../include/LightGBM/config.h
 - make
 - sed -i 's/std::string device_type = "gpu";/std::string device_type = "cpu";/' ../include/LightGBM/config.h
-- cd $TRAVIS_BUILD_DIR/python-package && python setup.py install
+- cd $TRAVIS_BUILD_DIR/python-package && python setup.py install --precompile
 - cd $TRAVIS_BUILD_DIR && pytest tests/c_api_test/test.py
 - cd $TRAVIS_BUILD_DIR && pytest tests/python_package_test
 - cd $TRAVIS_BUILD_DIR
-- rm -rf build && mkdir build && cd build && cmake .. && make
+- rm -rf build && rm -f lib_lightgbm.so
 - cd $TRAVIS_BUILD_DIR/python-package && python setup.py install
-- cd $TRAVIS_BUILD_DIR && pytest tests/c_api_test/test.py
 - cd $TRAVIS_BUILD_DIR && pytest tests/python_package_test
+- cd $TRAVIS_BUILD_DIR
+- LGB_VER=$(head -n 1 VERSION.txt)
+- cd $TRAVIS_BUILD_DIR/python-package && python setup.py sdist
+- cd $TRAVIS_BUILD_DIR/python-package/dist && pip install lightgbm-$LGB_VER.tar.gz -v
+- cd $TRAVIS_BUILD_DIR && pytest tests/python_package_test
+- cd $TRAVIS_BUILD_DIR
+- rm -rf build && mkdir build && cd build && cmake .. && make
 - cd $TRAVIS_BUILD_DIR/tests/cpp_test && ../../lightgbm config=train.conf && ../../lightgbm config=predict.conf output_result=origin.pred
 - cd $TRAVIS_BUILD_DIR/build && make
 - cd $TRAVIS_BUILD_DIR/tests/cpp_test && ../../lightgbm config=predict.conf output_result=ifelse.pred && python test.py

CMakeLists.txt

@@ -10,6 +10,12 @@ PROJECT(lightgbm)
 OPTION(USE_MPI "MPI based parallel learning" OFF)
 OPTION(USE_OPENMP "Enable OpenMP" ON)
 OPTION(USE_GPU "Enable GPU-acclerated training (EXPERIMENTAL)" OFF)
+OPTION(BUILD_EXE "Build execute program" ON)
+OPTION(BUILD_LIB "Build dynamic library" OFF)
+
+if(NOT BUILD_EXE AND NOT BUILD_LIB)
+    message( FATAL_ERROR "You cannot disable BUILD_EXE and BUILD_LIB at the same time." )
+endif()
 
 if(APPLE)
     OPTION(APPLE_OUTPUT_DYLIB "Output dylib shared library" OFF)
@@ -107,33 +113,51 @@ file(GLOB SOURCES
     src/treelearner/*.cpp
 )
 
-add_executable(lightgbm src/main.cpp ${SOURCES})
-add_library(_lightgbm SHARED src/c_api.cpp src/lightgbm_R.cpp ${SOURCES})
+if(BUILD_EXE)
+    add_executable(lightgbm src/main.cpp ${SOURCES})
+endif()
+if(BUILD_LIB)
+    add_library(_lightgbm SHARED src/c_api.cpp src/lightgbm_R.cpp ${SOURCES})
+endif()
 
-if(MSVC)
+if(MSVC AND BUILD_LIB)
     set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm")
-endif(MSVC)
+endif()
 
 if(USE_MPI)
-  TARGET_LINK_LIBRARIES(lightgbm ${MPI_CXX_LIBRARIES})
-  TARGET_LINK_LIBRARIES(_lightgbm ${MPI_CXX_LIBRARIES})
+    if(BUILD_EXE)
+        TARGET_LINK_LIBRARIES(lightgbm ${MPI_CXX_LIBRARIES})
+    endif()
+    if(BUILD_LIB)
+        TARGET_LINK_LIBRARIES(_lightgbm ${MPI_CXX_LIBRARIES})
+    endif()
 endif(USE_MPI)
 
 if(USE_GPU)
-  TARGET_LINK_LIBRARIES(lightgbm ${OpenCL_LIBRARY} ${Boost_LIBRARIES})
-  TARGET_LINK_LIBRARIES(_lightgbm ${OpenCL_LIBRARY} ${Boost_LIBRARIES})
+    if(BUILD_EXE)
+        TARGET_LINK_LIBRARIES(lightgbm ${OpenCL_LIBRARY} ${Boost_LIBRARIES})
+    endif()
+    if(BUILD_LIB)
+        TARGET_LINK_LIBRARIES(_lightgbm ${OpenCL_LIBRARY} ${Boost_LIBRARIES})
+    endif()
 endif(USE_GPU)
 
 if(WIN32 AND (MINGW OR CYGWIN))
-    TARGET_LINK_LIBRARIES(lightgbm Ws2_32)
-    TARGET_LINK_LIBRARIES(_lightgbm Ws2_32)
-    TARGET_LINK_LIBRARIES(lightgbm IPHLPAPI)
-    TARGET_LINK_LIBRARIES(_lightgbm IPHLPAPI)
+    if(BUILD_EXE)
+        TARGET_LINK_LIBRARIES(lightgbm Ws2_32)
+        TARGET_LINK_LIBRARIES(lightgbm IPHLPAPI)
+    endif()
+    if(BUILD_LIB)
+        TARGET_LINK_LIBRARIES(_lightgbm Ws2_32)
+        TARGET_LINK_LIBRARIES(_lightgbm IPHLPAPI)
+    endif()
 endif()
 
-install(TARGETS lightgbm _lightgbm
-        RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin
-        LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib
-        ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/lib)
+if(BUILD_EXE AND BUILD_LIB)
+    install(TARGETS lightgbm _lightgbm
+            RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin
+            LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib
+            ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/lib)
+    install(DIRECTORY ${LightGBM_HEADER_DIR}/LightGBM DESTINATION ${CMAKE_INSTALL_PREFIX}/include)
+endif()
 
-install(DIRECTORY ${LightGBM_HEADER_DIR}/LightGBM DESTINATION ${CMAKE_INSTALL_PREFIX}/include)

R-package/src/install.libs.R

@@ -34,7 +34,7 @@ if (!use_precompile) {
   setwd(build_dir)
   
   # Prepare installation steps
-  cmake_cmd <- "cmake"
+  cmake_cmd <- "cmake -DBUILD_EXE=OFF -DBUILD_LIB=ON "
   build_cmd <- "make -j4"
   lib_folder <- file.path(R_PACKAGE_SOURCE, "src", fsep = "/")
   
@@ -51,7 +51,7 @@ if (!use_precompile) {
   }
   
   if (use_gpu) {
-    cmake_cmd <- paste0(cmake_cmd, " -DUSE_GPU=1 ")
+    cmake_cmd <- paste0(cmake_cmd, " -DUSE_GPU=ON ")
   }
   
   # Install

README.md

@@ -3,6 +3,7 @@ LightGBM, Light Gradient Boosting Machine
 [![Build Status](https://travis-ci.org/Microsoft/LightGBM.svg?branch=master)](https://travis-ci.org/Microsoft/LightGBM)
 [![Windows Build status](https://ci.appveyor.com/api/projects/status/1ys5ot401m0fep6l/branch/master?svg=true)](https://ci.appveyor.com/project/guolinke/lightgbm/branch/master)
 [![Documentation Status](https://readthedocs.org/projects/lightgbm/badge/?version=latest)](http://lightgbm.readthedocs.io/)
+[![PyPI version](https://badge.fury.io/py/lightgbm.svg)](https://badge.fury.io/py/lightgbm)
 
 LightGBM is a gradient boosting framework that uses tree based learning algorithms. It is designed to be distributed and efficient with the following advantages:
 
@@ -19,6 +20,8 @@ For more details, please refer to [Features](https://github.com/Microsoft/LightG
 News
 ----
 
+06/20/2017: Python-package is on PyPI now.
+
 06/09/2017: [LightGBM Slack team](https://lightgbm.slack.com) is available.
 
 05/03/2017: LightGBM v2 stable release.

VERSION.txt

+2.0.2
\ No newline at end of file

appveyor.yml

@@ -13,7 +13,7 @@ init:
 
 build_script:
   - mkdir build && cd build
-  - cmake -DCMAKE_GENERATOR_PLATFORM=x64 .. && cmake --build . --target ALL_BUILD --config Release
+  - cmake -DCMAKE_GENERATOR_PLATFORM=x64 -DBUILD_EXE=ON -DBUILD_LIB=ON .. && cmake --build . --target ALL_BUILD --config Release
   - cd ..
 
 test_script:
@@ -23,10 +23,16 @@ test_script:
   - conda info -a
   - conda install --yes numpy scipy scikit-learn pandas matplotlib
   - pip install pep8 pytest
-  - cd python-package && python setup.py install && cd ..
+  - cd python-package && python setup.py install --precompile && cd ..
   - pytest tests/c_api_test/test.py
   - pytest tests/python_package_test
   - pep8 --ignore=E501 --exclude=./compute,./docs .
+  - "set /p LGB_VER=< VERSION.txt"
+  - cd python-package && python setup.py sdist --formats gztar
+  - cd dist 
+  - "pip install lightgbm-%LGB_VER%.tar.gz -v"
+  - cd ../..
+  - pytest tests/python_package_test
 
 nuget:
   project_feed: true

python-package/MANIFEST.in

+prune build
+include *.rst *.txt
+recursive-include lightgbm *.py *.txt *.so
+recursive-include lightgbm/Release *.dll
+recursive-include lightgbm/include *
+recursive-include lightgbm/src *
+global-exclude *.pyo
+global-exclude *.pyc

python-package/README.rst

 LightGBM Python Package
 =======================
 
+|PyPI version|
+
+
 Installation
 ------------
 
-1. Following `Installation Guide <https://github.com/Microsoft/LightGBM/wiki/Installation-Guide>`__ to build first.
-   For the windows user, please change the build config to ``DLL``.
-2. Install with ``cd python-package; python setup.py install`` 
+Preparation
+'''''''''''
+
+You need to install `cmake <https://cmake.org/>`_ and `setuptools <https://pypi.python.org/pypi/setuptools>`_ first. 
+
+For Windows users, Visual Studio (or `MS Build <https://www.visualstudio.com/downloads/#build-tools-for-visual-studio-2017>`_) is needed. You also can use MinGW instead if installing from GitHub.
+
+For Mac OS X users, gcc with OpenMP support must be installed first. Refer to `wiki <https://github.com/Microsoft/LightGBM/wiki/Installation-Guide#osx>`_ for installing gcc with OpenMP support.
+
+Note: 32-bit python is not supported. Please install 64-bit version.
+
+Install from pip
+''''''''''''''''
+
+``pip install lightgbm``
+
+For the MinGW build in Windows and GPU support, please install the latest version from GitHub.
 
-Note: Make sure you have `setuptools <https://pypi.python.org/pypi/setuptools>`__
+Install from GitHub
+'''''''''''''''''''
 
+.. code:: sh
+
+    git clone --recursive https://github.com/Microsoft/LightGBM
+    cd LightGBM/python-package
+    python setup.py install
+
+You may need to use ``sudo`` (or administrator rights in Windows) to perform ``python setup.py install``.
+
+Use ``python setup.py install --mingw`` to use MinGW in Windows.
+
+Use ``python setup.py install --gpu`` to enable GPU support. You will need to install Boost and OpenCL first: details for installation can be found in `gpu-support <https://github.com/Microsoft/LightGBM/wiki/Installation-Guide#with-gpu-support>`_.
 
 Examples
 --------
 
-Refer to the walk through examples in `python-guide folder <https://github.com/Microsoft/LightGBM/tree/master/examples/python-guide>`__
+Refer to the walk through examples in `python-guide folder <https://github.com/Microsoft/LightGBM/tree/master/examples/python-guide>`_
 
 
 Troubleshooting
---------
+---------------
 
-Refer to `FAQ <https://github.com/Microsoft/LightGBM/tree/master/docs/FAQ.md>`__ 
+Refer to `FAQ <https://github.com/Microsoft/LightGBM/tree/master/docs/FAQ.md>`_ 
 
 Developments
---------
+------------
 
-The code style of python package follows `pep8 <https://www.python.org/dev/peps/pep-0008/>`__. If you would like to make a contribution and not familiar with pep-8, please check the pep8 style guide first. Otherwise, you won't pass the check. You should be careful about:
+The code style of python package follows `pep8 <https://www.python.org/dev/peps/pep-0008/>`_. If you would like to make a contribution and not familiar with pep-8, please check the pep8 style guide first. Otherwise, you won't pass the check. You should be careful about:
 
 - E1 Indentation (check pep8 link above)
 - E202 whitespace before and after brackets
@@ -36,3 +65,7 @@ The code style of python package follows `pep8 <https://www.python.org/dev/peps/
 - E302 expected 2 blank lines in front of and at the end of a function or a class
 
 You can ignore E501 (line too long).
+
+
+.. |PyPI version| image:: https://badge.fury.io/py/lightgbm.svg
+    :target: https://badge.fury.io/py/lightgbm

python-package/lightgbm/libpath.py

@@ -16,11 +16,9 @@ def find_lib_path():
         return []
 
     curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
-    dll_path = [curr_path, os.path.join(curr_path, '../../lib/'),
-                os.path.join(curr_path, '../../'),
-                os.path.join(curr_path, './lib/'),
-                os.path.join(sys.prefix, 'lightgbm')]
+    dll_path = [curr_path, os.path.join(curr_path, '../../'), os.path.join(curr_path, '../../lib/')]
     if os.name == 'nt':
+        dll_path.append(os.path.join(curr_path, './Release/'))
         dll_path.append(os.path.join(curr_path, '../../Release/'))
         dll_path.append(os.path.join(curr_path, '../../windows/x64/DLL/'))
         dll_path = [os.path.join(p, 'lib_lightgbm.dll') for p in dll_path]

python-package/setup.py

@@ -3,34 +3,121 @@
 """Setup lightgbm package."""
 from __future__ import absolute_import
 
+import struct
 import os
 import sys
-
+import getopt
+import distutils
+import shutil
+from distutils import dir_util
+from distutils import file_util
 from setuptools import find_packages, setup
 
-sys.path.insert(0, '.')
+if __name__ == "__main__":
+    build_sdist = sys.argv[1] == 'sdist'
+    if (8 * struct.calcsize("P")) != 64:
+        raise Exception('Cannot install LightGBM in 32-bit python, please use 64-bit python instead.')
+    use_gpu = False
+    use_mingw = False
+    use_precompile = False
+    try:
+        opts, args = getopt.getopt(sys.argv[2:], 'mgp', ['mingw', 'gpu', 'precompile'])
+        for opt, arg in opts:
+            if opt in ('-m', '--mingw'):
+                use_mingw = True
+            elif opt in ('-g', '--gpu'):
+                use_gpu = True
+            elif opt in ('-p', '--precompile'):
+                use_precompile = True
+        sys.argv = sys.argv[0:2]
+    except getopt.GetoptError as err:
+        pass
+    if not use_precompile or build_sdist:
+        if not os.path.isfile('./_IS_SOURCE_PACKAGE.txt'):
+            if os.path.exists("../include"):
+                if os.path.exists("./lightgbm/include"):
+                    shutil.rmtree('./lightgbm/include')
+                distutils.dir_util.copy_tree("../include", "./lightgbm/include")
+            else:
+                raise Exception('Cannot copy ../include folder')
+            if os.path.exists("../src"):
+                if os.path.exists("./lightgbm/src"):
+                    shutil.rmtree('./lightgbm/src')
+                distutils.dir_util.copy_tree("../src", "./lightgbm/src")
+            else:
+                raise Exception('Cannot copy ../src folder')
+            if use_gpu:
+                if os.path.exists("../compute"):
+                    if os.path.exists("./lightgbm/compute"):
+                        shutil.rmtree('./lightgbm/compute')
+                    distutils.dir_util.copy_tree("../compute", "./lightgbm/compute")
+                else:
+                    raise Exception('Cannot copy ../compute folder')
+            distutils.file_util.copy_file("../CMakeLists.txt", "./lightgbm/")
+            distutils.file_util.copy_file("../VERSION.txt", "./lightgbm/")
+            file_flag = open("./_IS_SOURCE_PACKAGE.txt", 'w')
+            file_flag.close()
+
+        if not os.path.exists("build"):
+            os.makedirs("build")
+        os.chdir("build")
 
-CURRENT_DIR = os.path.dirname(__file__)
+        cmake_cmd = "cmake -DBUILD_EXE=OFF -DBUILD_LIB=ON "
+        build_cmd = "make"
 
-libpath_py = os.path.join(CURRENT_DIR, 'lightgbm/libpath.py')
-libpath = {'__file__': libpath_py}
-exec(compile(open(libpath_py, "rb").read(), libpath_py, 'exec'), libpath, libpath)
+        if os.name == "nt":
+            if use_mingw:
+                cmake_cmd = cmake_cmd + " -G \"MinGW Makefiles\" "
+                build_cmd = "mingw32-make.exe"
+            else:
+                cmake_cmd = cmake_cmd + " -DCMAKE_GENERATOR_PLATFORM=x64 "
+                build_cmd = "cmake --build . --target _lightgbm  --config Release"
+        if use_gpu:
+            cmake_cmd = cmake_cmd + " -DUSE_GPU=ON "
+        if not build_sdist:
+            print("Start to compile libarary.")
+            os.system(cmake_cmd + " ../lightgbm/")
+            os.system(build_cmd)
+        os.chdir("..")
 
-LIB_PATH = [os.path.relpath(path, CURRENT_DIR) for path in libpath['find_lib_path']()]
-print("Install lib_lightgbm from: %s" % LIB_PATH)
+    data_files = []
 
+    if build_sdist:
+        print("remove library when building source distribution")
+        if os.path.exists("./lightgbm/Release/"):
+            shutil.rmtree('./lightgbm/Release/')
+        if os.path.isfile('./lightgbm/lib_lightgbm.so'):
+            os.remove('./lightgbm/lib_lightgbm.so')
+    else:
+        sys.path.insert(0, '.')
+        CURRENT_DIR = os.path.dirname(__file__)
+        libpath_py = os.path.join(CURRENT_DIR, 'lightgbm/libpath.py')
+        libpath = {'__file__': libpath_py}
+        exec(compile(open(libpath_py, "rb").read(), libpath_py, 'exec'), libpath, libpath)
 
-setup(name='lightgbm',
-      version=0.2,
-      description="LightGBM Python Package",
-      install_requires=[
-          'numpy',
-          'scipy',
-      ],
-      maintainer='Guolin Ke',
-      maintainer_email='guolin.ke@microsoft.com',
-      zip_safe=False,
-      packages=find_packages(),
-      include_package_data=True,
-      data_files=[('lightgbm', LIB_PATH)],
-      url='https://github.com/Microsoft/LightGBM')
+        LIB_PATH = [os.path.relpath(path, CURRENT_DIR) for path in libpath['find_lib_path']()]
+        print("Install lib_lightgbm from: %s" % LIB_PATH)
+        data_files = [('lightgbm', LIB_PATH)]
+    version = '2.0.1'
+    if os.path.isfile('./lightgbm/VERSION.txt'):
+        version = open('./lightgbm/VERSION.txt').read().strip()
+    elif os.path.isfile('../VERSION.txt'):
+        version = open('../VERSION.txt').read().strip()
+    setup(name='lightgbm',
+          version=version,
+          description='LightGBM Python Package',
+          install_requires=[
+              'numpy',
+              'scipy',
+              'scikit-learn'
+          ],
+          maintainer='Guolin Ke',
+          maintainer_email='guolin.ke@microsoft.com',
+          zip_safe=False,
+          packages=find_packages(),
+          include_package_data=True,
+          data_files=data_files,
+          license='The MIT License(https://github.com/Microsoft/LightGBM/blob/master/LICENSE)',
+          url='https://github.com/Microsoft/LightGBM')
+    if build_sdist:
+        os.remove('./_IS_SOURCE_PACKAGE.txt')

tests/c_api_test/test.py

@@ -15,10 +15,7 @@ def find_lib_path():
         return []
 
     curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
-    dll_path = [curr_path, os.path.join(curr_path, '../../lib/'),
-                os.path.join(curr_path, '../../'),
-                os.path.join(curr_path, './lib/'),
-                os.path.join(sys.prefix, 'lightgbm')]
+    dll_path = [curr_path, os.path.join(curr_path, '../../'), os.path.join(curr_path, '../../lib/')]
     if os.name == 'nt':
         dll_path.append(os.path.join(curr_path, '../../Release/'))
         dll_path.append(os.path.join(curr_path, '../../windows/x64/DLL/'))