test.sh 8.87 KB
Newer Older
Guolin Ke's avatar
Guolin Ke committed
1
2
#!/bin/bash

3
if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "gcc" ]]; then
Nikita Titov's avatar
Nikita Titov committed
4
5
    export CXX=g++-9
    export CC=gcc-9
6
elif [[ $OS_NAME == "linux" ]] && [[ $COMPILER == "clang" ]]; then
7
8
    export CXX=clang++
    export CC=clang
Guolin Ke's avatar
Guolin Ke committed
9
10
fi

11
12
13
14
15
16
17
18
19
20
if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "clang" ]] && [[ $(sw_vers -productVersion | cut -d'.' -f2) -ge "14" ]]; then
    CMAKE_OPTS=(-DOpenMP_C_FLAGS="-Xpreprocessor -fopenmp -I$(brew --prefix libomp)/include"
                -DOpenMP_C_LIB_NAMES=omp
                -DOpenMP_CXX_FLAGS="-Xpreprocessor -fopenmp -I$(brew --prefix libomp)/include"
                -DOpenMP_CXX_LIB_NAMES=omp
                -DOpenMP_omp_LIBRARY=$(brew --prefix libomp)/lib/libomp.dylib)
else
    CMAKE_OPTS=()
fi

21
22
conda create -q -y -n $CONDA_ENV python=$PYTHON_VERSION
source activate $CONDA_ENV
23
24
25
26

cd $BUILD_DIRECTORY

if [[ $TRAVIS == "true" ]] && [[ $TASK == "check-docs" ]]; then
27
28
29
    cd $BUILD_DIRECTORY/docs
    conda install -q -y -n $CONDA_ENV -c conda-forge doxygen
    pip install --user -r requirements.txt rstcheck
30
31
32
33
    # check reStructuredText formatting
    cd $BUILD_DIRECTORY/python-package
    rstcheck --report warning `find . -type f -name "*.rst"` || exit -1
    cd $BUILD_DIRECTORY/docs
34
    rstcheck --report warning --ignore-directives=autoclass,autofunction,doxygenfile `find . -type f -name "*.rst"` || exit -1
35
36
37
38
39
    # build docs and check them for broken links
    make html || exit -1
    find ./_build/html/ -type f -name '*.html' -exec \
    sed -i'.bak' -e 's;\(\.\/[^.]*\.\)rst\([^[:space:]]*\);\1html\2;g' {} \;  # emulate js function
    if [[ $OS_NAME == "linux" ]]; then
40
        sudo apt-get update
41
42
43
44
45
46
        sudo apt-get install linkchecker
        linkchecker --config=.linkcheckerrc ./_build/html/*.html || exit -1
    fi
    # check the consistency of parameters' descriptions and other stuff
    cp $BUILD_DIRECTORY/docs/Parameters.rst $BUILD_DIRECTORY/docs/Parameters-backup.rst
    cp $BUILD_DIRECTORY/src/io/config_auto.cpp $BUILD_DIRECTORY/src/io/config_auto-backup.cpp
47
    python $BUILD_DIRECTORY/helpers/parameter_generator.py || exit -1
48
49
50
51
    diff $BUILD_DIRECTORY/docs/Parameters-backup.rst $BUILD_DIRECTORY/docs/Parameters.rst || exit -1
    diff $BUILD_DIRECTORY/src/io/config_auto-backup.cpp $BUILD_DIRECTORY/src/io/config_auto.cpp || exit -1
    exit 0
fi
52

53
if [[ $TASK == "lint" ]]; then
54
55
56
57
    conda install -q -y -n $CONDA_ENV \
        pycodestyle \
        pydocstyle \
        r-lintr
58
    pip install --user cpplint
59
    echo "Linting Python code"
60
    pycodestyle --ignore=E501,W503 --exclude=./compute,./.nuget . || exit -1
61
    pydocstyle --convention=numpy --add-ignore=D105 --match-dir="^(?!^compute|test|example).*" --match="(?!^test_|setup).*\.py" . || exit -1
62
63
64
    echo "Linting R code"
    Rscript ${BUILD_DIRECTORY}/.ci/lint_r_code.R ${BUILD_DIRECTORY} || exit -1
    echo "Linting C++ code"
65
    cpplint --filter=-build/c++11,-build/include_subdir,-build/header_guard,-whitespace/line_length --recursive ./src ./include || exit 0
Guolin Ke's avatar
Guolin Ke committed
66
67
68
69
    exit 0
fi

if [[ $TASK == "if-else" ]]; then
Nikita Titov's avatar
Nikita Titov committed
70
    conda install -q -y -n $CONDA_ENV numpy
71
    mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build && cmake "${CMAKE_OPTS[@]}" .. && make lightgbm -j4 || exit -1
72
    cd $BUILD_DIRECTORY/tests/cpp_test && ../../lightgbm config=train.conf convert_model_language=cpp convert_model=../../src/boosting/gbdt_prediction.cpp && ../../lightgbm config=predict.conf output_result=origin.pred || exit -1
73
    cd $BUILD_DIRECTORY/build && make lightgbm -j4 || exit -1
74
    cd $BUILD_DIRECTORY/tests/cpp_test && ../../lightgbm config=predict.conf output_result=ifelse.pred && python test.py || exit -1
Guolin Ke's avatar
Guolin Ke committed
75
76
77
    exit 0
fi

78
conda install -q -y -n $CONDA_ENV joblib matplotlib numpy pandas psutil pytest python-graphviz scikit-learn scipy
Guolin Ke's avatar
Guolin Ke committed
79

80
if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "clang" ]]; then
81
82
    # fix "OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized." (OpenMP library conflict due to conda's MKL)
    for LIBOMP_ALIAS in libgomp.dylib libiomp5.dylib libomp.dylib; do sudo ln -sf "$(brew --cellar libomp)"/*/lib/libomp.dylib $CONDA_PREFIX/lib/$LIBOMP_ALIAS || exit -1; done
83
84
fi

Guolin Ke's avatar
Guolin Ke committed
85
if [[ $TASK == "sdist" ]]; then
86
    cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
87
    pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v || exit -1
88
89
    if [[ $AZURE == "true" ]]; then
        cp $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz $BUILD_ARTIFACTSTAGINGDIRECTORY
90
91
92
93
94
95
96
97
98
99
100
101
102
        mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build
        if [[ $OS_NAME == "macos" ]]; then
            cmake -DUSE_SWIG=ON -DAPPLE_OUTPUT_DYLIB=ON "${CMAKE_OPTS[@]}" ..
        else
            cmake -DUSE_SWIG=ON "${CMAKE_OPTS[@]}" ..
        fi
        make -j4 || exit -1
        if [[ $OS_NAME == "linux" ]] && [[ $COMPILER == "gcc" ]]; then
            objdump -T $BUILD_DIRECTORY/lib_lightgbm.so > $BUILD_DIRECTORY/objdump.log || exit -1
            objdump -T $BUILD_DIRECTORY/lib_lightgbm_swig.so >> $BUILD_DIRECTORY/objdump.log || exit -1
            python $BUILD_DIRECTORY/helpers/check_dynamic_dependencies.py $BUILD_DIRECTORY/objdump.log || exit -1
        fi
        cp $BUILD_DIRECTORY/build/lightgbmlib.jar $BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_$OS_NAME.jar
103
104
    fi
    pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
Guolin Ke's avatar
Guolin Ke committed
105
106
    exit 0
elif [[ $TASK == "bdist" ]]; then
107
108
    if [[ $OS_NAME == "macos" ]]; then
        cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --plat-name=macosx --universal || exit -1
Nikita Titov's avatar
Nikita Titov committed
109
        mv dist/lightgbm-$LGB_VER-py2.py3-none-macosx.whl dist/lightgbm-$LGB_VER-py2.py3-none-macosx_10_9_x86_64.macosx_10_10_x86_64.macosx_10_11_x86_64.macosx_10_12_x86_64.macosx_10_13_x86_64.macosx_10_14_x86_64.macosx_10_15_x86_64.whl
110
111
112
        if [[ $AZURE == "true" ]]; then
            cp dist/lightgbm-$LGB_VER-py2.py3-none-macosx*.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
        fi
Guolin Ke's avatar
Guolin Ke committed
113
    else
114
115
116
117
        cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --plat-name=manylinux1_x86_64 --universal || exit -1
        if [[ $AZURE == "true" ]]; then
            cp dist/lightgbm-$LGB_VER-py2.py3-none-manylinux1_x86_64.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
        fi
Guolin Ke's avatar
Guolin Ke committed
118
    fi
119
    pip install --user $BUILD_DIRECTORY/python-package/dist/*.whl || exit -1
120
    pytest $BUILD_DIRECTORY/tests || exit -1
Guolin Ke's avatar
Guolin Ke committed
121
122
123
    exit 0
fi

124
125
mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build

Guolin Ke's avatar
Guolin Ke committed
126
if [[ $TASK == "gpu" ]]; then
127
128
    sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "gpu";/' $BUILD_DIRECTORY/include/LightGBM/config.h
    grep -q 'std::string device_type = "gpu"' $BUILD_DIRECTORY/include/LightGBM/config.h || exit -1  # make sure that changes were really done
Guolin Ke's avatar
Guolin Ke committed
129
    if [[ $METHOD == "pip" ]]; then
130
        cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
131
        pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu --install-option="--opencl-include-dir=$AMDAPPSDK_PATH/include/" || exit -1
132
        pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
Guolin Ke's avatar
Guolin Ke committed
133
134
        exit 0
    fi
135
    cmake -DUSE_GPU=ON -DOpenCL_INCLUDE_DIR=$AMDAPPSDK_PATH/include/ "${CMAKE_OPTS[@]}" ..
136
elif [[ $TASK == "mpi" ]]; then
137
    if [[ $METHOD == "pip" ]]; then
138
        cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
139
        pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--mpi || exit -1
140
        pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
141
142
        exit 0
    fi
143
    cmake -DUSE_MPI=ON "${CMAKE_OPTS[@]}" ..
Guolin Ke's avatar
Guolin Ke committed
144
else
145
    cmake "${CMAKE_OPTS[@]}" ..
Guolin Ke's avatar
Guolin Ke committed
146
147
fi

148
make _lightgbm -j4 || exit -1
Guolin Ke's avatar
Guolin Ke committed
149

150
cd $BUILD_DIRECTORY/python-package && python setup.py install --precompile --user || exit -1
151
pytest $BUILD_DIRECTORY/tests || exit -1
Guolin Ke's avatar
Guolin Ke committed
152
153

if [[ $TASK == "regular" ]]; then
154
155
156
157
    if [[ $AZURE == "true" ]]; then
        if [[ $OS_NAME == "macos" ]]; then
            cp $BUILD_DIRECTORY/lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.dylib
        else
158
159
160
161
            if [[ $COMPILER == "gcc" ]]; then
                objdump -T $BUILD_DIRECTORY/lib_lightgbm.so > $BUILD_DIRECTORY/objdump.log || exit -1
                python $BUILD_DIRECTORY/helpers/check_dynamic_dependencies.py $BUILD_DIRECTORY/objdump.log || exit -1
            fi
162
163
            cp $BUILD_DIRECTORY/lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.so
        fi
Guolin Ke's avatar
Guolin Ke committed
164
    fi
165
    cd $BUILD_DIRECTORY/examples/python-guide
Guolin Ke's avatar
Guolin Ke committed
166
167
168
169
    sed -i'.bak' '/import lightgbm as lgb/a\
import matplotlib\
matplotlib.use\(\"Agg\"\)\
' plot_example.py  # prevent interactive window mode
170
    sed -i'.bak' 's/graph.render(view=True)/graph.render(view=False)/' plot_example.py
Guolin Ke's avatar
Guolin Ke committed
171
    for f in *.py; do python $f || exit -1; done  # run all examples
172
    cd $BUILD_DIRECTORY/examples/python-guide/notebooks
Nikita Titov's avatar
Nikita Titov committed
173
    conda install -q -y -n $CONDA_ENV ipywidgets notebook
174
    jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb || exit -1  # run all notebooks
Guolin Ke's avatar
Guolin Ke committed
175
fi