test.sh 11.6 KB
Newer Older
Guolin Ke's avatar
Guolin Ke committed
1
2
#!/bin/bash

3
if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "gcc" ]]; then
4
5
    export CXX=g++-10
    export CC=gcc-10
6
elif [[ $OS_NAME == "linux" ]] && [[ $COMPILER == "clang" ]]; then
7
8
    export CXX=clang++
    export CC=clang
Guolin Ke's avatar
Guolin Ke committed
9
10
fi

11
if [[ "${TASK}" == "r-package" ]] || [[ "${TASK}" == "r-rchk" ]]; then
12
13
14
15
    bash ${BUILD_DIRECTORY}/.ci/test_r_package.sh || exit -1
    exit 0
fi

16
17
if [[ "$TASK" == "cpp-tests" ]]; then
    mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build
18
19
20
21
22
23
24
25
26
    if [[ $METHOD == "with-sanitizers" ]]; then
        extra_cmake_opts="-DUSE_SANITIZER=ON"
        if [[ -n $SANITIZERS ]]; then
            extra_cmake_opts="$extra_cmake_opts -DENABLED_SANITIZERS=$SANITIZERS"
        fi
    else
        extra_cmake_opts=""
    fi
    cmake -DBUILD_CPP_TEST=ON -DUSE_OPENMP=OFF -DUSE_DEBUG=ON $extra_cmake_opts ..
27
28
29
30
31
    make testlightgbm -j4 || exit -1
    ./../testlightgbm || exit -1
    exit 0
fi

32
33
conda create -q -y -n $CONDA_ENV python=$PYTHON_VERSION
source activate $CONDA_ENV
34
35
36

cd $BUILD_DIRECTORY

37
if [[ $TASK == "check-docs" ]] || [[ $TASK == "check-links" ]]; then
38
    cd $BUILD_DIRECTORY/docs
39
40
    conda install -q -y -n $CONDA_ENV -c conda-forge doxygen rstcheck
    pip install --user -r requirements.txt
41
42
    # check reStructuredText formatting
    cd $BUILD_DIRECTORY/python-package
43
    rstcheck --report warning $(find . -type f -name "*.rst") || exit -1
44
    cd $BUILD_DIRECTORY/docs
45
    rstcheck --report warning --ignore-directives=autoclass,autofunction,doxygenfile $(find . -type f -name "*.rst") || exit -1
46
    # build docs
47
    make html || exit -1
48
49
    if [[ $TASK == "check-links" ]]; then
        # check docs for broken links
50
        pip install --user linkchecker
51
52
53
        linkchecker --config=.linkcheckerrc ./_build/html/*.html || exit -1
        exit 0
    fi
54
55
56
    # check the consistency of parameters' descriptions and other stuff
    cp $BUILD_DIRECTORY/docs/Parameters.rst $BUILD_DIRECTORY/docs/Parameters-backup.rst
    cp $BUILD_DIRECTORY/src/io/config_auto.cpp $BUILD_DIRECTORY/src/io/config_auto-backup.cpp
57
    python $BUILD_DIRECTORY/helpers/parameter_generator.py || exit -1
58
59
60
61
    diff $BUILD_DIRECTORY/docs/Parameters-backup.rst $BUILD_DIRECTORY/docs/Parameters.rst || exit -1
    diff $BUILD_DIRECTORY/src/io/config_auto-backup.cpp $BUILD_DIRECTORY/src/io/config_auto.cpp || exit -1
    exit 0
fi
62

63
if [[ $TASK == "lint" ]]; then
64
65
66
    conda install -q -y -n $CONDA_ENV \
        pycodestyle \
        pydocstyle \
67
        r-stringi  # stringi needs to be installed separate from r-lintr to avoid issues like 'unable to load shared object stringi.so'
68
    # r-xfun below has to be upgraded because lintr requires > 0.19 for that package
69
70
    conda install -q -y -n $CONDA_ENV \
        -c conda-forge \
71
            libxml2 \
72
            "r-xfun>=0.19" \
73
            "r-lintr>=2.0"
74
    pip install --user cmakelint cpplint isort mypy
75
    echo "Linting Python code"
76
77
    pycodestyle --ignore=E501,W503 --exclude=./.nuget,./external_libs . || exit -1
    pydocstyle --convention=numpy --add-ignore=D105 --match-dir="^(?!^external_libs|test|example).*" --match="(?!^test_|setup).*\.py" . || exit -1
78
    isort . --check-only || exit -1
79
    mypy --ignore-missing-imports python-package/ || true
80
81
82
    echo "Linting R code"
    Rscript ${BUILD_DIRECTORY}/.ci/lint_r_code.R ${BUILD_DIRECTORY} || exit -1
    echo "Linting C++ code"
83
    cpplint --filter=-build/c++11,-build/include_subdir,-build/header_guard,-whitespace/line_length --recursive ./src ./include ./R-package ./swig ./tests || exit -1
84
85
    cmake_files=$(find . -name CMakeLists.txt -o -path "*/cmake/*.cmake")
    cmakelint --linelength=120 ${cmake_files} || true
Guolin Ke's avatar
Guolin Ke committed
86
87
88
89
    exit 0
fi

if [[ $TASK == "if-else" ]]; then
Nikita Titov's avatar
Nikita Titov committed
90
    conda install -q -y -n $CONDA_ENV numpy
91
    mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build && cmake .. && make lightgbm -j4 || exit -1
92
    cd $BUILD_DIRECTORY/tests/cpp_tests && ../../lightgbm config=train.conf convert_model_language=cpp convert_model=../../src/boosting/gbdt_prediction.cpp && ../../lightgbm config=predict.conf output_result=origin.pred || exit -1
93
    cd $BUILD_DIRECTORY/build && make lightgbm -j4 || exit -1
94
    cd $BUILD_DIRECTORY/tests/cpp_tests && ../../lightgbm config=predict.conf output_result=ifelse.pred && python test.py || exit -1
Guolin Ke's avatar
Guolin Ke committed
95
96
97
    exit 0
fi

98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
if [[ $TASK == "swig" ]]; then
    mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build
    if [[ $OS_NAME == "macos" ]]; then
        cmake -DUSE_SWIG=ON -DAPPLE_OUTPUT_DYLIB=ON ..
    else
        cmake -DUSE_SWIG=ON ..
    fi
    make -j4 || exit -1
    if [[ $OS_NAME == "linux" ]] && [[ $COMPILER == "gcc" ]]; then
        objdump -T $BUILD_DIRECTORY/lib_lightgbm.so > $BUILD_DIRECTORY/objdump.log || exit -1
        objdump -T $BUILD_DIRECTORY/lib_lightgbm_swig.so >> $BUILD_DIRECTORY/objdump.log || exit -1
        python $BUILD_DIRECTORY/helpers/check_dynamic_dependencies.py $BUILD_DIRECTORY/objdump.log || exit -1
    fi
    if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
        cp $BUILD_DIRECTORY/build/lightgbmlib.jar $BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_$OS_NAME.jar
    fi
    exit 0
fi

117
conda install -q -y -n $CONDA_ENV cloudpickle dask distributed joblib matplotlib numpy pandas psutil pytest scikit-learn scipy
118
pip install graphviz  # python-graphviz from Anaconda is not allowed to be installed with Python 3.9
Guolin Ke's avatar
Guolin Ke committed
119

120
if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "clang" ]]; then
121
122
    # fix "OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized." (OpenMP library conflict due to conda's MKL)
    for LIBOMP_ALIAS in libgomp.dylib libiomp5.dylib libomp.dylib; do sudo ln -sf "$(brew --cellar libomp)"/*/lib/libomp.dylib $CONDA_PREFIX/lib/$LIBOMP_ALIAS || exit -1; done
123
124
fi

Guolin Ke's avatar
Guolin Ke committed
125
if [[ $TASK == "sdist" ]]; then
126
    cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
127
    pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v || exit -1
128
    if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
129
130
131
        cp $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz $BUILD_ARTIFACTSTAGINGDIRECTORY
    fi
    pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
Guolin Ke's avatar
Guolin Ke committed
132
133
    exit 0
elif [[ $TASK == "bdist" ]]; then
134
    if [[ $OS_NAME == "macos" ]]; then
135
        cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --plat-name=macosx --python-tag py3 || exit -1
136
        mv dist/lightgbm-$LGB_VER-py3-none-macosx.whl dist/lightgbm-$LGB_VER-py3-none-macosx_10_14_x86_64.macosx_10_15_x86_64.macosx_11_0_x86_64.whl
137
        if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
138
            cp dist/lightgbm-$LGB_VER-py3-none-macosx*.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
139
        fi
Guolin Ke's avatar
Guolin Ke committed
140
    else
141
142
143
144
145
146
147
        ARCH=$(uname -m)
        if [[ $ARCH == "x86_64" ]]; then
            PLATFORM="manylinux1_x86_64"
        else
            PLATFORM="manylinux2014_$ARCH"
        fi
        cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --plat-name=$PLATFORM --python-tag py3 || exit -1
148
        if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
149
            cp dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
150
        fi
Guolin Ke's avatar
Guolin Ke committed
151
    fi
152
    pip install --user $BUILD_DIRECTORY/python-package/dist/*.whl || exit -1
153
    pytest $BUILD_DIRECTORY/tests || exit -1
Guolin Ke's avatar
Guolin Ke committed
154
155
156
    exit 0
fi

157
158
mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build

Guolin Ke's avatar
Guolin Ke committed
159
if [[ $TASK == "gpu" ]]; then
160
161
    sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "gpu";/' $BUILD_DIRECTORY/include/LightGBM/config.h
    grep -q 'std::string device_type = "gpu"' $BUILD_DIRECTORY/include/LightGBM/config.h || exit -1  # make sure that changes were really done
Guolin Ke's avatar
Guolin Ke committed
162
    if [[ $METHOD == "pip" ]]; then
163
        cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
164
        pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu --install-option="--opencl-include-dir=$AMDAPPSDK_PATH/include/" || exit -1
165
        pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
Guolin Ke's avatar
Guolin Ke committed
166
        exit 0
167
168
169
170
171
172
173
    elif [[ $METHOD == "wheel" ]]; then
        cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --gpu --opencl-include-dir="$AMDAPPSDK_PATH/include/" || exit -1
        pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER*.whl -v || exit -1
        pytest $BUILD_DIRECTORY/tests || exit -1
        exit 0
    elif [[ $METHOD == "source" ]]; then
        cmake -DUSE_GPU=ON -DOpenCL_INCLUDE_DIR=$AMDAPPSDK_PATH/include/ ..
Guolin Ke's avatar
Guolin Ke committed
174
    fi
175
176
177
178
179
180
181
182
elif [[ $TASK == "cuda" ]]; then
    sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "cuda";/' $BUILD_DIRECTORY/include/LightGBM/config.h
    grep -q 'std::string device_type = "cuda"' $BUILD_DIRECTORY/include/LightGBM/config.h || exit -1  # make sure that changes were really done
    if [[ $METHOD == "pip" ]]; then
        cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
        pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--cuda || exit -1
        pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
        exit 0
183
184
185
186
187
188
189
    elif [[ $METHOD == "wheel" ]]; then
        cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --cuda || exit -1
        pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER*.whl -v || exit -1
        pytest $BUILD_DIRECTORY/tests || exit -1
        exit 0
    elif [[ $METHOD == "source" ]]; then
        cmake -DUSE_CUDA=ON ..
190
    fi
191
elif [[ $TASK == "mpi" ]]; then
192
    if [[ $METHOD == "pip" ]]; then
193
        cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
194
        pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--mpi || exit -1
195
        pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
196
        exit 0
197
198
199
200
201
202
203
    elif [[ $METHOD == "wheel" ]]; then
        cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --mpi || exit -1
        pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER*.whl -v || exit -1
        pytest $BUILD_DIRECTORY/tests || exit -1
        exit 0
    elif [[ $METHOD == "source" ]]; then
        cmake -DUSE_MPI=ON -DUSE_DEBUG=ON ..
204
    fi
Guolin Ke's avatar
Guolin Ke committed
205
else
206
    cmake ..
Guolin Ke's avatar
Guolin Ke committed
207
208
fi

209
make _lightgbm -j4 || exit -1
Guolin Ke's avatar
Guolin Ke committed
210

211
cd $BUILD_DIRECTORY/python-package && python setup.py install --precompile --user || exit -1
212
pytest $BUILD_DIRECTORY/tests || exit -1
Guolin Ke's avatar
Guolin Ke committed
213
214

if [[ $TASK == "regular" ]]; then
215
    if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
216
217
218
        if [[ $OS_NAME == "macos" ]]; then
            cp $BUILD_DIRECTORY/lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.dylib
        else
219
220
221
222
            if [[ $COMPILER == "gcc" ]]; then
                objdump -T $BUILD_DIRECTORY/lib_lightgbm.so > $BUILD_DIRECTORY/objdump.log || exit -1
                python $BUILD_DIRECTORY/helpers/check_dynamic_dependencies.py $BUILD_DIRECTORY/objdump.log || exit -1
            fi
223
224
            cp $BUILD_DIRECTORY/lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.so
        fi
Guolin Ke's avatar
Guolin Ke committed
225
    fi
226
    cd $BUILD_DIRECTORY/examples/python-guide
Guolin Ke's avatar
Guolin Ke committed
227
228
229
230
    sed -i'.bak' '/import lightgbm as lgb/a\
import matplotlib\
matplotlib.use\(\"Agg\"\)\
' plot_example.py  # prevent interactive window mode
231
    sed -i'.bak' 's/graph.render(view=True)/graph.render(view=False)/' plot_example.py
232
    conda install -q -y -n $CONDA_ENV h5py ipywidgets notebook  # requirements for examples
233
    for f in *.py **/*.py; do python $f || exit -1; done  # run all examples
234
    cd $BUILD_DIRECTORY/examples/python-guide/notebooks
235
    sed -i'.bak' 's/INTERACTIVE = False/assert False, \\"Interactive mode disabled\\"/' interactive_plot_example.ipynb
236
    jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb || exit -1  # run all notebooks
Guolin Ke's avatar
Guolin Ke committed
237
fi