test.sh 13 KB
Newer Older
Guolin Ke's avatar
Guolin Ke committed
1
2
#!/bin/bash

3
if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "gcc" ]]; then
4
5
    export CXX=g++-11
    export CC=gcc-11
6
elif [[ $OS_NAME == "linux" ]] && [[ $COMPILER == "clang" ]]; then
7
8
    export CXX=clang++
    export CC=clang
Guolin Ke's avatar
Guolin Ke committed
9
10
fi

11
12
13
14
15
if [[ $IN_UBUNTU_BASE_CONTAINER == "true" ]]; then
    export LANG="en_US.UTF-8"
    export LC_ALL="en_US.UTF-8"
fi

16
if [[ "${TASK}" == "r-package" ]] || [[ "${TASK}" == "r-rchk" ]]; then
17
18
19
20
    bash ${BUILD_DIRECTORY}/.ci/test_r_package.sh || exit -1
    exit 0
fi

21
22
if [[ "$TASK" == "cpp-tests" ]]; then
    mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build
23
24
25
26
27
28
29
30
31
    if [[ $METHOD == "with-sanitizers" ]]; then
        extra_cmake_opts="-DUSE_SANITIZER=ON"
        if [[ -n $SANITIZERS ]]; then
            extra_cmake_opts="$extra_cmake_opts -DENABLED_SANITIZERS=$SANITIZERS"
        fi
    else
        extra_cmake_opts=""
    fi
    cmake -DBUILD_CPP_TEST=ON -DUSE_OPENMP=OFF -DUSE_DEBUG=ON $extra_cmake_opts ..
32
33
34
35
36
    make testlightgbm -j4 || exit -1
    ./../testlightgbm || exit -1
    exit 0
fi

37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
CONDA_PYTHON_REQUIREMENT="python=$PYTHON_VERSION[build=*cpython]"

if [[ $TASK == "if-else" ]]; then
    conda create -q -y -n $CONDA_ENV ${CONDA_PYTHON_REQUIREMENT} numpy
    source activate $CONDA_ENV
    mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build && cmake .. && make lightgbm -j4 || exit -1
    cd $BUILD_DIRECTORY/tests/cpp_tests && ../../lightgbm config=train.conf convert_model_language=cpp convert_model=../../src/boosting/gbdt_prediction.cpp && ../../lightgbm config=predict.conf output_result=origin.pred || exit -1
    cd $BUILD_DIRECTORY/build && make lightgbm -j4 || exit -1
    cd $BUILD_DIRECTORY/tests/cpp_tests && ../../lightgbm config=predict.conf output_result=ifelse.pred && python test.py || exit -1
    exit 0
fi

if [[ $TASK == "swig" ]]; then
    mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build
    if [[ $OS_NAME == "macos" ]]; then
        cmake -DUSE_SWIG=ON -DAPPLE_OUTPUT_DYLIB=ON ..
    else
        cmake -DUSE_SWIG=ON ..
    fi
    make -j4 || exit -1
    if [[ $OS_NAME == "linux" ]] && [[ $COMPILER == "gcc" ]]; then
        objdump -T $BUILD_DIRECTORY/lib_lightgbm.so > $BUILD_DIRECTORY/objdump.log || exit -1
        objdump -T $BUILD_DIRECTORY/lib_lightgbm_swig.so >> $BUILD_DIRECTORY/objdump.log || exit -1
        python $BUILD_DIRECTORY/helpers/check_dynamic_dependencies.py $BUILD_DIRECTORY/objdump.log || exit -1
    fi
    if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
        cp $BUILD_DIRECTORY/build/lightgbmlib.jar $BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_$OS_NAME.jar
    fi
    exit 0
fi

if [[ $TASK == "lint" ]]; then
69
    cd ${BUILD_DIRECTORY}
70
71
72
73
    conda create -q -y -n $CONDA_ENV \
        ${CONDA_PYTHON_REQUIREMENT} \
        cmakelint \
        cpplint \
74
        flake8 \
75
76
77
78
79
80
        isort \
        mypy \
        pydocstyle \
        "r-lintr>=3.0"
    source activate $CONDA_ENV
    echo "Linting Python code"
81
    sh ${BUILD_DIRECTORY}/.ci/lint-python.sh ${BUILD_DIRECTORY} || exit -1
82
83
84
    echo "Linting R code"
    Rscript ${BUILD_DIRECTORY}/.ci/lint_r_code.R ${BUILD_DIRECTORY} || exit -1
    echo "Linting C++ code"
85
    sh ${BUILD_DIRECTORY}/.ci/lint-cpp.sh || exit -1
86
87
88
    exit 0
fi

89
if [[ $TASK == "check-docs" ]] || [[ $TASK == "check-links" ]]; then
90
    cd $BUILD_DIRECTORY/docs
91
    conda env create \
92
93
94
95
96
97
98
        -n $CONDA_ENV \
        --file ./env.yml || exit -1
    conda install \
        -q \
        -y \
        -n $CONDA_ENV \
            doxygen \
99
            'rstcheck>=6.0.0' || exit -1
100
    source activate $CONDA_ENV
101
102
    # check reStructuredText formatting
    cd $BUILD_DIRECTORY/python-package
103
    rstcheck --report-level warning $(find . -type f -name "*.rst") || exit -1
104
    cd $BUILD_DIRECTORY/docs
105
    rstcheck --report-level warning --ignore-directives=autoclass,autofunction,autosummary,doxygenfile $(find . -type f -name "*.rst") || exit -1
106
    # build docs
107
    make html || exit -1
108
109
    if [[ $TASK == "check-links" ]]; then
        # check docs for broken links
110
        pip install --user linkchecker
111
112
113
        linkchecker --config=.linkcheckerrc ./_build/html/*.html || exit -1
        exit 0
    fi
114
115
116
    # check the consistency of parameters' descriptions and other stuff
    cp $BUILD_DIRECTORY/docs/Parameters.rst $BUILD_DIRECTORY/docs/Parameters-backup.rst
    cp $BUILD_DIRECTORY/src/io/config_auto.cpp $BUILD_DIRECTORY/src/io/config_auto-backup.cpp
117
    python $BUILD_DIRECTORY/helpers/parameter_generator.py || exit -1
118
119
120
121
    diff $BUILD_DIRECTORY/docs/Parameters-backup.rst $BUILD_DIRECTORY/docs/Parameters.rst || exit -1
    diff $BUILD_DIRECTORY/src/io/config_auto-backup.cpp $BUILD_DIRECTORY/src/io/config_auto.cpp || exit -1
    exit 0
fi
122

123
124
# including python=version[build=*cpython] to ensure that conda doesn't fall back to pypy
conda create -q -y -n $CONDA_ENV \
125
    cloudpickle \
126
127
    dask-core \
    distributed \
128
129
130
131
132
133
    joblib \
    matplotlib \
    numpy \
    pandas \
    psutil \
    pytest \
134
    ${CONDA_PYTHON_REQUIREMENT} \
135
    python-graphviz \
136
137
    scikit-learn \
    scipy || exit -1
138

139
140
141
142
source activate $CONDA_ENV

cd $BUILD_DIRECTORY

143
if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "clang" ]]; then
144
145
    # fix "OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized." (OpenMP library conflict due to conda's MKL)
    for LIBOMP_ALIAS in libgomp.dylib libiomp5.dylib libomp.dylib; do sudo ln -sf "$(brew --cellar libomp)"/*/lib/libomp.dylib $CONDA_PREFIX/lib/$LIBOMP_ALIAS || exit -1; done
146
147
fi

Guolin Ke's avatar
Guolin Ke committed
148
if [[ $TASK == "sdist" ]]; then
149
150
151
    cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit -1
    sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
    pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz -v || exit -1
152
    if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
153
        cp $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz $BUILD_ARTIFACTSTAGINGDIRECTORY
154
155
    fi
    pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
Guolin Ke's avatar
Guolin Ke committed
156
157
    exit 0
elif [[ $TASK == "bdist" ]]; then
158
    if [[ $OS_NAME == "macos" ]]; then
159
160
161
162
163
        cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel || exit -1
        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
        mv \
            dist/lightgbm-$LGB_VER-py3-none-macosx*.whl \
            dist/lightgbm-$LGB_VER-py3-none-macosx_10_15_x86_64.macosx_11_6_x86_64.macosx_12_5_x86_64.whl
164
        if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
165
            cp dist/lightgbm-$LGB_VER-py3-none-macosx*.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
166
        fi
Guolin Ke's avatar
Guolin Ke committed
167
    else
168
169
        ARCH=$(uname -m)
        if [[ $ARCH == "x86_64" ]]; then
170
            PLATFORM="manylinux_2_28_x86_64"
171
172
173
        else
            PLATFORM="manylinux2014_$ARCH"
        fi
174
175
176
177
178
        cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --integrated-opencl || exit -1
        mv \
            ./dist/*.whl \
            ./dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl
        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
179
        if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
180
            cp dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
181
        fi
182
183
        # Make sure we can do both CPU and GPU; see tests/python_package_test/test_dual.py
        export LIGHTGBM_TEST_DUAL_CPU_GPU=1
Guolin Ke's avatar
Guolin Ke committed
184
    fi
185
    pip install --user $BUILD_DIRECTORY/dist/*.whl || exit -1
186
    pytest $BUILD_DIRECTORY/tests || exit -1
Guolin Ke's avatar
Guolin Ke committed
187
188
189
    exit 0
fi

190
191
192
193
194
195
# temporarily pin pip to versions that support 'pip install --install-option'
# ref: https://github.com/microsoft/LightGBM/issues/5061#issuecomment-1510642287
if [[ $METHOD == "pip" ]]; then
    pip install 'pip<23.1'
fi

Guolin Ke's avatar
Guolin Ke committed
196
if [[ $TASK == "gpu" ]]; then
197
198
    sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "gpu";/' $BUILD_DIRECTORY/include/LightGBM/config.h
    grep -q 'std::string device_type = "gpu"' $BUILD_DIRECTORY/include/LightGBM/config.h || exit -1  # make sure that changes were really done
Guolin Ke's avatar
Guolin Ke committed
199
    if [[ $METHOD == "pip" ]]; then
200
201
202
        cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit -1
        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
        pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu || exit -1
203
        pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
Guolin Ke's avatar
Guolin Ke committed
204
        exit 0
205
    elif [[ $METHOD == "wheel" ]]; then
206
207
208
        cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --gpu || exit -1
        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
        pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER*.whl -v || exit -1
209
210
211
        pytest $BUILD_DIRECTORY/tests || exit -1
        exit 0
    elif [[ $METHOD == "source" ]]; then
212
213
        mkdir $BUILD_DIRECTORY/build
        cd $BUILD_DIRECTORY/build
214
        cmake -DUSE_GPU=ON ..
Guolin Ke's avatar
Guolin Ke committed
215
    fi
216
217
218
219
220
221
elif [[ $TASK == "cuda" ]]; then
    sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "cuda";/' $BUILD_DIRECTORY/include/LightGBM/config.h
    grep -q 'std::string device_type = "cuda"' $BUILD_DIRECTORY/include/LightGBM/config.h || exit -1  # make sure that changes were really done
    # by default ``gpu_use_dp=false`` for efficiency. change to ``true`` here for exact results in ci tests
    sed -i'.bak' 's/gpu_use_dp = false;/gpu_use_dp = true;/' $BUILD_DIRECTORY/include/LightGBM/config.h
    grep -q 'gpu_use_dp = true' $BUILD_DIRECTORY/include/LightGBM/config.h || exit -1  # make sure that changes were really done
222
    if [[ $METHOD == "pip" ]]; then
223
224
225
        cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit -1
        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
        pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--cuda || exit -1
226
227
        pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
        exit 0
228
    elif [[ $METHOD == "wheel" ]]; then
229
230
231
        cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --cuda || exit -1
        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
        pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER*.whl -v || exit -1
232
233
234
        pytest $BUILD_DIRECTORY/tests || exit -1
        exit 0
    elif [[ $METHOD == "source" ]]; then
235
236
        mkdir $BUILD_DIRECTORY/build
        cd $BUILD_DIRECTORY/build
237
        cmake -DUSE_CUDA=ON ..
238
    fi
239
elif [[ $TASK == "mpi" ]]; then
240
    if [[ $METHOD == "pip" ]]; then
241
242
243
        cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit -1
        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
        pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--mpi || exit -1
244
        pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
245
        exit 0
246
    elif [[ $METHOD == "wheel" ]]; then
247
248
249
        cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --mpi || exit -1
        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1
        pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER*.whl -v || exit -1
250
251
252
        pytest $BUILD_DIRECTORY/tests || exit -1
        exit 0
    elif [[ $METHOD == "source" ]]; then
253
254
        mkdir $BUILD_DIRECTORY/build
        cd $BUILD_DIRECTORY/build
255
        cmake -DUSE_MPI=ON -DUSE_DEBUG=ON ..
256
    fi
Guolin Ke's avatar
Guolin Ke committed
257
else
258
259
    mkdir $BUILD_DIRECTORY/build
    cd $BUILD_DIRECTORY/build
260
    cmake ..
Guolin Ke's avatar
Guolin Ke committed
261
262
fi

263
make _lightgbm -j4 || exit -1
Guolin Ke's avatar
Guolin Ke committed
264

265
cd $BUILD_DIRECTORY && sh ./build-python.sh install --precompile --user || exit -1
266
pytest $BUILD_DIRECTORY/tests || exit -1
Guolin Ke's avatar
Guolin Ke committed
267
268

if [[ $TASK == "regular" ]]; then
269
    if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
270
271
272
        if [[ $OS_NAME == "macos" ]]; then
            cp $BUILD_DIRECTORY/lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.dylib
        else
273
274
275
276
            if [[ $COMPILER == "gcc" ]]; then
                objdump -T $BUILD_DIRECTORY/lib_lightgbm.so > $BUILD_DIRECTORY/objdump.log || exit -1
                python $BUILD_DIRECTORY/helpers/check_dynamic_dependencies.py $BUILD_DIRECTORY/objdump.log || exit -1
            fi
277
278
            cp $BUILD_DIRECTORY/lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.so
        fi
Guolin Ke's avatar
Guolin Ke committed
279
    fi
280
    cd $BUILD_DIRECTORY/examples/python-guide
Guolin Ke's avatar
Guolin Ke committed
281
282
283
284
    sed -i'.bak' '/import lightgbm as lgb/a\
import matplotlib\
matplotlib.use\(\"Agg\"\)\
' plot_example.py  # prevent interactive window mode
285
    sed -i'.bak' 's/graph.render(view=True)/graph.render(view=False)/' plot_example.py
286
287
288
289
290
    # requirements for examples
    conda install -q -y -n $CONDA_ENV \
        h5py \
        ipywidgets \
        notebook
291
    for f in *.py **/*.py; do python $f || exit -1; done  # run all examples
292
    cd $BUILD_DIRECTORY/examples/python-guide/notebooks
293
    sed -i'.bak' 's/INTERACTIVE = False/assert False, \\"Interactive mode disabled\\"/' interactive_plot_example.ipynb
294
    jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb || exit -1  # run all notebooks
295
296
297
298
299
300
301
302
303
304
305

    # importing the library should succeed even if all optional dependencies are not present
    conda uninstall --force --yes \
        dask \
        distributed \
        joblib \
        matplotlib \
        psutil \
        python-graphviz \
        scikit-learn || exit -1
    python -c "import lightgbm" || exit -1
Guolin Ke's avatar
Guolin Ke committed
306
fi