test.sh 13.4 KB
Newer Older
Guolin Ke's avatar
Guolin Ke committed
1
2
#!/bin/bash

3
4
5
set -e -E -o -u pipefail

# defaults
6
CONDA_ENV="test-env"
7
8
9
10
IN_UBUNTU_BASE_CONTAINER=${IN_UBUNTU_BASE_CONTAINER:-"false"}
METHOD=${METHOD:-""}
PRODUCES_ARTIFACTS=${PRODUCES_ARTIFACTS:-"false"}
SANITIZERS=${SANITIZERS:-""}
11

12
13
ARCH=$(uname -m)

14
15
LGB_VER=$(head -n 1 ${BUILD_DIRECTORY}/VERSION.txt)

16
if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "gcc" ]]; then
17
18
    export CXX=g++-11
    export CC=gcc-11
19
elif [[ $OS_NAME == "linux" ]] && [[ $COMPILER == "clang" ]]; then
20
21
    export CXX=clang++
    export CC=clang
22
23
24
elif [[ $OS_NAME == "linux" ]] && [[ $COMPILER == "clang-17" ]]; then
    export CXX=clang++-17
    export CC=clang-17
Guolin Ke's avatar
Guolin Ke committed
25
26
fi

27
28
29
30
31
if [[ $IN_UBUNTU_BASE_CONTAINER == "true" ]]; then
    export LANG="en_US.UTF-8"
    export LC_ALL="en_US.UTF-8"
fi

32
if [[ "${TASK}" == "r-package" ]] || [[ "${TASK}" == "r-rchk" ]]; then
33
    bash ${BUILD_DIRECTORY}/.ci/test_r_package.sh || exit 1
34
35
36
    exit 0
fi

37
if [[ "$TASK" == "cpp-tests" ]]; then
38
39
40
41
42
43
44
45
    if [[ $METHOD == "with-sanitizers" ]]; then
        extra_cmake_opts="-DUSE_SANITIZER=ON"
        if [[ -n $SANITIZERS ]]; then
            extra_cmake_opts="$extra_cmake_opts -DENABLED_SANITIZERS=$SANITIZERS"
        fi
    else
        extra_cmake_opts=""
    fi
46
47
48
    cmake -B build -S . -DBUILD_CPP_TEST=ON -DUSE_OPENMP=OFF -DUSE_DEBUG=ON $extra_cmake_opts
    cmake --build build --target testlightgbm -j4 || exit 1
    ./testlightgbm || exit 1
49
50
51
    exit 0
fi

52
# including python=version[build=*cpython] to ensure that conda doesn't fall back to pypy
53
54
55
CONDA_PYTHON_REQUIREMENT="python=$PYTHON_VERSION[build=*cpython]"

if [[ $TASK == "if-else" ]]; then
56
    mamba create -q -y -n $CONDA_ENV ${CONDA_PYTHON_REQUIREMENT} numpy
57
    source activate $CONDA_ENV
58
59
    cmake -B build -S . || exit 1
    cmake --build build --target lightgbm -j4 || exit 1
60
61
    cd $BUILD_DIRECTORY/tests/cpp_tests && ../../lightgbm config=train.conf convert_model_language=cpp convert_model=../../src/boosting/gbdt_prediction.cpp && ../../lightgbm config=predict.conf output_result=origin.pred || exit 1
    cd $BUILD_DIRECTORY/tests/cpp_tests && ../../lightgbm config=predict.conf output_result=ifelse.pred && python test.py || exit 1
62
63
64
65
    exit 0
fi

if [[ $TASK == "swig" ]]; then
66
    cmake -B build -S . -DUSE_SWIG=ON
67
    cmake --build build -j4 || exit 1
68
    if [[ $OS_NAME == "linux" ]] && [[ $COMPILER == "gcc" ]]; then
69
70
71
        objdump -T $BUILD_DIRECTORY/lib_lightgbm.so > $BUILD_DIRECTORY/objdump.log || exit 1
        objdump -T $BUILD_DIRECTORY/lib_lightgbm_swig.so >> $BUILD_DIRECTORY/objdump.log || exit 1
        python $BUILD_DIRECTORY/helpers/check_dynamic_dependencies.py $BUILD_DIRECTORY/objdump.log || exit 1
72
73
74
75
76
77
78
79
    fi
    if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
        cp $BUILD_DIRECTORY/build/lightgbmlib.jar $BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_$OS_NAME.jar
    fi
    exit 0
fi

if [[ $TASK == "lint" ]]; then
80
    cd ${BUILD_DIRECTORY}
81
    mamba create -q -y -n $CONDA_ENV \
82
        ${CONDA_PYTHON_REQUIREMENT} \
83
84
        'cmakelint>=1.4.2' \
        'cpplint>=1.6.0' \
85
        'matplotlib-base>=3.8.3' \
86
        'mypy>=1.8.0' \
87
        'pre-commit>=3.6.0' \
88
        'pyarrow>=6.0' \
89
        'r-lintr>=3.1.2'
90
91
    source activate $CONDA_ENV
    echo "Linting Python code"
92
    bash ${BUILD_DIRECTORY}/.ci/lint-python.sh || exit 1
93
    echo "Linting R code"
94
    Rscript ${BUILD_DIRECTORY}/.ci/lint_r_code.R ${BUILD_DIRECTORY} || exit 1
95
    echo "Linting C++ code"
96
    bash ${BUILD_DIRECTORY}/.ci/lint-cpp.sh || exit 1
97
98
99
    exit 0
fi

100
if [[ $TASK == "check-docs" ]] || [[ $TASK == "check-links" ]]; then
101
    cd $BUILD_DIRECTORY/docs
102
    mamba env create \
103
        -n $CONDA_ENV \
104
        --file ./env.yml || exit 1
105
    mamba install \
106
107
108
        -q \
        -y \
        -n $CONDA_ENV \
109
110
            'doxygen>=1.10.0' \
            'rstcheck>=6.2.0' || exit 1
111
    source activate $CONDA_ENV
112
113
    # check reStructuredText formatting
    cd $BUILD_DIRECTORY/python-package
114
    rstcheck --report-level warning $(find . -type f -name "*.rst") || exit 1
115
    cd $BUILD_DIRECTORY/docs
116
    rstcheck --report-level warning --ignore-directives=autoclass,autofunction,autosummary,doxygenfile $(find . -type f -name "*.rst") || exit 1
117
    # build docs
118
    make html || exit 1
119
120
    if [[ $TASK == "check-links" ]]; then
        # check docs for broken links
121
        pip install linkchecker
122
        linkchecker --config=.linkcheckerrc ./_build/html/*.html || exit 1
123
124
        exit 0
    fi
125
126
127
    # check the consistency of parameters' descriptions and other stuff
    cp $BUILD_DIRECTORY/docs/Parameters.rst $BUILD_DIRECTORY/docs/Parameters-backup.rst
    cp $BUILD_DIRECTORY/src/io/config_auto.cpp $BUILD_DIRECTORY/src/io/config_auto-backup.cpp
128
129
130
    python $BUILD_DIRECTORY/helpers/parameter_generator.py || exit 1
    diff $BUILD_DIRECTORY/docs/Parameters-backup.rst $BUILD_DIRECTORY/docs/Parameters.rst || exit 1
    diff $BUILD_DIRECTORY/src/io/config_auto-backup.cpp $BUILD_DIRECTORY/src/io/config_auto.cpp || exit 1
131
132
    exit 0
fi
133

134
if [[ $PYTHON_VERSION == "3.7" ]]; then
135
    CONDA_REQUIREMENT_FILES="--file ${BUILD_DIRECTORY}/.ci/conda-envs/ci-core-py37.txt"
136
137
elif [[ $PYTHON_VERSION == "3.8" ]]; then
    CONDA_REQUIREMENT_FILES="--file ${BUILD_DIRECTORY}/.ci/conda-envs/ci-core-py38.txt"
138
139
else
    CONDA_REQUIREMENT_FILES="--file ${BUILD_DIRECTORY}/.ci/conda-envs/ci-core.txt"
140
141
fi

142
143
144
145
mamba create \
    -y \
    -n $CONDA_ENV \
    ${CONDA_REQUIREMENT_FILES} \
146
    ${CONDA_PYTHON_REQUIREMENT} \
147
|| exit 1
148

149
150
151
152
source activate $CONDA_ENV

cd $BUILD_DIRECTORY

153
if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "clang" ]]; then
154
    # fix "OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized." (OpenMP library conflict due to conda's MKL)
155
    for LIBOMP_ALIAS in libgomp.dylib libiomp5.dylib libomp.dylib; do sudo ln -sf "$(brew --cellar libomp)"/*/lib/libomp.dylib $CONDA_PREFIX/lib/$LIBOMP_ALIAS || exit 1; done
156
157
fi

Guolin Ke's avatar
Guolin Ke committed
158
if [[ $TASK == "sdist" ]]; then
159
160
    cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit 1
    sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit 1
161
    pip install $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz -v || exit 1
162
    if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
163
        cp $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz $BUILD_ARTIFACTSTAGINGDIRECTORY || exit 1
164
    fi
165
    pytest $BUILD_DIRECTORY/tests/python_package_test || exit 1
Guolin Ke's avatar
Guolin Ke committed
166
167
    exit 0
elif [[ $TASK == "bdist" ]]; then
168
    if [[ $OS_NAME == "macos" ]]; then
169
170
        cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel || exit 1
        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit 1
171
        mv \
172
            ./dist/*.whl \
173
            ./dist/tmp.whl || exit 1
174
175
176
        if [[ $ARCH == "x86_64" ]]; then
            PLATFORM="macosx_10_15_x86_64.macosx_11_6_x86_64.macosx_12_5_x86_64"
        else
177
            PLATFORM="macosx_14_0_arm64"
178
        fi
179
180
        mv \
            ./dist/tmp.whl \
181
            dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl || exit 1
182
        if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
183
            cp dist/lightgbm-$LGB_VER-py3-none-macosx*.whl $BUILD_ARTIFACTSTAGINGDIRECTORY || exit 1
184
        fi
Guolin Ke's avatar
Guolin Ke committed
185
    else
186
        if [[ $ARCH == "x86_64" ]]; then
187
            PLATFORM="manylinux_2_28_x86_64"
188
189
190
        else
            PLATFORM="manylinux2014_$ARCH"
        fi
191
        cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --integrated-opencl || exit 1
192
193
        mv \
            ./dist/*.whl \
194
            ./dist/tmp.whl || exit 1
195
196
        mv \
            ./dist/tmp.whl \
197
198
            ./dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl || exit 1
        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit 1
199
        if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
200
            cp dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl $BUILD_ARTIFACTSTAGINGDIRECTORY || exit 1
201
        fi
202
203
        # Make sure we can do both CPU and GPU; see tests/python_package_test/test_dual.py
        export LIGHTGBM_TEST_DUAL_CPU_GPU=1
Guolin Ke's avatar
Guolin Ke committed
204
    fi
205
    pip install -v $BUILD_DIRECTORY/dist/*.whl || exit 1
206
    pytest $BUILD_DIRECTORY/tests || exit 1
Guolin Ke's avatar
Guolin Ke committed
207
208
209
210
    exit 0
fi

if [[ $TASK == "gpu" ]]; then
211
    sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "gpu";/' $BUILD_DIRECTORY/include/LightGBM/config.h
212
    grep -q 'std::string device_type = "gpu"' $BUILD_DIRECTORY/include/LightGBM/config.h || exit 1  # make sure that changes were really done
Guolin Ke's avatar
Guolin Ke committed
213
    if [[ $METHOD == "pip" ]]; then
214
215
        cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit 1
        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit 1
216
217
        pip install \
            -v \
218
            --config-settings=cmake.define.USE_GPU=ON \
219
            $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz \
220
221
        || exit 1
        pytest $BUILD_DIRECTORY/tests/python_package_test || exit 1
Guolin Ke's avatar
Guolin Ke committed
222
        exit 0
223
    elif [[ $METHOD == "wheel" ]]; then
224
225
        cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --gpu || exit 1
        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit 1
226
        pip install $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER*.whl -v || exit 1
227
        pytest $BUILD_DIRECTORY/tests || exit 1
228
229
        exit 0
    elif [[ $METHOD == "source" ]]; then
230
        cmake -B build -S . -DUSE_GPU=ON
Guolin Ke's avatar
Guolin Ke committed
231
    fi
232
233
elif [[ $TASK == "cuda" ]]; then
    sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "cuda";/' $BUILD_DIRECTORY/include/LightGBM/config.h
234
    grep -q 'std::string device_type = "cuda"' $BUILD_DIRECTORY/include/LightGBM/config.h || exit 1  # make sure that changes were really done
235
236
    # by default ``gpu_use_dp=false`` for efficiency. change to ``true`` here for exact results in ci tests
    sed -i'.bak' 's/gpu_use_dp = false;/gpu_use_dp = true;/' $BUILD_DIRECTORY/include/LightGBM/config.h
237
    grep -q 'gpu_use_dp = true' $BUILD_DIRECTORY/include/LightGBM/config.h || exit 1  # make sure that changes were really done
238
    if [[ $METHOD == "pip" ]]; then
239
240
        cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit 1
        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit 1
241
242
        pip install \
            -v \
243
            --config-settings=cmake.define.USE_CUDA=ON \
244
            $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz \
245
246
        || exit 1
        pytest $BUILD_DIRECTORY/tests/python_package_test || exit 1
247
        exit 0
248
    elif [[ $METHOD == "wheel" ]]; then
249
250
        cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --cuda || exit 1
        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit 1
251
        pip install $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER*.whl -v || exit 1
252
        pytest $BUILD_DIRECTORY/tests || exit 1
253
254
        exit 0
    elif [[ $METHOD == "source" ]]; then
255
        cmake -B build -S . -DUSE_CUDA=ON
256
    fi
257
elif [[ $TASK == "mpi" ]]; then
258
    if [[ $METHOD == "pip" ]]; then
259
260
        cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit 1
        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit 1
261
262
        pip install \
            -v \
263
            --config-settings=cmake.define.USE_MPI=ON \
264
            $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz \
265
266
        || exit 1
        pytest $BUILD_DIRECTORY/tests/python_package_test || exit 1
267
        exit 0
268
    elif [[ $METHOD == "wheel" ]]; then
269
270
        cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --mpi || exit 1
        sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit 1
271
        pip install $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER*.whl -v || exit 1
272
        pytest $BUILD_DIRECTORY/tests || exit 1
273
274
        exit 0
    elif [[ $METHOD == "source" ]]; then
275
        cmake -B build -S . -DUSE_MPI=ON -DUSE_DEBUG=ON
276
    fi
Guolin Ke's avatar
Guolin Ke committed
277
else
278
    cmake -B build -S .
Guolin Ke's avatar
Guolin Ke committed
279
280
fi

281
cmake --build build --target _lightgbm -j4 || exit 1
Guolin Ke's avatar
Guolin Ke committed
282

283
cd $BUILD_DIRECTORY && sh ./build-python.sh install --precompile || exit 1
284
pytest $BUILD_DIRECTORY/tests || exit 1
Guolin Ke's avatar
Guolin Ke committed
285
286

if [[ $TASK == "regular" ]]; then
287
    if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
288
        if [[ $OS_NAME == "macos" ]]; then
289
            cp $BUILD_DIRECTORY/lib_lightgbm.dylib $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.dylib
290
        else
291
            if [[ $COMPILER == "gcc" ]]; then
292
293
                objdump -T $BUILD_DIRECTORY/lib_lightgbm.so > $BUILD_DIRECTORY/objdump.log || exit 1
                python $BUILD_DIRECTORY/helpers/check_dynamic_dependencies.py $BUILD_DIRECTORY/objdump.log || exit 1
294
            fi
295
296
            cp $BUILD_DIRECTORY/lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.so
        fi
Guolin Ke's avatar
Guolin Ke committed
297
    fi
298
    cd $BUILD_DIRECTORY/examples/python-guide
Guolin Ke's avatar
Guolin Ke committed
299
300
301
302
    sed -i'.bak' '/import lightgbm as lgb/a\
import matplotlib\
matplotlib.use\(\"Agg\"\)\
' plot_example.py  # prevent interactive window mode
303
    sed -i'.bak' 's/graph.render(view=True)/graph.render(view=False)/' plot_example.py
304
    # requirements for examples
305
306
307
308
    mamba install -y -n $CONDA_ENV \
        'h5py>=3.10' \
        'ipywidgets>=8.1.2' \
        'notebook>=7.1.2'
309
    for f in *.py **/*.py; do python $f || exit 1; done  # run all examples
310
    cd $BUILD_DIRECTORY/examples/python-guide/notebooks
311
    sed -i'.bak' 's/INTERACTIVE = False/assert False, \\"Interactive mode disabled\\"/' interactive_plot_example.ipynb
312
    jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb || exit 1  # run all notebooks
313
314

    # importing the library should succeed even if all optional dependencies are not present
315
    conda uninstall -n $CONDA_ENV --force --yes \
316
        cffi \
317
        dask \
318
319
        distributed \
        joblib \
320
        matplotlib-base \
321
        psutil \
322
        pyarrow \
323
        python-graphviz \
324
325
        scikit-learn || exit 1
    python -c "import lightgbm" || exit 1
Guolin Ke's avatar
Guolin Ke committed
326
fi