test_r_package.sh 10 KB
Newer Older
1
2
3
#!/bin/bash

# set up R environment
4
CRAN_MIRROR="https://cran.rstudio.com"
5
6
R_LIB_PATH=~/Rlib
mkdir -p $R_LIB_PATH
7
export R_LIBS=$R_LIB_PATH
8
9
export PATH="$R_LIB_PATH/R/bin:$PATH"

10
11
12
13
14
15
# don't fail builds for long-running examples unless they're very long.
# See https://github.com/microsoft/LightGBM/issues/4049#issuecomment-793412254.
if [[ $R_BUILD_TYPE != "cran" ]]; then
    export _R_CHECK_EXAMPLE_TIMING_THRESHOLD_=30
fi

16
# Get details needed for installing R components
17
18
19
R_MAJOR_VERSION=( ${R_VERSION//./ } )
if [[ "${R_MAJOR_VERSION}" == "3" ]]; then
    export R_MAC_VERSION=3.6.3
20
    export R_MAC_PKG_URL=${CRAN_MIRROR}/bin/macosx/R-${R_MAC_VERSION}.nn.pkg
21
22
23
    export R_LINUX_VERSION="3.6.3-1bionic"
    export R_APT_REPO="bionic-cran35/"
elif [[ "${R_MAJOR_VERSION}" == "4" ]]; then
24
25
26
    export R_MAC_VERSION=4.3.1
    export R_MAC_PKG_URL=${CRAN_MIRROR}/bin/macosx/big-sur-x86_64/base/R-${R_MAC_VERSION}-x86_64.pkg
    export R_LINUX_VERSION="4.3.1-1.2204.0"
27
    export R_APT_REPO="jammy-cran40/"
28
29
else
    echo "Unrecognized R version: ${R_VERSION}"
30
    exit 1
31
32
fi

33
34
35
# installing precompiled R for Ubuntu
# https://cran.r-project.org/bin/linux/ubuntu/#installation
# adding steps from https://stackoverflow.com/a/56378217/3986677 to get latest version
36
37
#
# `devscripts` is required for 'checkbashisms' (https://github.com/r-lib/actions/issues/111)
38
if [[ $OS_NAME == "linux" ]]; then
39
40
    mkdir -p ~/.gnupg
    echo "disable-ipv6" >> ~/.gnupg/dirmngr.conf
41
    sudo apt-key adv \
42
        --homedir ~/.gnupg \
43
        --keyserver keyserver.ubuntu.com \
44
        --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 || exit 1
45
    sudo add-apt-repository \
46
        "deb ${CRAN_MIRROR}/bin/linux/ubuntu ${R_APT_REPO}" || exit 1
47
48
49
    sudo apt-get update
    sudo apt-get install \
        --no-install-recommends \
50
        -y \
51
            devscripts \
52
            r-base-core=${R_LINUX_VERSION} \
53
            r-base-dev=${R_LINUX_VERSION} \
54
            texinfo \
55
            texlive-latex-extra \
56
57
58
            texlive-latex-recommended \
            texlive-fonts-recommended \
            texlive-fonts-extra \
59
            tidy \
60
            qpdf \
61
            || exit 1
62
63
64
65
66
67

    if [[ $R_BUILD_TYPE == "cran" ]]; then
        sudo apt-get install \
            --no-install-recommends \
            -y \
                autoconf=$(cat R-package/AUTOCONF_UBUNTU_VERSION) \
68
                automake \
69
                || exit 1
70
    fi
71
72
73
    if [[ $INSTALL_CMAKE_FROM_RELEASES == "true" ]]; then
        curl -O -L \
            https://github.com/Kitware/CMake/releases/download/v3.25.1/cmake-3.25.1-linux-x86_64.sh \
74
        || exit 1
75

76
77
78
        sudo mkdir /opt/cmake || exit 1
        sudo sh cmake-3.25.1-linux-x86_64.sh --skip-license --prefix=/opt/cmake || exit 1
        sudo ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake || exit 1
79
    fi
80
81
82
83
fi

# Installing R precompiled for Mac OS 10.11 or higher
if [[ $OS_NAME == "macos" ]]; then
84
    if [[ $R_BUILD_TYPE == "cran" ]]; then
85
        brew install automake || exit 1
86
    fi
87
88
    brew install \
        checkbashisms \
89
90
        qpdf || exit 1
    brew install basictex || exit 1
91
    export PATH="/Library/TeX/texbin:$PATH"
92
93
    sudo tlmgr --verify-repo=none update --self || exit 1
    sudo tlmgr --verify-repo=none install inconsolata helvetic rsfs || exit 1
94

95
    curl -sL ${R_MAC_PKG_URL} -o R.pkg || exit 1
96
97
    sudo installer \
        -pkg $(pwd)/R.pkg \
98
        -target / || exit 1
99
100
fi

101
102
103
# fix for issue where CRAN was not returning {lattice} when using R 3.6
# "Warning: dependency ‘lattice’ is not available"
if [[ "${R_MAJOR_VERSION}" == "3" ]]; then
104
    Rscript --vanilla -e "install.packages('https://cran.r-project.org/src/contrib/Archive/lattice/lattice_0.20-41.tar.gz', repos = NULL, lib = '${R_LIB_PATH}')"
105
106
fi

107
# Manually install Depends and Imports libraries + 'knitr', 'markdown', 'RhpcBLASctl', 'testthat'
108
# to avoid a CI-time dependency on devtools (for devtools::install_deps())
109
110
# NOTE: testthat is not required when running rchk
if [[ "${TASK}" == "r-rchk" ]]; then
111
    packages="c('data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'R6', 'RhpcBLASctl')"
112
else
113
    packages="c('data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'R6', 'RhpcBLASctl', 'testthat')"
114
fi
115
compile_from_source="both"
116
if [[ $OS_NAME == "macos" ]]; then
117
118
    packages+=", type = 'binary'"
    compile_from_source="never"
119
fi
120
Rscript --vanilla -e "options(install.packages.compile.from.source = '${compile_from_source}'); install.packages(${packages}, repos = '${CRAN_MIRROR}', lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'), Ncpus = parallel::detectCores())" || exit 1
121
122
123

cd ${BUILD_DIRECTORY}

Guolin Ke's avatar
Guolin Ke committed
124
PKG_TARBALL="lightgbm_*.tar.gz"
125
LOG_FILE_NAME="lightgbm.Rcheck/00check.log"
126
if [[ $R_BUILD_TYPE == "cmake" ]]; then
127
    Rscript build_r.R -j4 --skip-install || exit 1
128
129
130
131
132
elif [[ $R_BUILD_TYPE == "cran" ]]; then

    # on Linux, we recreate configure in CI to test if
    # a change in a PR has changed configure.ac
    if [[ $OS_NAME == "linux" ]]; then
133
        ${BUILD_DIRECTORY}/R-package/recreate-configure.sh
134
135
136
137
138
139
140
141
142
143

        num_files_changed=$(
            git diff --name-only | wc -l
        )
        if [[ ${num_files_changed} -gt 0 ]]; then
            echo "'configure' in the R package has changed. Please recreate it and commit the changes."
            echo "Changed files:"
            git diff --compact-summary
            echo "See R-package/README.md for details on how to recreate this script."
            echo ""
144
            exit 1
145
146
147
        fi
    fi

148
    ./build-cran-package.sh || exit 1
149

150
151
152
153
154
155
156
157
158
159
    if [[ "${TASK}" == "r-rchk" ]]; then
        echo "Checking R package with rchk"
        mkdir -p packages
        cp ${PKG_TARBALL} packages
        RCHK_LOG_FILE="rchk-logs.txt"
        docker run \
            -v $(pwd)/packages:/rchk/packages \
            kalibera/rchk:latest \
            "/rchk/packages/${PKG_TARBALL}" \
        2>&1 > ${RCHK_LOG_FILE} \
160
        || (cat ${RCHK_LOG_FILE} && exit 1)
161
162
        cat ${RCHK_LOG_FILE}

James Lamb's avatar
James Lamb committed
163
        # the exceptions below are from R itself and not LightGBM:
164
165
166
167
        # https://github.com/kalibera/rchk/issues/22#issuecomment-656036156
        exit $(
            cat ${RCHK_LOG_FILE} \
            | grep -v "in function strptime_internal" \
James Lamb's avatar
James Lamb committed
168
            | grep -v "in function RunGenCollect" \
169
170
171
172
            | grep --count -E '\[PB\]|ERROR'
        )
    fi

173
174
175
176
177
178
179
180
181
    # Test CRAN source .tar.gz in a directory that is not this repo or below it.
    # When people install.packages('lightgbm'), they won't have the LightGBM
    # git repo around. This is to protect against the use of relative paths
    # like ../../CMakeLists.txt that would only work if you are in the repo
    R_CMD_CHECK_DIR="${HOME}/tmp-r-cmd-check/"
    mkdir -p ${R_CMD_CHECK_DIR}
    mv ${PKG_TARBALL} ${R_CMD_CHECK_DIR}
    cd ${R_CMD_CHECK_DIR}
fi
182
183
184

# fails tests if either ERRORs or WARNINGs are thrown by
# R CMD CHECK
185
check_succeeded="yes"
186
187
188
(
    R CMD check ${PKG_TARBALL} \
        --as-cran \
189
        --run-donttest \
190
191
192
193
194
195
196
197
198
199
200
201
    || check_succeeded="no"
) &

# R CMD check suppresses output, some CIs kill builds after
# a few minutes with no output. This trick gives R CMD check more time
#     * https://github.com/travis-ci/travis-ci/issues/4190#issuecomment-169987525
#     * https://stackoverflow.com/a/29890106/3986677
CHECK_PID=$!
while kill -0 ${CHECK_PID} >/dev/null 2>&1; do
    echo -n -e " \b"
    sleep 5
done
202
203

echo "R CMD check build logs:"
204
205
BUILD_LOG_FILE=lightgbm.Rcheck/00install.out
cat ${BUILD_LOG_FILE}
206
207

if [[ $check_succeeded == "no" ]]; then
208
    exit 1
209
fi
210

211
212
213
214
215
216
used_correct_r_version=$(
    cat $LOG_FILE_NAME \
    | grep --count "using R version ${R_VERSION}"
)
if [[ $used_correct_r_version -ne 1 ]]; then
    echo "Unexpected R version was used. Expected '${R_VERSION}'."
217
    exit 1
218
219
220
221
222
223
224
225
226
fi

if [[ $R_BUILD_TYPE == "cmake" ]]; then
    passed_correct_r_version_to_cmake=$(
        cat $BUILD_LOG_FILE \
        | grep --count "R version passed into FindLibR.cmake: ${R_VERSION}"
    )
    if [[ $used_correct_r_version -ne 1 ]]; then
        echo "Unexpected R version was passed into cmake. Expected '${R_VERSION}'."
227
        exit 1
228
229
230
231
    fi
fi


232
233
if grep -q -E "NOTE|WARNING|ERROR" "$LOG_FILE_NAME"; then
    echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check"
234
    exit 1
235
fi
236

237
# this check makes sure that CI builds of the package actually use OpenMP
238
239
240
if [[ $OS_NAME == "macos" ]] && [[ $R_BUILD_TYPE == "cran" ]]; then
    omp_working=$(
        cat $BUILD_LOG_FILE \
241
        | grep --count -E "checking whether OpenMP will work .*yes"
242
    )
243
244
245
246
247
248
249
250
251
252
elif [[ $R_BUILD_TYPE == "cmake" ]]; then
    omp_working=$(
        cat $BUILD_LOG_FILE \
        | grep --count -E ".*Found OpenMP: TRUE.*"
    )
else
    omp_working=1
fi
if [[ $omp_working -ne 1 ]]; then
    echo "OpenMP was not found"
253
    exit 1
254
fi
255

256
# this check makes sure that CI builds of the package
257
258
259
260
261
262
# actually use MM_PREFETCH preprocessor definition
if [[ $R_BUILD_TYPE == "cran" ]]; then
    mm_prefetch_working=$(
        cat $BUILD_LOG_FILE \
        | grep --count -E "checking whether MM_PREFETCH work.*yes"
    )
263
264
265
266
267
268
269
270
else
    mm_prefetch_working=$(
        cat $BUILD_LOG_FILE \
        | grep --count -E ".*Performing Test MM_PREFETCH - Success"
    )
fi
if [[ $mm_prefetch_working -ne 1 ]]; then
    echo "MM_PREFETCH test was not passed"
271
    exit 1
272
273
fi

274
# this check makes sure that CI builds of the package
275
276
277
278
279
280
# actually use MM_MALLOC preprocessor definition
if [[ $R_BUILD_TYPE == "cran" ]]; then
    mm_malloc_working=$(
        cat $BUILD_LOG_FILE \
        | grep --count -E "checking whether MM_MALLOC work.*yes"
    )
281
282
283
284
285
286
287
288
else
    mm_malloc_working=$(
        cat $BUILD_LOG_FILE \
        | grep --count -E ".*Performing Test MM_MALLOC - Success"
    )
fi
if [[ $mm_malloc_working -ne 1 ]]; then
    echo "MM_MALLOC test was not passed"
289
    exit 1
290
291
fi

292
293
294
295
296
# this check makes sure that no "warning: unknown pragma ignored" logs
# reach the user leading them to believe that something went wrong
if [[ $R_BUILD_TYPE == "cran" ]]; then
    pragma_warning_present=$(
        cat $BUILD_LOG_FILE \
297
        | grep --count -E "warning: unknown pragma ignored"
298
299
300
    )
    if [[ $pragma_warning_present -ne 0 ]]; then
        echo "Unknown pragma warning is present, pragmas should have been removed before build"
301
        exit 1
302
303
    fi
fi