[ci] consolidate R package installs into a CI script (#6808)

* [ci] consolidate R package installs into a CI script

* use CRAN_MIRROR and R_LIB_PATH consistently

* shorter lines and fix reference

* fix type option

* add extension

* fix invocations on Windows

* more fixes

* binary

* use package_type binary on Windows

* Update .ci/test-r-package-windows.ps1

* more fixes

* Apply suggestions from code review
Co-authored-by: Nikita Titov <nekit94-08@mail.ru>

* make install-r-deps more flexible

* Apply suggestions from code review
Co-authored-by: Nikita Titov <nekit94-08@mail.ru>

---------
Co-authored-by: Nikita Titov <nekit94-08@mail.ru>

.ci/install-r-deps.R

+# Install R dependencies, using only base R.
+#
+# Supported arguments:
+#
+#  --all                     Install all the 'Depends', 'Imports', 'LinkingTo', and 'Suggests' dependencies
+#                            (automatically implies --build --test).
+#
+#  --build                   Install the packages needed to build.
+#
+#  --exclude=<pkg1,pkg2,...> Comma-delimited list of packages to NOT install.
+#
+#  --include=<pkg1,pkg2,...> Comma-delimited list of additional packages to install.
+#                            These will always be installed, unless also used in "--exclude".
+#
+#  --test                    Install packages needed to run tests.
+#
+
+
+# [description] Parse command line arguments into an R list.
+#               Returns a list where keys are arguments and values
+#               are either TRUE (for flags) or a vector of values passed via a
+#               comma-delimited list.
+.parse_args <- function(args) {
+    out <- list(
+        "--all"       = FALSE
+        , "--build"   = FALSE
+        , "--exclude" = character(0L)
+        , "--include" = character(0L)
+        , "--test"    = FALSE
+    )
+    for (arg in args) {
+        parsed_arg <- unlist(strsplit(arg, "=", fixed = TRUE))
+        arg_name <- parsed_arg[[1L]]
+        if (!(arg_name %in% names(out))) {
+            stop(sprintf("Unrecognized argument: '%s'", arg_name))
+        }
+        if (length(parsed_arg) == 2L) {
+            # lists, like "--include=roxygen2,testthat"
+            values <- unlist(strsplit(parsed_arg[[2L]], ",", fixed = TRUE))
+            out[[arg_name]] <- values
+        } else {
+            # flags, like "--build"
+            out[[arg]] <- TRUE
+        }
+    }
+    return(out)
+}
+
+args <- .parse_args(
+    commandArgs(trailingOnly = TRUE)
+)
+
+# which dependencies to install
+ALL_DEPS     <- isTRUE(args[["--all"]])
+BUILD_DEPS   <- ALL_DEPS || isTRUE(args[["--build"]])
+TEST_DEPS    <- ALL_DEPS || isTRUE(args[["--test"]])
+
+# force downloading of binary packages on macOS
+COMPILE_FROM_SOURCE <- "both"
+PACKAGE_TYPE <- getOption("pkgType")
+
+# CRAN has precompiled binaries for macOS and Windows... prefer those,
+# for faster installation.
+if (Sys.info()[["sysname"]] == "Darwin" || .Platform$OS.type == "windows") {
+    COMPILE_FROM_SOURCE <- "never"
+    PACKAGE_TYPE <- "binary"
+}
+options(
+    install.packages.check.source = "no"
+    , install.packages.compile.from.source = COMPILE_FROM_SOURCE
+)
+
+# always use the same CRAN mirror
+CRAN_MIRROR <- Sys.getenv("CRAN_MIRROR", unset = "https://cran.r-project.org")
+
+# we always want these
+deps_to_install <- c(
+    "data.table"
+    , "jsonlite"
+    , "Matrix"
+    , "R6"
+)
+
+if (isTRUE(BUILD_DEPS)) {
+    deps_to_install <- c(
+        deps_to_install
+        , "knitr"
+        , "markdown"
+    )
+}
+
+if (isTRUE(TEST_DEPS)) {
+    deps_to_install <- c(
+        deps_to_install
+        , "RhpcBLASctl"
+        , "testthat"
+    )
+}
+
+# add packages passed through '--include'
+deps_to_install <- unique(c(
+    deps_to_install
+    , args[["--include"]]
+))
+
+# remove packages passed through '--exclude'
+deps_to_install <- setdiff(
+    x = deps_to_install
+    , args[["--exclude"]]
+)
+
+msg <- sprintf(
+    "[install-r-deps] installing R packages: %s\n"
+    , toString(sort(deps_to_install))
+)
+cat(msg)
+
+install.packages(  # nolint[undesirable_function]
+    pkgs = deps_to_install
+    , dependencies = c("Depends", "Imports", "LinkingTo")
+    , lib = Sys.getenv("R_LIB_PATH", unset = .libPaths()[[1L]])
+    , repos = CRAN_MIRROR
+    , type = PACKAGE_TYPE
+    , Ncpus = parallel::detectCores()
+)

.ci/test-r-package-valgrind.sh

@@ -2,7 +2,7 @@
 
 set -e -E -u -o pipefail
 
-RDscriptvalgrind -e "install.packages(c('R6', 'data.table', 'jsonlite', 'Matrix', 'RhpcBLASctl', 'testthat'), repos = 'https://cran.rstudio.com')" || exit 1
+RDscriptvalgrind ./.ci/install-r-deps.R --test || exit 1
 sh build-cran-package.sh \
   --r-executable=RDvalgrind \
   --no-build-vignettes \

.ci/test-r-package-windows.ps1

@@ -176,16 +176,7 @@ Remove-Item "$env:RTOOLS_MINGW_BIN/cmake.exe" -Force -ErrorAction Ignore
 Write-Output "Done installing CMake"
 
 Write-Output "Installing dependencies"
-$packages = -join @(
-    "c('data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'processx', 'R6', 'RhpcBLASctl', 'testthat'), ",
-    "dependencies = c('Imports', 'Depends', 'LinkingTo')"
-)
-$params = -join @(
-    "options(install.packages.check.source = 'no'); ",
-    "install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', ",
-    "lib = '$env:R_LIB_PATH', Ncpus = parallel::detectCores())"
-)
-Invoke-R-Code-Redirect-Stderr $params ; Assert-Output $?
+Rscript.exe --vanilla ".ci/install-r-deps.R" --build --include=processx --test ; Assert-Output $?
 
 Write-Output "Building R-package"
 

.ci/test-r-package.sh

@@ -6,8 +6,8 @@ set -e -E -u -o pipefail
 ARCH=$(uname -m)
 
 # set up R environment
-CRAN_MIRROR="https://cran.rstudio.com"
-R_LIB_PATH=~/Rlib
+export CRAN_MIRROR="https://cran.rstudio.com"
+export R_LIB_PATH=~/Rlib
 mkdir -p $R_LIB_PATH
 export R_LIBS=$R_LIB_PATH
 export PATH="$R_LIB_PATH/R/bin:$PATH"
@@ -112,15 +112,8 @@ Rscript --vanilla -e "install.packages('lattice', repos = '${CRAN_MIRROR}', lib
 # ref: https://github.com/microsoft/LightGBM/issues/6433
 Rscript --vanilla -e "install.packages('https://cran.r-project.org/src/contrib/Archive/Matrix/Matrix_1.6-5.tar.gz', repos = NULL, lib = '${R_LIB_PATH}')"
 
-# Manually install Depends and Imports libraries + 'knitr', 'markdown', 'RhpcBLASctl', 'testthat'
-# to avoid a CI-time dependency on devtools (for devtools::install_deps())
-packages="c('data.table', 'jsonlite', 'knitr', 'markdown', 'R6', 'RhpcBLASctl', 'testthat')"
-compile_from_source="both"
-if [[ $OS_NAME == "macos" ]]; then
-    packages+=", type = 'binary'"
-    compile_from_source="never"
-fi
-Rscript --vanilla -e "options(install.packages.compile.from.source = '${compile_from_source}'); install.packages(${packages}, repos = '${CRAN_MIRROR}', lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'), Ncpus = parallel::detectCores())" || exit 1
+# Manually install dependencies to avoid a CI-time dependency on devtools (for devtools::install_deps())
+Rscript --vanilla ./.ci/install-r-deps.R --build --test --exclude=Matrix || exit 1
 
 cd "${BUILD_DIRECTORY}"
 PKG_TARBALL="lightgbm_$(head -1 VERSION.txt).tar.gz"

.github/workflows/r_package.yml

@@ -245,9 +245,7 @@ jobs:
       - name: Install packages
         shell: bash
         run: |
-          R_LIBS="c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl', 'testthat')"
-          RDscript${{ matrix.r_customization }} \
-            -e "install.packages(${R_LIBS}, repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())"
+          RDscript${{ matrix.r_customization }} ./.ci/install-r-deps.R --build --test
           sh build-cran-package.sh --r-executable=RD${{ matrix.r_customization }}
           RD${{ matrix.r_customization }} CMD INSTALL lightgbm_*.tar.gz || exit 1
       - name: Run tests with sanitizers
@@ -313,9 +311,7 @@ jobs:
       - name: Install packages and run tests
         shell: bash
         run: |
-          R_LIBS="c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl')"
-          Rscript \
-            -e "install.packages(${R_LIBS}, repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())"
+          Rscript ./.ci/install-r-deps.R --build --test --exclude=testthat
           sh build-cran-package.sh
 
           # 'rchk' isn't run through 'R CMD check', use the approach documented at

.github/workflows/static_analysis.yml

@@ -63,12 +63,10 @@ jobs:
           submodules: true
       - name: Install packages
         shell: bash
-        # yamllint disable rule:line-length
         run: |
-          Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl', 'roxygen2', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())"
+          Rscript ./.ci/install-r-deps.R --build --include=roxygen2
           sh build-cran-package.sh || exit 1
           R CMD INSTALL --with-keep.source lightgbm_*.tar.gz || exit 1
-        # yamllint enable rule:line-length
       - name: Test documentation
         shell: bash --noprofile --norc {0}
         run: |

.vsts-ci.yml

@@ -414,16 +414,14 @@ jobs:
       - script: |
           git clean -d -f -x
         displayName: 'Clean source directory'
-      # yamllint disable rule:line-length
       - script: |
           LGB_VER=$(head -n 1 VERSION.txt | sed "s/rc/-/g")
           R_LIB_PATH=~/Rlib
           export R_LIBS=${R_LIB_PATH}
           mkdir -p ${R_LIB_PATH}
-          RDscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl'),  lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" || exit 1
+          RDscript .ci/install-r-deps.R --build --include=RhpcBLASctl || exit 1
           sh build-cran-package.sh --r-executable=RD || exit 1
           mv lightgbm_${LGB_VER}.tar.gz $(Build.ArtifactStagingDirectory)/lightgbm-${LGB_VER}-r-cran.tar.gz
-        # yamllint enable rule:line-length
         displayName: 'Build CRAN R-package'
       - task: PublishBuildArtifacts@1
         condition: succeeded()