test_r_package_windows.ps1 12.4 KB
Newer Older
1
2
3
4
5
6
7
# Download a file and retry upon failure. This looks like
# an infinite loop but CI-level timeouts will kill it
function Download-File-With-Retries {
  param(
    [string]$url,
    [string]$destfile
  )
8
  $ProgressPreference = "SilentlyContinue"  # progress bar bug extremely slows down download speed
9
10
11
  do {
    Write-Output "Downloading ${url}"
    sleep 5;
12
    Invoke-WebRequest -Uri $url -OutFile $destfile
13
14
15
  } while(!$?);
}

16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
# External utilities like R.exe / Rscript.exe writing to stderr (even for harmless
# status information) can cause failures in GitHub Actions PowerShell jobs.
# See https://github.community/t/powershell-steps-fail-nondeterministically/115496
#
# Using standard PowerShell redirection does not work to avoid these errors.
# This function uses R's built-in redirection mechanism, sink(). Any place where
# this function is used is a command that writes harmless messages to stderr
function Run-R-Code-Redirect-Stderr {
  param(
    [string]$rcode
  )
  $decorated_code = "out_file <- file(tempfile(), open = 'wt'); sink(out_file, type = 'message'); $rcode; sink()"
  Rscript --vanilla -e $decorated_code
}

31
32
33
34
35
36
37
38
39
40
41
# Remove all items matching some pattern from PATH environment variable
function Remove-From-Path {
  param(
    [string]$pattern_to_remove
  )
  $env:PATH = ($env:PATH.Split(';') | Where-Object { $_ -notmatch "$pattern_to_remove" }) -join ';'
}

# remove some details that exist in the GitHub Actions images which might
# cause conflicts with R and other components installed by this script
$env:RTOOLS40_HOME = ""
42
43
44
45
Remove-From-Path ".*Amazon.*"
Remove-From-Path ".*Anaconda.*"
Remove-From-Path ".*android.*"
Remove-From-Path ".*Android.*"
46
47
Remove-From-Path ".*chocolatey.*"
Remove-From-Path ".*Chocolatey.*"
48
49
50
51
Remove-From-Path ".*\\Git\\.*"
Remove-From-Path "(?!.*pandoc.*).*hostedtoolcache.*"
Remove-From-Path ".*Microsoft SDKs.*"
Remove-From-Path ".*mingw.*"
52
Remove-From-Path ".*msys64.*"
53
54
55
Remove-From-Path ".*PostgreSQL.*"
Remove-From-Path ".*\\R\\.*"
Remove-From-Path ".*R Client.*"
56
Remove-From-Path ".*rtools40.*"
57
Remove-From-Path ".*rtools42.*"
58
Remove-From-Path ".*rtools43.*"
59
Remove-From-Path ".*shells.*"
60
Remove-From-Path ".*Strawberry.*"
61
Remove-From-Path ".*tools.*"
62
63

Remove-Item C:\rtools40 -Force -Recurse -ErrorAction Ignore
64
Remove-Item C:\rtools42 -Force -Recurse -ErrorAction Ignore
65
Remove-Item C:\rtools43 -Force -Recurse -ErrorAction Ignore
66

67
68
69
70
71
72
# Get details needed for installing R components
#
# NOTES:
#    * some paths and file names are different on R4.0
$env:R_MAJOR_VERSION = $env:R_VERSION.split('.')[0]
if ($env:R_MAJOR_VERSION -eq "3") {
73
74
75
  # Rtools 3.x has to be installed at C:\Rtools\
  #     * https://stackoverflow.com/a/46619260/3986677
  $RTOOLS_INSTALL_PATH = "C:\Rtools"
76
77
  $env:RTOOLS_BIN = "$RTOOLS_INSTALL_PATH\bin"
  $env:RTOOLS_MINGW_BIN = "$RTOOLS_INSTALL_PATH\mingw_64\bin"
78
  $env:RTOOLS_EXE_FILE = "rtools35-x86_64.exe"
79
80
  $env:R_WINDOWS_VERSION = "3.6.3"
} elseif ($env:R_MAJOR_VERSION -eq "4") {
81
  $RTOOLS_INSTALL_PATH = "C:\rtools43"
82
  $env:RTOOLS_BIN = "$RTOOLS_INSTALL_PATH\usr\bin"
83
  $env:RTOOLS_MINGW_BIN = "$RTOOLS_INSTALL_PATH\x86_64-w64-mingw32.static.posix\bin"
84
85
  $env:RTOOLS_EXE_FILE = "rtools43-5550-5548.exe"
  $env:R_WINDOWS_VERSION = "4.3.1"
86
87
88
89
90
} else {
  Write-Output "[ERROR] Unrecognized R version: $env:R_VERSION"
  Check-Output $false
}

91
92
$env:R_LIB_PATH = "$env:BUILD_SOURCESDIRECTORY/RLibrary" -replace '[\\]', '/'
$env:R_LIBS = "$env:R_LIB_PATH"
93
$env:PATH = "$env:RTOOLS_BIN;" + "$env:RTOOLS_MINGW_BIN;" + "$env:R_LIB_PATH/R/bin/x64;"+ $env:PATH
94
$env:CRAN_MIRROR = "https://cran.rstudio.com"
James Lamb's avatar
James Lamb committed
95
$env:MIKTEX_EXCEPTION_PATH = "$env:TEMP\miktex"
96

97
98
99
100
101
102
# don't fail builds for long-running examples unless they're very long.
# See https://github.com/microsoft/LightGBM/issues/4049#issuecomment-793412254.
if ($env:R_BUILD_TYPE -ne "cran") {
    $env:_R_CHECK_EXAMPLE_TIMING_THRESHOLD_ = 30
}

103
if (($env:COMPILER -eq "MINGW") -and ($env:R_BUILD_TYPE -eq "cmake")) {
104
105
  $env:CXX = "$env:RTOOLS_MINGW_BIN/g++.exe"
  $env:CC = "$env:RTOOLS_MINGW_BIN/gcc.exe"
106
107
108
109
110
}

cd $env:BUILD_SOURCESDIRECTORY
tzutil /s "GMT Standard Time"
[Void][System.IO.Directory]::CreateDirectory($env:R_LIB_PATH)
111

112
113
# download R and RTools
Write-Output "Downloading R and Rtools"
114
Download-File-With-Retries -url "$env:CRAN_MIRROR/bin/windows/base/old/$env:R_WINDOWS_VERSION/R-$env:R_WINDOWS_VERSION-win.exe" -destfile "R-win.exe"
115
Download-File-With-Retries -url "https://github.com/microsoft/LightGBM/releases/download/v2.0.12/$env:RTOOLS_EXE_FILE" -destfile "Rtools.exe"
116
117
118

# Install R
Write-Output "Installing R"
James Lamb's avatar
James Lamb committed
119
Start-Process -FilePath R-win.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT /DIR=$env:R_LIB_PATH/R /COMPONENTS=main,x64,i386" ; Check-Output $?
120
121
122
Write-Output "Done installing R"

Write-Output "Installing Rtools"
123
Start-Process -FilePath Rtools.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT /SUPPRESSMSGBOXES /DIR=$RTOOLS_INSTALL_PATH" ; Check-Output $?
124
125
Write-Output "Done installing Rtools"

126
Write-Output "Installing dependencies"
127
$packages = "c('data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'processx', 'R6', 'RhpcBLASctl', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')"
128
Run-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH', Ncpus = parallel::detectCores())" ; Check-Output $?
129

130
131
Write-Output "Building R package"

132
133
# R CMD check is not used for MSVC builds
if ($env:COMPILER -ne "MSVC") {
134

135
  $PKG_FILE_NAME = "lightgbm_$env:LGB_VER.tar.gz"
136
137
  $LOG_FILE_NAME = "lightgbm.Rcheck/00check.log"

138
  if ($env:R_BUILD_TYPE -eq "cmake") {
139
140
    if ($env:TOOLCHAIN -eq "MINGW") {
      Write-Output "Telling R to use MinGW"
141
      $env:BUILD_R_FLAGS = "c('--skip-install', '--use-mingw', '-j4')"
142
143
    } elseif ($env:TOOLCHAIN -eq "MSYS") {
      Write-Output "Telling R to use MSYS"
144
      $env:BUILD_R_FLAGS = "c('--skip-install', '--use-msys2', '-j4')"
145
146
147
148
149
150
151
    } elseif ($env:TOOLCHAIN -eq "MSVC") {
      $env:BUILD_R_FLAGS = "'--skip-install'"
    } else {
      Write-Output "[ERROR] Unrecognized toolchain: $env:TOOLCHAIN"
      Check-Output $false
    }
    Run-R-Code-Redirect-Stderr "commandArgs <- function(...){$env:BUILD_R_FLAGS}; source('build_r.R')"; Check-Output $?
152
  } elseif ($env:R_BUILD_TYPE -eq "cran") {
153
154
155
156
157
158
159
    # NOTE: gzip and tar are needed to create a CRAN package on Windows, but
    # some flavors of tar.exe can fail in some settings on Windows.
    # Putting the msys64 utilities at the beginning of PATH temporarily to be
    # sure they're used for that purpose.
    if ($env:R_MAJOR_VERSION -eq "3") {
      $env:PATH = "C:\msys64\usr\bin;" + $env:PATH
    }
160
    Run-R-Code-Redirect-Stderr "result <- processx::run(command = 'sh', args = 'build-cran-package.sh', echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; Check-Output $?
161
    Remove-From-Path ".*msys64.*"
162
163
164
165
166
167
168
169
170
171
    # Test CRAN source .tar.gz in a directory that is not this repo or below it.
    # When people install.packages('lightgbm'), they won't have the LightGBM
    # git repo around. This is to protect against the use of relative paths
    # like ../../CMakeLists.txt that would only work if you are in the repoo
    $R_CMD_CHECK_DIR = "tmp-r-cmd-check"
    New-Item -Path "C:\" -Name $R_CMD_CHECK_DIR -ItemType "directory" > $null
    Move-Item -Path "$PKG_FILE_NAME" -Destination "C:\$R_CMD_CHECK_DIR\" > $null
    cd "C:\$R_CMD_CHECK_DIR\"
  }

172
173
174
  Write-Output "Running R CMD check"
  if ($env:R_BUILD_TYPE -eq "cran") {
    # CRAN packages must pass without --no-multiarch (build on 64-bit and 32-bit)
175
    $check_args = "c('CMD', 'check', '--as-cran', '--run-donttest', '$PKG_FILE_NAME')"
176
  } else {
177
    $check_args = "c('CMD', 'check', '--no-multiarch', '--as-cran', '--run-donttest', '$PKG_FILE_NAME')"
178
  }
179
  Run-R-Code-Redirect-Stderr "result <- processx::run(command = 'R.exe', args = $check_args, echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; $check_succeeded = $?
180

181
  Write-Output "R CMD check build logs:"
182
  $INSTALL_LOG_FILE_NAME = "lightgbm.Rcheck\00install.out"
183
184
185
186
187
  Get-Content -Path "$INSTALL_LOG_FILE_NAME"

  Check-Output $check_succeeded

  Write-Output "Looking for issues with R CMD check results"
188
189
  if (Get-Content "$LOG_FILE_NAME" | Select-String -Pattern "NOTE|WARNING|ERROR" -CaseSensitive -Quiet) {
      echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check"
190
191
192
193
      Check-Output $False
  }

} else {
194
  $env:TMPDIR = $env:USERPROFILE  # to avoid warnings about incremental builds inside a temp directory
195
  $INSTALL_LOG_FILE_NAME = "$env:BUILD_SOURCESDIRECTORY\00install_out.txt"
196
  Run-R-Code-Redirect-Stderr "source('build_r.R')" 1> $INSTALL_LOG_FILE_NAME ; $install_succeeded = $?
197
198
199
200
  Write-Output "----- build and install logs -----"
  Get-Content -Path "$INSTALL_LOG_FILE_NAME"
  Write-Output "----- end of build and install logs -----"
  Check-Output $install_succeeded
201
202
203
204
205
  # some errors are not raised above, but can be found in the logs
  if (Get-Content "$INSTALL_LOG_FILE_NAME" | Select-String -Pattern "ERROR" -CaseSensitive -Quiet) {
      echo "ERRORs have been found installing lightgbm"
      Check-Output $False
  }
206
}
207

208
209
210
211
212
213
214
215
216
217
218
219
220
# Checking that the correct R version was used
if ($env:TOOLCHAIN -ne "MSVC") {
  $checks = Select-String -Path "${LOG_FILE_NAME}" -Pattern "using R version $env:R_WINDOWS_VERSION"
  $checks_cnt = $checks.Matches.length
} else {
  $checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "R version passed into FindLibR.* $env:R_WINDOWS_VERSION"
  $checks_cnt = $checks.Matches.length
}
if ($checks_cnt -eq 0) {
  Write-Output "Wrong R version was found (expected '$env:R_WINDOWS_VERSION'). Check the build logs."
  Check-Output $False
}

221
222
223
# Checking that we actually got the expected compiler. The R package has some logic
# to fail back to MinGW if MSVC fails, but for CI builds we need to check that the correct
# compiler was used.
224
225
226
227
228
229
if ($env:R_BUILD_TYPE -eq "cmake") {
  $checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "Check for working CXX compiler.*$env:COMPILER"
  if ($checks.Matches.length -eq 0) {
    Write-Output "The wrong compiler was used. Check the build logs."
    Check-Output $False
  }
230
231
}

232
233
# Checking that we got the right toolchain for MinGW. If using MinGW, both
# MinGW and MSYS toolchains are supported
234
if (($env:COMPILER -eq "MINGW") -and ($env:R_BUILD_TYPE -eq "cmake")) {
235
236
237
238
239
240
241
  $checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "Trying to build with.*$env:TOOLCHAIN"
  if ($checks.Matches.length -eq 0) {
    Write-Output "The wrong toolchain was used. Check the build logs."
    Check-Output $False
  }
}

242
# Checking that MM_PREFETCH preprocessor definition is actually used in CI builds.
243
244
if ($env:R_BUILD_TYPE -eq "cran") {
  $checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "checking whether MM_PREFETCH work.*yes"
245
246
247
248
249
250
251
252
253
254
  $checks_cnt = $checks.Matches.length
} elseif ($env:TOOLCHAIN -ne "MSVC") {
  $checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern ".*Performing Test MM_PREFETCH - Success"
  $checks_cnt = $checks.Matches.length
} else {
  $checks_cnt = 1
}
if ($checks_cnt -eq 0) {
  Write-Output "MM_PREFETCH preprocessor definition wasn't used. Check the build logs."
  Check-Output $False
255
256
}

257
# Checking that MM_MALLOC preprocessor definition is actually used in CI builds.
258
259
if ($env:R_BUILD_TYPE -eq "cran") {
  $checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "checking whether MM_MALLOC work.*yes"
260
261
262
263
264
265
266
267
268
269
  $checks_cnt = $checks.Matches.length
} elseif ($env:TOOLCHAIN -ne "MSVC") {
  $checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern ".*Performing Test MM_MALLOC - Success"
  $checks_cnt = $checks.Matches.length
} else {
  $checks_cnt = 1
}
if ($checks_cnt -eq 0) {
  Write-Output "MM_MALLOC preprocessor definition wasn't used. Check the build logs."
  Check-Output $False
270
271
}

272
273
274
275
276
277
278
279
280
# Checking that OpenMP is actually used in CMake builds.
if ($env:R_BUILD_TYPE -eq "cmake") {
  $checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern ".*Found OpenMP: TRUE.*"
  if ($checks.Matches.length -eq 0) {
    Write-Output "OpenMP wasn't found. Check the build logs."
    Check-Output $False
  }
}

281
282
283
if ($env:COMPILER -eq "MSVC") {
  Write-Output "Running tests with testthat.R"
  cd R-package/tests
284
285
286
287
  # NOTE: using Rscript.exe intentionally here, instead of Run-R-Code-Redirect-Stderr,
  #       because something about the interaction between Run-R-Code-Redirect-Stderr
  #       and testthat results in failing tests not exiting with a non-0 exit code.
  Rscript.exe --vanilla "testthat.R" ; Check-Output $?
288
289
290
}

Write-Output "No issues were found checking the R package"