[ci] [R-package] ensure that MSVC jobs fail when tests fail (fixes #5439) (#5448)

.ci/test_r_package_windows.ps1

@@ -283,7 +283,10 @@ if ($env:R_BUILD_TYPE -eq "cmake") {
 if ($env:COMPILER -eq "MSVC") {
   Write-Output "Running tests with testthat.R"
   cd R-package/tests
-  Run-R-Code-Redirect-Stderr "source('testthat.R')" ; Check-Output $?
+  # NOTE: using Rscript.exe intentionally here, instead of Run-R-Code-Redirect-Stderr,
+  #       because something about the interaction between Run-R-Code-Redirect-Stderr
+  #       and testthat results in failing tests not exiting with a non-0 exit code.
+  Rscript.exe --vanilla "testthat.R" ; Check-Output $?
 }
 
 Write-Output "No issues were found checking the R package"

R-package/tests/testthat/test_dataset.R

@@ -218,6 +218,10 @@ test_that("lgb.Dataset: Dataset should be able to construct from matrix and retu
 })
 
 test_that("cpp errors should be raised as proper R errors", {
+  testthat::skip_if(
+    Sys.getenv("COMPILER", "") == "MSVC" && as.integer(R.Version()[["major"]]) < 4L
+    , message = "Skipping on R 3.x and Visual Studio"
+  )
   data(agaricus.train, package = "lightgbm")
   train <- agaricus.train
   dtrain <- lgb.Dataset(