[R] Fix 32-bit build (#3307)

* use integer type for handle

* Apply suggestions from code review

* fix 64bit handle

R-package/R/lgb.Booster.R

@@ -31,7 +31,7 @@ Booster <- R6::R6Class(
 
       # Create parameters and handle
       params <- append(params, list(...))
-      handle <- 0.0
+      handle <- lgb.null.handle()
 
       # Attempts to create a handle for the dataset
       try({

R-package/R/lgb.Dataset.R

@@ -186,7 +186,7 @@ Dataset <- R6::R6Class(
       if (!is.null(private$reference)) {
         ref_handle <- private$reference$.__enclos_env__$private$get_handle()
       }
-      handle <- NA_real_
+      handle <- lgb.null.handle()
 
       # Not subsetting
       if (is.null(private$used_indices)) {

R-package/R/utils.R

@@ -6,6 +6,14 @@ lgb.is.Dataset <- function(x) {
   lgb.check.r6.class(x, "lgb.Dataset") # Checking if it is of class lgb.Dataset or not
 }
 
+lgb.null.handle <- function() {
+  if (.Machine$sizeof.pointer == 8L) {
+    return(NA_real_)
+  } else {
+    return(NA_integer_)
+  }
+}
+
 lgb.is.null.handle <- function(x) {
   is.null(x) || is.na(x)
 }

R-package/tests/testthat/test_dataset.R

@@ -73,7 +73,7 @@ test_that("lgb.Dataset: nrow is correct for a very sparse matrix", {
 
 test_that("lgb.Dataset: Dataset should be able to construct from matrix and return non-null handle", {
   rawData <- matrix(runif(1000L), ncol = 10L)
-  handle <- NA_real_
+  handle <- lgb.null.handle()
   ref_handle <- NULL
   handle <- lightgbm:::lgb.call(
     "LGBM_DatasetCreateFromMat_R"