fix R's overflow (#1960)

include/LightGBM/R_object_helper.h

@@ -124,10 +124,14 @@ typedef union { VECTOR_SER s; double align; } SEXPREC_ALIGN;
 
 #define R_INT_PTR(x)  ((int *) DATAPTR(x))
 
+#define R_INT64_PTR(x)  ((int64_t *) DATAPTR(x))
+
 #define R_REAL_PTR(x)     ((double *) DATAPTR(x))
 
 #define R_AS_INT(x) (*((int *) DATAPTR(x)))
 
+#define R_AS_INT64(x) (*((int64_t *) DATAPTR(x)))
+
 #define R_IS_NULL(x) ((*(LGBM_SE)(x)).sxpinfo.type == 0)
 
 // 64bit pointer

src/lightgbm_R.cpp

@@ -474,7 +474,7 @@ LGBM_SE LGBM_BoosterGetNumPredict_R(LGBM_SE handle,
   R_API_BEGIN();
   int64_t len;
   CHECK_CALL(LGBM_BoosterGetNumPredict(R_GET_PTR(handle), R_AS_INT(data_idx), &len));
-  R_INT_PTR(out)[0] = static_cast<int>(len);
+  R_INT64_PTR(out)[0] = len;
   R_API_END();
 }