[CUDA] Initial work for boosting and evaluation with CUDA (#5279)

* initial work for boosting and evaluation with CUDA

* fix compatibility with CPU code

* fix creating objective without USE_CUDA_EXP

* fix static analysis errors

* fix static analysis errors

src/treelearner/cuda/cuda_single_gpu_tree_learner.hpp

@@ -24,7 +24,7 @@ namespace LightGBM {
 
 class CUDASingleGPUTreeLearner: public SerialTreeLearner {
  public:
-  explicit CUDASingleGPUTreeLearner(const Config* config);
+  explicit CUDASingleGPUTreeLearner(const Config* config, const bool boosting_on_cuda);
 
   ~CUDASingleGPUTreeLearner();
 
@@ -118,6 +118,15 @@ class CUDASingleGPUTreeLearner: public SerialTreeLearner {
   score_t* cuda_gradients_;
   /*! \brief hessians on CUDA */
   score_t* cuda_hessians_;
+  /*! \brief whether boosting is done on CUDA */
+  const bool boosting_on_cuda_;
+
+  #ifdef DEBUG
+  /*! \brief gradients on CPU */
+  std::vector<score_t> host_gradients_;
+  /*! \brief hessians on CPU */
+  std::vector<score_t> host_hessians_;
+  #endif  // DEBUG
 };
 
 }  // namespace LightGBM
@@ -131,7 +140,7 @@ namespace LightGBM {
 class CUDASingleGPUTreeLearner: public SerialTreeLearner {
  public:
     #pragma warning(disable : 4702)
-    explicit CUDASingleGPUTreeLearner(const Config* tree_config) : SerialTreeLearner(tree_config) {
+    explicit CUDASingleGPUTreeLearner(const Config* tree_config, const bool /*boosting_on_cuda*/) : SerialTreeLearner(tree_config) {
       Log::Fatal("CUDA Tree Learner experimental version was not enabled in this build.\n"
                  "Please recompile with CMake option -DUSE_CUDA_EXP=1");
     }

src/treelearner/tree_learner.cpp

@@ -14,7 +14,7 @@
 namespace LightGBM {
 
 TreeLearner* TreeLearner::CreateTreeLearner(const std::string& learner_type, const std::string& device_type,
-                                            const Config* config) {
+                                            const Config* config, const bool boosting_on_cuda) {
   if (device_type == std::string("cpu")) {
     if (learner_type == std::string("serial")) {
       if (config->linear_tree) {
@@ -52,7 +52,7 @@ TreeLearner* TreeLearner::CreateTreeLearner(const std::string& learner_type, con
   } else if (device_type == std::string("cuda_exp")) {
     if (learner_type == std::string("serial")) {
       if (config->num_gpu == 1) {
-        return new CUDASingleGPUTreeLearner(config);
+        return new CUDASingleGPUTreeLearner(config, boosting_on_cuda);
       } else {
         Log::Fatal("cuda_exp only supports training on a single GPU.");
       }