gbdt.cpp 38.2 KB
Newer Older
1
2
3
4
/*!
 * Copyright (c) 2016 Microsoft Corporation. All rights reserved.
 * Licensed under the MIT License. See LICENSE file in the project root for license information.
 */
Guolin Ke's avatar
Guolin Ke committed
5
6
7
#include "gbdt.h"

#include <LightGBM/metric.h>
Guolin Ke's avatar
Guolin Ke committed
8
#include <LightGBM/network.h>
9
10
11
12
#include <LightGBM/objective_function.h>
#include <LightGBM/prediction_early_stop.h>
#include <LightGBM/utils/common.h>
#include <LightGBM/utils/openmp_wrapper.h>
Guolin Ke's avatar
Guolin Ke committed
13

14
15
16
17
#include <chrono>
#include <ctime>
#include <sstream>

Guolin Ke's avatar
Guolin Ke committed
18
19
namespace LightGBM {

20
21
Common::Timer global_timer;

22
23
24
int LGBM_config_::current_device = lgbm_device_cpu;
int LGBM_config_::current_learner = use_cpu_learner;

25
26
27
GBDT::GBDT()
    : iter_(0),
      train_data_(nullptr),
28
      config_(nullptr),
29
30
31
32
33
34
35
36
37
38
39
40
      objective_function_(nullptr),
      early_stopping_round_(0),
      es_first_metric_only_(false),
      max_feature_idx_(0),
      num_tree_per_iteration_(1),
      num_class_(1),
      num_iteration_for_pred_(0),
      shrinkage_rate_(0.1f),
      num_init_iteration_(0),
      need_re_bagging_(false),
      balanced_bagging_(false),
      bagging_runner_(0, bagging_rand_block_) {
Guolin Ke's avatar
Guolin Ke committed
41
  average_output_ = false;
Guolin Ke's avatar
Guolin Ke committed
42
  tree_learner_ = nullptr;
43
  linear_tree_ = false;
Guolin Ke's avatar
Guolin Ke committed
44
45
46
47
48
}

GBDT::~GBDT() {
}

Guolin Ke's avatar
Guolin Ke committed
49
void GBDT::Init(const Config* config, const Dataset* train_data, const ObjectiveFunction* objective_function,
50
                const std::vector<const Metric*>& training_metrics) {
Nikita Titov's avatar
Nikita Titov committed
51
  CHECK_NOTNULL(train_data);
52
  train_data_ = train_data;
53
  if (!config->monotone_constraints.empty()) {
Nikita Titov's avatar
Nikita Titov committed
54
    CHECK_EQ(static_cast<size_t>(train_data_->num_total_features()), config->monotone_constraints.size());
Nikita Titov's avatar
Nikita Titov committed
55
  }
56
  if (!config->feature_contri.empty()) {
Nikita Titov's avatar
Nikita Titov committed
57
    CHECK_EQ(static_cast<size_t>(train_data_->num_total_features()), config->feature_contri.size());
58
  }
59
  iter_ = 0;
wxchan's avatar
wxchan committed
60
  num_iteration_for_pred_ = 0;
61
  max_feature_idx_ = 0;
wxchan's avatar
wxchan committed
62
  num_class_ = config->num_class;
Guolin Ke's avatar
Guolin Ke committed
63
64
  config_ = std::unique_ptr<Config>(new Config(*config));
  early_stopping_round_ = config_->early_stopping_round;
65
  es_first_metric_only_ = config_->first_metric_only;
Guolin Ke's avatar
Guolin Ke committed
66
  shrinkage_rate_ = config_->learning_rate;
67

68
  if (config_->device_type == std::string("cuda") || config_->device_type == std::string("cuda_exp")) {
69
    LGBM_config_::current_learner = use_cuda_learner;
70
71
72
73
74
75
    #ifdef USE_CUDA_EXP
    if (config_->device_type == std::string("cuda_exp")) {
      const int gpu_device_id = config_->gpu_device_id >= 0 ? config_->gpu_device_id : 0;
      CUDASUCCESS_OR_FATAL(cudaSetDevice(gpu_device_id));
    }
    #endif  // USE_CUDA_EXP
76
77
  }

78
  // load forced_splits file
79
80
81
82
83
  if (!config->forcedsplits_filename.empty()) {
    std::ifstream forced_splits_file(config->forcedsplits_filename.c_str());
    std::stringstream buffer;
    buffer << forced_splits_file.rdbuf();
    std::string err;
Guolin Ke's avatar
Guolin Ke committed
84
    forced_splits_json_ = Json::parse(buffer.str(), &err);
85
86
  }

87
88
89
  objective_function_ = objective_function;
  num_tree_per_iteration_ = num_class_;
  if (objective_function_ != nullptr) {
Guolin Ke's avatar
Guolin Ke committed
90
    num_tree_per_iteration_ = objective_function_->NumModelPerIteration();
91
92
93
    if (objective_function_->IsRenewTreeOutput() && !config->monotone_constraints.empty()) {
      Log::Fatal("Cannot use ``monotone_constraints`` in %s objective, please disable it.", objective_function_->GetName());
    }
94
95
  }

96
97
  is_constant_hessian_ = GetIsConstHessian(objective_function);

98
  const bool boosting_on_gpu = objective_function_ != nullptr && objective_function_->IsCUDAObjective();
99
  tree_learner_ = std::unique_ptr<TreeLearner>(TreeLearner::CreateTreeLearner(config_->tree_learner, config_->device_type,
100
                                                                              config_.get(), boosting_on_gpu));
101
102
103

  // init tree learner
  tree_learner_->Init(train_data_, is_constant_hessian_);
104
  tree_learner_->SetForcedSplit(&forced_splits_json_);
105
106
107
108
109
110
111
112

  // push training metrics
  training_metrics_.clear();
  for (const auto& metric : training_metrics) {
    training_metrics_.push_back(metric);
  }
  training_metrics_.shrink_to_fit();

113
114
115
116
117
118
119
120
121
  #ifdef USE_CUDA_EXP
  if (config_->device_type == std::string("cuda_exp")) {
    train_score_updater_.reset(new CUDAScoreUpdater(train_data_, num_tree_per_iteration_, boosting_on_gpu));
  } else {
  #endif  // USE_CUDA_EXP
    train_score_updater_.reset(new ScoreUpdater(train_data_, num_tree_per_iteration_));
  #ifdef USE_CUDA_EXP
  }
  #endif  // USE_CUDA_EXP
122
123

  num_data_ = train_data_->num_data();
Andrew Ziem's avatar
Andrew Ziem committed
124
  // create buffer for gradients and Hessians
125
126
  if (objective_function_ != nullptr) {
    size_t total_size = static_cast<size_t>(num_data_) * num_tree_per_iteration_;
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
    #ifdef USE_CUDA_EXP
    if (config_->device_type == std::string("cuda_exp") && boosting_on_gpu) {
      AllocateCUDAMemory<score_t>(&gradients_pointer_, total_size, __FILE__, __LINE__);
      AllocateCUDAMemory<score_t>(&hessians_pointer_, total_size, __FILE__, __LINE__);
    } else {
    #endif  // USE_CUDA_EXP
      gradients_.resize(total_size);
      hessians_.resize(total_size);
      gradients_pointer_ = gradients_.data();
      hessians_pointer_ = hessians_.data();
    #ifdef USE_CUDA_EXP
    }
    #endif  // USE_CUDA_EXP
  #ifndef USE_CUDA_EXP
  }
  #else  // USE_CUDA_EXP
  } else {
    if (config_->device_type == std::string("cuda_exp")) {
      size_t total_size = static_cast<size_t>(num_data_) * num_tree_per_iteration_;
      AllocateCUDAMemory<score_t>(&gradients_pointer_, total_size, __FILE__, __LINE__);
      AllocateCUDAMemory<score_t>(&hessians_pointer_, total_size, __FILE__, __LINE__);
    }
149
  }
150
  #endif  // USE_CUDA_EXP
151
152
153
154
155
156
157
  // get max feature index
  max_feature_idx_ = train_data_->num_total_features() - 1;
  // get label index
  label_idx_ = train_data_->label_idx();
  // get feature names
  feature_names_ = train_data_->feature_names();
  feature_infos_ = train_data_->feature_infos();
158
  monotone_constraints_ = config->monotone_constraints;
159
160
  // get parser config file content
  parser_config_str_ = train_data_->parser_config_str();
161
162

  // if need bagging, create buffer
Guolin Ke's avatar
Guolin Ke committed
163
  ResetBaggingConfig(config_.get(), true);
164
165
166

  class_need_train_ = std::vector<bool>(num_tree_per_iteration_, true);
  if (objective_function_ != nullptr && objective_function_->SkipEmptyClass()) {
Nikita Titov's avatar
Nikita Titov committed
167
    CHECK_EQ(num_tree_per_iteration_, num_class_);
168
169
    for (int i = 0; i < num_class_; ++i) {
      class_need_train_[i] = objective_function_->ClassNeedTrain(i);
170
171
    }
  }
172
173
174
175

  if (config_->linear_tree) {
    linear_tree_ = true;
  }
wxchan's avatar
wxchan committed
176
177
178
}

void GBDT::AddValidDataset(const Dataset* valid_data,
179
                           const std::vector<const Metric*>& valid_metrics) {
wxchan's avatar
wxchan committed
180
  if (!train_data_->CheckAlign(*valid_data)) {
181
    Log::Fatal("Cannot add validation data, since it has different bin mappers with training data");
182
  }
Guolin Ke's avatar
Guolin Ke committed
183
  // for a validation dataset, we need its score and metric
184
185
186
187
188
189
190
  auto new_score_updater =
    #ifdef USE_CUDA_EXP
    config_->device_type == std::string("cuda_exp") ?
    std::unique_ptr<CUDAScoreUpdater>(new CUDAScoreUpdater(valid_data, num_tree_per_iteration_,
      objective_function_ != nullptr && objective_function_->IsCUDAObjective())) :
    #endif  // USE_CUDA_EXP
    std::unique_ptr<ScoreUpdater>(new ScoreUpdater(valid_data, num_tree_per_iteration_));
wxchan's avatar
wxchan committed
191
192
  // update score
  for (int i = 0; i < iter_; ++i) {
193
194
195
    for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
      auto curr_tree = (i + num_init_iteration_) * num_tree_per_iteration_ + cur_tree_id;
      new_score_updater->AddScore(models_[curr_tree].get(), cur_tree_id);
wxchan's avatar
wxchan committed
196
197
    }
  }
Guolin Ke's avatar
Guolin Ke committed
198
  valid_score_updater_.push_back(std::move(new_score_updater));
Guolin Ke's avatar
Guolin Ke committed
199
200
201
202
  valid_metrics_.emplace_back();
  for (const auto& metric : valid_metrics) {
    valid_metrics_.back().push_back(metric);
  }
Guolin Ke's avatar
Guolin Ke committed
203
  valid_metrics_.back().shrink_to_fit();
204

205
206
207
208
209
210
211
  if (early_stopping_round_ > 0) {
    auto num_metrics = valid_metrics.size();
    if (es_first_metric_only_) { num_metrics = 1; }
    best_iter_.emplace_back(num_metrics, 0);
    best_score_.emplace_back(num_metrics, kMinScore);
    best_msg_.emplace_back(num_metrics);
  }
Guolin Ke's avatar
Guolin Ke committed
212
213
}

Guolin Ke's avatar
Guolin Ke committed
214
void GBDT::Boosting() {
215
  Common::FunctionTimer fun_timer("GBDT::Boosting", global_timer);
Guolin Ke's avatar
Guolin Ke committed
216
217
218
219
220
221
  if (objective_function_ == nullptr) {
    Log::Fatal("No object function provided");
  }
  // objective function will calculate gradients and hessians
  int64_t num_score = 0;
  objective_function_->
222
    GetGradients(GetTrainingScore(&num_score), gradients_pointer_, hessians_pointer_);
Guolin Ke's avatar
Guolin Ke committed
223
224
}

225
data_size_t GBDT::BaggingHelper(data_size_t start, data_size_t cnt, data_size_t* buffer) {
226
227
228
  if (cnt <= 0) {
    return 0;
  }
229
  data_size_t cur_left_cnt = 0;
230
  data_size_t cur_right_pos = cnt;
231
232
  // random bagging, minimal unit is one record
  for (data_size_t i = 0; i < cnt; ++i) {
233
234
235
    auto cur_idx = start + i;
    if (bagging_rands_[cur_idx / bagging_rand_block_].NextFloat() < config_->bagging_fraction) {
      buffer[cur_left_cnt++] = cur_idx;
236
    } else {
237
      buffer[--cur_right_pos] = cur_idx;
238
239
240
241
    }
  }
  return cur_left_cnt;
}
Guolin Ke's avatar
Guolin Ke committed
242

243
244
data_size_t GBDT::BalancedBaggingHelper(data_size_t start, data_size_t cnt,
                                        data_size_t* buffer) {
Guolin Ke's avatar
Guolin Ke committed
245
246
247
248
249
  if (cnt <= 0) {
    return 0;
  }
  auto label_ptr = train_data_->metadata().label();
  data_size_t cur_left_cnt = 0;
250
  data_size_t cur_right_pos = cnt;
Guolin Ke's avatar
Guolin Ke committed
251
252
  // random bagging, minimal unit is one record
  for (data_size_t i = 0; i < cnt; ++i) {
253
    auto cur_idx = start + i;
Guolin Ke's avatar
Guolin Ke committed
254
255
256
    bool is_pos = label_ptr[start + i] > 0;
    bool is_in_bag = false;
    if (is_pos) {
257
258
      is_in_bag = bagging_rands_[cur_idx / bagging_rand_block_].NextFloat() <
                  config_->pos_bagging_fraction;
Guolin Ke's avatar
Guolin Ke committed
259
    } else {
260
261
      is_in_bag = bagging_rands_[cur_idx / bagging_rand_block_].NextFloat() <
                  config_->neg_bagging_fraction;
Guolin Ke's avatar
Guolin Ke committed
262
263
    }
    if (is_in_bag) {
264
      buffer[cur_left_cnt++] = cur_idx;
Guolin Ke's avatar
Guolin Ke committed
265
    } else {
266
      buffer[--cur_right_pos] = cur_idx;
Guolin Ke's avatar
Guolin Ke committed
267
268
269
270
271
    }
  }
  return cur_left_cnt;
}

272
void GBDT::Bagging(int iter) {
273
  Common::FunctionTimer fun_timer("GBDT::Bagging", global_timer);
Guolin Ke's avatar
Guolin Ke committed
274
  // if need bagging
Guolin Ke's avatar
Guolin Ke committed
275
276
  if ((bag_data_cnt_ < num_data_ && iter % config_->bagging_freq == 0) ||
      need_re_bagging_) {
Guolin Ke's avatar
Guolin Ke committed
277
    need_re_bagging_ = false;
278
279
280
281
282
283
284
285
    auto left_cnt = bagging_runner_.Run<true>(
        num_data_,
        [=](int, data_size_t cur_start, data_size_t cur_cnt, data_size_t* left,
            data_size_t*) {
          data_size_t cur_left_count = 0;
          if (balanced_bagging_) {
            cur_left_count =
                BalancedBaggingHelper(cur_start, cur_cnt, left);
Guolin Ke's avatar
Guolin Ke committed
286
          } else {
287
            cur_left_count = BaggingHelper(cur_start, cur_cnt, left);
Guolin Ke's avatar
Guolin Ke committed
288
          }
289
290
291
          return cur_left_count;
        },
        bag_data_indices_.data());
Guolin Ke's avatar
Guolin Ke committed
292
    bag_data_cnt_ = left_cnt;
Guolin Ke's avatar
Guolin Ke committed
293
    Log::Debug("Re-bagging, using %d data to train", bag_data_cnt_);
Guolin Ke's avatar
Guolin Ke committed
294
    // set bagging data to tree learner
Guolin Ke's avatar
Guolin Ke committed
295
    if (!is_use_subset_) {
296
297
298
299
300
301
302
303
304
305
      #ifdef USE_CUDA_EXP
      if (config_->device_type == std::string("cuda_exp")) {
        CopyFromHostToCUDADevice<data_size_t>(cuda_bag_data_indices_.RawData(), bag_data_indices_.data(), static_cast<size_t>(num_data_), __FILE__, __LINE__);
        tree_learner_->SetBaggingData(nullptr, cuda_bag_data_indices_.RawData(), bag_data_cnt_);
      } else {
      #endif  // USE_CUDA_EXP
        tree_learner_->SetBaggingData(nullptr, bag_data_indices_.data(), bag_data_cnt_);
      #ifdef USE_CUDA_EXP
      }
      #endif  // USE_CUDA_EXP
Guolin Ke's avatar
Guolin Ke committed
306
307
    } else {
      // get subset
Guolin Ke's avatar
Guolin Ke committed
308
      tmp_subset_->ReSize(bag_data_cnt_);
309
      tmp_subset_->CopySubrow(train_data_, bag_data_indices_.data(),
Guolin Ke's avatar
Guolin Ke committed
310
                              bag_data_cnt_, false);
311
312
313
314
315
316
317
318
319
320
321
322
      #ifdef USE_CUDA_EXP
      if (config_->device_type == std::string("cuda_exp")) {
        CopyFromHostToCUDADevice<data_size_t>(cuda_bag_data_indices_.RawData(), bag_data_indices_.data(), static_cast<size_t>(num_data_), __FILE__, __LINE__);
        tree_learner_->SetBaggingData(tmp_subset_.get(), cuda_bag_data_indices_.RawData(),
                                      bag_data_cnt_);
      } else {
      #endif  // USE_CUDA_EXP
        tree_learner_->SetBaggingData(tmp_subset_.get(), bag_data_indices_.data(),
                                      bag_data_cnt_);
      #ifdef USE_CUDA_EXP
      }
      #endif  // USE_CUDA_EXP
Guolin Ke's avatar
Guolin Ke committed
323
    }
Guolin Ke's avatar
Guolin Ke committed
324
325
326
  }
}

Guolin Ke's avatar
Guolin Ke committed
327
void GBDT::Train(int snapshot_freq, const std::string& model_output_path) {
328
  Common::FunctionTimer fun_timer("GBDT::Train", global_timer);
Guolin Ke's avatar
Guolin Ke committed
329
330
  bool is_finished = false;
  auto start_time = std::chrono::steady_clock::now();
Guolin Ke's avatar
Guolin Ke committed
331
  for (int iter = 0; iter < config_->num_iterations && !is_finished; ++iter) {
Guolin Ke's avatar
Guolin Ke committed
332
333
334
335
    is_finished = TrainOneIter(nullptr, nullptr);
    if (!is_finished) {
      is_finished = EvalAndCheckEarlyStopping();
    }
Guolin Ke's avatar
Guolin Ke committed
336
337
338
339
340
341
342
    auto end_time = std::chrono::steady_clock::now();
    // output used time per iteration
    Log::Info("%f seconds elapsed, finished iteration %d", std::chrono::duration<double,
              std::milli>(end_time - start_time) * 1e-3, iter + 1);
    if (snapshot_freq > 0
        && (iter + 1) % snapshot_freq == 0) {
      std::string snapshot_out = model_output_path + ".snapshot_iter_" + std::to_string(iter + 1);
343
      SaveModelToFile(0, -1, config_->saved_feature_importance_type, snapshot_out.c_str());
Guolin Ke's avatar
Guolin Ke committed
344
345
346
347
    }
  }
}

348
void GBDT::RefitTree(const std::vector<std::vector<int>>& tree_leaf_prediction) {
349
350
351
  CHECK_GT(tree_leaf_prediction.size(), 0);
  CHECK_EQ(static_cast<size_t>(num_data_), tree_leaf_prediction.size());
  CHECK_EQ(static_cast<size_t>(models_.size()), tree_leaf_prediction[0].size());
352
353
  int num_iterations = static_cast<int>(models_.size() / num_tree_per_iteration_);
  std::vector<int> leaf_pred(num_data_);
354
355
356
357
358
359
360
361
362
363
364
365
366
  if (linear_tree_) {
    std::vector<int> max_leaves_by_thread = std::vector<int>(OMP_NUM_THREADS(), 0);
    #pragma omp parallel for schedule(static)
    for (int i = 0; i < static_cast<int>(tree_leaf_prediction.size()); ++i) {
      int tid = omp_get_thread_num();
      for (size_t j = 0; j < tree_leaf_prediction[i].size(); ++j) {
        max_leaves_by_thread[tid] = std::max(max_leaves_by_thread[tid], tree_leaf_prediction[i][j]);
      }
    }
    int max_leaves = *std::max_element(max_leaves_by_thread.begin(), max_leaves_by_thread.end());
    max_leaves += 1;
    tree_learner_->InitLinear(train_data_, max_leaves);
  }
367
368
369
370
371
372
373
  for (int iter = 0; iter < num_iterations; ++iter) {
    Boosting();
    for (int tree_id = 0; tree_id < num_tree_per_iteration_; ++tree_id) {
      int model_index = iter * num_tree_per_iteration_ + tree_id;
      #pragma omp parallel for schedule(static)
      for (int i = 0; i < num_data_; ++i) {
        leaf_pred[i] = tree_leaf_prediction[i][model_index];
Nikita Titov's avatar
Nikita Titov committed
374
        CHECK_LT(leaf_pred[i], models_[model_index]->num_leaves());
375
      }
376
      size_t offset = static_cast<size_t>(tree_id) * num_data_;
377
378
      auto grad = gradients_pointer_ + offset;
      auto hess = hessians_pointer_ + offset;
379
380
381
382
383
384
385
      auto new_tree = tree_learner_->FitByExistingTree(models_[model_index].get(), leaf_pred, grad, hess);
      train_score_updater_->AddScore(tree_learner_.get(), new_tree, tree_id);
      models_[model_index].reset(new_tree);
    }
  }
}

Andrew Ziem's avatar
Andrew Ziem committed
386
/* If the custom "average" is implemented it will be used in place of the label average (if enabled)
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
*
* An improvement to this is to have options to explicitly choose
* (i) standard average
* (ii) custom average if available
* (iii) any user defined scalar bias (e.g. using a new option "init_score" that overrides (i) and (ii) )
*
* (i) and (ii) could be selected as say "auto_init_score" = 0 or 1 etc..
*
*/
double ObtainAutomaticInitialScore(const ObjectiveFunction* fobj, int class_id) {
  double init_score = 0.0;
  if (fobj != nullptr) {
    init_score = fobj->BoostFromScore(class_id);
  }
  if (Network::num_machines() > 1) {
    init_score = Network::GlobalSyncUpByMean(init_score);
  }
  return init_score;
}

Guolin Ke's avatar
Guolin Ke committed
407
double GBDT::BoostFromAverage(int class_id, bool update_scorer) {
408
  Common::FunctionTimer fun_timer("GBDT::BoostFromAverage", global_timer);
409
  // boosting from average label; or customized "average" if implemented for the current objective
410
411
412
  if (models_.empty() && !train_score_updater_->has_init_score() && objective_function_ != nullptr) {
    if (config_->boost_from_average || (train_data_ != nullptr && train_data_->num_features() == 0)) {
      double init_score = ObtainAutomaticInitialScore(objective_function_, class_id);
413
      if (std::fabs(init_score) > kEpsilon) {
Guolin Ke's avatar
Guolin Ke committed
414
415
416
417
418
        if (update_scorer) {
          train_score_updater_->AddScore(init_score, class_id);
          for (auto& score_updater : valid_score_updater_) {
            score_updater->AddScore(init_score, class_id);
          }
419
420
421
        }
        Log::Info("Start training from score %lf", init_score);
        return init_score;
Guolin Ke's avatar
Guolin Ke committed
422
      }
423
424
425
    } else if (std::string(objective_function_->GetName()) == std::string("regression_l1")
               || std::string(objective_function_->GetName()) == std::string("quantile")
               || std::string(objective_function_->GetName()) == std::string("mape")) {
426
      Log::Warning("Disabling boost_from_average in %s may cause the slow convergence", objective_function_->GetName());
427
    }
428
  }
Guolin Ke's avatar
Guolin Ke committed
429
430
  return 0.0f;
}
Guolin Ke's avatar
Guolin Ke committed
431

Guolin Ke's avatar
Guolin Ke committed
432
bool GBDT::TrainOneIter(const score_t* gradients, const score_t* hessians) {
433
  Common::FunctionTimer fun_timer("GBDT::TrainOneIter", global_timer);
434
  std::vector<double> init_scores(num_tree_per_iteration_, 0.0);
Guolin Ke's avatar
Guolin Ke committed
435
  // boosting first
Guolin Ke's avatar
Guolin Ke committed
436
  if (gradients == nullptr || hessians == nullptr) {
437
    for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
Guolin Ke's avatar
Guolin Ke committed
438
      init_scores[cur_tree_id] = BoostFromAverage(cur_tree_id, true);
439
    }
Guolin Ke's avatar
Guolin Ke committed
440
    Boosting();
441
442
443
    gradients = gradients_pointer_;
    hessians = hessians_pointer_;
  #ifndef USE_CUDA_EXP
Guolin Ke's avatar
Guolin Ke committed
444
  }
445
446
447
448
449
450
451
452
453
454
455
456
  #else  // USE_CUDA_EXP
  } else {
    if (config_->device_type == std::string("cuda_exp")) {
      const size_t total_size = static_cast<size_t>(num_data_ * num_class_);
      CopyFromHostToCUDADevice<score_t>(gradients_pointer_, gradients, total_size, __FILE__, __LINE__);
      CopyFromHostToCUDADevice<score_t>(hessians_pointer_, hessians, total_size, __FILE__, __LINE__);
      gradients = gradients_pointer_;
      hessians = hessians_pointer_;
    }
  }
  #endif  // USE_CUDA_EXP

457
458
  // bagging logic
  Bagging(iter_);
Guolin Ke's avatar
Guolin Ke committed
459

Guolin Ke's avatar
Guolin Ke committed
460
  bool should_continue = false;
461
  for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
462
    const size_t offset = static_cast<size_t>(cur_tree_id) * num_data_;
463
    std::unique_ptr<Tree> new_tree(new Tree(2, false, false));
464
    if (class_need_train_[cur_tree_id] && train_data_->num_features() > 0) {
465
466
      auto grad = gradients + offset;
      auto hess = hessians + offset;
Guolin Ke's avatar
Guolin Ke committed
467
      // need to copy gradients for bagging subset.
468
      if (is_use_subset_ && bag_data_cnt_ < num_data_ && config_->device_type != std::string("cuda_exp")) {
Guolin Ke's avatar
Guolin Ke committed
469
        for (int i = 0; i < bag_data_cnt_; ++i) {
470
471
          gradients_pointer_[offset + i] = grad[bag_data_indices_[i]];
          hessians_pointer_[offset + i] = hess[bag_data_indices_[i]];
Guolin Ke's avatar
Guolin Ke committed
472
        }
473
474
        grad = gradients_pointer_ + offset;
        hess = hessians_pointer_ + offset;
Guolin Ke's avatar
Guolin Ke committed
475
      }
476
477
      bool is_first_tree = models_.size() < static_cast<size_t>(num_tree_per_iteration_);
      new_tree.reset(tree_learner_->Train(grad, hess, is_first_tree));
478
    }
Guolin Ke's avatar
Guolin Ke committed
479

Guolin Ke's avatar
Guolin Ke committed
480
    if (new_tree->num_leaves() > 1) {
Guolin Ke's avatar
Guolin Ke committed
481
      should_continue = true;
482
      auto score_ptr = train_score_updater_->score() + offset;
483
484
      auto residual_getter = [score_ptr](const label_t* label, int i) {return static_cast<double>(label[i]) - score_ptr[i]; };
      tree_learner_->RenewTreeOutput(new_tree.get(), objective_function_, residual_getter,
485
                                     num_data_, bag_data_indices_.data(), bag_data_cnt_);
Guolin Ke's avatar
Guolin Ke committed
486
487
488
      // shrinkage by learning rate
      new_tree->Shrinkage(shrinkage_rate_);
      // update score
489
      UpdateScore(new_tree.get(), cur_tree_id);
490
491
      if (std::fabs(init_scores[cur_tree_id]) > kEpsilon) {
        new_tree->AddBias(init_scores[cur_tree_id]);
Guolin Ke's avatar
Guolin Ke committed
492
      }
493
494
    } else {
      // only add default score one-time
495
      if (models_.size() < static_cast<size_t>(num_tree_per_iteration_)) {
496
497
498
499
500
501
        if (objective_function_ != nullptr && !config_->boost_from_average && !train_score_updater_->has_init_score()) {
          init_scores[cur_tree_id] = ObtainAutomaticInitialScore(objective_function_, cur_tree_id);
          // updates scores
          train_score_updater_->AddScore(init_scores[cur_tree_id], cur_tree_id);
          for (auto& score_updater : valid_score_updater_) {
            score_updater->AddScore(init_scores[cur_tree_id], cur_tree_id);
502
          }
503
        }
504
        new_tree->AsConstantTree(init_scores[cur_tree_id]);
505
506
      }
    }
Guolin Ke's avatar
Guolin Ke committed
507
508
509
    // add model
    models_.push_back(std::move(new_tree));
  }
Guolin Ke's avatar
Guolin Ke committed
510

Guolin Ke's avatar
Guolin Ke committed
511
  if (!should_continue) {
512
    Log::Warning("Stopped training because there are no more leaves that meet the split requirements");
513
514
515
516
    if (models_.size() > static_cast<size_t>(num_tree_per_iteration_)) {
      for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
        models_.pop_back();
      }
Guolin Ke's avatar
Guolin Ke committed
517
518
519
    }
    return true;
  }
520

Guolin Ke's avatar
Guolin Ke committed
521
522
  ++iter_;
  return false;
Guolin Ke's avatar
Guolin Ke committed
523
}
524

wxchan's avatar
wxchan committed
525
void GBDT::RollbackOneIter() {
526
  if (iter_ <= 0) { return; }
wxchan's avatar
wxchan committed
527
  // reset score
528
  for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
Guolin Ke's avatar
Guolin Ke committed
529
    auto curr_tree = models_.size() - num_tree_per_iteration_ + cur_tree_id;
wxchan's avatar
wxchan committed
530
    models_[curr_tree]->Shrinkage(-1.0);
531
    train_score_updater_->AddScore(models_[curr_tree].get(), cur_tree_id);
wxchan's avatar
wxchan committed
532
    for (auto& score_updater : valid_score_updater_) {
533
      score_updater->AddScore(models_[curr_tree].get(), cur_tree_id);
wxchan's avatar
wxchan committed
534
535
536
    }
  }
  // remove model
537
  for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
wxchan's avatar
wxchan committed
538
539
540
541
542
    models_.pop_back();
  }
  --iter_;
}

Guolin Ke's avatar
Guolin Ke committed
543
bool GBDT::EvalAndCheckEarlyStopping() {
544
545
  bool is_met_early_stopping = false;
  // print message for metric
Guolin Ke's avatar
Guolin Ke committed
546
  auto best_msg = OutputMetric(iter_);
Guolin Ke's avatar
Guolin Ke committed
547
548


Guolin Ke's avatar
Guolin Ke committed
549
  is_met_early_stopping = !best_msg.empty();
550
551
  if (is_met_early_stopping) {
    Log::Info("Early stopping at iteration %d, the best iteration round is %d",
552
              iter_, iter_ - early_stopping_round_);
Guolin Ke's avatar
Guolin Ke committed
553
    Log::Info("Output of best iteration round:\n%s", best_msg.c_str());
554
    // pop last early_stopping_round_ models
555
    for (int i = 0; i < early_stopping_round_ * num_tree_per_iteration_; ++i) {
556
557
558
559
      models_.pop_back();
    }
  }
  return is_met_early_stopping;
Guolin Ke's avatar
Guolin Ke committed
560
561
}

562
void GBDT::UpdateScore(const Tree* tree, const int cur_tree_id) {
563
  Common::FunctionTimer fun_timer("GBDT::UpdateScore", global_timer);
Guolin Ke's avatar
Guolin Ke committed
564
  // update training score
Guolin Ke's avatar
Guolin Ke committed
565
  if (!is_use_subset_) {
566
    train_score_updater_->AddScore(tree_learner_.get(), tree, cur_tree_id);
Guolin Ke's avatar
Guolin Ke committed
567
568
569

    // we need to predict out-of-bag scores of data for boosting
    if (num_data_ - bag_data_cnt_ > 0) {
570
571
572
573
574
575
576
577
578
      #ifdef USE_CUDA_EXP
      if (config_->device_type == std::string("cuda_exp")) {
        train_score_updater_->AddScore(tree, cuda_bag_data_indices_.RawData() + bag_data_cnt_, num_data_ - bag_data_cnt_, cur_tree_id);
      } else {
      #endif  // USE_CUDA_EXP
        train_score_updater_->AddScore(tree, bag_data_indices_.data() + bag_data_cnt_, num_data_ - bag_data_cnt_, cur_tree_id);
      #ifdef USE_CUDA_EXP
      }
      #endif  // USE_CUDA_EXP
Guolin Ke's avatar
Guolin Ke committed
579
580
    }

Guolin Ke's avatar
Guolin Ke committed
581
  } else {
582
    train_score_updater_->AddScore(tree, cur_tree_id);
Guolin Ke's avatar
Guolin Ke committed
583
  }
Guolin Ke's avatar
Guolin Ke committed
584
585


Guolin Ke's avatar
Guolin Ke committed
586
  // update validation score
Guolin Ke's avatar
Guolin Ke committed
587
  for (auto& score_updater : valid_score_updater_) {
588
    score_updater->AddScore(tree, cur_tree_id);
Guolin Ke's avatar
Guolin Ke committed
589
590
591
  }
}

Guolin Ke's avatar
Guolin Ke committed
592
std::vector<double> GBDT::EvalOneMetric(const Metric* metric, const double* score) const {
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
  #ifdef USE_CUDA_EXP
  const bool boosting_on_cuda = objective_function_ != nullptr && objective_function_->IsCUDAObjective();
  const bool evaluation_on_cuda = metric->IsCUDAMetric();
  if ((boosting_on_cuda && evaluation_on_cuda) || (!boosting_on_cuda && !evaluation_on_cuda)) {
  #endif  // USE_CUDA_EXP
    return metric->Eval(score, objective_function_);
  #ifdef USE_CUDA_EXP
  } else if (boosting_on_cuda && !evaluation_on_cuda) {
    const size_t total_size = static_cast<size_t>(num_data_) * static_cast<size_t>(num_tree_per_iteration_);
    if (total_size > host_score_.size()) {
      host_score_.resize(total_size, 0.0f);
    }
    CopyFromCUDADeviceToHost<double>(host_score_.data(), score, total_size, __FILE__, __LINE__);
    return metric->Eval(host_score_.data(), objective_function_);
  } else {
    const size_t total_size = static_cast<size_t>(num_data_) * static_cast<size_t>(num_tree_per_iteration_);
    if (total_size > cuda_score_.Size()) {
      cuda_score_.Resize(total_size);
    }
    CopyFromHostToCUDADevice<double>(cuda_score_.RawData(), score, total_size, __FILE__, __LINE__);
    return metric->Eval(cuda_score_.RawData(), objective_function_);
  }
  #endif  // USE_CUDA_EXP
Guolin Ke's avatar
Guolin Ke committed
616
617
}

Guolin Ke's avatar
Guolin Ke committed
618
std::string GBDT::OutputMetric(int iter) {
Guolin Ke's avatar
Guolin Ke committed
619
  bool need_output = (iter % config_->metric_freq) == 0;
Guolin Ke's avatar
Guolin Ke committed
620
621
  std::string ret = "";
  std::stringstream msg_buf;
622
  std::vector<std::pair<size_t, size_t>> meet_early_stopping_pairs;
Guolin Ke's avatar
Guolin Ke committed
623
  // print training metric
Guolin Ke's avatar
Guolin Ke committed
624
  if (need_output) {
625
626
    for (auto& sub_metric : training_metrics_) {
      auto name = sub_metric->GetName();
Guolin Ke's avatar
Guolin Ke committed
627
      auto scores = EvalOneMetric(sub_metric, train_score_updater_->score());
Guolin Ke's avatar
Guolin Ke committed
628
      for (size_t k = 0; k < name.size(); ++k) {
Guolin Ke's avatar
Guolin Ke committed
629
630
631
632
633
634
        std::stringstream tmp_buf;
        tmp_buf << "Iteration:" << iter
          << ", training " << name[k]
          << " : " << scores[k];
        Log::Info(tmp_buf.str().c_str());
        if (early_stopping_round_ > 0) {
635
          msg_buf << tmp_buf.str() << '\n';
Guolin Ke's avatar
Guolin Ke committed
636
        }
637
      }
638
    }
Guolin Ke's avatar
Guolin Ke committed
639
640
  }
  // print validation metric
Guolin Ke's avatar
Guolin Ke committed
641
  if (need_output || early_stopping_round_ > 0) {
642
643
    for (size_t i = 0; i < valid_metrics_.size(); ++i) {
      for (size_t j = 0; j < valid_metrics_[i].size(); ++j) {
Guolin Ke's avatar
Guolin Ke committed
644
        auto test_scores = EvalOneMetric(valid_metrics_[i][j], valid_score_updater_[i]->score());
Guolin Ke's avatar
Guolin Ke committed
645
646
647
648
649
650
651
652
653
654
        auto name = valid_metrics_[i][j]->GetName();
        for (size_t k = 0; k < name.size(); ++k) {
          std::stringstream tmp_buf;
          tmp_buf << "Iteration:" << iter
            << ", valid_" << i + 1 << " " << name[k]
            << " : " << test_scores[k];
          if (need_output) {
            Log::Info(tmp_buf.str().c_str());
          }
          if (early_stopping_round_ > 0) {
655
            msg_buf << tmp_buf.str() << '\n';
656
          }
wxchan's avatar
wxchan committed
657
        }
658
        if (es_first_metric_only_ && j > 0) { continue; }
Guolin Ke's avatar
Guolin Ke committed
659
        if (ret.empty() && early_stopping_round_ > 0) {
660
661
662
          auto cur_score = valid_metrics_[i][j]->factor_to_bigger_better() * test_scores.back();
          if (cur_score > best_score_[i][j]) {
            best_score_[i][j] = cur_score;
663
            best_iter_[i][j] = iter;
Guolin Ke's avatar
Guolin Ke committed
664
            meet_early_stopping_pairs.emplace_back(i, j);
665
          } else {
Guolin Ke's avatar
Guolin Ke committed
666
            if (iter - best_iter_[i][j] >= early_stopping_round_) { ret = best_msg_[i][j]; }
667
          }
wxchan's avatar
wxchan committed
668
669
        }
      }
Guolin Ke's avatar
Guolin Ke committed
670
671
    }
  }
Guolin Ke's avatar
Guolin Ke committed
672
673
674
  for (auto& pair : meet_early_stopping_pairs) {
    best_msg_[pair.first][pair.second] = msg_buf.str();
  }
wxchan's avatar
wxchan committed
675
  return ret;
Guolin Ke's avatar
Guolin Ke committed
676
677
}

678
/*! \brief Get eval result */
679
std::vector<double> GBDT::GetEvalAt(int data_idx) const {
Guolin Ke's avatar
Guolin Ke committed
680
  CHECK(data_idx >= 0 && data_idx <= static_cast<int>(valid_score_updater_.size()));
681
682
  std::vector<double> ret;
  if (data_idx == 0) {
683
    for (auto& sub_metric : training_metrics_) {
Guolin Ke's avatar
Guolin Ke committed
684
      auto scores = EvalOneMetric(sub_metric, train_score_updater_->score());
685
686
687
      for (auto score : scores) {
        ret.push_back(score);
      }
688
    }
689
  } else {
690
691
    auto used_idx = data_idx - 1;
    for (size_t j = 0; j < valid_metrics_[used_idx].size(); ++j) {
Guolin Ke's avatar
Guolin Ke committed
692
      auto test_scores = EvalOneMetric(valid_metrics_[used_idx][j], valid_score_updater_[used_idx]->score());
693
694
695
      for (auto score : test_scores) {
        ret.push_back(score);
      }
696
697
698
699
700
    }
  }
  return ret;
}

Guolin Ke's avatar
Guolin Ke committed
701
/*! \brief Get training scores result */
702
const double* GBDT::GetTrainingScore(int64_t* out_len) {
703
  *out_len = static_cast<int64_t>(train_score_updater_->num_data()) * num_class_;
Guolin Ke's avatar
Guolin Ke committed
704
  return train_score_updater_->score();
705
706
}

707
void GBDT::PredictContrib(const double* features, double* output) const {
708
  // set zero
Guolin Ke's avatar
Guolin Ke committed
709
710
  const int num_features = max_feature_idx_ + 1;
  std::memset(output, 0, sizeof(double) * num_tree_per_iteration_ * (num_features + 1));
711
712
  const int end_iteration_for_pred = start_iteration_for_pred_ + num_iteration_for_pred_;
  for (int i = start_iteration_for_pred_; i < end_iteration_for_pred; ++i) {
713
714
    // predict all the trees for one iteration
    for (int k = 0; k < num_tree_per_iteration_; ++k) {
Guolin Ke's avatar
Guolin Ke committed
715
      models_[i * num_tree_per_iteration_ + k]->PredictContrib(features, num_features, output + k*(num_features + 1));
716
    }
717
718
719
720
721
722
  }
}

void GBDT::PredictContribByMap(const std::unordered_map<int, double>& features,
                               std::vector<std::unordered_map<int, double>>* output) const {
  const int num_features = max_feature_idx_ + 1;
723
724
  const int end_iteration_for_pred = start_iteration_for_pred_ + num_iteration_for_pred_;
  for (int i = start_iteration_for_pred_; i < end_iteration_for_pred; ++i) {
725
726
727
    // predict all the trees for one iteration
    for (int k = 0; k < num_tree_per_iteration_; ++k) {
      models_[i * num_tree_per_iteration_ + k]->PredictContribByMap(features, num_features, &((*output)[k]));
728
729
730
731
    }
  }
}

Guolin Ke's avatar
Guolin Ke committed
732
733
void GBDT::GetPredictAt(int data_idx, double* out_result, int64_t* out_len) {
  CHECK(data_idx >= 0 && data_idx <= static_cast<int>(valid_score_updater_.size()));
Guolin Ke's avatar
Guolin Ke committed
734

735
  const double* raw_scores = nullptr;
Guolin Ke's avatar
Guolin Ke committed
736
737
  data_size_t num_data = 0;
  if (data_idx == 0) {
wxchan's avatar
wxchan committed
738
    raw_scores = GetTrainingScore(out_len);
Guolin Ke's avatar
Guolin Ke committed
739
740
741
742
743
    num_data = train_score_updater_->num_data();
  } else {
    auto used_idx = data_idx - 1;
    raw_scores = valid_score_updater_[used_idx]->score();
    num_data = valid_score_updater_[used_idx]->num_data();
744
    *out_len = static_cast<int64_t>(num_data) * num_class_;
Guolin Ke's avatar
Guolin Ke committed
745
  }
Guolin Ke's avatar
Guolin Ke committed
746
  if (objective_function_ != nullptr) {
Guolin Ke's avatar
Guolin Ke committed
747
748
    #pragma omp parallel for schedule(static)
    for (data_size_t i = 0; i < num_data; ++i) {
Guolin Ke's avatar
Guolin Ke committed
749
      std::vector<double> tree_pred(num_tree_per_iteration_);
750
      for (int j = 0; j < num_tree_per_iteration_; ++j) {
Guolin Ke's avatar
Guolin Ke committed
751
        tree_pred[j] = raw_scores[j * num_data + i];
752
      }
Guolin Ke's avatar
Guolin Ke committed
753
754
      std::vector<double> tmp_result(num_class_);
      objective_function_->ConvertOutput(tree_pred.data(), tmp_result.data());
Guolin Ke's avatar
Guolin Ke committed
755
      for (int j = 0; j < num_class_; ++j) {
756
        out_result[j * num_data + i] = static_cast<double>(tmp_result[j]);
Guolin Ke's avatar
Guolin Ke committed
757
758
      }
    }
759
  } else {
Guolin Ke's avatar
Guolin Ke committed
760
    #pragma omp parallel for schedule(static)
Guolin Ke's avatar
Guolin Ke committed
761
    for (data_size_t i = 0; i < num_data; ++i) {
762
      for (int j = 0; j < num_tree_per_iteration_; ++j) {
Guolin Ke's avatar
Guolin Ke committed
763
        out_result[j * num_data + i] = static_cast<double>(raw_scores[j * num_data + i]);
Guolin Ke's avatar
Guolin Ke committed
764
765
766
767
768
      }
    }
  }
}

769
770
double GBDT::GetUpperBoundValue() const {
  double max_value = 0.0;
Nikita Titov's avatar
Nikita Titov committed
771
  for (const auto &tree : models_) {
772
773
774
775
776
777
778
    max_value += tree->GetUpperBoundValue();
  }
  return max_value;
}

double GBDT::GetLowerBoundValue() const {
  double min_value = 0.0;
Nikita Titov's avatar
Nikita Titov committed
779
  for (const auto &tree : models_) {
780
781
782
783
784
    min_value += tree->GetLowerBoundValue();
  }
  return min_value;
}

Guolin Ke's avatar
Guolin Ke committed
785
786
787
void GBDT::ResetTrainingData(const Dataset* train_data, const ObjectiveFunction* objective_function,
                             const std::vector<const Metric*>& training_metrics) {
  if (train_data != train_data_ && !train_data_->CheckAlign(*train_data)) {
788
    Log::Fatal("Cannot reset training data, since new training data has different bin mappers");
wxchan's avatar
wxchan committed
789
790
  }

Guolin Ke's avatar
Guolin Ke committed
791
792
  objective_function_ = objective_function;
  if (objective_function_ != nullptr) {
Nikita Titov's avatar
Nikita Titov committed
793
    CHECK_EQ(num_tree_per_iteration_, objective_function_->NumModelPerIteration());
794
795
796
    if (objective_function_->IsRenewTreeOutput() && !config_->monotone_constraints.empty()) {
      Log::Fatal("Cannot use ``monotone_constraints`` in %s objective, please disable it.", objective_function_->GetName());
    }
797
  }
798
  is_constant_hessian_ = GetIsConstHessian(objective_function);
799

Guolin Ke's avatar
Guolin Ke committed
800
801
802
803
  // push training metrics
  training_metrics_.clear();
  for (const auto& metric : training_metrics) {
    training_metrics_.push_back(metric);
804
  }
Guolin Ke's avatar
Guolin Ke committed
805
  training_metrics_.shrink_to_fit();
806

807
808
809
810
  #ifdef USE_CUDA_EXP
  const bool boosting_on_gpu = objective_function_ != nullptr && objective_function_->IsCUDAObjective();
  #endif  // USE_CUDA_EXP

Guolin Ke's avatar
Guolin Ke committed
811
812
813
814
  if (train_data != train_data_) {
    train_data_ = train_data;
    // not same training data, need reset score and others
    // create score tracker
815
816
817
818
819
820
821
822
823
    #ifdef USE_CUDA_EXP
    if (config_->device_type == std::string("cuda_exp")) {
      train_score_updater_.reset(new CUDAScoreUpdater(train_data_, num_tree_per_iteration_, boosting_on_gpu));
    } else {
    #endif  // USE_CUDA_EXP
      train_score_updater_.reset(new ScoreUpdater(train_data_, num_tree_per_iteration_));
    #ifdef USE_CUDA_EXP
    }
    #endif  // USE_CUDA_EXP
824

Guolin Ke's avatar
Guolin Ke committed
825
826
827
828
829
830
    // update score
    for (int i = 0; i < iter_; ++i) {
      for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
        auto curr_tree = (i + num_init_iteration_) * num_tree_per_iteration_ + cur_tree_id;
        train_score_updater_->AddScore(models_[curr_tree].get(), cur_tree_id);
      }
831
832
    }

Guolin Ke's avatar
Guolin Ke committed
833
    num_data_ = train_data_->num_data();
834

Guolin Ke's avatar
Guolin Ke committed
835
836
837
    // create buffer for gradients and hessians
    if (objective_function_ != nullptr) {
      size_t total_size = static_cast<size_t>(num_data_) * num_tree_per_iteration_;
838
839
840
841
842
843
844
845
846
847
848
849
850
      #ifdef USE_CUDA_EXP
      if (config_->device_type == std::string("cuda_exp") && boosting_on_gpu) {
        AllocateCUDAMemory<score_t>(&gradients_pointer_, total_size, __FILE__, __LINE__);
        AllocateCUDAMemory<score_t>(&hessians_pointer_, total_size, __FILE__, __LINE__);
      } else {
      #endif  // USE_CUDA_EXP
        gradients_.resize(total_size);
        hessians_.resize(total_size);
        gradients_pointer_ = gradients_.data();
        hessians_pointer_ = hessians_.data();
      #ifdef USE_CUDA_EXP
      }
      #endif  // USE_CUDA_EXP
Guolin Ke's avatar
Guolin Ke committed
851
    }
852

Guolin Ke's avatar
Guolin Ke committed
853
854
855
856
    max_feature_idx_ = train_data_->num_total_features() - 1;
    label_idx_ = train_data_->label_idx();
    feature_names_ = train_data_->feature_names();
    feature_infos_ = train_data_->feature_infos();
857
    parser_config_str_ = train_data_->parser_config_str();
858

859
    tree_learner_->ResetTrainingData(train_data, is_constant_hessian_);
Guolin Ke's avatar
Guolin Ke committed
860
    ResetBaggingConfig(config_.get(), true);
861
862
  } else {
    tree_learner_->ResetIsConstantHessian(is_constant_hessian_);
863
  }
864
865
}

Guolin Ke's avatar
Guolin Ke committed
866
867
void GBDT::ResetConfig(const Config* config) {
  auto new_config = std::unique_ptr<Config>(new Config(*config));
868
  if (!config->monotone_constraints.empty()) {
Nikita Titov's avatar
Nikita Titov committed
869
    CHECK_EQ(static_cast<size_t>(train_data_->num_total_features()), config->monotone_constraints.size());
870
871
  }
  if (!config->feature_contri.empty()) {
Nikita Titov's avatar
Nikita Titov committed
872
    CHECK_EQ(static_cast<size_t>(train_data_->num_total_features()), config->feature_contri.size());
873
  }
874
875
876
  if (objective_function_ != nullptr && objective_function_->IsRenewTreeOutput() && !config->monotone_constraints.empty()) {
    Log::Fatal("Cannot use ``monotone_constraints`` in %s objective, please disable it.", objective_function_->GetName());
  }
Guolin Ke's avatar
Guolin Ke committed
877
878
879
  early_stopping_round_ = new_config->early_stopping_round;
  shrinkage_rate_ = new_config->learning_rate;
  if (tree_learner_ != nullptr) {
Guolin Ke's avatar
Guolin Ke committed
880
    tree_learner_->ResetConfig(new_config.get());
881
  }
Guolin Ke's avatar
Guolin Ke committed
882
883
  if (train_data_ != nullptr) {
    ResetBaggingConfig(new_config.get(), false);
884
  }
885
  if (config_.get() != nullptr && config_->forcedsplits_filename != new_config->forcedsplits_filename) {
886
887
888
889
890
891
892
    // load forced_splits file
    if (!new_config->forcedsplits_filename.empty()) {
      std::ifstream forced_splits_file(
          new_config->forcedsplits_filename.c_str());
      std::stringstream buffer;
      buffer << forced_splits_file.rdbuf();
      std::string err;
Guolin Ke's avatar
Guolin Ke committed
893
      forced_splits_json_ = Json::parse(buffer.str(), &err);
894
895
896
897
898
899
      tree_learner_->SetForcedSplit(&forced_splits_json_);
    } else {
      forced_splits_json_ = Json();
      tree_learner_->SetForcedSplit(nullptr);
    }
  }
Guolin Ke's avatar
Guolin Ke committed
900
  config_.reset(new_config.release());
Guolin Ke's avatar
Guolin Ke committed
901
902
}

Guolin Ke's avatar
Guolin Ke committed
903
void GBDT::ResetBaggingConfig(const Config* config, bool is_change_dataset) {
Guolin Ke's avatar
Guolin Ke committed
904
  // if need bagging, create buffer
Guolin Ke's avatar
Guolin Ke committed
905
906
907
908
909
910
  data_size_t num_pos_data = 0;
  if (objective_function_ != nullptr) {
    num_pos_data = objective_function_->NumPositiveData();
  }
  bool balance_bagging_cond = (config->pos_bagging_fraction < 1.0 || config->neg_bagging_fraction < 1.0) && (num_pos_data > 0);
  if ((config->bagging_fraction < 1.0 || balance_bagging_cond) && config->bagging_freq > 0) {
911
912
    need_re_bagging_ = false;
    if (!is_change_dataset &&
Guolin Ke's avatar
Guolin Ke committed
913
914
      config_.get() != nullptr && config_->bagging_fraction == config->bagging_fraction && config_->bagging_freq == config->bagging_freq
      && config_->pos_bagging_fraction == config->pos_bagging_fraction && config_->neg_bagging_fraction == config->neg_bagging_fraction) {
915
916
      return;
    }
Guolin Ke's avatar
Guolin Ke committed
917
918
    if (balance_bagging_cond) {
      balanced_bagging_ = true;
919
      bag_data_cnt_ = static_cast<data_size_t>(num_pos_data * config->pos_bagging_fraction)
Guolin Ke's avatar
Guolin Ke committed
920
921
922
923
                      + static_cast<data_size_t>((num_data_ - num_pos_data) * config->neg_bagging_fraction);
    } else {
      bag_data_cnt_ = static_cast<data_size_t>(config->bagging_fraction * num_data_);
    }
Guolin Ke's avatar
Guolin Ke committed
924
    bag_data_indices_.resize(num_data_);
925
926
927
928
929
    #ifdef USE_CUDA_EXP
    if (config->device_type == std::string("cuda_exp")) {
      cuda_bag_data_indices_.Resize(num_data_);
    }
    #endif  // USE_CUDA_EXP
930
931
932
933
934
935
    bagging_runner_.ReSize(num_data_);
    bagging_rands_.clear();
    for (int i = 0;
         i < (num_data_ + bagging_rand_block_ - 1) / bagging_rand_block_; ++i) {
      bagging_rands_.emplace_back(config_->bagging_seed + i);
    }
936

937
938
    double average_bag_rate =
        (static_cast<double>(bag_data_cnt_) / num_data_) / config->bagging_freq;
Guolin Ke's avatar
Guolin Ke committed
939
    is_use_subset_ = false;
940
941
942
943
944
945
946
947
948
949
    if (config_->device_type != std::string("cuda_exp")) {
      const int group_threshold_usesubset = 100;
      if (average_bag_rate <= 0.5
          && (train_data_->num_feature_groups() < group_threshold_usesubset)) {
        if (tmp_subset_ == nullptr || is_change_dataset) {
          tmp_subset_.reset(new Dataset(bag_data_cnt_));
          tmp_subset_->CopyFeatureMapperFrom(train_data_);
        }
        is_use_subset_ = true;
        Log::Debug("Use subset for bagging");
Guolin Ke's avatar
Guolin Ke committed
950
      }
Guolin Ke's avatar
Guolin Ke committed
951
952
    }

953
    need_re_bagging_ = true;
954

Guolin Ke's avatar
Guolin Ke committed
955
956
957
    if (is_use_subset_ && bag_data_cnt_ < num_data_) {
      if (objective_function_ == nullptr) {
        size_t total_size = static_cast<size_t>(num_data_) * num_tree_per_iteration_;
958
959
960
961
962
963
964
965
966
967
968
969
970
        #ifdef USE_CUDA_EXP
        if (config_->device_type == std::string("cuda_exp") && objective_function_ != nullptr && objective_function_->IsCUDAObjective()) {
          AllocateCUDAMemory<score_t>(&gradients_pointer_, total_size, __FILE__, __LINE__);
          AllocateCUDAMemory<score_t>(&hessians_pointer_, total_size, __FILE__, __LINE__);
        } else {
        #endif  // USE_CUDA_EXP
          gradients_.resize(total_size);
          hessians_.resize(total_size);
          gradients_pointer_ = gradients_.data();
          hessians_pointer_ = hessians_.data();
        #ifdef USE_CUDA_EXP
        }
        #endif  // USE_CUDA_EXP
971
      }
972
    }
973
  } else {
Guolin Ke's avatar
Guolin Ke committed
974
975
    bag_data_cnt_ = num_data_;
    bag_data_indices_.clear();
976
977
978
    #ifdef USE_CUDA_EXP
    cuda_bag_data_indices_.Clear();
    #endif  // USE_CUDA_EXP
979
    bagging_runner_.ReSize(0);
Guolin Ke's avatar
Guolin Ke committed
980
    is_use_subset_ = false;
981
  }
wxchan's avatar
wxchan committed
982
983
}

Guolin Ke's avatar
Guolin Ke committed
984
}  // namespace LightGBM