gbdt.h 21.7 KB
Newer Older
1
2
3
4
/*!
 * Copyright (c) 2016 Microsoft Corporation. All rights reserved.
 * Licensed under the MIT License. See LICENSE file in the project root for license information.
 */
5
6
#ifndef LIGHTGBM_SRC_BOOSTING_GBDT_H_
#define LIGHTGBM_SRC_BOOSTING_GBDT_H_
Guolin Ke's avatar
Guolin Ke committed
7

8
9
10
#include <LightGBM/boosting.h>
#include <LightGBM/objective_function.h>
#include <LightGBM/prediction_early_stop.h>
11
#include <LightGBM/cuda/vector_cudahost.h>
12
13
#include <LightGBM/utils/json11.h>
#include <LightGBM/utils/threading.h>
14
#include <LightGBM/sample_strategy.h>
15

Guolin Ke's avatar
Guolin Ke committed
16
#include <string>
17
18
#include <algorithm>
#include <cstdio>
19
#include <fstream>
20
#include <map>
Guolin Ke's avatar
Guolin Ke committed
21
#include <memory>
22
#include <mutex>
23
24
25
26
#include <unordered_map>
#include <utility>
#include <vector>

27
#include "cuda/cuda_score_updater.hpp"
28
#include "score_updater.hpp"
Guolin Ke's avatar
Guolin Ke committed
29
30

namespace LightGBM {
Guolin Ke's avatar
Guolin Ke committed
31

32
using json11_internal_lightgbm::Json;
33

Guolin Ke's avatar
Guolin Ke committed
34
35
36
/*!
* \brief GBDT algorithm implementation. including Training, prediction, bagging.
*/
Guolin Ke's avatar
Guolin Ke committed
37
class GBDT : public GBDTBase {
Nikita Titov's avatar
Nikita Titov committed
38
 public:
Guolin Ke's avatar
Guolin Ke committed
39
40
41
  /*!
  * \brief Constructor
  */
42
  GBDT();
Guolin Ke's avatar
Guolin Ke committed
43

Guolin Ke's avatar
Guolin Ke committed
44
45
46
47
  /*!
  * \brief Destructor
  */
  ~GBDT();
Guolin Ke's avatar
Guolin Ke committed
48

49

Guolin Ke's avatar
Guolin Ke committed
50
  /*!
Qiwei Ye's avatar
Qiwei Ye committed
51
  * \brief Initialization logic
zhangyafeikimi's avatar
zhangyafeikimi committed
52
  * \param gbdt_config Config for boosting
Guolin Ke's avatar
Guolin Ke committed
53
  * \param train_data Training data
54
  * \param objective_function Training objective function
Guolin Ke's avatar
Guolin Ke committed
55
56
  * \param training_metrics Training metrics
  */
Guolin Ke's avatar
Guolin Ke committed
57
  void Init(const Config* gbdt_config, const Dataset* train_data,
58
            const ObjectiveFunction* objective_function,
Guolin Ke's avatar
Guolin Ke committed
59
            const std::vector<const Metric*>& training_metrics) override;
wxchan's avatar
wxchan committed
60

61
62
63
64
65
  /*!
  * \brief Traverse the tree of forced splits and check that all indices are less than the number of features.
  */
  void CheckForcedSplitFeatures();

wxchan's avatar
wxchan committed
66
  /*!
Guolin Ke's avatar
Guolin Ke committed
67
  * \brief Merge model from other boosting object. Will insert to the front of current boosting object
wxchan's avatar
wxchan committed
68
69
70
71
72
73
74
75
76
77
78
79
  * \param other
  */
  void MergeFrom(const Boosting* other) override {
    auto other_gbdt = reinterpret_cast<const GBDT*>(other);
    // tmp move to other vector
    auto original_models = std::move(models_);
    models_ = std::vector<std::unique_ptr<Tree>>();
    // push model from other first
    for (const auto& tree : other_gbdt->models_) {
      auto new_tree = std::unique_ptr<Tree>(new Tree(*(tree.get())));
      models_.push_back(std::move(new_tree));
    }
Guolin Ke's avatar
Guolin Ke committed
80
    num_init_iteration_ = static_cast<int>(models_.size()) / num_tree_per_iteration_;
wxchan's avatar
wxchan committed
81
82
83
84
85
    // push model in current object
    for (const auto& tree : original_models) {
      auto new_tree = std::unique_ptr<Tree>(new Tree(*(tree.get())));
      models_.push_back(std::move(new_tree));
    }
Guolin Ke's avatar
Guolin Ke committed
86
    num_iteration_for_pred_ = static_cast<int>(models_.size()) / num_tree_per_iteration_;
wxchan's avatar
wxchan committed
87
88
  }

89
  void ShuffleModels(int start_iter, int end_iter) override {
90
    int total_iter = static_cast<int>(models_.size()) / num_tree_per_iteration_;
91
92
93
94
95
    start_iter = std::max(0, start_iter);
    if (end_iter <= 0) {
      end_iter = total_iter;
    }
    end_iter = std::min(total_iter, end_iter);
96
97
98
99
100
101
    auto original_models = std::move(models_);
    std::vector<int> indices(total_iter);
    for (int i = 0; i < total_iter; ++i) {
      indices[i] = i;
    }
    Random tmp_rand(17);
102
103
    for (int i = start_iter; i < end_iter - 1; ++i) {
      int j = tmp_rand.NextShort(i + 1, end_iter);
104
105
106
107
108
109
110
111
112
113
114
115
      std::swap(indices[i], indices[j]);
    }
    models_ = std::vector<std::unique_ptr<Tree>>();
    for (int i = 0; i < total_iter; ++i) {
      for (int j = 0; j < num_tree_per_iteration_; ++j) {
        int tree_idx = indices[i] * num_tree_per_iteration_ + j;
        auto new_tree = std::unique_ptr<Tree>(new Tree(*(original_models[tree_idx].get())));
        models_.push_back(std::move(new_tree));
      }
    }
  }

Guolin Ke's avatar
Guolin Ke committed
116
117
118
119
120
121
  /*!
  * \brief Reset the training data
  * \param train_data New Training data
  * \param objective_function Training objective function
  * \param training_metrics Training metrics
  */
122
123
  void ResetTrainingData(const Dataset* train_data, const ObjectiveFunction* objective_function,
                         const std::vector<const Metric*>& training_metrics) override;
wxchan's avatar
wxchan committed
124

Guolin Ke's avatar
Guolin Ke committed
125
126
127
128
  /*!
  * \brief Reset Boosting Config
  * \param gbdt_config Config for boosting
  */
Guolin Ke's avatar
Guolin Ke committed
129
  void ResetConfig(const Config* gbdt_config) override;
Guolin Ke's avatar
Guolin Ke committed
130

Guolin Ke's avatar
Guolin Ke committed
131
  /*!
Qiwei Ye's avatar
Qiwei Ye committed
132
133
134
  * \brief Adding a validation dataset
  * \param valid_data Validation dataset
  * \param valid_metrics Metrics for validation dataset
Guolin Ke's avatar
Guolin Ke committed
135
  */
wxchan's avatar
wxchan committed
136
  void AddValidDataset(const Dataset* valid_data,
137
                       const std::vector<const Metric*>& valid_metrics) override;
Guolin Ke's avatar
Guolin Ke committed
138

Guolin Ke's avatar
Guolin Ke committed
139
140
  /*!
  * \brief Perform a full training procedure
Andrew Ziem's avatar
Andrew Ziem committed
141
  * \param snapshot_freq frequency of snapshot
Guolin Ke's avatar
Guolin Ke committed
142
143
  * \param model_output_path path of model file
  */
Guolin Ke's avatar
Guolin Ke committed
144
145
  void Train(int snapshot_freq, const std::string& model_output_path) override;

146
  void RefitTree(const int* tree_leaf_prediction, const size_t nrow, const size_t ncol) override;
147

Guolin Ke's avatar
Guolin Ke committed
148
  /*!
Guolin Ke's avatar
Guolin Ke committed
149
  * \brief Training logic
Guolin Ke's avatar
Guolin Ke committed
150
  * \param gradients nullptr for using default objective, otherwise use self-defined boosting
Nikita Titov's avatar
Nikita Titov committed
151
  * \param hessians nullptr for using default objective, otherwise use self-defined boosting
Guolin Ke's avatar
Guolin Ke committed
152
  * \return True if cannot train any more
Guolin Ke's avatar
Guolin Ke committed
153
  */
Guolin Ke's avatar
Guolin Ke committed
154
  bool TrainOneIter(const score_t* gradients, const score_t* hessians) override;
155

wxchan's avatar
wxchan committed
156
157
158
159
160
  /*!
  * \brief Rollback one iteration
  */
  void RollbackOneIter() override;

Guolin Ke's avatar
Guolin Ke committed
161
162
163
  /*!
  * \brief Get current iteration
  */
Guolin Ke's avatar
Guolin Ke committed
164
  int GetCurrentIteration() const override { return static_cast<int>(models_.size()) / num_tree_per_iteration_; }
wxchan's avatar
wxchan committed
165

166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
  /*!
  * \brief Get parameters as a JSON string
  */
  std::string GetLoadedParam() const override {
    if (loaded_parameter_.empty()) {
      return std::string("{}");
    }
    const auto param_types = Config::ParameterTypes();
    const auto lines = Common::Split(loaded_parameter_.c_str(), "\n");
    bool first = true;
    std::stringstream str_buf;
    str_buf << "{";
    for (const auto& line : lines) {
      const auto pair = Common::Split(line.c_str(), ":");
      if (pair[1] == " ]")
        continue;
182
183
184
185
186
187
188
189
      const auto param = pair[0].substr(1);
      const auto value_str = pair[1].substr(1, pair[1].size() - 2);
      auto iter = param_types.find(param);
      if (iter == param_types.end()) {
        Log::Warning("Ignoring unrecognized parameter '%s' found in model string.", param.c_str());
        continue;
      }
      std::string param_type = iter->second;
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
      if (first) {
        first = false;
        str_buf << "\"";
      } else {
        str_buf << ",\"";
      }
      str_buf << param << "\": ";
      if (param_type == "string") {
        str_buf << "\"" << value_str << "\"";
      } else if (param_type == "int") {
        int value;
        Common::Atoi(value_str.c_str(), &value);
        str_buf << value;
      } else if (param_type == "double") {
        double value;
        Common::Atof(value_str.c_str(), &value);
        str_buf << value;
      } else if (param_type == "bool") {
        bool value = value_str == "1";
        str_buf << std::boolalpha << value;
      } else if (param_type.substr(0, 6) == "vector") {
        str_buf << "[";
        if (param_type.substr(7, 6) == "string") {
          const auto parts = Common::Split(value_str.c_str(), ",");
          str_buf << "\"" << Common::Join(parts, "\",\"") << "\"";
        } else {
          str_buf << value_str;
        }
        str_buf << "]";
      }
    }
    str_buf << "}";
    return str_buf.str();
  }

Guolin Ke's avatar
Guolin Ke committed
225
226
227
228
  /*!
  * \brief Can use early stopping for prediction or not
  * \return True if cannot use early stopping for prediction
  */
229
  bool NeedAccuratePrediction() const override {
230
231
232
233
234
235
236
    if (objective_function_ == nullptr) {
      return true;
    } else {
      return objective_function_->NeedAccuratePrediction();
    }
  }

Guolin Ke's avatar
Guolin Ke committed
237
238
239
240
241
  /*!
  * \brief Get evaluation result at data_idx data
  * \param data_idx 0: training data, 1: 1st validation data
  * \return evaluation result
  */
242
  std::vector<double> GetEvalAt(int data_idx) const override;
243

Guolin Ke's avatar
Guolin Ke committed
244
245
  /*!
  * \brief Get current training score
Guolin Ke's avatar
Guolin Ke committed
246
  * \param out_len length of returned score
Guolin Ke's avatar
Guolin Ke committed
247
248
  * \return training score
  */
Guolin Ke's avatar
Guolin Ke committed
249
  const double* GetTrainingScore(int64_t* out_len) override;
250

Guolin Ke's avatar
Guolin Ke committed
251
252
253
254
255
  /*!
  * \brief Get size of prediction at data_idx data
  * \param data_idx 0: training data, 1: 1st validation data
  * \return The size of prediction
  */
Guolin Ke's avatar
Guolin Ke committed
256
  int64_t GetNumPredictAt(int data_idx) const override {
Guolin Ke's avatar
Guolin Ke committed
257
258
259
260
261
    CHECK(data_idx >= 0 && data_idx <= static_cast<int>(valid_score_updater_.size()));
    data_size_t num_data = train_data_->num_data();
    if (data_idx > 0) {
      num_data = valid_score_updater_[data_idx - 1]->num_data();
    }
262
    return static_cast<int64_t>(num_data) * num_class_;
Guolin Ke's avatar
Guolin Ke committed
263
  }
Guolin Ke's avatar
Guolin Ke committed
264

Guolin Ke's avatar
Guolin Ke committed
265
266
267
268
  /*!
  * \brief Get prediction result at data_idx data
  * \param data_idx 0: training data, 1: 1st validation data
  * \param result used to store prediction result, should allocate memory before call this function
269
  * \param out_len length of returned score
Guolin Ke's avatar
Guolin Ke committed
270
  */
Guolin Ke's avatar
Guolin Ke committed
271
  void GetPredictAt(int data_idx, double* out_result, int64_t* out_len) override;
Guolin Ke's avatar
Guolin Ke committed
272

Guolin Ke's avatar
Guolin Ke committed
273
274
  /*!
  * \brief Get number of prediction for one data
275
  * \param start_iteration Start index of the iteration to predict
Guolin Ke's avatar
Guolin Ke committed
276
  * \param num_iteration number of used iterations
277
  * \param is_pred_leaf True if predicting leaf index
Guolin Ke's avatar
Guolin Ke committed
278
279
280
  * \param is_pred_contrib True if predicting feature contribution
  * \return number of prediction
  */
281
  inline int NumPredictOneRow(int start_iteration, int num_iteration, bool is_pred_leaf, bool is_pred_contrib) const override {
282
    int num_pred_in_one_row = num_class_;
Guolin Ke's avatar
Guolin Ke committed
283
284
    if (is_pred_leaf) {
      int max_iteration = GetCurrentIteration();
285
286
      start_iteration = std::max(start_iteration, 0);
      start_iteration = std::min(start_iteration, max_iteration);
Guolin Ke's avatar
Guolin Ke committed
287
      if (num_iteration > 0) {
288
        num_pred_in_one_row *= static_cast<int>(std::min(max_iteration - start_iteration, num_iteration));
Guolin Ke's avatar
Guolin Ke committed
289
      } else {
290
        num_pred_in_one_row *= (max_iteration - start_iteration);
Guolin Ke's avatar
Guolin Ke committed
291
      }
292
    } else if (is_pred_contrib) {
293
      num_pred_in_one_row = num_tree_per_iteration_ * (max_feature_idx_ + 2);  // +1 for 0-based indexing, +1 for baseline
Guolin Ke's avatar
Guolin Ke committed
294
    }
295
    return num_pred_in_one_row;
Guolin Ke's avatar
Guolin Ke committed
296
  }
Guolin Ke's avatar
Guolin Ke committed
297

cbecker's avatar
cbecker committed
298
  void PredictRaw(const double* features, double* output,
299
                  const PredictionEarlyStopInstance* earlyStop) const override;
wxchan's avatar
wxchan committed
300

Guolin Ke's avatar
Guolin Ke committed
301
302
  void PredictRawByMap(const std::unordered_map<int, double>& features, double* output,
                       const PredictionEarlyStopInstance* early_stop) const override;
303

cbecker's avatar
cbecker committed
304
305
  void Predict(const double* features, double* output,
               const PredictionEarlyStopInstance* earlyStop) const override;
Guolin Ke's avatar
Guolin Ke committed
306

Guolin Ke's avatar
Guolin Ke committed
307
308
  void PredictByMap(const std::unordered_map<int, double>& features, double* output,
                    const PredictionEarlyStopInstance* early_stop) const override;
309

310
  void PredictLeafIndex(const double* features, double* output) const override;
wxchan's avatar
wxchan committed
311

312
313
  void PredictLeafIndexByMap(const std::unordered_map<int, double>& features, double* output) const override;

314
315
316
317
  void PredictContrib(const double* features, double* output) const override;

  void PredictContribByMap(const std::unordered_map<int, double>& features,
                           std::vector<std::unordered_map<int, double>>* output) const override;
318

Guolin Ke's avatar
Guolin Ke committed
319
  /*!
wxchan's avatar
wxchan committed
320
  * \brief Dump model to json format string
321
  * \param start_iteration The model will be saved start from
322
  * \param num_iteration Number of iterations that want to dump, -1 means dump all
323
  * \param feature_importance_type Type of feature importance, 0: split, 1: gain
wxchan's avatar
wxchan committed
324
  * \return Json format string of model
Guolin Ke's avatar
Guolin Ke committed
325
  */
326
327
  std::string DumpModel(int start_iteration, int num_iteration,
                        int feature_importance_type) const override;
wxchan's avatar
wxchan committed
328

329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
  /*!
  * \brief Translate model to if-else statement
  * \param num_iteration Number of iterations that want to translate, -1 means translate all
  * \return if-else format codes of model
  */
  std::string ModelToIfElse(int num_iteration) const override;

  /*!
  * \brief Translate model to if-else statement
  * \param num_iteration Number of iterations that want to translate, -1 means translate all
  * \param filename Filename that want to save to
  * \return is_finish Is training finished or not
  */
  bool SaveModelToIfElse(int num_iteration, const char* filename) const override;

wxchan's avatar
wxchan committed
344
345
  /*!
  * \brief Save model to file
346
  * \param start_iteration The model will be saved start from
wxchan's avatar
wxchan committed
347
  * \param num_iterations Number of model that want to save, -1 means save all
348
  * \param feature_importance_type Type of feature importance, 0: split, 1: gain
wxchan's avatar
wxchan committed
349
  * \param filename Filename that want to save to
350
  * \return is_finish Is training finished or not
wxchan's avatar
wxchan committed
351
  */
352
353
354
  bool SaveModelToFile(int start_iteration, int num_iterations,
                       int feature_importance_type,
                       const char* filename) const override;
wxchan's avatar
wxchan committed
355

356
357
  /*!
  * \brief Save model to string
358
  * \param start_iteration The model will be saved start from
wxchan's avatar
wxchan committed
359
  * \param num_iterations Number of model that want to save, -1 means save all
360
  * \param feature_importance_type Type of feature importance, 0: split, 1: gain
361
362
  * \return Non-empty string if succeeded
  */
363
  std::string SaveModelToString(int start_iteration, int num_iterations, int feature_importance_type) const override;
364

Guolin Ke's avatar
Guolin Ke committed
365
  /*!
366
  * \brief Restore from a serialized buffer
Guolin Ke's avatar
Guolin Ke committed
367
  */
368
  bool LoadModelFromString(const char* buffer, size_t len) override;
wxchan's avatar
wxchan committed
369

370
371
372
373
374
375
376
377
  /*!
  * \brief Calculate feature importances
  * \param num_iteration Number of model that want to use for feature importance, -1 means use all
  * \param importance_type: 0 for split, 1 for gain
  * \return vector of feature_importance
  */
  std::vector<double> FeatureImportance(int num_iteration, int importance_type) const override;

378
379
380
381
382
383
384
385
386
387
388
389
  /*!
  * \brief Calculate upper bound value
  * \return upper bound value
  */
  double GetUpperBoundValue() const override;

  /*!
  * \brief Calculate lower bound value
  * \return lower bound value
  */
  double GetLowerBoundValue() const override;

Guolin Ke's avatar
Guolin Ke committed
390
391
392
393
394
  /*!
  * \brief Get max feature index of this model
  * \return Max feature index of this model
  */
  inline int MaxFeatureIdx() const override { return max_feature_idx_; }
Guolin Ke's avatar
Guolin Ke committed
395

wxchan's avatar
wxchan committed
396
397
398
399
400
401
  /*!
  * \brief Get feature names of this model
  * \return Feature names of this model
  */
  inline std::vector<std::string> FeatureNames() const override { return feature_names_; }

Guolin Ke's avatar
Guolin Ke committed
402
403
404
405
406
407
  /*!
  * \brief Get index of label column
  * \return index of label column
  */
  inline int LabelIdx() const override { return label_idx_; }

Guolin Ke's avatar
Guolin Ke committed
408
409
410
411
  /*!
  * \brief Get number of weak sub-models
  * \return Number of weak sub-models
  */
wxchan's avatar
wxchan committed
412
  inline int NumberOfTotalModel() const override { return static_cast<int>(models_.size()); }
Guolin Ke's avatar
Guolin Ke committed
413

Guolin Ke's avatar
Guolin Ke committed
414
415
416
417
  /*!
  * \brief Get number of tree per iteration
  * \return number of tree per iteration
  */
Guolin Ke's avatar
Guolin Ke committed
418
  inline int NumModelPerIteration() const override { return num_tree_per_iteration_; }
Guolin Ke's avatar
Guolin Ke committed
419

420
421
422
423
  /*!
  * \brief Get number of classes
  * \return Number of classes
  */
Guolin Ke's avatar
Guolin Ke committed
424
  inline int NumberOfClasses() const override { return num_class_; }
425

426
  inline void InitPredict(int start_iteration, int num_iteration, bool is_pred_contrib) override {
Guolin Ke's avatar
Guolin Ke committed
427
    num_iteration_for_pred_ = static_cast<int>(models_.size()) / num_tree_per_iteration_;
428
429
    start_iteration = std::max(start_iteration, 0);
    start_iteration = std::min(start_iteration, num_iteration_for_pred_);
wxchan's avatar
wxchan committed
430
    if (num_iteration > 0) {
431
432
433
      num_iteration_for_pred_ = std::min(num_iteration, num_iteration_for_pred_ - start_iteration);
    } else {
      num_iteration_for_pred_ = num_iteration_for_pred_ - start_iteration;
434
    }
435
    start_iteration_for_pred_ = start_iteration;
436
437
438
439
440
441

    if (is_pred_contrib && !models_initialized_) {
      std::lock_guard<std::mutex> lock(instance_mutex_);
      if (models_initialized_)
        return;

442
      #pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
443
444
445
      for (int i = 0; i < static_cast<int>(models_.size()); ++i) {
        models_[i]->RecomputeMaxDepth();
      }
446
447

      models_initialized_ = true;
448
    }
449
  }
wxchan's avatar
wxchan committed
450

Guolin Ke's avatar
Guolin Ke committed
451
  inline double GetLeafValue(int tree_idx, int leaf_idx) const override {
Guolin Ke's avatar
Guolin Ke committed
452
453
454
455
456
    CHECK(tree_idx >= 0 && static_cast<size_t>(tree_idx) < models_.size());
    CHECK(leaf_idx >= 0 && leaf_idx < models_[tree_idx]->num_leaves());
    return models_[tree_idx]->LeafOutput(leaf_idx);
  }

Guolin Ke's avatar
Guolin Ke committed
457
  inline void SetLeafValue(int tree_idx, int leaf_idx, double val) override {
Guolin Ke's avatar
Guolin Ke committed
458
459
460
461
462
    CHECK(tree_idx >= 0 && static_cast<size_t>(tree_idx) < models_.size());
    CHECK(leaf_idx >= 0 && leaf_idx < models_[tree_idx]->num_leaves());
    models_[tree_idx]->SetLeafOutput(leaf_idx, val);
  }

463
464
465
  /*!
  * \brief Get Type name of this boosting object
  */
Guolin Ke's avatar
Guolin Ke committed
466
  const char* SubModelName() const override { return "tree"; }
467

468
469
  bool IsLinear() const override { return linear_tree_; }

470
471
  inline std::string ParserConfigStr() const override {return parser_config_str_;}

Nikita Titov's avatar
Nikita Titov committed
472
 protected:
473
  virtual bool GetIsConstHessian(const ObjectiveFunction* objective_function) {
474
    if (objective_function != nullptr && !data_sample_strategy_->IsHessianChange()) {
475
476
477
478
479
      return objective_function->IsConstantHessian();
    } else {
      return false;
    }
  }
Guolin Ke's avatar
Guolin Ke committed
480
481
482
  /*!
  * \brief Print eval result and check early stopping
  */
483
  virtual bool EvalAndCheckEarlyStopping();
Guolin Ke's avatar
Guolin Ke committed
484
485
486
487

  /*!
  * \brief reset config for bagging
  */
Guolin Ke's avatar
Guolin Ke committed
488
  void ResetBaggingConfig(const Config* config, bool is_change_dataset);
Guolin Ke's avatar
Guolin Ke committed
489

Guolin Ke's avatar
Guolin Ke committed
490
  /*!
491
  * \brief calculate the objective function
Guolin Ke's avatar
Guolin Ke committed
492
  */
Guolin Ke's avatar
Guolin Ke committed
493
  virtual void Boosting();
Guolin Ke's avatar
Guolin Ke committed
494

Guolin Ke's avatar
Guolin Ke committed
495
  /*!
Qiwei Ye's avatar
Qiwei Ye committed
496
  * \brief updating score after tree was trained
Guolin Ke's avatar
Guolin Ke committed
497
  * \param tree Trained tree of this iteration
498
  * \param cur_tree_id Current tree for multiclass training
Guolin Ke's avatar
Guolin Ke committed
499
  */
500
  virtual void UpdateScore(const Tree* tree, const int cur_tree_id);
Guolin Ke's avatar
Guolin Ke committed
501

Guolin Ke's avatar
Guolin Ke committed
502
503
504
505
  /*!
  * \brief eval results for one metric

  */
506
  virtual std::vector<double> EvalOneMetric(const Metric* metric, const double* score, const data_size_t num_data) const;
Guolin Ke's avatar
Guolin Ke committed
507

Guolin Ke's avatar
Guolin Ke committed
508
  /*!
Hui Xue's avatar
Hui Xue committed
509
  * \brief Print metric result of current iteration
Andrew Ziem's avatar
Andrew Ziem committed
510
  * \param iter Current iteration
Guolin Ke's avatar
Guolin Ke committed
511
  * \return best_msg if met early_stopping
Guolin Ke's avatar
Guolin Ke committed
512
  */
Guolin Ke's avatar
Guolin Ke committed
513
  std::string OutputMetric(int iter);
514

Guolin Ke's avatar
Guolin Ke committed
515
  double BoostFromAverage(int class_id, bool update_scorer);
Guolin Ke's avatar
Guolin Ke committed
516

517
518
519
520
521
  /*!
  * \brief Reset gradient buffers, must be called after sample strategy is reset
  */
  void ResetGradientBuffers();

522
523
  /*! \brief current iteration */
  int iter_;
Guolin Ke's avatar
Guolin Ke committed
524
525
526
  /*! \brief Pointer to training data */
  const Dataset* train_data_;
  /*! \brief Config of gbdt */
Guolin Ke's avatar
Guolin Ke committed
527
  std::unique_ptr<Config> config_;
Hui Xue's avatar
Hui Xue committed
528
  /*! \brief Tree learner, will use this class to learn trees */
529
  std::unique_ptr<TreeLearner> tree_learner_;
Guolin Ke's avatar
Guolin Ke committed
530
  /*! \brief Objective function */
531
  const ObjectiveFunction* objective_function_;
Hui Xue's avatar
Hui Xue committed
532
  /*! \brief Store and update training data's score */
Guolin Ke's avatar
Guolin Ke committed
533
  std::unique_ptr<ScoreUpdater> train_score_updater_;
Guolin Ke's avatar
Guolin Ke committed
534
535
536
  /*! \brief Metrics for training data */
  std::vector<const Metric*> training_metrics_;
  /*! \brief Store and update validation data's scores */
Guolin Ke's avatar
Guolin Ke committed
537
  std::vector<std::unique_ptr<ScoreUpdater>> valid_score_updater_;
Guolin Ke's avatar
Guolin Ke committed
538
539
  /*! \brief Metric for validation data */
  std::vector<std::vector<const Metric*>> valid_metrics_;
wxchan's avatar
wxchan committed
540
541
  /*! \brief Number of rounds for early stopping */
  int early_stopping_round_;
542
543
  /*! \brief Minimum improvement for early stopping */
  double early_stopping_min_delta_;
544
545
  /*! \brief Only use first metric for early stopping */
  bool es_first_metric_only_;
Guolin Ke's avatar
Guolin Ke committed
546
  /*! \brief Best iteration(s) for early stopping */
wxchan's avatar
wxchan committed
547
  std::vector<std::vector<int>> best_iter_;
Guolin Ke's avatar
Guolin Ke committed
548
  /*! \brief Best score(s) for early stopping */
549
  std::vector<std::vector<double>> best_score_;
Guolin Ke's avatar
Guolin Ke committed
550
551
  /*! \brief output message of best iteration */
  std::vector<std::vector<std::string>> best_msg_;
Guolin Ke's avatar
Guolin Ke committed
552
  /*! \brief Trained models(trees) */
Guolin Ke's avatar
Guolin Ke committed
553
  std::vector<std::unique_ptr<Tree>> models_;
Guolin Ke's avatar
Guolin Ke committed
554
555
  /*! \brief Max feature index of training data*/
  int max_feature_idx_;
556
557
  /*! \brief Parser config file content */
  std::string parser_config_str_ = "";
558
559
560
561
  /*! \brief Are the models initialized (passed RecomputeMaxDepth phase) */
  bool models_initialized_ = false;
  /*! \brief Mutex for exclusive models initialization */
  std::mutex instance_mutex_;
562

563
#ifdef USE_CUDA
564
565
566
567
568
  /*! \brief First order derivative of training data */
  std::vector<score_t, CHAllocator<score_t>> gradients_;
  /*! \brief Second order derivative of training data */
  std::vector<score_t, CHAllocator<score_t>> hessians_;
#else
Guolin Ke's avatar
Guolin Ke committed
569
  /*! \brief First order derivative of training data */
570
  std::vector<score_t, Common::AlignmentAllocator<score_t, kAlignedSize>> gradients_;
571
  /*! \brief Second order derivative of training data */
572
  std::vector<score_t, Common::AlignmentAllocator<score_t, kAlignedSize>> hessians_;
573
#endif
574
575
576
577
  /*! \brief Pointer to gradient vector, can be on CPU or GPU */
  score_t* gradients_pointer_;
  /*! \brief Pointer to hessian vector, can be on CPU or GPU */
  score_t* hessians_pointer_;
578
  /*! \brief Whether boosting is done on GPU, used for device_type=cuda */
shiyu1994's avatar
shiyu1994 committed
579
  bool boosting_on_gpu_;
580
  #ifdef USE_CUDA
581
582
583
584
  /*! \brief Gradient vector on GPU */
  CUDAVector<score_t> cuda_gradients_;
  /*! \brief Hessian vector on GPU */
  CUDAVector<score_t> cuda_hessians_;
585
  /*! \brief Buffer for scores when boosting is on GPU but evaluation is not, used only with device_type=cuda */
586
  mutable std::vector<double> host_score_;
587
  /*! \brief Buffer for scores when boosting is not on GPU but evaluation is, used only with device_type=cuda */
588
  mutable CUDAVector<double> cuda_score_;
589
  #endif  // USE_CUDA
590

wxchan's avatar
wxchan committed
591
  /*! \brief Number of training data */
Guolin Ke's avatar
Guolin Ke committed
592
  data_size_t num_data_;
593
594
595
  /*! \brief Number of trees per iterations */
  int num_tree_per_iteration_;
  /*! \brief Number of class */
596
  int num_class_;
Guolin Ke's avatar
Guolin Ke committed
597
598
  /*! \brief Index of label column */
  data_size_t label_idx_;
599
  /*! \brief number of used model */
wxchan's avatar
wxchan committed
600
  int num_iteration_for_pred_;
601
602
  /*! \brief Start iteration of used model */
  int start_iteration_for_pred_;
Guolin Ke's avatar
Guolin Ke committed
603
604
  /*! \brief Shrinkage rate for one iteration */
  double shrinkage_rate_;
wxchan's avatar
wxchan committed
605
606
  /*! \brief Number of loaded initial models */
  int num_init_iteration_;
Guolin Ke's avatar
Guolin Ke committed
607
608
  /*! \brief Feature names */
  std::vector<std::string> feature_names_;
Guolin Ke's avatar
Guolin Ke committed
609
  std::vector<std::string> feature_infos_;
610
  std::vector<bool> class_need_train_;
611
  bool is_constant_hessian_;
612
  std::unique_ptr<ObjectiveFunction> loaded_objective_;
Guolin Ke's avatar
Guolin Ke committed
613
  bool average_output_;
Guolin Ke's avatar
Guolin Ke committed
614
  bool need_re_bagging_;
Guolin Ke's avatar
Guolin Ke committed
615
  bool balanced_bagging_;
Guolin Ke's avatar
Guolin Ke committed
616
  std::string loaded_parameter_;
617
  std::vector<int8_t> monotone_constraints_;
618
  Json forced_splits_json_;
619
  bool linear_tree_;
620
  std::unique_ptr<SampleStrategy> data_sample_strategy_;
Guolin Ke's avatar
Guolin Ke committed
621
622
623
};

}  // namespace LightGBM
624
#endif   // LIGHTGBM_SRC_BOOSTING_GBDT_H_