gbdt.h 8.99 KB
Newer Older
Guolin Ke's avatar
Guolin Ke committed
1
2
3
4
5
6
7
8
9
#ifndef LIGHTGBM_BOOSTING_GBDT_H_
#define LIGHTGBM_BOOSTING_GBDT_H_

#include <LightGBM/boosting.h>
#include "score_updater.hpp"

#include <cstdio>
#include <vector>
#include <string>
10
#include <fstream>
Guolin Ke's avatar
Guolin Ke committed
11
#include <memory>
Guolin Ke's avatar
Guolin Ke committed
12
13
14
15
16
17
18
19
20
21

namespace LightGBM {
/*!
* \brief GBDT algorithm implementation. including Training, prediction, bagging.
*/
class GBDT: public Boosting {
public:
  /*!
  * \brief Constructor
  */
22
  GBDT();
Guolin Ke's avatar
Guolin Ke committed
23
24
25
26
27
  /*!
  * \brief Destructor
  */
  ~GBDT();
  /*!
Qiwei Ye's avatar
Qiwei Ye committed
28
  * \brief Initialization logic
Guolin Ke's avatar
Guolin Ke committed
29
30
31
32
33
34
  * \param config Config for boosting
  * \param train_data Training data
  * \param object_function Training objective function
  * \param training_metrics Training metrics
  * \param output_model_filename Filename of output model
  */
35
36
  void Init(const BoostingConfig* gbdt_config, const Dataset* train_data, const ObjectiveFunction* object_function,
                             const std::vector<const Metric*>& training_metrics)
Guolin Ke's avatar
Guolin Ke committed
37
                                                                       override;
38

Guolin Ke's avatar
Guolin Ke committed
39
40
41
42
43
44
45
46
  void MergeFrom(const Boosting* other) override {
    auto other_gbdt = reinterpret_cast<const GBDT*>(other);
    for (const auto& tree : other_gbdt->models_) {
      auto new_tree = std::unique_ptr<Tree>(new Tree(*(tree.get())));
      models_.push_back(std::move(new_tree));
    }
  }

47
48
49
50
51
52
  /*!
  * \brief Reset training data for current boosting
  * \param train_data Training data
  * \param object_function Training objective function
  * \param training_metrics Training metric
  */
53
  void ResetTrainingData(const BoostingConfig* config, const Dataset* train_data, const ObjectiveFunction* object_function, const std::vector<const Metric*>& training_metrics) override;
54

Guolin Ke's avatar
Guolin Ke committed
55
  /*!
Qiwei Ye's avatar
Qiwei Ye committed
56
57
58
  * \brief Adding a validation dataset
  * \param valid_data Validation dataset
  * \param valid_metrics Metrics for validation dataset
Guolin Ke's avatar
Guolin Ke committed
59
  */
60
  void AddValidDataset(const Dataset* valid_data,
Guolin Ke's avatar
Guolin Ke committed
61
62
       const std::vector<const Metric*>& valid_metrics) override;
  /*!
Guolin Ke's avatar
Guolin Ke committed
63
64
65
  * \brief Training logic
  * \param gradient nullptr for using default objective, otherwise use self-defined boosting
  * \param hessian nullptr for using default objective, otherwise use self-defined boosting
Guolin Ke's avatar
Guolin Ke committed
66
  * \param is_eval true if need evaluation or early stop
Guolin Ke's avatar
Guolin Ke committed
67
  * \return True if meet early stopping or cannot boosting
Guolin Ke's avatar
Guolin Ke committed
68
  */
69
  virtual bool TrainOneIter(const score_t* gradient, const score_t* hessian, bool is_eval) override;
70

71
72
73
74
75
  /*!
  * \brief Rollback one iteration
  */
  void RollbackOneIter() override;

76
  int GetCurrentIteration() const override { return iter_ + num_init_iteration_; }
77

Guolin Ke's avatar
Guolin Ke committed
78
79
  bool EvalAndCheckEarlyStopping() override;

Guolin Ke's avatar
Guolin Ke committed
80
81
82
83
84
  /*!
  * \brief Get evaluation result at data_idx data
  * \param data_idx 0: training data, 1: 1st validation data
  * \return evaluation result
  */
85
  std::vector<double> GetEvalAt(int data_idx) const override;
86

Guolin Ke's avatar
Guolin Ke committed
87
88
  /*!
  * \brief Get current training score
Guolin Ke's avatar
Guolin Ke committed
89
  * \param out_len length of returned score
Guolin Ke's avatar
Guolin Ke committed
90
91
  * \return training score
  */
92
  virtual const score_t* GetTrainingScore(data_size_t* out_len) override;
93

Guolin Ke's avatar
Guolin Ke committed
94
95
96
97
98
99
  /*!
  * \brief Get prediction result at data_idx data
  * \param data_idx 0: training data, 1: 1st validation data
  * \param result used to store prediction result, should allocate memory before call this function
  * \param out_len lenght of returned score
  */
Guolin Ke's avatar
Guolin Ke committed
100
  void GetPredictAt(int data_idx, score_t* out_result, data_size_t* out_len) override;
Guolin Ke's avatar
Guolin Ke committed
101

Guolin Ke's avatar
Guolin Ke committed
102
  /*!
Qiwei Ye's avatar
Qiwei Ye committed
103
  * \brief Predtion for one record without sigmoid transformation
Guolin Ke's avatar
Guolin Ke committed
104
105
106
  * \param feature_values Feature value on this record
  * \return Prediction result for this record
  */
107
  std::vector<double> PredictRaw(const double* feature_values) const override;
Guolin Ke's avatar
Guolin Ke committed
108
109

  /*!
Qiwei Ye's avatar
Qiwei Ye committed
110
  * \brief Predtion for one record with sigmoid transformation if enabled
Guolin Ke's avatar
Guolin Ke committed
111
112
113
  * \param feature_values Feature value on this record
  * \return Prediction result for this record
  */
114
  std::vector<double> Predict(const double* feature_values) const override;
115
  
wxchan's avatar
wxchan committed
116
117
118
119
120
  /*!
  * \brief Predtion for one record with leaf index
  * \param feature_values Feature value on this record
  * \return Predicted leaf index for this record
  */
121
  std::vector<int> PredictLeafIndex(const double* value) const override;
wxchan's avatar
wxchan committed
122
  
Guolin Ke's avatar
Guolin Ke committed
123
  /*!
Guolin Ke's avatar
Guolin Ke committed
124
  * \brief save model to file
Guolin Ke's avatar
Guolin Ke committed
125
  * \param num_iteration -1 means save all
Guolin Ke's avatar
Guolin Ke committed
126
127
  * \param is_finish is training finished or not
  * \param filename filename that want to save to
Guolin Ke's avatar
Guolin Ke committed
128
  */
Guolin Ke's avatar
Guolin Ke committed
129
  virtual void SaveModelToFile(int num_iteration, bool is_finish, const char* filename) override;
Guolin Ke's avatar
Guolin Ke committed
130
131
132
  /*!
  * \brief Restore from a serialized string
  */
Guolin Ke's avatar
Guolin Ke committed
133
  void LoadModelFromString(const std::string& model_str) override;
Guolin Ke's avatar
Guolin Ke committed
134
135
136
137
138
  /*!
  * \brief Get max feature index of this model
  * \return Max feature index of this model
  */
  inline int MaxFeatureIdx() const override { return max_feature_idx_; }
Guolin Ke's avatar
Guolin Ke committed
139
140
141
142
143
144
145

  /*!
  * \brief Get index of label column
  * \return index of label column
  */
  inline int LabelIdx() const override { return label_idx_; }

Guolin Ke's avatar
Guolin Ke committed
146

Guolin Ke's avatar
Guolin Ke committed
147
148
149
150
  /*!
  * \brief Get number of weak sub-models
  * \return Number of weak sub-models
  */
Guolin Ke's avatar
Guolin Ke committed
151
  inline int NumberOfTotalModel() const override { return static_cast<int>(models_.size()); }
Guolin Ke's avatar
Guolin Ke committed
152

153
154
155
156
  /*!
  * \brief Get number of classes
  * \return Number of classes
  */
Guolin Ke's avatar
Guolin Ke committed
157
  inline int NumberOfClasses() const override { return num_class_; }
158
159

  /*!
Guolin Ke's avatar
Guolin Ke committed
160
  * \brief Set number of iterations for prediction
161
  */
Guolin Ke's avatar
Guolin Ke committed
162
163
164
165
166
  inline void SetNumIterationForPred(int num_iteration) override {
    if (num_iteration > 0) {
      num_iteration_for_pred_ = num_iteration;
    } else {
      num_iteration_for_pred_ = static_cast<int>(models_.size()) / num_class_;
167
168
    }
  }
169
  
170
171
172
  /*!
  * \brief Get Type name of this boosting object
  */
173
  virtual const char* Name() const override { return "gbdt"; }
174

175
protected:
Guolin Ke's avatar
Guolin Ke committed
176
177
178
  /*!
  * \brief Implement bagging logic
  * \param iter Current interation
179
  * \param curr_class Current class for multiclass training
Guolin Ke's avatar
Guolin Ke committed
180
  */
181
  void Bagging(int iter, const int curr_class);
Guolin Ke's avatar
Guolin Ke committed
182
  /*!
Qiwei Ye's avatar
Qiwei Ye committed
183
184
  * \brief updating score for out-of-bag data.
  *        Data should be update since we may re-bagging data on training
Guolin Ke's avatar
Guolin Ke committed
185
  * \param tree Trained tree of this iteration
186
  * \param curr_class Current class for multiclass training
Guolin Ke's avatar
Guolin Ke committed
187
  */
188
  void UpdateScoreOutOfBag(const Tree* tree, const int curr_class);
Guolin Ke's avatar
Guolin Ke committed
189
190
191
192
193
  /*!
  * \brief calculate the object function
  */
  void Boosting();
  /*!
Qiwei Ye's avatar
Qiwei Ye committed
194
  * \brief updating score after tree was trained
Guolin Ke's avatar
Guolin Ke committed
195
  * \param tree Trained tree of this iteration
196
  * \param curr_class Current class for multiclass training
Guolin Ke's avatar
Guolin Ke committed
197
  */
198
  virtual void UpdateScore(const Tree* tree, const int curr_class);
Guolin Ke's avatar
Guolin Ke committed
199
  /*!
Hui Xue's avatar
Hui Xue committed
200
  * \brief Print metric result of current iteration
Guolin Ke's avatar
Guolin Ke committed
201
202
  * \param iter Current interation
  */
wxchan's avatar
wxchan committed
203
  bool OutputMetric(int iter);
wxchan's avatar
wxchan committed
204
205
206
207
  /*!
  * \brief Calculate feature importances
  * \param last_iter Last tree use to calculate
  */
208
209
210
  std::string FeatureImportance() const;
  /*! \brief current iteration */
  int iter_;
Guolin Ke's avatar
Guolin Ke committed
211
212
213
  /*! \brief Pointer to training data */
  const Dataset* train_data_;
  /*! \brief Config of gbdt */
Guolin Ke's avatar
Guolin Ke committed
214
  const BoostingConfig* gbdt_config_;
Hui Xue's avatar
Hui Xue committed
215
  /*! \brief Tree learner, will use this class to learn trees */
Guolin Ke's avatar
Guolin Ke committed
216
  std::vector<std::unique_ptr<TreeLearner>> tree_learner_;
Guolin Ke's avatar
Guolin Ke committed
217
218
  /*! \brief Objective function */
  const ObjectiveFunction* object_function_;
Hui Xue's avatar
Hui Xue committed
219
  /*! \brief Store and update training data's score */
Guolin Ke's avatar
Guolin Ke committed
220
  std::unique_ptr<ScoreUpdater> train_score_updater_;
Guolin Ke's avatar
Guolin Ke committed
221
222
223
  /*! \brief Metrics for training data */
  std::vector<const Metric*> training_metrics_;
  /*! \brief Store and update validation data's scores */
Guolin Ke's avatar
Guolin Ke committed
224
  std::vector<std::unique_ptr<ScoreUpdater>> valid_score_updater_;
Guolin Ke's avatar
Guolin Ke committed
225
226
  /*! \brief Metric for validation data */
  std::vector<std::vector<const Metric*>> valid_metrics_;
wxchan's avatar
wxchan committed
227
228
  /*! \brief Number of rounds for early stopping */
  int early_stopping_round_;
wxchan's avatar
wxchan committed
229
230
  /*! \brief Best score(s) for early stopping */
  std::vector<std::vector<int>> best_iter_;
231
  std::vector<std::vector<double>> best_score_;
Guolin Ke's avatar
Guolin Ke committed
232
  /*! \brief Trained models(trees) */
Guolin Ke's avatar
Guolin Ke committed
233
  std::vector<std::unique_ptr<Tree>> models_;
Guolin Ke's avatar
Guolin Ke committed
234
235
236
  /*! \brief Max feature index of training data*/
  int max_feature_idx_;
  /*! \brief First order derivative of training data */
Guolin Ke's avatar
Guolin Ke committed
237
  std::vector<score_t> gradients_;
Guolin Ke's avatar
Guolin Ke committed
238
  /*! \brief Secend order derivative of training data */
Guolin Ke's avatar
Guolin Ke committed
239
  std::vector<score_t> hessians_;
Guolin Ke's avatar
Guolin Ke committed
240
  /*! \brief Store the data indices of out-of-bag */
Guolin Ke's avatar
Guolin Ke committed
241
  std::vector<data_size_t> out_of_bag_data_indices_;
Guolin Ke's avatar
Guolin Ke committed
242
243
244
  /*! \brief Number of out-of-bag data */
  data_size_t out_of_bag_data_cnt_;
  /*! \brief Store the indices of in-bag data */
Guolin Ke's avatar
Guolin Ke committed
245
  std::vector<data_size_t> bag_data_indices_;
Guolin Ke's avatar
Guolin Ke committed
246
247
248
249
  /*! \brief Number of in-bag data */
  data_size_t bag_data_cnt_;
  /*! \brief Number of traning data */
  data_size_t num_data_;
250
251
  /*! \brief Number of classes */
  int num_class_;
Guolin Ke's avatar
Guolin Ke committed
252
253
254
255
256
257
  /*! \brief Random generator, used for bagging */
  Random random_;
  /*!
  *   \brief Sigmoid parameter, used for prediction.
  *          if > 0 meas output score will transform by sigmoid function
  */
258
  double sigmoid_;
Guolin Ke's avatar
Guolin Ke committed
259
260
  /*! \brief Index of label column */
  data_size_t label_idx_;
261
  /*! \brief Saved number of models */
262
  int saved_model_size_;
263
264
  /*! \brief File to write models */
  std::ofstream model_output_file_;
265
  /*! \brief number of used model */
Guolin Ke's avatar
Guolin Ke committed
266
  int num_iteration_for_pred_;
Guolin Ke's avatar
Guolin Ke committed
267
268
  /*! \brief Shrinkage rate for one iteration */
  double shrinkage_rate_;
269
  int num_init_iteration_;
Guolin Ke's avatar
Guolin Ke committed
270
271
272
};

}  // namespace LightGBM
Guolin Ke's avatar
Guolin Ke committed
273
#endif   // LightGBM_BOOSTING_GBDT_H_