gbdt.cpp 21.8 KB
Newer Older
Guolin Ke's avatar
Guolin Ke committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
#include "gbdt.h"

#include <LightGBM/utils/common.h>

#include <LightGBM/feature.h>
#include <LightGBM/objective_function.h>
#include <LightGBM/metric.h>

#include <ctime>

#include <sstream>
#include <chrono>
#include <string>
#include <vector>
15
#include <utility>
Guolin Ke's avatar
Guolin Ke committed
16
17
18

namespace LightGBM {

wxchan's avatar
wxchan committed
19
20
21
GBDT::GBDT() 
  :num_iteration_for_pred_(0), 
  num_init_iteration_(0) {
Guolin Ke's avatar
Guolin Ke committed
22

Guolin Ke's avatar
Guolin Ke committed
23
24
25
}

GBDT::~GBDT() {
Guolin Ke's avatar
Guolin Ke committed
26

Guolin Ke's avatar
Guolin Ke committed
27
28
}

29
30
31
void GBDT::Init(const BoostingConfig* config, const Dataset* train_data, const ObjectiveFunction* object_function,
     const std::vector<const Metric*>& training_metrics) {
  iter_ = 0;
wxchan's avatar
wxchan committed
32
  num_iteration_for_pred_ = 0;
33
  max_feature_idx_ = 0;
wxchan's avatar
wxchan committed
34
  num_class_ = config->num_class;
Guolin Ke's avatar
Guolin Ke committed
35
  random_ = Random(config->bagging_seed);
wxchan's avatar
wxchan committed
36
  train_data_ = nullptr;
Guolin Ke's avatar
Guolin Ke committed
37
  gbdt_config_ = nullptr;
38
  tree_learner_ = nullptr;
wxchan's avatar
wxchan committed
39
40
41
42
43
  ResetTrainingData(config, train_data, object_function, training_metrics);
}

void GBDT::ResetTrainingData(const BoostingConfig* config, const Dataset* train_data, const ObjectiveFunction* object_function,
  const std::vector<const Metric*>& training_metrics) {
Guolin Ke's avatar
Guolin Ke committed
44
  auto new_config = std::unique_ptr<BoostingConfig>(new BoostingConfig(*config));
wxchan's avatar
wxchan committed
45
46
47
  if (train_data_ != nullptr && !train_data_->CheckAlign(*train_data)) {
    Log::Fatal("cannot reset training data, since new training data has different bin mappers");
  }
Guolin Ke's avatar
Guolin Ke committed
48
49
50
  early_stopping_round_ = new_config->early_stopping_round;
  shrinkage_rate_ = new_config->learning_rate;

Guolin Ke's avatar
Guolin Ke committed
51
  object_function_ = object_function;
Guolin Ke's avatar
Guolin Ke committed
52

Guolin Ke's avatar
Guolin Ke committed
53
  sigmoid_ = -1.0f;
wxchan's avatar
wxchan committed
54
  if (object_function_ != nullptr
Guolin Ke's avatar
Guolin Ke committed
55
56
    && std::string(object_function_->GetName()) == std::string("binary")) {
    // only binary classification need sigmoid transform
Guolin Ke's avatar
Guolin Ke committed
57
    sigmoid_ = new_config->sigmoid;
58
  }
Guolin Ke's avatar
Guolin Ke committed
59

Guolin Ke's avatar
Guolin Ke committed
60
  if (train_data_ != train_data && train_data != nullptr) {
61
62
    if (tree_learner_ == nullptr) {
      tree_learner_ = std::unique_ptr<TreeLearner>(TreeLearner::CreateTreeLearner(new_config->tree_learner_type, &new_config->tree_config));
Guolin Ke's avatar
Guolin Ke committed
63
64
    }
    // init tree learner
65
    tree_learner_->Init(train_data);
Guolin Ke's avatar
Guolin Ke committed
66

Guolin Ke's avatar
Guolin Ke committed
67
68
69
70
71
72
    // push training metrics
    training_metrics_.clear();
    for (const auto& metric : training_metrics) {
      training_metrics_.push_back(metric);
    }
    training_metrics_.shrink_to_fit();
wxchan's avatar
wxchan committed
73
74
75
76
77
78
79
80
81
82
83
84
85
    // not same training data, need reset score and others
    // create score tracker
    train_score_updater_.reset(new ScoreUpdater(train_data, num_class_));
    // update score
    for (int i = 0; i < iter_; ++i) {
      for (int curr_class = 0; curr_class < num_class_; ++curr_class) {
        auto curr_tree = (i + num_init_iteration_) * num_class_ + curr_class;
        train_score_updater_->AddScore(models_[curr_tree].get(), curr_class);
      }
    }
    num_data_ = train_data->num_data();
    // create buffer for gradients and hessians
    if (object_function_ != nullptr) {
86
87
      gradients_.resize(num_data_ * num_class_);
      hessians_.resize(num_data_ * num_class_);
wxchan's avatar
wxchan committed
88
89
90
91
92
    }
    // get max feature index
    max_feature_idx_ = train_data->num_total_features() - 1;
    // get label index
    label_idx_ = train_data->label_idx();
Guolin Ke's avatar
Guolin Ke committed
93
94
  }

Guolin Ke's avatar
Guolin Ke committed
95
96
  if ((train_data_ != train_data && train_data != nullptr)
    || (gbdt_config_ != nullptr && gbdt_config_->bagging_fraction != new_config->bagging_fraction)) {
wxchan's avatar
wxchan committed
97
    // if need bagging, create buffer
Guolin Ke's avatar
Guolin Ke committed
98
    if (new_config->bagging_fraction < 1.0 && new_config->bagging_freq > 0) {
99
100
      out_of_bag_data_indices_.resize(num_data_);
      bag_data_indices_.resize(num_data_);
wxchan's avatar
wxchan committed
101
102
103
104
105
106
    } else {
      out_of_bag_data_cnt_ = 0;
      out_of_bag_data_indices_.clear();
      bag_data_cnt_ = num_data_;
      bag_data_indices_.clear();
    }
Guolin Ke's avatar
Guolin Ke committed
107
  }
wxchan's avatar
wxchan committed
108
  train_data_ = train_data;
Guolin Ke's avatar
Guolin Ke committed
109
110
  if (train_data_ != nullptr) {
    // reset config for tree learner
111
    tree_learner_->ResetConfig(&new_config->tree_config);
Guolin Ke's avatar
Guolin Ke committed
112
  }
Guolin Ke's avatar
Guolin Ke committed
113
  gbdt_config_.reset(new_config.release());
Guolin Ke's avatar
Guolin Ke committed
114
115
}

wxchan's avatar
wxchan committed
116
void GBDT::AddValidDataset(const Dataset* valid_data,
Guolin Ke's avatar
Guolin Ke committed
117
  const std::vector<const Metric*>& valid_metrics) {
wxchan's avatar
wxchan committed
118
119
  if (!train_data_->CheckAlign(*valid_data)) {
    Log::Fatal("cannot add validation data, since it has different bin mappers with training data");
120
  }
Guolin Ke's avatar
Guolin Ke committed
121
  // for a validation dataset, we need its score and metric
Guolin Ke's avatar
Guolin Ke committed
122
  auto new_score_updater = std::unique_ptr<ScoreUpdater>(new ScoreUpdater(valid_data, num_class_));
wxchan's avatar
wxchan committed
123
124
125
126
127
128
129
  // update score
  for (int i = 0; i < iter_; ++i) {
    for (int curr_class = 0; curr_class < num_class_; ++curr_class) {
      auto curr_tree = (i + num_init_iteration_) * num_class_ + curr_class;
      new_score_updater->AddScore(models_[curr_tree].get(), curr_class);
    }
  }
Guolin Ke's avatar
Guolin Ke committed
130
  valid_score_updater_.push_back(std::move(new_score_updater));
Guolin Ke's avatar
Guolin Ke committed
131
  valid_metrics_.emplace_back();
132
133
134
  if (early_stopping_round_ > 0) {
    best_iter_.emplace_back();
    best_score_.emplace_back();
Guolin Ke's avatar
Guolin Ke committed
135
    best_msg_.emplace_back();
136
  }
Guolin Ke's avatar
Guolin Ke committed
137
138
  for (const auto& metric : valid_metrics) {
    valid_metrics_.back().push_back(metric);
139
140
141
    if (early_stopping_round_ > 0) {
      best_iter_.back().push_back(0);
      best_score_.back().push_back(kMinScore);
Guolin Ke's avatar
Guolin Ke committed
142
      best_msg_.back().emplace_back();
143
    }
Guolin Ke's avatar
Guolin Ke committed
144
  }
Guolin Ke's avatar
Guolin Ke committed
145
  valid_metrics_.back().shrink_to_fit();
Guolin Ke's avatar
Guolin Ke committed
146
147
148
}


149
void GBDT::Bagging(int iter) {
Guolin Ke's avatar
Guolin Ke committed
150
  // if need bagging
Guolin Ke's avatar
Guolin Ke committed
151
  if (!out_of_bag_data_indices_.empty() && iter % gbdt_config_->bagging_freq == 0) {
Guolin Ke's avatar
Guolin Ke committed
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
    // if doesn't have query data
    if (train_data_->metadata().query_boundaries() == nullptr) {
      bag_data_cnt_ =
        static_cast<data_size_t>(gbdt_config_->bagging_fraction * num_data_);
      out_of_bag_data_cnt_ = num_data_ - bag_data_cnt_;
      data_size_t cur_left_cnt = 0;
      data_size_t cur_right_cnt = 0;
      // random bagging, minimal unit is one record
      for (data_size_t i = 0; i < num_data_; ++i) {
        double prob =
          (bag_data_cnt_ - cur_left_cnt) / static_cast<double>(num_data_ - i);
        if (random_.NextDouble() < prob) {
          bag_data_indices_[cur_left_cnt++] = i;
        } else {
          out_of_bag_data_indices_[cur_right_cnt++] = i;
        }
      }
    } else {
      // if have query data
      const data_size_t* query_boundaries = train_data_->metadata().query_boundaries();
      data_size_t num_query = train_data_->metadata().num_queries();
      data_size_t bag_query_cnt =
          static_cast<data_size_t>(num_query * gbdt_config_->bagging_fraction);
      data_size_t cur_left_query_cnt = 0;
      data_size_t cur_left_cnt = 0;
      data_size_t cur_right_cnt = 0;
      // random bagging, minimal unit is one query
      for (data_size_t i = 0; i < num_query; ++i) {
        double prob =
            (bag_query_cnt - cur_left_query_cnt) / static_cast<double>(num_query - i);
        if (random_.NextDouble() < prob) {
          for (data_size_t j = query_boundaries[i]; j < query_boundaries[i + 1]; ++j) {
            bag_data_indices_[cur_left_cnt++] = j;
          }
          cur_left_query_cnt++;
        } else {
          for (data_size_t j = query_boundaries[i]; j < query_boundaries[i + 1]; ++j) {
            out_of_bag_data_indices_[cur_right_cnt++] = j;
          }
        }
      }
      bag_data_cnt_ = cur_left_cnt;
      out_of_bag_data_cnt_ = num_data_ - bag_data_cnt_;
    }
Guolin Ke's avatar
Guolin Ke committed
196
    Log::Debug("Re-bagging, using %d data to train", bag_data_cnt_);
Guolin Ke's avatar
Guolin Ke committed
197
    // set bagging data to tree learner
198
    tree_learner_->SetBaggingData(bag_data_indices_.data(), bag_data_cnt_);
Guolin Ke's avatar
Guolin Ke committed
199
200
201
  }
}

202
void GBDT::UpdateScoreOutOfBag(const Tree* tree, const int curr_class) {
Hui Xue's avatar
Hui Xue committed
203
  // we need to predict out-of-bag socres of data for boosting
Guolin Ke's avatar
Guolin Ke committed
204
  if (!out_of_bag_data_indices_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
205
    train_score_updater_->AddScore(tree, out_of_bag_data_indices_.data(), out_of_bag_data_cnt_, curr_class);
Guolin Ke's avatar
Guolin Ke committed
206
207
208
  }
}

209
bool GBDT::TrainOneIter(const score_t* gradient, const score_t* hessian, bool is_eval) {
Guolin Ke's avatar
Guolin Ke committed
210
211
212
213
214
215
  // boosting first
  if (gradient == nullptr || hessian == nullptr) {
    Boosting();
    gradient = gradients_.data();
    hessian = hessians_.data();
  }
216
217
  // bagging logic
  Bagging(iter_);
Guolin Ke's avatar
Guolin Ke committed
218
219
220
  for (int curr_class = 0; curr_class < num_class_; ++curr_class) {

    // train a new tree
221
    std::unique_ptr<Tree> new_tree(tree_learner_->Train(gradient + curr_class * num_data_, hessian + curr_class * num_data_));
Guolin Ke's avatar
Guolin Ke committed
222
223
224
225
    // if cannot learn a new tree, then stop
    if (new_tree->num_leaves() <= 1) {
      Log::Info("Stopped training because there are no more leafs that meet the split requirements.");
      return true;
226
    }
227

Guolin Ke's avatar
Guolin Ke committed
228
229
230
231
232
    // shrinkage by learning rate
    new_tree->Shrinkage(shrinkage_rate_);
    // update score
    UpdateScore(new_tree.get(), curr_class);
    UpdateScoreOutOfBag(new_tree.get(), curr_class);
233

Guolin Ke's avatar
Guolin Ke committed
234
235
236
237
238
239
240
241
242
    // add model
    models_.push_back(std::move(new_tree));
  }
  ++iter_;
  if (is_eval) {
    return EvalAndCheckEarlyStopping();
  } else {
    return false;
  }
243

Guolin Ke's avatar
Guolin Ke committed
244
}
245

wxchan's avatar
wxchan committed
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
void GBDT::RollbackOneIter() {
  if (iter_ == 0) { return; }
  int cur_iter = iter_ + num_init_iteration_ - 1;
  // reset score
  for (int curr_class = 0; curr_class < num_class_; ++curr_class) {
    auto curr_tree = cur_iter * num_class_ + curr_class;
    models_[curr_tree]->Shrinkage(-1.0);
    train_score_updater_->AddScore(models_[curr_tree].get(), curr_class);
    for (auto& score_updater : valid_score_updater_) {
      score_updater->AddScore(models_[curr_tree].get(), curr_class);
    }
  }
  // remove model
  for (int curr_class = 0; curr_class < num_class_; ++curr_class) {
    models_.pop_back();
  }
  --iter_;
}

Guolin Ke's avatar
Guolin Ke committed
265
bool GBDT::EvalAndCheckEarlyStopping() {
266
267
  bool is_met_early_stopping = false;
  // print message for metric
Guolin Ke's avatar
Guolin Ke committed
268
269
  auto best_msg = OutputMetric(iter_);
  is_met_early_stopping = !best_msg.empty();
270
271
272
  if (is_met_early_stopping) {
    Log::Info("Early stopping at iteration %d, the best iteration round is %d",
      iter_, iter_ - early_stopping_round_);
Guolin Ke's avatar
Guolin Ke committed
273
    Log::Info("Output of best iteration round:\n%s", best_msg.c_str());
274
    // pop last early_stopping_round_ models
275
    for (int i = 0; i < early_stopping_round_ * num_class_; ++i) {
276
277
278
279
      models_.pop_back();
    }
  }
  return is_met_early_stopping;
Guolin Ke's avatar
Guolin Ke committed
280
281
}

282
void GBDT::UpdateScore(const Tree* tree, const int curr_class) {
Guolin Ke's avatar
Guolin Ke committed
283
  // update training score
284
  train_score_updater_->AddScore(tree_learner_.get(), curr_class);
Guolin Ke's avatar
Guolin Ke committed
285
  // update validation score
Guolin Ke's avatar
Guolin Ke committed
286
287
  for (auto& score_updater : valid_score_updater_) {
    score_updater->AddScore(tree, curr_class);
Guolin Ke's avatar
Guolin Ke committed
288
289
290
  }
}

Guolin Ke's avatar
Guolin Ke committed
291
292
293
294
std::string GBDT::OutputMetric(int iter) {
  bool need_output = (iter % gbdt_config_->output_freq) == 0;
  std::string ret = "";
  std::stringstream msg_buf;
295
  std::vector<std::pair<size_t, size_t>> meet_early_stopping_pairs;
Guolin Ke's avatar
Guolin Ke committed
296
  // print training metric
Guolin Ke's avatar
Guolin Ke committed
297
  if (need_output) {
298
299
300
    for (auto& sub_metric : training_metrics_) {
      auto name = sub_metric->GetName();
      auto scores = sub_metric->Eval(train_score_updater_->score());
Guolin Ke's avatar
Guolin Ke committed
301
      for (size_t k = 0; k < name.size(); ++k) {
Guolin Ke's avatar
Guolin Ke committed
302
303
304
305
306
307
308
309
        std::stringstream tmp_buf;
        tmp_buf << "Iteration:" << iter
          << ", training " << name[k]
          << " : " << scores[k];
        Log::Info(tmp_buf.str().c_str());
        if (early_stopping_round_ > 0) {
          msg_buf << tmp_buf.str() << std::endl;
        }
310
      }
311
    }
Guolin Ke's avatar
Guolin Ke committed
312
313
  }
  // print validation metric
Guolin Ke's avatar
Guolin Ke committed
314
  if (need_output || early_stopping_round_ > 0) {
315
316
317
    for (size_t i = 0; i < valid_metrics_.size(); ++i) {
      for (size_t j = 0; j < valid_metrics_[i].size(); ++j) {
        auto test_scores = valid_metrics_[i][j]->Eval(valid_score_updater_[i]->score());
Guolin Ke's avatar
Guolin Ke committed
318
319
320
321
322
323
324
325
326
327
328
        auto name = valid_metrics_[i][j]->GetName();
        for (size_t k = 0; k < name.size(); ++k) {
          std::stringstream tmp_buf;
          tmp_buf << "Iteration:" << iter
            << ", valid_" << i + 1 << " " << name[k]
            << " : " << test_scores[k];
          if (need_output) {
            Log::Info(tmp_buf.str().c_str());
          }
          if (early_stopping_round_ > 0) {
            msg_buf << tmp_buf.str() << std::endl;
329
          }
wxchan's avatar
wxchan committed
330
        }
Guolin Ke's avatar
Guolin Ke committed
331
        if (ret.empty() && early_stopping_round_ > 0) {
332
333
334
          auto cur_score = valid_metrics_[i][j]->factor_to_bigger_better() * test_scores.back();
          if (cur_score > best_score_[i][j]) {
            best_score_[i][j] = cur_score;
335
            best_iter_[i][j] = iter;
Guolin Ke's avatar
Guolin Ke committed
336
            meet_early_stopping_pairs.emplace_back(i, j);
337
          } else {
Guolin Ke's avatar
Guolin Ke committed
338
            if (iter - best_iter_[i][j] >= early_stopping_round_) { ret = best_msg_[i][j]; }
339
          }
wxchan's avatar
wxchan committed
340
341
        }
      }
Guolin Ke's avatar
Guolin Ke committed
342
343
    }
  }
Guolin Ke's avatar
Guolin Ke committed
344
345
346
  for (auto& pair : meet_early_stopping_pairs) {
    best_msg_[pair.first][pair.second] = msg_buf.str();
  }
wxchan's avatar
wxchan committed
347
  return ret;
Guolin Ke's avatar
Guolin Ke committed
348
349
}

350
/*! \brief Get eval result */
351
352
353
354
std::vector<double> GBDT::GetEvalAt(int data_idx) const {
  CHECK(data_idx >= 0 && data_idx <= static_cast<int>(valid_metrics_.size()));
  std::vector<double> ret;
  if (data_idx == 0) {
355
356
    for (auto& sub_metric : training_metrics_) {
      auto scores = sub_metric->Eval(train_score_updater_->score());
357
358
359
      for (auto score : scores) {
        ret.push_back(score);
      }
360
361
    }
  }
362
363
364
365
366
367
368
  else {
    auto used_idx = data_idx - 1;
    for (size_t j = 0; j < valid_metrics_[used_idx].size(); ++j) {
      auto test_scores = valid_metrics_[used_idx][j]->Eval(valid_score_updater_[used_idx]->score());
      for (auto score : test_scores) {
        ret.push_back(score);
      }
369
370
371
372
373
    }
  }
  return ret;
}

Guolin Ke's avatar
Guolin Ke committed
374
/*! \brief Get training scores result */
375
const score_t* GBDT::GetTrainingScore(data_size_t* out_len) {
Guolin Ke's avatar
Guolin Ke committed
376
377
  *out_len = train_score_updater_->num_data() * num_class_;
  return train_score_updater_->score();
378
379
}

wxchan's avatar
wxchan committed
380
void GBDT::GetPredictAt(int data_idx, score_t* out_result, data_size_t* out_len) {
Guolin Ke's avatar
Guolin Ke committed
381
382
383
384
385
  CHECK(data_idx >= 0 && data_idx <= static_cast<int>(valid_metrics_.size()));

  const score_t* raw_scores = nullptr;
  data_size_t num_data = 0;
  if (data_idx == 0) {
wxchan's avatar
wxchan committed
386
    raw_scores = GetTrainingScore(out_len);
Guolin Ke's avatar
Guolin Ke committed
387
388
389
390
391
    num_data = train_score_updater_->num_data();
  } else {
    auto used_idx = data_idx - 1;
    raw_scores = valid_score_updater_[used_idx]->score();
    num_data = valid_score_updater_[used_idx]->num_data();
wxchan's avatar
wxchan committed
392
    *out_len = num_data * num_class_;
Guolin Ke's avatar
Guolin Ke committed
393
394
  }
  if (num_class_ > 1) {
wxchan's avatar
wxchan committed
395
#pragma omp parallel for schedule(static)
Guolin Ke's avatar
Guolin Ke committed
396
    for (data_size_t i = 0; i < num_data; ++i) {
397
      std::vector<double> tmp_result(num_class_);
Guolin Ke's avatar
Guolin Ke committed
398
      for (int j = 0; j < num_class_; ++j) {
399
        tmp_result[j] = raw_scores[j * num_data + i];
Guolin Ke's avatar
Guolin Ke committed
400
401
402
      }
      Common::Softmax(&tmp_result);
      for (int j = 0; j < num_class_; ++j) {
403
        out_result[j * num_data + i] = static_cast<score_t>(tmp_result[j]);
Guolin Ke's avatar
Guolin Ke committed
404
405
      }
    }
Guolin Ke's avatar
Guolin Ke committed
406
  } else if(sigmoid_ > 0.0f){
wxchan's avatar
wxchan committed
407
#pragma omp parallel for schedule(static)
Guolin Ke's avatar
Guolin Ke committed
408
409
410
411
    for (data_size_t i = 0; i < num_data; ++i) {
      out_result[i] = static_cast<score_t>(1.0f / (1.0f + std::exp(-2.0f * sigmoid_ * raw_scores[i])));
    }
  } else {
wxchan's avatar
wxchan committed
412
#pragma omp parallel for schedule(static)
Guolin Ke's avatar
Guolin Ke committed
413
414
415
416
417
418
419
    for (data_size_t i = 0; i < num_data; ++i) {
      out_result[i] = raw_scores[i];
    }
  }

}

Guolin Ke's avatar
Guolin Ke committed
420
void GBDT::Boosting() {
421
422
423
  if (object_function_ == nullptr) {
    Log::Fatal("No object function provided");
  }
Hui Xue's avatar
Hui Xue committed
424
  // objective function will calculate gradients and hessians
Guolin Ke's avatar
Guolin Ke committed
425
  int num_score = 0;
Guolin Ke's avatar
Guolin Ke committed
426
  object_function_->
Guolin Ke's avatar
Guolin Ke committed
427
    GetGradients(GetTrainingScore(&num_score), gradients_.data(), hessians_.data());
Guolin Ke's avatar
Guolin Ke committed
428
429
}

wxchan's avatar
wxchan committed
430
std::string GBDT::DumpModel() const {
Guolin Ke's avatar
Guolin Ke committed
431
  std::stringstream str_buf;
wxchan's avatar
wxchan committed
432

Guolin Ke's avatar
Guolin Ke committed
433
434
435
436
437
438
  str_buf << "{";
  str_buf << "\"name\":\"" << Name() << "\"," << std::endl;
  str_buf << "\"num_class\":" << num_class_ << "," << std::endl;
  str_buf << "\"label_index\":" << label_idx_ << "," << std::endl;
  str_buf << "\"max_feature_idx\":" << max_feature_idx_ << "," << std::endl;
  str_buf << "\"sigmoid\":" << sigmoid_ << "," << std::endl;
wxchan's avatar
wxchan committed
439

Guolin Ke's avatar
Guolin Ke committed
440
441
442
443
444
445
  // output feature names
  auto feature_names = std::ref(feature_names_);
  if (train_data_ != nullptr) {
    feature_names = std::ref(train_data_->feature_names());
  }

Guolin Ke's avatar
Guolin Ke committed
446
  str_buf << "\"feature_names\":[\"" 
Guolin Ke's avatar
Guolin Ke committed
447
448
449
     << Common::Join(feature_names.get(), "\",\"") << "\"]," 
     << std::endl;

Guolin Ke's avatar
Guolin Ke committed
450
  str_buf << "\"tree_info\":[";
wxchan's avatar
wxchan committed
451
452
  for (int i = 0; i < static_cast<int>(models_.size()); ++i) {
    if (i > 0) {
Guolin Ke's avatar
Guolin Ke committed
453
      str_buf << ",";
wxchan's avatar
wxchan committed
454
    }
Guolin Ke's avatar
Guolin Ke committed
455
456
457
458
    str_buf << "{";
    str_buf << "\"tree_index\":" << i << ",";
    str_buf << models_[i]->ToJSON();
    str_buf << "}";
wxchan's avatar
wxchan committed
459
  }
Guolin Ke's avatar
Guolin Ke committed
460
  str_buf << "]" << std::endl;
wxchan's avatar
wxchan committed
461

Guolin Ke's avatar
Guolin Ke committed
462
  str_buf << "}" << std::endl;
wxchan's avatar
wxchan committed
463

Guolin Ke's avatar
Guolin Ke committed
464
  return str_buf.str();
wxchan's avatar
wxchan committed
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
}

void GBDT::SaveModelToFile(int num_iteration, const char* filename) const {
  /*! \brief File to write models */
  std::ofstream output_file;
  output_file.open(filename);
  // output model type
  output_file << Name() << std::endl;
  // output number of class
  output_file << "num_class=" << num_class_ << std::endl;
  // output label index
  output_file << "label_index=" << label_idx_ << std::endl;
  // output max_feature_idx
  output_file << "max_feature_idx=" << max_feature_idx_ << std::endl;
  // output objective name
  if (object_function_ != nullptr) {
    output_file << "objective=" << object_function_->GetName() << std::endl;
482
  }
wxchan's avatar
wxchan committed
483
484
  // output sigmoid parameter
  output_file << "sigmoid=" << sigmoid_ << std::endl;
Guolin Ke's avatar
Guolin Ke committed
485
486
487
488
489
490
  // output feature names
  auto feature_names = std::ref(feature_names_);
  if (train_data_ != nullptr) {
    feature_names = std::ref(train_data_->feature_names());
  }
  output_file << "feature_names=" << Common::Join(feature_names.get(), " ") << std::endl;
wxchan's avatar
wxchan committed
491

Guolin Ke's avatar
Guolin Ke committed
492
  output_file << std::endl;
wxchan's avatar
wxchan committed
493
494
  int num_used_model = 0;
  if (num_iteration <= 0) {
Guolin Ke's avatar
Guolin Ke committed
495
496
    num_used_model = static_cast<int>(models_.size());
  } else {
wxchan's avatar
wxchan committed
497
    num_used_model = num_iteration * num_class_;
Guolin Ke's avatar
Guolin Ke committed
498
  }
wxchan's avatar
wxchan committed
499
  num_used_model = std::min(num_used_model, static_cast<int>(models_.size()));
500
  // output tree models
wxchan's avatar
wxchan committed
501
502
503
  for (int i = 0; i < num_used_model; ++i) {
    output_file << "Tree=" << i << std::endl;
    output_file << models_[i]->ToString() << std::endl;
504
  }
505

wxchan's avatar
wxchan committed
506
507
508
509
  std::vector<std::pair<size_t, std::string>> pairs = FeatureImportance();
  output_file << std::endl << "feature importances:" << std::endl;
  for (size_t i = 0; i < pairs.size(); ++i) {
    output_file << pairs[i].second << "=" << std::to_string(pairs[i].first) << std::endl;
Guolin Ke's avatar
Guolin Ke committed
510
  }
wxchan's avatar
wxchan committed
511
  output_file.close();
Guolin Ke's avatar
Guolin Ke committed
512
513
}

Guolin Ke's avatar
Guolin Ke committed
514
void GBDT::LoadModelFromString(const std::string& model_str) {
Guolin Ke's avatar
Guolin Ke committed
515
516
517
  // use serialized string to restore this object
  models_.clear();
  std::vector<std::string> lines = Common::Split(model_str.c_str(), '\n');
518
519

  // get number of classes
520
521
522
523
  auto line = Common::FindFromLines(lines, "num_class=");
  if (line.size() > 0) {
    Common::Atoi(Common::Split(line.c_str(), '=')[1].c_str(), &num_class_);
  } else {
524
    Log::Fatal("Model file doesn't specify the number of classes");
525
526
    return;
  }
Guolin Ke's avatar
Guolin Ke committed
527
  // get index of label
528
529
530
531
  line = Common::FindFromLines(lines, "label_index=");
  if (line.size() > 0) {
    Common::Atoi(Common::Split(line.c_str(), '=')[1].c_str(), &label_idx_);
  } else {
532
    Log::Fatal("Model file doesn't specify the label index");
Guolin Ke's avatar
Guolin Ke committed
533
534
    return;
  }
Guolin Ke's avatar
Guolin Ke committed
535
  // get max_feature_idx first
536
537
538
539
  line = Common::FindFromLines(lines, "max_feature_idx=");
  if (line.size() > 0) {
    Common::Atoi(Common::Split(line.c_str(), '=')[1].c_str(), &max_feature_idx_);
  } else {
540
    Log::Fatal("Model file doesn't specify max_feature_idx");
Guolin Ke's avatar
Guolin Ke committed
541
542
543
    return;
  }
  // get sigmoid parameter
544
545
546
547
  line = Common::FindFromLines(lines, "sigmoid=");
  if (line.size() > 0) {
    Common::Atof(Common::Split(line.c_str(), '=')[1].c_str(), &sigmoid_);
  } else {
548
    sigmoid_ = -1.0f;
Guolin Ke's avatar
Guolin Ke committed
549
  }
Guolin Ke's avatar
Guolin Ke committed
550
551
552
553
554
555
556
557
558
559
560
561
562
  // get feature names
  line = Common::FindFromLines(lines, "feature_names=");
  if (line.size() > 0) {
    feature_names_ = Common::Split(Common::Split(line.c_str(), '=')[1].c_str(), ' ');
    if (feature_names_.size() != static_cast<size_t>(max_feature_idx_ + 1)) {
      Log::Fatal("Wrong size of feature_names");
      return;
    }
  } else {
    Log::Fatal("Model file doesn't contain feature names");
    return;
  }

Guolin Ke's avatar
Guolin Ke committed
563
  // get tree models
564
  size_t i = 0;
Guolin Ke's avatar
Guolin Ke committed
565
566
567
568
569
570
571
  while (i < lines.size()) {
    size_t find_pos = lines[i].find("Tree=");
    if (find_pos != std::string::npos) {
      ++i;
      int start = static_cast<int>(i);
      while (i < lines.size() && lines[i].find("Tree=") == std::string::npos) { ++i; }
      int end = static_cast<int>(i);
Guolin Ke's avatar
Guolin Ke committed
572
      std::string tree_str = Common::Join<std::string>(lines, start, end, "\n");
Guolin Ke's avatar
Guolin Ke committed
573
574
      auto new_tree = std::unique_ptr<Tree>(new Tree(tree_str));
      models_.push_back(std::move(new_tree));
Guolin Ke's avatar
Guolin Ke committed
575
576
577
578
    } else {
      ++i;
    }
  }
579
  Log::Info("Finished loading %d models", models_.size());
wxchan's avatar
wxchan committed
580
581
  num_iteration_for_pred_ = static_cast<int>(models_.size()) / num_class_;
  num_init_iteration_ = num_iteration_for_pred_;
Guolin Ke's avatar
Guolin Ke committed
582
583
}

wxchan's avatar
wxchan committed
584
std::vector<std::pair<size_t, std::string>> GBDT::FeatureImportance() const {
Guolin Ke's avatar
Guolin Ke committed
585
586
587
588
  auto feature_names = std::ref(feature_names_);
  if (train_data_ != nullptr) {
    feature_names = std::ref(train_data_->feature_names());
  }
589
  std::vector<size_t> feature_importances(max_feature_idx_ + 1, 0);
590
    for (size_t iter = 0; iter < models_.size(); ++iter) {
591
592
        for (int split_idx = 0; split_idx < models_[iter]->num_leaves() - 1; ++split_idx) {
            ++feature_importances[models_[iter]->split_feature_real(split_idx)];
wxchan's avatar
wxchan committed
593
594
        }
    }
595
596
597
    // store the importance first
    std::vector<std::pair<size_t, std::string>> pairs;
    for (size_t i = 0; i < feature_importances.size(); ++i) {
Guolin Ke's avatar
Guolin Ke committed
598
      if (feature_importances[i] > 0) {
Guolin Ke's avatar
Guolin Ke committed
599
        pairs.emplace_back(feature_importances[i], feature_names.get().at(i));
Guolin Ke's avatar
Guolin Ke committed
600
      }
601
602
603
604
605
    }
    // sort the importance
    std::sort(pairs.begin(), pairs.end(),
      [](const std::pair<size_t, std::string>& lhs,
        const std::pair<size_t, std::string>& rhs) {
606
      return lhs.first > rhs.first;
607
    });
wxchan's avatar
wxchan committed
608
    return pairs;
wxchan's avatar
wxchan committed
609
610
}

611
612
std::vector<double> GBDT::PredictRaw(const double* value) const {
  std::vector<double> ret(num_class_, 0.0f);
wxchan's avatar
wxchan committed
613
  for (int i = 0; i < num_iteration_for_pred_; ++i) {
614
615
616
    for (int j = 0; j < num_class_; ++j) {
      ret[j] += models_[i * num_class_ + j]->Predict(value);
    }
Guolin Ke's avatar
Guolin Ke committed
617
618
619
620
  }
  return ret;
}

621
std::vector<double> GBDT::Predict(const double* value) const {
622
  std::vector<double> ret(num_class_, 0.0f);
wxchan's avatar
wxchan committed
623
  for (int i = 0; i < num_iteration_for_pred_; ++i) {
624
625
    for (int j = 0; j < num_class_; ++j) {
      ret[j] += models_[i * num_class_ + j]->Predict(value);
626
627
    }
  }
628
629
630
631
632
633
  // if need sigmoid transform
  if (sigmoid_ > 0 && num_class_ == 1) {
    ret[0] = 1.0f / (1.0f + std::exp(- 2.0f * sigmoid_ * ret[0]));
  } else if (num_class_ > 1) {
    Common::Softmax(&ret);
  }
634
635
636
  return ret;
}

637
std::vector<int> GBDT::PredictLeafIndex(const double* value) const {
wxchan's avatar
wxchan committed
638
  std::vector<int> ret;
wxchan's avatar
wxchan committed
639
  for (int i = 0; i < num_iteration_for_pred_; ++i) {
640
641
642
    for (int j = 0; j < num_class_; ++j) {
      ret.push_back(models_[i * num_class_ + j]->PredictLeafIndex(value));
    }
wxchan's avatar
wxchan committed
643
644
645
646
  }
  return ret;
}

Guolin Ke's avatar
Guolin Ke committed
647
}  // namespace LightGBM