Merge branch 'master' of https://github.com/Microsoft/LightGBM

Conflicts:
	include/LightGBM/config.h
	src/metric/binary_metric.hpp

include/LightGBM/config.h

@@ -313,7 +313,9 @@ struct ParameterAlias {
       { "mlist", "machine_list_file" },
       { "is_save_binary", "is_save_binary_file" },
       { "save_binary", "is_save_binary_file" },
-      { "verbose", "verbosity" }
+      { "early_stopping_rounds", "early_stopping_round"},
+      { "early_stopping", "early_stopping_round"},
+      { "verbosity", "verbose" }
     });
     std::unordered_map<std::string, std::string> tmp_map;
     for (const auto& pair : *params) {

include/LightGBM/metric.h

@@ -32,7 +32,7 @@ public:
   * \param iter Current iteration
   * \param score Current prediction score
   */
-  virtual void Print(int iter, const score_t* score, score_t& loss) const = 0;
+  virtual score_t PrintAndGetLoss(int iter, const score_t* score) const = 0;
 
   /*!
   * \brief Create object of metrics

src/boosting/gbdt.cpp

@@ -13,7 +13,6 @@
 #include <string>
 #include <vector>
 
-
 namespace LightGBM {
 
 GBDT::GBDT(const BoostingConfig* config)
@@ -185,19 +184,44 @@ void GBDT::Train() {
     UpdateScore(new_tree);
     UpdateScoreOutOfBag(new_tree);
     // print message for metric
-    if (OutputMetric(iter + 1)) return;
+    bool is_early_stopping = OutputMetric(iter + 1);
     // add model
     models_.push_back(new_tree);
     // save model to file per iteration
-    fprintf(output_model_file, "Tree=%d\n", iter);
-    fprintf(output_model_file, "%s\n", new_tree->ToString().c_str());
-    fflush(output_model_file);
+    if (early_stopping_round_ > 0){
+        // if use early stopping, save previous model at (iter - early_stopping_round_) iteration
+        if (iter >= early_stopping_round_){
+            fprintf(output_model_file, "Tree=%d\n", iter - early_stopping_round_);
+            Tree * printing_tree = models_.at(iter - early_stopping_round_);
+            fprintf(output_model_file, "%s\n", printing_tree->ToString().c_str());
+            fflush(output_model_file);
+        }
+    }
+    else{
+        fprintf(output_model_file, "Tree=%d\n", iter);
+        fprintf(output_model_file, "%s\n", new_tree->ToString().c_str());
+        fflush(output_model_file);
+    }
     auto end_time = std::chrono::high_resolution_clock::now();
     // output used time per iteration
     Log::Info("%f seconds elapsed, finished %d iteration\n", std::chrono::duration<double,
                                      std::milli>(end_time - start_time) * 1e-3, iter + 1);
+    if (is_early_stopping) {
+        // close file with an early-stopping message
+        Log::Stdout("early stopping at iteration %d, the best iteration round is %d", iter + 1, iter + 1 - early_stopping_round_);
+        fclose(output_model_file);
+        return;
+    }
   }
   // close file
+  if (early_stopping_round_ > 0) {
+      // save remaining models
+      for (int iter = gbdt_config_->num_iterations - early_stopping_round_; iter < static_cast<int>(models_.size()); ++iter){
+        fprintf(output_model_file, "Tree=%d\n", iter);
+        fprintf(output_model_file, "%s\n", models_.at(iter)->ToString().c_str());
+      }
+      fflush(output_model_file);
+  }
   fclose(output_model_file);
 }
 
@@ -215,16 +239,15 @@ void GBDT::UpdateScore(const Tree* tree) {
 }
 
 bool GBDT::OutputMetric(int iter) {
-  score_t train_score_ = 0, test_score_ = 0; 
   bool ret = false;
   // print training metric
   for (auto& sub_metric : training_metrics_) {
-    sub_metric->Print(iter, train_score_updater_->score(), train_score_);
+    sub_metric->PrintAndGetLoss(iter, train_score_updater_->score());
   }
   // print validation metric
   for (size_t i = 0; i < valid_metrics_.size(); ++i) {
     for (size_t j = 0; j < valid_metrics_[i].size(); ++j) {
-      valid_metrics_[i][j]->Print(iter, valid_score_updater_[i]->score(), test_score_);
+      score_t test_score_ = valid_metrics_[i][j]->PrintAndGetLoss(iter, valid_score_updater_[i]->score());
       if (!ret && early_stopping_round_ > 0){
         bool the_bigger_the_better_ = valid_metrics_[i][j]->the_bigger_the_better;
         if (best_score_[i][j] < 0 
@@ -341,7 +364,7 @@ double GBDT::Predict(const double* value) const {
   }
   // if need sigmoid transform
   if (sigmoid_ > 0) {
-    ret = 1.0 / (1.0 + std::exp(-sigmoid_ * ret));
+    ret = 1.0 / (1.0 + std::exp(- 2.0f * sigmoid_ * ret));
   }
   return ret;
 }

src/metric/binary_metric.hpp

@@ -50,14 +50,14 @@ public:
     }
   }
 
-  void Print(int iter, const score_t* score, score_t& loss) const override {
+  score_t PrintAndGetLoss(int iter, const score_t* score) const override {
     score_t sum_loss = 0.0f;
-    if (early_stopping_round_ > 0 || output_freq_ > 0 && iter % output_freq_ == 0) {
+    if (early_stopping_round_ > 0 || (output_freq_ > 0 && iter % output_freq_ == 0)) {
       if (weights_ == nullptr) {
         #pragma omp parallel for schedule(static) reduction(+:sum_loss)
         for (data_size_t i = 0; i < num_data_; ++i) {
           // sigmoid transform
-          score_t prob = 1.0f / (1.0f + std::exp(-sigmoid_ * score[i]));
+          score_t prob = 1.0f / (1.0f + std::exp(-2.0f * sigmoid_ * score[i]));
           // add loss
           sum_loss += PointWiseLossCalculator::LossOnPoint(label_[i], prob);
         }
@@ -65,16 +65,18 @@ public:
         #pragma omp parallel for schedule(static) reduction(+:sum_loss)
         for (data_size_t i = 0; i < num_data_; ++i) {
           // sigmoid transform
-          score_t prob = 1.0f / (1.0f + std::exp(-sigmoid_ * score[i]));
+          score_t prob = 1.0f / (1.0f + std::exp(-2.0f * sigmoid_ * score[i]));
           // add loss
           sum_loss += PointWiseLossCalculator::LossOnPoint(label_[i], prob) * weights_[i];
         }
       }
-      loss = sum_loss / sum_weights_;
+      score_t loss = sum_loss / sum_weights_;
       if (output_freq_ > 0 && iter % output_freq_ == 0){
         Log::Info("Iteration:%d, %s's %s: %f\n", iter, name, PointWiseLossCalculator::Name(), loss);
       }
+      return loss;
     }
+    return 0.0f;
   }
 
 private:
@@ -170,8 +172,8 @@ public:
     }
   }
 
-  void Print(int iter, const score_t* score, score_t& loss) const override {
-    if (early_stopping_round_ > 0 || output_freq_ > 0 && iter % output_freq_ == 0) {
+  score_t PrintAndGetLoss(int iter, const score_t* score) const override {
+    if (early_stopping_round_ > 0 || (output_freq_ > 0 && iter % output_freq_ == 0)) {
       // get indices sorted by score, descent order
       std::vector<data_size_t> sorted_idx;
       for (data_size_t i = 0; i < num_data_; ++i) {
@@ -227,11 +229,12 @@ public:
       if (sum_pos > 0.0f && sum_pos != sum_weights_) {
         auc = accum / (sum_pos *(sum_weights_ - sum_pos));
       }
-      loss = auc;
       if (output_freq_ > 0 && iter % output_freq_ == 0){
         Log::Info("Iteration:%d, %s's %s: %f\n", iter, name, "auc", loss);
       }
+      return auc;
     }
+    return 0.0f;
   }
 
 private:

src/metric/rank_metric.hpp

@@ -75,8 +75,8 @@ public:
     }
   }
 
-  void Print(int iter, const score_t* score, score_t& loss) const override {
-    if (early_stopping_round_ > 0 || output_freq_ > 0 && iter % output_freq_ == 0) {
+  score_t PrintAndGetLoss(int iter, const score_t* score) const override {
+    if (early_stopping_round_ > 0 || (output_freq_ > 0 && iter % output_freq_ == 0)) {
       // some buffers for multi-threading sum up
       std::vector<std::vector<double>> result_buffer_;
       for (int i = 0; i < num_threads_; ++i) {
@@ -134,11 +134,12 @@ public:
         result[j] /= sum_query_weights_;
         result_ss << "NDCG@" << eval_at_[j] << ":" << result[j] << "\t";
       }
-      loss = result[0];
       if (output_freq_ > 0 && iter % output_freq_ == 0){
         Log::Info("Iteration:%d, Test:%s, %s \n", iter, name, result_ss.str().c_str());
       }
+      return result[0];
     }
+    return 0.0f;
   }
 
 private:

src/metric/regression_metric.hpp

@@ -42,8 +42,8 @@ public:
     }
   }
   
-  void Print(int iter, const score_t* score, score_t& loss) const override {
-    if (early_stopping_round_ > 0 || output_freq_ > 0 && iter % output_freq_ == 0) {
+  score_t PrintAndGetLoss(int iter, const score_t* score) const override {
+    if (early_stopping_round_ > 0 || (output_freq_ > 0 && iter % output_freq_ == 0)) {
       score_t sum_loss = 0.0;
       if (weights_ == nullptr) {
         #pragma omp parallel for schedule(static) reduction(+:sum_loss)
@@ -58,11 +58,13 @@ public:
           sum_loss += PointWiseLossCalculator::LossOnPoint(label_[i], score[i]) * weights_[i];
         }
       }
-      loss = PointWiseLossCalculator::AverageLoss(sum_loss, sum_weights_);
+      score_t loss = PointWiseLossCalculator::AverageLoss(sum_loss, sum_weights_);
       if (output_freq_ > 0 && iter % output_freq_ == 0){
         Log::Info("Iteration:%d, %s's %s : %f", iter, name, PointWiseLossCalculator::Name(), loss);
       }
+      return loss;
     }
+    return 0.0f;
   }
 
   inline static score_t AverageLoss(score_t sum_loss, score_t sum_weights) {