Support Poisson regression (#270)

* Support Poisson regression

Close #233

* update docs

docs/Parameters.md

@@ -16,12 +16,13 @@ The parameter format is ```key1=value1 key2=value2 ... ``` . And parameters can
 * ```task```, default=```train```, type=enum, options=```train```,```prediction```
   * ```train``` for training
   * ```prediction``` for prediction.
-* ```application```, default=```regression```, type=enum, options=```regression```,```regression_l1```,```huber```,```binary```,```lambdarank```,```multiclass```, alias=```objective```,```app```
+* ```application```, default=```regression```, type=enum, options=```regression```,```regression_l1```,```huber```,```fair```,```poisson```,```binary```,```lambdarank```,```multiclass```, alias=```objective```,```app```
   * ```regression```, regression application
     * ```regression_l2```, L2 loss, alias=```mean_squared_error```,```mse```
     * ```regression_l1```, L1 loss, alias=```mean_absolute_error```,```mae```
     * ```huber```, [Huber loss](https://en.wikipedia.org/wiki/Huber_loss "Huber loss - Wikipedia")
     * ```fair```, [Fair loss](http://research.microsoft.com/en-us/um/people/zhang/INRIA/Publis/Tutorial-Estim/node24.html)
+    * ```poisson```, [Poisson regression](https://en.wikipedia.org/wiki/Poisson_regression "Poisson regression")
   * ```binary```, binary classification application 
   * ```lambdarank```, lambdarank application
   * ```multiclass```, multi-class classification application, should set ```num_class``` as well
@@ -173,6 +174,8 @@ The parameter format is ```key1=value1 key2=value2 ... ``` . And parameters can
   * parameter for [Huber loss](https://en.wikipedia.org/wiki/Huber_loss "Huber loss - Wikipedia"). Will be used in regression task.
 * ```fair_c```, default=```1.0```, type=double
   * parameter for [Fair loss](http://research.microsoft.com/en-us/um/people/zhang/INRIA/Publis/Tutorial-Estim/node24.html). Will be used in regression task.
+* ```poission_max_delta_step```, default=```0.7```, type=double
+  * parameter used to safeguard optimization
 * ```scale_pos_weight```, default=```1.0```, type=double
   * weight of positive class in binary classification task
 * ```is_unbalance```, default=```false```, type=bool
@@ -192,6 +195,7 @@ The parameter format is ```key1=value1 key2=value2 ... ``` . And parameters can
   * ```l2```, square loss, alias=```mean_squared_error```, ```mse```
   * ```huber```, [Huber loss](https://en.wikipedia.org/wiki/Huber_loss "Huber loss - Wikipedia")
   * ```fair```, [Fair loss](http://research.microsoft.com/en-us/um/people/zhang/INRIA/Publis/Tutorial-Estim/node24.html)
+  * ```poisson```, [Poisson regression](https://en.wikipedia.org/wiki/Poisson_regression "Poisson regression")
   * ```ndcg```, [NDCG](https://en.wikipedia.org/wiki/Discounted_cumulative_gain#Normalized_DCG)
   * ```map```, [MAP](https://www.kaggle.com/wiki/MeanAveragePrecision)
   * ```auc```, [AUC](https://en.wikipedia.org/wiki/Area_under_the_curve_(pharmacokinetics))

docs/Python-API.md

@@ -592,7 +592,7 @@ The methods of each Class is in alphabetical order.
 
 ###Common Methods
 
-####__init__(boosting_type="gbdt", num_leaves=31, max_depth=-1, learning_rate=0.1, n_estimators=10, max_bin=255, subsample_for_bin=50000, objective="regression", min_split_gain=0, min_child_weight=5, min_child_samples=10, subsample=1, subsample_freq=1, colsample_bytree=1, reg_alpha=0, reg_lambda=0, scale_pos_weight=1, is_unbalance=False, seed=0, nthread=-1, silent=True, sigmoid=1.0, huber_delta=1.0, max_position=20, label_gain=None, drop_rate=0.1, skip_drop=0.5, max_drop=50, uniform_drop=False, xgboost_dart_mode=False)
+####__init__(boosting_type="gbdt", num_leaves=31, max_depth=-1, learning_rate=0.1, n_estimators=10, max_bin=255, subsample_for_bin=50000, objective="regression", min_split_gain=0, min_child_weight=5, min_child_samples=10, subsample=1, subsample_freq=1, colsample_bytree=1, reg_alpha=0, reg_lambda=0, scale_pos_weight=1, is_unbalance=False, seed=0, nthread=-1, silent=True, sigmoid=1.0, huber_delta=1.0, gaussian_eta=1.0, fair_c=1.0, poisson_max_delta_step=0.7, max_position=20, label_gain=None, drop_rate=0.1, skip_drop=0.5, max_drop=50, uniform_drop=False, xgboost_dart_mode=False)
 
     Implementation of the Scikit-Learn API for LightGBM.
 
@@ -652,6 +652,8 @@ The methods of each Class is in alphabetical order.
         It is used to control the width of Gaussian function to approximate hessian.
     fair_c : float
         Only used in regression. Parameter for Fair loss function.
+    poisson_max_delta_step : float
+        parameter used to safeguard optimization in Poisson regression.
     max_position : int
         Only used in lambdarank, will optimize NDCG at this position.
     label_gain : list of float

include/LightGBM/config.h

@@ -136,6 +136,7 @@ public:
   double fair_c = 1.0f;
   // for ApproximateHessianWithGaussian
   double gaussian_eta = 1.0f;
+  double poisson_max_delta_step = 0.7f;
   // for lambdarank
   std::vector<double> label_gain;
   // for lambdarank

python-package/lightgbm/sklearn.py

@@ -130,6 +130,7 @@ class LGBMModel(LGBMModelBase):
                  reg_alpha=0, reg_lambda=0, scale_pos_weight=1,
                  is_unbalance=False, seed=0, nthread=-1, silent=True,
                  sigmoid=1.0, huber_delta=1.0, gaussian_eta=1.0, fair_c=1.0,
+                 poisson_max_delta_step=0.7,
                  max_position=20, label_gain=None,
                  drop_rate=0.1, skip_drop=0.5, max_drop=50,
                  uniform_drop=False, xgboost_dart_mode=False):
@@ -192,6 +193,8 @@ class LGBMModel(LGBMModelBase):
             It is used to control the width of Gaussian function to approximate hessian.
         fair_c : float
             Only used in regression. Parameter for Fair loss function.
+        poisson_max_delta_step : float
+            parameter used to safeguard optimization in Poisson regression.
         max_position : int
             Only used in lambdarank, will optimize NDCG at this position.
         label_gain : list of float
@@ -259,6 +262,7 @@ class LGBMModel(LGBMModelBase):
         self.huber_delta = huber_delta
         self.gaussian_eta = gaussian_eta
         self.fair_c = fair_c
+        self.poisson_max_delta_step = poisson_max_delta_step
         self.max_position = max_position
         self.label_gain = label_gain
         self.drop_rate = drop_rate
@@ -487,6 +491,7 @@ class LGBMRegressor(LGBMModel, LGBMRegressorBase):
                  reg_alpha=0, reg_lambda=0,
                  seed=0, nthread=-1, silent=True,
                  huber_delta=1.0, gaussian_eta=1.0, fair_c=1.0,
+                 poisson_max_delta_step=0.7,
                  drop_rate=0.1, skip_drop=0.5, max_drop=50,
                  uniform_drop=False, xgboost_dart_mode=False):
         super(LGBMRegressor, self).__init__(boosting_type=boosting_type, num_leaves=num_leaves,
@@ -499,6 +504,7 @@ class LGBMRegressor(LGBMModel, LGBMRegressorBase):
                                             reg_alpha=reg_alpha, reg_lambda=reg_lambda,
                                             seed=seed, nthread=nthread, silent=silent,
                                             huber_delta=huber_delta, gaussian_eta=gaussian_eta, fair_c=fair_c,
+                                            poisson_max_delta_step=poisson_max_delta_step,
                                             drop_rate=drop_rate, skip_drop=skip_drop, max_drop=max_drop,
                                             uniform_drop=uniform_drop, xgboost_dart_mode=xgboost_dart_mode)
 

src/io/config.cpp

@@ -224,6 +224,7 @@ void ObjectiveConfig::Set(const std::unordered_map<std::string, std::string>& pa
   GetDouble(params, "huber_delta", &huber_delta);
   GetDouble(params, "fair_c", &fair_c);
   GetDouble(params, "gaussian_eta", &gaussian_eta);
+  GetDouble(params, "poisson_max_delta_step", &poisson_max_delta_step);
   GetInt(params, "max_position", &max_position);
   CHECK(max_position > 0);
   GetInt(params, "num_class", &num_class);

src/metric/metric.cpp

@@ -16,6 +16,8 @@ Metric* Metric::CreateMetric(const std::string& type, const MetricConfig& config
     return new HuberLossMetric(config);
   } else if (type == std::string("fair")) {
     return new FairLossMetric(config);
+  } else if (type == std::string("poisson")) {
+    return new PoissonMetric(config);
   } else if (type == std::string("binary_logloss")) {
     return new BinaryLoglossMetric(config);
   } else if (type == std::string("binary_error")) {

src/metric/regression_metric.hpp

@@ -162,5 +162,23 @@ public:
   }
 };
 
+/*! \brief Poisson regression loss for regression task */
+class PoissonMetric: public RegressionMetric<PoissonMetric> {
+public:
+  explicit PoissonMetric(const MetricConfig& config) :RegressionMetric<PoissonMetric>(config) {
+  }
+
+  inline static double LossOnPoint(float label, double score, double, double) {
+    const double eps = 1e-10f;
+    if (score < eps) {
+      score = eps;
+    }
+    return score - label * std::log(score);
+  }
+  inline static const char* Name() {
+    return "poisson";
+  }
+};
+
 }  // namespace LightGBM
 #endif   // LightGBM_METRIC_REGRESSION_METRIC_HPP_

src/objective/objective_function.cpp

@@ -16,6 +16,8 @@ ObjectiveFunction* ObjectiveFunction::CreateObjectiveFunction(const std::string&
     return new RegressionHuberLoss(config);
   } else if (type == std::string("fair")) {
     return new RegressionFairLoss(config);
+  } else if (type == std::string("poisson")) {
+    return new RegressionPoissonLoss(config);
   } else if (type == std::string("binary")) {
     return new BinaryLogloss(config);
   } else if (type == std::string("lambdarank")) {

src/objective/regression_objective.hpp

@@ -236,5 +236,55 @@ private:
   double c_;
 };
 
+
+/*!
+* \brief Objective function for Poisson regression
+*/
+class RegressionPoissonLoss: public ObjectiveFunction {
+public:
+  explicit RegressionPoissonLoss(const ObjectiveConfig& config) {
+    max_delta_step_ =  static_cast<double>(config.poisson_max_delta_step);
+  }
+
+  ~RegressionPoissonLoss() {}
+
+  void Init(const Metadata& metadata, data_size_t num_data) override {
+    num_data_ = num_data;
+    label_ = metadata.label();
+    weights_ = metadata.weights();
+  }
+
+  void GetGradients(const double* score, score_t* gradients,
+    score_t* hessians) const override {
+    if (weights_ == nullptr) {
+#pragma omp parallel for schedule(static)
+      for (data_size_t i = 0; i < num_data_; ++i) {
+        gradients[i] = score[i] - label_[i];
+        hessians[i] = score[i] + max_delta_step_;
+      }
+    } else {
+#pragma omp parallel for schedule(static)
+      for (data_size_t i = 0; i < num_data_; ++i) {
+        gradients[i] = (score[i] - label_[i]) * weights_[i];
+        hessians[i] = (score[i] + max_delta_step_) * weights_[i];
+      }
+    }
+  }
+
+  const char* GetName() const override {
+    return "poisson";
+  }
+
+private:
+  /*! \brief Number of data */
+  data_size_t num_data_;
+  /*! \brief Pointer of label */
+  const float* label_;
+  /*! \brief Pointer of weights */
+  const float* weights_;
+  /*! \brief used to safeguard optimization */
+  double max_delta_step_;
+};
+
 }  // namespace LightGBM
 #endif   // LightGBM_OBJECTIVE_REGRESSION_OBJECTIVE_HPP_