disable monotone constraint in objective functions with renew_tree_output (#3380)

* Update gbdt.cpp

* Update gbdt.cpp

* Apply suggestions from code review

src/boosting/gbdt.cpp

@@ -79,6 +79,9 @@ void GBDT::Init(const Config* config, const Dataset* train_data, const Objective
   num_tree_per_iteration_ = num_class_;
   if (objective_function_ != nullptr) {
     num_tree_per_iteration_ = objective_function_->NumModelPerIteration();
+    if (objective_function_->IsRenewTreeOutput() && !config->monotone_constraints.empty()) {
+      Log::Fatal("Cannot use ``monotone_constraints`` in %s objective, please disable it.", objective_function_->GetName());
+    }
   }
 
   is_constant_hessian_ = GetIsConstHessian(objective_function);
@@ -665,6 +668,9 @@ void GBDT::ResetTrainingData(const Dataset* train_data, const ObjectiveFunction*
   objective_function_ = objective_function;
   if (objective_function_ != nullptr) {
     CHECK_EQ(num_tree_per_iteration_, objective_function_->NumModelPerIteration());
+    if (objective_function_->IsRenewTreeOutput() && !config_->monotone_constraints.empty()) {
+      Log::Fatal("Cannot use ``monotone_constraints`` in %s objective, please disable it.", objective_function_->GetName());
+    }
   }
   is_constant_hessian_ = GetIsConstHessian(objective_function);
 
@@ -718,6 +724,9 @@ void GBDT::ResetConfig(const Config* config) {
   if (!config->feature_contri.empty()) {
     CHECK_EQ(static_cast<size_t>(train_data_->num_total_features()), config->feature_contri.size());
   }
+  if (objective_function_ != nullptr && objective_function_->IsRenewTreeOutput() && !config->monotone_constraints.empty()) {
+    Log::Fatal("Cannot use ``monotone_constraints`` in %s objective, please disable it.", objective_function_->GetName());
+  }
   early_stopping_round_ = new_config->early_stopping_round;
   shrinkage_rate_ = new_config->learning_rate;
   if (tree_learner_ != nullptr) {