[docs] document how to use a custom objective function via the C API (#6396)

docs/Parameters.rst

@@ -123,6 +123,14 @@ Core Parameters
 
       -  label should be ``int`` type, and larger number represents the higher relevance (e.g. 0:bad, 1:fair, 2:good, 3:perfect)
 
+   -  custom objective function (gradients and hessians not computed directly by LightGBM)
+
+      -  ``custom``
+
+      -  **Note**: Not supported in CLI version
+
+      -  must be passed through parameters explicitly in the C API
+
 -  ``boosting`` :raw-html:`<a id="boosting" title="Permalink to this parameter" href="#boosting">&#x1F517;&#xFE0E;</a>`, default = ``gbdt``, type = enum, options: ``gbdt``, ``rf``, ``dart``, aliases: ``boosting_type``, ``boost``
 
    -  ``gbdt``, traditional Gradient Boosting Decision Tree, aliases: ``gbrt``

include/LightGBM/config.h

@@ -158,6 +158,10 @@ struct Config {
   // descl2 = ``rank_xendcg``, `XE_NDCG_MART <https://arxiv.org/abs/1911.09798>`__ ranking objective function, aliases: ``xendcg``, ``xe_ndcg``, ``xe_ndcg_mart``, ``xendcg_mart``
   // descl2 = ``rank_xendcg`` is faster than and achieves the similar performance as ``lambdarank``
   // descl2 = label should be ``int`` type, and larger number represents the higher relevance (e.g. 0:bad, 1:fair, 2:good, 3:perfect)
+  // desc = custom objective function (gradients and hessians not computed directly by LightGBM)
+  // descl2 = ``custom``
+  // descl2 = **Note**: Not supported in CLI version
+  // descl2 = must be passed through parameters explicitly in the C API
   std::string objective = "regression";
 
   // [no-automatically-extract]

src/boosting/gbdt.cpp

@@ -344,6 +344,7 @@ bool GBDT::TrainOneIter(const score_t* gradients, const score_t* hessians) {
     hessians = hessians_pointer_;
   } else {
     // use customized objective function
+    // the check below fails unless objective=custom is provided in the parameters on Booster creation
     CHECK(objective_function_ == nullptr);
     if (data_sample_strategy_->IsHessianChange()) {
       // need to copy customized gradients when using GOSS