Fix loss computation in rank cross-entropy objective (#3031)

* Fix loss computation

* fix test

src/objective/rank_objective.hpp

@@ -323,7 +323,7 @@ class RankXENDCG : public RankingObjective {
     double sum_l1 = 0.0f;
     for (data_size_t i = 0; i < cnt; ++i) {
       l1s[i] = -l1s[i] / sum_labels + rho[i];
-      sum_l1 += l1s[i];
+      sum_l1 += l1s[i] / (1. - rho[i]);
     }
     if (cnt <= 1) {
       // when cnt <= 1, the l2 and l3 are zeros
@@ -336,13 +336,13 @@ class RankXENDCG : public RankingObjective {
       std::vector<double> l2s(cnt, 0.0);
       double sum_l2 = 0.0;
       for (data_size_t i = 0; i < cnt; ++i) {
-        l2s[i] = (sum_l1 - l1s[i]) / (1 - rho[i]);
-        sum_l2 += l2s[i];
+        l2s[i] = sum_l1 - (l1s[i] / (1. - rho[i]));
+        sum_l2 += l2s[i] * rho[i] / (1. - rho[i]);
       }
       for (data_size_t i = 0; i < cnt; ++i) {
-        auto l3 = (sum_l2 - l2s[i]) / (1 - rho[i]);
+        auto l3 = sum_l2 - (l2s[i] * rho[i] / (1. - rho[i]));
         lambdas[i] = static_cast<score_t>(l1s[i] + rho[i] * l2s[i] +
-                                          rho[i] * rho[i] * l3);
+                                          rho[i] * l3);
         hessians[i] = static_cast<score_t>(rho[i] * (1.0 - rho[i]));
       }
     }

tests/python_package_test/test_sklearn.py

@@ -130,8 +130,8 @@ class TestSklearn(unittest.TestCase):
                 eval_metric='ndcg',
                 callbacks=[lgb.reset_parameter(learning_rate=lambda x: max(0.01, 0.1 - 0.01 * x))])
         self.assertLessEqual(gbm.best_iteration_, 24)
-        self.assertGreater(gbm.best_score_['valid_0']['ndcg@1'], 0.6382)
-        self.assertGreater(gbm.best_score_['valid_0']['ndcg@3'], 0.6319)
+        self.assertGreater(gbm.best_score_['valid_0']['ndcg@1'], 0.6211)
+        self.assertGreater(gbm.best_score_['valid_0']['ndcg@3'], 0.6253)
 
     def test_regression_with_custom_objective(self):
         X, y = load_boston(True)