[tests][python] added tests for metrics' behavior and fixed case for...

[tests][python] added tests for metrics' behavior and fixed case for multiclass task with custom objective (#1954)

* added metrics test for standard interface

* simplified code

* less trees

* less trees

* use dummy custom objective and metric

* added tests for multiclass metrics aliases

* fixed bug in case of custom obj and num_class > 1

* added metric test for sklearn wrapper

python-package/lightgbm/engine.py

@@ -109,6 +109,8 @@ def train(params, train_set, num_boost_round=100,
     """
     # create predictor first
     params = copy.deepcopy(params)
+    if fobj is not None:
+        params['objective'] = 'none'
     for alias in ["num_iterations", "num_iteration", "n_iter", "num_tree", "num_trees",
                   "num_round", "num_rounds", "num_boost_round", "n_estimators"]:
         if alias in params:
@@ -420,6 +422,8 @@ def cv(params, train_set, num_boost_round=100,
         raise TypeError("Traninig only accepts Dataset object")
 
     params = copy.deepcopy(params)
+    if fobj is not None:
+        params['objective'] = 'none'
     for alias in ["num_iterations", "num_iteration", "n_iter", "num_tree", "num_trees",
                   "num_round", "num_rounds", "num_boost_round", "n_estimators"]:
         if alias in params:

tests/python_package_test/test_engine.py

@@ -3,16 +3,16 @@
 import copy
 import math
 import os
+import random
 import unittest
 
 import lightgbm as lgb
-import random
 import numpy as np
+from scipy.sparse import csr_matrix
 from sklearn.datasets import (load_boston, load_breast_cancer, load_digits,
                               load_iris, load_svmlight_file)
 from sklearn.metrics import log_loss, mean_absolute_error, mean_squared_error, roc_auc_score
 from sklearn.model_selection import train_test_split, TimeSeriesSplit, GroupKFold
-from scipy.sparse import csr_matrix
 
 try:
     import cPickle as pickle
@@ -828,3 +828,383 @@ class TestEngine(unittest.TestCase):
         results = lgb.cv(params, dataset, num_boost_round=10, fpreproc=preprocess_data)
         self.assertIn('multi_logloss-mean', results)
         self.assertEqual(len(results['multi_logloss-mean']), 10)
+
+    def test_metrics(self):
+        def custom_obj(preds, train_data):
+            return np.zeros(preds.shape), np.zeros(preds.shape)
+
+        def custom_metric(preds, train_data):
+            return 'error', 0, False
+
+        X, y = load_digits(2, True)
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
+        lgb_train = lgb.Dataset(X_train, y_train, silent=True)
+        lgb_valid = lgb.Dataset(X_test, y_test, reference=lgb_train, silent=True)
+
+        evals_result = {}
+        params_verbose = {'verbose': -1}
+        params_obj_verbose = {'objective': 'binary', 'verbose': -1}
+        params_obj_metric_log_verbose = {'objective': 'binary', 'metric': 'binary_logloss', 'verbose': -1}
+        params_obj_metric_err_verbose = {'objective': 'binary', 'metric': 'binary_error', 'verbose': -1}
+        params_obj_metric_inv_verbose = {'objective': 'binary', 'metric': 'invalid_metric', 'verbose': -1}
+        params_obj_metric_multi_verbose = {'objective': 'binary',
+                                           'metric': ['binary_logloss', 'binary_error'],
+                                           'verbose': -1}
+        params_obj_metric_none_verbose = {'objective': 'binary', 'metric': 'None', 'verbose': -1}
+        params_metric_log_verbose = {'metric': 'binary_logloss', 'verbose': -1}
+        params_metric_err_verbose = {'metric': 'binary_error', 'verbose': -1}
+        params_metric_inv_verbose = {'metric_types': 'invalid_metric', 'verbose': -1}
+        params_metric_multi_verbose = {'metric': ['binary_logloss', 'binary_error'], 'verbose': -1}
+        params_metric_none_verbose = {'metric': 'None', 'verbose': -1}
+
+        def get_cv_result(params=params_obj_verbose, **kwargs):
+            return lgb.cv(params, lgb_train, num_boost_round=5, verbose_eval=False, **kwargs)
+
+        def train_booster(params=params_obj_verbose, **kwargs):
+            lgb.train(params, lgb_train,
+                      num_boost_round=5,
+                      valid_sets=[lgb_valid],
+                      evals_result=evals_result,
+                      verbose_eval=False, **kwargs)
+
+        # no fobj, no feval
+        # default metric
+        res = get_cv_result()
+        self.assertEqual(len(res), 2)
+        self.assertIn('binary_logloss-mean', res)
+
+        # non-default metric in params
+        res = get_cv_result(params=params_obj_metric_err_verbose)
+        self.assertEqual(len(res), 2)
+        self.assertIn('binary_error-mean', res)
+
+        # default metric in args
+        res = get_cv_result(metrics='binary_logloss')
+        self.assertEqual(len(res), 2)
+        self.assertIn('binary_logloss-mean', res)
+
+        # non-default metric in args
+        res = get_cv_result(metrics='binary_error')
+        self.assertEqual(len(res), 2)
+        self.assertIn('binary_error-mean', res)
+
+        # metric in args overwrites one in params
+        res = get_cv_result(params=params_obj_metric_inv_verbose, metrics='binary_error')
+        self.assertEqual(len(res), 2)
+        self.assertIn('binary_error-mean', res)
+
+        # multiple metrics in params
+        res = get_cv_result(params=params_obj_metric_multi_verbose)
+        self.assertEqual(len(res), 4)
+        self.assertIn('binary_logloss-mean', res)
+        self.assertIn('binary_error-mean', res)
+
+        # multiple metrics in args
+        res = get_cv_result(metrics=['binary_logloss', 'binary_error'])
+        self.assertEqual(len(res), 4)
+        self.assertIn('binary_logloss-mean', res)
+        self.assertIn('binary_error-mean', res)
+
+        # remove default metric by 'None' in list
+        res = get_cv_result(metrics=['None'])
+        self.assertEqual(len(res), 0)
+
+        # remove default metric by 'None' aliases
+        for na_alias in ('None', 'na', 'null', 'custom'):
+            res = get_cv_result(metrics=na_alias)
+            self.assertEqual(len(res), 0)
+
+        # fobj, no feval
+        # no default metric
+        res = get_cv_result(params=params_verbose, fobj=custom_obj)
+        self.assertEqual(len(res), 0)
+
+        # metric in params
+        res = get_cv_result(params=params_metric_err_verbose, fobj=custom_obj)
+        self.assertEqual(len(res), 2)
+        self.assertIn('binary_error-mean', res)
+
+        # metric in args
+        res = get_cv_result(params=params_verbose, fobj=custom_obj, metrics='binary_error')
+        self.assertEqual(len(res), 2)
+        self.assertIn('binary_error-mean', res)
+
+        # metric in args overwrites its' alias in params
+        res = get_cv_result(params=params_metric_inv_verbose, fobj=custom_obj, metrics='binary_error')
+        self.assertEqual(len(res), 2)
+        self.assertIn('binary_error-mean', res)
+
+        # multiple metrics in params
+        res = get_cv_result(params=params_metric_multi_verbose, fobj=custom_obj)
+        self.assertEqual(len(res), 4)
+        self.assertIn('binary_logloss-mean', res)
+        self.assertIn('binary_error-mean', res)
+
+        # multiple metrics in args
+        res = get_cv_result(params=params_verbose, fobj=custom_obj,
+                            metrics=['binary_logloss', 'binary_error'])
+        self.assertEqual(len(res), 4)
+        self.assertIn('binary_logloss-mean', res)
+        self.assertIn('binary_error-mean', res)
+
+        # no fobj, feval
+        # default metric with custom one
+        res = get_cv_result(feval=custom_metric)
+        self.assertEqual(len(res), 4)
+        self.assertIn('binary_logloss-mean', res)
+        self.assertIn('error-mean', res)
+
+        # non-default metric in params with custom one
+        res = get_cv_result(params=params_obj_metric_err_verbose, feval=custom_metric)
+        self.assertEqual(len(res), 4)
+        self.assertIn('binary_error-mean', res)
+        self.assertIn('error-mean', res)
+
+        # default metric in args with custom one
+        res = get_cv_result(metrics='binary_logloss', feval=custom_metric)
+        self.assertEqual(len(res), 4)
+        self.assertIn('binary_logloss-mean', res)
+        self.assertIn('error-mean', res)
+
+        # non-default metric in args with custom one
+        res = get_cv_result(metrics='binary_error', feval=custom_metric)
+        self.assertEqual(len(res), 4)
+        self.assertIn('binary_error-mean', res)
+        self.assertIn('error-mean', res)
+
+        # metric in args overwrites one in params, custom one is evaluated too
+        res = get_cv_result(params=params_obj_metric_inv_verbose, metrics='binary_error', feval=custom_metric)
+        self.assertEqual(len(res), 4)
+        self.assertIn('binary_error-mean', res)
+        self.assertIn('error-mean', res)
+
+        # multiple metrics in params with custom one
+        res = get_cv_result(params=params_obj_metric_multi_verbose, feval=custom_metric)
+        self.assertEqual(len(res), 6)
+        self.assertIn('binary_logloss-mean', res)
+        self.assertIn('binary_error-mean', res)
+        self.assertIn('error-mean', res)
+
+        # multiple metrics in args with custom one
+        res = get_cv_result(metrics=['binary_logloss', 'binary_error'], feval=custom_metric)
+        self.assertEqual(len(res), 6)
+        self.assertIn('binary_logloss-mean', res)
+        self.assertIn('binary_error-mean', res)
+        self.assertIn('error-mean', res)
+
+        # custom metric is evaluated despite 'None' is passed
+        res = get_cv_result(metrics=['None'], feval=custom_metric)
+        self.assertEqual(len(res), 2)
+        self.assertIn('error-mean', res)
+
+        # fobj, feval
+        # no default metric, only custom one
+        res = get_cv_result(params=params_verbose, fobj=custom_obj, feval=custom_metric)
+        self.assertEqual(len(res), 2)
+        self.assertIn('error-mean', res)
+
+        # metric in params with custom one
+        res = get_cv_result(params=params_metric_err_verbose, fobj=custom_obj, feval=custom_metric)
+        self.assertEqual(len(res), 4)
+        self.assertIn('binary_error-mean', res)
+        self.assertIn('error-mean', res)
+
+        # metric in args with custom one
+        res = get_cv_result(params=params_verbose, fobj=custom_obj,
+                            feval=custom_metric, metrics='binary_error')
+        self.assertEqual(len(res), 4)
+        self.assertIn('binary_error-mean', res)
+        self.assertIn('error-mean', res)
+
+        # metric in args overwrites one in params, custom one is evaluated too
+        res = get_cv_result(params=params_metric_inv_verbose, fobj=custom_obj,
+                            feval=custom_metric, metrics='binary_error')
+        self.assertEqual(len(res), 4)
+        self.assertIn('binary_error-mean', res)
+        self.assertIn('error-mean', res)
+
+        # multiple metrics in params with custom one
+        res = get_cv_result(params=params_metric_multi_verbose, fobj=custom_obj, feval=custom_metric)
+        self.assertEqual(len(res), 6)
+        self.assertIn('binary_logloss-mean', res)
+        self.assertIn('binary_error-mean', res)
+        self.assertIn('error-mean', res)
+
+        # multiple metrics in args with custom one
+        res = get_cv_result(params=params_verbose, fobj=custom_obj, feval=custom_metric,
+                            metrics=['binary_logloss', 'binary_error'])
+        self.assertEqual(len(res), 6)
+        self.assertIn('binary_logloss-mean', res)
+        self.assertIn('binary_error-mean', res)
+        self.assertIn('error-mean', res)
+
+        # custom metric is evaluated despite 'None' is passed
+        res = get_cv_result(params=params_metric_none_verbose, fobj=custom_obj, feval=custom_metric)
+        self.assertEqual(len(res), 2)
+        self.assertIn('error-mean', res)
+
+        # no fobj, no feval
+        # default metric
+        train_booster()
+        self.assertEqual(len(evals_result['valid_0']), 1)
+        self.assertIn('binary_logloss', evals_result['valid_0'])
+
+        # default metric in params
+        train_booster(params=params_obj_metric_log_verbose)
+        self.assertEqual(len(evals_result['valid_0']), 1)
+        self.assertIn('binary_logloss', evals_result['valid_0'])
+
+        # non-default metric in params
+        train_booster(params=params_obj_metric_err_verbose)
+        self.assertEqual(len(evals_result['valid_0']), 1)
+        self.assertIn('binary_error', evals_result['valid_0'])
+
+        # multiple metrics in params
+        train_booster(params=params_obj_metric_multi_verbose)
+        self.assertEqual(len(evals_result['valid_0']), 2)
+        self.assertIn('binary_logloss', evals_result['valid_0'])
+        self.assertIn('binary_error', evals_result['valid_0'])
+
+        # remove default metric by 'None' aliases
+        for na_alias in ('None', 'na', 'null', 'custom'):
+            params = {'objective': 'binary', 'metric': na_alias, 'verbose': -1}
+            train_booster(params=params)
+            self.assertEqual(len(evals_result), 0)
+
+        # fobj, no feval
+        # no default metric
+        train_booster(params=params_verbose, fobj=custom_obj)
+        self.assertEqual(len(evals_result), 0)
+
+        # metric in params
+        train_booster(params=params_metric_log_verbose, fobj=custom_obj)
+        self.assertEqual(len(evals_result['valid_0']), 1)
+        self.assertIn('binary_logloss', evals_result['valid_0'])
+
+        # multiple metrics in params
+        train_booster(params=params_metric_multi_verbose, fobj=custom_obj)
+        self.assertEqual(len(evals_result['valid_0']), 2)
+        self.assertIn('binary_logloss', evals_result['valid_0'])
+        self.assertIn('binary_error', evals_result['valid_0'])
+
+        # no fobj, feval
+        # default metric with custom one
+        train_booster(feval=custom_metric)
+        self.assertEqual(len(evals_result['valid_0']), 2)
+        self.assertIn('binary_logloss', evals_result['valid_0'])
+        self.assertIn('error', evals_result['valid_0'])
+
+        # default metric in params with custom one
+        train_booster(params=params_obj_metric_log_verbose, feval=custom_metric)
+        self.assertEqual(len(evals_result['valid_0']), 2)
+        self.assertIn('binary_logloss', evals_result['valid_0'])
+        self.assertIn('error', evals_result['valid_0'])
+
+        # non-default metric in params with custom one
+        train_booster(params=params_obj_metric_err_verbose, feval=custom_metric)
+        self.assertEqual(len(evals_result['valid_0']), 2)
+        self.assertIn('binary_error', evals_result['valid_0'])
+        self.assertIn('error', evals_result['valid_0'])
+
+        # multiple metrics in params with custom one
+        train_booster(params=params_obj_metric_multi_verbose, feval=custom_metric)
+        self.assertEqual(len(evals_result['valid_0']), 3)
+        self.assertIn('binary_logloss', evals_result['valid_0'])
+        self.assertIn('binary_error', evals_result['valid_0'])
+        self.assertIn('error', evals_result['valid_0'])
+
+        # custom metric is evaluated despite 'None' is passed
+        train_booster(params=params_obj_metric_none_verbose, feval=custom_metric)
+        self.assertEqual(len(evals_result), 1)
+        self.assertIn('error', evals_result['valid_0'])
+
+        # fobj, feval
+        # no default metric, only custom one
+        train_booster(params=params_verbose, fobj=custom_obj, feval=custom_metric)
+        self.assertEqual(len(evals_result['valid_0']), 1)
+        self.assertIn('error', evals_result['valid_0'])
+
+        # metric in params with custom one
+        train_booster(params=params_metric_log_verbose, fobj=custom_obj, feval=custom_metric)
+        self.assertEqual(len(evals_result['valid_0']), 2)
+        self.assertIn('binary_logloss', evals_result['valid_0'])
+        self.assertIn('error', evals_result['valid_0'])
+
+        # multiple metrics in params with custom one
+        train_booster(params=params_metric_multi_verbose, fobj=custom_obj, feval=custom_metric)
+        self.assertEqual(len(evals_result['valid_0']), 3)
+        self.assertIn('binary_logloss', evals_result['valid_0'])
+        self.assertIn('binary_error', evals_result['valid_0'])
+        self.assertIn('error', evals_result['valid_0'])
+
+        # custom metric is evaluated despite 'None' is passed
+        train_booster(params=params_metric_none_verbose, fobj=custom_obj, feval=custom_metric)
+        self.assertEqual(len(evals_result), 1)
+        self.assertIn('error', evals_result['valid_0'])
+
+        X, y = load_digits(3, True)
+        lgb_train = lgb.Dataset(X, y, silent=True)
+
+        obj_multi_aliases = ['multiclass', 'softmax', 'multiclassova', 'multiclass_ova', 'ova', 'ovr']
+        for obj_multi_alias in obj_multi_aliases:
+            params_obj_class_3_verbose = {'objective': obj_multi_alias, 'num_class': 3, 'verbose': -1}
+            params_obj_class_1_verbose = {'objective': obj_multi_alias, 'num_class': 1, 'verbose': -1}
+            params_obj_verbose = {'objective': obj_multi_alias, 'verbose': -1}
+            # multiclass default metric
+            res = get_cv_result(params_obj_class_3_verbose)
+            self.assertEqual(len(res), 2)
+            self.assertIn('multi_logloss-mean', res)
+            # multiclass default metric with custom one
+            res = get_cv_result(params_obj_class_3_verbose, feval=custom_metric)
+            self.assertEqual(len(res), 4)
+            self.assertIn('multi_logloss-mean', res)
+            self.assertIn('error-mean', res)
+            # multiclass metric alias with custom one for custom objective
+            res = get_cv_result(params_obj_class_3_verbose, fobj=custom_obj, feval=custom_metric)
+            self.assertEqual(len(res), 2)
+            self.assertIn('error-mean', res)
+            # no metric for invalid class_num
+            res = get_cv_result(params_obj_class_1_verbose, fobj=custom_obj)
+            self.assertEqual(len(res), 0)
+            # custom metric for invalid class_num
+            res = get_cv_result(params_obj_class_1_verbose, fobj=custom_obj, feval=custom_metric)
+            self.assertEqual(len(res), 2)
+            self.assertIn('error-mean', res)
+            # multiclass metric alias with custom one with invalid class_num
+            self.assertRaises(lgb.basic.LightGBMError, get_cv_result,
+                              params_obj_class_1_verbose, metrics=obj_multi_alias,
+                              fobj=custom_obj, feval=custom_metric)
+            # multiclass default metric without num_class
+            self.assertRaises(lgb.basic.LightGBMError, get_cv_result,
+                              params_obj_verbose)
+            for metric_multi_alias in obj_multi_aliases + ['multi_logloss']:
+                # multiclass metric alias
+                res = get_cv_result(params_obj_class_3_verbose, metrics=metric_multi_alias)
+                self.assertEqual(len(res), 2)
+                self.assertIn('multi_logloss-mean', res)
+            # multiclass metric
+            res = get_cv_result(params_obj_class_3_verbose, metrics='multi_error')
+            self.assertEqual(len(res), 2)
+            self.assertIn('multi_error-mean', res)
+            # non-valid metric for multiclass objective
+            self.assertRaises(lgb.basic.LightGBMError, get_cv_result,
+                              params_obj_class_3_verbose, metrics='binary_logloss')
+        params_class_3_verbose = {'num_class': 3, 'verbose': -1}
+        # non-default num_class for default objective
+        self.assertRaises(lgb.basic.LightGBMError, get_cv_result,
+                          params_class_3_verbose)
+        # no metric with non-default num_class for custom objective
+        res = get_cv_result(params_class_3_verbose, fobj=custom_obj)
+        self.assertEqual(len(res), 0)
+        for metric_multi_alias in obj_multi_aliases + ['multi_logloss']:
+            # multiclass metric alias for custom objective
+            res = get_cv_result(params_class_3_verbose, metrics=metric_multi_alias, fobj=custom_obj)
+            self.assertEqual(len(res), 2)
+            self.assertIn('multi_logloss-mean', res)
+        # multiclass metric for custom objective
+        res = get_cv_result(params_class_3_verbose, metrics='multi_error', fobj=custom_obj)
+        self.assertEqual(len(res), 2)
+        self.assertIn('multi_error-mean', res)
+        # binary metric with non-default num_class for custom objective
+        self.assertRaises(lgb.basic.LightGBMError, get_cv_result,
+                          params_class_3_verbose, metrics='binary_error', fobj=custom_obj)

tests/python_package_test/test_sklearn.py

@@ -285,7 +285,268 @@ class TestSklearn(unittest.TestCase):
         X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
         gbm = lgb.LGBMRegressor(n_estimators=10, silent=True)
         gbm.fit(X_train, y_train, eval_set=[(X_train, y_train), (X_test, y_test)], verbose=False)
+        self.assertEqual(len(gbm.evals_result_), 2)
         self.assertIn('training', gbm.evals_result_)
+        self.assertEqual(len(gbm.evals_result_['training']), 1)
         self.assertIn('l2', gbm.evals_result_['training'])
         self.assertIn('valid_1', gbm.evals_result_)
+        self.assertEqual(len(gbm.evals_result_['valid_1']), 1)
         self.assertIn('l2', gbm.evals_result_['valid_1'])
+
+    def test_metrics(self):
+        def custom_obj(y_true, y_pred):
+            return np.zeros(y_true.shape), np.zeros(y_true.shape)
+
+        def custom_metric(y_true, y_pred):
+            return 'error', 0, False
+
+        X, y = load_boston(True)
+        params = {'n_estimators': 5, 'verbose': -1}
+        params_fit = {'X': X, 'y': y, 'eval_set': (X, y), 'verbose': False}
+
+        # no custom objective, no custom metric
+        # default metric
+        gbm = lgb.LGBMRegressor(**params).fit(**params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 1)
+        self.assertIn('l2', gbm.evals_result_['training'])
+
+        # non-default metric
+        gbm = lgb.LGBMRegressor(metric='mape', **params).fit(**params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 1)
+        self.assertIn('mape', gbm.evals_result_['training'])
+
+        # no metric
+        gbm = lgb.LGBMRegressor(metric='None', **params).fit(**params_fit)
+        self.assertIs(gbm.evals_result_, None)
+
+        # non-default metric in eval_metric
+        gbm = lgb.LGBMRegressor(**params).fit(eval_metric='mape', **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 2)
+        self.assertIn('l2', gbm.evals_result_['training'])
+        self.assertIn('mape', gbm.evals_result_['training'])
+
+        # non-default metric with non-default metric in eval_metric
+        gbm = lgb.LGBMRegressor(metric='gamma', **params).fit(eval_metric='mape', **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 2)
+        self.assertIn('gamma', gbm.evals_result_['training'])
+        self.assertIn('mape', gbm.evals_result_['training'])
+
+        # non-default metric with multiple metrics in eval_metric
+        gbm = lgb.LGBMRegressor(metric='gamma',
+                                **params).fit(eval_metric=['l2', 'mape'], **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 3)
+        self.assertIn('gamma', gbm.evals_result_['training'])
+        self.assertIn('l2', gbm.evals_result_['training'])
+        self.assertIn('mape', gbm.evals_result_['training'])
+
+        # default metric for non-default objective
+        gbm = lgb.LGBMRegressor(objective='regression_l1', **params).fit(**params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 1)
+        self.assertIn('l1', gbm.evals_result_['training'])
+
+        # non-default metric for non-default objective
+        gbm = lgb.LGBMRegressor(objective='regression_l1', metric='mape',
+                                **params).fit(**params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 1)
+        self.assertIn('mape', gbm.evals_result_['training'])
+
+        # no metric
+        gbm = lgb.LGBMRegressor(objective='regression_l1', metric='None',
+                                **params).fit(**params_fit)
+        self.assertIs(gbm.evals_result_, None)
+
+        # non-default metric in eval_metric for non-default objective
+        gbm = lgb.LGBMRegressor(objective='regression_l1',
+                                **params).fit(eval_metric='mape', **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 2)
+        self.assertIn('l1', gbm.evals_result_['training'])
+        self.assertIn('mape', gbm.evals_result_['training'])
+
+        # non-default metric with non-default metric in eval_metric for non-default objective
+        gbm = lgb.LGBMRegressor(objective='regression_l1', metric='gamma',
+                                **params).fit(eval_metric='mape', **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 2)
+        self.assertIn('gamma', gbm.evals_result_['training'])
+        self.assertIn('mape', gbm.evals_result_['training'])
+
+        # non-default metric with multiple metrics in eval_metric for non-default objective
+        gbm = lgb.LGBMRegressor(objective='regression_l1', metric='gamma',
+                                **params).fit(eval_metric=['l2', 'mape'], **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 3)
+        self.assertIn('gamma', gbm.evals_result_['training'])
+        self.assertIn('l2', gbm.evals_result_['training'])
+        self.assertIn('mape', gbm.evals_result_['training'])
+
+        # custom objective, no custom metric
+        # default regression metric for custom objective
+        gbm = lgb.LGBMRegressor(objective=custom_obj, **params).fit(**params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 1)
+        self.assertIn('l2', gbm.evals_result_['training'])
+
+        # non-default regression metric for custom objective
+        gbm = lgb.LGBMRegressor(objective=custom_obj, metric='mape', **params).fit(**params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 1)
+        self.assertIn('mape', gbm.evals_result_['training'])
+
+        # multiple regression metrics for custom objective
+        gbm = lgb.LGBMRegressor(objective=custom_obj, metric=['l1', 'gamma'],
+                                **params).fit(**params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 2)
+        self.assertIn('l1', gbm.evals_result_['training'])
+        self.assertIn('gamma', gbm.evals_result_['training'])
+
+        # no metric
+        gbm = lgb.LGBMRegressor(objective=custom_obj, metric='None',
+                                **params).fit(**params_fit)
+        self.assertIs(gbm.evals_result_, None)
+
+        # default regression metric with non-default metric in eval_metric for custom objective
+        gbm = lgb.LGBMRegressor(objective=custom_obj,
+                                **params).fit(eval_metric='mape', **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 2)
+        self.assertIn('l2', gbm.evals_result_['training'])
+        self.assertIn('mape', gbm.evals_result_['training'])
+
+        # non-default regression metric with metric in eval_metric for custom objective
+        gbm = lgb.LGBMRegressor(objective=custom_obj, metric='mape',
+                                **params).fit(eval_metric='gamma', **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 2)
+        self.assertIn('mape', gbm.evals_result_['training'])
+        self.assertIn('gamma', gbm.evals_result_['training'])
+
+        # multiple regression metrics with metric in eval_metric for custom objective
+        gbm = lgb.LGBMRegressor(objective=custom_obj, metric=['l1', 'gamma'],
+                                **params).fit(eval_metric='l2', **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 3)
+        self.assertIn('l1', gbm.evals_result_['training'])
+        self.assertIn('gamma', gbm.evals_result_['training'])
+        self.assertIn('l2', gbm.evals_result_['training'])
+
+        # multiple regression metrics with multiple metrics in eval_metric for custom objective
+        gbm = lgb.LGBMRegressor(objective=custom_obj, metric=['l1', 'gamma'],
+                                **params).fit(eval_metric=['l2', 'mape'], **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 4)
+        self.assertIn('l1', gbm.evals_result_['training'])
+        self.assertIn('gamma', gbm.evals_result_['training'])
+        self.assertIn('l2', gbm.evals_result_['training'])
+        self.assertIn('mape', gbm.evals_result_['training'])
+
+        # no custom objective, custom metric
+        # default metric with custom metric
+        gbm = lgb.LGBMRegressor(**params).fit(eval_metric=custom_metric, **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 2)
+        self.assertIn('l2', gbm.evals_result_['training'])
+        self.assertIn('error', gbm.evals_result_['training'])
+
+        # non-default metric with custom metric
+        gbm = lgb.LGBMRegressor(metric='mape',
+                                **params).fit(eval_metric=custom_metric, **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 2)
+        self.assertIn('mape', gbm.evals_result_['training'])
+        self.assertIn('error', gbm.evals_result_['training'])
+
+        # multiple metrics with custom metric
+        gbm = lgb.LGBMRegressor(metric=['l1', 'gamma'],
+                                **params).fit(eval_metric=custom_metric, **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 3)
+        self.assertIn('l1', gbm.evals_result_['training'])
+        self.assertIn('gamma', gbm.evals_result_['training'])
+        self.assertIn('error', gbm.evals_result_['training'])
+
+        # custom metric (disable default metric)
+        gbm = lgb.LGBMRegressor(metric='None',
+                                **params).fit(eval_metric=custom_metric, **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 1)
+        self.assertIn('error', gbm.evals_result_['training'])
+
+        # default metric for non-default objective with custom metric
+        gbm = lgb.LGBMRegressor(objective='regression_l1',
+                                **params).fit(eval_metric=custom_metric, **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 2)
+        self.assertIn('l1', gbm.evals_result_['training'])
+        self.assertIn('error', gbm.evals_result_['training'])
+
+        # non-default metric for non-default objective with custom metric
+        gbm = lgb.LGBMRegressor(objective='regression_l1', metric='mape',
+                                **params).fit(eval_metric=custom_metric, **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 2)
+        self.assertIn('mape', gbm.evals_result_['training'])
+        self.assertIn('error', gbm.evals_result_['training'])
+
+        # multiple metrics for non-default objective with custom metric
+        gbm = lgb.LGBMRegressor(objective='regression_l1', metric=['l1', 'gamma'],
+                                **params).fit(eval_metric=custom_metric, **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 3)
+        self.assertIn('l1', gbm.evals_result_['training'])
+        self.assertIn('gamma', gbm.evals_result_['training'])
+        self.assertIn('error', gbm.evals_result_['training'])
+
+        # custom metric (disable default metric for non-default objective)
+        gbm = lgb.LGBMRegressor(objective='regression_l1', metric='None',
+                                **params).fit(eval_metric=custom_metric, **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 1)
+        self.assertIn('error', gbm.evals_result_['training'])
+
+        # custom objective, custom metric
+        # custom metric for custom objective
+        gbm = lgb.LGBMRegressor(objective=custom_obj,
+                                **params).fit(eval_metric=custom_metric, **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 1)
+        self.assertIn('error', gbm.evals_result_['training'])
+
+        # non-default regression metric with custom metric for custom objective
+        gbm = lgb.LGBMRegressor(objective=custom_obj, metric='mape',
+                                **params).fit(eval_metric=custom_metric, **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 2)
+        self.assertIn('mape', gbm.evals_result_['training'])
+        self.assertIn('error', gbm.evals_result_['training'])
+
+        # multiple regression metrics with custom metric for custom objective
+        gbm = lgb.LGBMRegressor(objective=custom_obj, metric=['l2', 'mape'],
+                                **params).fit(eval_metric=custom_metric, **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 3)
+        self.assertIn('l2', gbm.evals_result_['training'])
+        self.assertIn('mape', gbm.evals_result_['training'])
+        self.assertIn('error', gbm.evals_result_['training'])
+
+        X, y = load_digits(3, True)
+        params_fit = {'X': X, 'y': y, 'eval_set': (X, y), 'verbose': False}
+
+        # default metric and invalid binary metric is replaced with multiclass alternative
+        gbm = lgb.LGBMClassifier(**params).fit(eval_metric='binary_error', **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 2)
+        self.assertIn('multi_logloss', gbm.evals_result_['training'])
+        self.assertIn('multi_error', gbm.evals_result_['training'])
+
+        # invalid objective is replaced with default multiclass one
+        # and invalid binary metric is replaced with multiclass alternative
+        gbm = lgb.LGBMClassifier(objective='invalid_obj',
+                                 **params).fit(eval_metric='binary_error', **params_fit)
+        self.assertEqual(gbm.objective_, 'multiclass')
+        self.assertEqual(len(gbm.evals_result_['training']), 2)
+        self.assertIn('multi_logloss', gbm.evals_result_['training'])
+        self.assertIn('multi_error', gbm.evals_result_['training'])
+
+        # default metric for non-default multiclass objective
+        # and invalid binary metric is replaced with multiclass alternative
+        gbm = lgb.LGBMClassifier(objective='ovr',
+                                 **params).fit(eval_metric='binary_error', **params_fit)
+        self.assertEqual(gbm.objective_, 'ovr')
+        self.assertEqual(len(gbm.evals_result_['training']), 2)
+        self.assertIn('multi_logloss', gbm.evals_result_['training'])
+        self.assertIn('multi_error', gbm.evals_result_['training'])
+
+        X, y = load_digits(2, True)
+        params_fit = {'X': X, 'y': y, 'eval_set': (X, y), 'verbose': False}
+
+        # default metric and invalid multiclass metric is replaced with binary alternative
+        gbm = lgb.LGBMClassifier(**params).fit(eval_metric='multi_error', **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 2)
+        self.assertIn('binary_logloss', gbm.evals_result_['training'])
+        self.assertIn('binary_error', gbm.evals_result_['training'])
+
+        # invalid multiclass metric is replaced with binary alternative for custom objective
+        gbm = lgb.LGBMClassifier(objective=custom_obj,
+                                 **params).fit(eval_metric='multi_logloss', **params_fit)
+        self.assertEqual(len(gbm.evals_result_['training']), 1)
+        self.assertIn('binary_logloss', gbm.evals_result_['training'])