test_sklearn.py 46.2 KB
Newer Older
wxchan's avatar
wxchan committed
1
# coding: utf-8
2
import itertools
3
import joblib
4
import math
5
import os
wxchan's avatar
wxchan committed
6
import unittest
7
import warnings
wxchan's avatar
wxchan committed
8

Guolin Ke's avatar
Guolin Ke committed
9
import lightgbm as lgb
wxchan's avatar
wxchan committed
10
import numpy as np
11
from sklearn import __version__ as sk_version
wxchan's avatar
wxchan committed
12
from sklearn.base import clone
wxchan's avatar
wxchan committed
13
from sklearn.datasets import (load_boston, load_breast_cancer, load_digits,
14
                              load_iris, load_svmlight_file)
15
from sklearn.exceptions import SkipTestWarning
wxchan's avatar
wxchan committed
16
17
from sklearn.metrics import log_loss, mean_squared_error
from sklearn.model_selection import GridSearchCV, train_test_split
18
from sklearn.utils.estimator_checks import (_yield_all_checks, SkipTest,
19
                                            check_parameters_default_constructible)
wxchan's avatar
wxchan committed
20

wxchan's avatar
wxchan committed
21

22
decreasing_generator = itertools.count(0, -1)
wxchan's avatar
wxchan committed
23

wxchan's avatar
wxchan committed
24

25
26
27
28
29
30
31
def custom_asymmetric_obj(y_true, y_pred):
    residual = (y_true - y_pred).astype("float")
    grad = np.where(residual < 0, -2 * 10.0 * residual, -2 * residual)
    hess = np.where(residual < 0, 2 * 10.0, 2.0)
    return grad, hess


32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
def objective_ls(y_true, y_pred):
    grad = (y_pred - y_true)
    hess = np.ones(len(y_true))
    return grad, hess


def logregobj(y_true, y_pred):
    y_pred = 1.0 / (1.0 + np.exp(-y_pred))
    grad = y_pred - y_true
    hess = y_pred * (1.0 - y_pred)
    return grad, hess


def custom_dummy_obj(y_true, y_pred):
    return np.ones(y_true.shape), np.ones(y_true.shape)


def constant_metric(y_true, y_pred):
    return 'error', 0, False


def decreasing_metric(y_true, y_pred):
    return ('decreasing_metric', next(decreasing_generator), False)


57
58
59
60
def mse(y_true, y_pred):
    return 'custom MSE', mean_squared_error(y_true, y_pred), False


61
62
63
64
65
66
67
68
69
70
71
72
def binary_error(y_true, y_pred):
    return np.mean((y_pred > 0.5) != y_true)


def multi_error(y_true, y_pred):
    return np.mean(y_true != y_pred)


def multi_logloss(y_true, y_pred):
    return np.mean([-math.log(y_pred[i][y]) for i, y in enumerate(y_true)])


wxchan's avatar
wxchan committed
73
74
75
class TestSklearn(unittest.TestCase):

    def test_binary(self):
76
77
78
        X, y = load_breast_cancer(True)
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
        gbm = lgb.LGBMClassifier(n_estimators=50, silent=True)
79
        gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=5, verbose=False)
80
        ret = log_loss(y_test, gbm.predict_proba(X_test))
81
        self.assertLess(ret, 0.12)
82
        self.assertAlmostEqual(ret, gbm.evals_result_['valid_0']['binary_logloss'][gbm.best_iteration_ - 1], places=5)
wxchan's avatar
wxchan committed
83

84
    def test_regression(self):
85
86
87
        X, y = load_boston(True)
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
        gbm = lgb.LGBMRegressor(n_estimators=50, silent=True)
88
        gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=5, verbose=False)
89
        ret = mean_squared_error(y_test, gbm.predict(X_test))
90
        self.assertLess(ret, 7)
91
        self.assertAlmostEqual(ret, gbm.evals_result_['valid_0']['l2'][gbm.best_iteration_ - 1], places=5)
wxchan's avatar
wxchan committed
92

wxchan's avatar
wxchan committed
93
    def test_multiclass(self):
94
95
96
        X, y = load_digits(10, True)
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
        gbm = lgb.LGBMClassifier(n_estimators=50, silent=True)
97
        gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=5, verbose=False)
98
        ret = multi_error(y_test, gbm.predict(X_test))
99
        self.assertLess(ret, 0.05)
100
        ret = multi_logloss(y_test, gbm.predict_proba(X_test))
101
        self.assertLess(ret, 0.16)
102
        self.assertAlmostEqual(ret, gbm.evals_result_['valid_0']['multi_logloss'][gbm.best_iteration_ - 1], places=5)
wxchan's avatar
wxchan committed
103

wxchan's avatar
wxchan committed
104
    def test_lambdarank(self):
105
106
107
108
109
110
111
112
        X_train, y_train = load_svmlight_file(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                                           '../../examples/lambdarank/rank.train'))
        X_test, y_test = load_svmlight_file(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                                         '../../examples/lambdarank/rank.test'))
        q_train = np.loadtxt(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                          '../../examples/lambdarank/rank.train.query'))
        q_test = np.loadtxt(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                         '../../examples/lambdarank/rank.test.query'))
113
        gbm = lgb.LGBMRanker(n_estimators=50)
114
        gbm.fit(X_train, y_train, group=q_train, eval_set=[(X_test, y_test)],
115
116
                eval_group=[q_test], eval_at=[1, 3], early_stopping_rounds=10, verbose=False,
                callbacks=[lgb.reset_parameter(learning_rate=lambda x: max(0.01, 0.1 - 0.01 * x))])
117
        self.assertLessEqual(gbm.best_iteration_, 24)
118
119
        self.assertGreater(gbm.best_score_['valid_0']['ndcg@1'], 0.5769)
        self.assertGreater(gbm.best_score_['valid_0']['ndcg@3'], 0.5920)
wxchan's avatar
wxchan committed
120

121
122
123
124
125
126
127
128
129
130
131
132
    def test_xendcg(self):
        dir_path = os.path.dirname(os.path.realpath(__file__))
        X_train, y_train = load_svmlight_file(os.path.join(dir_path, '../../examples/xendcg/rank.train'))
        X_test, y_test = load_svmlight_file(os.path.join(dir_path, '../../examples/xendcg/rank.test'))
        q_train = np.loadtxt(os.path.join(dir_path, '../../examples/xendcg/rank.train.query'))
        q_test = np.loadtxt(os.path.join(dir_path, '../../examples/xendcg/rank.test.query'))
        gbm = lgb.LGBMRanker(n_estimators=50, objective='rank_xendcg', random_state=5, n_jobs=1)
        gbm.fit(X_train, y_train, group=q_train, eval_set=[(X_test, y_test)],
                eval_group=[q_test], eval_at=[1, 3], early_stopping_rounds=10, verbose=False,
                eval_metric='ndcg',
                callbacks=[lgb.reset_parameter(learning_rate=lambda x: max(0.01, 0.1 - 0.01 * x))])
        self.assertLessEqual(gbm.best_iteration_, 24)
133
134
        self.assertGreater(gbm.best_score_['valid_0']['ndcg@1'], 0.6382)
        self.assertGreater(gbm.best_score_['valid_0']['ndcg@3'], 0.6319)
135

wxchan's avatar
wxchan committed
136
    def test_regression_with_custom_objective(self):
137
138
139
        X, y = load_boston(True)
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
        gbm = lgb.LGBMRegressor(n_estimators=50, silent=True, objective=objective_ls)
140
        gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=5, verbose=False)
141
        ret = mean_squared_error(y_test, gbm.predict(X_test))
142
        self.assertLess(ret, 7.0)
143
        self.assertAlmostEqual(ret, gbm.evals_result_['valid_0']['l2'][gbm.best_iteration_ - 1], places=5)
wxchan's avatar
wxchan committed
144
145

    def test_binary_classification_with_custom_objective(self):
146
        X, y = load_digits(2, True)
147
148
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
        gbm = lgb.LGBMClassifier(n_estimators=50, silent=True, objective=logregobj)
149
        gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=5, verbose=False)
150
151
152
153
154
        # prediction result is actually not transformed (is raw) due to custom objective
        y_pred_raw = gbm.predict_proba(X_test)
        self.assertFalse(np.all(y_pred_raw >= 0))
        y_pred = 1.0 / (1.0 + np.exp(-y_pred_raw))
        ret = binary_error(y_test, y_pred)
155
        self.assertLess(ret, 0.05)
wxchan's avatar
wxchan committed
156

157
    def test_dart(self):
158
159
        X, y = load_boston(True)
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
160
        gbm = lgb.LGBMRegressor(boosting_type='dart', n_estimators=50)
161
        gbm.fit(X_train, y_train)
162
163
164
        score = gbm.score(X_test, y_test)
        self.assertGreaterEqual(score, 0.8)
        self.assertLessEqual(score, 1.)
165

wxchan's avatar
wxchan committed
166
    def test_grid_search(self):
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
        X, y = load_iris(True)
        y = np.array(list(map(str, y)))  # utilize label encoder at it's max power
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
        params = {'subsample': 0.8,
                  'subsample_freq': 1}
        grid_params = {'boosting_type': ['rf', 'gbdt'],
                       'n_estimators': [4, 6],
                       'reg_alpha': [0.01, 0.005]}
        fit_params = {'verbose': False,
                      'eval_set': [(X_test, y_test)],
                      'eval_metric': constant_metric,
                      'early_stopping_rounds': 2}
        grid = GridSearchCV(lgb.LGBMClassifier(**params), grid_params, cv=2)
        grid.fit(X, y, **fit_params)
        self.assertIn(grid.best_params_['boosting_type'], ['rf', 'gbdt'])
        self.assertIn(grid.best_params_['n_estimators'], [4, 6])
        self.assertIn(grid.best_params_['reg_alpha'], [0.01, 0.005])
        self.assertLess(grid.best_score_, 0.9)
        self.assertEqual(grid.best_estimator_.best_iteration_, 1)
        self.assertLess(grid.best_estimator_.best_score_['valid_0']['multi_logloss'], 0.25)
        self.assertEqual(grid.best_estimator_.best_score_['valid_0']['error'], 0)
wxchan's avatar
wxchan committed
188

189
    def test_clone_and_property(self):
190
        X, y = load_boston(True)
191
192
        gbm = lgb.LGBMRegressor(n_estimators=10, silent=True)
        gbm.fit(X, y, verbose=False)
193

wxchan's avatar
wxchan committed
194
        gbm_clone = clone(gbm)
195
        self.assertIsInstance(gbm.booster_, lgb.Booster)
196
        self.assertIsInstance(gbm.feature_importances_, np.ndarray)
197
198

        X, y = load_digits(2, True)
199
200
        clf = lgb.LGBMClassifier(n_estimators=10, silent=True)
        clf.fit(X, y, verbose=False)
201
202
203
        self.assertListEqual(sorted(clf.classes_), [0, 1])
        self.assertEqual(clf.n_classes_, 2)
        self.assertIsInstance(clf.booster_, lgb.Booster)
204
        self.assertIsInstance(clf.feature_importances_, np.ndarray)
wxchan's avatar
wxchan committed
205

wxchan's avatar
wxchan committed
206
    def test_joblib(self):
207
208
        X, y = load_boston(True)
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
209
210
211
212
213
        gbm = lgb.LGBMRegressor(n_estimators=10, objective=custom_asymmetric_obj,
                                silent=True, importance_type='split')
        gbm.fit(X_train, y_train, eval_set=[(X_train, y_train), (X_test, y_test)],
                eval_metric=mse, early_stopping_rounds=5, verbose=False,
                callbacks=[lgb.reset_parameter(learning_rate=list(np.arange(1, 0, -0.1)))])
214

215
        joblib.dump(gbm, 'lgb.pkl')  # test model with custom functions
wxchan's avatar
wxchan committed
216
        gbm_pickle = joblib.load('lgb.pkl')
217
        self.assertIsInstance(gbm_pickle.booster_, lgb.Booster)
wxchan's avatar
wxchan committed
218
        self.assertDictEqual(gbm.get_params(), gbm_pickle.get_params())
219
220
221
222
223
224
        np.testing.assert_array_equal(gbm.feature_importances_, gbm_pickle.feature_importances_)
        self.assertAlmostEqual(gbm_pickle.learning_rate, 0.1)
        self.assertTrue(callable(gbm_pickle.objective))

        for eval_set in gbm.evals_result_:
            for metric in gbm.evals_result_[eval_set]:
225
226
                np.testing.assert_allclose(gbm.evals_result_[eval_set][metric],
                                           gbm_pickle.evals_result_[eval_set][metric])
wxchan's avatar
wxchan committed
227
228
        pred_origin = gbm.predict(X_test)
        pred_pickle = gbm_pickle.predict(X_test)
229
        np.testing.assert_allclose(pred_origin, pred_pickle)
230

231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
    def test_random_state_object(self):
        X, y = load_iris(True)
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
        state1 = np.random.RandomState(123)
        state2 = np.random.RandomState(123)
        clf1 = lgb.LGBMClassifier(n_estimators=10, subsample=0.5, subsample_freq=1, random_state=state1)
        clf2 = lgb.LGBMClassifier(n_estimators=10, subsample=0.5, subsample_freq=1, random_state=state2)
        # Test if random_state is properly stored
        self.assertIs(clf1.random_state, state1)
        self.assertIs(clf2.random_state, state2)
        # Test if two random states produce identical models
        clf1.fit(X_train, y_train)
        clf2.fit(X_train, y_train)
        y_pred1 = clf1.predict(X_test, raw_score=True)
        y_pred2 = clf2.predict(X_test, raw_score=True)
        np.testing.assert_allclose(y_pred1, y_pred2)
        np.testing.assert_array_equal(clf1.feature_importances_, clf2.feature_importances_)
        df1 = clf1.booster_.model_to_string(num_iteration=0)
        df2 = clf2.booster_.model_to_string(num_iteration=0)
        self.assertMultiLineEqual(df1, df2)
        # Test if subsequent fits sample from random_state object and produce different models
        clf1.fit(X_train, y_train)
        y_pred1_refit = clf1.predict(X_test, raw_score=True)
        df3 = clf1.booster_.model_to_string(num_iteration=0)
        self.assertIs(clf1.random_state, state1)
        self.assertIs(clf2.random_state, state2)
        self.assertRaises(AssertionError,
                          np.testing.assert_allclose,
                          y_pred1, y_pred1_refit)
        self.assertRaises(AssertionError,
                          self.assertMultiLineEqual,
                          df1, df3)

264
265
    def test_feature_importances_single_leaf(self):
        data = load_iris()
266
        clf = lgb.LGBMClassifier(n_estimators=10)
267
268
269
        clf.fit(data.data, data.target)
        importances = clf.feature_importances_
        self.assertEqual(len(importances), 4)
270

271
272
    def test_feature_importances_type(self):
        data = load_iris()
273
        clf = lgb.LGBMClassifier(n_estimators=10)
274
275
276
277
278
279
280
281
282
283
        clf.fit(data.data, data.target)
        clf.set_params(importance_type='split')
        importances_split = clf.feature_importances_
        clf.set_params(importance_type='gain')
        importances_gain = clf.feature_importances_
        # Test that the largest element is NOT the same, the smallest can be the same, i.e. zero
        importance_split_top1 = sorted(importances_split, reverse=True)[0]
        importance_gain_top1 = sorted(importances_gain, reverse=True)[0]
        self.assertNotEqual(importance_split_top1, importance_gain_top1)

284
    # sklearn <0.19 cannot accept instance, but many tests could be passed only with min_data=1 and min_data_in_bin=1
285
    @unittest.skipIf(sk_version < '0.19.0', 'scikit-learn version is less than 0.19')
286
    def test_sklearn_integration(self):
287
288
289
290
        # we cannot use `check_estimator` directly since there is no skip test mechanism
        for name, estimator in ((lgb.sklearn.LGBMClassifier.__name__, lgb.sklearn.LGBMClassifier),
                                (lgb.sklearn.LGBMRegressor.__name__, lgb.sklearn.LGBMRegressor)):
            check_parameters_default_constructible(name, estimator)
291
            # we cannot leave default params (see https://github.com/microsoft/LightGBM/issues/833)
292
293
            estimator = estimator(min_child_samples=1, min_data_in_bin=1)
            for check in _yield_all_checks(name, estimator):
294
295
                check_name = check.func.__name__ if hasattr(check, 'func') else check.__name__
                if check_name == 'check_estimators_nan_inf':
296
                    continue  # skip test because LightGBM deals with nan
297
298
299
300
301
                elif check_name == "check_no_attributes_set_in_init":
                    # skip test because scikit-learn incorrectly asserts that
                    # private attributes cannot be set in __init__
                    # (see https://github.com/microsoft/LightGBM/issues/2628)
                    continue
302
303
304
305
306
307
                try:
                    check(name, estimator)
                except SkipTest as message:
                    warnings.warn(message, SkipTestWarning)

    @unittest.skipIf(not lgb.compat.PANDAS_INSTALLED, 'pandas is not installed')
308
    def test_pandas_categorical(self):
309
        import pandas as pd
310
        np.random.seed(42)  # sometimes there is no difference how cols are treated (cat or not cat)
311
312
313
        X = pd.DataFrame({"A": np.random.permutation(['a', 'b', 'c', 'd'] * 75),  # str
                          "B": np.random.permutation([1, 2, 3] * 100),  # int
                          "C": np.random.permutation([0.1, 0.2, -0.1, -0.1, 0.2] * 60),  # float
314
315
316
                          "D": np.random.permutation([True, False] * 150),  # bool
                          "E": pd.Categorical(np.random.permutation(['z', 'y', 'x', 'w', 'v'] * 60),
                                              ordered=True)})  # str and ordered categorical
317
        y = np.random.permutation([0, 1] * 150)
318
        X_test = pd.DataFrame({"A": np.random.permutation(['a', 'b', 'e'] * 20),  # unseen category
319
320
                               "B": np.random.permutation([1, 3] * 30),
                               "C": np.random.permutation([0.1, -0.1, 0.2, 0.2] * 15),
321
                               "D": np.random.permutation([True, False] * 30),
322
                               "E": pd.Categorical(np.random.permutation(['z', 'y'] * 30),
323
324
325
326
327
328
329
                                                   ordered=True)})
        np.random.seed()  # reset seed
        cat_cols_actual = ["A", "B", "C", "D"]
        cat_cols_to_store = cat_cols_actual + ["E"]
        X[cat_cols_actual] = X[cat_cols_actual].astype('category')
        X_test[cat_cols_actual] = X_test[cat_cols_actual].astype('category')
        cat_values = [X[col].cat.categories.tolist() for col in cat_cols_to_store]
330
        gbm0 = lgb.sklearn.LGBMClassifier(n_estimators=10).fit(X, y)
331
        pred0 = gbm0.predict(X_test, raw_score=True)
332
        pred_prob = gbm0.predict_proba(X_test)[:, 1]
333
        gbm1 = lgb.sklearn.LGBMClassifier(n_estimators=10).fit(X, pd.Series(y), categorical_feature=[0])
334
        pred1 = gbm1.predict(X_test, raw_score=True)
335
        gbm2 = lgb.sklearn.LGBMClassifier(n_estimators=10).fit(X, y, categorical_feature=['A'])
336
        pred2 = gbm2.predict(X_test, raw_score=True)
337
        gbm3 = lgb.sklearn.LGBMClassifier(n_estimators=10).fit(X, y, categorical_feature=['A', 'B', 'C', 'D'])
338
        pred3 = gbm3.predict(X_test, raw_score=True)
339
340
        gbm3.booster_.save_model('categorical.model')
        gbm4 = lgb.Booster(model_file='categorical.model')
341
        pred4 = gbm4.predict(X_test)
342
        gbm5 = lgb.sklearn.LGBMClassifier(n_estimators=10).fit(X, y, categorical_feature=['A', 'B', 'C', 'D', 'E'])
343
        pred5 = gbm5.predict(X_test, raw_score=True)
344
        gbm6 = lgb.sklearn.LGBMClassifier(n_estimators=10).fit(X, y, categorical_feature=[])
345
346
        pred6 = gbm6.predict(X_test, raw_score=True)
        self.assertRaises(AssertionError,
347
                          np.testing.assert_allclose,
348
349
                          pred0, pred1)
        self.assertRaises(AssertionError,
350
                          np.testing.assert_allclose,
351
                          pred0, pred2)
352
353
354
        np.testing.assert_allclose(pred1, pred2)
        np.testing.assert_allclose(pred0, pred3)
        np.testing.assert_allclose(pred_prob, pred4)
355
        self.assertRaises(AssertionError,
356
                          np.testing.assert_allclose,
357
                          pred0, pred5)  # ordered cat features aren't treated as cat features by default
358
        self.assertRaises(AssertionError,
359
                          np.testing.assert_allclose,
360
                          pred0, pred6)
361
362
363
364
365
366
        self.assertListEqual(gbm0.booster_.pandas_categorical, cat_values)
        self.assertListEqual(gbm1.booster_.pandas_categorical, cat_values)
        self.assertListEqual(gbm2.booster_.pandas_categorical, cat_values)
        self.assertListEqual(gbm3.booster_.pandas_categorical, cat_values)
        self.assertListEqual(gbm4.pandas_categorical, cat_values)
        self.assertListEqual(gbm5.booster_.pandas_categorical, cat_values)
367
        self.assertListEqual(gbm6.booster_.pandas_categorical, cat_values)
368

369
370
371
    @unittest.skipIf(not lgb.compat.PANDAS_INSTALLED, 'pandas is not installed')
    def test_pandas_sparse(self):
        import pandas as pd
372
373
374
375
376
377
378
379
380
381
382
        try:
            from pandas.arrays import SparseArray
        except ImportError:  # support old versions
            from pandas import SparseArray
        X = pd.DataFrame({"A": SparseArray(np.random.permutation([0, 1, 2] * 100)),
                          "B": SparseArray(np.random.permutation([0.0, 0.1, 0.2, -0.1, 0.2] * 60)),
                          "C": SparseArray(np.random.permutation([True, False] * 150))})
        y = pd.Series(SparseArray(np.random.permutation([0, 1] * 150)))
        X_test = pd.DataFrame({"A": SparseArray(np.random.permutation([0, 2] * 30)),
                               "B": SparseArray(np.random.permutation([0.0, 0.1, 0.2, -0.1] * 15)),
                               "C": SparseArray(np.random.permutation([True, False] * 30))})
383
384
385
        if pd.__version__ >= '0.24.0':
            for dtype in pd.concat([X.dtypes, X_test.dtypes, pd.Series(y.dtypes)]):
                self.assertTrue(pd.api.types.is_sparse(dtype))
386
        gbm = lgb.sklearn.LGBMClassifier(n_estimators=10).fit(X, y)
387
388
389
390
391
392
393
        pred_sparse = gbm.predict(X_test, raw_score=True)
        if hasattr(X_test, 'sparse'):
            pred_dense = gbm.predict(X_test.sparse.to_dense(), raw_score=True)
        else:
            pred_dense = gbm.predict(X_test.to_dense(), raw_score=True)
        np.testing.assert_allclose(pred_sparse, pred_dense)

394
    def test_predict(self):
395
        # With default params
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
        iris = load_iris()
        X_train, X_test, y_train, y_test = train_test_split(iris.data, iris.target,
                                                            test_size=0.2, random_state=42)

        gbm = lgb.train({'objective': 'multiclass',
                         'num_class': 3,
                         'verbose': -1},
                        lgb.Dataset(X_train, y_train))
        clf = lgb.LGBMClassifier(verbose=-1).fit(X_train, y_train)

        # Tests same probabilities
        res_engine = gbm.predict(X_test)
        res_sklearn = clf.predict_proba(X_test)
        np.testing.assert_allclose(res_engine, res_sklearn)

        # Tests same predictions
        res_engine = np.argmax(gbm.predict(X_test), axis=1)
        res_sklearn = clf.predict(X_test)
        np.testing.assert_equal(res_engine, res_sklearn)

        # Tests same raw scores
        res_engine = gbm.predict(X_test, raw_score=True)
        res_sklearn = clf.predict(X_test, raw_score=True)
        np.testing.assert_allclose(res_engine, res_sklearn)

        # Tests same leaf indices
        res_engine = gbm.predict(X_test, pred_leaf=True)
        res_sklearn = clf.predict(X_test, pred_leaf=True)
        np.testing.assert_equal(res_engine, res_sklearn)

        # Tests same feature contributions
        res_engine = gbm.predict(X_test, pred_contrib=True)
        res_sklearn = clf.predict(X_test, pred_contrib=True)
        np.testing.assert_allclose(res_engine, res_sklearn)

        # Tests other parameters for the prediction works
        res_engine = gbm.predict(X_test)
        res_sklearn_params = clf.predict_proba(X_test,
                                               pred_early_stop=True,
                                               pred_early_stop_margin=1.0)
        self.assertRaises(AssertionError,
                          np.testing.assert_allclose,
                          res_engine, res_sklearn_params)
439
440
441
442
443
444

    def test_evaluate_train_set(self):
        X, y = load_boston(True)
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
        gbm = lgb.LGBMRegressor(n_estimators=10, silent=True)
        gbm.fit(X_train, y_train, eval_set=[(X_train, y_train), (X_test, y_test)], verbose=False)
445
        self.assertEqual(len(gbm.evals_result_), 2)
446
        self.assertIn('training', gbm.evals_result_)
447
        self.assertEqual(len(gbm.evals_result_['training']), 1)
448
449
        self.assertIn('l2', gbm.evals_result_['training'])
        self.assertIn('valid_1', gbm.evals_result_)
450
        self.assertEqual(len(gbm.evals_result_['valid_1']), 1)
451
        self.assertIn('l2', gbm.evals_result_['valid_1'])
452
453
454

    def test_metrics(self):
        X, y = load_boston(True)
455
        params = {'n_estimators': 2, 'verbose': -1}
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
        params_fit = {'X': X, 'y': y, 'eval_set': (X, y), 'verbose': False}

        # no custom objective, no custom metric
        # default metric
        gbm = lgb.LGBMRegressor(**params).fit(**params_fit)
        self.assertEqual(len(gbm.evals_result_['training']), 1)
        self.assertIn('l2', gbm.evals_result_['training'])

        # non-default metric
        gbm = lgb.LGBMRegressor(metric='mape', **params).fit(**params_fit)
        self.assertEqual(len(gbm.evals_result_['training']), 1)
        self.assertIn('mape', gbm.evals_result_['training'])

        # no metric
        gbm = lgb.LGBMRegressor(metric='None', **params).fit(**params_fit)
        self.assertIs(gbm.evals_result_, None)

        # non-default metric in eval_metric
        gbm = lgb.LGBMRegressor(**params).fit(eval_metric='mape', **params_fit)
        self.assertEqual(len(gbm.evals_result_['training']), 2)
        self.assertIn('l2', gbm.evals_result_['training'])
        self.assertIn('mape', gbm.evals_result_['training'])

        # non-default metric with non-default metric in eval_metric
        gbm = lgb.LGBMRegressor(metric='gamma', **params).fit(eval_metric='mape', **params_fit)
        self.assertEqual(len(gbm.evals_result_['training']), 2)
        self.assertIn('gamma', gbm.evals_result_['training'])
        self.assertIn('mape', gbm.evals_result_['training'])

        # non-default metric with multiple metrics in eval_metric
        gbm = lgb.LGBMRegressor(metric='gamma',
                                **params).fit(eval_metric=['l2', 'mape'], **params_fit)
        self.assertEqual(len(gbm.evals_result_['training']), 3)
        self.assertIn('gamma', gbm.evals_result_['training'])
        self.assertIn('l2', gbm.evals_result_['training'])
        self.assertIn('mape', gbm.evals_result_['training'])

        # default metric for non-default objective
        gbm = lgb.LGBMRegressor(objective='regression_l1', **params).fit(**params_fit)
        self.assertEqual(len(gbm.evals_result_['training']), 1)
        self.assertIn('l1', gbm.evals_result_['training'])

        # non-default metric for non-default objective
        gbm = lgb.LGBMRegressor(objective='regression_l1', metric='mape',
                                **params).fit(**params_fit)
        self.assertEqual(len(gbm.evals_result_['training']), 1)
        self.assertIn('mape', gbm.evals_result_['training'])

        # no metric
        gbm = lgb.LGBMRegressor(objective='regression_l1', metric='None',
                                **params).fit(**params_fit)
        self.assertIs(gbm.evals_result_, None)

        # non-default metric in eval_metric for non-default objective
        gbm = lgb.LGBMRegressor(objective='regression_l1',
                                **params).fit(eval_metric='mape', **params_fit)
        self.assertEqual(len(gbm.evals_result_['training']), 2)
        self.assertIn('l1', gbm.evals_result_['training'])
        self.assertIn('mape', gbm.evals_result_['training'])

        # non-default metric with non-default metric in eval_metric for non-default objective
        gbm = lgb.LGBMRegressor(objective='regression_l1', metric='gamma',
                                **params).fit(eval_metric='mape', **params_fit)
        self.assertEqual(len(gbm.evals_result_['training']), 2)
        self.assertIn('gamma', gbm.evals_result_['training'])
        self.assertIn('mape', gbm.evals_result_['training'])

        # non-default metric with multiple metrics in eval_metric for non-default objective
        gbm = lgb.LGBMRegressor(objective='regression_l1', metric='gamma',
                                **params).fit(eval_metric=['l2', 'mape'], **params_fit)
        self.assertEqual(len(gbm.evals_result_['training']), 3)
        self.assertIn('gamma', gbm.evals_result_['training'])
        self.assertIn('l2', gbm.evals_result_['training'])
        self.assertIn('mape', gbm.evals_result_['training'])

        # custom objective, no custom metric
        # default regression metric for custom objective
533
        gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, **params).fit(**params_fit)
534
535
536
537
        self.assertEqual(len(gbm.evals_result_['training']), 1)
        self.assertIn('l2', gbm.evals_result_['training'])

        # non-default regression metric for custom objective
538
        gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric='mape', **params).fit(**params_fit)
539
540
541
542
        self.assertEqual(len(gbm.evals_result_['training']), 1)
        self.assertIn('mape', gbm.evals_result_['training'])

        # multiple regression metrics for custom objective
543
        gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric=['l1', 'gamma'],
544
545
546
547
548
549
                                **params).fit(**params_fit)
        self.assertEqual(len(gbm.evals_result_['training']), 2)
        self.assertIn('l1', gbm.evals_result_['training'])
        self.assertIn('gamma', gbm.evals_result_['training'])

        # no metric
550
        gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric='None',
551
552
553
554
                                **params).fit(**params_fit)
        self.assertIs(gbm.evals_result_, None)

        # default regression metric with non-default metric in eval_metric for custom objective
555
        gbm = lgb.LGBMRegressor(objective=custom_dummy_obj,
556
557
558
559
560
561
                                **params).fit(eval_metric='mape', **params_fit)
        self.assertEqual(len(gbm.evals_result_['training']), 2)
        self.assertIn('l2', gbm.evals_result_['training'])
        self.assertIn('mape', gbm.evals_result_['training'])

        # non-default regression metric with metric in eval_metric for custom objective
562
        gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric='mape',
563
564
565
566
567
568
                                **params).fit(eval_metric='gamma', **params_fit)
        self.assertEqual(len(gbm.evals_result_['training']), 2)
        self.assertIn('mape', gbm.evals_result_['training'])
        self.assertIn('gamma', gbm.evals_result_['training'])

        # multiple regression metrics with metric in eval_metric for custom objective
569
        gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric=['l1', 'gamma'],
570
571
572
573
574
575
576
                                **params).fit(eval_metric='l2', **params_fit)
        self.assertEqual(len(gbm.evals_result_['training']), 3)
        self.assertIn('l1', gbm.evals_result_['training'])
        self.assertIn('gamma', gbm.evals_result_['training'])
        self.assertIn('l2', gbm.evals_result_['training'])

        # multiple regression metrics with multiple metrics in eval_metric for custom objective
577
        gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric=['l1', 'gamma'],
578
579
580
581
582
583
584
585
586
                                **params).fit(eval_metric=['l2', 'mape'], **params_fit)
        self.assertEqual(len(gbm.evals_result_['training']), 4)
        self.assertIn('l1', gbm.evals_result_['training'])
        self.assertIn('gamma', gbm.evals_result_['training'])
        self.assertIn('l2', gbm.evals_result_['training'])
        self.assertIn('mape', gbm.evals_result_['training'])

        # no custom objective, custom metric
        # default metric with custom metric
587
        gbm = lgb.LGBMRegressor(**params).fit(eval_metric=constant_metric, **params_fit)
588
589
590
591
592
593
        self.assertEqual(len(gbm.evals_result_['training']), 2)
        self.assertIn('l2', gbm.evals_result_['training'])
        self.assertIn('error', gbm.evals_result_['training'])

        # non-default metric with custom metric
        gbm = lgb.LGBMRegressor(metric='mape',
594
                                **params).fit(eval_metric=constant_metric, **params_fit)
595
596
597
598
599
600
        self.assertEqual(len(gbm.evals_result_['training']), 2)
        self.assertIn('mape', gbm.evals_result_['training'])
        self.assertIn('error', gbm.evals_result_['training'])

        # multiple metrics with custom metric
        gbm = lgb.LGBMRegressor(metric=['l1', 'gamma'],
601
                                **params).fit(eval_metric=constant_metric, **params_fit)
602
603
604
605
606
607
608
        self.assertEqual(len(gbm.evals_result_['training']), 3)
        self.assertIn('l1', gbm.evals_result_['training'])
        self.assertIn('gamma', gbm.evals_result_['training'])
        self.assertIn('error', gbm.evals_result_['training'])

        # custom metric (disable default metric)
        gbm = lgb.LGBMRegressor(metric='None',
609
                                **params).fit(eval_metric=constant_metric, **params_fit)
610
611
612
613
614
        self.assertEqual(len(gbm.evals_result_['training']), 1)
        self.assertIn('error', gbm.evals_result_['training'])

        # default metric for non-default objective with custom metric
        gbm = lgb.LGBMRegressor(objective='regression_l1',
615
                                **params).fit(eval_metric=constant_metric, **params_fit)
616
617
618
619
620
621
        self.assertEqual(len(gbm.evals_result_['training']), 2)
        self.assertIn('l1', gbm.evals_result_['training'])
        self.assertIn('error', gbm.evals_result_['training'])

        # non-default metric for non-default objective with custom metric
        gbm = lgb.LGBMRegressor(objective='regression_l1', metric='mape',
622
                                **params).fit(eval_metric=constant_metric, **params_fit)
623
624
625
626
627
628
        self.assertEqual(len(gbm.evals_result_['training']), 2)
        self.assertIn('mape', gbm.evals_result_['training'])
        self.assertIn('error', gbm.evals_result_['training'])

        # multiple metrics for non-default objective with custom metric
        gbm = lgb.LGBMRegressor(objective='regression_l1', metric=['l1', 'gamma'],
629
                                **params).fit(eval_metric=constant_metric, **params_fit)
630
631
632
633
634
635
636
        self.assertEqual(len(gbm.evals_result_['training']), 3)
        self.assertIn('l1', gbm.evals_result_['training'])
        self.assertIn('gamma', gbm.evals_result_['training'])
        self.assertIn('error', gbm.evals_result_['training'])

        # custom metric (disable default metric for non-default objective)
        gbm = lgb.LGBMRegressor(objective='regression_l1', metric='None',
637
                                **params).fit(eval_metric=constant_metric, **params_fit)
638
639
640
641
642
        self.assertEqual(len(gbm.evals_result_['training']), 1)
        self.assertIn('error', gbm.evals_result_['training'])

        # custom objective, custom metric
        # custom metric for custom objective
643
644
        gbm = lgb.LGBMRegressor(objective=custom_dummy_obj,
                                **params).fit(eval_metric=constant_metric, **params_fit)
645
646
647
648
        self.assertEqual(len(gbm.evals_result_['training']), 1)
        self.assertIn('error', gbm.evals_result_['training'])

        # non-default regression metric with custom metric for custom objective
649
650
        gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric='mape',
                                **params).fit(eval_metric=constant_metric, **params_fit)
651
652
653
654
655
        self.assertEqual(len(gbm.evals_result_['training']), 2)
        self.assertIn('mape', gbm.evals_result_['training'])
        self.assertIn('error', gbm.evals_result_['training'])

        # multiple regression metrics with custom metric for custom objective
656
657
        gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric=['l2', 'mape'],
                                **params).fit(eval_metric=constant_metric, **params_fit)
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
        self.assertEqual(len(gbm.evals_result_['training']), 3)
        self.assertIn('l2', gbm.evals_result_['training'])
        self.assertIn('mape', gbm.evals_result_['training'])
        self.assertIn('error', gbm.evals_result_['training'])

        X, y = load_digits(3, True)
        params_fit = {'X': X, 'y': y, 'eval_set': (X, y), 'verbose': False}

        # default metric and invalid binary metric is replaced with multiclass alternative
        gbm = lgb.LGBMClassifier(**params).fit(eval_metric='binary_error', **params_fit)
        self.assertEqual(len(gbm.evals_result_['training']), 2)
        self.assertIn('multi_logloss', gbm.evals_result_['training'])
        self.assertIn('multi_error', gbm.evals_result_['training'])

        # invalid objective is replaced with default multiclass one
        # and invalid binary metric is replaced with multiclass alternative
        gbm = lgb.LGBMClassifier(objective='invalid_obj',
                                 **params).fit(eval_metric='binary_error', **params_fit)
        self.assertEqual(gbm.objective_, 'multiclass')
        self.assertEqual(len(gbm.evals_result_['training']), 2)
        self.assertIn('multi_logloss', gbm.evals_result_['training'])
        self.assertIn('multi_error', gbm.evals_result_['training'])

        # default metric for non-default multiclass objective
        # and invalid binary metric is replaced with multiclass alternative
        gbm = lgb.LGBMClassifier(objective='ovr',
                                 **params).fit(eval_metric='binary_error', **params_fit)
        self.assertEqual(gbm.objective_, 'ovr')
        self.assertEqual(len(gbm.evals_result_['training']), 2)
        self.assertIn('multi_logloss', gbm.evals_result_['training'])
        self.assertIn('multi_error', gbm.evals_result_['training'])

        X, y = load_digits(2, True)
        params_fit = {'X': X, 'y': y, 'eval_set': (X, y), 'verbose': False}

        # default metric and invalid multiclass metric is replaced with binary alternative
        gbm = lgb.LGBMClassifier(**params).fit(eval_metric='multi_error', **params_fit)
        self.assertEqual(len(gbm.evals_result_['training']), 2)
        self.assertIn('binary_logloss', gbm.evals_result_['training'])
        self.assertIn('binary_error', gbm.evals_result_['training'])

        # invalid multiclass metric is replaced with binary alternative for custom objective
700
        gbm = lgb.LGBMClassifier(objective=custom_dummy_obj,
701
702
703
                                 **params).fit(eval_metric='multi_logloss', **params_fit)
        self.assertEqual(len(gbm.evals_result_['training']), 1)
        self.assertIn('binary_logloss', gbm.evals_result_['training'])
Guolin Ke's avatar
Guolin Ke committed
704
705

    def test_inf_handle(self):
706
        nrows = 100
Guolin Ke's avatar
Guolin Ke committed
707
708
709
710
711
712
713
714
        ncols = 10
        X = np.random.randn(nrows, ncols)
        y = np.random.randn(nrows) + np.full(nrows, 1e30)
        weight = np.full(nrows, 1e10)
        params = {'n_estimators': 20, 'verbose': -1}
        params_fit = {'X': X, 'y': y, 'sample_weight': weight, 'eval_set': (X, y),
                      'verbose': False, 'early_stopping_rounds': 5}
        gbm = lgb.LGBMRegressor(**params).fit(**params_fit)
715
        np.testing.assert_allclose(gbm.evals_result_['training']['l2'], np.inf)
Guolin Ke's avatar
Guolin Ke committed
716
717

    def test_nan_handle(self):
718
        nrows = 100
Guolin Ke's avatar
Guolin Ke committed
719
720
721
722
723
724
725
726
        ncols = 10
        X = np.random.randn(nrows, ncols)
        y = np.random.randn(nrows) + np.full(nrows, 1e30)
        weight = np.zeros(nrows)
        params = {'n_estimators': 20, 'verbose': -1}
        params_fit = {'X': X, 'y': y, 'sample_weight': weight, 'eval_set': (X, y),
                      'verbose': False, 'early_stopping_rounds': 5}
        gbm = lgb.LGBMRegressor(**params).fit(**params_fit)
727
        np.testing.assert_allclose(gbm.evals_result_['training']['l2'], np.nan)
728

729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
    def test_first_metric_only(self):

        def fit_and_check(eval_set_names, metric_names, assumed_iteration, first_metric_only):
            params['first_metric_only'] = first_metric_only
            gbm = lgb.LGBMRegressor(**params).fit(**params_fit)
            self.assertEqual(len(gbm.evals_result_), len(eval_set_names))
            for eval_set_name in eval_set_names:
                self.assertIn(eval_set_name, gbm.evals_result_)
                self.assertEqual(len(gbm.evals_result_[eval_set_name]), len(metric_names))
                for metric_name in metric_names:
                    self.assertIn(metric_name, gbm.evals_result_[eval_set_name])

                    actual = len(gbm.evals_result_[eval_set_name][metric_name])
                    expected = assumed_iteration + (params_fit['early_stopping_rounds']
                                                    if eval_set_name != 'training'
                                                    and assumed_iteration != gbm.n_estimators else 0)
                    self.assertEqual(expected, actual)
746
                    self.assertEqual(assumed_iteration if eval_set_name != 'training' else gbm.n_estimators,
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
                                     gbm.best_iteration_)

        X, y = load_boston(True)
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
        X_test1, X_test2, y_test1, y_test2 = train_test_split(X_test, y_test, test_size=0.5, random_state=72)
        params = {'n_estimators': 30,
                  'learning_rate': 0.8,
                  'num_leaves': 15,
                  'verbose': -1,
                  'seed': 123}
        params_fit = {'X': X_train,
                      'y': y_train,
                      'early_stopping_rounds': 5,
                      'verbose': False}

        iter_valid1_l1 = 3
        iter_valid1_l2 = 18
        iter_valid2_l1 = 11
        iter_valid2_l2 = 7
        self.assertEqual(len(set([iter_valid1_l1, iter_valid1_l2, iter_valid2_l1, iter_valid2_l2])), 4)
        iter_min_l1 = min([iter_valid1_l1, iter_valid2_l1])
        iter_min_l2 = min([iter_valid1_l2, iter_valid2_l2])
        iter_min = min([iter_min_l1, iter_min_l2])
        iter_min_valid1 = min([iter_valid1_l1, iter_valid1_l2])

        # training data as eval_set
        params_fit['eval_set'] = (X_train, y_train)
        fit_and_check(['training'], ['l2'], 30, False)
        fit_and_check(['training'], ['l2'], 30, True)

        # feval
        params['metric'] = 'None'
        params_fit['eval_metric'] = lambda preds, train_data: [decreasing_metric(preds, train_data),
                                                               constant_metric(preds, train_data)]
        params_fit['eval_set'] = (X_test1, y_test1)
782
783
        fit_and_check(['valid_0'], ['decreasing_metric', 'error'], 1, False)
        fit_and_check(['valid_0'], ['decreasing_metric', 'error'], 30, True)
784
785
        params_fit['eval_metric'] = lambda preds, train_data: [constant_metric(preds, train_data),
                                                               decreasing_metric(preds, train_data)]
786
        fit_and_check(['valid_0'], ['decreasing_metric', 'error'], 1, True)
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828

        # single eval_set
        params.pop('metric')
        params_fit.pop('eval_metric')
        fit_and_check(['valid_0'], ['l2'], iter_valid1_l2, False)
        fit_and_check(['valid_0'], ['l2'], iter_valid1_l2, True)

        params_fit['eval_metric'] = "l2"
        fit_and_check(['valid_0'], ['l2'], iter_valid1_l2, False)
        fit_and_check(['valid_0'], ['l2'], iter_valid1_l2, True)

        params_fit['eval_metric'] = "l1"
        fit_and_check(['valid_0'], ['l1', 'l2'], iter_min_valid1, False)
        fit_and_check(['valid_0'], ['l1', 'l2'], iter_valid1_l1, True)

        params_fit['eval_metric'] = ["l1", "l2"]
        fit_and_check(['valid_0'], ['l1', 'l2'], iter_min_valid1, False)
        fit_and_check(['valid_0'], ['l1', 'l2'], iter_valid1_l1, True)

        params_fit['eval_metric'] = ["l2", "l1"]
        fit_and_check(['valid_0'], ['l1', 'l2'], iter_min_valid1, False)
        fit_and_check(['valid_0'], ['l1', 'l2'], iter_valid1_l2, True)

        params_fit['eval_metric'] = ["l2", "regression", "mse"]  # test aliases
        fit_and_check(['valid_0'], ['l2'], iter_valid1_l2, False)
        fit_and_check(['valid_0'], ['l2'], iter_valid1_l2, True)

        # two eval_set
        params_fit['eval_set'] = [(X_test1, y_test1), (X_test2, y_test2)]
        params_fit['eval_metric'] = ["l1", "l2"]
        fit_and_check(['valid_0', 'valid_1'], ['l1', 'l2'], iter_min_l1, True)
        params_fit['eval_metric'] = ["l2", "l1"]
        fit_and_check(['valid_0', 'valid_1'], ['l1', 'l2'], iter_min_l2, True)

        params_fit['eval_set'] = [(X_test2, y_test2), (X_test1, y_test1)]
        params_fit['eval_metric'] = ["l1", "l2"]
        fit_and_check(['valid_0', 'valid_1'], ['l1', 'l2'], iter_min, False)
        fit_and_check(['valid_0', 'valid_1'], ['l1', 'l2'], iter_min_l1, True)
        params_fit['eval_metric'] = ["l2", "l1"]
        fit_and_check(['valid_0', 'valid_1'], ['l1', 'l2'], iter_min, False)
        fit_and_check(['valid_0', 'valid_1'], ['l1', 'l2'], iter_min_l2, True)

829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
    def test_class_weight(self):
        X, y = load_digits(10, True)
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
        y_train_str = y_train.astype('str')
        y_test_str = y_test.astype('str')
        gbm = lgb.LGBMClassifier(n_estimators=10, class_weight='balanced', silent=True)
        gbm.fit(X_train, y_train,
                eval_set=[(X_train, y_train), (X_test, y_test), (X_test, y_test),
                          (X_test, y_test), (X_test, y_test)],
                eval_class_weight=['balanced', None, 'balanced', {1: 10, 4: 20}, {5: 30, 2: 40}],
                verbose=False)
        for eval_set1, eval_set2 in itertools.combinations(gbm.evals_result_.keys(), 2):
            for metric in gbm.evals_result_[eval_set1]:
                np.testing.assert_raises(AssertionError,
                                         np.testing.assert_allclose,
                                         gbm.evals_result_[eval_set1][metric],
                                         gbm.evals_result_[eval_set2][metric])
        gbm_str = lgb.LGBMClassifier(n_estimators=10, class_weight='balanced', silent=True)
        gbm_str.fit(X_train, y_train_str,
                    eval_set=[(X_train, y_train_str), (X_test, y_test_str),
                              (X_test, y_test_str), (X_test, y_test_str), (X_test, y_test_str)],
                    eval_class_weight=['balanced', None, 'balanced', {'1': 10, '4': 20}, {'5': 30, '2': 40}],
                    verbose=False)
        for eval_set1, eval_set2 in itertools.combinations(gbm_str.evals_result_.keys(), 2):
            for metric in gbm_str.evals_result_[eval_set1]:
                np.testing.assert_raises(AssertionError,
                                         np.testing.assert_allclose,
                                         gbm_str.evals_result_[eval_set1][metric],
                                         gbm_str.evals_result_[eval_set2][metric])
        for eval_set in gbm.evals_result_:
            for metric in gbm.evals_result_[eval_set]:
                np.testing.assert_allclose(gbm.evals_result_[eval_set][metric],
                                           gbm_str.evals_result_[eval_set][metric])
862
863
864
865
866
867
868
869
870
871
872
873
874

    def test_continue_training_with_model(self):
        X, y = load_digits(3, True)
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
        init_gbm = lgb.LGBMClassifier(n_estimators=5).fit(X_train, y_train, eval_set=(X_test, y_test),
                                                          verbose=False)
        gbm = lgb.LGBMClassifier(n_estimators=5).fit(X_train, y_train, eval_set=(X_test, y_test),
                                                     verbose=False, init_model=init_gbm)
        self.assertEqual(len(init_gbm.evals_result_['valid_0']['multi_logloss']),
                         len(gbm.evals_result_['valid_0']['multi_logloss']))
        self.assertEqual(len(init_gbm.evals_result_['valid_0']['multi_logloss']), 5)
        self.assertLess(gbm.evals_result_['valid_0']['multi_logloss'][-1],
                        init_gbm.evals_result_['valid_0']['multi_logloss'][-1])