Skip to content

GitLab

Unverified Commit 439c721a authored by Thomas J. Fan's avatar Thomas J. Fan Committed by GitHub
Browse files

[python-package] migrate test_sklearn.py to pytest (#3844) 

* TST Migrates test_sklearn.py to pytest

* STY Fixes linting

* FIX Adds reason

* ENH Address comments
parent 113da3af
Hide whitespace changes
Inline Side-by-side
Showing with 1064 additions and 1050 deletions
tests/python_package_test/test_sklearn.py
View file @ 439c721a
...@@ -3,7 +3,6 @@ import itertools ...@@ -3,7 +3,6 @@ import itertools
import joblib import joblib
import math import math
import os import os
import unittest
import lightgbm as lgb import lightgbm as lgb
import numpy as np import numpy as np
...@@ -80,1058 +79,1073 @@ def multi_logloss(y_true, y_pred): ...@@ -80,1058 +79,1073 @@ def multi_logloss(y_true, y_pred):
return np.mean([-math.log(y_pred[i][y]) for i, y in enumerate(y_true)]) return np.mean([-math.log(y_pred[i][y]) for i, y in enumerate(y_true)])
class TestSklearn(unittest.TestCase): def test_binary():
X, y = load_breast_cancer(return_X_y=True)
def test_binary(self): X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
X, y = load_breast_cancer(return_X_y=True) gbm = lgb.LGBMClassifier(n_estimators=50, silent=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42) gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=5, verbose=False)
gbm = lgb.LGBMClassifier(n_estimators=50, silent=True) ret = log_loss(y_test, gbm.predict_proba(X_test))
gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=5, verbose=False) assert ret < 0.12
ret = log_loss(y_test, gbm.predict_proba(X_test)) assert ret == pytest.approx(gbm.evals_result_['valid_0']['binary_logloss'][gbm.best_iteration_ - 1], abs=1e-5)
self.assertLess(ret, 0.12)
self.assertAlmostEqual(ret, gbm.evals_result_['valid_0']['binary_logloss'][gbm.best_iteration_ - 1], places=5)
def test_regression():
def test_regression(self): X, y = load_boston(return_X_y=True)
X, y = load_boston(return_X_y=True) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42) gbm = lgb.LGBMRegressor(n_estimators=50, silent=True)
gbm = lgb.LGBMRegressor(n_estimators=50, silent=True) gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=5, verbose=False)
gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=5, verbose=False) ret = mean_squared_error(y_test, gbm.predict(X_test))
ret = mean_squared_error(y_test, gbm.predict(X_test)) assert ret < 7
self.assertLess(ret, 7) assert ret == pytest.approx(gbm.evals_result_['valid_0']['l2'][gbm.best_iteration_ - 1], abs=1e-5)
self.assertAlmostEqual(ret, gbm.evals_result_['valid_0']['l2'][gbm.best_iteration_ - 1], places=5)
def test_multiclass(self): def test_multiclass():
X, y = load_digits(n_class=10, return_X_y=True) X, y = load_digits(n_class=10, return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
gbm = lgb.LGBMClassifier(n_estimators=50, silent=True) gbm = lgb.LGBMClassifier(n_estimators=50, silent=True)
gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=5, verbose=False) gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=5, verbose=False)
ret = multi_error(y_test, gbm.predict(X_test)) ret = multi_error(y_test, gbm.predict(X_test))
self.assertLess(ret, 0.05) assert ret < 0.05
ret = multi_logloss(y_test, gbm.predict_proba(X_test)) ret = multi_logloss(y_test, gbm.predict_proba(X_test))
self.assertLess(ret, 0.16) assert ret < 0.16
self.assertAlmostEqual(ret, gbm.evals_result_['valid_0']['multi_logloss'][gbm.best_iteration_ - 1], places=5) assert ret == pytest.approx(gbm.evals_result_['valid_0']['multi_logloss'][gbm.best_iteration_ - 1], abs=1e-5)
def test_lambdarank(self):
X_train, y_train = load_svmlight_file(os.path.join(os.path.dirname(os.path.realpath(__file__)), def test_lambdarank():
'../../examples/lambdarank/rank.train')) X_train, y_train = load_svmlight_file(os.path.join(os.path.dirname(os.path.realpath(__file__)),
X_test, y_test = load_svmlight_file(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../examples/lambdarank/rank.train'))
'../../examples/lambdarank/rank.test')) X_test, y_test = load_svmlight_file(os.path.join(os.path.dirname(os.path.realpath(__file__)),
q_train = np.loadtxt(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../examples/lambdarank/rank.test'))
'../../examples/lambdarank/rank.train.query')) q_train = np.loadtxt(os.path.join(os.path.dirname(os.path.realpath(__file__)),
q_test = np.loadtxt(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../examples/lambdarank/rank.train.query'))
'../../examples/lambdarank/rank.test.query')) q_test = np.loadtxt(os.path.join(os.path.dirname(os.path.realpath(__file__)),
gbm = lgb.LGBMRanker(n_estimators=50) '../../examples/lambdarank/rank.test.query'))
gbm.fit(X_train, y_train, group=q_train, eval_set=[(X_test, y_test)], gbm = lgb.LGBMRanker(n_estimators=50)
eval_group=[q_test], eval_at=[1, 3], early_stopping_rounds=10, verbose=False, gbm.fit(X_train, y_train, group=q_train, eval_set=[(X_test, y_test)],
callbacks=[lgb.reset_parameter(learning_rate=lambda x: max(0.01, 0.1 - 0.01 * x))]) eval_group=[q_test], eval_at=[1, 3], early_stopping_rounds=10, verbose=False,
self.assertLessEqual(gbm.best_iteration_, 24) callbacks=[lgb.reset_parameter(learning_rate=lambda x: max(0.01, 0.1 - 0.01 * x))])
self.assertGreater(gbm.best_score_['valid_0']['ndcg@1'], 0.5674) assert gbm.best_iteration_ <= 24
self.assertGreater(gbm.best_score_['valid_0']['ndcg@3'], 0.578) assert gbm.best_score_['valid_0']['ndcg@1'] > 0.5674
assert gbm.best_score_['valid_0']['ndcg@3'] > 0.578
def test_xendcg(self):
dir_path = os.path.dirname(os.path.realpath(__file__))
X_train, y_train = load_svmlight_file(os.path.join(dir_path, '../../examples/xendcg/rank.train')) def test_xendcg():
X_test, y_test = load_svmlight_file(os.path.join(dir_path, '../../examples/xendcg/rank.test')) dir_path = os.path.dirname(os.path.realpath(__file__))
q_train = np.loadtxt(os.path.join(dir_path, '../../examples/xendcg/rank.train.query')) X_train, y_train = load_svmlight_file(os.path.join(dir_path, '../../examples/xendcg/rank.train'))
q_test = np.loadtxt(os.path.join(dir_path, '../../examples/xendcg/rank.test.query')) X_test, y_test = load_svmlight_file(os.path.join(dir_path, '../../examples/xendcg/rank.test'))
gbm = lgb.LGBMRanker(n_estimators=50, objective='rank_xendcg', random_state=5, n_jobs=1) q_train = np.loadtxt(os.path.join(dir_path, '../../examples/xendcg/rank.train.query'))
gbm.fit(X_train, y_train, group=q_train, eval_set=[(X_test, y_test)], q_test = np.loadtxt(os.path.join(dir_path, '../../examples/xendcg/rank.test.query'))
eval_group=[q_test], eval_at=[1, 3], early_stopping_rounds=10, verbose=False, gbm = lgb.LGBMRanker(n_estimators=50, objective='rank_xendcg', random_state=5, n_jobs=1)
eval_metric='ndcg', gbm.fit(X_train, y_train, group=q_train, eval_set=[(X_test, y_test)],
callbacks=[lgb.reset_parameter(learning_rate=lambda x: max(0.01, 0.1 - 0.01 * x))]) eval_group=[q_test], eval_at=[1, 3], early_stopping_rounds=10, verbose=False,
self.assertLessEqual(gbm.best_iteration_, 24) eval_metric='ndcg',
self.assertGreater(gbm.best_score_['valid_0']['ndcg@1'], 0.6211) callbacks=[lgb.reset_parameter(learning_rate=lambda x: max(0.01, 0.1 - 0.01 * x))])
self.assertGreater(gbm.best_score_['valid_0']['ndcg@3'], 0.6253) assert gbm.best_iteration_ <= 24
assert gbm.best_score_['valid_0']['ndcg@1'] > 0.6211
def test_regression_with_custom_objective(self): assert gbm.best_score_['valid_0']['ndcg@3'] > 0.6253
X, y = load_boston(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
gbm = lgb.LGBMRegressor(n_estimators=50, silent=True, objective=objective_ls) def test_regression_with_custom_objective():
gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=5, verbose=False) X, y = load_boston(return_X_y=True)
ret = mean_squared_error(y_test, gbm.predict(X_test)) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
self.assertLess(ret, 7.0) gbm = lgb.LGBMRegressor(n_estimators=50, silent=True, objective=objective_ls)
self.assertAlmostEqual(ret, gbm.evals_result_['valid_0']['l2'][gbm.best_iteration_ - 1], places=5) gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=5, verbose=False)
ret = mean_squared_error(y_test, gbm.predict(X_test))
def test_binary_classification_with_custom_objective(self): assert ret < 7.0
X, y = load_digits(n_class=2, return_X_y=True) assert ret == pytest.approx(gbm.evals_result_['valid_0']['l2'][gbm.best_iteration_ - 1], abs=1e-5)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
gbm = lgb.LGBMClassifier(n_estimators=50, silent=True, objective=logregobj)
gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=5, verbose=False) def test_binary_classification_with_custom_objective():
# prediction result is actually not transformed (is raw) due to custom objective X, y = load_digits(n_class=2, return_X_y=True)
y_pred_raw = gbm.predict_proba(X_test) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
self.assertFalse(np.all(y_pred_raw >= 0)) gbm = lgb.LGBMClassifier(n_estimators=50, silent=True, objective=logregobj)
y_pred = 1.0 / (1.0 + np.exp(-y_pred_raw)) gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=5, verbose=False)
ret = binary_error(y_test, y_pred) # prediction result is actually not transformed (is raw) due to custom objective
self.assertLess(ret, 0.05) y_pred_raw = gbm.predict_proba(X_test)
assert not np.all(y_pred_raw >= 0)
def test_dart(self): y_pred = 1.0 / (1.0 + np.exp(-y_pred_raw))
X, y = load_boston(return_X_y=True) ret = binary_error(y_test, y_pred)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42) assert ret < 0.05
gbm = lgb.LGBMRegressor(boosting_type='dart', n_estimators=50)
gbm.fit(X_train, y_train)
score = gbm.score(X_test, y_test) def test_dart():
self.assertGreaterEqual(score, 0.8) X, y = load_boston(return_X_y=True)
self.assertLessEqual(score, 1.) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
gbm = lgb.LGBMRegressor(boosting_type='dart', n_estimators=50)
# sklearn <0.23 does not have a stacking classifier and n_features_in_ property gbm.fit(X_train, y_train)
@unittest.skipIf(sk_version < parse_version("0.23"), 'scikit-learn version is less than 0.23') score = gbm.score(X_test, y_test)
def test_stacking_classifier(self): assert score >= 0.8
from sklearn.ensemble import StackingClassifier assert score <= 1.
X, y = load_iris(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42) # sklearn <0.23 does not have a stacking classifier and n_features_in_ property
classifiers = [('gbm1', lgb.LGBMClassifier(n_estimators=3)), @pytest.mark.skipif(sk_version < parse_version("0.23"), reason='scikit-learn version is less than 0.23')
('gbm2', lgb.LGBMClassifier(n_estimators=3))] def test_stacking_classifier():
clf = StackingClassifier(estimators=classifiers, from sklearn.ensemble import StackingClassifier
final_estimator=lgb.LGBMClassifier(n_estimators=3),
passthrough=True) X, y = load_iris(return_X_y=True)
clf.fit(X_train, y_train) X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
score = clf.score(X_test, y_test) classifiers = [('gbm1', lgb.LGBMClassifier(n_estimators=3)),
self.assertGreaterEqual(score, 0.8) ('gbm2', lgb.LGBMClassifier(n_estimators=3))]
self.assertLessEqual(score, 1.) clf = StackingClassifier(estimators=classifiers,
self.assertEqual(clf.n_features_in_, 4) # number of input features final_estimator=lgb.LGBMClassifier(n_estimators=3),
self.assertEqual(len(clf.named_estimators_['gbm1'].feature_importances_), 4) passthrough=True)
self.assertEqual(clf.named_estimators_['gbm1'].n_features_in_, clf.fit(X_train, y_train)
clf.named_estimators_['gbm2'].n_features_in_) score = clf.score(X_test, y_test)
self.assertEqual(clf.final_estimator_.n_features_in_, 10) # number of concatenated features assert score >= 0.8
self.assertEqual(len(clf.final_estimator_.feature_importances_), 10) assert score <= 1.
classes = clf.named_estimators_['gbm1'].classes_ == clf.named_estimators_['gbm2'].classes_ assert clf.n_features_in_ == 4 # number of input features
self.assertTrue(all(classes)) assert len(clf.named_estimators_['gbm1'].feature_importances_) == 4
classes = clf.classes_ == clf.named_estimators_['gbm1'].classes_ assert clf.named_estimators_['gbm1'].n_features_in_ == clf.named_estimators_['gbm2'].n_features_in_
self.assertTrue(all(classes)) assert clf.final_estimator_.n_features_in_ == 10 # number of concatenated features
assert len(clf.final_estimator_.feature_importances_) == 10
# sklearn <0.23 does not have a stacking regressor and n_features_in_ property classes = clf.named_estimators_['gbm1'].classes_ == clf.named_estimators_['gbm2'].classes_
@unittest.skipIf(sk_version < parse_version('0.23'), 'scikit-learn version is less than 0.23') assert all(classes)
def test_stacking_regressor(self): classes = clf.classes_ == clf.named_estimators_['gbm1'].classes_
from sklearn.ensemble import StackingRegressor assert all(classes)
X, y = load_boston(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42) # sklearn <0.23 does not have a stacking regressor and n_features_in_ property
regressors = [('gbm1', lgb.LGBMRegressor(n_estimators=3)), @pytest.mark.skipif(sk_version < parse_version('0.23'), reason='scikit-learn version is less than 0.23')
('gbm2', lgb.LGBMRegressor(n_estimators=3))] def test_stacking_regressor():
reg = StackingRegressor(estimators=regressors, from sklearn.ensemble import StackingRegressor
final_estimator=lgb.LGBMRegressor(n_estimators=3),
passthrough=True) X, y = load_boston(return_X_y=True)
reg.fit(X_train, y_train) X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
score = reg.score(X_test, y_test) regressors = [('gbm1', lgb.LGBMRegressor(n_estimators=3)),
self.assertGreaterEqual(score, 0.2) ('gbm2', lgb.LGBMRegressor(n_estimators=3))]
self.assertLessEqual(score, 1.) reg = StackingRegressor(estimators=regressors,
self.assertEqual(reg.n_features_in_, 13) # number of input features final_estimator=lgb.LGBMRegressor(n_estimators=3),
self.assertEqual(len(reg.named_estimators_['gbm1'].feature_importances_), 13) passthrough=True)
self.assertEqual(reg.named_estimators_['gbm1'].n_features_in_, reg.fit(X_train, y_train)
reg.named_estimators_['gbm2'].n_features_in_) score = reg.score(X_test, y_test)
self.assertEqual(reg.final_estimator_.n_features_in_, 15) # number of concatenated features assert score >= 0.2
self.assertEqual(len(reg.final_estimator_.feature_importances_), 15) assert score <= 1.
assert reg.n_features_in_ == 13 # number of input features
def test_grid_search(self): assert len(reg.named_estimators_['gbm1'].feature_importances_) == 13
X, y = load_iris(return_X_y=True) assert reg.named_estimators_['gbm1'].n_features_in_ == reg.named_estimators_['gbm2'].n_features_in_
y = y.astype(str) # utilize label encoder at it's max power assert reg.final_estimator_.n_features_in_ == 15 # number of concatenated features
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, assert len(reg.final_estimator_.feature_importances_) == 15
random_state=42)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1,
random_state=42) def test_grid_search():
params = dict(subsample=0.8, X, y = load_iris(return_X_y=True)
subsample_freq=1) y = y.astype(str) # utilize label encoder at it's max power
grid_params = dict(boosting_type=['rf', 'gbdt'], X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1,
n_estimators=[4, 6], random_state=42)
reg_alpha=[0.01, 0.005]) X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1,
fit_params = dict(verbose=False, random_state=42)
eval_set=[(X_val, y_val)], params = dict(subsample=0.8,
eval_metric=constant_metric, subsample_freq=1)
early_stopping_rounds=2) grid_params = dict(boosting_type=['rf', 'gbdt'],
grid = GridSearchCV(estimator=lgb.LGBMClassifier(**params), param_grid=grid_params, n_estimators=[4, 6],
cv=2) reg_alpha=[0.01, 0.005])
grid.fit(X_train, y_train, **fit_params) fit_params = dict(verbose=False,
score = grid.score(X_test, y_test) # utilizes GridSearchCV default refit=True eval_set=[(X_val, y_val)],
self.assertIn(grid.best_params_['boosting_type'], ['rf', 'gbdt']) eval_metric=constant_metric,
self.assertIn(grid.best_params_['n_estimators'], [4, 6]) early_stopping_rounds=2)
self.assertIn(grid.best_params_['reg_alpha'], [0.01, 0.005]) grid = GridSearchCV(estimator=lgb.LGBMClassifier(**params), param_grid=grid_params,
self.assertLessEqual(grid.best_score_, 1.) cv=2)
self.assertEqual(grid.best_estimator_.best_iteration_, 1) grid.fit(X_train, y_train, **fit_params)
self.assertLess(grid.best_estimator_.best_score_['valid_0']['multi_logloss'], 0.25) score = grid.score(X_test, y_test) # utilizes GridSearchCV default refit=True
self.assertEqual(grid.best_estimator_.best_score_['valid_0']['error'], 0) assert grid.best_params_['boosting_type'] in ['rf', 'gbdt']
self.assertGreaterEqual(score, 0.2) assert grid.best_params_['n_estimators'] in [4, 6]
self.assertLessEqual(score, 1.) assert grid.best_params_['reg_alpha'] in [0.01, 0.005]
assert grid.best_score_ <= 1.
def test_random_search(self): assert grid.best_estimator_.best_iteration_ == 1
X, y = load_iris(return_X_y=True) assert grid.best_estimator_.best_score_['valid_0']['multi_logloss'] < 0.25
y = y.astype(str) # utilize label encoder at it's max power assert grid.best_estimator_.best_score_['valid_0']['error'] == 0
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, assert score >= 0.2
random_state=42) assert score <= 1.
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1,
random_state=42)
n_iter = 3 # Number of samples def test_random_search():
params = dict(subsample=0.8, X, y = load_iris(return_X_y=True)
subsample_freq=1) y = y.astype(str) # utilize label encoder at it's max power
param_dist = dict(boosting_type=['rf', 'gbdt'], X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1,
n_estimators=[np.random.randint(low=3, high=10) for i in range(n_iter)], random_state=42)
reg_alpha=[np.random.uniform(low=0.01, high=0.06) for i in range(n_iter)]) X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1,
fit_params = dict(verbose=False, random_state=42)
eval_set=[(X_val, y_val)], n_iter = 3 # Number of samples
eval_metric=constant_metric, params = dict(subsample=0.8,
early_stopping_rounds=2) subsample_freq=1)
rand = RandomizedSearchCV(estimator=lgb.LGBMClassifier(**params), param_dist = dict(boosting_type=['rf', 'gbdt'],
param_distributions=param_dist, cv=2, n_estimators=[np.random.randint(low=3, high=10) for i in range(n_iter)],
n_iter=n_iter, random_state=42) reg_alpha=[np.random.uniform(low=0.01, high=0.06) for i in range(n_iter)])
rand.fit(X_train, y_train, **fit_params) fit_params = dict(verbose=False,
score = rand.score(X_test, y_test) # utilizes RandomizedSearchCV default refit=True eval_set=[(X_val, y_val)],
self.assertIn(rand.best_params_['boosting_type'], ['rf', 'gbdt']) eval_metric=constant_metric,
self.assertIn(rand.best_params_['n_estimators'], list(range(3, 10))) early_stopping_rounds=2)
self.assertGreaterEqual(rand.best_params_['reg_alpha'], 0.01) # Left-closed boundary point rand = RandomizedSearchCV(estimator=lgb.LGBMClassifier(**params),
self.assertLessEqual(rand.best_params_['reg_alpha'], 0.06) # Right-closed boundary point param_distributions=param_dist, cv=2,
self.assertLessEqual(rand.best_score_, 1.) n_iter=n_iter, random_state=42)
self.assertLess(rand.best_estimator_.best_score_['valid_0']['multi_logloss'], 0.25) rand.fit(X_train, y_train, **fit_params)
self.assertEqual(rand.best_estimator_.best_score_['valid_0']['error'], 0) score = rand.score(X_test, y_test) # utilizes RandomizedSearchCV default refit=True
self.assertGreaterEqual(score, 0.2) assert rand.best_params_['boosting_type'] in ['rf', 'gbdt']
self.assertLessEqual(score, 1.) assert rand.best_params_['n_estimators'] in list(range(3, 10))
assert rand.best_params_['reg_alpha'] >= 0.01 # Left-closed boundary point
# sklearn < 0.22 does not have the post fit attribute: classes_ assert rand.best_params_['reg_alpha'] <= 0.06 # Right-closed boundary point
@unittest.skipIf(sk_version < parse_version('0.22'), 'scikit-learn version is less than 0.22') assert rand.best_score_ <= 1.
def test_multioutput_classifier(self): assert rand.best_estimator_.best_score_['valid_0']['multi_logloss'] < 0.25
n_outputs = 3 assert rand.best_estimator_.best_score_['valid_0']['error'] == 0
X, y = make_multilabel_classification(n_samples=100, n_features=20, assert score >= 0.2
n_classes=n_outputs, random_state=0) assert score <= 1.
y = y.astype(str) # utilize label encoder at it's max power
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1,
random_state=42) # sklearn < 0.22 does not have the post fit attribute: classes_
clf = MultiOutputClassifier(estimator=lgb.LGBMClassifier(n_estimators=10)) @pytest.mark.skipif(sk_version < parse_version('0.22'), reason='scikit-learn version is less than 0.22')
clf.fit(X_train, y_train) def test_multioutput_classifier():
score = clf.score(X_test, y_test) n_outputs = 3
self.assertGreaterEqual(score, 0.2) X, y = make_multilabel_classification(n_samples=100, n_features=20,
self.assertLessEqual(score, 1.) n_classes=n_outputs, random_state=0)
np.testing.assert_array_equal(np.tile(np.unique(y_train), n_outputs), y = y.astype(str) # utilize label encoder at it's max power
np.concatenate(clf.classes_)) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1,
for classifier in clf.estimators_: random_state=42)
self.assertIsInstance(classifier, lgb.LGBMClassifier) clf = MultiOutputClassifier(estimator=lgb.LGBMClassifier(n_estimators=10))
self.assertIsInstance(classifier.booster_, lgb.Booster) clf.fit(X_train, y_train)
score = clf.score(X_test, y_test)
# sklearn < 0.23 does not have as_frame parameter assert score >= 0.2
@unittest.skipIf(sk_version < parse_version('0.23'), 'scikit-learn version is less than 0.23') assert score <= 1.
def test_multioutput_regressor(self): np.testing.assert_array_equal(np.tile(np.unique(y_train), n_outputs),
bunch = load_linnerud(as_frame=True) # returns a Bunch instance np.concatenate(clf.classes_))
X, y = bunch['data'], bunch['target'] for classifier in clf.estimators_:
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, assert isinstance(classifier, lgb.LGBMClassifier)
random_state=42) assert isinstance(classifier.booster_, lgb.Booster)
reg = MultiOutputRegressor(estimator=lgb.LGBMRegressor(n_estimators=10))
reg.fit(X_train, y_train)
y_pred = reg.predict(X_test) # sklearn < 0.23 does not have as_frame parameter
_, score, _ = mse(y_test, y_pred) @pytest.mark.skipif(sk_version < parse_version('0.23'), reason='scikit-learn version is less than 0.23')
self.assertGreaterEqual(score, 0.2) def test_multioutput_regressor():
self.assertLessEqual(score, 120.) bunch = load_linnerud(as_frame=True) # returns a Bunch instance
for regressor in reg.estimators_: X, y = bunch['data'], bunch['target']
self.assertIsInstance(regressor, lgb.LGBMRegressor) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1,
self.assertIsInstance(regressor.booster_, lgb.Booster) random_state=42)
reg = MultiOutputRegressor(estimator=lgb.LGBMRegressor(n_estimators=10))
# sklearn < 0.22 does not have the post fit attribute: classes_ reg.fit(X_train, y_train)
@unittest.skipIf(sk_version < parse_version('0.22'), 'scikit-learn version is less than 0.22') y_pred = reg.predict(X_test)
def test_classifier_chain(self): _, score, _ = mse(y_test, y_pred)
n_outputs = 3 assert score >= 0.2
X, y = make_multilabel_classification(n_samples=100, n_features=20, assert score <= 120.
n_classes=n_outputs, random_state=0) for regressor in reg.estimators_:
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, assert isinstance(regressor, lgb.LGBMRegressor)
random_state=42) assert isinstance(regressor.booster_, lgb.Booster)
order = [2, 0, 1]
clf = ClassifierChain(base_estimator=lgb.LGBMClassifier(n_estimators=10),
order=order, random_state=42) # sklearn < 0.22 does not have the post fit attribute: classes_
clf.fit(X_train, y_train) @pytest.mark.skipif(sk_version < parse_version('0.22'), reason='scikit-learn version is less than 0.22')
score = clf.score(X_test, y_test) def test_classifier_chain():
self.assertGreaterEqual(score, 0.2) n_outputs = 3
self.assertLessEqual(score, 1.) X, y = make_multilabel_classification(n_samples=100, n_features=20,
np.testing.assert_array_equal(np.tile(np.unique(y_train), n_outputs), n_classes=n_outputs, random_state=0)
np.concatenate(clf.classes_)) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1,
self.assertListEqual(order, clf.order_) random_state=42)
for classifier in clf.estimators_: order = [2, 0, 1]
self.assertIsInstance(classifier, lgb.LGBMClassifier) clf = ClassifierChain(base_estimator=lgb.LGBMClassifier(n_estimators=10),
self.assertIsInstance(classifier.booster_, lgb.Booster) order=order, random_state=42)
clf.fit(X_train, y_train)
# sklearn < 0.23 does not have as_frame parameter score = clf.score(X_test, y_test)
@unittest.skipIf(sk_version < parse_version('0.23'), 'scikit-learn version is less than 0.23') assert score >= 0.2
def test_regressor_chain(self): assert score <= 1.
bunch = load_linnerud(as_frame=True) # returns a Bunch instance np.testing.assert_array_equal(np.tile(np.unique(y_train), n_outputs),
X, y = bunch['data'], bunch['target'] np.concatenate(clf.classes_))
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42) assert order == clf.order_
order = [2, 0, 1] for classifier in clf.estimators_:
reg = RegressorChain(base_estimator=lgb.LGBMRegressor(n_estimators=10), order=order, assert isinstance(classifier, lgb.LGBMClassifier)
random_state=42) assert isinstance(classifier.booster_, lgb.Booster)
reg.fit(X_train, y_train)
y_pred = reg.predict(X_test)
_, score, _ = mse(y_test, y_pred) # sklearn < 0.23 does not have as_frame parameter
self.assertGreaterEqual(score, 0.2) @pytest.mark.skipif(sk_version < parse_version('0.23'), reason='scikit-learn version is less than 0.23')
self.assertLessEqual(score, 120.) def test_regressor_chain():
self.assertListEqual(order, reg.order_) bunch = load_linnerud(as_frame=True) # returns a Bunch instance
for regressor in reg.estimators_: X, y = bunch['data'], bunch['target']
self.assertIsInstance(regressor, lgb.LGBMRegressor) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
self.assertIsInstance(regressor.booster_, lgb.Booster) order = [2, 0, 1]
reg = RegressorChain(base_estimator=lgb.LGBMRegressor(n_estimators=10), order=order,
def test_clone_and_property(self): random_state=42)
X, y = load_boston(return_X_y=True) reg.fit(X_train, y_train)
gbm = lgb.LGBMRegressor(n_estimators=10, silent=True) y_pred = reg.predict(X_test)
gbm.fit(X, y, verbose=False) _, score, _ = mse(y_test, y_pred)
assert score >= 0.2
gbm_clone = clone(gbm) assert score <= 120.
self.assertIsInstance(gbm.booster_, lgb.Booster) assert order == reg.order_
self.assertIsInstance(gbm.feature_importances_, np.ndarray) for regressor in reg.estimators_:
assert isinstance(regressor, lgb.LGBMRegressor)
X, y = load_digits(n_class=2, return_X_y=True) assert isinstance(regressor.booster_, lgb.Booster)
clf = lgb.LGBMClassifier(n_estimators=10, silent=True)
clf.fit(X, y, verbose=False)
self.assertListEqual(sorted(clf.classes_), [0, 1]) def test_clone_and_property():
self.assertEqual(clf.n_classes_, 2) X, y = load_boston(return_X_y=True)
self.assertIsInstance(clf.booster_, lgb.Booster) gbm = lgb.LGBMRegressor(n_estimators=10, silent=True)
self.assertIsInstance(clf.feature_importances_, np.ndarray) gbm.fit(X, y, verbose=False)
def test_joblib(self): gbm_clone = clone(gbm)
X, y = load_boston(return_X_y=True) assert isinstance(gbm.booster_, lgb.Booster)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42) assert isinstance(gbm.feature_importances_, np.ndarray)
gbm = lgb.LGBMRegressor(n_estimators=10, objective=custom_asymmetric_obj,
silent=True, importance_type='split') X, y = load_digits(n_class=2, return_X_y=True)
gbm.fit(X_train, y_train, eval_set=[(X_train, y_train), (X_test, y_test)], clf = lgb.LGBMClassifier(n_estimators=10, silent=True)
eval_metric=mse, early_stopping_rounds=5, verbose=False, clf.fit(X, y, verbose=False)
callbacks=[lgb.reset_parameter(learning_rate=list(np.arange(1, 0, -0.1)))]) assert sorted(clf.classes_) == [0, 1]
assert clf.n_classes_ == 2
joblib.dump(gbm, 'lgb.pkl') # test model with custom functions assert isinstance(clf.booster_, lgb.Booster)
gbm_pickle = joblib.load('lgb.pkl') assert isinstance(clf.feature_importances_, np.ndarray)
self.assertIsInstance(gbm_pickle.booster_, lgb.Booster)
self.assertDictEqual(gbm.get_params(), gbm_pickle.get_params())
np.testing.assert_array_equal(gbm.feature_importances_, gbm_pickle.feature_importances_) def test_joblib():
self.assertAlmostEqual(gbm_pickle.learning_rate, 0.1) X, y = load_boston(return_X_y=True)
self.assertTrue(callable(gbm_pickle.objective)) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
gbm = lgb.LGBMRegressor(n_estimators=10, objective=custom_asymmetric_obj,
for eval_set in gbm.evals_result_: silent=True, importance_type='split')
for metric in gbm.evals_result_[eval_set]: gbm.fit(X_train, y_train, eval_set=[(X_train, y_train), (X_test, y_test)],
np.testing.assert_allclose(gbm.evals_result_[eval_set][metric], eval_metric=mse, early_stopping_rounds=5, verbose=False,
gbm_pickle.evals_result_[eval_set][metric]) callbacks=[lgb.reset_parameter(learning_rate=list(np.arange(1, 0, -0.1)))])
pred_origin = gbm.predict(X_test)
pred_pickle = gbm_pickle.predict(X_test) joblib.dump(gbm, 'lgb.pkl') # test model with custom functions
np.testing.assert_allclose(pred_origin, pred_pickle) gbm_pickle = joblib.load('lgb.pkl')
assert isinstance(gbm_pickle.booster_, lgb.Booster)
def test_random_state_object(self): assert gbm.get_params() == gbm_pickle.get_params()
X, y = load_iris(return_X_y=True) np.testing.assert_array_equal(gbm.feature_importances_, gbm_pickle.feature_importances_)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42) assert gbm_pickle.learning_rate == pytest.approx(0.1)
state1 = np.random.RandomState(123) assert callable(gbm_pickle.objective)
state2 = np.random.RandomState(123)
clf1 = lgb.LGBMClassifier(n_estimators=10, subsample=0.5, subsample_freq=1, random_state=state1) for eval_set in gbm.evals_result_:
clf2 = lgb.LGBMClassifier(n_estimators=10, subsample=0.5, subsample_freq=1, random_state=state2) for metric in gbm.evals_result_[eval_set]:
# Test if random_state is properly stored np.testing.assert_allclose(gbm.evals_result_[eval_set][metric],
self.assertIs(clf1.random_state, state1) gbm_pickle.evals_result_[eval_set][metric])
self.assertIs(clf2.random_state, state2) pred_origin = gbm.predict(X_test)
# Test if two random states produce identical models pred_pickle = gbm_pickle.predict(X_test)
clf1.fit(X_train, y_train) np.testing.assert_allclose(pred_origin, pred_pickle)
clf2.fit(X_train, y_train)
y_pred1 = clf1.predict(X_test, raw_score=True)
y_pred2 = clf2.predict(X_test, raw_score=True) def test_random_state_object():
np.testing.assert_allclose(y_pred1, y_pred2) X, y = load_iris(return_X_y=True)
np.testing.assert_array_equal(clf1.feature_importances_, clf2.feature_importances_) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
df1 = clf1.booster_.model_to_string(num_iteration=0) state1 = np.random.RandomState(123)
df2 = clf2.booster_.model_to_string(num_iteration=0) state2 = np.random.RandomState(123)
self.assertMultiLineEqual(df1, df2) clf1 = lgb.LGBMClassifier(n_estimators=10, subsample=0.5, subsample_freq=1, random_state=state1)
# Test if subsequent fits sample from random_state object and produce different models clf2 = lgb.LGBMClassifier(n_estimators=10, subsample=0.5, subsample_freq=1, random_state=state2)
clf1.fit(X_train, y_train) # Test if random_state is properly stored
y_pred1_refit = clf1.predict(X_test, raw_score=True) assert clf1.random_state is state1
df3 = clf1.booster_.model_to_string(num_iteration=0) assert clf2.random_state is state2
self.assertIs(clf1.random_state, state1) # Test if two random states produce identical models
self.assertIs(clf2.random_state, state2) clf1.fit(X_train, y_train)
self.assertRaises(AssertionError, clf2.fit(X_train, y_train)
np.testing.assert_allclose, y_pred1 = clf1.predict(X_test, raw_score=True)
y_pred1, y_pred1_refit) y_pred2 = clf2.predict(X_test, raw_score=True)
self.assertRaises(AssertionError, np.testing.assert_allclose(y_pred1, y_pred2)
self.assertMultiLineEqual, np.testing.assert_array_equal(clf1.feature_importances_, clf2.feature_importances_)
df1, df3) df1 = clf1.booster_.model_to_string(num_iteration=0)
df2 = clf2.booster_.model_to_string(num_iteration=0)
def test_feature_importances_single_leaf(self): assert df1 == df2
data = load_iris(return_X_y=False) # Test if subsequent fits sample from random_state object and produce different models
clf = lgb.LGBMClassifier(n_estimators=10) clf1.fit(X_train, y_train)
clf.fit(data.data, data.target) y_pred1_refit = clf1.predict(X_test, raw_score=True)
importances = clf.feature_importances_ df3 = clf1.booster_.model_to_string(num_iteration=0)
self.assertEqual(len(importances), 4) assert clf1.random_state is state1
assert clf2.random_state is state2
def test_feature_importances_type(self): with pytest.raises(AssertionError):
data = load_iris(return_X_y=False) np.testing.assert_allclose(y_pred1, y_pred1_refit)
clf = lgb.LGBMClassifier(n_estimators=10) assert df1 != df3
clf.fit(data.data, data.target)
clf.set_params(importance_type='split')
importances_split = clf.feature_importances_ def test_feature_importances_single_leaf():
clf.set_params(importance_type='gain') data = load_iris(return_X_y=False)
importances_gain = clf.feature_importances_ clf = lgb.LGBMClassifier(n_estimators=10)
# Test that the largest element is NOT the same, the smallest can be the same, i.e. zero clf.fit(data.data, data.target)
importance_split_top1 = sorted(importances_split, reverse=True)[0] importances = clf.feature_importances_
importance_gain_top1 = sorted(importances_gain, reverse=True)[0] assert len(importances) == 4
self.assertNotEqual(importance_split_top1, importance_gain_top1)
@unittest.skipIf(not lgb.compat.PANDAS_INSTALLED, 'pandas is not installed') def test_feature_importances_type():
def test_pandas_categorical(self): data = load_iris(return_X_y=False)
import pandas as pd clf = lgb.LGBMClassifier(n_estimators=10)
np.random.seed(42) # sometimes there is no difference how cols are treated (cat or not cat) clf.fit(data.data, data.target)
X = pd.DataFrame({"A": np.random.permutation(['a', 'b', 'c', 'd'] * 75), # str clf.set_params(importance_type='split')
"B": np.random.permutation([1, 2, 3] * 100), # int importances_split = clf.feature_importances_
"C": np.random.permutation([0.1, 0.2, -0.1, -0.1, 0.2] * 60), # float clf.set_params(importance_type='gain')
"D": np.random.permutation([True, False] * 150), # bool importances_gain = clf.feature_importances_
"E": pd.Categorical(np.random.permutation(['z', 'y', 'x', 'w', 'v'] * 60), # Test that the largest element is NOT the same, the smallest can be the same, i.e. zero
ordered=True)}) # str and ordered categorical importance_split_top1 = sorted(importances_split, reverse=True)[0]
y = np.random.permutation([0, 1] * 150) importance_gain_top1 = sorted(importances_gain, reverse=True)[0]
X_test = pd.DataFrame({"A": np.random.permutation(['a', 'b', 'e'] * 20), # unseen category assert importance_split_top1 != importance_gain_top1
"B": np.random.permutation([1, 3] * 30),
"C": np.random.permutation([0.1, -0.1, 0.2, 0.2] * 15),
"D": np.random.permutation([True, False] * 30), def test_pandas_categorical():
"E": pd.Categorical(np.random.permutation(['z', 'y'] * 30), pd = pytest.importorskip("pandas")
ordered=True)}) np.random.seed(42) # sometimes there is no difference how cols are treated (cat or not cat)
np.random.seed() # reset seed X = pd.DataFrame({"A": np.random.permutation(['a', 'b', 'c', 'd'] * 75), # str
cat_cols_actual = ["A", "B", "C", "D"] "B": np.random.permutation([1, 2, 3] * 100), # int
cat_cols_to_store = cat_cols_actual + ["E"] "C": np.random.permutation([0.1, 0.2, -0.1, -0.1, 0.2] * 60), # float
X[cat_cols_actual] = X[cat_cols_actual].astype('category') "D": np.random.permutation([True, False] * 150), # bool
X_test[cat_cols_actual] = X_test[cat_cols_actual].astype('category') "E": pd.Categorical(np.random.permutation(['z', 'y', 'x', 'w', 'v'] * 60),
cat_values = [X[col].cat.categories.tolist() for col in cat_cols_to_store] ordered=True)}) # str and ordered categorical
gbm0 = lgb.sklearn.LGBMClassifier(n_estimators=10).fit(X, y) y = np.random.permutation([0, 1] * 150)
pred0 = gbm0.predict(X_test, raw_score=True) X_test = pd.DataFrame({"A": np.random.permutation(['a', 'b', 'e'] * 20), # unseen category
pred_prob = gbm0.predict_proba(X_test)[:, 1] "B": np.random.permutation([1, 3] * 30),
gbm1 = lgb.sklearn.LGBMClassifier(n_estimators=10).fit(X, pd.Series(y), categorical_feature=[0]) "C": np.random.permutation([0.1, -0.1, 0.2, 0.2] * 15),
pred1 = gbm1.predict(X_test, raw_score=True) "D": np.random.permutation([True, False] * 30),
gbm2 = lgb.sklearn.LGBMClassifier(n_estimators=10).fit(X, y, categorical_feature=['A']) "E": pd.Categorical(np.random.permutation(['z', 'y'] * 30),
pred2 = gbm2.predict(X_test, raw_score=True) ordered=True)})
gbm3 = lgb.sklearn.LGBMClassifier(n_estimators=10).fit(X, y, categorical_feature=['A', 'B', 'C', 'D']) np.random.seed() # reset seed
pred3 = gbm3.predict(X_test, raw_score=True) cat_cols_actual = ["A", "B", "C", "D"]
gbm3.booster_.save_model('categorical.model') cat_cols_to_store = cat_cols_actual + ["E"]
gbm4 = lgb.Booster(model_file='categorical.model') X[cat_cols_actual] = X[cat_cols_actual].astype('category')
pred4 = gbm4.predict(X_test) X_test[cat_cols_actual] = X_test[cat_cols_actual].astype('category')
gbm5 = lgb.sklearn.LGBMClassifier(n_estimators=10).fit(X, y, categorical_feature=['A', 'B', 'C', 'D', 'E']) cat_values = [X[col].cat.categories.tolist() for col in cat_cols_to_store]
pred5 = gbm5.predict(X_test, raw_score=True) gbm0 = lgb.sklearn.LGBMClassifier(n_estimators=10).fit(X, y)
gbm6 = lgb.sklearn.LGBMClassifier(n_estimators=10).fit(X, y, categorical_feature=[]) pred0 = gbm0.predict(X_test, raw_score=True)
pred6 = gbm6.predict(X_test, raw_score=True) pred_prob = gbm0.predict_proba(X_test)[:, 1]
self.assertRaises(AssertionError, gbm1 = lgb.sklearn.LGBMClassifier(n_estimators=10).fit(X, pd.Series(y), categorical_feature=[0])
np.testing.assert_allclose, pred1 = gbm1.predict(X_test, raw_score=True)
pred0, pred1) gbm2 = lgb.sklearn.LGBMClassifier(n_estimators=10).fit(X, y, categorical_feature=['A'])
self.assertRaises(AssertionError, pred2 = gbm2.predict(X_test, raw_score=True)
np.testing.assert_allclose, gbm3 = lgb.sklearn.LGBMClassifier(n_estimators=10).fit(X, y, categorical_feature=['A', 'B', 'C', 'D'])
pred0, pred2) pred3 = gbm3.predict(X_test, raw_score=True)
np.testing.assert_allclose(pred1, pred2) gbm3.booster_.save_model('categorical.model')
np.testing.assert_allclose(pred0, pred3) gbm4 = lgb.Booster(model_file='categorical.model')
np.testing.assert_allclose(pred_prob, pred4) pred4 = gbm4.predict(X_test)
self.assertRaises(AssertionError, gbm5 = lgb.sklearn.LGBMClassifier(n_estimators=10).fit(X, y, categorical_feature=['A', 'B', 'C', 'D', 'E'])
np.testing.assert_allclose, pred5 = gbm5.predict(X_test, raw_score=True)
pred0, pred5) # ordered cat features aren't treated as cat features by default gbm6 = lgb.sklearn.LGBMClassifier(n_estimators=10).fit(X, y, categorical_feature=[])
self.assertRaises(AssertionError, pred6 = gbm6.predict(X_test, raw_score=True)
np.testing.assert_allclose, with pytest.raises(AssertionError):
pred0, pred6) np.testing.assert_allclose(pred0, pred1)
self.assertListEqual(gbm0.booster_.pandas_categorical, cat_values) with pytest.raises(AssertionError):
self.assertListEqual(gbm1.booster_.pandas_categorical, cat_values) np.testing.assert_allclose(pred0, pred2)
self.assertListEqual(gbm2.booster_.pandas_categorical, cat_values) np.testing.assert_allclose(pred1, pred2)
self.assertListEqual(gbm3.booster_.pandas_categorical, cat_values) np.testing.assert_allclose(pred0, pred3)
self.assertListEqual(gbm4.pandas_categorical, cat_values) np.testing.assert_allclose(pred_prob, pred4)
self.assertListEqual(gbm5.booster_.pandas_categorical, cat_values) with pytest.raises(AssertionError):
self.assertListEqual(gbm6.booster_.pandas_categorical, cat_values) np.testing.assert_allclose(pred0, pred5) # ordered cat features aren't treated as cat features by default
with pytest.raises(AssertionError):
@unittest.skipIf(not lgb.compat.PANDAS_INSTALLED, 'pandas is not installed') np.testing.assert_allclose(pred0, pred6)
def test_pandas_sparse(self): assert gbm0.booster_.pandas_categorical == cat_values
import pandas as pd assert gbm1.booster_.pandas_categorical == cat_values
try: assert gbm2.booster_.pandas_categorical == cat_values
from pandas.arrays import SparseArray assert gbm3.booster_.pandas_categorical == cat_values
except ImportError: # support old versions assert gbm4.pandas_categorical == cat_values
from pandas import SparseArray assert gbm5.booster_.pandas_categorical == cat_values
X = pd.DataFrame({"A": SparseArray(np.random.permutation([0, 1, 2] * 100)), assert gbm6.booster_.pandas_categorical == cat_values
"B": SparseArray(np.random.permutation([0.0, 0.1, 0.2, -0.1, 0.2] * 60)),
"C": SparseArray(np.random.permutation([True, False] * 150))})
y = pd.Series(SparseArray(np.random.permutation([0, 1] * 150))) def test_pandas_sparse():
X_test = pd.DataFrame({"A": SparseArray(np.random.permutation([0, 2] * 30)), pd = pytest.importorskip("pandas")
"B": SparseArray(np.random.permutation([0.0, 0.1, 0.2, -0.1] * 15)), try:
"C": SparseArray(np.random.permutation([True, False] * 30))}) from pandas.arrays import SparseArray
if pd.__version__ >= '0.24.0': except ImportError: # support old versions
for dtype in pd.concat([X.dtypes, X_test.dtypes, pd.Series(y.dtypes)]): from pandas import SparseArray
self.assertTrue(pd.api.types.is_sparse(dtype)) X = pd.DataFrame({"A": SparseArray(np.random.permutation([0, 1, 2] * 100)),
gbm = lgb.sklearn.LGBMClassifier(n_estimators=10).fit(X, y) "B": SparseArray(np.random.permutation([0.0, 0.1, 0.2, -0.1, 0.2] * 60)),
pred_sparse = gbm.predict(X_test, raw_score=True) "C": SparseArray(np.random.permutation([True, False] * 150))})
if hasattr(X_test, 'sparse'): y = pd.Series(SparseArray(np.random.permutation([0, 1] * 150)))
pred_dense = gbm.predict(X_test.sparse.to_dense(), raw_score=True) X_test = pd.DataFrame({"A": SparseArray(np.random.permutation([0, 2] * 30)),
else: "B": SparseArray(np.random.permutation([0.0, 0.1, 0.2, -0.1] * 15)),
pred_dense = gbm.predict(X_test.to_dense(), raw_score=True) "C": SparseArray(np.random.permutation([True, False] * 30))})
np.testing.assert_allclose(pred_sparse, pred_dense) if pd.__version__ >= '0.24.0':
for dtype in pd.concat([X.dtypes, X_test.dtypes, pd.Series(y.dtypes)]):
def test_predict(self): assert pd.api.types.is_sparse(dtype)
# With default params gbm = lgb.sklearn.LGBMClassifier(n_estimators=10).fit(X, y)
iris = load_iris(return_X_y=False) pred_sparse = gbm.predict(X_test, raw_score=True)
X_train, X_test, y_train, y_test = train_test_split(iris.data, iris.target, if hasattr(X_test, 'sparse'):
test_size=0.2, random_state=42) pred_dense = gbm.predict(X_test.sparse.to_dense(), raw_score=True)
else:
gbm = lgb.train({'objective': 'multiclass', pred_dense = gbm.predict(X_test.to_dense(), raw_score=True)
'num_class': 3, np.testing.assert_allclose(pred_sparse, pred_dense)
'verbose': -1},
lgb.Dataset(X_train, y_train))
clf = lgb.LGBMClassifier(verbose=-1).fit(X_train, y_train) def test_predict():
# With default params
# Tests same probabilities iris = load_iris(return_X_y=False)
res_engine = gbm.predict(X_test) X_train, X_test, y_train, y_test = train_test_split(iris.data, iris.target,
res_sklearn = clf.predict_proba(X_test) test_size=0.2, random_state=42)
np.testing.assert_allclose(res_engine, res_sklearn)
gbm = lgb.train({'objective': 'multiclass',
# Tests same predictions 'num_class': 3,
res_engine = np.argmax(gbm.predict(X_test), axis=1) 'verbose': -1},
res_sklearn = clf.predict(X_test) lgb.Dataset(X_train, y_train))
np.testing.assert_equal(res_engine, res_sklearn) clf = lgb.LGBMClassifier(verbose=-1).fit(X_train, y_train)
# Tests same raw scores # Tests same probabilities
res_engine = gbm.predict(X_test, raw_score=True) res_engine = gbm.predict(X_test)
res_sklearn = clf.predict(X_test, raw_score=True) res_sklearn = clf.predict_proba(X_test)
np.testing.assert_allclose(res_engine, res_sklearn) np.testing.assert_allclose(res_engine, res_sklearn)
# Tests same leaf indices # Tests same predictions
res_engine = gbm.predict(X_test, pred_leaf=True) res_engine = np.argmax(gbm.predict(X_test), axis=1)
res_sklearn = clf.predict(X_test, pred_leaf=True) res_sklearn = clf.predict(X_test)
np.testing.assert_equal(res_engine, res_sklearn) np.testing.assert_equal(res_engine, res_sklearn)
# Tests same feature contributions # Tests same raw scores
res_engine = gbm.predict(X_test, pred_contrib=True) res_engine = gbm.predict(X_test, raw_score=True)
res_sklearn = clf.predict(X_test, pred_contrib=True) res_sklearn = clf.predict(X_test, raw_score=True)
np.testing.assert_allclose(res_engine, res_sklearn) np.testing.assert_allclose(res_engine, res_sklearn)
# Tests other parameters for the prediction works # Tests same leaf indices
res_engine = gbm.predict(X_test) res_engine = gbm.predict(X_test, pred_leaf=True)
res_sklearn_params = clf.predict_proba(X_test, res_sklearn = clf.predict(X_test, pred_leaf=True)
pred_early_stop=True, np.testing.assert_equal(res_engine, res_sklearn)
pred_early_stop_margin=1.0)
self.assertRaises(AssertionError, # Tests same feature contributions
np.testing.assert_allclose, res_engine = gbm.predict(X_test, pred_contrib=True)
res_engine, res_sklearn_params) res_sklearn = clf.predict(X_test, pred_contrib=True)
np.testing.assert_allclose(res_engine, res_sklearn)
# Tests start_iteration
# Tests same probabilities, starting from iteration 10 # Tests other parameters for the prediction works
res_engine = gbm.predict(X_test, start_iteration=10) res_engine = gbm.predict(X_test)
res_sklearn = clf.predict_proba(X_test, start_iteration=10) res_sklearn_params = clf.predict_proba(X_test,
np.testing.assert_allclose(res_engine, res_sklearn) pred_early_stop=True,
pred_early_stop_margin=1.0)
# Tests same predictions, starting from iteration 10 with pytest.raises(AssertionError):
res_engine = np.argmax(gbm.predict(X_test, start_iteration=10), axis=1) np.testing.assert_allclose(res_engine, res_sklearn_params)
res_sklearn = clf.predict(X_test, start_iteration=10)
np.testing.assert_equal(res_engine, res_sklearn) # Tests start_iteration
# Tests same probabilities, starting from iteration 10
# Tests same raw scores, starting from iteration 10 res_engine = gbm.predict(X_test, start_iteration=10)
res_engine = gbm.predict(X_test, raw_score=True, start_iteration=10) res_sklearn = clf.predict_proba(X_test, start_iteration=10)
res_sklearn = clf.predict(X_test, raw_score=True, start_iteration=10) np.testing.assert_allclose(res_engine, res_sklearn)
np.testing.assert_allclose(res_engine, res_sklearn)
# Tests same predictions, starting from iteration 10
# Tests same leaf indices, starting from iteration 10 res_engine = np.argmax(gbm.predict(X_test, start_iteration=10), axis=1)
res_engine = gbm.predict(X_test, pred_leaf=True, start_iteration=10) res_sklearn = clf.predict(X_test, start_iteration=10)
res_sklearn = clf.predict(X_test, pred_leaf=True, start_iteration=10) np.testing.assert_equal(res_engine, res_sklearn)
np.testing.assert_equal(res_engine, res_sklearn)
# Tests same raw scores, starting from iteration 10
# Tests same feature contributions, starting from iteration 10 res_engine = gbm.predict(X_test, raw_score=True, start_iteration=10)
res_engine = gbm.predict(X_test, pred_contrib=True, start_iteration=10) res_sklearn = clf.predict(X_test, raw_score=True, start_iteration=10)
res_sklearn = clf.predict(X_test, pred_contrib=True, start_iteration=10) np.testing.assert_allclose(res_engine, res_sklearn)
np.testing.assert_allclose(res_engine, res_sklearn)
# Tests same leaf indices, starting from iteration 10
# Tests other parameters for the prediction works, starting from iteration 10 res_engine = gbm.predict(X_test, pred_leaf=True, start_iteration=10)
res_engine = gbm.predict(X_test, start_iteration=10) res_sklearn = clf.predict(X_test, pred_leaf=True, start_iteration=10)
res_sklearn_params = clf.predict_proba(X_test, np.testing.assert_equal(res_engine, res_sklearn)
pred_early_stop=True,
pred_early_stop_margin=1.0, start_iteration=10) # Tests same feature contributions, starting from iteration 10
self.assertRaises(AssertionError, res_engine = gbm.predict(X_test, pred_contrib=True, start_iteration=10)
np.testing.assert_allclose, res_sklearn = clf.predict(X_test, pred_contrib=True, start_iteration=10)
res_engine, res_sklearn_params) np.testing.assert_allclose(res_engine, res_sklearn)
def test_evaluate_train_set(self): # Tests other parameters for the prediction works, starting from iteration 10
X, y = load_boston(return_X_y=True) res_engine = gbm.predict(X_test, start_iteration=10)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42) res_sklearn_params = clf.predict_proba(X_test,
gbm = lgb.LGBMRegressor(n_estimators=10, silent=True) pred_early_stop=True,
gbm.fit(X_train, y_train, eval_set=[(X_train, y_train), (X_test, y_test)], verbose=False) pred_early_stop_margin=1.0, start_iteration=10)
self.assertEqual(len(gbm.evals_result_), 2) with pytest.raises(AssertionError):
self.assertIn('training', gbm.evals_result_) np.testing.assert_allclose(res_engine, res_sklearn_params)
self.assertEqual(len(gbm.evals_result_['training']), 1)
self.assertIn('l2', gbm.evals_result_['training'])
self.assertIn('valid_1', gbm.evals_result_) def test_evaluate_train_set():
self.assertEqual(len(gbm.evals_result_['valid_1']), 1) X, y = load_boston(return_X_y=True)
self.assertIn('l2', gbm.evals_result_['valid_1']) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
gbm = lgb.LGBMRegressor(n_estimators=10, silent=True)
def test_metrics(self): gbm.fit(X_train, y_train, eval_set=[(X_train, y_train), (X_test, y_test)], verbose=False)
X, y = load_boston(return_X_y=True) assert len(gbm.evals_result_) == 2
params = {'n_estimators': 2, 'verbose': -1} assert 'training' in gbm.evals_result_
params_fit = {'X': X, 'y': y, 'eval_set': (X, y), 'verbose': False} assert len(gbm.evals_result_['training']) == 1
assert 'l2' in gbm.evals_result_['training']
# no custom objective, no custom metric assert 'valid_1' in gbm.evals_result_
# default metric assert len(gbm.evals_result_['valid_1']) == 1
assert 'l2' in gbm.evals_result_['valid_1']
def test_metrics():
X, y = load_boston(return_X_y=True)
params = {'n_estimators': 2, 'verbose': -1}
params_fit = {'X': X, 'y': y, 'eval_set': (X, y), 'verbose': False}
# no custom objective, no custom metric
# default metric
gbm = lgb.LGBMRegressor(**params).fit(**params_fit)
assert len(gbm.evals_result_['training']) == 1
assert 'l2' in gbm.evals_result_['training']
# non-default metric
gbm = lgb.LGBMRegressor(metric='mape', **params).fit(**params_fit)
assert len(gbm.evals_result_['training']) == 1
assert 'mape' in gbm.evals_result_['training']
# no metric
gbm = lgb.LGBMRegressor(metric='None', **params).fit(**params_fit)
assert gbm.evals_result_ is None
# non-default metric in eval_metric
gbm = lgb.LGBMRegressor(**params).fit(eval_metric='mape', **params_fit)
assert len(gbm.evals_result_['training']) == 2
assert 'l2' in gbm.evals_result_['training']
assert 'mape' in gbm.evals_result_['training']
# non-default metric with non-default metric in eval_metric
gbm = lgb.LGBMRegressor(metric='gamma', **params).fit(eval_metric='mape', **params_fit)
assert len(gbm.evals_result_['training']) == 2
assert 'gamma' in gbm.evals_result_['training']
assert 'mape' in gbm.evals_result_['training']
# non-default metric with multiple metrics in eval_metric
gbm = lgb.LGBMRegressor(metric='gamma',
**params).fit(eval_metric=['l2', 'mape'], **params_fit)
assert len(gbm.evals_result_['training']) == 3
assert 'gamma' in gbm.evals_result_['training']
assert 'l2' in gbm.evals_result_['training']
assert 'mape' in gbm.evals_result_['training']
# non-default metric with multiple metrics in eval_metric for LGBMClassifier
X_classification, y_classification = load_breast_cancer(return_X_y=True)
params_classification = {'n_estimators': 2, 'verbose': -1,
'objective': 'binary', 'metric': 'binary_logloss'}
params_fit_classification = {'X': X_classification, 'y': y_classification,
'eval_set': (X_classification, y_classification),
'verbose': False}
gbm = lgb.LGBMClassifier(**params_classification).fit(eval_metric=['fair', 'error'],
**params_fit_classification)
assert len(gbm.evals_result_['training']) == 3
assert 'fair' in gbm.evals_result_['training']
assert 'binary_error' in gbm.evals_result_['training']
assert 'binary_logloss' in gbm.evals_result_['training']
# default metric for non-default objective
gbm = lgb.LGBMRegressor(objective='regression_l1', **params).fit(**params_fit)
assert len(gbm.evals_result_['training']) == 1
assert 'l1' in gbm.evals_result_['training']
# non-default metric for non-default objective
gbm = lgb.LGBMRegressor(objective='regression_l1', metric='mape',
**params).fit(**params_fit)
assert len(gbm.evals_result_['training']) == 1
assert 'mape' in gbm.evals_result_['training']
# no metric
gbm = lgb.LGBMRegressor(objective='regression_l1', metric='None',
**params).fit(**params_fit)
assert gbm.evals_result_ is None
# non-default metric in eval_metric for non-default objective
gbm = lgb.LGBMRegressor(objective='regression_l1',
**params).fit(eval_metric='mape', **params_fit)
assert len(gbm.evals_result_['training']) == 2
assert 'l1' in gbm.evals_result_['training']
assert 'mape' in gbm.evals_result_['training']
# non-default metric with non-default metric in eval_metric for non-default objective
gbm = lgb.LGBMRegressor(objective='regression_l1', metric='gamma',
**params).fit(eval_metric='mape', **params_fit)
assert len(gbm.evals_result_['training']) == 2
assert 'gamma' in gbm.evals_result_['training']
assert 'mape' in gbm.evals_result_['training']
# non-default metric with multiple metrics in eval_metric for non-default objective
gbm = lgb.LGBMRegressor(objective='regression_l1', metric='gamma',
**params).fit(eval_metric=['l2', 'mape'], **params_fit)
assert len(gbm.evals_result_['training']) == 3
assert 'gamma' in gbm.evals_result_['training']
assert 'l2' in gbm.evals_result_['training']
assert 'mape' in gbm.evals_result_['training']
# custom objective, no custom metric
# default regression metric for custom objective
gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, **params).fit(**params_fit)
assert len(gbm.evals_result_['training']) == 1
assert 'l2' in gbm.evals_result_['training']
# non-default regression metric for custom objective
gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric='mape', **params).fit(**params_fit)
assert len(gbm.evals_result_['training']) == 1
assert 'mape' in gbm.evals_result_['training']
# multiple regression metrics for custom objective
gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric=['l1', 'gamma'],
**params).fit(**params_fit)
assert len(gbm.evals_result_['training']) == 2
assert 'l1' in gbm.evals_result_['training']
assert 'gamma' in gbm.evals_result_['training']
# no metric
gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric='None',
**params).fit(**params_fit)
assert gbm.evals_result_ is None
# default regression metric with non-default metric in eval_metric for custom objective
gbm = lgb.LGBMRegressor(objective=custom_dummy_obj,
**params).fit(eval_metric='mape', **params_fit)
assert len(gbm.evals_result_['training']) == 2
assert 'l2' in gbm.evals_result_['training']
assert 'mape' in gbm.evals_result_['training']
# non-default regression metric with metric in eval_metric for custom objective
gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric='mape',
**params).fit(eval_metric='gamma', **params_fit)
assert len(gbm.evals_result_['training']) == 2
assert 'mape' in gbm.evals_result_['training']
assert 'gamma' in gbm.evals_result_['training']
# multiple regression metrics with metric in eval_metric for custom objective
gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric=['l1', 'gamma'],
**params).fit(eval_metric='l2', **params_fit)
assert len(gbm.evals_result_['training']) == 3
assert 'l1' in gbm.evals_result_['training']
assert 'gamma' in gbm.evals_result_['training']
assert 'l2' in gbm.evals_result_['training']
# multiple regression metrics with multiple metrics in eval_metric for custom objective
gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric=['l1', 'gamma'],
**params).fit(eval_metric=['l2', 'mape'], **params_fit)
assert len(gbm.evals_result_['training']) == 4
assert 'l1' in gbm.evals_result_['training']
assert 'gamma' in gbm.evals_result_['training']
assert 'l2' in gbm.evals_result_['training']
assert 'mape' in gbm.evals_result_['training']
# no custom objective, custom metric
# default metric with custom metric
gbm = lgb.LGBMRegressor(**params).fit(eval_metric=constant_metric, **params_fit)
assert len(gbm.evals_result_['training']) == 2
assert 'l2' in gbm.evals_result_['training']
assert 'error' in gbm.evals_result_['training']
# non-default metric with custom metric
gbm = lgb.LGBMRegressor(metric='mape',
**params).fit(eval_metric=constant_metric, **params_fit)
assert len(gbm.evals_result_['training']) == 2
assert 'mape' in gbm.evals_result_['training']
assert 'error' in gbm.evals_result_['training']
# multiple metrics with custom metric
gbm = lgb.LGBMRegressor(metric=['l1', 'gamma'],
**params).fit(eval_metric=constant_metric, **params_fit)
assert len(gbm.evals_result_['training']) == 3
assert 'l1' in gbm.evals_result_['training']
assert 'gamma' in gbm.evals_result_['training']
assert 'error' in gbm.evals_result_['training']
# custom metric (disable default metric)
gbm = lgb.LGBMRegressor(metric='None',
**params).fit(eval_metric=constant_metric, **params_fit)
assert len(gbm.evals_result_['training']) == 1
assert 'error' in gbm.evals_result_['training']
# default metric for non-default objective with custom metric
gbm = lgb.LGBMRegressor(objective='regression_l1',
**params).fit(eval_metric=constant_metric, **params_fit)
assert len(gbm.evals_result_['training']) == 2
assert 'l1' in gbm.evals_result_['training']
assert 'error' in gbm.evals_result_['training']
# non-default metric for non-default objective with custom metric
gbm = lgb.LGBMRegressor(objective='regression_l1', metric='mape',
**params).fit(eval_metric=constant_metric, **params_fit)
assert len(gbm.evals_result_['training']) == 2
assert 'mape' in gbm.evals_result_['training']
assert 'error' in gbm.evals_result_['training']
# multiple metrics for non-default objective with custom metric
gbm = lgb.LGBMRegressor(objective='regression_l1', metric=['l1', 'gamma'],
**params).fit(eval_metric=constant_metric, **params_fit)
assert len(gbm.evals_result_['training']) == 3
assert 'l1' in gbm.evals_result_['training']
assert 'gamma' in gbm.evals_result_['training']
assert 'error' in gbm.evals_result_['training']
# custom metric (disable default metric for non-default objective)
gbm = lgb.LGBMRegressor(objective='regression_l1', metric='None',
**params).fit(eval_metric=constant_metric, **params_fit)
assert len(gbm.evals_result_['training']) == 1
assert 'error' in gbm.evals_result_['training']
# custom objective, custom metric
# custom metric for custom objective
gbm = lgb.LGBMRegressor(objective=custom_dummy_obj,
**params).fit(eval_metric=constant_metric, **params_fit)
assert len(gbm.evals_result_['training']) == 2
assert 'error' in gbm.evals_result_['training']
# non-default regression metric with custom metric for custom objective
gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric='mape',
**params).fit(eval_metric=constant_metric, **params_fit)
assert len(gbm.evals_result_['training']) == 2
assert 'mape' in gbm.evals_result_['training']
assert 'error' in gbm.evals_result_['training']
# multiple regression metrics with custom metric for custom objective
gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric=['l2', 'mape'],
**params).fit(eval_metric=constant_metric, **params_fit)
assert len(gbm.evals_result_['training']) == 3
assert 'l2' in gbm.evals_result_['training']
assert 'mape' in gbm.evals_result_['training']
assert 'error' in gbm.evals_result_['training']
X, y = load_digits(n_class=3, return_X_y=True)
params_fit = {'X': X, 'y': y, 'eval_set': (X, y), 'verbose': False}
# default metric and invalid binary metric is replaced with multiclass alternative
gbm = lgb.LGBMClassifier(**params).fit(eval_metric='binary_error', **params_fit)
assert len(gbm.evals_result_['training']) == 2
assert 'multi_logloss' in gbm.evals_result_['training']
assert 'multi_error' in gbm.evals_result_['training']
# invalid objective is replaced with default multiclass one
# and invalid binary metric is replaced with multiclass alternative
gbm = lgb.LGBMClassifier(objective='invalid_obj',
**params).fit(eval_metric='binary_error', **params_fit)
assert gbm.objective_ == 'multiclass'
assert len(gbm.evals_result_['training']) == 2
assert 'multi_logloss' in gbm.evals_result_['training']
assert 'multi_error' in gbm.evals_result_['training']
# default metric for non-default multiclass objective
# and invalid binary metric is replaced with multiclass alternative
gbm = lgb.LGBMClassifier(objective='ovr',
**params).fit(eval_metric='binary_error', **params_fit)
assert gbm.objective_ == 'ovr'
assert len(gbm.evals_result_['training']) == 2
assert 'multi_logloss' in gbm.evals_result_['training']
assert 'multi_error' in gbm.evals_result_['training']
X, y = load_digits(n_class=2, return_X_y=True)
params_fit = {'X': X, 'y': y, 'eval_set': (X, y), 'verbose': False}
# default metric and invalid multiclass metric is replaced with binary alternative
gbm = lgb.LGBMClassifier(**params).fit(eval_metric='multi_error', **params_fit)
assert len(gbm.evals_result_['training']) == 2
assert 'binary_logloss' in gbm.evals_result_['training']
assert 'binary_error' in gbm.evals_result_['training']
# invalid multiclass metric is replaced with binary alternative for custom objective
gbm = lgb.LGBMClassifier(objective=custom_dummy_obj,
**params).fit(eval_metric='multi_logloss', **params_fit)
assert len(gbm.evals_result_['training']) == 1
assert 'binary_logloss' in gbm.evals_result_['training']
def test_multiple_eval_metrics():
X, y = load_breast_cancer(return_X_y=True)
params = {'n_estimators': 2, 'verbose': -1, 'objective': 'binary', 'metric': 'binary_logloss'}
params_fit = {'X': X, 'y': y, 'eval_set': (X, y), 'verbose': False}
# Verify that can receive a list of metrics, only callable
gbm = lgb.LGBMClassifier(**params).fit(eval_metric=[constant_metric, decreasing_metric], **params_fit)
assert len(gbm.evals_result_['training']) == 3
assert 'error' in gbm.evals_result_['training']
assert 'decreasing_metric' in gbm.evals_result_['training']
assert 'binary_logloss' in gbm.evals_result_['training']
# Verify that can receive a list of custom and built-in metrics
gbm = lgb.LGBMClassifier(**params).fit(eval_metric=[constant_metric, decreasing_metric, 'fair'], **params_fit)
assert len(gbm.evals_result_['training']) == 4
assert 'error' in gbm.evals_result_['training']
assert 'decreasing_metric' in gbm.evals_result_['training']
assert 'binary_logloss' in gbm.evals_result_['training']
assert 'fair' in gbm.evals_result_['training']
# Verify that works as expected when eval_metric is empty
gbm = lgb.LGBMClassifier(**params).fit(eval_metric=[], **params_fit)
assert len(gbm.evals_result_['training']) == 1
assert 'binary_logloss' in gbm.evals_result_['training']
# Verify that can receive a list of metrics, only built-in
gbm = lgb.LGBMClassifier(**params).fit(eval_metric=['fair', 'error'], **params_fit)
assert len(gbm.evals_result_['training']) == 3
assert 'binary_logloss' in gbm.evals_result_['training']
# Verify that eval_metric is robust to receiving a list with None
gbm = lgb.LGBMClassifier(**params).fit(eval_metric=['fair', 'error', None], **params_fit)
assert len(gbm.evals_result_['training']) == 3
assert 'binary_logloss' in gbm.evals_result_['training']
def test_inf_handle():
nrows = 100
ncols = 10
X = np.random.randn(nrows, ncols)
y = np.random.randn(nrows) + np.full(nrows, 1e30)
weight = np.full(nrows, 1e10)
params = {'n_estimators': 20, 'verbose': -1}
params_fit = {'X': X, 'y': y, 'sample_weight': weight, 'eval_set': (X, y),
'verbose': False, 'early_stopping_rounds': 5}
gbm = lgb.LGBMRegressor(**params).fit(**params_fit)
np.testing.assert_allclose(gbm.evals_result_['training']['l2'], np.inf)
def test_nan_handle():
nrows = 100
ncols = 10
X = np.random.randn(nrows, ncols)
y = np.random.randn(nrows) + np.full(nrows, 1e30)
weight = np.zeros(nrows)
params = {'n_estimators': 20, 'verbose': -1}
params_fit = {'X': X, 'y': y, 'sample_weight': weight, 'eval_set': (X, y),
'verbose': False, 'early_stopping_rounds': 5}
gbm = lgb.LGBMRegressor(**params).fit(**params_fit)
np.testing.assert_allclose(gbm.evals_result_['training']['l2'], np.nan)
def test_first_metric_only():
def fit_and_check(eval_set_names, metric_names, assumed_iteration, first_metric_only):
params['first_metric_only'] = first_metric_only
gbm = lgb.LGBMRegressor(**params).fit(**params_fit) gbm = lgb.LGBMRegressor(**params).fit(**params_fit)
self.assertEqual(len(gbm.evals_result_['training']), 1) assert len(gbm.evals_result_) == len(eval_set_names)
self.assertIn('l2', gbm.evals_result_['training']) for eval_set_name in eval_set_names:
assert eval_set_name in gbm.evals_result_
# non-default metric assert len(gbm.evals_result_[eval_set_name]) == len(metric_names)
gbm = lgb.LGBMRegressor(metric='mape', **params).fit(**params_fit) for metric_name in metric_names:
self.assertEqual(len(gbm.evals_result_['training']), 1) assert metric_name in gbm.evals_result_[eval_set_name]
self.assertIn('mape', gbm.evals_result_['training'])
actual = len(gbm.evals_result_[eval_set_name][metric_name])
# no metric expected = assumed_iteration + (params_fit['early_stopping_rounds']
gbm = lgb.LGBMRegressor(metric='None', **params).fit(**params_fit) if eval_set_name != 'training'
self.assertIs(gbm.evals_result_, None) and assumed_iteration != gbm.n_estimators else 0)
if eval_set_name != 'training':
# non-default metric in eval_metric assert assumed_iteration == gbm.best_iteration_
gbm = lgb.LGBMRegressor(**params).fit(eval_metric='mape', **params_fit) else:
self.assertEqual(len(gbm.evals_result_['training']), 2) assert gbm.n_estimators == gbm.best_iteration_
self.assertIn('l2', gbm.evals_result_['training'])
self.assertIn('mape', gbm.evals_result_['training']) X, y = load_boston(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
# non-default metric with non-default metric in eval_metric X_test1, X_test2, y_test1, y_test2 = train_test_split(X_test, y_test, test_size=0.5, random_state=72)
gbm = lgb.LGBMRegressor(metric='gamma', **params).fit(eval_metric='mape', **params_fit) params = {'n_estimators': 30,
self.assertEqual(len(gbm.evals_result_['training']), 2) 'learning_rate': 0.8,
self.assertIn('gamma', gbm.evals_result_['training']) 'num_leaves': 15,
self.assertIn('mape', gbm.evals_result_['training']) 'verbose': -1,
'seed': 123}
# non-default metric with multiple metrics in eval_metric params_fit = {'X': X_train,
gbm = lgb.LGBMRegressor(metric='gamma', 'y': y_train,
**params).fit(eval_metric=['l2', 'mape'], **params_fit) 'early_stopping_rounds': 5,
self.assertEqual(len(gbm.evals_result_['training']), 3) 'verbose': False}
self.assertIn('gamma', gbm.evals_result_['training'])
self.assertIn('l2', gbm.evals_result_['training']) iter_valid1_l1 = 3
self.assertIn('mape', gbm.evals_result_['training']) iter_valid1_l2 = 18
iter_valid2_l1 = 11
# non-default metric with multiple metrics in eval_metric for LGBMClassifier iter_valid2_l2 = 7
X_classification, y_classification = load_breast_cancer(return_X_y=True) assert len(set([iter_valid1_l1, iter_valid1_l2, iter_valid2_l1, iter_valid2_l2])) == 4
params_classification = {'n_estimators': 2, 'verbose': -1, iter_min_l1 = min([iter_valid1_l1, iter_valid2_l1])
'objective': 'binary', 'metric': 'binary_logloss'} iter_min_l2 = min([iter_valid1_l2, iter_valid2_l2])
params_fit_classification = {'X': X_classification, 'y': y_classification, iter_min = min([iter_min_l1, iter_min_l2])
'eval_set': (X_classification, y_classification), iter_min_valid1 = min([iter_valid1_l1, iter_valid1_l2])
'verbose': False}
gbm = lgb.LGBMClassifier(**params_classification).fit(eval_metric=['fair', 'error'], # training data as eval_set
**params_fit_classification) params_fit['eval_set'] = (X_train, y_train)
self.assertEqual(len(gbm.evals_result_['training']), 3) fit_and_check(['training'], ['l2'], 30, False)
self.assertIn('fair', gbm.evals_result_['training']) fit_and_check(['training'], ['l2'], 30, True)
self.assertIn('binary_error', gbm.evals_result_['training'])
self.assertIn('binary_logloss', gbm.evals_result_['training']) # feval
params['metric'] = 'None'
# default metric for non-default objective params_fit['eval_metric'] = lambda preds, train_data: [decreasing_metric(preds, train_data),
gbm = lgb.LGBMRegressor(objective='regression_l1', **params).fit(**params_fit) constant_metric(preds, train_data)]
self.assertEqual(len(gbm.evals_result_['training']), 1) params_fit['eval_set'] = (X_test1, y_test1)
self.assertIn('l1', gbm.evals_result_['training']) fit_and_check(['valid_0'], ['decreasing_metric', 'error'], 1, False)
fit_and_check(['valid_0'], ['decreasing_metric', 'error'], 30, True)
# non-default metric for non-default objective params_fit['eval_metric'] = lambda preds, train_data: [constant_metric(preds, train_data),
gbm = lgb.LGBMRegressor(objective='regression_l1', metric='mape', decreasing_metric(preds, train_data)]
**params).fit(**params_fit) fit_and_check(['valid_0'], ['decreasing_metric', 'error'], 1, True)
self.assertEqual(len(gbm.evals_result_['training']), 1)
self.assertIn('mape', gbm.evals_result_['training']) # single eval_set
params.pop('metric')
# no metric params_fit.pop('eval_metric')
gbm = lgb.LGBMRegressor(objective='regression_l1', metric='None', fit_and_check(['valid_0'], ['l2'], iter_valid1_l2, False)
**params).fit(**params_fit) fit_and_check(['valid_0'], ['l2'], iter_valid1_l2, True)
self.assertIs(gbm.evals_result_, None)
params_fit['eval_metric'] = "l2"
# non-default metric in eval_metric for non-default objective fit_and_check(['valid_0'], ['l2'], iter_valid1_l2, False)
gbm = lgb.LGBMRegressor(objective='regression_l1', fit_and_check(['valid_0'], ['l2'], iter_valid1_l2, True)
**params).fit(eval_metric='mape', **params_fit)
self.assertEqual(len(gbm.evals_result_['training']), 2) params_fit['eval_metric'] = "l1"
self.assertIn('l1', gbm.evals_result_['training']) fit_and_check(['valid_0'], ['l1', 'l2'], iter_min_valid1, False)
self.assertIn('mape', gbm.evals_result_['training']) fit_and_check(['valid_0'], ['l1', 'l2'], iter_valid1_l1, True)
# non-default metric with non-default metric in eval_metric for non-default objective params_fit['eval_metric'] = ["l1", "l2"]
gbm = lgb.LGBMRegressor(objective='regression_l1', metric='gamma', fit_and_check(['valid_0'], ['l1', 'l2'], iter_min_valid1, False)
**params).fit(eval_metric='mape', **params_fit) fit_and_check(['valid_0'], ['l1', 'l2'], iter_valid1_l1, True)
self.assertEqual(len(gbm.evals_result_['training']), 2)
self.assertIn('gamma', gbm.evals_result_['training']) params_fit['eval_metric'] = ["l2", "l1"]
self.assertIn('mape', gbm.evals_result_['training']) fit_and_check(['valid_0'], ['l1', 'l2'], iter_min_valid1, False)
fit_and_check(['valid_0'], ['l1', 'l2'], iter_valid1_l2, True)
# non-default metric with multiple metrics in eval_metric for non-default objective
gbm = lgb.LGBMRegressor(objective='regression_l1', metric='gamma', params_fit['eval_metric'] = ["l2", "regression", "mse"] # test aliases
**params).fit(eval_metric=['l2', 'mape'], **params_fit) fit_and_check(['valid_0'], ['l2'], iter_valid1_l2, False)
self.assertEqual(len(gbm.evals_result_['training']), 3) fit_and_check(['valid_0'], ['l2'], iter_valid1_l2, True)
self.assertIn('gamma', gbm.evals_result_['training'])
self.assertIn('l2', gbm.evals_result_['training']) # two eval_set
self.assertIn('mape', gbm.evals_result_['training']) params_fit['eval_set'] = [(X_test1, y_test1), (X_test2, y_test2)]
params_fit['eval_metric'] = ["l1", "l2"]
# custom objective, no custom metric fit_and_check(['valid_0', 'valid_1'], ['l1', 'l2'], iter_min_l1, True)
# default regression metric for custom objective params_fit['eval_metric'] = ["l2", "l1"]
gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, **params).fit(**params_fit) fit_and_check(['valid_0', 'valid_1'], ['l1', 'l2'], iter_min_l2, True)
self.assertEqual(len(gbm.evals_result_['training']), 1)
self.assertIn('l2', gbm.evals_result_['training']) params_fit['eval_set'] = [(X_test2, y_test2), (X_test1, y_test1)]
params_fit['eval_metric'] = ["l1", "l2"]
# non-default regression metric for custom objective fit_and_check(['valid_0', 'valid_1'], ['l1', 'l2'], iter_min, False)
gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric='mape', **params).fit(**params_fit) fit_and_check(['valid_0', 'valid_1'], ['l1', 'l2'], iter_min_l1, True)
self.assertEqual(len(gbm.evals_result_['training']), 1) params_fit['eval_metric'] = ["l2", "l1"]
self.assertIn('mape', gbm.evals_result_['training']) fit_and_check(['valid_0', 'valid_1'], ['l1', 'l2'], iter_min, False)
fit_and_check(['valid_0', 'valid_1'], ['l1', 'l2'], iter_min_l2, True)
# multiple regression metrics for custom objective
gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric=['l1', 'gamma'],
**params).fit(**params_fit) def test_class_weight():
self.assertEqual(len(gbm.evals_result_['training']), 2) X, y = load_digits(n_class=10, return_X_y=True)
self.assertIn('l1', gbm.evals_result_['training']) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
self.assertIn('gamma', gbm.evals_result_['training']) y_train_str = y_train.astype('str')
y_test_str = y_test.astype('str')
# no metric gbm = lgb.LGBMClassifier(n_estimators=10, class_weight='balanced', silent=True)
gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric='None', gbm.fit(X_train, y_train,
**params).fit(**params_fit) eval_set=[(X_train, y_train), (X_test, y_test), (X_test, y_test),
self.assertIs(gbm.evals_result_, None) (X_test, y_test), (X_test, y_test)],
eval_class_weight=['balanced', None, 'balanced', {1: 10, 4: 20}, {5: 30, 2: 40}],
# default regression metric with non-default metric in eval_metric for custom objective verbose=False)
gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, for eval_set1, eval_set2 in itertools.combinations(gbm.evals_result_.keys(), 2):
**params).fit(eval_metric='mape', **params_fit) for metric in gbm.evals_result_[eval_set1]:
self.assertEqual(len(gbm.evals_result_['training']), 2) np.testing.assert_raises(AssertionError,
self.assertIn('l2', gbm.evals_result_['training']) np.testing.assert_allclose,
self.assertIn('mape', gbm.evals_result_['training']) gbm.evals_result_[eval_set1][metric],
gbm.evals_result_[eval_set2][metric])
# non-default regression metric with metric in eval_metric for custom objective gbm_str = lgb.LGBMClassifier(n_estimators=10, class_weight='balanced', silent=True)
gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric='mape', gbm_str.fit(X_train, y_train_str,
**params).fit(eval_metric='gamma', **params_fit) eval_set=[(X_train, y_train_str), (X_test, y_test_str),
self.assertEqual(len(gbm.evals_result_['training']), 2) (X_test, y_test_str), (X_test, y_test_str), (X_test, y_test_str)],
self.assertIn('mape', gbm.evals_result_['training']) eval_class_weight=['balanced', None, 'balanced', {'1': 10, '4': 20}, {'5': 30, '2': 40}],
self.assertIn('gamma', gbm.evals_result_['training'])
# multiple regression metrics with metric in eval_metric for custom objective
gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric=['l1', 'gamma'],
**params).fit(eval_metric='l2', **params_fit)
self.assertEqual(len(gbm.evals_result_['training']), 3)
self.assertIn('l1', gbm.evals_result_['training'])
self.assertIn('gamma', gbm.evals_result_['training'])
self.assertIn('l2', gbm.evals_result_['training'])
# multiple regression metrics with multiple metrics in eval_metric for custom objective
gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric=['l1', 'gamma'],
**params).fit(eval_metric=['l2', 'mape'], **params_fit)
self.assertEqual(len(gbm.evals_result_['training']), 4)
self.assertIn('l1', gbm.evals_result_['training'])
self.assertIn('gamma', gbm.evals_result_['training'])
self.assertIn('l2', gbm.evals_result_['training'])
self.assertIn('mape', gbm.evals_result_['training'])
# no custom objective, custom metric
# default metric with custom metric
gbm = lgb.LGBMRegressor(**params).fit(eval_metric=constant_metric, **params_fit)
self.assertEqual(len(gbm.evals_result_['training']), 2)
self.assertIn('l2', gbm.evals_result_['training'])
self.assertIn('error', gbm.evals_result_['training'])
# non-default metric with custom metric
gbm = lgb.LGBMRegressor(metric='mape',
**params).fit(eval_metric=constant_metric, **params_fit)
self.assertEqual(len(gbm.evals_result_['training']), 2)
self.assertIn('mape', gbm.evals_result_['training'])
self.assertIn('error', gbm.evals_result_['training'])
# multiple metrics with custom metric
gbm = lgb.LGBMRegressor(metric=['l1', 'gamma'],
**params).fit(eval_metric=constant_metric, **params_fit)
self.assertEqual(len(gbm.evals_result_['training']), 3)
self.assertIn('l1', gbm.evals_result_['training'])
self.assertIn('gamma', gbm.evals_result_['training'])
self.assertIn('error', gbm.evals_result_['training'])
# custom metric (disable default metric)
gbm = lgb.LGBMRegressor(metric='None',
**params).fit(eval_metric=constant_metric, **params_fit)
self.assertEqual(len(gbm.evals_result_['training']), 1)
self.assertIn('error', gbm.evals_result_['training'])
# default metric for non-default objective with custom metric
gbm = lgb.LGBMRegressor(objective='regression_l1',
**params).fit(eval_metric=constant_metric, **params_fit)
self.assertEqual(len(gbm.evals_result_['training']), 2)
self.assertIn('l1', gbm.evals_result_['training'])
self.assertIn('error', gbm.evals_result_['training'])
# non-default metric for non-default objective with custom metric
gbm = lgb.LGBMRegressor(objective='regression_l1', metric='mape',
**params).fit(eval_metric=constant_metric, **params_fit)
self.assertEqual(len(gbm.evals_result_['training']), 2)
self.assertIn('mape', gbm.evals_result_['training'])
self.assertIn('error', gbm.evals_result_['training'])
# multiple metrics for non-default objective with custom metric
gbm = lgb.LGBMRegressor(objective='regression_l1', metric=['l1', 'gamma'],
**params).fit(eval_metric=constant_metric, **params_fit)
self.assertEqual(len(gbm.evals_result_['training']), 3)
self.assertIn('l1', gbm.evals_result_['training'])
self.assertIn('gamma', gbm.evals_result_['training'])
self.assertIn('error', gbm.evals_result_['training'])
# custom metric (disable default metric for non-default objective)
gbm = lgb.LGBMRegressor(objective='regression_l1', metric='None',
**params).fit(eval_metric=constant_metric, **params_fit)
self.assertEqual(len(gbm.evals_result_['training']), 1)
self.assertIn('error', gbm.evals_result_['training'])
# custom objective, custom metric
# custom metric for custom objective
gbm = lgb.LGBMRegressor(objective=custom_dummy_obj,
**params).fit(eval_metric=constant_metric, **params_fit)
self.assertEqual(len(gbm.evals_result_['training']), 2)
self.assertIn('error', gbm.evals_result_['training'])
# non-default regression metric with custom metric for custom objective
gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric='mape',
**params).fit(eval_metric=constant_metric, **params_fit)
self.assertEqual(len(gbm.evals_result_['training']), 2)
self.assertIn('mape', gbm.evals_result_['training'])
self.assertIn('error', gbm.evals_result_['training'])
# multiple regression metrics with custom metric for custom objective
gbm = lgb.LGBMRegressor(objective=custom_dummy_obj, metric=['l2', 'mape'],
**params).fit(eval_metric=constant_metric, **params_fit)
self.assertEqual(len(gbm.evals_result_['training']), 3)
self.assertIn('l2', gbm.evals_result_['training'])
self.assertIn('mape', gbm.evals_result_['training'])
self.assertIn('error', gbm.evals_result_['training'])
X, y = load_digits(n_class=3, return_X_y=True)
params_fit = {'X': X, 'y': y, 'eval_set': (X, y), 'verbose': False}
# default metric and invalid binary metric is replaced with multiclass alternative
gbm = lgb.LGBMClassifier(**params).fit(eval_metric='binary_error', **params_fit)
self.assertEqual(len(gbm.evals_result_['training']), 2)
self.assertIn('multi_logloss', gbm.evals_result_['training'])
self.assertIn('multi_error', gbm.evals_result_['training'])
# invalid objective is replaced with default multiclass one
# and invalid binary metric is replaced with multiclass alternative
gbm = lgb.LGBMClassifier(objective='invalid_obj',
**params).fit(eval_metric='binary_error', **params_fit)
self.assertEqual(gbm.objective_, 'multiclass')
self.assertEqual(len(gbm.evals_result_['training']), 2)
self.assertIn('multi_logloss', gbm.evals_result_['training'])
self.assertIn('multi_error', gbm.evals_result_['training'])
# default metric for non-default multiclass objective
# and invalid binary metric is replaced with multiclass alternative
gbm = lgb.LGBMClassifier(objective='ovr',
**params).fit(eval_metric='binary_error', **params_fit)
self.assertEqual(gbm.objective_, 'ovr')
self.assertEqual(len(gbm.evals_result_['training']), 2)
self.assertIn('multi_logloss', gbm.evals_result_['training'])
self.assertIn('multi_error', gbm.evals_result_['training'])
X, y = load_digits(n_class=2, return_X_y=True)
params_fit = {'X': X, 'y': y, 'eval_set': (X, y), 'verbose': False}
# default metric and invalid multiclass metric is replaced with binary alternative
gbm = lgb.LGBMClassifier(**params).fit(eval_metric='multi_error', **params_fit)
self.assertEqual(len(gbm.evals_result_['training']), 2)
self.assertIn('binary_logloss', gbm.evals_result_['training'])
self.assertIn('binary_error', gbm.evals_result_['training'])
# invalid multiclass metric is replaced with binary alternative for custom objective
gbm = lgb.LGBMClassifier(objective=custom_dummy_obj,
**params).fit(eval_metric='multi_logloss', **params_fit)
self.assertEqual(len(gbm.evals_result_['training']), 1)
self.assertIn('binary_logloss', gbm.evals_result_['training'])
def test_multiple_eval_metrics(self):
X, y = load_breast_cancer(return_X_y=True)
params = {'n_estimators': 2, 'verbose': -1, 'objective': 'binary', 'metric': 'binary_logloss'}
params_fit = {'X': X, 'y': y, 'eval_set': (X, y), 'verbose': False}
# Verify that can receive a list of metrics, only callable
gbm = lgb.LGBMClassifier(**params).fit(eval_metric=[constant_metric, decreasing_metric], **params_fit)
self.assertEqual(len(gbm.evals_result_['training']), 3)
self.assertIn('error', gbm.evals_result_['training'])
self.assertIn('decreasing_metric', gbm.evals_result_['training'])
self.assertIn('binary_logloss', gbm.evals_result_['training'])
# Verify that can receive a list of custom and built-in metrics
gbm = lgb.LGBMClassifier(**params).fit(eval_metric=[constant_metric, decreasing_metric, 'fair'], **params_fit)
self.assertEqual(len(gbm.evals_result_['training']), 4)
self.assertIn('error', gbm.evals_result_['training'])
self.assertIn('decreasing_metric', gbm.evals_result_['training'])
self.assertIn('binary_logloss', gbm.evals_result_['training'])
self.assertIn('fair', gbm.evals_result_['training'])
# Verify that works as expected when eval_metric is empty
gbm = lgb.LGBMClassifier(**params).fit(eval_metric=[], **params_fit)
self.assertEqual(len(gbm.evals_result_['training']), 1)
self.assertIn('binary_logloss', gbm.evals_result_['training'])
# Verify that can receive a list of metrics, only built-in
gbm = lgb.LGBMClassifier(**params).fit(eval_metric=['fair', 'error'], **params_fit)
self.assertEqual(len(gbm.evals_result_['training']), 3)
self.assertIn('binary_logloss', gbm.evals_result_['training'])
# Verify that eval_metric is robust to receiving a list with None
gbm = lgb.LGBMClassifier(**params).fit(eval_metric=['fair', 'error', None], **params_fit)
self.assertEqual(len(gbm.evals_result_['training']), 3)
self.assertIn('binary_logloss', gbm.evals_result_['training'])
def test_inf_handle(self):
nrows = 100
ncols = 10
X = np.random.randn(nrows, ncols)
y = np.random.randn(nrows) + np.full(nrows, 1e30)
weight = np.full(nrows, 1e10)
params = {'n_estimators': 20, 'verbose': -1}
params_fit = {'X': X, 'y': y, 'sample_weight': weight, 'eval_set': (X, y),
'verbose': False, 'early_stopping_rounds': 5}
gbm = lgb.LGBMRegressor(**params).fit(**params_fit)
np.testing.assert_allclose(gbm.evals_result_['training']['l2'], np.inf)
def test_nan_handle(self):
nrows = 100
ncols = 10
X = np.random.randn(nrows, ncols)
y = np.random.randn(nrows) + np.full(nrows, 1e30)
weight = np.zeros(nrows)
params = {'n_estimators': 20, 'verbose': -1}
params_fit = {'X': X, 'y': y, 'sample_weight': weight, 'eval_set': (X, y),
'verbose': False, 'early_stopping_rounds': 5}
gbm = lgb.LGBMRegressor(**params).fit(**params_fit)
np.testing.assert_allclose(gbm.evals_result_['training']['l2'], np.nan)
def test_first_metric_only(self):
def fit_and_check(eval_set_names, metric_names, assumed_iteration, first_metric_only):
params['first_metric_only'] = first_metric_only
gbm = lgb.LGBMRegressor(**params).fit(**params_fit)
self.assertEqual(len(gbm.evals_result_), len(eval_set_names))
for eval_set_name in eval_set_names:
self.assertIn(eval_set_name, gbm.evals_result_)
self.assertEqual(len(gbm.evals_result_[eval_set_name]), len(metric_names))
for metric_name in metric_names:
self.assertIn(metric_name, gbm.evals_result_[eval_set_name])
actual = len(gbm.evals_result_[eval_set_name][metric_name])
expected = assumed_iteration + (params_fit['early_stopping_rounds']
if eval_set_name != 'training'
and assumed_iteration != gbm.n_estimators else 0)
self.assertEqual(expected, actual)
self.assertEqual(assumed_iteration if eval_set_name != 'training' else gbm.n_estimators,
gbm.best_iteration_)
X, y = load_boston(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
X_test1, X_test2, y_test1, y_test2 = train_test_split(X_test, y_test, test_size=0.5, random_state=72)
params = {'n_estimators': 30,
'learning_rate': 0.8,
'num_leaves': 15,
'verbose': -1,
'seed': 123}
params_fit = {'X': X_train,
'y': y_train,
'early_stopping_rounds': 5,
'verbose': False}
iter_valid1_l1 = 3
iter_valid1_l2 = 18
iter_valid2_l1 = 11
iter_valid2_l2 = 7
self.assertEqual(len(set([iter_valid1_l1, iter_valid1_l2, iter_valid2_l1, iter_valid2_l2])), 4)
iter_min_l1 = min([iter_valid1_l1, iter_valid2_l1])
iter_min_l2 = min([iter_valid1_l2, iter_valid2_l2])
iter_min = min([iter_min_l1, iter_min_l2])
iter_min_valid1 = min([iter_valid1_l1, iter_valid1_l2])
# training data as eval_set
params_fit['eval_set'] = (X_train, y_train)
fit_and_check(['training'], ['l2'], 30, False)
fit_and_check(['training'], ['l2'], 30, True)
# feval
params['metric'] = 'None'
params_fit['eval_metric'] = lambda preds, train_data: [decreasing_metric(preds, train_data),
constant_metric(preds, train_data)]
params_fit['eval_set'] = (X_test1, y_test1)
fit_and_check(['valid_0'], ['decreasing_metric', 'error'], 1, False)
fit_and_check(['valid_0'], ['decreasing_metric', 'error'], 30, True)
params_fit['eval_metric'] = lambda preds, train_data: [constant_metric(preds, train_data),
decreasing_metric(preds, train_data)]
fit_and_check(['valid_0'], ['decreasing_metric', 'error'], 1, True)
# single eval_set
params.pop('metric')
params_fit.pop('eval_metric')
fit_and_check(['valid_0'], ['l2'], iter_valid1_l2, False)
fit_and_check(['valid_0'], ['l2'], iter_valid1_l2, True)
params_fit['eval_metric'] = "l2"
fit_and_check(['valid_0'], ['l2'], iter_valid1_l2, False)
fit_and_check(['valid_0'], ['l2'], iter_valid1_l2, True)
params_fit['eval_metric'] = "l1"
fit_and_check(['valid_0'], ['l1', 'l2'], iter_min_valid1, False)
fit_and_check(['valid_0'], ['l1', 'l2'], iter_valid1_l1, True)
params_fit['eval_metric'] = ["l1", "l2"]
fit_and_check(['valid_0'], ['l1', 'l2'], iter_min_valid1, False)
fit_and_check(['valid_0'], ['l1', 'l2'], iter_valid1_l1, True)
params_fit['eval_metric'] = ["l2", "l1"]
fit_and_check(['valid_0'], ['l1', 'l2'], iter_min_valid1, False)
fit_and_check(['valid_0'], ['l1', 'l2'], iter_valid1_l2, True)
params_fit['eval_metric'] = ["l2", "regression", "mse"] # test aliases
fit_and_check(['valid_0'], ['l2'], iter_valid1_l2, False)
fit_and_check(['valid_0'], ['l2'], iter_valid1_l2, True)
# two eval_set
params_fit['eval_set'] = [(X_test1, y_test1), (X_test2, y_test2)]
params_fit['eval_metric'] = ["l1", "l2"]
fit_and_check(['valid_0', 'valid_1'], ['l1', 'l2'], iter_min_l1, True)
params_fit['eval_metric'] = ["l2", "l1"]
fit_and_check(['valid_0', 'valid_1'], ['l1', 'l2'], iter_min_l2, True)
params_fit['eval_set'] = [(X_test2, y_test2), (X_test1, y_test1)]
params_fit['eval_metric'] = ["l1", "l2"]
fit_and_check(['valid_0', 'valid_1'], ['l1', 'l2'], iter_min, False)
fit_and_check(['valid_0', 'valid_1'], ['l1', 'l2'], iter_min_l1, True)
params_fit['eval_metric'] = ["l2", "l1"]
fit_and_check(['valid_0', 'valid_1'], ['l1', 'l2'], iter_min, False)
fit_and_check(['valid_0', 'valid_1'], ['l1', 'l2'], iter_min_l2, True)
def test_class_weight(self):
X, y = load_digits(n_class=10, return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
y_train_str = y_train.astype('str')
y_test_str = y_test.astype('str')
gbm = lgb.LGBMClassifier(n_estimators=10, class_weight='balanced', silent=True)
gbm.fit(X_train, y_train,
eval_set=[(X_train, y_train), (X_test, y_test), (X_test, y_test),
(X_test, y_test), (X_test, y_test)],
eval_class_weight=['balanced', None, 'balanced', {1: 10, 4: 20}, {5: 30, 2: 40}],
verbose=False) verbose=False)
for eval_set1, eval_set2 in itertools.combinations(gbm.evals_result_.keys(), 2): for eval_set1, eval_set2 in itertools.combinations(gbm_str.evals_result_.keys(), 2):
for metric in gbm.evals_result_[eval_set1]: for metric in gbm_str.evals_result_[eval_set1]:
np.testing.assert_raises(AssertionError, np.testing.assert_raises(AssertionError,
np.testing.assert_allclose, np.testing.assert_allclose,
gbm.evals_result_[eval_set1][metric], gbm_str.evals_result_[eval_set1][metric],
gbm.evals_result_[eval_set2][metric]) gbm_str.evals_result_[eval_set2][metric])
gbm_str = lgb.LGBMClassifier(n_estimators=10, class_weight='balanced', silent=True) for eval_set in gbm.evals_result_:
gbm_str.fit(X_train, y_train_str, for metric in gbm.evals_result_[eval_set]:
eval_set=[(X_train, y_train_str), (X_test, y_test_str), np.testing.assert_allclose(gbm.evals_result_[eval_set][metric],
(X_test, y_test_str), (X_test, y_test_str), (X_test, y_test_str)], gbm_str.evals_result_[eval_set][metric])
eval_class_weight=['balanced', None, 'balanced', {'1': 10, '4': 20}, {'5': 30, '2': 40}],
verbose=False)
for eval_set1, eval_set2 in itertools.combinations(gbm_str.evals_result_.keys(), 2): def test_continue_training_with_model():
for metric in gbm_str.evals_result_[eval_set1]: X, y = load_digits(n_class=3, return_X_y=True)
np.testing.assert_raises(AssertionError, X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
np.testing.assert_allclose, init_gbm = lgb.LGBMClassifier(n_estimators=5).fit(X_train, y_train, eval_set=(X_test, y_test),
gbm_str.evals_result_[eval_set1][metric], verbose=False)
gbm_str.evals_result_[eval_set2][metric]) gbm = lgb.LGBMClassifier(n_estimators=5).fit(X_train, y_train, eval_set=(X_test, y_test),
for eval_set in gbm.evals_result_: verbose=False, init_model=init_gbm)
for metric in gbm.evals_result_[eval_set]: assert len(init_gbm.evals_result_['valid_0']['multi_logloss']) == len(gbm.evals_result_['valid_0']['multi_logloss'])
np.testing.assert_allclose(gbm.evals_result_[eval_set][metric], assert len(init_gbm.evals_result_['valid_0']['multi_logloss']) == 5
gbm_str.evals_result_[eval_set][metric]) assert gbm.evals_result_['valid_0']['multi_logloss'][-1] < init_gbm.evals_result_['valid_0']['multi_logloss'][-1]
def test_continue_training_with_model(self):
X, y = load_digits(n_class=3, return_X_y=True) # sklearn < 0.22 requires passing "attributes" argument
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42) @pytest.mark.skipif(sk_version < parse_version('0.22'), reason='scikit-learn version is less than 0.22')
init_gbm = lgb.LGBMClassifier(n_estimators=5).fit(X_train, y_train, eval_set=(X_test, y_test), def test_check_is_fitted():
verbose=False) X, y = load_digits(n_class=2, return_X_y=True)
gbm = lgb.LGBMClassifier(n_estimators=5).fit(X_train, y_train, eval_set=(X_test, y_test), est = lgb.LGBMModel(n_estimators=5, objective="binary")
verbose=False, init_model=init_gbm) clf = lgb.LGBMClassifier(n_estimators=5)
self.assertEqual(len(init_gbm.evals_result_['valid_0']['multi_logloss']), reg = lgb.LGBMRegressor(n_estimators=5)
len(gbm.evals_result_['valid_0']['multi_logloss'])) rnk = lgb.LGBMRanker(n_estimators=5)
self.assertEqual(len(init_gbm.evals_result_['valid_0']['multi_logloss']), 5) models = (est, clf, reg, rnk)
self.assertLess(gbm.evals_result_['valid_0']['multi_logloss'][-1], for model in models:
init_gbm.evals_result_['valid_0']['multi_logloss'][-1]) with pytest.raises(lgb.compat.LGBMNotFittedError):
# sklearn < 0.22 requires passing "attributes" argument
@unittest.skipIf(sk_version < parse_version('0.22'), 'scikit-learn version is less than 0.22')
def test_check_is_fitted(self):
X, y = load_digits(n_class=2, return_X_y=True)
est = lgb.LGBMModel(n_estimators=5, objective="binary")
clf = lgb.LGBMClassifier(n_estimators=5)
reg = lgb.LGBMRegressor(n_estimators=5)
rnk = lgb.LGBMRanker(n_estimators=5)
models = (est, clf, reg, rnk)
for model in models:
self.assertRaises(lgb.compat.LGBMNotFittedError,
check_is_fitted,
model)
est.fit(X, y)
clf.fit(X, y)
reg.fit(X, y)
rnk.fit(X, y, group=np.ones(X.shape[0]))
for model in models:
check_is_fitted(model) check_is_fitted(model)
est.fit(X, y)
clf.fit(X, y)
reg.fit(X, y)
rnk.fit(X, y, group=np.ones(X.shape[0]))
for model in models:
check_is_fitted(model)
def _tested_estimators(): def _tested_estimators():
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment