re-write test cases: remove global template (#479)

tests/python_package_test/test_engine.py

@@ -28,72 +28,76 @@ def multi_logloss(y_true, y_pred):
     return np.mean([-math.log(y_pred[i][y]) for i, y in enumerate(y_true)])
 
 
-class template(object):
-    @staticmethod
-    def test_template(params={'objective': 'regression', 'metric': 'l2'},
-                      X_y=load_boston(True), feval=mean_squared_error,
-                      num_round=50, init_model=None, custom_eval=None,
-                      early_stopping_rounds=2,
-                      return_data=False, return_model=False):
-        params['verbose'], params['seed'] = -1, 42
-        X_train, X_test, y_train, y_test = train_test_split(*X_y, test_size=0.1, random_state=42)
-        lgb_train = lgb.Dataset(X_train, y_train, params=params)
-        lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train, params=params)
-        if return_data:
-            return lgb_train, lgb_eval
-        evals_result = {}
-        gbm = lgb.train(params, lgb_train,
-                        num_boost_round=num_round,
-                        valid_sets=lgb_eval,
-                        valid_names='eval',
-                        verbose_eval=False,
-                        feval=custom_eval,
-                        evals_result=evals_result,
-                        early_stopping_rounds=early_stopping_rounds,
-                        init_model=init_model)
-        if return_model:
-            return gbm
-        else:
-            return evals_result, feval(y_test, gbm.predict(X_test, gbm.best_iteration))
-
-
 class TestEngine(unittest.TestCase):
 
     def test_binary(self):
-        X_y = load_breast_cancer(True)
+        X, y = load_breast_cancer(True)
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
         params = {
             'objective': 'binary',
-            'metric': 'binary_logloss'
+            'metric': 'binary_logloss',
+            'verbose': -1
         }
-        evals_result, ret = template.test_template(params, X_y, log_loss)
+        lgb_train = lgb.Dataset(X_train, y_train)
+        lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
+        evals_result = {}
+        gbm = lgb.train(params, lgb_train,
+                        num_boost_round=50,
+                        valid_sets=lgb_eval,
+                        verbose_eval=False,
+                        evals_result=evals_result)
+        ret = log_loss(y_test, gbm.predict(X_test))
         self.assertLess(ret, 0.15)
-        self.assertAlmostEqual(min(evals_result['eval']['binary_logloss']), ret, places=5)
+        self.assertAlmostEqual(evals_result['valid_0']['binary_logloss'][-1], ret, places=5)
 
     def test_regreesion(self):
-        evals_result, ret = template.test_template()
+        X, y = load_boston(True)
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
+        params = {
+            'metric': 'l2',
+            'verbose': -1
+        }
+        lgb_train = lgb.Dataset(X_train, y_train)
+        lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
+        evals_result = {}
+        gbm = lgb.train(params, lgb_train,
+                        num_boost_round=50,
+                        valid_sets=lgb_eval,
+                        verbose_eval=False,
+                        evals_result=evals_result)
+        ret = mean_squared_error(y_test, gbm.predict(X_test))
         self.assertLess(ret, 16)
-        self.assertAlmostEqual(min(evals_result['eval']['l2']), ret, places=5)
+        self.assertAlmostEqual(evals_result['valid_0']['l2'][-1], ret, places=5)
 
     def test_multiclass(self):
-        X_y = load_digits(10, True)
+        X, y = load_digits(10, True)
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
         params = {
             'objective': 'multiclass',
             'metric': 'multi_logloss',
-            'num_class': 10
+            'num_class': 10,
+            'verbose': -1
         }
-        evals_result, ret = template.test_template(params, X_y, multi_logloss)
+        lgb_train = lgb.Dataset(X_train, y_train, params=params)
+        lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train, params=params)
+        evals_result = {}
+        gbm = lgb.train(params, lgb_train,
+                        num_boost_round=50,
+                        valid_sets=lgb_eval,
+                        verbose_eval=False,
+                        evals_result=evals_result)
+        ret = multi_logloss(y_test, gbm.predict(X_test))
         self.assertLess(ret, 0.2)
-        self.assertAlmostEqual(min(evals_result['eval']['multi_logloss']), ret, places=5)
+        self.assertAlmostEqual(evals_result['valid_0']['multi_logloss'][-1], ret, places=5)
 
     def test_early_stopping(self):
-        X_y = load_breast_cancer(True)
+        X, y = load_breast_cancer(True)
         params = {
             'objective': 'binary',
             'metric': 'binary_logloss',
-            'verbose': -1,
-            'seed': 42
+            'verbose': -1
         }
-        X_train, X_test, y_train, y_test = train_test_split(*X_y, test_size=0.1, random_state=42)
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
         lgb_train = lgb.Dataset(X_train, y_train)
         lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
         valid_set_name = 'valid_set'
@@ -109,7 +113,6 @@ class TestEngine(unittest.TestCase):
         self.assertIn('binary_logloss', gbm.best_score[valid_set_name])
         # early stopping occurs
         gbm = lgb.train(params, lgb_train,
-                        num_boost_round=100,
                         valid_sets=lgb_eval,
                         valid_names=valid_set_name,
                         verbose_eval=False,
@@ -118,85 +121,126 @@ class TestEngine(unittest.TestCase):
         self.assertIn(valid_set_name, gbm.best_score)
         self.assertIn('binary_logloss', gbm.best_score[valid_set_name])
 
-    def test_continue_train_and_other(self):
+    def test_continue_train_and_dump_model(self):
+        X, y = load_boston(True)
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
         params = {
             'objective': 'regression',
-            'metric': 'l1'
+            'metric': 'l1',
+            'verbose': -1
         }
+        lgb_train = lgb.Dataset(X_train, y_train, free_raw_data=False)
+        lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train, free_raw_data=False)
+        init_gbm = lgb.train(params, lgb_train, num_boost_round=20)
         model_name = 'model.txt'
-        gbm = template.test_template(params, num_round=20, return_model=True, early_stopping_rounds=-1)
-        gbm.save_model(model_name)
-        evals_result, ret = template.test_template(params, feval=mean_absolute_error,
-                                                   num_round=80, init_model=model_name,
-                                                   custom_eval=(lambda p, d: ('mae', mean_absolute_error(p, d.get_label()), False)))
+        init_gbm.save_model(model_name)
+        evals_result = {}
+        gbm = lgb.train(params, lgb_train,
+                        num_boost_round=30,
+                        valid_sets=lgb_eval,
+                        verbose_eval=False,
+                        # test custom eval metrics
+                        feval=(lambda p, d: ('mae', mean_absolute_error(p, d.get_label()), False)),
+                        evals_result=evals_result,
+                        init_model='model.txt')
+        ret = mean_absolute_error(y_test, gbm.predict(X_test))
         self.assertLess(ret, 3.5)
-        self.assertAlmostEqual(min(evals_result['eval']['l1']), ret, places=5)
-        for l1, mae in zip(evals_result['eval']['l1'], evals_result['eval']['mae']):
+        self.assertAlmostEqual(evals_result['valid_0']['l1'][-1], ret, places=5)
+        for l1, mae in zip(evals_result['valid_0']['l1'], evals_result['valid_0']['mae']):
             self.assertAlmostEqual(l1, mae, places=5)
+        # test dump model
         self.assertIn('tree_info', gbm.dump_model())
         self.assertIsInstance(gbm.feature_importance(), np.ndarray)
         os.remove(model_name)
 
     def test_continue_train_multiclass(self):
-        X_y = load_iris(True)
+        X, y = load_iris(True)
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
         params = {
             'objective': 'multiclass',
             'metric': 'multi_logloss',
-            'num_class': 3
+            'num_class': 3,
+            'verbose': -1
         }
-        gbm = template.test_template(params, X_y, num_round=20, return_model=True, early_stopping_rounds=-1)
-        evals_result, ret = template.test_template(params, X_y, feval=multi_logloss,
-                                                   num_round=80, init_model=gbm)
+        lgb_train = lgb.Dataset(X_train, y_train, params=params, free_raw_data=False)
+        lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train, params=params, free_raw_data=False)
+        init_gbm = lgb.train(params, lgb_train, num_boost_round=20)
+        evals_result = {}
+        gbm = lgb.train(params, lgb_train,
+                        num_boost_round=30,
+                        valid_sets=lgb_eval,
+                        verbose_eval=False,
+                        evals_result=evals_result,
+                        init_model=init_gbm)
+        ret = multi_logloss(y_test, gbm.predict(X_test))
         self.assertLess(ret, 1.5)
-        self.assertAlmostEqual(min(evals_result['eval']['multi_logloss']), ret, places=5)
+        self.assertAlmostEqual(evals_result['valid_0']['multi_logloss'][-1], ret, places=5)
 
     def test_cv(self):
-        lgb_train, _ = template.test_template(return_data=True)
-        # shuffle = False
-        lgb.cv({'verbose': -1}, lgb_train, num_boost_round=10, nfold=3, shuffle=False,
+        X, y = load_boston(True)
+        X_train, _, y_train, _ = train_test_split(X, y, test_size=0.1, random_state=42)
+        params = {'verbose': -1}
+        lgb_train = lgb.Dataset(X_train, y_train)
+        # shuffle = False, override metric in params
+        params_with_metric = {'metric': 'l2', 'verbose': -1}
+        lgb.cv(params_with_metric, lgb_train, num_boost_round=10, nfold=3, shuffle=False,
                metrics='l1', verbose_eval=False)
         # shuffle = True, callbacks
-        lgb.cv({'verbose': -1}, lgb_train, num_boost_round=10, nfold=3, shuffle=True,
+        lgb.cv(params, lgb_train, num_boost_round=10, nfold=3, shuffle=True,
                metrics='l1', verbose_eval=False,
                callbacks=[lgb.reset_parameter(learning_rate=lambda i: 0.1 - 0.001 * i)])
         # self defined data_splitter
         tss = TimeSeriesSplit(3)
-        lgb.cv({'verbose': -1}, lgb_train, num_boost_round=10, data_splitter=tss, nfold=5,  # test if wrong nfold is ignored
+        lgb.cv(params, lgb_train, num_boost_round=10, data_splitter=tss, nfold=5,  # test if wrong nfold is ignored
                metrics='l2', verbose_eval=False)
         # lambdarank
         X_train, y_train = load_svmlight_file('../../examples/lambdarank/rank.train')
         q_train = np.loadtxt('../../examples/lambdarank/rank.train.query')
-        params = {'objective': 'lambdarank', 'verbose': -1}
-        lgb_train = lgb.Dataset(X_train, y_train, group=q_train, params=params)
-        lgb.cv(params, lgb_train, num_boost_round=20, nfold=3, metrics='l2', verbose_eval=False)
+        params_lambdarank = {'objective': 'lambdarank', 'verbose': -1}
+        lgb_train = lgb.Dataset(X_train, y_train, group=q_train)
+        lgb.cv(params_lambdarank, lgb_train, num_boost_round=10, nfold=3, metrics='l2', verbose_eval=False)
 
     def test_feature_name(self):
-        lgb_train, _ = template.test_template(return_data=True)
+        X, y = load_boston(True)
+        X_train, _, y_train, _ = train_test_split(X, y, test_size=0.1, random_state=42)
+        params = {'verbose': -1}
+        lgb_train = lgb.Dataset(X_train, y_train)
         feature_names = ['f_' + str(i) for i in range(13)]
-        gbm = lgb.train({'verbose': -1}, lgb_train, num_boost_round=5, feature_name=feature_names)
+        gbm = lgb.train(params, lgb_train, num_boost_round=5, feature_name=feature_names)
         self.assertListEqual(feature_names, gbm.feature_name())
         # test feature_names with whitespaces
         feature_names_with_space = ['f ' + str(i) for i in range(13)]
-        gbm = lgb.train({'verbose': -1}, lgb_train, num_boost_round=5, feature_name=feature_names_with_space)
+        gbm = lgb.train(params, lgb_train, num_boost_round=5, feature_name=feature_names_with_space)
         self.assertListEqual(feature_names, gbm.feature_name())
 
     def test_save_load_copy_pickle(self):
-        gbm = template.test_template(num_round=20, return_model=True)
-        _, ret_origin = template.test_template(init_model=gbm)
+        def test_template(init_model=None, return_model=False):
+            X, y = load_boston(True)
+            params = {
+                'objective': 'regression',
+                'metric': 'l2',
+                'verbose': -1
+            }
+            X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
+            lgb_train = lgb.Dataset(X_train, y_train)
+            gbm_template = lgb.train(params, lgb_train, num_boost_round=10, init_model=init_model)
+            return gbm_template if return_model else mean_squared_error(y_test, gbm_template.predict(X_test))
+        gbm = test_template(return_model=True)
+        ret_origin = test_template(init_model=gbm)
         other_ret = []
         gbm.save_model('lgb.model')
-        other_ret.append(template.test_template(init_model='lgb.model')[1])
+        other_ret.append(test_template(init_model='lgb.model'))
         gbm_load = lgb.Booster(model_file='lgb.model')
-        other_ret.append(template.test_template(init_model=gbm_load)[1])
-        other_ret.append(template.test_template(init_model=copy.copy(gbm))[1])
-        other_ret.append(template.test_template(init_model=copy.deepcopy(gbm))[1])
+        other_ret.append(test_template(init_model=gbm_load))
+        other_ret.append(test_template(init_model=copy.copy(gbm)))
+        other_ret.append(test_template(init_model=copy.deepcopy(gbm)))
         with open('lgb.pkl', 'wb') as f:
             pickle.dump(gbm, f)
         with open('lgb.pkl', 'rb') as f:
             gbm_pickle = pickle.load(f)
-        other_ret.append(template.test_template(init_model=gbm_pickle)[1])
+        other_ret.append(test_template(init_model=gbm_pickle))
         gbm_pickles = pickle.loads(pickle.dumps(gbm))
-        other_ret.append(template.test_template(init_model=gbm_pickles)[1])
+        other_ret.append(test_template(init_model=gbm_pickles))
         for ret in other_ret:
             self.assertAlmostEqual(ret_origin, ret, places=5)
 

tests/python_package_test/test_sklearn.py

 # coding: utf-8
 # pylint: skip-file
+import math
 import unittest
 
 import lightgbm as lgb
@@ -12,57 +13,52 @@ from sklearn.metrics import log_loss, mean_squared_error
 from sklearn.model_selection import GridSearchCV, train_test_split
 
 
-class template(object):
-    @staticmethod
-    def test_template(X_y=load_boston(True), model=lgb.LGBMRegressor,
-                      feval=mean_squared_error, num_round=100,
-                      custom_obj=None, predict_proba=False,
-                      return_data=False, return_model=False):
-        X_train, X_test, y_train, y_test = train_test_split(*X_y, test_size=0.1, random_state=42)
-        if return_data:
-            return X_train, X_test, y_train, y_test
-        arguments = {'n_estimators': num_round, 'silent': True}
-        if custom_obj:
-            arguments['objective'] = custom_obj
-        gbm = model(**arguments)
-        gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=10, verbose=False)
-        if return_model:
-            return gbm
-        elif predict_proba:
-            return feval(y_test, gbm.predict_proba(X_test))
-        else:
-            return feval(y_test, gbm.predict(X_test))
+def multi_error(y_true, y_pred):
+    return np.mean(y_true != y_pred)
+
+
+def multi_logloss(y_true, y_pred):
+    return np.mean([-math.log(y_pred[i][y]) for i, y in enumerate(y_true)])
 
 
 class TestSklearn(unittest.TestCase):
 
     def test_binary(self):
-        X_y = load_breast_cancer(True)
-        ret = template.test_template(X_y, lgb.LGBMClassifier, log_loss, predict_proba=True)
+        X, y = load_breast_cancer(True)
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
+        gbm = lgb.LGBMClassifier(n_estimators=50, silent=True)
+        gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], verbose=False)
+        ret = log_loss(y_test, gbm.predict_proba(X_test))
         self.assertLess(ret, 0.15)
+        self.assertAlmostEqual(ret, gbm.evals_result['valid_0']['binary_logloss'][-1], places=5)
 
     def test_regreesion(self):
-        self.assertLess(template.test_template() ** 0.5, 4)
+        X, y = load_boston(True)
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
+        gbm = lgb.LGBMRegressor(n_estimators=50, silent=True)
+        gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], verbose=False)
+        ret = mean_squared_error(y_test, gbm.predict(X_test))
+        self.assertLess(ret, 16)
+        self.assertAlmostEqual(ret, gbm.evals_result['valid_0']['l2'][-1], places=5)
 
     def test_multiclass(self):
-        X_y = load_digits(10, True)
-
-        def multi_error(y_true, y_pred):
-            return np.mean(y_true != y_pred)
-        ret = template.test_template(X_y, lgb.LGBMClassifier, multi_error)
+        X, y = load_digits(10, True)
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
+        gbm = lgb.LGBMClassifier(n_estimators=50, silent=True)
+        gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], verbose=False)
+        ret = multi_error(y_test, gbm.predict(X_test))
         self.assertLess(ret, 0.2)
+        ret = multi_logloss(y_test, gbm.predict_proba(X_test))
+        self.assertAlmostEqual(ret, gbm.evals_result['valid_0']['multi_logloss'][-1], places=5)
 
     def test_lambdarank(self):
         X_train, y_train = load_svmlight_file('../../examples/lambdarank/rank.train')
         X_test, y_test = load_svmlight_file('../../examples/lambdarank/rank.test')
         q_train = np.loadtxt('../../examples/lambdarank/rank.train.query')
         q_test = np.loadtxt('../../examples/lambdarank/rank.test.query')
-        lgb_model = lgb.LGBMRanker().fit(X_train, y_train,
-                                         group=q_train,
-                                         eval_set=[(X_test, y_test)],
-                                         eval_group=[q_test],
-                                         eval_at=[1],
-                                         verbose=False,
+        gbm = lgb.LGBMRanker()
+        gbm.fit(X_train, y_train, group=q_train, eval_set=[(X_test, y_test)],
+                eval_group=[q_test], eval_at=[1, 3], verbose=False,
                 callbacks=[lgb.reset_parameter(learning_rate=lambda x: 0.95 ** x * 0.1)])
 
     def test_regression_with_custom_objective(self):
@@ -70,8 +66,13 @@ class TestSklearn(unittest.TestCase):
             grad = (y_pred - y_true)
             hess = np.ones(len(y_true))
             return grad, hess
-        ret = template.test_template(custom_obj=objective_ls)
+        X, y = load_boston(True)
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
+        gbm = lgb.LGBMRegressor(n_estimators=50, silent=True, objective=objective_ls)
+        gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], verbose=False)
+        ret = mean_squared_error(y_test, gbm.predict(X_test))
         self.assertLess(ret, 100)
+        self.assertAlmostEqual(ret, gbm.evals_result['valid_0']['l2'][-1], places=5)
 
     def test_binary_classification_with_custom_objective(self):
         def logregobj(y_true, y_pred):
@@ -79,21 +80,26 @@ class TestSklearn(unittest.TestCase):
             grad = y_pred - y_true
             hess = y_pred * (1.0 - y_pred)
             return grad, hess
-        X_y = load_digits(2, True)
+        X, y = load_digits(2, True)
 
         def binary_error(y_test, y_pred):
             return np.mean([int(p > 0.5) != y for y, p in zip(y_test, y_pred)])
-        ret = template.test_template(X_y, lgb.LGBMClassifier, feval=binary_error, custom_obj=logregobj)
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
+        gbm = lgb.LGBMClassifier(n_estimators=50, silent=True, objective=logregobj)
+        gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], verbose=False)
+        ret = binary_error(y_test, gbm.predict(X_test))
         self.assertLess(ret, 0.1)
 
     def test_dart(self):
-        X_train, X_test, y_train, y_test = template.test_template(return_data=True)
+        X, y = load_boston(True)
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
         gbm = lgb.LGBMRegressor(boosting_type='dart')
         gbm.fit(X_train, y_train)
         self.assertLessEqual(gbm.score(X_train, y_train), 1.)
 
     def test_grid_search(self):
-        X_train, X_test, y_train, y_test = template.test_template(return_data=True)
+        X, y = load_boston(True)
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
         params = {'boosting_type': ['dart', 'gbdt'],
                   'n_estimators': [5, 8],
                   'drop_rate': [0.05, 0.1]}
@@ -102,24 +108,38 @@ class TestSklearn(unittest.TestCase):
         self.assertIn(gbm.best_params_['n_estimators'], [5, 8])
 
     def test_clone_and_property(self):
-        gbm = template.test_template(return_model=True)
+        X, y = load_boston(True)
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
+        gbm = lgb.LGBMRegressor(n_estimators=100, silent=True)
+        gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=10, verbose=False)
+
         gbm_clone = clone(gbm)
         self.assertIsInstance(gbm.booster_, lgb.Booster)
         self.assertIsInstance(gbm.feature_importances_, np.ndarray)
-        clf = template.test_template(load_digits(2, True), model=lgb.LGBMClassifier, return_model=True)
+
+        X, y = load_digits(2, True)
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
+        clf = lgb.LGBMClassifier()
+        clf.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=10, verbose=False)
         self.assertListEqual(sorted(clf.classes_), [0, 1])
         self.assertEqual(clf.n_classes_, 2)
         self.assertIsInstance(clf.booster_, lgb.Booster)
         self.assertIsInstance(clf.feature_importances_, np.ndarray)
 
     def test_joblib(self):
-        gbm = template.test_template(num_round=10, return_model=True)
+        X, y = load_boston(True)
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
+        gbm = lgb.LGBMRegressor(n_estimators=100, silent=True)
+        gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=10, verbose=False)
+
         joblib.dump(gbm, 'lgb.pkl')
         gbm_pickle = joblib.load('lgb.pkl')
         self.assertIsInstance(gbm_pickle.booster_, lgb.Booster)
         self.assertDictEqual(gbm.get_params(), gbm_pickle.get_params())
         self.assertListEqual(list(gbm.feature_importances_), list(gbm_pickle.feature_importances_))
-        X_train, X_test, y_train, y_test = template.test_template(return_data=True)
+
+        X, y = load_boston(True)
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
         gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], verbose=False)
         gbm_pickle.fit(X_train, y_train, eval_set=[(X_test, y_test)], verbose=False)
         for key in gbm.evals_result_: