rewrite test with unittest (#142)

* add troubleshooting

* use unittest

* update unittest version

* fix test_engine.py

* fix test_sklearn.py

* default eval_metric by subclass

* add test grid search

* remove verbose_eval

.travis.yml

@@ -14,7 +14,7 @@ before_install:
 
 install:
 - sudo apt-get install -y libopenmpi-dev openmpi-bin build-essential 
-- conda install --yes atlas numpy scipy scikit-learn pandas
+- conda install --yes atlas numpy scipy scikit-learn
 
 
 script:

docs/Python-API.md

@@ -6,7 +6,7 @@
 
 * [Training API](Python-API.md#training-api)
     - [train](Python-API.md#trainparams-train_set-num_boost_round100-valid_setsnone-valid_namesnone-fobjnone-fevalnone-init_modelnone-feature_namenone-categorical_featurenone-early_stopping_roundsnone-evals_resultnone-verbose_evaltrue-learning_ratesnone-callbacksnone)
-    - [cv](Python-API.md#cvparams-train_set-num_boost_round10-nfold5-stratifiedfalse-metricsnone-fobjnone-fevalnone-init_modelnone-feature_namenone-categorical_featurenone-early_stopping_roundsnone-fpreprocnone-verbose_evalnone-show_stdvtrue-seed0-callbacksnone)
+    - [cv](Python-API.md#cvparams-train_set-num_boost_round10-nfold5-stratifiedfalse-shuffletrue-metricsnone-fobjnone-fevalnone-init_modelnone-feature_namenone-categorical_featurenone-early_stopping_roundsnone-fpreprocnone-verbose_evalnone-show_stdvtrue-seed0-callbacksnone)
 
 * [Scikit-learn API](Python-API.md#scikit-learn-api)
     - [Common Methods](Python-API.md#common-methods)
@@ -516,7 +516,7 @@ The methods of each Class is in alphabetical order.
     booster : a trained booster model
     
 
-####cv(params, train_set, num_boost_round=10, nfold=5, stratified=False, metrics=None, fobj=None, feval=None, init_model=None, feature_name=None, categorical_feature=None, early_stopping_rounds=None, fpreproc=None, verbose_eval=None, show_stdv=True, seed=0, callbacks=None)
+####cv(params, train_set, num_boost_round=10, nfold=5, stratified=False, shuffle=True, metrics=None, fobj=None, feval=None, init_model=None, feature_name=None, categorical_feature=None, early_stopping_rounds=None, fpreproc=None, verbose_eval=None, show_stdv=True, seed=0, callbacks=None)
 
     Cross-validation with given paramaters.
 
@@ -532,6 +532,8 @@ The methods of each Class is in alphabetical order.
         Number of folds in CV.
     stratified : bool
         Perform stratified sampling.
+    shuffle: bool
+        Whether shuffle before split data.
     folds : a KFold or StratifiedKFold instance
         Sklearn KFolds or StratifiedKFolds.
     metrics : str or list of str
@@ -723,6 +725,7 @@ The methods of each Class is in alphabetical order.
     eval_metric : str, list of str, callable, optional
         If a str, should be a built-in evaluation metric to use.
         If callable, a custom evaluation metric, see note for more details.
+        default: binary_error for LGBMClassifier, l2 for LGBMRegressor, ndcg for LGBMRanker
     early_stopping_rounds : int
     verbose : bool
         If `verbose` and an evaluation set is used, writes the evaluation
@@ -806,11 +809,11 @@ The methods of each Class is in alphabetical order.
 
 ###LGBMRanker
 
-####fit(X, y, sample_weight=None, init_score=None, group=None, eval_set=None, eval_sample_weight=None, eval_init_score=None, eval_group=None, eval_metric=None, eval_at=None, early_stopping_rounds=None, verbose=True, feature_name=None, categorical_feature=None, other_params=None)
+####fit(X, y, sample_weight=None, init_score=None, group=None, eval_set=None, eval_sample_weight=None, eval_init_score=None, eval_group=None, eval_metric='ndcg', eval_at=1, early_stopping_rounds=None, verbose=True, feature_name=None, categorical_feature=None, other_params=None)
 
     Most arguments are same as Common Methods except:
 
-    eval_at : list of int
+    eval_at : int or list of int, default=1
         The evaulation positions of NDCG
 
 ## Callbacks

python-package/README.rst

@@ -11,9 +11,23 @@ Installation
 Note: Make sure you have `setuptools <https://pypi.python.org/pypi/setuptools>`__
 
 
-
 Examples
 --------
 
 -  Refer also to the walk through examples in `python-guide
    folder <https://github.com/Microsoft/LightGBM/tree/master/examples/python-guide>`__
+
+
+Troubleshooting
+--------
+
+- **Trouble 1**: I see error messages like this when install from github using `python setup.py install`.
+
+    error: Error: setup script specifies an absolute path:
+
+    /Users/Microsoft/LightGBM/python-package/lightgbm/../../lib_lightgbm.so
+
+    setup() arguments must *always* be /-separated paths relative to the
+    setup.py directory, *never* absolute paths.
+
+- **Solution 1**: please check `here <http://stackoverflow.com/questions/18085571/pip-install-error-setup-script-specifies-an-absolute-path>`__.

python-package/lightgbm/basic.py

@@ -527,7 +527,7 @@ class _InnerDataset(object):
                                                 is_reshape=False)
             if self.predictor.num_class > 1:
                 # need re group init score
-                new_init_score = np.zeros(init_score.size(), dtype=np.float32)
+                new_init_score = np.zeros(init_score.size, dtype=np.float32)
                 num_data = self.num_data()
                 for i in range(num_data):
                     for j in range(self.predictor.num_class):
@@ -981,8 +981,7 @@ class Dataset(object):
             self._predictor = predictor
             self.inner_dataset = None
         else:
-            raise LightGBMError("Cannot set predictor after freed raw data,\
-             Set free_raw_data=False when construct Dataset to avoid this.")
+            raise LightGBMError("Cannot set predictor after freed raw data,Set free_raw_data=False when construct Dataset to avoid this.")
 
     def set_reference(self, reference):
         """

python-package/lightgbm/callback.py

@@ -155,7 +155,7 @@ def early_stopping(stopping_rounds, verbose=True):
     def init(env):
         """internal function"""
         if not env.evaluation_result_list:
-            raise ValueError('For early stopping, at least one dataset is required for evaluation')
+            raise ValueError('For early stopping, at least one dataset or eval metric is required for evaluation')
 
         if verbose:
             msg = "Train until valid scores didn't improve in {} rounds."

python-package/lightgbm/sklearn.py

@@ -344,13 +344,6 @@ class LGBMModel(LGBMModelBase):
             params["objective"] = "None"
         else:
             params["objective"] = self.objective
-            if eval_metric is None and eval_set is not None:
-                eval_metric = {
-                    'regression': 'l2',
-                    'binary': 'binary_logloss',
-                    'lambdarank': 'ndcg',
-                    'multiclass': 'multi_logloss'
-                }.get(self.objective, None)
 
         if callable(eval_metric):
             feval = _eval_function_wrapper(eval_metric)
@@ -471,7 +464,7 @@ class LGBMRegressor(LGBMModel, LGBMRegressorBase):
             sample_weight=None, init_score=None,
             eval_set=None, eval_sample_weight=None,
             eval_init_score=None,
-            eval_metric=None,
+            eval_metric="l2",
             early_stopping_rounds=None, verbose=True,
             feature_name=None, categorical_feature=None,
             other_params=None):
@@ -504,7 +497,7 @@ class LGBMClassifier(LGBMModel, LGBMClassifierBase):
             sample_weight=None, init_score=None,
             eval_set=None, eval_sample_weight=None,
             eval_init_score=None,
-            eval_metric=None,
+            eval_metric="binary_logloss",
             early_stopping_rounds=None, verbose=True,
             feature_name=None, categorical_feature=None,
             other_params=None):
@@ -517,6 +510,8 @@ class LGBMClassifier(LGBMModel, LGBMClassifierBase):
             # Switch to using a multiclass objective in the underlying LGBM instance
             self.objective = "multiclass"
             other_params['num_class'] = self.n_classes_
+            if eval_set is not None and eval_metric == "binary_logloss":
+                eval_metric = "multi_logloss"
 
         self._le = LGBMLabelEncoder().fit(y)
         training_labels = self._le.transform(y)
@@ -589,7 +584,7 @@ class LGBMRanker(LGBMModel):
             sample_weight=None, init_score=None, group=None,
             eval_set=None, eval_sample_weight=None,
             eval_init_score=None, eval_group=None,
-            eval_metric=None, eval_at=None,
+            eval_metric='ndcg', eval_at=1,
             early_stopping_rounds=None, verbose=True,
             feature_name=None, categorical_feature=None,
             other_params=None):
@@ -616,6 +611,8 @@ class LGBMRanker(LGBMModel):
 
         if eval_at is not None:
             other_params = {} if other_params is None else other_params
+            if isinstance(eval_at, int):
+                eval_at = [eval_at]
             other_params['ndcg_eval_at'] = list(eval_at)
         super(LGBMRanker, self).fit(X, y, sample_weight, init_score, group,
                                     eval_set, eval_sample_weight, eval_init_score, eval_group,

tests/python_package_test/test_engine.py

 # coding: utf-8
-# pylint: disable = invalid-name, C0111
-import json
+# pylint: skip-file
+import os, unittest, math
+import numpy as np
 import lightgbm as lgb
-import pandas as pd
-from sklearn.metrics import mean_squared_error
-
-# load or create your dataset
-df_train = pd.read_csv('../../examples/regression/regression.train', header=None, sep='\t')
-df_test = pd.read_csv('../../examples/regression/regression.test', header=None, sep='\t')
-
-y_train = df_train[0]
-y_test = df_test[0]
-X_train = df_train.drop(0, axis=1)
-X_test = df_test.drop(0, axis=1)
-
-# create dataset for lightgbm
-lgb_train = lgb.Dataset(X_train, y_train, free_raw_data=False)
-lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train,free_raw_data=False)
-
-
-# specify your configurations as a dict
-params = {
-    'task' : 'train',
-    'boosting_type' : 'gbdt',
-    'objective' : 'regression',
-    'metric' : {'l2', 'auc'},
-    'num_leaves' : 31,
-    'learning_rate' : 0.05,
-    'feature_fraction' : 0.9,
-    'bagging_fraction' : 0.8,
-    'bagging_freq': 5,
-    'verbose' : 0
-}
-
-# train
-init_gbm = lgb.train(params,
-                lgb_train,
-                num_boost_round=5,
-                valid_sets=lgb_eval)
-
-print('Start continue train')
-
-gbm = lgb.train(params,
-                lgb_train,
-                num_boost_round=100,
+from sklearn.metrics import log_loss, mean_squared_error, mean_absolute_error
+from sklearn.datasets import load_breast_cancer, load_boston, load_digits, load_iris
+from sklearn.model_selection import train_test_split
+
+def multi_logloss(y_true, y_pred):
+    return np.mean([-math.log(y_pred[i][y]) for i, y in enumerate(y_true)])
+
+def test_template(params = {'objective' : 'regression', 'metric' : 'l2'},
+                X_y=load_boston(True), feval=mean_squared_error,
+                stratify=None, num_round=100, return_data=False,
+                return_model=False, init_model=None, custom_eval=None):
+    X, y = X_y
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1,
+                                                        stratify=stratify,
+                                                        random_state=42)
+    lgb_train = lgb.Dataset(X_train, y_train, free_raw_data=not return_model, params=params)
+    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train, free_raw_data=not return_model, params=params)
+    if return_data: return lgb_train, lgb_eval
+    evals_result = {}
+    params['verbose'] = 0
+    gbm = lgb.train(params, lgb_train,
+                    num_boost_round=num_round,
                     valid_sets=lgb_eval,
+                    valid_names='eval',
+                    verbose_eval=False,
+                    feval=custom_eval,
+                    evals_result=evals_result,
                     early_stopping_rounds=10,
-                init_model=init_gbm)
-
-
-# save model to file
-gbm.save_model('model.txt')
-
-# predict
-y_pred = gbm.predict(X_test, num_iteration=gbm.best_iteration)
-# eval
-print('The rmse of prediction is:', mean_squared_error(y_test, y_pred) ** 0.5)
-
-# dump model to json (and save to file)
-model_json = gbm.dump_model()
-
-with open('model.json', 'w+') as f:
-    json.dump(model_json, f, indent=4)
-
-# feature importances
-print('Feature importances:', gbm.feature_importance())
-print('Feature importances:', gbm.feature_importance("gain"))
-
-print('Start test cv')
-
-lgb.cv(params,
-        lgb_train,
-        num_boost_round=100,
-        nfold=5,
-        verbose_eval=5,
-        init_model=init_gbm)
+                    init_model=init_model)
+    if return_model: return gbm
+    else: return evals_result, feval(y_test, gbm.predict(X_test, gbm.best_iteration))
+
+class TestBasic(unittest.TestCase):
+
+    def test_binary(self):
+        X_y= load_breast_cancer(True)
+        params = {
+            'objective' : 'binary',
+            'metric' : 'binary_logloss'
+        }
+        evals_result, ret = test_template(params, X_y, log_loss, stratify=X_y[1])
+        self.assertLess(ret, 0.15)
+        self.assertAlmostEqual(min(evals_result['eval']['logloss']), ret, places=5)
+    
+    def test_regreesion(self):
+        evals_result, ret = test_template()
+        ret **= 0.5
+        self.assertLess(ret, 4)
+        self.assertAlmostEqual(min(evals_result['eval']['l2']), ret, places=5)
+
+    def test_multiclass(self):
+        X_y = load_digits(10, True)
+        params = {
+            'objective' : 'multiclass',
+            'metric' : 'multi_logloss',
+            'num_class' : 10
+        }
+        evals_result, ret = test_template(params, X_y, multi_logloss, stratify=X_y[1])
+        self.assertLess(ret, 0.2)
+        self.assertAlmostEqual(min(evals_result['eval']['multi_logloss']), ret, places=5)
+
+    def test_continue_train_and_other(self):
+        params = {
+            'objective' : 'regression',
+            'metric' : 'l1'
+        }
+        model_name = 'model.txt'
+        gbm = test_template(params, num_round=20, return_model=True)
+        gbm.save_model(model_name)
+        evals_result, ret = test_template(params, feval=mean_absolute_error,
+                                        num_round=80, init_model=model_name,
+                                        custom_eval=(lambda p, d: ('mae', mean_absolute_error(p, d.get_label()), False)))
+        self.assertLess(ret, 3)
+        self.assertAlmostEqual(min(evals_result['eval']['l1']), ret, places=5)
+        for l1, mae in zip(evals_result['eval']['l1'], evals_result['eval']['mae']):
+            self.assertAlmostEqual(l1, mae, places=5)
+        self.assertIn('tree_info', gbm.dump_model())
+        self.assertIsInstance(gbm.feature_importance(), np.ndarray)
+        os.remove(model_name)
+
+    def test_continue_train_multiclass(self):
+        X_y = load_iris(True)
+        params = {
+            'objective' : 'multiclass',
+            'metric' : 'multi_logloss',
+            'num_class' : 3
+        }
+        gbm = test_template(params, X_y, num_round=20, return_model=True, stratify=X_y[1])
+        evals_result, ret = test_template(params, X_y, feval=multi_logloss,
+                                        num_round=80, init_model=gbm)
+        self.assertLess(ret, 1.5)
+        self.assertAlmostEqual(min(evals_result['eval']['multi_logloss']), ret, places=5)
+
+    def test_cv(self):
+        lgb_train, lgb_eval = test_template(return_data=True)
+        lgb.cv({'verbose':0}, lgb_train, num_boost_round=200, nfold=5,
+                metrics='l1', verbose_eval=False)
+
+print("----------------------------------------------------------------------")
+print("running test_engine.py")
+unittest.main()

tests/python_package_test/test_sklearn.py

+# coding: utf-8
+# pylint: skip-file
+import os, unittest
 import numpy as np
-import random
 import lightgbm as lgb
-
-
-rng = np.random.RandomState(2016)
-
-def test_binary_classification():
-
-    from sklearn import datasets, metrics, model_selection
-
-    X, y = datasets.make_classification(n_samples=10000, n_features=100)
-    x_train, x_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.1, random_state=1)
-    lgb_model = lgb.LGBMClassifier().fit(x_train, y_train)
-    from sklearn.datasets import load_digits
-    digits = load_digits(2)
-    y = digits['target']
-    X = digits['data']
-    x_train, x_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.1, random_state=1)
-    lgb_model = lgb.LGBMClassifier().fit(x_train, y_train)
-    preds = lgb_model.predict(x_test)
-    err = sum(1 for i in range(len(preds))
-          if int(preds[i] > 0.5) != y_test[i]) / float(len(preds))
-    assert err < 0.1
-
-def test_multiclass_classification():
-    from sklearn.datasets import load_iris
-    from sklearn import datasets, metrics, model_selection
-
-    def check_pred(preds, labels):
-        err = sum(1 for i in range(len(preds))
-                  if int(preds[i] > 0.5) != labels[i]) / float(len(preds))
-        assert err < 0.7
-
-
-    X, y = datasets.make_classification(n_samples=10000, n_features=100, n_classes=4, n_informative=3)
-
-    x_train, x_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.1, random_state=1)
-
-    lgb_model = lgb.LGBMClassifier().fit(x_train, y_train)
-    preds = lgb_model.predict(x_test)
-
-    check_pred(preds, y_test)
-
-def test_regression():
-    from sklearn.metrics import mean_squared_error
-    from sklearn.datasets import load_boston
-    from sklearn.cross_validation import KFold
-    from sklearn import datasets, metrics, model_selection
-
-    boston = load_boston()
-    y = boston['target']
-    X = boston['data']
-    x_train, x_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.1, random_state=1)
-    lgb_model = lgb.LGBMRegressor().fit(x_train, y_train)
-    preds = lgb_model.predict(x_test)
-    assert mean_squared_error(preds, y_test) < 100
-
-def test_lambdarank():
-    from sklearn.datasets import load_svmlight_file
+from sklearn.metrics import log_loss, mean_squared_error, mean_absolute_error
+from sklearn.datasets import load_breast_cancer, load_boston, load_digits, load_iris, load_svmlight_file
+from sklearn.model_selection import train_test_split, GridSearchCV
+from sklearn.base import clone
+
+def test_template(X_y=load_boston(True), model=lgb.LGBMRegressor,
+                feval=mean_squared_error, stratify=None, num_round=100, return_data=False,
+                return_model=False, init_model=None, custom_obj=None, proba=False):
+    X, y = X_y
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1,
+                                                        stratify=stratify,
+                                                        random_state=42)
+    if return_data: return X_train, X_test, y_train, y_test
+    gbm = model(n_estimators=num_round, objective=custom_obj) if custom_obj else model(n_estimators=num_round)
+    gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=10, verbose=False)
+    if return_model: return gbm
+    else: return feval(y_test, gbm.predict_proba(X_test) if proba else gbm.predict(X_test))
+
+class TestSklearn(unittest.TestCase):
+
+    def test_binary(self):
+        X_y= load_breast_cancer(True)
+        ret = test_template(X_y, lgb.LGBMClassifier, log_loss, stratify=X_y[1], proba=True)
+        self.assertLess(ret, 0.15)
+
+    def test_regreesion(self):
+        self.assertLess(test_template() ** 0.5, 4)
+ 
+    def test_multiclass(self):
+        X_y = load_digits(10, True)
+        def multi_error(y_true, y_pred):
+            return np.mean(y_true != y_pred)
+        ret = test_template(X_y, lgb.LGBMClassifier, multi_error, stratify=X_y[1])
+        self.assertLess(ret, 0.2)
+        
+    def test_lambdarank(self):
         X_train, y_train = load_svmlight_file('../../examples/lambdarank/rank.train')
         X_test, y_test = load_svmlight_file('../../examples/lambdarank/rank.test')
         q_train = np.loadtxt('../../examples/lambdarank/rank.train.query')
         lgb_model = lgb.LGBMRanker().fit(X_train, y_train, group=q_train, eval_at=[1])
 
-def test_regression_with_custom_objective():
-    from sklearn.metrics import mean_squared_error
-    from sklearn.datasets import load_boston
-    from sklearn.cross_validation import KFold
-    from sklearn import datasets, metrics, model_selection
+    def test_regression_with_custom_objective(self):
         def objective_ls(y_true, y_pred):
             grad = (y_pred - y_true)
             hess = np.ones(len(y_true))
             return grad, hess
-    boston = load_boston()
-    y = boston['target']
-    X = boston['data']
-    x_train, x_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.1, random_state=1)
-    lgb_model = lgb.LGBMRegressor(objective=objective_ls).fit(x_train, y_train)
-    preds = lgb_model.predict(x_test)
-    assert mean_squared_error(preds, y_test) < 100
+        ret = test_template(custom_obj=objective_ls)
+        self.assertLess(ret, 100)
 
-
-def test_binary_classification_with_custom_objective():
-
-    from sklearn import datasets, metrics, model_selection
+    def test_binary_classification_with_custom_objective(self):
         def logregobj(y_true, y_pred):
             y_pred = 1.0 / (1.0 + np.exp(-y_pred))
             grad = y_pred - y_true
             hess = y_pred * (1.0 - y_pred)
             return grad, hess
-    X, y = datasets.make_classification(n_samples=10000, n_features=100)
-    x_train, x_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.1, random_state=1)
-    lgb_model = lgb.LGBMClassifier(objective=logregobj).fit(x_train, y_train)
-    from sklearn.datasets import load_digits
-    digits = load_digits(2)
-    y = digits['target']
-    X = digits['data']
-    x_train, x_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.2, random_state=1)
-    lgb_model = lgb.LGBMClassifier(objective=logregobj).fit(x_train, y_train)
-    preds = lgb_model.predict(x_test)
-    err = sum(1 for i in range(len(preds))
-          if int(preds[i] > 0.5) != y_test[i]) / float(len(preds))
-    assert err < 0.1
-
-def test_early_stopping():
-    from sklearn.metrics import mean_squared_error
-    from sklearn.datasets import load_boston
-    from sklearn.cross_validation import KFold
-    from sklearn import datasets, metrics, model_selection
-    from sklearn.base import clone
-
-    boston = load_boston()
-    y = boston['target']
-    X = boston['data']
-    x_train, x_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.1, random_state=1)
-    lgb_model = lgb.LGBMRegressor(n_estimators=500) \
-            .fit(x_train, y_train, eval_set=[(x_test, y_test)], 
-                eval_metric='l2', 
-                early_stopping_rounds=10,
-                verbose=10)
-    lgb_model_clone = clone(lgb_model)
-    print(lgb_model.best_iteration)
-
-test_binary_classification()
-test_multiclass_classification()
-test_regression()
-test_lambdarank()
-test_regression_with_custom_objective()
-test_binary_classification_with_custom_objective()
-test_early_stopping()
+        X_y = load_digits(2, True)
+        def binary_error(y_test, y_pred):
+            return np.mean([int(p > 0.5) != y for y, p in zip(y_test, y_pred)])
+        ret = test_template(X_y, lgb.LGBMClassifier, feval=binary_error, custom_obj=logregobj)
+        self.assertLess(ret, 0.1)
+
+    def test_grid_search(self):
+        X_train, X_test, y_train, y_test = test_template(return_data=True)
+        params = {'n_estimators': [10, 15, 20]}
+        gbm = GridSearchCV(lgb.LGBMRegressor(), params, cv=5)
+        gbm.fit(X_train, y_train)
+        self.assertIn(gbm.best_params_['n_estimators'], [10, 15, 20])
+
+    def test_clone(self):
+        gbm = test_template(return_model=True)
+        gbm_clone = clone(gbm)
+
+print("----------------------------------------------------------------------")
+print("running test_sklearn.py")
+unittest.main()