re-define callback order (#114)

current problem:
callback order: user-defined in callback parameter -> reset_learning_rate/print_evaluation/record_evaluation/early_stop
user can't insert a callback between last 4 callbacks

solution:
set order variable:
reset_learning_rate = 10
print_evaluation = 10
record_evaluation = 20
early_stop = 30
user-defined = some int
default: according to index in callback parameter list, = index - len(callbacks) (all < 0)

current callback order:
before iter: user-defined -> reset_learning_rate -> user-defined
after iter: user-defined -> print_evaluation -> user-defined -> record_evaluation -> user-defined -> early_stop -> user-defined

.gitignore

@@ -362,3 +362,6 @@ ENV/
 
 # Rope project settings
 .ropeproject
+
+# macOS
+.DS_Store

python-package/lightgbm/callback.py

 # coding: utf-8
-# pylint: disable = invalid-name, W0105
+# pylint: disable = invalid-name, W0105, C0301
 from __future__ import absolute_import
 import collections
+import inspect
 
 class EarlyStopException(Exception):
     """Exception of early stopping.
@@ -57,10 +58,11 @@ def print_evaluation(period=1, show_stdv=True):
         """internal function"""
         if not env.evaluation_result_list or period <= 0:
             return
-        if env.iteration % period == 0 or env.iteration + 1 == env.begin_iteration:
+        if (env.iteration + 1) % period == 0:
             result = '\t'.join([_format_eval_result(x, show_stdv) \
                 for x in env.evaluation_result_list])
-            print('[%d]\t%s' % (env.iteration, result))
+            print('[%d]\t%s' % (env.iteration + 1, result))
+    callback.order = 10
     return callback
 
 
@@ -92,6 +94,7 @@ def record_evaluation(eval_result):
             init(env)
         for data_name, eval_name, result, _ in env.evaluation_result_list:
             eval_result[data_name][eval_name].append(result)
+    callback.order = 20
     return callback
 
 
@@ -108,8 +111,8 @@ def reset_learning_rate(learning_rates):
         current number of round and the total number of boosting round \
         (e.g. yields learning rate decay)
         - list l: learning_rate = l[current_round]
-        - function f: learning_rate = f(current_round, total_boost_round)
-
+        - function f: learning_rate = f(current_round, total_boost_round) \
+                   or learning_rate = f(current_round)
     Returns
     -------
     callback : function
@@ -117,15 +120,21 @@ def reset_learning_rate(learning_rates):
     """
     def callback(env):
         """internal function"""
-        booster = env.model
-        iteration = env.iteration
         if isinstance(learning_rates, list):
-            if len(learning_rates) != env.end_iteration:
-                raise ValueError("Length of list 'learning_rates' has to equal 'num_boost_round'.")
-            booster.reset_parameter({'learning_rate':learning_rates[iteration]})
+            if len(learning_rates) != env.end_iteration - env.begin_iteration:
+                raise ValueError("Length of list 'learning_rates' has to equal to 'num_boost_round'.")
+            env.model.reset_parameter({'learning_rate':learning_rates[env.iteration]})
         else:
-            booster.reset_parameter({'learning_rate':learning_rates(iteration, env.end_iteration)})
+            argc = len(inspect.getargspec(learning_rates).args)
+            if argc is 1:
+                env.model.reset_parameter({"learning_rate": learning_rates(env.iteration - env.begin_iteration)})
+            elif argc is 2:
+                env.model.reset_parameter({"learning_rate": \
+                    learning_rates(env.iteration - env.begin_iteration, env.end_iteration - env.begin_iteration)})
+            else:
+                raise ValueError("Self-defined function 'learning_rates' should have 1 or 2 arguments")
     callback.before_iteration = True
+    callback.order = 10
     return callback
 
 
@@ -178,7 +187,7 @@ def early_stop(stopping_rounds, verbose=True):
                 best_score[i] = score
                 best_iter[i] = env.iteration
                 if verbose:
-                    best_msg[i] = '[%d]\t%s' % (env.iteration, \
+                    best_msg[i] = '[%d]\t%s' % (env.iteration + 1, \
                         '\t'.join([_format_eval_result(x) for x in env.evaluation_result_list]))
             else:
                 if env.iteration - best_iter[i] >= stopping_rounds:
@@ -188,4 +197,5 @@ def early_stop(stopping_rounds, verbose=True):
                         print('early stopping, best iteration is:')
                         print(best_msg[i])
                     raise EarlyStopException(best_iter[i])
+    callback.order = 30
     return callback

python-package/lightgbm/engine.py

@@ -3,6 +3,8 @@
 """Training Library containing training routines of LightGBM."""
 from __future__ import absolute_import
 
+import collections
+from operator import attrgetter
 import numpy as np
 from .basic import LightGBMError, Predictor, Dataset, Booster, is_str
 from . import callback
@@ -20,7 +22,7 @@ def _construct_dataset(X_y, reference=None,
     init_score = None
     if other_fields is not None:
         if not isinstance(other_fields, dict):
-            raise TypeError("other filed data should be dict type")
+            raise TypeError("type of other filed data should be dict")
         weight = other_fields.get('weight', None)
         group = other_fields.get('group', None)
         init_score = other_fields.get('init_score', None)
@@ -29,7 +31,7 @@ def _construct_dataset(X_y, reference=None,
         label = None
     else:
         if len(X_y) != 2:
-            raise TypeError("should pass (data, label) pair")
+            raise TypeError("should pass (data, label) tuple for dataset")
         data = X_y[0]
         label = X_y[1]
     if reference is None:
@@ -114,7 +116,8 @@ def train(params, train_data, num_boost_round=100,
         current number of round and the total number of boosting round \
         (e.g. yields learning rate decay)
         - list l: learning_rate = l[current_round]
-        - function f: learning_rate = f(current_round, total_boost_round)
+        - function f: learning_rate = f(current_round, total_boost_round) \
+        or learning_rate = f(current_round)
     callbacks : list of callback functions
         List of callback functions that are applied at end of each iteration.
 
@@ -148,7 +151,7 @@ def train(params, train_data, num_boost_round=100,
     train_data_name = "training"
     valid_sets = []
     name_valid_sets = []
-    if valid_datas is not None:
+    if valid_datas:
         if isinstance(valid_datas, (Dataset, tuple)):
             valid_datas = [valid_datas]
         if isinstance(valid_names, str):
@@ -180,35 +183,43 @@ def train(params, train_data, num_boost_round=100,
                 name_valid_sets.append(valid_names[i])
             else:
                 name_valid_sets.append('valid_'+str(i))
+
     """process callbacks"""
-    callbacks = [] if callbacks is None else callbacks
+    if not callbacks:
+        callbacks = set()
+    else:
+        for i, cb in enumerate(callbacks):
+            cb.__dict__.setdefault('order', i - len(callbacks))
+        callbacks = set(callbacks)
 
     # Most of legacy advanced options becomes callbacks
-    if isinstance(verbose_eval, bool) and verbose_eval:
-        callbacks.append(callback.print_evaluation())
+    if verbose_eval is True:
+        callbacks.add(callback.print_evaluation())
     elif isinstance(verbose_eval, int):
-        callbacks.append(callback.print_evaluation(verbose_eval))
+        callbacks.add(callback.print_evaluation(verbose_eval))
 
-    if early_stopping_rounds is not None:
-        callbacks.append(callback.early_stop(early_stopping_rounds,
-                                             verbose=bool(verbose_eval)))
+    if early_stopping_rounds:
+        callbacks.add(callback.early_stop(early_stopping_rounds,
+                                          verbose=bool(verbose_eval)))
 
     if learning_rates is not None:
-        callbacks.append(callback.reset_learning_rate(learning_rates))
+        callbacks.add(callback.reset_learning_rate(learning_rates))
 
     if evals_result is not None:
-        callbacks.append(callback.record_evaluation(evals_result))
+        callbacks.add(callback.record_evaluation(evals_result))
+
+    callbacks_before_iter = {cb for cb in callbacks if getattr(cb, 'before_iteration', False)}
+    callbacks_after_iter = callbacks - callbacks_before_iter
+    callbacks_before_iter = sorted(callbacks_before_iter, key=attrgetter('order'))
+    callbacks_after_iter = sorted(callbacks_after_iter, key=attrgetter('order'))
 
-    callbacks_before_iter = [
-        cb for cb in callbacks if cb.__dict__.get('before_iteration', False)]
-    callbacks_after_iter = [
-        cb for cb in callbacks if not cb.__dict__.get('before_iteration', False)]
     """construct booster"""
     booster = Booster(params=params, train_set=train_set)
     if is_valid_contain_train:
         booster.set_train_data_name(train_data_name)
     for valid_set, name_valid_set in zip(valid_sets, name_valid_sets):
         booster.add_valid(valid_set, name_valid_set)
+
     """start training"""
     for i in range(init_iteration, init_iteration + num_boost_round):
         for cb in callbacks_before_iter:
@@ -271,7 +282,7 @@ except ImportError:
     except ImportError:
         SKLEARN_StratifiedKFold = False
 
-def _make_n_folds(full_data, nfold, param, seed, fpreproc=None, stratified=False):
+def _make_n_folds(full_data, nfold, params, seed, fpreproc=None, stratified=False):
     """
     Make an n-fold list of CVBooster from random indices.
     """
@@ -293,9 +304,9 @@ def _make_n_folds(full_data, nfold, param, seed, fpreproc=None, stratified=False
         valid_set = full_data.subset(idset[k])
         # run preprocessing on the data set if needed
         if fpreproc is not None:
-            train_set, valid_set, tparam = fpreproc(train_set, valid_set, param.copy())
+            train_set, valid_set, tparam = fpreproc(train_set, valid_set, params.copy())
         else:
-            tparam = param
+            tparam = params
         ret.append(CVBooster(train_set, valid_set, tparam))
     return ret
 
@@ -303,21 +314,13 @@ def _agg_cv_result(raw_results):
     """
     Aggregate cross-validation results.
     """
-    cvmap = {}
+    cvmap = collections.defaultdict(list)
     metric_type = {}
     for one_result in raw_results:
         for one_line in one_result:
-            key = one_line[1]
-            metric_type[key] = one_line[3]
-            if key not in cvmap:
-                cvmap[key] = []
-            cvmap[key].append(one_line[2])
-    results = []
-    for k, v in cvmap.items():
-        v = np.array(v)
-        mean, std = np.mean(v), np.std(v)
-        results.append(('cv_agg', k, mean, metric_type[k], std))
-    return results
+            metric_type[one_line[1]] = one_line[3]
+            cvmap[one_line[1]].append(one_line[2])
+    return [('cv_agg', k, np.mean(v), metric_type[k], np.std(v)) for k, v in cvmap.items()]
 
 def cv(params, train_data, num_boost_round=10, nfold=5, stratified=False,
        metrics=(), fobj=None, feval=None, train_fields=None,
@@ -331,7 +334,7 @@ def cv(params, train_data, num_boost_round=10, nfold=5, stratified=False,
     ----------
     params : dict
         Booster params.
-    train_data : pair, (X, y) or filename of data
+    train_data : tuple (X, y) or filename of data
         Data to be trained.
     num_boost_round : int
         Number of boosting iterations.
@@ -391,23 +394,27 @@ def cv(params, train_data, num_boost_round=10, nfold=5, stratified=False,
                                    feature_name=feature_name,
                                    categorical_feature=categorical_feature)
 
-    results = {}
+    results = collections.defaultdict(list)
     cvfolds = _make_n_folds(train_set, nfold, params, seed, fpreproc, stratified)
 
     # setup callbacks
-    callbacks = [] if callbacks is None else callbacks
-    if early_stopping_rounds is not None:
-        callbacks.append(callback.early_stop(early_stopping_rounds,
-                                             verbose=False))
-    if isinstance(verbose_eval, bool) and verbose_eval:
-        callbacks.append(callback.print_evaluation(show_stdv=show_stdv))
+    if not callbacks:
+        callbacks = set()
+    else:
+        for i, cb in enumerate(callbacks):
+            cb.__dict__.setdefault('order', i - len(callbacks))
+        callbacks = set(callbacks)
+    if early_stopping_rounds:
+        callbacks.add(callback.early_stop(early_stopping_rounds, verbose=False))
+    if verbose_eval is True:
+        callbacks.add(callback.print_evaluation(show_stdv=show_stdv))
     elif isinstance(verbose_eval, int):
-        callbacks.append(callback.print_evaluation(verbose_eval, show_stdv=show_stdv))
+        callbacks.add(callback.print_evaluation(verbose_eval, show_stdv=show_stdv))
 
-    callbacks_before_iter = [
-        cb for cb in callbacks if cb.__dict__.get('before_iteration', False)]
-    callbacks_after_iter = [
-        cb for cb in callbacks if not cb.__dict__.get('before_iteration', False)]
+    callbacks_before_iter = {cb for cb in callbacks if getattr(cb, 'before_iteration', False)}
+    callbacks_after_iter = callbacks - callbacks_before_iter
+    callbacks_before_iter = sorted(callbacks_before_iter, key=attrgetter('order'))
+    callbacks_after_iter = sorted(callbacks_after_iter, key=attrgetter('order'))
 
     for i in range(num_boost_round):
         for cb in callbacks_before_iter:
@@ -421,12 +428,8 @@ def cv(params, train_data, num_boost_round=10, nfold=5, stratified=False,
             fold.update(fobj)
         res = _agg_cv_result([f.eval(feval) for f in cvfolds])
         for _, key, mean, _, std in res:
-            if key + '-mean' not in results:
-                results[key + '-mean'] = []
-            if key + '-std' not in results:
-                results[key + '-std'] = []
             results[key + '-mean'].append(mean)
-            results[key + '-std'].append(std)
+            results[key + '-stdv'].append(std)
         try:
             for cb in callbacks_after_iter:
                 cb(callback.CallbackEnv(model=None,
@@ -437,6 +440,6 @@ def cv(params, train_data, num_boost_round=10, nfold=5, stratified=False,
                                         evaluation_result_list=res))
         except callback.EarlyStopException as e:
             for k in results:
-                results[k] = results[k][:(e.best_iteration + 1)]
+                results[k] = results[k][:e.best_iteration + 1]
             break
-    return results
+    return dict(results)