engine.py 29 KB
Newer Older
wxchan's avatar
wxchan committed
1
# coding: utf-8
2
"""Library with training routines of LightGBM."""
wxchan's avatar
wxchan committed
3
import collections
4
import copy
wxchan's avatar
wxchan committed
5
from operator import attrgetter
6

wxchan's avatar
wxchan committed
7
import numpy as np
8

wxchan's avatar
wxchan committed
9
from . import callback
10
from .basic import Booster, Dataset, LightGBMError, _ConfigAliases, _InnerPredictor, _log_warning
11
from .compat import SKLEARN_INSTALLED, _LGBMGroupKFold, _LGBMStratifiedKFold
wxchan's avatar
wxchan committed
12

wxchan's avatar
wxchan committed
13

Guolin Ke's avatar
Guolin Ke committed
14
15
def train(params, train_set, num_boost_round=100,
          valid_sets=None, valid_names=None,
wxchan's avatar
wxchan committed
16
          fobj=None, feval=None, init_model=None,
17
          feature_name='auto', categorical_feature='auto',
wxchan's avatar
wxchan committed
18
          early_stopping_rounds=None, evals_result=None,
19
20
          verbose_eval=True, learning_rates=None,
          keep_training_booster=False, callbacks=None):
21
    """Perform the training with given parameters.
wxchan's avatar
wxchan committed
22
23
24
25

    Parameters
    ----------
    params : dict
26
        Parameters for training.
Guolin Ke's avatar
Guolin Ke committed
27
    train_set : Dataset
28
29
        Data to be trained on.
    num_boost_round : int, optional (default=100)
wxchan's avatar
wxchan committed
30
        Number of boosting iterations.
31
32
33
    valid_sets : list of Datasets or None, optional (default=None)
        List of data to be evaluated on during training.
    valid_names : list of strings or None, optional (default=None)
34
35
        Names of ``valid_sets``.
    fobj : callable or None, optional (default=None)
wxchan's avatar
wxchan committed
36
        Customized objective function.
37
38
39
40
41
42
43
44
45
46
47
48
        Should accept two parameters: preds, train_data,
        and return (grad, hess).

            preds : list or numpy 1-D array
                The predicted values.
            train_data : Dataset
                The training dataset.
            grad : list or numpy 1-D array
                The value of the first order derivative (gradient) for each sample point.
            hess : list or numpy 1-D array
                The value of the second order derivative (Hessian) for each sample point.

49
        For binary task, the preds is margin.
50
51
52
53
        For multi-class task, the preds is group by class_id first, then group by row_id.
        If you want to get i-th row preds in j-th class, the access way is score[j * num_data + i]
        and you should group grad and hess in this way as well.

54
    feval : callable, list of callable functions or None, optional (default=None)
wxchan's avatar
wxchan committed
55
        Customized evaluation function.
56
        Each evaluation function should accept two parameters: preds, train_data,
57
        and return (eval_name, eval_result, is_higher_better) or list of such tuples.
58
59
60
61
62
63

            preds : list or numpy 1-D array
                The predicted values.
            train_data : Dataset
                The training dataset.
            eval_name : string
64
                The name of evaluation function (without whitespaces).
65
66
67
68
69
            eval_result : float
                The eval result.
            is_higher_better : bool
                Is eval result higher better, e.g. AUC is ``is_higher_better``.

70
        For binary task, the preds is probability of positive class (or margin in case of specified ``fobj``).
71
72
        For multi-class task, the preds is group by class_id first, then group by row_id.
        If you want to get i-th row preds in j-th class, the access way is preds[j * num_data + i].
73
74
        To ignore the default metric corresponding to the used objective,
        set the ``metric`` parameter to the string ``"None"`` in ``params``.
75
    init_model : string, Booster or None, optional (default=None)
76
77
78
79
80
81
82
83
        Filename of LightGBM model or Booster instance used for continue training.
    feature_name : list of strings or 'auto', optional (default="auto")
        Feature names.
        If 'auto' and data is pandas DataFrame, data columns names are used.
    categorical_feature : list of strings or int, or 'auto', optional (default="auto")
        Categorical features.
        If list of int, interpreted as indices.
        If list of strings, interpreted as feature names (need to specify ``feature_name`` as well).
84
        If 'auto' and data is pandas DataFrame, pandas unordered categorical columns are used.
85
        All values in categorical features should be less than int32 max value (2147483647).
86
        Large values could be memory consuming. Consider using consecutive integers starting from zero.
87
        All negative values in categorical features will be treated as missing values.
88
        The output cannot be monotonically constrained with respect to a categorical feature.
89
    early_stopping_rounds : int or None, optional (default=None)
90
        Activates early stopping. The model will train until the validation score stops improving.
91
92
93
94
        Validation score needs to improve at least every ``early_stopping_rounds`` round(s)
        to continue training.
        Requires at least one validation data and one metric.
        If there's more than one, will check all of them. But the training data is ignored anyway.
95
        To check only the first metric, set the ``first_metric_only`` parameter to ``True`` in ``params``.
96
97
98
        The index of iteration that has the best performance will be saved in the ``best_iteration`` field
        if early stopping logic is enabled by setting ``early_stopping_rounds``.
    evals_result: dict or None, optional (default=None)
99
100
        This dictionary used to store all evaluation results of all the items in ``valid_sets``.

Nikita Titov's avatar
Nikita Titov committed
101
102
        .. rubric:: Example

103
104
        With a ``valid_sets`` = [valid_set, train_set],
        ``valid_names`` = ['eval', 'train']
105
106
        and a ``params`` = {'metric': 'logloss'}
        returns {'train': {'logloss': ['0.48253', '0.35953', ...]},
107
        'eval': {'logloss': ['0.480385', '0.357756', ...]}}.
108

109
110
111
112
113
114
    verbose_eval : bool or int, optional (default=True)
        Requires at least one validation data.
        If True, the eval metric on the valid set is printed at each boosting stage.
        If int, the eval metric on the valid set is printed at every ``verbose_eval`` boosting stage.
        The last boosting stage or the boosting stage found by using ``early_stopping_rounds`` is also printed.

Nikita Titov's avatar
Nikita Titov committed
115
116
        .. rubric:: Example

117
        With ``verbose_eval`` = 4 and at least one item in ``valid_sets``,
118
        an evaluation metric is printed every 4 (instead of 1) boosting stages.
119
120

    learning_rates : list, callable or None, optional (default=None)
121
122
123
124
125
126
        List of learning rates for each boosting round
        or a customized function that calculates ``learning_rate``
        in terms of current number of round (e.g. yields learning rate decay).
    keep_training_booster : bool, optional (default=False)
        Whether the returned Booster will be used to keep training.
        If False, the returned value will be converted into _InnerPredictor before returning.
127
128
        When your model is very large and cause the memory error,
        you can try to set this param to ``True`` to avoid the model conversion performed during the internal call of ``model_to_string``.
129
130
        You can still use _InnerPredictor as ``init_model`` for future continue training.
    callbacks : list of callables or None, optional (default=None)
131
        List of callback functions that are applied at each iteration.
132
        See Callbacks in Python API for more information.
wxchan's avatar
wxchan committed
133
134
135

    Returns
    -------
136
137
    booster : Booster
        The trained Booster model.
wxchan's avatar
wxchan committed
138
    """
139
    # create predictor first
140
    params = copy.deepcopy(params)
141
    if fobj is not None:
142
143
        for obj_alias in _ConfigAliases.get("objective"):
            params.pop(obj_alias, None)
144
        params['objective'] = 'none'
145
    for alias in _ConfigAliases.get("num_iterations"):
146
        if alias in params:
147
            num_boost_round = params.pop(alias)
148
            _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
149
    params["num_iterations"] = num_boost_round
150
    for alias in _ConfigAliases.get("early_stopping_round"):
151
152
        if alias in params:
            early_stopping_rounds = params.pop(alias)
153
            _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
154
155
    params["early_stopping_round"] = early_stopping_rounds
    first_metric_only = params.get('first_metric_only', False)
156

157
158
    if num_boost_round <= 0:
        raise ValueError("num_boost_round should be greater than zero.")
159
    if isinstance(init_model, str):
160
        predictor = _InnerPredictor(model_file=init_model, pred_parameter=params)
wxchan's avatar
wxchan committed
161
    elif isinstance(init_model, Booster):
162
        predictor = init_model._to_predictor(dict(init_model.params, **params))
wxchan's avatar
wxchan committed
163
164
    else:
        predictor = None
165
    init_iteration = predictor.num_total_iteration if predictor is not None else 0
166
    # check dataset
Guolin Ke's avatar
Guolin Ke committed
167
    if not isinstance(train_set, Dataset):
168
        raise TypeError("Training only accepts Dataset object")
Guolin Ke's avatar
Guolin Ke committed
169

170
171
172
173
    train_set._update_params(params) \
             ._set_predictor(predictor) \
             .set_feature_name(feature_name) \
             .set_categorical_feature(categorical_feature)
Guolin Ke's avatar
Guolin Ke committed
174

wxchan's avatar
wxchan committed
175
176
    is_valid_contain_train = False
    train_data_name = "training"
Guolin Ke's avatar
Guolin Ke committed
177
    reduced_valid_sets = []
wxchan's avatar
wxchan committed
178
    name_valid_sets = []
179
    if valid_sets is not None:
Guolin Ke's avatar
Guolin Ke committed
180
181
        if isinstance(valid_sets, Dataset):
            valid_sets = [valid_sets]
182
        if isinstance(valid_names, str):
wxchan's avatar
wxchan committed
183
            valid_names = [valid_names]
Guolin Ke's avatar
Guolin Ke committed
184
        for i, valid_data in enumerate(valid_sets):
185
            # reduce cost for prediction training data
Guolin Ke's avatar
Guolin Ke committed
186
            if valid_data is train_set:
wxchan's avatar
wxchan committed
187
188
189
190
                is_valid_contain_train = True
                if valid_names is not None:
                    train_data_name = valid_names[i]
                continue
Guolin Ke's avatar
Guolin Ke committed
191
            if not isinstance(valid_data, Dataset):
192
                raise TypeError("Training only accepts Dataset object")
Nikita Titov's avatar
Nikita Titov committed
193
            reduced_valid_sets.append(valid_data._update_params(params).set_reference(train_set))
194
            if valid_names is not None and len(valid_names) > i:
wxchan's avatar
wxchan committed
195
196
                name_valid_sets.append(valid_names[i])
            else:
wxchan's avatar
wxchan committed
197
                name_valid_sets.append('valid_' + str(i))
198
    # process callbacks
199
    if callbacks is None:
wxchan's avatar
wxchan committed
200
201
202
203
204
        callbacks = set()
    else:
        for i, cb in enumerate(callbacks):
            cb.__dict__.setdefault('order', i - len(callbacks))
        callbacks = set(callbacks)
wxchan's avatar
wxchan committed
205
206

    # Most of legacy advanced options becomes callbacks
wxchan's avatar
wxchan committed
207
208
    if verbose_eval is True:
        callbacks.add(callback.print_evaluation())
209
    elif isinstance(verbose_eval, int):
wxchan's avatar
wxchan committed
210
        callbacks.add(callback.print_evaluation(verbose_eval))
wxchan's avatar
wxchan committed
211

212
    if early_stopping_rounds is not None and early_stopping_rounds > 0:
213
        callbacks.add(callback.early_stopping(early_stopping_rounds, first_metric_only, verbose=bool(verbose_eval)))
214

wxchan's avatar
wxchan committed
215
    if learning_rates is not None:
216
        callbacks.add(callback.reset_parameter(learning_rate=learning_rates))
wxchan's avatar
wxchan committed
217
218

    if evals_result is not None:
wxchan's avatar
wxchan committed
219
220
221
222
223
224
        callbacks.add(callback.record_evaluation(evals_result))

    callbacks_before_iter = {cb for cb in callbacks if getattr(cb, 'before_iteration', False)}
    callbacks_after_iter = callbacks - callbacks_before_iter
    callbacks_before_iter = sorted(callbacks_before_iter, key=attrgetter('order'))
    callbacks_after_iter = sorted(callbacks_after_iter, key=attrgetter('order'))
wxchan's avatar
wxchan committed
225

226
    # construct booster
227
228
229
230
    try:
        booster = Booster(params=params, train_set=train_set)
        if is_valid_contain_train:
            booster.set_train_data_name(train_data_name)
231
        for valid_set, name_valid_set in zip(reduced_valid_sets, name_valid_sets):
232
233
234
235
236
            booster.add_valid(valid_set, name_valid_set)
    finally:
        train_set._reverse_update_params()
        for valid_set in reduced_valid_sets:
            valid_set._reverse_update_params()
237
    booster.best_iteration = 0
wxchan's avatar
wxchan committed
238

239
    # start training
240
    for i in range(init_iteration, init_iteration + num_boost_round):
wxchan's avatar
wxchan committed
241
242
        for cb in callbacks_before_iter:
            cb(callback.CallbackEnv(model=booster,
243
                                    params=params,
wxchan's avatar
wxchan committed
244
                                    iteration=i,
245
246
                                    begin_iteration=init_iteration,
                                    end_iteration=init_iteration + num_boost_round,
wxchan's avatar
wxchan committed
247
248
249
250
251
252
                                    evaluation_result_list=None))

        booster.update(fobj=fobj)

        evaluation_result_list = []
        # check evaluation result.
253
        if valid_sets is not None:
wxchan's avatar
wxchan committed
254
255
256
257
258
259
            if is_valid_contain_train:
                evaluation_result_list.extend(booster.eval_train(feval))
            evaluation_result_list.extend(booster.eval_valid(feval))
        try:
            for cb in callbacks_after_iter:
                cb(callback.CallbackEnv(model=booster,
260
                                        params=params,
wxchan's avatar
wxchan committed
261
                                        iteration=i,
262
263
                                        begin_iteration=init_iteration,
                                        end_iteration=init_iteration + num_boost_round,
wxchan's avatar
wxchan committed
264
                                        evaluation_result_list=evaluation_result_list))
265
266
        except callback.EarlyStopException as earlyStopException:
            booster.best_iteration = earlyStopException.best_iteration + 1
wxchan's avatar
wxchan committed
267
            evaluation_result_list = earlyStopException.best_score
wxchan's avatar
wxchan committed
268
            break
269
    booster.best_score = collections.defaultdict(collections.OrderedDict)
wxchan's avatar
wxchan committed
270
271
    for dataset_name, eval_name, score, _ in evaluation_result_list:
        booster.best_score[dataset_name][eval_name] = score
272
    if not keep_training_booster:
Nikita Titov's avatar
Nikita Titov committed
273
        booster.model_from_string(booster.model_to_string(), False).free_dataset()
wxchan's avatar
wxchan committed
274
275
276
    return booster


277
class CVBooster:
278
279
280
281
282
283
284
285
286
287
288
289
290
    """CVBooster in LightGBM.

    Auxiliary data structure to hold and redirect all boosters of ``cv`` function.
    This class has the same methods as Booster class.
    All method calls are actually performed for underlying Boosters and then all returned results are returned in a list.

    Attributes
    ----------
    boosters : list of Booster
        The list of underlying fitted models.
    best_iteration : int
        The best iteration of fitted model.
    """
291

292
    def __init__(self):
293
294
295
296
        """Initialize the CVBooster.

        Generally, no need to instantiate manually.
        """
297
        self.boosters = []
298
        self.best_iteration = -1
299

300
301
    def _append(self, booster):
        """Add a booster to CVBooster."""
302
303
304
        self.boosters.append(booster)

    def __getattr__(self, name):
305
        """Redirect methods call of CVBooster."""
306
307
        def handler_function(*args, **kwargs):
            """Call methods with each booster, and concatenate their results."""
308
309
310
311
            ret = []
            for booster in self.boosters:
                ret.append(getattr(booster, name)(*args, **kwargs))
            return ret
312
        return handler_function
wxchan's avatar
wxchan committed
313

314

315
316
def _make_n_folds(full_data, folds, nfold, params, seed, fpreproc=None, stratified=True,
                  shuffle=True, eval_train_metric=False):
317
    """Make a n-fold list of Booster from random indices."""
wxchan's avatar
wxchan committed
318
319
    full_data = full_data.construct()
    num_data = full_data.num_data()
320
    if folds is not None:
321
322
323
324
325
326
        if not hasattr(folds, '__iter__') and not hasattr(folds, 'split'):
            raise AttributeError("folds should be a generator or iterator of (train_idx, test_idx) tuples "
                                 "or scikit-learn splitter object with split method")
        if hasattr(folds, 'split'):
            group_info = full_data.get_group()
            if group_info is not None:
327
                group_info = np.array(group_info, dtype=np.int32, copy=False)
328
                flatted_group = np.repeat(range(len(group_info)), repeats=group_info)
329
            else:
330
                flatted_group = np.zeros(num_data, dtype=np.int32)
331
            folds = folds.split(X=np.zeros(num_data), y=full_data.get_label(), groups=flatted_group)
wxchan's avatar
wxchan committed
332
    else:
333
334
335
        if any(params.get(obj_alias, "") in {"lambdarank", "rank_xendcg", "xendcg",
                                             "xe_ndcg", "xe_ndcg_mart", "xendcg_mart"}
               for obj_alias in _ConfigAliases.get("objective")):
wxchan's avatar
wxchan committed
336
            if not SKLEARN_INSTALLED:
337
                raise LightGBMError('scikit-learn is required for ranking cv')
338
            # ranking task, split according to groups
339
            group_info = np.array(full_data.get_group(), dtype=np.int32, copy=False)
340
            flatted_group = np.repeat(range(len(group_info)), repeats=group_info)
341
            group_kfold = _LGBMGroupKFold(n_splits=nfold)
wxchan's avatar
wxchan committed
342
343
344
            folds = group_kfold.split(X=np.zeros(num_data), groups=flatted_group)
        elif stratified:
            if not SKLEARN_INSTALLED:
345
                raise LightGBMError('scikit-learn is required for stratified cv')
346
            skf = _LGBMStratifiedKFold(n_splits=nfold, shuffle=shuffle, random_state=seed)
wxchan's avatar
wxchan committed
347
            folds = skf.split(X=np.zeros(num_data), y=full_data.get_label())
extremin's avatar
extremin committed
348
        else:
wxchan's avatar
wxchan committed
349
350
351
352
353
            if shuffle:
                randidx = np.random.RandomState(seed).permutation(num_data)
            else:
                randidx = np.arange(num_data)
            kstep = int(num_data / nfold)
354
355
356
            test_id = [randidx[i: i + kstep] for i in range(0, num_data, kstep)]
            train_id = [np.concatenate([test_id[i] for i in range(nfold) if k != i]) for k in range(nfold)]
            folds = zip(train_id, test_id)
wxchan's avatar
wxchan committed
357

358
    ret = CVBooster()
wxchan's avatar
wxchan committed
359
    for train_idx, test_idx in folds:
360
361
        train_set = full_data.subset(sorted(train_idx))
        valid_set = full_data.subset(sorted(test_idx))
wxchan's avatar
wxchan committed
362
363
        # run preprocessing on the data set if needed
        if fpreproc is not None:
wxchan's avatar
wxchan committed
364
            train_set, valid_set, tparam = fpreproc(train_set, valid_set, params.copy())
wxchan's avatar
wxchan committed
365
        else:
wxchan's avatar
wxchan committed
366
            tparam = params
367
        cvbooster = Booster(tparam, train_set)
368
369
        if eval_train_metric:
            cvbooster.add_valid(train_set, 'train')
370
        cvbooster.add_valid(valid_set, 'valid')
371
        ret._append(cvbooster)
wxchan's avatar
wxchan committed
372
373
    return ret

wxchan's avatar
wxchan committed
374

375
def _agg_cv_result(raw_results, eval_train_metric=False):
376
    """Aggregate cross-validation results."""
377
    cvmap = collections.OrderedDict()
wxchan's avatar
wxchan committed
378
379
380
    metric_type = {}
    for one_result in raw_results:
        for one_line in one_result:
381
382
383
384
385
            if eval_train_metric:
                key = "{} {}".format(one_line[0], one_line[1])
            else:
                key = one_line[1]
            metric_type[key] = one_line[3]
386
            cvmap.setdefault(key, [])
387
            cvmap[key].append(one_line[2])
wxchan's avatar
wxchan committed
388
    return [('cv_agg', k, np.mean(v), metric_type[k], np.std(v)) for k, v in cvmap.items()]
wxchan's avatar
wxchan committed
389

wxchan's avatar
wxchan committed
390

391
def cv(params, train_set, num_boost_round=100,
392
       folds=None, nfold=5, stratified=True, shuffle=True,
wxchan's avatar
wxchan committed
393
       metrics=None, fobj=None, feval=None, init_model=None,
394
       feature_name='auto', categorical_feature='auto',
Guolin Ke's avatar
Guolin Ke committed
395
396
       early_stopping_rounds=None, fpreproc=None,
       verbose_eval=None, show_stdv=True, seed=0,
397
398
       callbacks=None, eval_train_metric=False,
       return_cvbooster=False):
399
    """Perform the cross-validation with given paramaters.
wxchan's avatar
wxchan committed
400
401
402
403

    Parameters
    ----------
    params : dict
404
        Parameters for Booster.
Guolin Ke's avatar
Guolin Ke committed
405
    train_set : Dataset
406
        Data to be trained on.
407
    num_boost_round : int, optional (default=100)
wxchan's avatar
wxchan committed
408
        Number of boosting iterations.
409
    folds : generator or iterator of (train_idx, test_idx) tuples, scikit-learn splitter object or None, optional (default=None)
410
        If generator or iterator, it should yield the train and test indices for each fold.
411
        If object, it should be one of the scikit-learn splitter classes
412
        (https://scikit-learn.org/stable/modules/classes.html#splitter-classes)
413
        and have ``split`` method.
414
        This argument has highest priority over other data split arguments.
415
    nfold : int, optional (default=5)
wxchan's avatar
wxchan committed
416
        Number of folds in CV.
417
418
    stratified : bool, optional (default=True)
        Whether to perform stratified sampling.
419
    shuffle : bool, optional (default=True)
420
421
422
423
424
        Whether to shuffle before splitting data.
    metrics : string, list of strings or None, optional (default=None)
        Evaluation metrics to be monitored while CV.
        If not None, the metric in ``params`` will be overridden.
    fobj : callable or None, optional (default=None)
425
426
427
428
429
430
431
432
433
434
435
436
437
        Customized objective function.
        Should accept two parameters: preds, train_data,
        and return (grad, hess).

            preds : list or numpy 1-D array
                The predicted values.
            train_data : Dataset
                The training dataset.
            grad : list or numpy 1-D array
                The value of the first order derivative (gradient) for each sample point.
            hess : list or numpy 1-D array
                The value of the second order derivative (Hessian) for each sample point.

438
        For binary task, the preds is margin.
439
440
441
442
        For multi-class task, the preds is group by class_id first, then group by row_id.
        If you want to get i-th row preds in j-th class, the access way is score[j * num_data + i]
        and you should group grad and hess in this way as well.

443
    feval : callable, list of callable functions or None, optional (default=None)
444
        Customized evaluation function.
445
        Each evaluation function should accept two parameters: preds, train_data,
446
        and return (eval_name, eval_result, is_higher_better) or list of such tuples.
447
448
449
450
451
452

            preds : list or numpy 1-D array
                The predicted values.
            train_data : Dataset
                The training dataset.
            eval_name : string
453
                The name of evaluation function (without whitespaces).
454
455
456
457
458
            eval_result : float
                The eval result.
            is_higher_better : bool
                Is eval result higher better, e.g. AUC is ``is_higher_better``.

459
        For binary task, the preds is probability of positive class (or margin in case of specified ``fobj``).
460
461
        For multi-class task, the preds is group by class_id first, then group by row_id.
        If you want to get i-th row preds in j-th class, the access way is preds[j * num_data + i].
462
463
        To ignore the default metric corresponding to the used objective,
        set ``metrics`` to the string ``"None"``.
464
    init_model : string, Booster or None, optional (default=None)
465
466
467
468
469
470
471
472
        Filename of LightGBM model or Booster instance used for continue training.
    feature_name : list of strings or 'auto', optional (default="auto")
        Feature names.
        If 'auto' and data is pandas DataFrame, data columns names are used.
    categorical_feature : list of strings or int, or 'auto', optional (default="auto")
        Categorical features.
        If list of int, interpreted as indices.
        If list of strings, interpreted as feature names (need to specify ``feature_name`` as well).
473
        If 'auto' and data is pandas DataFrame, pandas unordered categorical columns are used.
474
        All values in categorical features should be less than int32 max value (2147483647).
475
        Large values could be memory consuming. Consider using consecutive integers starting from zero.
476
        All negative values in categorical features will be treated as missing values.
477
        The output cannot be monotonically constrained with respect to a categorical feature.
478
    early_stopping_rounds : int or None, optional (default=None)
479
480
481
482
        Activates early stopping.
        CV score needs to improve at least every ``early_stopping_rounds`` round(s)
        to continue.
        Requires at least one metric. If there's more than one, will check all of them.
483
        To check only the first metric, set the ``first_metric_only`` parameter to ``True`` in ``params``.
484
        Last entry in evaluation history is the one from the best iteration.
485
486
    fpreproc : callable or None, optional (default=None)
        Preprocessing function that takes (dtrain, dtest, params)
wxchan's avatar
wxchan committed
487
        and returns transformed versions of those.
488
    verbose_eval : bool, int, or None, optional (default=None)
wxchan's avatar
wxchan committed
489
490
        Whether to display the progress.
        If None, progress will be displayed when np.ndarray is returned.
491
492
493
        If True, progress will be displayed at every boosting stage.
        If int, progress will be displayed at every given ``verbose_eval`` boosting stage.
    show_stdv : bool, optional (default=True)
wxchan's avatar
wxchan committed
494
        Whether to display the standard deviation in progress.
495
        Results are not affected by this parameter, and always contain std.
496
    seed : int, optional (default=0)
wxchan's avatar
wxchan committed
497
        Seed used to generate the folds (passed to numpy.random.seed).
498
    callbacks : list of callables or None, optional (default=None)
499
        List of callback functions that are applied at each iteration.
500
        See Callbacks in Python API for more information.
501
502
503
    eval_train_metric : bool, optional (default=False)
        Whether to display the train metric in progress.
        The score of the metric is calculated again after each training step, so there is some impact on performance.
504
505
    return_cvbooster : bool, optional (default=False)
        Whether to return Booster models trained on each fold through ``CVBooster``.
wxchan's avatar
wxchan committed
506
507
508

    Returns
    -------
509
510
511
512
    eval_hist : dict
        Evaluation history.
        The dictionary has the following format:
        {'metric1-mean': [values], 'metric1-stdv': [values],
Qiwei Ye's avatar
Qiwei Ye committed
513
        'metric2-mean': [values], 'metric2-stdv': [values],
514
        ...}.
515
        If ``return_cvbooster=True``, also returns trained boosters via ``cvbooster`` key.
wxchan's avatar
wxchan committed
516
    """
Guolin Ke's avatar
Guolin Ke committed
517
    if not isinstance(train_set, Dataset):
518
        raise TypeError("Training only accepts Dataset object")
Guolin Ke's avatar
Guolin Ke committed
519

520
    params = copy.deepcopy(params)
521
    if fobj is not None:
522
523
        for obj_alias in _ConfigAliases.get("objective"):
            params.pop(obj_alias, None)
524
        params['objective'] = 'none'
525
    for alias in _ConfigAliases.get("num_iterations"):
526
        if alias in params:
527
            _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
528
            num_boost_round = params.pop(alias)
529
    params["num_iterations"] = num_boost_round
530
    for alias in _ConfigAliases.get("early_stopping_round"):
531
        if alias in params:
532
            _log_warning("Found `{}` in params. Will use it instead of argument".format(alias))
533
            early_stopping_rounds = params.pop(alias)
534
535
    params["early_stopping_round"] = early_stopping_rounds
    first_metric_only = params.get('first_metric_only', False)
536

537
538
    if num_boost_round <= 0:
        raise ValueError("num_boost_round should be greater than zero.")
539
    if isinstance(init_model, str):
540
        predictor = _InnerPredictor(model_file=init_model, pred_parameter=params)
Guolin Ke's avatar
Guolin Ke committed
541
    elif isinstance(init_model, Booster):
542
        predictor = init_model._to_predictor(dict(init_model.params, **params))
Guolin Ke's avatar
Guolin Ke committed
543
544
545
    else:
        predictor = None

Peter's avatar
Peter committed
546
    if metrics is not None:
547
548
        for metric_alias in _ConfigAliases.get("metric"):
            params.pop(metric_alias, None)
Peter's avatar
Peter committed
549
        params['metric'] = metrics
wxchan's avatar
wxchan committed
550

551
552
553
554
555
    train_set._update_params(params) \
             ._set_predictor(predictor) \
             .set_feature_name(feature_name) \
             .set_categorical_feature(categorical_feature)

wxchan's avatar
wxchan committed
556
    results = collections.defaultdict(list)
557
558
    cvfolds = _make_n_folds(train_set, folds=folds, nfold=nfold,
                            params=params, seed=seed, fpreproc=fpreproc,
559
560
                            stratified=stratified, shuffle=shuffle,
                            eval_train_metric=eval_train_metric)
wxchan's avatar
wxchan committed
561
562

    # setup callbacks
563
    if callbacks is None:
wxchan's avatar
wxchan committed
564
565
566
567
568
        callbacks = set()
    else:
        for i, cb in enumerate(callbacks):
            cb.__dict__.setdefault('order', i - len(callbacks))
        callbacks = set(callbacks)
569
    if early_stopping_rounds is not None and early_stopping_rounds > 0:
570
        callbacks.add(callback.early_stopping(early_stopping_rounds, first_metric_only, verbose=False))
wxchan's avatar
wxchan committed
571
572
    if verbose_eval is True:
        callbacks.add(callback.print_evaluation(show_stdv=show_stdv))
573
    elif isinstance(verbose_eval, int):
wxchan's avatar
wxchan committed
574
        callbacks.add(callback.print_evaluation(verbose_eval, show_stdv=show_stdv))
wxchan's avatar
wxchan committed
575

wxchan's avatar
wxchan committed
576
577
578
579
    callbacks_before_iter = {cb for cb in callbacks if getattr(cb, 'before_iteration', False)}
    callbacks_after_iter = callbacks - callbacks_before_iter
    callbacks_before_iter = sorted(callbacks_before_iter, key=attrgetter('order'))
    callbacks_after_iter = sorted(callbacks_after_iter, key=attrgetter('order'))
wxchan's avatar
wxchan committed
580

581
    for i in range(num_boost_round):
wxchan's avatar
wxchan committed
582
        for cb in callbacks_before_iter:
583
584
            cb(callback.CallbackEnv(model=cvfolds,
                                    params=params,
wxchan's avatar
wxchan committed
585
586
587
588
                                    iteration=i,
                                    begin_iteration=0,
                                    end_iteration=num_boost_round,
                                    evaluation_result_list=None))
wxchan's avatar
wxchan committed
589
        cvfolds.update(fobj=fobj)
590
        res = _agg_cv_result(cvfolds.eval_valid(feval), eval_train_metric)
wxchan's avatar
wxchan committed
591
592
        for _, key, mean, _, std in res:
            results[key + '-mean'].append(mean)
wxchan's avatar
wxchan committed
593
            results[key + '-stdv'].append(std)
wxchan's avatar
wxchan committed
594
595
        try:
            for cb in callbacks_after_iter:
596
597
                cb(callback.CallbackEnv(model=cvfolds,
                                        params=params,
wxchan's avatar
wxchan committed
598
599
600
601
                                        iteration=i,
                                        begin_iteration=0,
                                        end_iteration=num_boost_round,
                                        evaluation_result_list=res))
602
603
        except callback.EarlyStopException as earlyStopException:
            cvfolds.best_iteration = earlyStopException.best_iteration + 1
wxchan's avatar
wxchan committed
604
            for k in results:
605
                results[k] = results[k][:cvfolds.best_iteration]
wxchan's avatar
wxchan committed
606
            break
607
608
609
610

    if return_cvbooster:
        results['cvbooster'] = cvfolds

wxchan's avatar
wxchan committed
611
    return dict(results)