engine.py 28 KB
Newer Older
wxchan's avatar
wxchan committed
1
# coding: utf-8
2
"""Library with training routines of LightGBM."""
wxchan's avatar
wxchan committed
3
import collections
4
import copy
wxchan's avatar
wxchan committed
5
from operator import attrgetter
6
from pathlib import Path
7
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
8

wxchan's avatar
wxchan committed
9
import numpy as np
10

wxchan's avatar
wxchan committed
11
from . import callback
12
from .basic import Booster, Dataset, LightGBMError, _choose_param_value, _ConfigAliases, _InnerPredictor, _log_warning
13
from .compat import SKLEARN_INSTALLED, _LGBMBaseCrossValidator, _LGBMGroupKFold, _LGBMStratifiedKFold
wxchan's avatar
wxchan committed
14

15
_LGBM_CustomMetricFunction = Callable[
16
    [np.ndarray, Dataset],
17
18
    Tuple[str, float, bool]
]
wxchan's avatar
wxchan committed
19

20
21
22
23
24
_LGBM_PreprocFunction = Callable[
    [Dataset, Dataset, Dict[str, Any]],
    Tuple[Dataset, Dataset, Dict[str, Any]]
]

25
26
27
28
29
30
31
32
33
34
35
36
37
38

def train(
    params: Dict[str, Any],
    train_set: Dataset,
    num_boost_round: int = 100,
    valid_sets: Optional[List[Dataset]] = None,
    valid_names: Optional[List[str]] = None,
    feval: Optional[Union[_LGBM_CustomMetricFunction, List[_LGBM_CustomMetricFunction]]] = None,
    init_model: Optional[Union[str, Path, Booster]] = None,
    feature_name: Union[List[str], str] = 'auto',
    categorical_feature: Union[List[str], List[int], str] = 'auto',
    keep_training_booster: bool = False,
    callbacks: Optional[List[Callable]] = None
) -> Booster:
39
    """Perform the training with given parameters.
wxchan's avatar
wxchan committed
40
41
42
43

    Parameters
    ----------
    params : dict
44
45
        Parameters for training. Values passed through ``params`` take precedence over those
        supplied via arguments.
Guolin Ke's avatar
Guolin Ke committed
46
    train_set : Dataset
47
48
        Data to be trained on.
    num_boost_round : int, optional (default=100)
wxchan's avatar
wxchan committed
49
        Number of boosting iterations.
50
    valid_sets : list of Dataset, or None, optional (default=None)
51
        List of data to be evaluated on during training.
52
    valid_names : list of str, or None, optional (default=None)
53
        Names of ``valid_sets``.
54
    feval : callable, list of callable, or None, optional (default=None)
wxchan's avatar
wxchan committed
55
        Customized evaluation function.
Akshita Dixit's avatar
Akshita Dixit committed
56
        Each evaluation function should accept two parameters: preds, eval_data,
57
        and return (eval_name, eval_result, is_higher_better) or list of such tuples.
58

59
            preds : numpy 1-D array or numpy 2-D array (for multi-class task)
60
                The predicted values.
61
                For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes].
62
                If custom objective function is used, predicted values are returned before any transformation,
63
                e.g. they are raw margin instead of probability of positive class for binary task in this case.
Akshita Dixit's avatar
Akshita Dixit committed
64
            eval_data : Dataset
65
                A ``Dataset`` to evaluate.
66
            eval_name : str
67
                The name of evaluation function (without whitespaces).
68
69
70
71
72
            eval_result : float
                The eval result.
            is_higher_better : bool
                Is eval result higher better, e.g. AUC is ``is_higher_better``.

73
74
        To ignore the default metric corresponding to the used objective,
        set the ``metric`` parameter to the string ``"None"`` in ``params``.
75
    init_model : str, pathlib.Path, Booster or None, optional (default=None)
76
        Filename of LightGBM model or Booster instance used for continue training.
77
    feature_name : list of str, or 'auto', optional (default="auto")
78
79
        Feature names.
        If 'auto' and data is pandas DataFrame, data columns names are used.
80
    categorical_feature : list of str or int, or 'auto', optional (default="auto")
81
82
        Categorical features.
        If list of int, interpreted as indices.
83
        If list of str, interpreted as feature names (need to specify ``feature_name`` as well).
84
        If 'auto' and data is pandas DataFrame, pandas unordered categorical columns are used.
85
        All values in categorical features will be cast to int32 and thus should be less than int32 max value (2147483647).
86
        Large values could be memory consuming. Consider using consecutive integers starting from zero.
87
        All negative values in categorical features will be treated as missing values.
88
        The output cannot be monotonically constrained with respect to a categorical feature.
89
        Floating point numbers in categorical features will be rounded towards 0.
90
91
92
    keep_training_booster : bool, optional (default=False)
        Whether the returned Booster will be used to keep training.
        If False, the returned value will be converted into _InnerPredictor before returning.
93
        This means you won't be able to use ``eval``, ``eval_train`` or ``eval_valid`` methods of the returned Booster.
94
95
        When your model is very large and cause the memory error,
        you can try to set this param to ``True`` to avoid the model conversion performed during the internal call of ``model_to_string``.
96
        You can still use _InnerPredictor as ``init_model`` for future continue training.
97
    callbacks : list of callable, or None, optional (default=None)
98
        List of callback functions that are applied at each iteration.
99
        See Callbacks in Python API for more information.
wxchan's avatar
wxchan committed
100

101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
    Note
    ----
    A custom objective function can be provided for the ``objective`` parameter.
    It should accept two parameters: preds, train_data and return (grad, hess).

        preds : numpy 1-D array or numpy 2-D array (for multi-class task)
            The predicted values.
            Predicted values are returned before any transformation,
            e.g. they are raw margin instead of probability of positive class for binary task.
        train_data : Dataset
            The training dataset.
        grad : numpy 1-D array or numpy 2-D array (for multi-class task)
            The value of the first order derivative (gradient) of the loss
            with respect to the elements of preds for each sample point.
        hess : numpy 1-D array or numpy 2-D array (for multi-class task)
            The value of the second order derivative (Hessian) of the loss
            with respect to the elements of preds for each sample point.

    For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes],
    and grad and hess should be returned in the same format.

wxchan's avatar
wxchan committed
122
123
    Returns
    -------
124
125
    booster : Booster
        The trained Booster model.
wxchan's avatar
wxchan committed
126
    """
127
    # create predictor first
128
    params = copy.deepcopy(params)
129
130
131
132
133
134
135
136
137
    params = _choose_param_value(
        main_param_name='objective',
        params=params,
        default_value=None
    )
    fobj = None
    if callable(params["objective"]):
        fobj = params["objective"]
        params["objective"] = 'none'
138
    for alias in _ConfigAliases.get("num_iterations"):
139
        if alias in params:
140
            num_boost_round = params.pop(alias)
141
            _log_warning(f"Found `{alias}` in params. Will use it instead of argument")
142
    params["num_iterations"] = num_boost_round
143
144
145
146
147
148
149
150
    # setting early stopping via global params should be possible
    params = _choose_param_value(
        main_param_name="early_stopping_round",
        params=params,
        default_value=None
    )
    if params["early_stopping_round"] is None:
        params.pop("early_stopping_round")
151
    first_metric_only = params.get('first_metric_only', False)
152

153
154
    if num_boost_round <= 0:
        raise ValueError("num_boost_round should be greater than zero.")
155
    predictor: Optional[_InnerPredictor] = None
156
    if isinstance(init_model, (str, Path)):
157
        predictor = _InnerPredictor(model_file=init_model, pred_parameter=params)
wxchan's avatar
wxchan committed
158
    elif isinstance(init_model, Booster):
159
        predictor = init_model._to_predictor(dict(init_model.params, **params))
160
    init_iteration = predictor.num_total_iteration if predictor is not None else 0
161
    # check dataset
Guolin Ke's avatar
Guolin Ke committed
162
    if not isinstance(train_set, Dataset):
163
        raise TypeError("Training only accepts Dataset object")
Guolin Ke's avatar
Guolin Ke committed
164

165
166
167
168
    train_set._update_params(params) \
             ._set_predictor(predictor) \
             .set_feature_name(feature_name) \
             .set_categorical_feature(categorical_feature)
Guolin Ke's avatar
Guolin Ke committed
169

wxchan's avatar
wxchan committed
170
171
    is_valid_contain_train = False
    train_data_name = "training"
Guolin Ke's avatar
Guolin Ke committed
172
    reduced_valid_sets = []
wxchan's avatar
wxchan committed
173
    name_valid_sets = []
174
    if valid_sets is not None:
Guolin Ke's avatar
Guolin Ke committed
175
176
        if isinstance(valid_sets, Dataset):
            valid_sets = [valid_sets]
177
        if isinstance(valid_names, str):
wxchan's avatar
wxchan committed
178
            valid_names = [valid_names]
Guolin Ke's avatar
Guolin Ke committed
179
        for i, valid_data in enumerate(valid_sets):
180
            # reduce cost for prediction training data
Guolin Ke's avatar
Guolin Ke committed
181
            if valid_data is train_set:
wxchan's avatar
wxchan committed
182
183
184
185
                is_valid_contain_train = True
                if valid_names is not None:
                    train_data_name = valid_names[i]
                continue
Guolin Ke's avatar
Guolin Ke committed
186
            if not isinstance(valid_data, Dataset):
187
                raise TypeError("Training only accepts Dataset object")
Nikita Titov's avatar
Nikita Titov committed
188
            reduced_valid_sets.append(valid_data._update_params(params).set_reference(train_set))
189
            if valid_names is not None and len(valid_names) > i:
wxchan's avatar
wxchan committed
190
191
                name_valid_sets.append(valid_names[i])
            else:
192
                name_valid_sets.append(f'valid_{i}')
193
    # process callbacks
194
    if callbacks is None:
195
        callbacks_set = set()
wxchan's avatar
wxchan committed
196
197
198
    else:
        for i, cb in enumerate(callbacks):
            cb.__dict__.setdefault('order', i - len(callbacks))
199
        callbacks_set = set(callbacks)
wxchan's avatar
wxchan committed
200

201
202
203
204
205
206
207
208
209
210
211
212
    if "early_stopping_round" in params:
        callbacks_set.add(
            callback.early_stopping(
                stopping_rounds=params["early_stopping_round"],
                first_metric_only=first_metric_only,
                verbose=_choose_param_value(
                    main_param_name="verbosity",
                    params=params,
                    default_value=1
                ).pop("verbosity") > 0
            )
        )
213

214
215
216
217
    callbacks_before_iter_set = {cb for cb in callbacks_set if getattr(cb, 'before_iteration', False)}
    callbacks_after_iter_set = callbacks_set - callbacks_before_iter_set
    callbacks_before_iter = sorted(callbacks_before_iter_set, key=attrgetter('order'))
    callbacks_after_iter = sorted(callbacks_after_iter_set, key=attrgetter('order'))
wxchan's avatar
wxchan committed
218

219
    # construct booster
220
221
222
223
    try:
        booster = Booster(params=params, train_set=train_set)
        if is_valid_contain_train:
            booster.set_train_data_name(train_data_name)
224
        for valid_set, name_valid_set in zip(reduced_valid_sets, name_valid_sets):
225
226
227
228
229
            booster.add_valid(valid_set, name_valid_set)
    finally:
        train_set._reverse_update_params()
        for valid_set in reduced_valid_sets:
            valid_set._reverse_update_params()
230
    booster.best_iteration = 0
wxchan's avatar
wxchan committed
231

232
    # start training
233
    for i in range(init_iteration, init_iteration + num_boost_round):
wxchan's avatar
wxchan committed
234
235
        for cb in callbacks_before_iter:
            cb(callback.CallbackEnv(model=booster,
236
                                    params=params,
wxchan's avatar
wxchan committed
237
                                    iteration=i,
238
239
                                    begin_iteration=init_iteration,
                                    end_iteration=init_iteration + num_boost_round,
wxchan's avatar
wxchan committed
240
241
242
243
244
245
                                    evaluation_result_list=None))

        booster.update(fobj=fobj)

        evaluation_result_list = []
        # check evaluation result.
246
        if valid_sets is not None:
wxchan's avatar
wxchan committed
247
248
249
250
251
252
            if is_valid_contain_train:
                evaluation_result_list.extend(booster.eval_train(feval))
            evaluation_result_list.extend(booster.eval_valid(feval))
        try:
            for cb in callbacks_after_iter:
                cb(callback.CallbackEnv(model=booster,
253
                                        params=params,
wxchan's avatar
wxchan committed
254
                                        iteration=i,
255
256
                                        begin_iteration=init_iteration,
                                        end_iteration=init_iteration + num_boost_round,
wxchan's avatar
wxchan committed
257
                                        evaluation_result_list=evaluation_result_list))
258
259
        except callback.EarlyStopException as earlyStopException:
            booster.best_iteration = earlyStopException.best_iteration + 1
wxchan's avatar
wxchan committed
260
            evaluation_result_list = earlyStopException.best_score
wxchan's avatar
wxchan committed
261
            break
262
    booster.best_score = collections.defaultdict(collections.OrderedDict)
wxchan's avatar
wxchan committed
263
264
    for dataset_name, eval_name, score, _ in evaluation_result_list:
        booster.best_score[dataset_name][eval_name] = score
265
    if not keep_training_booster:
266
        booster.model_from_string(booster.model_to_string()).free_dataset()
wxchan's avatar
wxchan committed
267
268
269
    return booster


270
class CVBooster:
271
272
273
274
275
276
277
278
279
280
281
282
283
    """CVBooster in LightGBM.

    Auxiliary data structure to hold and redirect all boosters of ``cv`` function.
    This class has the same methods as Booster class.
    All method calls are actually performed for underlying Boosters and then all returned results are returned in a list.

    Attributes
    ----------
    boosters : list of Booster
        The list of underlying fitted models.
    best_iteration : int
        The best iteration of fitted model.
    """
284

285
    def __init__(self):
286
287
288
289
        """Initialize the CVBooster.

        Generally, no need to instantiate manually.
        """
290
        self.boosters = []
291
        self.best_iteration = -1
292

293
294
    def _append(self, booster):
        """Add a booster to CVBooster."""
295
296
297
        self.boosters.append(booster)

    def __getattr__(self, name):
298
        """Redirect methods call of CVBooster."""
299
300
        def handler_function(*args, **kwargs):
            """Call methods with each booster, and concatenate their results."""
301
302
303
304
            ret = []
            for booster in self.boosters:
                ret.append(getattr(booster, name)(*args, **kwargs))
            return ret
305
        return handler_function
wxchan's avatar
wxchan committed
306

307

308
309
def _make_n_folds(full_data, folds, nfold, params, seed, fpreproc=None, stratified=True,
                  shuffle=True, eval_train_metric=False):
310
    """Make a n-fold list of Booster from random indices."""
wxchan's avatar
wxchan committed
311
312
    full_data = full_data.construct()
    num_data = full_data.num_data()
313
    if folds is not None:
314
315
316
317
318
319
        if not hasattr(folds, '__iter__') and not hasattr(folds, 'split'):
            raise AttributeError("folds should be a generator or iterator of (train_idx, test_idx) tuples "
                                 "or scikit-learn splitter object with split method")
        if hasattr(folds, 'split'):
            group_info = full_data.get_group()
            if group_info is not None:
320
                group_info = np.array(group_info, dtype=np.int32, copy=False)
321
                flatted_group = np.repeat(range(len(group_info)), repeats=group_info)
322
            else:
323
                flatted_group = np.zeros(num_data, dtype=np.int32)
324
            folds = folds.split(X=np.empty(num_data), y=full_data.get_label(), groups=flatted_group)
wxchan's avatar
wxchan committed
325
    else:
326
327
328
        if any(params.get(obj_alias, "") in {"lambdarank", "rank_xendcg", "xendcg",
                                             "xe_ndcg", "xe_ndcg_mart", "xendcg_mart"}
               for obj_alias in _ConfigAliases.get("objective")):
wxchan's avatar
wxchan committed
329
            if not SKLEARN_INSTALLED:
330
                raise LightGBMError('scikit-learn is required for ranking cv')
331
            # ranking task, split according to groups
332
            group_info = np.array(full_data.get_group(), dtype=np.int32, copy=False)
333
            flatted_group = np.repeat(range(len(group_info)), repeats=group_info)
334
            group_kfold = _LGBMGroupKFold(n_splits=nfold)
335
            folds = group_kfold.split(X=np.empty(num_data), groups=flatted_group)
wxchan's avatar
wxchan committed
336
337
        elif stratified:
            if not SKLEARN_INSTALLED:
338
                raise LightGBMError('scikit-learn is required for stratified cv')
339
            skf = _LGBMStratifiedKFold(n_splits=nfold, shuffle=shuffle, random_state=seed)
340
            folds = skf.split(X=np.empty(num_data), y=full_data.get_label())
extremin's avatar
extremin committed
341
        else:
wxchan's avatar
wxchan committed
342
343
344
345
346
            if shuffle:
                randidx = np.random.RandomState(seed).permutation(num_data)
            else:
                randidx = np.arange(num_data)
            kstep = int(num_data / nfold)
347
348
349
            test_id = [randidx[i: i + kstep] for i in range(0, num_data, kstep)]
            train_id = [np.concatenate([test_id[i] for i in range(nfold) if k != i]) for k in range(nfold)]
            folds = zip(train_id, test_id)
wxchan's avatar
wxchan committed
350

351
    ret = CVBooster()
wxchan's avatar
wxchan committed
352
    for train_idx, test_idx in folds:
353
354
        train_set = full_data.subset(sorted(train_idx))
        valid_set = full_data.subset(sorted(test_idx))
wxchan's avatar
wxchan committed
355
356
        # run preprocessing on the data set if needed
        if fpreproc is not None:
wxchan's avatar
wxchan committed
357
            train_set, valid_set, tparam = fpreproc(train_set, valid_set, params.copy())
wxchan's avatar
wxchan committed
358
        else:
wxchan's avatar
wxchan committed
359
            tparam = params
360
        cvbooster = Booster(tparam, train_set)
361
362
        if eval_train_metric:
            cvbooster.add_valid(train_set, 'train')
363
        cvbooster.add_valid(valid_set, 'valid')
364
        ret._append(cvbooster)
wxchan's avatar
wxchan committed
365
366
    return ret

wxchan's avatar
wxchan committed
367

368
def _agg_cv_result(raw_results):
369
    """Aggregate cross-validation results."""
370
    cvmap = collections.OrderedDict()
wxchan's avatar
wxchan committed
371
372
373
    metric_type = {}
    for one_result in raw_results:
        for one_line in one_result:
374
            key = f"{one_line[0]} {one_line[1]}"
375
            metric_type[key] = one_line[3]
376
            cvmap.setdefault(key, [])
377
            cvmap[key].append(one_line[2])
wxchan's avatar
wxchan committed
378
    return [('cv_agg', k, np.mean(v), metric_type[k], np.std(v)) for k, v in cvmap.items()]
wxchan's avatar
wxchan committed
379

wxchan's avatar
wxchan committed
380

381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
def cv(
    params: Dict[str, Any],
    train_set: Dataset,
    num_boost_round: int = 100,
    folds: Optional[Union[Iterable[Tuple[np.ndarray, np.ndarray]], _LGBMBaseCrossValidator]] = None,
    nfold: int = 5,
    stratified: bool = True,
    shuffle: bool = True,
    metrics: Optional[Union[str, List[str]]] = None,
    feval: Optional[Union[_LGBM_CustomMetricFunction, List[_LGBM_CustomMetricFunction]]] = None,
    init_model: Optional[Union[str, Path, Booster]] = None,
    feature_name: Union[str, List[str]] = 'auto',
    categorical_feature: Union[str, List[str], List[int]] = 'auto',
    fpreproc: Optional[_LGBM_PreprocFunction] = None,
    seed: int = 0,
    callbacks: Optional[List[Callable]] = None,
    eval_train_metric: bool = False,
    return_cvbooster: bool = False
) -> Dict[str, Any]:
Andrew Ziem's avatar
Andrew Ziem committed
400
    """Perform the cross-validation with given parameters.
wxchan's avatar
wxchan committed
401
402
403
404

    Parameters
    ----------
    params : dict
405
406
        Parameters for training. Values passed through ``params`` take precedence over those
        supplied via arguments.
Guolin Ke's avatar
Guolin Ke committed
407
    train_set : Dataset
408
        Data to be trained on.
409
    num_boost_round : int, optional (default=100)
wxchan's avatar
wxchan committed
410
        Number of boosting iterations.
411
    folds : generator or iterator of (train_idx, test_idx) tuples, scikit-learn splitter object or None, optional (default=None)
412
        If generator or iterator, it should yield the train and test indices for each fold.
413
        If object, it should be one of the scikit-learn splitter classes
414
        (https://scikit-learn.org/stable/modules/classes.html#splitter-classes)
415
        and have ``split`` method.
416
        This argument has highest priority over other data split arguments.
417
    nfold : int, optional (default=5)
wxchan's avatar
wxchan committed
418
        Number of folds in CV.
419
420
    stratified : bool, optional (default=True)
        Whether to perform stratified sampling.
421
    shuffle : bool, optional (default=True)
422
        Whether to shuffle before splitting data.
423
    metrics : str, list of str, or None, optional (default=None)
424
425
        Evaluation metrics to be monitored while CV.
        If not None, the metric in ``params`` will be overridden.
426
    feval : callable, list of callable, or None, optional (default=None)
427
        Customized evaluation function.
428
        Each evaluation function should accept two parameters: preds, eval_data,
429
        and return (eval_name, eval_result, is_higher_better) or list of such tuples.
430

431
            preds : numpy 1-D array or numpy 2-D array (for multi-class task)
432
                The predicted values.
433
                For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes].
434
                If custom objective function is used, predicted values are returned before any transformation,
435
                e.g. they are raw margin instead of probability of positive class for binary task in this case.
436
437
            eval_data : Dataset
                A ``Dataset`` to evaluate.
438
            eval_name : str
Andrew Ziem's avatar
Andrew Ziem committed
439
                The name of evaluation function (without whitespace).
440
441
442
443
444
            eval_result : float
                The eval result.
            is_higher_better : bool
                Is eval result higher better, e.g. AUC is ``is_higher_better``.

445
446
        To ignore the default metric corresponding to the used objective,
        set ``metrics`` to the string ``"None"``.
447
    init_model : str, pathlib.Path, Booster or None, optional (default=None)
448
        Filename of LightGBM model or Booster instance used for continue training.
449
    feature_name : list of str, or 'auto', optional (default="auto")
450
451
        Feature names.
        If 'auto' and data is pandas DataFrame, data columns names are used.
452
    categorical_feature : list of str or int, or 'auto', optional (default="auto")
453
454
        Categorical features.
        If list of int, interpreted as indices.
455
        If list of str, interpreted as feature names (need to specify ``feature_name`` as well).
456
        If 'auto' and data is pandas DataFrame, pandas unordered categorical columns are used.
457
        All values in categorical features will be cast to int32 and thus should be less than int32 max value (2147483647).
458
        Large values could be memory consuming. Consider using consecutive integers starting from zero.
459
        All negative values in categorical features will be treated as missing values.
460
        The output cannot be monotonically constrained with respect to a categorical feature.
461
        Floating point numbers in categorical features will be rounded towards 0.
462
463
    fpreproc : callable or None, optional (default=None)
        Preprocessing function that takes (dtrain, dtest, params)
wxchan's avatar
wxchan committed
464
        and returns transformed versions of those.
465
    seed : int, optional (default=0)
wxchan's avatar
wxchan committed
466
        Seed used to generate the folds (passed to numpy.random.seed).
467
    callbacks : list of callable, or None, optional (default=None)
468
        List of callback functions that are applied at each iteration.
469
        See Callbacks in Python API for more information.
470
471
472
    eval_train_metric : bool, optional (default=False)
        Whether to display the train metric in progress.
        The score of the metric is calculated again after each training step, so there is some impact on performance.
473
474
    return_cvbooster : bool, optional (default=False)
        Whether to return Booster models trained on each fold through ``CVBooster``.
wxchan's avatar
wxchan committed
475

476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
    Note
    ----
    A custom objective function can be provided for the ``objective`` parameter.
    It should accept two parameters: preds, train_data and return (grad, hess).

        preds : numpy 1-D array or numpy 2-D array (for multi-class task)
            The predicted values.
            Predicted values are returned before any transformation,
            e.g. they are raw margin instead of probability of positive class for binary task.
        train_data : Dataset
            The training dataset.
        grad : numpy 1-D array or numpy 2-D array (for multi-class task)
            The value of the first order derivative (gradient) of the loss
            with respect to the elements of preds for each sample point.
        hess : numpy 1-D array or numpy 2-D array (for multi-class task)
            The value of the second order derivative (Hessian) of the loss
            with respect to the elements of preds for each sample point.

    For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes],
    and grad and hess should be returned in the same format.

wxchan's avatar
wxchan committed
497
498
    Returns
    -------
499
500
501
502
    eval_hist : dict
        Evaluation history.
        The dictionary has the following format:
        {'metric1-mean': [values], 'metric1-stdv': [values],
Qiwei Ye's avatar
Qiwei Ye committed
503
        'metric2-mean': [values], 'metric2-stdv': [values],
504
        ...}.
505
        If ``return_cvbooster=True``, also returns trained boosters via ``cvbooster`` key.
wxchan's avatar
wxchan committed
506
    """
Guolin Ke's avatar
Guolin Ke committed
507
    if not isinstance(train_set, Dataset):
508
        raise TypeError("Training only accepts Dataset object")
509
    params = copy.deepcopy(params)
510
511
512
513
514
515
516
517
518
    params = _choose_param_value(
        main_param_name='objective',
        params=params,
        default_value=None
    )
    fobj = None
    if callable(params["objective"]):
        fobj = params["objective"]
        params["objective"] = 'none'
519
    for alias in _ConfigAliases.get("num_iterations"):
520
        if alias in params:
521
            _log_warning(f"Found '{alias}' in params. Will use it instead of 'num_boost_round' argument")
522
            num_boost_round = params.pop(alias)
523
    params["num_iterations"] = num_boost_round
524
525
526
527
528
529
530
531
    # setting early stopping via global params should be possible
    params = _choose_param_value(
        main_param_name="early_stopping_round",
        params=params,
        default_value=None
    )
    if params["early_stopping_round"] is None:
        params.pop("early_stopping_round")
532
    first_metric_only = params.get('first_metric_only', False)
533

534
535
    if num_boost_round <= 0:
        raise ValueError("num_boost_round should be greater than zero.")
536
    if isinstance(init_model, (str, Path)):
537
        predictor = _InnerPredictor(model_file=init_model, pred_parameter=params)
Guolin Ke's avatar
Guolin Ke committed
538
    elif isinstance(init_model, Booster):
539
        predictor = init_model._to_predictor(dict(init_model.params, **params))
Guolin Ke's avatar
Guolin Ke committed
540
541
542
    else:
        predictor = None

Peter's avatar
Peter committed
543
    if metrics is not None:
544
545
        for metric_alias in _ConfigAliases.get("metric"):
            params.pop(metric_alias, None)
Peter's avatar
Peter committed
546
        params['metric'] = metrics
wxchan's avatar
wxchan committed
547

548
549
550
551
552
    train_set._update_params(params) \
             ._set_predictor(predictor) \
             .set_feature_name(feature_name) \
             .set_categorical_feature(categorical_feature)

wxchan's avatar
wxchan committed
553
    results = collections.defaultdict(list)
554
555
    cvfolds = _make_n_folds(train_set, folds=folds, nfold=nfold,
                            params=params, seed=seed, fpreproc=fpreproc,
556
557
                            stratified=stratified, shuffle=shuffle,
                            eval_train_metric=eval_train_metric)
wxchan's avatar
wxchan committed
558
559

    # setup callbacks
560
    if callbacks is None:
wxchan's avatar
wxchan committed
561
562
563
564
565
        callbacks = set()
    else:
        for i, cb in enumerate(callbacks):
            cb.__dict__.setdefault('order', i - len(callbacks))
        callbacks = set(callbacks)
566
567
568
569
570
571
572
573
574
575
576
577
578

    if "early_stopping_round" in params:
        callbacks.add(
            callback.early_stopping(
                stopping_rounds=params["early_stopping_round"],
                first_metric_only=first_metric_only,
                verbose=_choose_param_value(
                    main_param_name="verbosity",
                    params=params,
                    default_value=1
                ).pop("verbosity") > 0
            )
        )
wxchan's avatar
wxchan committed
579

wxchan's avatar
wxchan committed
580
581
582
583
    callbacks_before_iter = {cb for cb in callbacks if getattr(cb, 'before_iteration', False)}
    callbacks_after_iter = callbacks - callbacks_before_iter
    callbacks_before_iter = sorted(callbacks_before_iter, key=attrgetter('order'))
    callbacks_after_iter = sorted(callbacks_after_iter, key=attrgetter('order'))
wxchan's avatar
wxchan committed
584

585
    for i in range(num_boost_round):
wxchan's avatar
wxchan committed
586
        for cb in callbacks_before_iter:
587
588
            cb(callback.CallbackEnv(model=cvfolds,
                                    params=params,
wxchan's avatar
wxchan committed
589
590
591
592
                                    iteration=i,
                                    begin_iteration=0,
                                    end_iteration=num_boost_round,
                                    evaluation_result_list=None))
wxchan's avatar
wxchan committed
593
        cvfolds.update(fobj=fobj)
594
        res = _agg_cv_result(cvfolds.eval_valid(feval))
wxchan's avatar
wxchan committed
595
        for _, key, mean, _, std in res:
596
597
            results[f'{key}-mean'].append(mean)
            results[f'{key}-stdv'].append(std)
wxchan's avatar
wxchan committed
598
599
        try:
            for cb in callbacks_after_iter:
600
601
                cb(callback.CallbackEnv(model=cvfolds,
                                        params=params,
wxchan's avatar
wxchan committed
602
603
604
605
                                        iteration=i,
                                        begin_iteration=0,
                                        end_iteration=num_boost_round,
                                        evaluation_result_list=res))
606
607
        except callback.EarlyStopException as earlyStopException:
            cvfolds.best_iteration = earlyStopException.best_iteration + 1
608
609
            for bst in cvfolds.boosters:
                bst.best_iteration = cvfolds.best_iteration
wxchan's avatar
wxchan committed
610
            for k in results:
611
                results[k] = results[k][:cvfolds.best_iteration]
wxchan's avatar
wxchan committed
612
            break
613
614
615
616

    if return_cvbooster:
        results['cvbooster'] = cvfolds

wxchan's avatar
wxchan committed
617
    return dict(results)