engine.py 33.1 KB
Newer Older
wxchan's avatar
wxchan committed
1
# coding: utf-8
2
"""Library with training routines of LightGBM."""
wxchan's avatar
wxchan committed
3
import collections
4
import copy
5
import json
wxchan's avatar
wxchan committed
6
from operator import attrgetter
7
from pathlib import Path
8
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
9

wxchan's avatar
wxchan committed
10
import numpy as np
11

wxchan's avatar
wxchan committed
12
from . import callback
13
from .basic import (Booster, Dataset, LightGBMError, _choose_param_value, _ConfigAliases, _InnerPredictor,
14
                    _LGBM_CategoricalFeatureConfiguration, _LGBM_CustomObjectiveFunction, _LGBM_EvalFunctionResultType,
15
                    _LGBM_FeatureNameConfiguration, _log_warning)
16
from .compat import SKLEARN_INSTALLED, _LGBMBaseCrossValidator, _LGBMGroupKFold, _LGBMStratifiedKFold
wxchan's avatar
wxchan committed
17

18
19
20
21
22
23
24
__all__ = [
    'cv',
    'CVBooster',
    'train',
]


25
26
27
28
29
30
31
32
33
_LGBM_CustomMetricFunction = Union[
    Callable[
        [np.ndarray, Dataset],
        _LGBM_EvalFunctionResultType,
    ],
    Callable[
        [np.ndarray, Dataset],
        List[_LGBM_EvalFunctionResultType]
    ],
34
]
wxchan's avatar
wxchan committed
35

36
37
38
39
40
_LGBM_PreprocFunction = Callable[
    [Dataset, Dataset, Dict[str, Any]],
    Tuple[Dataset, Dataset, Dict[str, Any]]
]

41
42
43
44
45
46
47
48
49

def train(
    params: Dict[str, Any],
    train_set: Dataset,
    num_boost_round: int = 100,
    valid_sets: Optional[List[Dataset]] = None,
    valid_names: Optional[List[str]] = None,
    feval: Optional[Union[_LGBM_CustomMetricFunction, List[_LGBM_CustomMetricFunction]]] = None,
    init_model: Optional[Union[str, Path, Booster]] = None,
50
51
    feature_name: _LGBM_FeatureNameConfiguration = 'auto',
    categorical_feature: _LGBM_CategoricalFeatureConfiguration = 'auto',
52
53
54
    keep_training_booster: bool = False,
    callbacks: Optional[List[Callable]] = None
) -> Booster:
55
    """Perform the training with given parameters.
wxchan's avatar
wxchan committed
56
57
58
59

    Parameters
    ----------
    params : dict
60
61
        Parameters for training. Values passed through ``params`` take precedence over those
        supplied via arguments.
Guolin Ke's avatar
Guolin Ke committed
62
    train_set : Dataset
63
64
        Data to be trained on.
    num_boost_round : int, optional (default=100)
wxchan's avatar
wxchan committed
65
        Number of boosting iterations.
66
    valid_sets : list of Dataset, or None, optional (default=None)
67
        List of data to be evaluated on during training.
68
    valid_names : list of str, or None, optional (default=None)
69
        Names of ``valid_sets``.
70
    feval : callable, list of callable, or None, optional (default=None)
wxchan's avatar
wxchan committed
71
        Customized evaluation function.
Akshita Dixit's avatar
Akshita Dixit committed
72
        Each evaluation function should accept two parameters: preds, eval_data,
73
        and return (eval_name, eval_result, is_higher_better) or list of such tuples.
74

75
            preds : numpy 1-D array or numpy 2-D array (for multi-class task)
76
                The predicted values.
77
                For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes].
78
                If custom objective function is used, predicted values are returned before any transformation,
79
                e.g. they are raw margin instead of probability of positive class for binary task in this case.
Akshita Dixit's avatar
Akshita Dixit committed
80
            eval_data : Dataset
81
                A ``Dataset`` to evaluate.
82
            eval_name : str
83
                The name of evaluation function (without whitespaces).
84
85
86
87
88
            eval_result : float
                The eval result.
            is_higher_better : bool
                Is eval result higher better, e.g. AUC is ``is_higher_better``.

89
90
        To ignore the default metric corresponding to the used objective,
        set the ``metric`` parameter to the string ``"None"`` in ``params``.
91
    init_model : str, pathlib.Path, Booster or None, optional (default=None)
92
        Filename of LightGBM model or Booster instance used for continue training.
93
    feature_name : list of str, or 'auto', optional (default="auto")
94
95
        Feature names.
        If 'auto' and data is pandas DataFrame, data columns names are used.
96
    categorical_feature : list of str or int, or 'auto', optional (default="auto")
97
98
        Categorical features.
        If list of int, interpreted as indices.
99
        If list of str, interpreted as feature names (need to specify ``feature_name`` as well).
100
        If 'auto' and data is pandas DataFrame, pandas unordered categorical columns are used.
101
        All values in categorical features will be cast to int32 and thus should be less than int32 max value (2147483647).
102
        Large values could be memory consuming. Consider using consecutive integers starting from zero.
103
        All negative values in categorical features will be treated as missing values.
104
        The output cannot be monotonically constrained with respect to a categorical feature.
105
        Floating point numbers in categorical features will be rounded towards 0.
106
107
108
    keep_training_booster : bool, optional (default=False)
        Whether the returned Booster will be used to keep training.
        If False, the returned value will be converted into _InnerPredictor before returning.
109
        This means you won't be able to use ``eval``, ``eval_train`` or ``eval_valid`` methods of the returned Booster.
110
111
        When your model is very large and cause the memory error,
        you can try to set this param to ``True`` to avoid the model conversion performed during the internal call of ``model_to_string``.
112
        You can still use _InnerPredictor as ``init_model`` for future continue training.
113
    callbacks : list of callable, or None, optional (default=None)
114
        List of callback functions that are applied at each iteration.
115
        See Callbacks in Python API for more information.
wxchan's avatar
wxchan committed
116

117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
    Note
    ----
    A custom objective function can be provided for the ``objective`` parameter.
    It should accept two parameters: preds, train_data and return (grad, hess).

        preds : numpy 1-D array or numpy 2-D array (for multi-class task)
            The predicted values.
            Predicted values are returned before any transformation,
            e.g. they are raw margin instead of probability of positive class for binary task.
        train_data : Dataset
            The training dataset.
        grad : numpy 1-D array or numpy 2-D array (for multi-class task)
            The value of the first order derivative (gradient) of the loss
            with respect to the elements of preds for each sample point.
        hess : numpy 1-D array or numpy 2-D array (for multi-class task)
            The value of the second order derivative (Hessian) of the loss
            with respect to the elements of preds for each sample point.

    For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes],
    and grad and hess should be returned in the same format.

wxchan's avatar
wxchan committed
138
139
    Returns
    -------
140
141
    booster : Booster
        The trained Booster model.
wxchan's avatar
wxchan committed
142
    """
143
    # create predictor first
144
    params = copy.deepcopy(params)
145
146
147
148
149
    params = _choose_param_value(
        main_param_name='objective',
        params=params,
        default_value=None
    )
150
    fobj: Optional[_LGBM_CustomObjectiveFunction] = None
151
152
153
    if callable(params["objective"]):
        fobj = params["objective"]
        params["objective"] = 'none'
154
    for alias in _ConfigAliases.get("num_iterations"):
155
        if alias in params:
156
            num_boost_round = params.pop(alias)
157
            _log_warning(f"Found `{alias}` in params. Will use it instead of argument")
158
    params["num_iterations"] = num_boost_round
159
160
161
162
163
164
165
166
    # setting early stopping via global params should be possible
    params = _choose_param_value(
        main_param_name="early_stopping_round",
        params=params,
        default_value=None
    )
    if params["early_stopping_round"] is None:
        params.pop("early_stopping_round")
167
    first_metric_only = params.get('first_metric_only', False)
168

169
170
    if num_boost_round <= 0:
        raise ValueError("num_boost_round should be greater than zero.")
171
    predictor: Optional[_InnerPredictor] = None
172
    if isinstance(init_model, (str, Path)):
173
        predictor = _InnerPredictor(model_file=init_model, pred_parameter=params)
wxchan's avatar
wxchan committed
174
    elif isinstance(init_model, Booster):
175
        predictor = init_model._to_predictor(dict(init_model.params, **params))
176
    init_iteration = predictor.num_total_iteration if predictor is not None else 0
177
    # check dataset
Guolin Ke's avatar
Guolin Ke committed
178
    if not isinstance(train_set, Dataset):
179
        raise TypeError("Training only accepts Dataset object")
Guolin Ke's avatar
Guolin Ke committed
180

181
182
183
184
    train_set._update_params(params) \
             ._set_predictor(predictor) \
             .set_feature_name(feature_name) \
             .set_categorical_feature(categorical_feature)
Guolin Ke's avatar
Guolin Ke committed
185

wxchan's avatar
wxchan committed
186
187
    is_valid_contain_train = False
    train_data_name = "training"
Guolin Ke's avatar
Guolin Ke committed
188
    reduced_valid_sets = []
wxchan's avatar
wxchan committed
189
    name_valid_sets = []
190
    if valid_sets is not None:
Guolin Ke's avatar
Guolin Ke committed
191
192
        if isinstance(valid_sets, Dataset):
            valid_sets = [valid_sets]
193
        if isinstance(valid_names, str):
wxchan's avatar
wxchan committed
194
            valid_names = [valid_names]
Guolin Ke's avatar
Guolin Ke committed
195
        for i, valid_data in enumerate(valid_sets):
196
            # reduce cost for prediction training data
Guolin Ke's avatar
Guolin Ke committed
197
            if valid_data is train_set:
wxchan's avatar
wxchan committed
198
199
200
201
                is_valid_contain_train = True
                if valid_names is not None:
                    train_data_name = valid_names[i]
                continue
Guolin Ke's avatar
Guolin Ke committed
202
            if not isinstance(valid_data, Dataset):
203
                raise TypeError("Training only accepts Dataset object")
Nikita Titov's avatar
Nikita Titov committed
204
            reduced_valid_sets.append(valid_data._update_params(params).set_reference(train_set))
205
            if valid_names is not None and len(valid_names) > i:
wxchan's avatar
wxchan committed
206
207
                name_valid_sets.append(valid_names[i])
            else:
208
                name_valid_sets.append(f'valid_{i}')
209
    # process callbacks
210
    if callbacks is None:
211
        callbacks_set = set()
wxchan's avatar
wxchan committed
212
213
214
    else:
        for i, cb in enumerate(callbacks):
            cb.__dict__.setdefault('order', i - len(callbacks))
215
        callbacks_set = set(callbacks)
wxchan's avatar
wxchan committed
216

217
218
219
220
221
222
223
224
225
226
227
228
    if "early_stopping_round" in params:
        callbacks_set.add(
            callback.early_stopping(
                stopping_rounds=params["early_stopping_round"],
                first_metric_only=first_metric_only,
                verbose=_choose_param_value(
                    main_param_name="verbosity",
                    params=params,
                    default_value=1
                ).pop("verbosity") > 0
            )
        )
229

230
231
232
233
    callbacks_before_iter_set = {cb for cb in callbacks_set if getattr(cb, 'before_iteration', False)}
    callbacks_after_iter_set = callbacks_set - callbacks_before_iter_set
    callbacks_before_iter = sorted(callbacks_before_iter_set, key=attrgetter('order'))
    callbacks_after_iter = sorted(callbacks_after_iter_set, key=attrgetter('order'))
wxchan's avatar
wxchan committed
234

235
    # construct booster
236
237
238
239
    try:
        booster = Booster(params=params, train_set=train_set)
        if is_valid_contain_train:
            booster.set_train_data_name(train_data_name)
240
        for valid_set, name_valid_set in zip(reduced_valid_sets, name_valid_sets):
241
242
243
244
245
            booster.add_valid(valid_set, name_valid_set)
    finally:
        train_set._reverse_update_params()
        for valid_set in reduced_valid_sets:
            valid_set._reverse_update_params()
246
    booster.best_iteration = 0
wxchan's avatar
wxchan committed
247

248
    # start training
249
    for i in range(init_iteration, init_iteration + num_boost_round):
wxchan's avatar
wxchan committed
250
251
        for cb in callbacks_before_iter:
            cb(callback.CallbackEnv(model=booster,
252
                                    params=params,
wxchan's avatar
wxchan committed
253
                                    iteration=i,
254
255
                                    begin_iteration=init_iteration,
                                    end_iteration=init_iteration + num_boost_round,
wxchan's avatar
wxchan committed
256
257
258
259
260
261
                                    evaluation_result_list=None))

        booster.update(fobj=fobj)

        evaluation_result_list = []
        # check evaluation result.
262
        if valid_sets is not None:
wxchan's avatar
wxchan committed
263
264
265
266
267
268
            if is_valid_contain_train:
                evaluation_result_list.extend(booster.eval_train(feval))
            evaluation_result_list.extend(booster.eval_valid(feval))
        try:
            for cb in callbacks_after_iter:
                cb(callback.CallbackEnv(model=booster,
269
                                        params=params,
wxchan's avatar
wxchan committed
270
                                        iteration=i,
271
272
                                        begin_iteration=init_iteration,
                                        end_iteration=init_iteration + num_boost_round,
wxchan's avatar
wxchan committed
273
                                        evaluation_result_list=evaluation_result_list))
274
275
        except callback.EarlyStopException as earlyStopException:
            booster.best_iteration = earlyStopException.best_iteration + 1
wxchan's avatar
wxchan committed
276
            evaluation_result_list = earlyStopException.best_score
wxchan's avatar
wxchan committed
277
            break
278
    booster.best_score = collections.defaultdict(collections.OrderedDict)
wxchan's avatar
wxchan committed
279
280
    for dataset_name, eval_name, score, _ in evaluation_result_list:
        booster.best_score[dataset_name][eval_name] = score
281
    if not keep_training_booster:
282
        booster.model_from_string(booster.model_to_string()).free_dataset()
wxchan's avatar
wxchan committed
283
284
285
    return booster


286
class CVBooster:
287
288
    """CVBooster in LightGBM.

289
    Auxiliary data structure to hold and redirect all boosters of ``cv()`` function.
290
    This class has the same methods as Booster class.
291
292
293
294
295
296
    All method calls, except for the following methods, are actually performed for underlying Boosters and
    then all returned results are returned in a list.

    - ``model_from_string()``
    - ``model_to_string()``
    - ``save_model()``
297
298
299
300
301
302
303
304

    Attributes
    ----------
    boosters : list of Booster
        The list of underlying fitted models.
    best_iteration : int
        The best iteration of fitted model.
    """
305

306
307
308
309
    def __init__(
        self,
        model_file: Optional[Union[str, Path]] = None
    ):
310
311
        """Initialize the CVBooster.

312
313
314
315
        Parameters
        ----------
        model_file : str, pathlib.Path or None, optional (default=None)
            Path to the CVBooster model file.
316
        """
317
        self.boosters: List[Booster] = []
318
        self.best_iteration = -1
319

320
321
322
323
        if model_file is not None:
            with open(model_file, "r") as file:
                self._from_dict(json.load(file))

324
    def _append(self, booster: Booster) -> None:
325
        """Add a booster to CVBooster."""
326
327
        self.boosters.append(booster)

328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
    def _from_dict(self, models: Dict[str, Any]) -> None:
        """Load CVBooster from dict."""
        self.best_iteration = models["best_iteration"]
        self.boosters = []
        for model_str in models["boosters"]:
            self._append(Booster(model_str=model_str))

    def _to_dict(self, num_iteration: Optional[int], start_iteration: int, importance_type: str) -> Dict[str, Any]:
        """Serialize CVBooster to dict."""
        models_str = []
        for booster in self.boosters:
            models_str.append(booster.model_to_string(num_iteration=num_iteration, start_iteration=start_iteration,
                                                      importance_type=importance_type))
        return {"boosters": models_str, "best_iteration": self.best_iteration}

343
    def __getattr__(self, name: str) -> Callable[[Any, Any], List[Any]]:
344
        """Redirect methods call of CVBooster."""
345
        def handler_function(*args: Any, **kwargs: Any) -> List[Any]:
346
            """Call methods with each booster, and concatenate their results."""
347
348
349
350
            ret = []
            for booster in self.boosters:
                ret.append(getattr(booster, name)(*args, **kwargs))
            return ret
351
        return handler_function
wxchan's avatar
wxchan committed
352

353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
    def __getstate__(self) -> Dict[str, Any]:
        return vars(self)

    def __setstate__(self, state: Dict[str, Any]) -> None:
        vars(self).update(state)

    def model_from_string(self, model_str: str) -> "CVBooster":
        """Load CVBooster from a string.

        Parameters
        ----------
        model_str : str
            Model will be loaded from this string.

        Returns
        -------
        self : CVBooster
            Loaded CVBooster object.
        """
        self._from_dict(json.loads(model_str))
        return self

    def model_to_string(
        self,
        num_iteration: Optional[int] = None,
        start_iteration: int = 0,
        importance_type: str = 'split'
    ) -> str:
        """Save CVBooster to JSON string.

        Parameters
        ----------
        num_iteration : int or None, optional (default=None)
            Index of the iteration that should be saved.
            If None, if the best iteration exists, it is saved; otherwise, all iterations are saved.
            If <= 0, all iterations are saved.
        start_iteration : int, optional (default=0)
            Start index of the iteration that should be saved.
        importance_type : str, optional (default="split")
            What type of feature importance should be saved.
            If "split", result contains numbers of times the feature is used in a model.
            If "gain", result contains total gains of splits which use the feature.

        Returns
        -------
        str_repr : str
            JSON string representation of CVBooster.
        """
        return json.dumps(self._to_dict(num_iteration, start_iteration, importance_type))

    def save_model(
        self,
        filename: Union[str, Path],
        num_iteration: Optional[int] = None,
        start_iteration: int = 0,
        importance_type: str = 'split'
    ) -> "CVBooster":
        """Save CVBooster to a file as JSON text.

        Parameters
        ----------
        filename : str or pathlib.Path
            Filename to save CVBooster.
        num_iteration : int or None, optional (default=None)
            Index of the iteration that should be saved.
            If None, if the best iteration exists, it is saved; otherwise, all iterations are saved.
            If <= 0, all iterations are saved.
        start_iteration : int, optional (default=0)
            Start index of the iteration that should be saved.
        importance_type : str, optional (default="split")
            What type of feature importance should be saved.
            If "split", result contains numbers of times the feature is used in a model.
            If "gain", result contains total gains of splits which use the feature.

        Returns
        -------
        self : CVBooster
            Returns self.
        """
        with open(filename, "w") as file:
            json.dump(self._to_dict(num_iteration, start_iteration, importance_type), file)

        return self

437

438
439
440
441
442
443
def _make_n_folds(
    full_data: Dataset,
    folds: Optional[Union[Iterable[Tuple[np.ndarray, np.ndarray]], _LGBMBaseCrossValidator]],
    nfold: int,
    params: Dict[str, Any],
    seed: int,
444
445
446
447
    fpreproc: Optional[_LGBM_PreprocFunction],
    stratified: bool,
    shuffle: bool,
    eval_train_metric: bool
448
) -> CVBooster:
449
    """Make a n-fold list of Booster from random indices."""
wxchan's avatar
wxchan committed
450
451
    full_data = full_data.construct()
    num_data = full_data.num_data()
452
    if folds is not None:
453
454
455
456
457
458
        if not hasattr(folds, '__iter__') and not hasattr(folds, 'split'):
            raise AttributeError("folds should be a generator or iterator of (train_idx, test_idx) tuples "
                                 "or scikit-learn splitter object with split method")
        if hasattr(folds, 'split'):
            group_info = full_data.get_group()
            if group_info is not None:
459
                group_info = np.array(group_info, dtype=np.int32, copy=False)
460
                flatted_group = np.repeat(range(len(group_info)), repeats=group_info)
461
            else:
462
                flatted_group = np.zeros(num_data, dtype=np.int32)
463
            folds = folds.split(X=np.empty(num_data), y=full_data.get_label(), groups=flatted_group)
wxchan's avatar
wxchan committed
464
    else:
465
466
467
        if any(params.get(obj_alias, "") in {"lambdarank", "rank_xendcg", "xendcg",
                                             "xe_ndcg", "xe_ndcg_mart", "xendcg_mart"}
               for obj_alias in _ConfigAliases.get("objective")):
wxchan's avatar
wxchan committed
468
            if not SKLEARN_INSTALLED:
469
                raise LightGBMError('scikit-learn is required for ranking cv')
470
            # ranking task, split according to groups
471
            group_info = np.array(full_data.get_group(), dtype=np.int32, copy=False)
472
            flatted_group = np.repeat(range(len(group_info)), repeats=group_info)
473
            group_kfold = _LGBMGroupKFold(n_splits=nfold)
474
            folds = group_kfold.split(X=np.empty(num_data), groups=flatted_group)
wxchan's avatar
wxchan committed
475
476
        elif stratified:
            if not SKLEARN_INSTALLED:
477
                raise LightGBMError('scikit-learn is required for stratified cv')
478
            skf = _LGBMStratifiedKFold(n_splits=nfold, shuffle=shuffle, random_state=seed)
479
            folds = skf.split(X=np.empty(num_data), y=full_data.get_label())
extremin's avatar
extremin committed
480
        else:
wxchan's avatar
wxchan committed
481
482
483
484
485
            if shuffle:
                randidx = np.random.RandomState(seed).permutation(num_data)
            else:
                randidx = np.arange(num_data)
            kstep = int(num_data / nfold)
486
487
488
            test_id = [randidx[i: i + kstep] for i in range(0, num_data, kstep)]
            train_id = [np.concatenate([test_id[i] for i in range(nfold) if k != i]) for k in range(nfold)]
            folds = zip(train_id, test_id)
wxchan's avatar
wxchan committed
489

490
    ret = CVBooster()
wxchan's avatar
wxchan committed
491
    for train_idx, test_idx in folds:
492
493
        train_set = full_data.subset(sorted(train_idx))
        valid_set = full_data.subset(sorted(test_idx))
wxchan's avatar
wxchan committed
494
495
        # run preprocessing on the data set if needed
        if fpreproc is not None:
wxchan's avatar
wxchan committed
496
            train_set, valid_set, tparam = fpreproc(train_set, valid_set, params.copy())
wxchan's avatar
wxchan committed
497
        else:
wxchan's avatar
wxchan committed
498
            tparam = params
499
        cvbooster = Booster(tparam, train_set)
500
501
        if eval_train_metric:
            cvbooster.add_valid(train_set, 'train')
502
        cvbooster.add_valid(valid_set, 'valid')
503
        ret._append(cvbooster)
wxchan's avatar
wxchan committed
504
505
    return ret

wxchan's avatar
wxchan committed
506

507
508
509
def _agg_cv_result(
    raw_results: List[List[Tuple[str, str, float, bool]]]
) -> List[Tuple[str, str, float, bool, float]]:
510
    """Aggregate cross-validation results."""
511
512
    cvmap: Dict[str, List[float]] = collections.OrderedDict()
    metric_type: Dict[str, bool] = {}
wxchan's avatar
wxchan committed
513
514
    for one_result in raw_results:
        for one_line in one_result:
515
            key = f"{one_line[0]} {one_line[1]}"
516
            metric_type[key] = one_line[3]
517
            cvmap.setdefault(key, [])
518
            cvmap[key].append(one_line[2])
wxchan's avatar
wxchan committed
519
    return [('cv_agg', k, np.mean(v), metric_type[k], np.std(v)) for k, v in cvmap.items()]
wxchan's avatar
wxchan committed
520

wxchan's avatar
wxchan committed
521

522
523
524
525
526
527
528
529
530
531
532
def cv(
    params: Dict[str, Any],
    train_set: Dataset,
    num_boost_round: int = 100,
    folds: Optional[Union[Iterable[Tuple[np.ndarray, np.ndarray]], _LGBMBaseCrossValidator]] = None,
    nfold: int = 5,
    stratified: bool = True,
    shuffle: bool = True,
    metrics: Optional[Union[str, List[str]]] = None,
    feval: Optional[Union[_LGBM_CustomMetricFunction, List[_LGBM_CustomMetricFunction]]] = None,
    init_model: Optional[Union[str, Path, Booster]] = None,
533
534
    feature_name: _LGBM_FeatureNameConfiguration = 'auto',
    categorical_feature: _LGBM_CategoricalFeatureConfiguration = 'auto',
535
536
537
538
539
540
    fpreproc: Optional[_LGBM_PreprocFunction] = None,
    seed: int = 0,
    callbacks: Optional[List[Callable]] = None,
    eval_train_metric: bool = False,
    return_cvbooster: bool = False
) -> Dict[str, Any]:
Andrew Ziem's avatar
Andrew Ziem committed
541
    """Perform the cross-validation with given parameters.
wxchan's avatar
wxchan committed
542
543
544
545

    Parameters
    ----------
    params : dict
546
547
        Parameters for training. Values passed through ``params`` take precedence over those
        supplied via arguments.
Guolin Ke's avatar
Guolin Ke committed
548
    train_set : Dataset
549
        Data to be trained on.
550
    num_boost_round : int, optional (default=100)
wxchan's avatar
wxchan committed
551
        Number of boosting iterations.
552
    folds : generator or iterator of (train_idx, test_idx) tuples, scikit-learn splitter object or None, optional (default=None)
553
        If generator or iterator, it should yield the train and test indices for each fold.
554
        If object, it should be one of the scikit-learn splitter classes
555
        (https://scikit-learn.org/stable/modules/classes.html#splitter-classes)
556
        and have ``split`` method.
557
        This argument has highest priority over other data split arguments.
558
    nfold : int, optional (default=5)
wxchan's avatar
wxchan committed
559
        Number of folds in CV.
560
561
    stratified : bool, optional (default=True)
        Whether to perform stratified sampling.
562
    shuffle : bool, optional (default=True)
563
        Whether to shuffle before splitting data.
564
    metrics : str, list of str, or None, optional (default=None)
565
566
        Evaluation metrics to be monitored while CV.
        If not None, the metric in ``params`` will be overridden.
567
    feval : callable, list of callable, or None, optional (default=None)
568
        Customized evaluation function.
569
        Each evaluation function should accept two parameters: preds, eval_data,
570
        and return (eval_name, eval_result, is_higher_better) or list of such tuples.
571

572
            preds : numpy 1-D array or numpy 2-D array (for multi-class task)
573
                The predicted values.
574
                For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes].
575
                If custom objective function is used, predicted values are returned before any transformation,
576
                e.g. they are raw margin instead of probability of positive class for binary task in this case.
577
578
            eval_data : Dataset
                A ``Dataset`` to evaluate.
579
            eval_name : str
Andrew Ziem's avatar
Andrew Ziem committed
580
                The name of evaluation function (without whitespace).
581
582
583
584
585
            eval_result : float
                The eval result.
            is_higher_better : bool
                Is eval result higher better, e.g. AUC is ``is_higher_better``.

586
587
        To ignore the default metric corresponding to the used objective,
        set ``metrics`` to the string ``"None"``.
588
    init_model : str, pathlib.Path, Booster or None, optional (default=None)
589
        Filename of LightGBM model or Booster instance used for continue training.
590
    feature_name : list of str, or 'auto', optional (default="auto")
591
592
        Feature names.
        If 'auto' and data is pandas DataFrame, data columns names are used.
593
    categorical_feature : list of str or int, or 'auto', optional (default="auto")
594
595
        Categorical features.
        If list of int, interpreted as indices.
596
        If list of str, interpreted as feature names (need to specify ``feature_name`` as well).
597
        If 'auto' and data is pandas DataFrame, pandas unordered categorical columns are used.
598
        All values in categorical features will be cast to int32 and thus should be less than int32 max value (2147483647).
599
        Large values could be memory consuming. Consider using consecutive integers starting from zero.
600
        All negative values in categorical features will be treated as missing values.
601
        The output cannot be monotonically constrained with respect to a categorical feature.
602
        Floating point numbers in categorical features will be rounded towards 0.
603
604
    fpreproc : callable or None, optional (default=None)
        Preprocessing function that takes (dtrain, dtest, params)
wxchan's avatar
wxchan committed
605
        and returns transformed versions of those.
606
    seed : int, optional (default=0)
wxchan's avatar
wxchan committed
607
        Seed used to generate the folds (passed to numpy.random.seed).
608
    callbacks : list of callable, or None, optional (default=None)
609
        List of callback functions that are applied at each iteration.
610
        See Callbacks in Python API for more information.
611
612
613
    eval_train_metric : bool, optional (default=False)
        Whether to display the train metric in progress.
        The score of the metric is calculated again after each training step, so there is some impact on performance.
614
615
    return_cvbooster : bool, optional (default=False)
        Whether to return Booster models trained on each fold through ``CVBooster``.
wxchan's avatar
wxchan committed
616

617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
    Note
    ----
    A custom objective function can be provided for the ``objective`` parameter.
    It should accept two parameters: preds, train_data and return (grad, hess).

        preds : numpy 1-D array or numpy 2-D array (for multi-class task)
            The predicted values.
            Predicted values are returned before any transformation,
            e.g. they are raw margin instead of probability of positive class for binary task.
        train_data : Dataset
            The training dataset.
        grad : numpy 1-D array or numpy 2-D array (for multi-class task)
            The value of the first order derivative (gradient) of the loss
            with respect to the elements of preds for each sample point.
        hess : numpy 1-D array or numpy 2-D array (for multi-class task)
            The value of the second order derivative (Hessian) of the loss
            with respect to the elements of preds for each sample point.

    For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes],
    and grad and hess should be returned in the same format.

wxchan's avatar
wxchan committed
638
639
    Returns
    -------
640
641
642
643
    eval_hist : dict
        Evaluation history.
        The dictionary has the following format:
        {'metric1-mean': [values], 'metric1-stdv': [values],
Qiwei Ye's avatar
Qiwei Ye committed
644
        'metric2-mean': [values], 'metric2-stdv': [values],
645
        ...}.
646
        If ``return_cvbooster=True``, also returns trained boosters via ``cvbooster`` key.
wxchan's avatar
wxchan committed
647
    """
Guolin Ke's avatar
Guolin Ke committed
648
    if not isinstance(train_set, Dataset):
649
        raise TypeError("Training only accepts Dataset object")
650
    params = copy.deepcopy(params)
651
652
653
654
655
    params = _choose_param_value(
        main_param_name='objective',
        params=params,
        default_value=None
    )
656
    fobj: Optional[_LGBM_CustomObjectiveFunction] = None
657
658
659
    if callable(params["objective"]):
        fobj = params["objective"]
        params["objective"] = 'none'
660
    for alias in _ConfigAliases.get("num_iterations"):
661
        if alias in params:
662
            _log_warning(f"Found '{alias}' in params. Will use it instead of 'num_boost_round' argument")
663
            num_boost_round = params.pop(alias)
664
    params["num_iterations"] = num_boost_round
665
666
667
668
669
670
671
672
    # setting early stopping via global params should be possible
    params = _choose_param_value(
        main_param_name="early_stopping_round",
        params=params,
        default_value=None
    )
    if params["early_stopping_round"] is None:
        params.pop("early_stopping_round")
673
    first_metric_only = params.get('first_metric_only', False)
674

675
676
    if num_boost_round <= 0:
        raise ValueError("num_boost_round should be greater than zero.")
677
    if isinstance(init_model, (str, Path)):
678
        predictor = _InnerPredictor(model_file=init_model, pred_parameter=params)
Guolin Ke's avatar
Guolin Ke committed
679
    elif isinstance(init_model, Booster):
680
        predictor = init_model._to_predictor(dict(init_model.params, **params))
Guolin Ke's avatar
Guolin Ke committed
681
682
683
    else:
        predictor = None

Peter's avatar
Peter committed
684
    if metrics is not None:
685
686
        for metric_alias in _ConfigAliases.get("metric"):
            params.pop(metric_alias, None)
Peter's avatar
Peter committed
687
        params['metric'] = metrics
wxchan's avatar
wxchan committed
688

689
690
691
692
693
    train_set._update_params(params) \
             ._set_predictor(predictor) \
             .set_feature_name(feature_name) \
             .set_categorical_feature(categorical_feature)

wxchan's avatar
wxchan committed
694
    results = collections.defaultdict(list)
695
    cvfolds = _make_n_folds(full_data=train_set, folds=folds, nfold=nfold,
696
                            params=params, seed=seed, fpreproc=fpreproc,
697
698
                            stratified=stratified, shuffle=shuffle,
                            eval_train_metric=eval_train_metric)
wxchan's avatar
wxchan committed
699
700

    # setup callbacks
701
    if callbacks is None:
702
        callbacks_set = set()
wxchan's avatar
wxchan committed
703
704
705
    else:
        for i, cb in enumerate(callbacks):
            cb.__dict__.setdefault('order', i - len(callbacks))
706
        callbacks_set = set(callbacks)
707
708

    if "early_stopping_round" in params:
709
        callbacks_set.add(
710
711
712
713
714
715
716
717
718
719
            callback.early_stopping(
                stopping_rounds=params["early_stopping_round"],
                first_metric_only=first_metric_only,
                verbose=_choose_param_value(
                    main_param_name="verbosity",
                    params=params,
                    default_value=1
                ).pop("verbosity") > 0
            )
        )
wxchan's avatar
wxchan committed
720

721
722
723
724
    callbacks_before_iter_set = {cb for cb in callbacks_set if getattr(cb, 'before_iteration', False)}
    callbacks_after_iter_set = callbacks_set - callbacks_before_iter_set
    callbacks_before_iter = sorted(callbacks_before_iter_set, key=attrgetter('order'))
    callbacks_after_iter = sorted(callbacks_after_iter_set, key=attrgetter('order'))
wxchan's avatar
wxchan committed
725

726
    for i in range(num_boost_round):
wxchan's avatar
wxchan committed
727
        for cb in callbacks_before_iter:
728
729
            cb(callback.CallbackEnv(model=cvfolds,
                                    params=params,
wxchan's avatar
wxchan committed
730
731
732
733
                                    iteration=i,
                                    begin_iteration=0,
                                    end_iteration=num_boost_round,
                                    evaluation_result_list=None))
wxchan's avatar
wxchan committed
734
        cvfolds.update(fobj=fobj)
735
        res = _agg_cv_result(cvfolds.eval_valid(feval))
wxchan's avatar
wxchan committed
736
        for _, key, mean, _, std in res:
737
738
            results[f'{key}-mean'].append(mean)
            results[f'{key}-stdv'].append(std)
wxchan's avatar
wxchan committed
739
740
        try:
            for cb in callbacks_after_iter:
741
742
                cb(callback.CallbackEnv(model=cvfolds,
                                        params=params,
wxchan's avatar
wxchan committed
743
744
745
746
                                        iteration=i,
                                        begin_iteration=0,
                                        end_iteration=num_boost_round,
                                        evaluation_result_list=res))
747
748
        except callback.EarlyStopException as earlyStopException:
            cvfolds.best_iteration = earlyStopException.best_iteration + 1
749
750
            for bst in cvfolds.boosters:
                bst.best_iteration = cvfolds.best_iteration
wxchan's avatar
wxchan committed
751
            for k in results:
752
                results[k] = results[k][:cvfolds.best_iteration]
wxchan's avatar
wxchan committed
753
            break
754
755
756
757

    if return_cvbooster:
        results['cvbooster'] = cvfolds

wxchan's avatar
wxchan committed
758
    return dict(results)