engine.py 34.9 KB
Newer Older
wxchan's avatar
wxchan committed
1
# coding: utf-8
2
"""Library with training routines of LightGBM."""
3

4
import copy
5
import json
6
from collections import OrderedDict, defaultdict
wxchan's avatar
wxchan committed
7
from operator import attrgetter
8
from pathlib import Path
9
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
10

wxchan's avatar
wxchan committed
11
import numpy as np
12

wxchan's avatar
wxchan committed
13
from . import callback
14
15
16
17
18
19
20
21
22
23
24
25
26
from .basic import (
    Booster,
    Dataset,
    LightGBMError,
    _choose_param_value,
    _ConfigAliases,
    _InnerPredictor,
    _LGBM_BoosterEvalMethodResultType,
    _LGBM_BoosterEvalMethodResultWithStandardDeviationType,
    _LGBM_CustomObjectiveFunction,
    _LGBM_EvalFunctionResultType,
    _log_warning,
)
27
from .compat import SKLEARN_INSTALLED, _LGBMBaseCrossValidator, _LGBMGroupKFold, _LGBMStratifiedKFold
wxchan's avatar
wxchan committed
28

29
__all__ = [
30
31
32
    "cv",
    "CVBooster",
    "train",
33
34
35
]


36
37
38
39
40
41
42
_LGBM_CustomMetricFunction = Union[
    Callable[
        [np.ndarray, Dataset],
        _LGBM_EvalFunctionResultType,
    ],
    Callable[
        [np.ndarray, Dataset],
43
        List[_LGBM_EvalFunctionResultType],
44
    ],
45
]
wxchan's avatar
wxchan committed
46

47
48
_LGBM_PreprocFunction = Callable[
    [Dataset, Dataset, Dict[str, Any]],
49
    Tuple[Dataset, Dataset, Dict[str, Any]],
50
51
]

52

53
def _choose_num_iterations(*, num_boost_round_kwarg: int, params: Dict[str, Any]) -> Dict[str, Any]:
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
    """Choose number of boosting rounds.

    In ``train()`` and ``cv()``, there are multiple ways to provide configuration for
    the number of boosting rounds to perform:

      * the ``num_boost_round`` keyword argument
      * any of the ``num_iterations`` or its aliases via the ``params`` dictionary

    These should be preferred in the following order (first one found wins):

      1. ``num_iterations`` provided via ``params`` (because it's the main parameter name)
      2. any other aliases of ``num_iterations`` provided via ``params``
      3. the ``num_boost_round`` keyword argument

    This function handles that choice, and issuing helpful warnings in the cases where the
    result might be surprising.

    Returns
    -------
    params : dict
        Parameters, with ``"num_iterations"`` set to the preferred value and all other
        aliases of ``num_iterations`` removed.
    """
    num_iteration_configs_provided = {
        alias: params[alias] for alias in _ConfigAliases.get("num_iterations") if alias in params
    }

    # now that the relevant information has been pulled out of params, it's safe to overwrite it
    # with the content that should be used for training (i.e. with aliases resolved)
    params = _choose_param_value(
        main_param_name="num_iterations",
        params=params,
        default_value=num_boost_round_kwarg,
    )

    # if there were not multiple boosting rounds configurations provided in params,
    # then by definition they cannot have conflicting values... no need to warn
    if len(num_iteration_configs_provided) <= 1:
        return params

    # if all the aliases have the same value, no need to warn
    if len(set(num_iteration_configs_provided.values())) <= 1:
        return params

    # if this line is reached, lightgbm should warn
    value_string = ", ".join(f"{alias}={val}" for alias, val in num_iteration_configs_provided.items())
    _log_warning(
        f"Found conflicting values for num_iterations provided via 'params': {value_string}. "
        f"LightGBM will perform up to {params['num_iterations']} boosting rounds. "
        "To be confident in the maximum number of boosting rounds LightGBM will perform and to "
        "suppress this warning, modify 'params' so that only one of those is present."
    )
    return params


109
110
111
112
113
114
115
116
117
def train(
    params: Dict[str, Any],
    train_set: Dataset,
    num_boost_round: int = 100,
    valid_sets: Optional[List[Dataset]] = None,
    valid_names: Optional[List[str]] = None,
    feval: Optional[Union[_LGBM_CustomMetricFunction, List[_LGBM_CustomMetricFunction]]] = None,
    init_model: Optional[Union[str, Path, Booster]] = None,
    keep_training_booster: bool = False,
118
    callbacks: Optional[List[Callable]] = None,
119
) -> Booster:
120
    """Perform the training with given parameters.
wxchan's avatar
wxchan committed
121
122
123
124

    Parameters
    ----------
    params : dict
125
126
        Parameters for training. Values passed through ``params`` take precedence over those
        supplied via arguments.
Guolin Ke's avatar
Guolin Ke committed
127
    train_set : Dataset
128
129
        Data to be trained on.
    num_boost_round : int, optional (default=100)
wxchan's avatar
wxchan committed
130
        Number of boosting iterations.
131
    valid_sets : list of Dataset, or None, optional (default=None)
132
        List of data to be evaluated on during training.
133
    valid_names : list of str, or None, optional (default=None)
134
        Names of ``valid_sets``.
135
    feval : callable, list of callable, or None, optional (default=None)
wxchan's avatar
wxchan committed
136
        Customized evaluation function.
Akshita Dixit's avatar
Akshita Dixit committed
137
        Each evaluation function should accept two parameters: preds, eval_data,
138
        and return (eval_name, eval_result, is_higher_better) or list of such tuples.
139

140
            preds : numpy 1-D array or numpy 2-D array (for multi-class task)
141
                The predicted values.
142
                For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes].
143
                If custom objective function is used, predicted values are returned before any transformation,
144
                e.g. they are raw margin instead of probability of positive class for binary task in this case.
Akshita Dixit's avatar
Akshita Dixit committed
145
            eval_data : Dataset
146
                A ``Dataset`` to evaluate.
147
            eval_name : str
148
                The name of evaluation function (without whitespaces).
149
150
151
152
153
            eval_result : float
                The eval result.
            is_higher_better : bool
                Is eval result higher better, e.g. AUC is ``is_higher_better``.

154
155
        To ignore the default metric corresponding to the used objective,
        set the ``metric`` parameter to the string ``"None"`` in ``params``.
156
    init_model : str, pathlib.Path, Booster or None, optional (default=None)
157
158
159
160
        Filename of LightGBM model or Booster instance used for continue training.
    keep_training_booster : bool, optional (default=False)
        Whether the returned Booster will be used to keep training.
        If False, the returned value will be converted into _InnerPredictor before returning.
161
        This means you won't be able to use ``eval``, ``eval_train`` or ``eval_valid`` methods of the returned Booster.
162
163
        When your model is very large and cause the memory error,
        you can try to set this param to ``True`` to avoid the model conversion performed during the internal call of ``model_to_string``.
164
        You can still use _InnerPredictor as ``init_model`` for future continue training.
165
    callbacks : list of callable, or None, optional (default=None)
166
        List of callback functions that are applied at each iteration.
167
        See Callbacks in Python API for more information.
wxchan's avatar
wxchan committed
168

169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
    Note
    ----
    A custom objective function can be provided for the ``objective`` parameter.
    It should accept two parameters: preds, train_data and return (grad, hess).

        preds : numpy 1-D array or numpy 2-D array (for multi-class task)
            The predicted values.
            Predicted values are returned before any transformation,
            e.g. they are raw margin instead of probability of positive class for binary task.
        train_data : Dataset
            The training dataset.
        grad : numpy 1-D array or numpy 2-D array (for multi-class task)
            The value of the first order derivative (gradient) of the loss
            with respect to the elements of preds for each sample point.
        hess : numpy 1-D array or numpy 2-D array (for multi-class task)
            The value of the second order derivative (Hessian) of the loss
            with respect to the elements of preds for each sample point.

    For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes],
    and grad and hess should be returned in the same format.

wxchan's avatar
wxchan committed
190
191
    Returns
    -------
192
193
    booster : Booster
        The trained Booster model.
wxchan's avatar
wxchan committed
194
    """
195
196
197
198
199
200
201
202
203
204
205
    if not isinstance(train_set, Dataset):
        raise TypeError(f"train() only accepts Dataset object, train_set has type '{type(train_set).__name__}'.")

    if isinstance(valid_sets, list):
        for i, valid_item in enumerate(valid_sets):
            if not isinstance(valid_item, Dataset):
                raise TypeError(
                    "Every item in valid_sets must be a Dataset object. "
                    f"Item {i} has type '{type(valid_item).__name__}'."
                )

206
    # create predictor first
207
    params = copy.deepcopy(params)
208
    params = _choose_param_value(
209
        main_param_name="objective",
210
        params=params,
211
        default_value=None,
212
    )
213
    fobj: Optional[_LGBM_CustomObjectiveFunction] = None
214
215
    if callable(params["objective"]):
        fobj = params["objective"]
216
        params["objective"] = "none"
217
218
219
220
221
222

    params = _choose_num_iterations(num_boost_round_kwarg=num_boost_round, params=params)
    num_boost_round = params["num_iterations"]
    if num_boost_round <= 0:
        raise ValueError(f"Number of boosting rounds must be greater than 0. Got {num_boost_round}.")

223
224
225
226
    # setting early stopping via global params should be possible
    params = _choose_param_value(
        main_param_name="early_stopping_round",
        params=params,
227
        default_value=None,
228
229
230
    )
    if params["early_stopping_round"] is None:
        params.pop("early_stopping_round")
231
    first_metric_only = params.get("first_metric_only", False)
232

233
    predictor: Optional[_InnerPredictor] = None
234
    if isinstance(init_model, (str, Path)):
235
        predictor = _InnerPredictor.from_model_file(model_file=init_model, pred_parameter=params)
wxchan's avatar
wxchan committed
236
    elif isinstance(init_model, Booster):
237
        predictor = _InnerPredictor.from_booster(booster=init_model, pred_parameter=dict(init_model.params, **params))
238
239
240
241
242

    if predictor is not None:
        init_iteration = predictor.current_iteration()
    else:
        init_iteration = 0
Guolin Ke's avatar
Guolin Ke committed
243

244
    train_set._update_params(params)._set_predictor(predictor)
Guolin Ke's avatar
Guolin Ke committed
245

wxchan's avatar
wxchan committed
246
247
    is_valid_contain_train = False
    train_data_name = "training"
Guolin Ke's avatar
Guolin Ke committed
248
    reduced_valid_sets = []
wxchan's avatar
wxchan committed
249
    name_valid_sets = []
250
    if valid_sets is not None:
Guolin Ke's avatar
Guolin Ke committed
251
252
        if isinstance(valid_sets, Dataset):
            valid_sets = [valid_sets]
253
        if isinstance(valid_names, str):
wxchan's avatar
wxchan committed
254
            valid_names = [valid_names]
Guolin Ke's avatar
Guolin Ke committed
255
        for i, valid_data in enumerate(valid_sets):
256
            # reduce cost for prediction training data
Guolin Ke's avatar
Guolin Ke committed
257
            if valid_data is train_set:
wxchan's avatar
wxchan committed
258
259
260
261
                is_valid_contain_train = True
                if valid_names is not None:
                    train_data_name = valid_names[i]
                continue
Nikita Titov's avatar
Nikita Titov committed
262
            reduced_valid_sets.append(valid_data._update_params(params).set_reference(train_set))
263
            if valid_names is not None and len(valid_names) > i:
wxchan's avatar
wxchan committed
264
265
                name_valid_sets.append(valid_names[i])
            else:
266
                name_valid_sets.append(f"valid_{i}")
267
    # process callbacks
268
    if callbacks is None:
269
        callbacks_set = set()
wxchan's avatar
wxchan committed
270
271
    else:
        for i, cb in enumerate(callbacks):
272
            cb.__dict__.setdefault("order", i - len(callbacks))
273
        callbacks_set = set(callbacks)
wxchan's avatar
wxchan committed
274

275
    if callback._should_enable_early_stopping(params.get("early_stopping_round", 0)):
276
277
        callbacks_set.add(
            callback.early_stopping(
278
                stopping_rounds=params["early_stopping_round"],  # type: ignore[arg-type]
279
                first_metric_only=first_metric_only,
280
                min_delta=params.get("early_stopping_min_delta", 0.0),
281
282
283
                verbose=_choose_param_value(
                    main_param_name="verbosity",
                    params=params,
284
285
286
                    default_value=1,
                ).pop("verbosity")
                > 0,
287
288
            )
        )
289

290
    callbacks_before_iter_set = {cb for cb in callbacks_set if getattr(cb, "before_iteration", False)}
291
    callbacks_after_iter_set = callbacks_set - callbacks_before_iter_set
292
293
    callbacks_before_iter = sorted(callbacks_before_iter_set, key=attrgetter("order"))
    callbacks_after_iter = sorted(callbacks_after_iter_set, key=attrgetter("order"))
wxchan's avatar
wxchan committed
294

295
    # construct booster
296
297
298
299
    try:
        booster = Booster(params=params, train_set=train_set)
        if is_valid_contain_train:
            booster.set_train_data_name(train_data_name)
300
        for valid_set, name_valid_set in zip(reduced_valid_sets, name_valid_sets):
301
302
303
304
305
            booster.add_valid(valid_set, name_valid_set)
    finally:
        train_set._reverse_update_params()
        for valid_set in reduced_valid_sets:
            valid_set._reverse_update_params()
306
    booster.best_iteration = 0
wxchan's avatar
wxchan committed
307

308
    # start training
309
    for i in range(init_iteration, init_iteration + num_boost_round):
wxchan's avatar
wxchan committed
310
        for cb in callbacks_before_iter:
311
312
313
314
315
316
317
318
319
320
            cb(
                callback.CallbackEnv(
                    model=booster,
                    params=params,
                    iteration=i,
                    begin_iteration=init_iteration,
                    end_iteration=init_iteration + num_boost_round,
                    evaluation_result_list=None,
                )
            )
wxchan's avatar
wxchan committed
321
322
323

        booster.update(fobj=fobj)

324
        evaluation_result_list: List[_LGBM_BoosterEvalMethodResultType] = []
wxchan's avatar
wxchan committed
325
        # check evaluation result.
326
        if valid_sets is not None:
wxchan's avatar
wxchan committed
327
328
329
330
331
            if is_valid_contain_train:
                evaluation_result_list.extend(booster.eval_train(feval))
            evaluation_result_list.extend(booster.eval_valid(feval))
        try:
            for cb in callbacks_after_iter:
332
333
334
335
336
337
338
339
340
341
                cb(
                    callback.CallbackEnv(
                        model=booster,
                        params=params,
                        iteration=i,
                        begin_iteration=init_iteration,
                        end_iteration=init_iteration + num_boost_round,
                        evaluation_result_list=evaluation_result_list,
                    )
                )
342
343
        except callback.EarlyStopException as earlyStopException:
            booster.best_iteration = earlyStopException.best_iteration + 1
wxchan's avatar
wxchan committed
344
            evaluation_result_list = earlyStopException.best_score
wxchan's avatar
wxchan committed
345
            break
346
    booster.best_score = defaultdict(OrderedDict)
wxchan's avatar
wxchan committed
347
348
    for dataset_name, eval_name, score, _ in evaluation_result_list:
        booster.best_score[dataset_name][eval_name] = score
349
    if not keep_training_booster:
350
        booster.model_from_string(booster.model_to_string()).free_dataset()
wxchan's avatar
wxchan committed
351
352
353
    return booster


354
class CVBooster:
355
356
    """CVBooster in LightGBM.

357
    Auxiliary data structure to hold and redirect all boosters of ``cv()`` function.
358
    This class has the same methods as Booster class.
359
360
361
362
363
364
    All method calls, except for the following methods, are actually performed for underlying Boosters and
    then all returned results are returned in a list.

    - ``model_from_string()``
    - ``model_to_string()``
    - ``save_model()``
365
366
367
368
369
370
371
372

    Attributes
    ----------
    boosters : list of Booster
        The list of underlying fitted models.
    best_iteration : int
        The best iteration of fitted model.
    """
373

374
375
    def __init__(
        self,
376
        model_file: Optional[Union[str, Path]] = None,
377
    ):
378
379
        """Initialize the CVBooster.

380
381
382
383
        Parameters
        ----------
        model_file : str, pathlib.Path or None, optional (default=None)
            Path to the CVBooster model file.
384
        """
385
        self.boosters: List[Booster] = []
386
        self.best_iteration = -1
387

388
389
390
391
392
393
394
395
396
        if model_file is not None:
            with open(model_file, "r") as file:
                self._from_dict(json.load(file))

    def _from_dict(self, models: Dict[str, Any]) -> None:
        """Load CVBooster from dict."""
        self.best_iteration = models["best_iteration"]
        self.boosters = []
        for model_str in models["boosters"]:
397
            self.boosters.append(Booster(model_str=model_str))
398

399
400
401
402
403
404
405
    def _to_dict(
        self,
        *,
        num_iteration: Optional[int],
        start_iteration: int,
        importance_type: str,
    ) -> Dict[str, Any]:
406
407
408
        """Serialize CVBooster to dict."""
        models_str = []
        for booster in self.boosters:
409
410
411
412
413
            models_str.append(
                booster.model_to_string(
                    num_iteration=num_iteration, start_iteration=start_iteration, importance_type=importance_type
                )
            )
414
415
        return {"boosters": models_str, "best_iteration": self.best_iteration}

416
    def __getattr__(self, name: str) -> Callable[[Any, Any], List[Any]]:
417
        """Redirect methods call of CVBooster."""
418

419
        def handler_function(*args: Any, **kwargs: Any) -> List[Any]:
420
            """Call methods with each booster, and concatenate their results."""
421
422
423
424
            ret = []
            for booster in self.boosters:
                ret.append(getattr(booster, name)(*args, **kwargs))
            return ret
425

426
        return handler_function
wxchan's avatar
wxchan committed
427

428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
    def __getstate__(self) -> Dict[str, Any]:
        return vars(self)

    def __setstate__(self, state: Dict[str, Any]) -> None:
        vars(self).update(state)

    def model_from_string(self, model_str: str) -> "CVBooster":
        """Load CVBooster from a string.

        Parameters
        ----------
        model_str : str
            Model will be loaded from this string.

        Returns
        -------
        self : CVBooster
            Loaded CVBooster object.
        """
        self._from_dict(json.loads(model_str))
        return self

    def model_to_string(
        self,
        num_iteration: Optional[int] = None,
        start_iteration: int = 0,
454
        importance_type: str = "split",
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
    ) -> str:
        """Save CVBooster to JSON string.

        Parameters
        ----------
        num_iteration : int or None, optional (default=None)
            Index of the iteration that should be saved.
            If None, if the best iteration exists, it is saved; otherwise, all iterations are saved.
            If <= 0, all iterations are saved.
        start_iteration : int, optional (default=0)
            Start index of the iteration that should be saved.
        importance_type : str, optional (default="split")
            What type of feature importance should be saved.
            If "split", result contains numbers of times the feature is used in a model.
            If "gain", result contains total gains of splits which use the feature.

        Returns
        -------
        str_repr : str
            JSON string representation of CVBooster.
        """
476
477
478
        return json.dumps(
            self._to_dict(num_iteration=num_iteration, start_iteration=start_iteration, importance_type=importance_type)
        )
479
480
481
482
483
484

    def save_model(
        self,
        filename: Union[str, Path],
        num_iteration: Optional[int] = None,
        start_iteration: int = 0,
485
        importance_type: str = "split",
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
    ) -> "CVBooster":
        """Save CVBooster to a file as JSON text.

        Parameters
        ----------
        filename : str or pathlib.Path
            Filename to save CVBooster.
        num_iteration : int or None, optional (default=None)
            Index of the iteration that should be saved.
            If None, if the best iteration exists, it is saved; otherwise, all iterations are saved.
            If <= 0, all iterations are saved.
        start_iteration : int, optional (default=0)
            Start index of the iteration that should be saved.
        importance_type : str, optional (default="split")
            What type of feature importance should be saved.
            If "split", result contains numbers of times the feature is used in a model.
            If "gain", result contains total gains of splits which use the feature.

        Returns
        -------
        self : CVBooster
            Returns self.
        """
        with open(filename, "w") as file:
510
511
512
513
514
515
            json.dump(
                self._to_dict(
                    num_iteration=num_iteration, start_iteration=start_iteration, importance_type=importance_type
                ),
                file,
            )
516
517
518

        return self

519

520
def _make_n_folds(
521
    *,
522
523
524
525
526
    full_data: Dataset,
    folds: Optional[Union[Iterable[Tuple[np.ndarray, np.ndarray]], _LGBMBaseCrossValidator]],
    nfold: int,
    params: Dict[str, Any],
    seed: int,
527
528
529
    fpreproc: Optional[_LGBM_PreprocFunction],
    stratified: bool,
    shuffle: bool,
530
    eval_train_metric: bool,
531
) -> CVBooster:
532
    """Make a n-fold list of Booster from random indices."""
wxchan's avatar
wxchan committed
533
534
    full_data = full_data.construct()
    num_data = full_data.num_data()
535
    if folds is not None:
536
537
538
539
540
541
        if not hasattr(folds, "__iter__") and not hasattr(folds, "split"):
            raise AttributeError(
                "folds should be a generator or iterator of (train_idx, test_idx) tuples "
                "or scikit-learn splitter object with split method"
            )
        if hasattr(folds, "split"):
542
543
            group_info = full_data.get_group()
            if group_info is not None:
544
                group_info = np.asarray(group_info, dtype=np.int32)
545
                flatted_group = np.repeat(range(len(group_info)), repeats=group_info)
546
            else:
547
                flatted_group = np.zeros(num_data, dtype=np.int32)
548
            folds = folds.split(X=np.empty(num_data), y=full_data.get_label(), groups=flatted_group)
wxchan's avatar
wxchan committed
549
    else:
550
551
552
553
554
        if any(
            params.get(obj_alias, "")
            in {"lambdarank", "rank_xendcg", "xendcg", "xe_ndcg", "xe_ndcg_mart", "xendcg_mart"}
            for obj_alias in _ConfigAliases.get("objective")
        ):
wxchan's avatar
wxchan committed
555
            if not SKLEARN_INSTALLED:
556
                raise LightGBMError("scikit-learn is required for ranking cv")
557
            # ranking task, split according to groups
558
            group_info = np.asarray(full_data.get_group(), dtype=np.int32)
559
            flatted_group = np.repeat(range(len(group_info)), repeats=group_info)
560
            group_kfold = _LGBMGroupKFold(n_splits=nfold)
561
            folds = group_kfold.split(X=np.empty(num_data), groups=flatted_group)
wxchan's avatar
wxchan committed
562
563
        elif stratified:
            if not SKLEARN_INSTALLED:
564
                raise LightGBMError("scikit-learn is required for stratified cv")
565
            skf = _LGBMStratifiedKFold(n_splits=nfold, shuffle=shuffle, random_state=seed)
566
            folds = skf.split(X=np.empty(num_data), y=full_data.get_label())
extremin's avatar
extremin committed
567
        else:
wxchan's avatar
wxchan committed
568
569
570
571
572
            if shuffle:
                randidx = np.random.RandomState(seed).permutation(num_data)
            else:
                randidx = np.arange(num_data)
            kstep = int(num_data / nfold)
573
            test_id = [randidx[i : i + kstep] for i in range(0, num_data, kstep)]
574
575
            train_id = [np.concatenate([test_id[i] for i in range(nfold) if k != i]) for k in range(nfold)]
            folds = zip(train_id, test_id)
wxchan's avatar
wxchan committed
576

577
    ret = CVBooster()
wxchan's avatar
wxchan committed
578
    for train_idx, test_idx in folds:
579
580
        train_set = full_data.subset(sorted(train_idx))
        valid_set = full_data.subset(sorted(test_idx))
wxchan's avatar
wxchan committed
581
582
        # run preprocessing on the data set if needed
        if fpreproc is not None:
wxchan's avatar
wxchan committed
583
            train_set, valid_set, tparam = fpreproc(train_set, valid_set, params.copy())
wxchan's avatar
wxchan committed
584
        else:
wxchan's avatar
wxchan committed
585
            tparam = params
586
        booster_for_fold = Booster(tparam, train_set)
587
        if eval_train_metric:
588
589
            booster_for_fold.add_valid(train_set, "train")
        booster_for_fold.add_valid(valid_set, "valid")
590
        ret.boosters.append(booster_for_fold)
wxchan's avatar
wxchan committed
591
592
    return ret

wxchan's avatar
wxchan committed
593

594
def _agg_cv_result(
595
    raw_results: List[List[_LGBM_BoosterEvalMethodResultType]],
596
) -> List[_LGBM_BoosterEvalMethodResultWithStandardDeviationType]:
597
    """Aggregate cross-validation results."""
598
599
600
601
602
603
604
605
606
607
608
609
    # build up 2 maps, of the form:
    #
    # OrderedDict{
    #     (<dataset_name>, <metric_name>): <is_higher_better>
    # }
    #
    # OrderedDict{
    #     (<dataset_name>, <metric_name>): list[<metric_value>]
    # }
    #
    metric_types: Dict[Tuple[str, str], bool] = OrderedDict()
    metric_values: Dict[Tuple[str, str], List[float]] = OrderedDict()
wxchan's avatar
wxchan committed
610
    for one_result in raw_results:
611
612
613
614
615
616
617
618
619
620
621
622
        for dataset_name, metric_name, metric_value, is_higher_better in one_result:
            key = (dataset_name, metric_name)
            metric_types[key] = is_higher_better
            metric_values.setdefault(key, [])
            metric_values[key].append(metric_value)

    # turn that into a list of tuples of the form:
    #
    # [
    #     (<dataset_name>, <metric_name>, mean(<values>), <is_higher_better>, std_dev(<values>))
    # ]
    return [(k[0], k[1], float(np.mean(v)), metric_types[k], float(np.std(v))) for k, v in metric_values.items()]
wxchan's avatar
wxchan committed
623

wxchan's avatar
wxchan committed
624

625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
def cv(
    params: Dict[str, Any],
    train_set: Dataset,
    num_boost_round: int = 100,
    folds: Optional[Union[Iterable[Tuple[np.ndarray, np.ndarray]], _LGBMBaseCrossValidator]] = None,
    nfold: int = 5,
    stratified: bool = True,
    shuffle: bool = True,
    metrics: Optional[Union[str, List[str]]] = None,
    feval: Optional[Union[_LGBM_CustomMetricFunction, List[_LGBM_CustomMetricFunction]]] = None,
    init_model: Optional[Union[str, Path, Booster]] = None,
    fpreproc: Optional[_LGBM_PreprocFunction] = None,
    seed: int = 0,
    callbacks: Optional[List[Callable]] = None,
    eval_train_metric: bool = False,
640
    return_cvbooster: bool = False,
641
) -> Dict[str, Union[List[float], CVBooster]]:
Andrew Ziem's avatar
Andrew Ziem committed
642
    """Perform the cross-validation with given parameters.
wxchan's avatar
wxchan committed
643
644
645
646

    Parameters
    ----------
    params : dict
647
648
        Parameters for training. Values passed through ``params`` take precedence over those
        supplied via arguments.
Guolin Ke's avatar
Guolin Ke committed
649
    train_set : Dataset
650
        Data to be trained on.
651
    num_boost_round : int, optional (default=100)
wxchan's avatar
wxchan committed
652
        Number of boosting iterations.
653
    folds : generator or iterator of (train_idx, test_idx) tuples, scikit-learn splitter object or None, optional (default=None)
654
        If generator or iterator, it should yield the train and test indices for each fold.
655
        If object, it should be one of the scikit-learn splitter classes
656
        (https://scikit-learn.org/stable/modules/classes.html#splitter-classes)
657
        and have ``split`` method.
658
        This argument has highest priority over other data split arguments.
659
    nfold : int, optional (default=5)
wxchan's avatar
wxchan committed
660
        Number of folds in CV.
661
662
    stratified : bool, optional (default=True)
        Whether to perform stratified sampling.
663
    shuffle : bool, optional (default=True)
664
        Whether to shuffle before splitting data.
665
    metrics : str, list of str, or None, optional (default=None)
666
667
        Evaluation metrics to be monitored while CV.
        If not None, the metric in ``params`` will be overridden.
668
    feval : callable, list of callable, or None, optional (default=None)
669
        Customized evaluation function.
670
        Each evaluation function should accept two parameters: preds, eval_data,
671
        and return (eval_name, eval_result, is_higher_better) or list of such tuples.
672

673
            preds : numpy 1-D array or numpy 2-D array (for multi-class task)
674
                The predicted values.
675
                For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes].
676
                If custom objective function is used, predicted values are returned before any transformation,
677
                e.g. they are raw margin instead of probability of positive class for binary task in this case.
678
679
            eval_data : Dataset
                A ``Dataset`` to evaluate.
680
            eval_name : str
Andrew Ziem's avatar
Andrew Ziem committed
681
                The name of evaluation function (without whitespace).
682
683
684
685
686
            eval_result : float
                The eval result.
            is_higher_better : bool
                Is eval result higher better, e.g. AUC is ``is_higher_better``.

687
688
        To ignore the default metric corresponding to the used objective,
        set ``metrics`` to the string ``"None"``.
689
    init_model : str, pathlib.Path, Booster or None, optional (default=None)
690
691
692
        Filename of LightGBM model or Booster instance used for continue training.
    fpreproc : callable or None, optional (default=None)
        Preprocessing function that takes (dtrain, dtest, params)
wxchan's avatar
wxchan committed
693
        and returns transformed versions of those.
694
    seed : int, optional (default=0)
wxchan's avatar
wxchan committed
695
        Seed used to generate the folds (passed to numpy.random.seed).
696
    callbacks : list of callable, or None, optional (default=None)
697
        List of callback functions that are applied at each iteration.
698
        See Callbacks in Python API for more information.
699
700
701
    eval_train_metric : bool, optional (default=False)
        Whether to display the train metric in progress.
        The score of the metric is calculated again after each training step, so there is some impact on performance.
702
703
    return_cvbooster : bool, optional (default=False)
        Whether to return Booster models trained on each fold through ``CVBooster``.
wxchan's avatar
wxchan committed
704

705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
    Note
    ----
    A custom objective function can be provided for the ``objective`` parameter.
    It should accept two parameters: preds, train_data and return (grad, hess).

        preds : numpy 1-D array or numpy 2-D array (for multi-class task)
            The predicted values.
            Predicted values are returned before any transformation,
            e.g. they are raw margin instead of probability of positive class for binary task.
        train_data : Dataset
            The training dataset.
        grad : numpy 1-D array or numpy 2-D array (for multi-class task)
            The value of the first order derivative (gradient) of the loss
            with respect to the elements of preds for each sample point.
        hess : numpy 1-D array or numpy 2-D array (for multi-class task)
            The value of the second order derivative (Hessian) of the loss
            with respect to the elements of preds for each sample point.

    For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes],
    and grad and hess should be returned in the same format.

wxchan's avatar
wxchan committed
726
727
    Returns
    -------
728
729
    eval_results : dict
        History of evaluation results of each metric.
730
        The dictionary has the following format:
731
732
        {'valid metric1-mean': [values], 'valid metric1-stdv': [values],
        'valid metric2-mean': [values], 'valid metric2-stdv': [values],
733
        ...}.
734
        If ``return_cvbooster=True``, also returns trained boosters wrapped in a ``CVBooster`` object via ``cvbooster`` key.
735
736
737
738
739
        If ``eval_train_metric=True``, also returns the train metric history.
        In this case, the dictionary has the following format:
        {'train metric1-mean': [values], 'valid metric1-mean': [values],
        'train metric2-mean': [values], 'valid metric2-mean': [values],
        ...}.
wxchan's avatar
wxchan committed
740
    """
Guolin Ke's avatar
Guolin Ke committed
741
    if not isinstance(train_set, Dataset):
742
743
        raise TypeError(f"cv() only accepts Dataset object, train_set has type '{type(train_set).__name__}'.")

744
    params = copy.deepcopy(params)
745
    params = _choose_param_value(
746
        main_param_name="objective",
747
        params=params,
748
        default_value=None,
749
    )
750
    fobj: Optional[_LGBM_CustomObjectiveFunction] = None
751
752
    if callable(params["objective"]):
        fobj = params["objective"]
753
        params["objective"] = "none"
754
755
756
757
758
759

    params = _choose_num_iterations(num_boost_round_kwarg=num_boost_round, params=params)
    num_boost_round = params["num_iterations"]
    if num_boost_round <= 0:
        raise ValueError(f"Number of boosting rounds must be greater than 0. Got {num_boost_round}.")

760
761
762
763
    # setting early stopping via global params should be possible
    params = _choose_param_value(
        main_param_name="early_stopping_round",
        params=params,
764
        default_value=None,
765
766
767
    )
    if params["early_stopping_round"] is None:
        params.pop("early_stopping_round")
768
    first_metric_only = params.get("first_metric_only", False)
769

770
    if isinstance(init_model, (str, Path)):
771
772
        predictor = _InnerPredictor.from_model_file(
            model_file=init_model,
773
            pred_parameter=params,
774
        )
Guolin Ke's avatar
Guolin Ke committed
775
    elif isinstance(init_model, Booster):
776
777
        predictor = _InnerPredictor.from_booster(
            booster=init_model,
778
            pred_parameter=dict(init_model.params, **params),
779
        )
Guolin Ke's avatar
Guolin Ke committed
780
781
782
    else:
        predictor = None

Peter's avatar
Peter committed
783
    if metrics is not None:
784
785
        for metric_alias in _ConfigAliases.get("metric"):
            params.pop(metric_alias, None)
786
        params["metric"] = metrics
wxchan's avatar
wxchan committed
787

788
    train_set._update_params(params)._set_predictor(predictor)
789

790
    results = defaultdict(list)
791
    cvbooster = _make_n_folds(
792
793
794
795
796
797
798
799
800
801
        full_data=train_set,
        folds=folds,
        nfold=nfold,
        params=params,
        seed=seed,
        fpreproc=fpreproc,
        stratified=stratified,
        shuffle=shuffle,
        eval_train_metric=eval_train_metric,
    )
wxchan's avatar
wxchan committed
802
803

    # setup callbacks
804
    if callbacks is None:
805
        callbacks_set = set()
wxchan's avatar
wxchan committed
806
807
    else:
        for i, cb in enumerate(callbacks):
808
            cb.__dict__.setdefault("order", i - len(callbacks))
809
        callbacks_set = set(callbacks)
810

811
    if callback._should_enable_early_stopping(params.get("early_stopping_round", 0)):
812
        callbacks_set.add(
813
            callback.early_stopping(
814
                stopping_rounds=params["early_stopping_round"],  # type: ignore[arg-type]
815
                first_metric_only=first_metric_only,
816
                min_delta=params.get("early_stopping_min_delta", 0.0),
817
818
819
                verbose=_choose_param_value(
                    main_param_name="verbosity",
                    params=params,
820
821
822
                    default_value=1,
                ).pop("verbosity")
                > 0,
823
824
            )
        )
wxchan's avatar
wxchan committed
825

826
    callbacks_before_iter_set = {cb for cb in callbacks_set if getattr(cb, "before_iteration", False)}
827
    callbacks_after_iter_set = callbacks_set - callbacks_before_iter_set
828
829
    callbacks_before_iter = sorted(callbacks_before_iter_set, key=attrgetter("order"))
    callbacks_after_iter = sorted(callbacks_after_iter_set, key=attrgetter("order"))
wxchan's avatar
wxchan committed
830

831
    for i in range(num_boost_round):
wxchan's avatar
wxchan committed
832
        for cb in callbacks_before_iter:
833
834
            cb(
                callback.CallbackEnv(
835
                    model=cvbooster,
836
837
838
839
840
841
842
                    params=params,
                    iteration=i,
                    begin_iteration=0,
                    end_iteration=num_boost_round,
                    evaluation_result_list=None,
                )
            )
843
844
        cvbooster.update(fobj=fobj)  # type: ignore[call-arg]
        res = _agg_cv_result(cvbooster.eval_valid(feval))  # type: ignore[call-arg]
845
846
847
        for dataset_name, metric_name, metric_mean, _, metric_std_dev in res:
            results[f"{dataset_name} {metric_name}-mean"].append(metric_mean)
            results[f"{dataset_name} {metric_name}-stdv"].append(metric_std_dev)
wxchan's avatar
wxchan committed
848
849
        try:
            for cb in callbacks_after_iter:
850
851
                cb(
                    callback.CallbackEnv(
852
                        model=cvbooster,
853
854
855
856
857
858
859
                        params=params,
                        iteration=i,
                        begin_iteration=0,
                        end_iteration=num_boost_round,
                        evaluation_result_list=res,
                    )
                )
860
        except callback.EarlyStopException as earlyStopException:
861
862
863
            cvbooster.best_iteration = earlyStopException.best_iteration + 1
            for bst in cvbooster.boosters:
                bst.best_iteration = cvbooster.best_iteration
wxchan's avatar
wxchan committed
864
            for k in results:
865
                results[k] = results[k][: cvbooster.best_iteration]
wxchan's avatar
wxchan committed
866
            break
867
868

    if return_cvbooster:
869
        results["cvbooster"] = cvbooster  # type: ignore[assignment]
870

wxchan's avatar
wxchan committed
871
    return dict(results)