sklearn.py 62.3 KB
Newer Older
wxchan's avatar
wxchan committed
1
# coding: utf-8
2
"""Scikit-learn wrapper interface for LightGBM."""
3
import copy
4
from inspect import signature
5
from pathlib import Path
6
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
7

wxchan's avatar
wxchan committed
8
import numpy as np
9
import scipy.sparse
10

11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
from .basic import (
    Booster,
    Dataset,
    LightGBMError,
    _choose_param_value,
    _ConfigAliases,
    _LGBM_BoosterBestScoreType,
    _LGBM_CategoricalFeatureConfiguration,
    _LGBM_EvalFunctionResultType,
    _LGBM_FeatureNameConfiguration,
    _LGBM_GroupType,
    _LGBM_InitScoreType,
    _LGBM_LabelType,
    _LGBM_WeightType,
    _log_warning,
)
27
from .callback import _EvalResultDict, record_evaluation
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
from .compat import (
    SKLEARN_INSTALLED,
    LGBMNotFittedError,
    _LGBMAssertAllFinite,
    _LGBMCheckArray,
    _LGBMCheckClassificationTargets,
    _LGBMCheckSampleWeight,
    _LGBMCheckXY,
    _LGBMClassifierBase,
    _LGBMComputeSampleWeight,
    _LGBMCpuCount,
    _LGBMLabelEncoder,
    _LGBMModelBase,
    _LGBMRegressorBase,
    dt_DataTable,
    np_random_Generator,
    pd_DataFrame,
)
wxchan's avatar
wxchan committed
46
from .engine import train
47

48
49
50
51
52
53
54
__all__ = [
    'LGBMClassifier',
    'LGBMModel',
    'LGBMRanker',
    'LGBMRegressor',
]

55
56
57
58
59
60
61
_LGBM_ScikitMatrixLike = Union[
    dt_DataTable,
    List[Union[List[float], List[int]]],
    np.ndarray,
    pd_DataFrame,
    scipy.sparse.spmatrix
]
62
_LGBM_ScikitCustomObjectiveFunction = Union[
63
    # f(labels, preds)
64
    Callable[
65
        [Optional[np.ndarray], np.ndarray],
66
        Tuple[np.ndarray, np.ndarray]
67
    ],
68
    # f(labels, preds, weights)
69
    Callable[
70
        [Optional[np.ndarray], np.ndarray, Optional[np.ndarray]],
71
        Tuple[np.ndarray, np.ndarray]
72
    ],
73
    # f(labels, preds, weights, group)
74
    Callable[
75
        [Optional[np.ndarray], np.ndarray, Optional[np.ndarray], Optional[np.ndarray]],
76
77
        Tuple[np.ndarray, np.ndarray]
    ],
78
79
]
_LGBM_ScikitCustomEvalFunction = Union[
80
    # f(labels, preds)
81
    Callable[
82
83
        [Optional[np.ndarray], np.ndarray],
        _LGBM_EvalFunctionResultType
84
85
    ],
    Callable[
86
87
        [Optional[np.ndarray], np.ndarray],
        List[_LGBM_EvalFunctionResultType]
88
    ],
89
    # f(labels, preds, weights)
90
    Callable[
91
92
        [Optional[np.ndarray], np.ndarray, Optional[np.ndarray]],
        _LGBM_EvalFunctionResultType
93
    ],
94
95
96
97
98
99
100
101
102
103
104
105
106
    Callable[
        [Optional[np.ndarray], np.ndarray, Optional[np.ndarray]],
        List[_LGBM_EvalFunctionResultType]
    ],
    # f(labels, preds, weights, group)
    Callable[
        [Optional[np.ndarray], np.ndarray, Optional[np.ndarray], Optional[np.ndarray]],
        _LGBM_EvalFunctionResultType
    ],
    Callable[
        [Optional[np.ndarray], np.ndarray, Optional[np.ndarray], Optional[np.ndarray]],
        List[_LGBM_EvalFunctionResultType]
    ]
107
]
108
109
110
111
112
_LGBM_ScikitEvalMetricType = Union[
    str,
    _LGBM_ScikitCustomEvalFunction,
    List[Union[str, _LGBM_ScikitCustomEvalFunction]]
]
113
_LGBM_ScikitValidSet = Tuple[_LGBM_ScikitMatrixLike, _LGBM_LabelType]
114

wxchan's avatar
wxchan committed
115

116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
def _get_group_from_constructed_dataset(dataset: Dataset) -> Optional[np.ndarray]:
    group = dataset.get_group()
    error_msg = (
        "Estimators in lightgbm.sklearn should only retrieve query groups from a constructed Dataset. "
        "If you're seeing this message, it's a bug in lightgbm. Please report it at https://github.com/microsoft/LightGBM/issues."
    )
    assert (group is None or isinstance(group, np.ndarray)), error_msg
    return group


def _get_label_from_constructed_dataset(dataset: Dataset) -> np.ndarray:
    label = dataset.get_label()
    error_msg = (
        "Estimators in lightgbm.sklearn should only retrieve labels from a constructed Dataset. "
        "If you're seeing this message, it's a bug in lightgbm. Please report it at https://github.com/microsoft/LightGBM/issues."
    )
    assert isinstance(label, np.ndarray), error_msg
    return label


def _get_weight_from_constructed_dataset(dataset: Dataset) -> Optional[np.ndarray]:
    weight = dataset.get_weight()
    error_msg = (
        "Estimators in lightgbm.sklearn should only retrieve weights from a constructed Dataset. "
        "If you're seeing this message, it's a bug in lightgbm. Please report it at https://github.com/microsoft/LightGBM/issues."
    )
    assert (weight is None or isinstance(weight, np.ndarray)), error_msg
    return weight


146
class _ObjectiveFunctionWrapper:
147
    """Proxy class for objective function."""
148

149
    def __init__(self, func: _LGBM_ScikitCustomObjectiveFunction):
150
        """Construct a proxy class.
151

152
153
        This class transforms objective function to match objective function with signature ``new_func(preds, dataset)``
        as expected by ``lightgbm.engine.train``.
154

155
156
157
        Parameters
        ----------
        func : callable
158
159
160
161
            Expects a callable with following signatures:
            ``func(y_true, y_pred)``,
            ``func(y_true, y_pred, weight)``
            or ``func(y_true, y_pred, weight, group)``
162
163
            and returns (grad, hess):

164
                y_true : numpy 1-D array of shape = [n_samples]
165
                    The target values.
166
                y_pred : numpy 1-D array of shape = [n_samples] or numpy 2-D array of shape = [n_samples, n_classes] (for multi-class task)
167
                    The predicted values.
168
169
                    Predicted values are returned before any transformation,
                    e.g. they are raw margin instead of probability of positive class for binary task.
170
171
                weight : numpy 1-D array of shape = [n_samples]
                    The weight of samples. Weights should be non-negative.
172
                group : numpy 1-D array
173
174
175
                    Group/query data.
                    Only used in the learning-to-rank task.
                    sum(group) = n_samples.
176
177
                    For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
                    where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
178
                grad : numpy 1-D array of shape = [n_samples] or numpy 2-D array of shape [n_samples, n_classes] (for multi-class task)
179
180
                    The value of the first order derivative (gradient) of the loss
                    with respect to the elements of y_pred for each sample point.
181
                hess : numpy 1-D array of shape = [n_samples] or numpy 2-D array of shape = [n_samples, n_classes] (for multi-class task)
182
183
                    The value of the second order derivative (Hessian) of the loss
                    with respect to the elements of y_pred for each sample point.
wxchan's avatar
wxchan committed
184

Nikita Titov's avatar
Nikita Titov committed
185
186
        .. note::

187
            For multi-class task, y_pred is a numpy 2-D array of shape = [n_samples, n_classes],
188
            and grad and hess should be returned in the same format.
189
190
        """
        self.func = func
wxchan's avatar
wxchan committed
191

192
    def __call__(self, preds: np.ndarray, dataset: Dataset) -> Tuple[np.ndarray, np.ndarray]:
193
194
195
196
        """Call passed function with appropriate arguments.

        Parameters
        ----------
197
        preds : numpy 1-D array of shape = [n_samples] or numpy 2-D array of shape = [n_samples, n_classes] (for multi-class task)
198
199
200
201
202
203
            The predicted values.
        dataset : Dataset
            The training dataset.

        Returns
        -------
204
        grad : numpy 1-D array of shape = [n_samples] or numpy 2-D array of shape = [n_samples, n_classes] (for multi-class task)
205
206
            The value of the first order derivative (gradient) of the loss
            with respect to the elements of preds for each sample point.
207
        hess : numpy 1-D array of shape = [n_samples] or numpy 2-D array of shape = [n_samples, n_classes] (for multi-class task)
208
209
            The value of the second order derivative (Hessian) of the loss
            with respect to the elements of preds for each sample point.
210
        """
211
        labels = _get_label_from_constructed_dataset(dataset)
212
        argc = len(signature(self.func).parameters)
213
        if argc == 2:
214
            grad, hess = self.func(labels, preds)  # type: ignore[call-arg]
215
216
217
218
219
220
221
222
223
224
225
226
            return grad, hess

        weight = _get_weight_from_constructed_dataset(dataset)
        if argc == 3:
            grad, hess = self.func(labels, preds, weight)  # type: ignore[call-arg]
            return grad, hess

        if argc == 4:
            group = _get_group_from_constructed_dataset(dataset)
            return self.func(labels, preds, weight, group)  # type: ignore[call-arg]

        raise TypeError(f"Self-defined objective function should have 2, 3 or 4 arguments, got {argc}")
wxchan's avatar
wxchan committed
227

wxchan's avatar
wxchan committed
228

229
class _EvalFunctionWrapper:
230
    """Proxy class for evaluation function."""
231

232
    def __init__(self, func: _LGBM_ScikitCustomEvalFunction):
233
        """Construct a proxy class.
234

235
236
        This class transforms evaluation function to match evaluation function with signature ``new_func(preds, dataset)``
        as expected by ``lightgbm.engine.train``.
237

238
239
240
241
242
243
244
245
246
247
        Parameters
        ----------
        func : callable
            Expects a callable with following signatures:
            ``func(y_true, y_pred)``,
            ``func(y_true, y_pred, weight)``
            or ``func(y_true, y_pred, weight, group)``
            and returns (eval_name, eval_result, is_higher_better) or
            list of (eval_name, eval_result, is_higher_better):

248
                y_true : numpy 1-D array of shape = [n_samples]
249
                    The target values.
250
                y_pred : numpy 1-D array of shape = [n_samples] or numpy 2-D array shape = [n_samples, n_classes] (for multi-class task)
251
                    The predicted values.
252
253
                    In case of custom ``objective``, predicted values are returned before any transformation,
                    e.g. they are raw margin instead of probability of positive class for binary task in this case.
254
                weight : numpy 1-D array of shape = [n_samples]
255
                    The weight of samples. Weights should be non-negative.
256
                group : numpy 1-D array
257
258
259
                    Group/query data.
                    Only used in the learning-to-rank task.
                    sum(group) = n_samples.
260
261
                    For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
                    where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
262
                eval_name : str
Andrew Ziem's avatar
Andrew Ziem committed
263
                    The name of evaluation function (without whitespace).
264
265
266
267
268
269
                eval_result : float
                    The eval result.
                is_higher_better : bool
                    Is eval result higher better, e.g. AUC is ``is_higher_better``.
        """
        self.func = func
270

271
272
273
274
275
    def __call__(
        self,
        preds: np.ndarray,
        dataset: Dataset
    ) -> Union[_LGBM_EvalFunctionResultType, List[_LGBM_EvalFunctionResultType]]:
276
        """Call passed function with appropriate arguments.
277

278
279
        Parameters
        ----------
280
        preds : numpy 1-D array of shape = [n_samples] or numpy 2-D array of shape = [n_samples, n_classes] (for multi-class task)
281
282
283
284
285
286
            The predicted values.
        dataset : Dataset
            The training dataset.

        Returns
        -------
287
        eval_name : str
Andrew Ziem's avatar
Andrew Ziem committed
288
            The name of evaluation function (without whitespace).
289
290
291
292
293
        eval_result : float
            The eval result.
        is_higher_better : bool
            Is eval result higher better, e.g. AUC is ``is_higher_better``.
        """
294
        labels = _get_label_from_constructed_dataset(dataset)
295
        argc = len(signature(self.func).parameters)
296
        if argc == 2:
297
            return self.func(labels, preds)  # type: ignore[call-arg]
298
299
300
301
302
303
304
305
306
307

        weight = _get_weight_from_constructed_dataset(dataset)
        if argc == 3:
            return self.func(labels, preds, weight)  # type: ignore[call-arg]

        if argc == 4:
            group = _get_group_from_constructed_dataset(dataset)
            return self.func(labels, preds, weight, group)  # type: ignore[call-arg]

        raise TypeError(f"Self-defined eval function should have 2, 3 or 4 arguments, got {argc}")
308

wxchan's avatar
wxchan committed
309

310
311
312
313
314
315
316
317
318
319
320
321
322
323
# documentation templates for LGBMModel methods are shared between the classes in
# this module and those in the ``dask`` module

_lgbmmodel_doc_fit = (
    """
    Build a gradient boosting model from the training set (X, y).

    Parameters
    ----------
    X : {X_shape}
        Input feature matrix.
    y : {y_shape}
        The target values (class labels in classification, real numbers in regression).
    sample_weight : {sample_weight_shape}
324
        Weights of training data. Weights should be non-negative.
325
    init_score : {init_score_shape}
326
327
328
329
330
331
332
333
334
        Init score of training data.
    group : {group_shape}
        Group/query data.
        Only used in the learning-to-rank task.
        sum(group) = n_samples.
        For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
        where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
    eval_set : list or None, optional (default=None)
        A list of (X, y) tuple pairs to use as validation sets.
335
    eval_names : list of str, or None, optional (default=None)
336
        Names of eval_set.
337
    eval_sample_weight : {eval_sample_weight_shape}
338
        Weights of eval data. Weights should be non-negative.
339
340
    eval_class_weight : list or None, optional (default=None)
        Class weights of eval data.
341
    eval_init_score : {eval_init_score_shape}
342
        Init score of eval data.
343
    eval_group : {eval_group_shape}
344
        Group data of eval data.
345
346
    eval_metric : str, callable, list or None, optional (default=None)
        If str, it should be a built-in evaluation metric to use.
347
348
349
350
        If callable, it should be a custom evaluation metric, see note below for more details.
        If list, it can be a list of built-in metrics, a list of custom evaluation metrics, or a mix of both.
        In either case, the ``metric`` from the model parameters will be evaluated and used as well.
        Default: 'l2' for LGBMRegressor, 'logloss' for LGBMClassifier, 'ndcg' for LGBMRanker.
351
    feature_name : list of str, or 'auto', optional (default='auto')
352
353
        Feature names.
        If 'auto' and data is pandas DataFrame, data columns names are used.
354
    categorical_feature : list of str or int, or 'auto', optional (default='auto')
355
356
        Categorical features.
        If list of int, interpreted as indices.
357
        If list of str, interpreted as feature names (need to specify ``feature_name`` as well).
358
        If 'auto' and data is pandas DataFrame, pandas unordered categorical columns are used.
359
        All values in categorical features will be cast to int32 and thus should be less than int32 max value (2147483647).
360
361
362
        Large values could be memory consuming. Consider using consecutive integers starting from zero.
        All negative values in categorical features will be treated as missing values.
        The output cannot be monotonically constrained with respect to a categorical feature.
363
        Floating point numbers in categorical features will be rounded towards 0.
364
    callbacks : list of callable, or None, optional (default=None)
365
366
        List of callback functions that are applied at each iteration.
        See Callbacks in Python API for more information.
367
    init_model : str, pathlib.Path, Booster, LGBMModel or None, optional (default=None)
368
369
370
371
        Filename of LightGBM model, Booster instance or LGBMModel instance used for continue training.

    Returns
    -------
372
    self : LGBMModel
373
374
375
376
377
378
379
380
381
382
383
384
385
        Returns self.
    """
)

_lgbmmodel_doc_custom_eval_note = """
    Note
    ----
    Custom eval function expects a callable with following signatures:
    ``func(y_true, y_pred)``, ``func(y_true, y_pred, weight)`` or
    ``func(y_true, y_pred, weight, group)``
    and returns (eval_name, eval_result, is_higher_better) or
    list of (eval_name, eval_result, is_higher_better):

386
        y_true : numpy 1-D array of shape = [n_samples]
387
            The target values.
388
        y_pred : numpy 1-D array of shape = [n_samples] or numpy 2-D array of shape = [n_samples, n_classes] (for multi-class task)
389
            The predicted values.
390
391
            In case of custom ``objective``, predicted values are returned before any transformation,
            e.g. they are raw margin instead of probability of positive class for binary task in this case.
392
        weight : numpy 1-D array of shape = [n_samples]
393
            The weight of samples. Weights should be non-negative.
394
        group : numpy 1-D array
395
396
397
398
399
            Group/query data.
            Only used in the learning-to-rank task.
            sum(group) = n_samples.
            For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
            where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
400
        eval_name : str
Andrew Ziem's avatar
Andrew Ziem committed
401
            The name of evaluation function (without whitespace).
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
        eval_result : float
            The eval result.
        is_higher_better : bool
            Is eval result higher better, e.g. AUC is ``is_higher_better``.
"""

_lgbmmodel_doc_predict = (
    """
    {description}

    Parameters
    ----------
    X : {X_shape}
        Input features matrix.
    raw_score : bool, optional (default=False)
        Whether to predict raw scores.
    start_iteration : int, optional (default=0)
        Start index of the iteration to predict.
        If <= 0, starts from the first iteration.
    num_iteration : int or None, optional (default=None)
        Total number of iterations used in the prediction.
        If None, if the best iteration exists and start_iteration <= 0, the best iteration is used;
        otherwise, all iterations from ``start_iteration`` are used (no limits).
        If <= 0, all iterations from ``start_iteration`` are used (no limits).
    pred_leaf : bool, optional (default=False)
        Whether to predict leaf index.
    pred_contrib : bool, optional (default=False)
        Whether to predict feature contributions.

        .. note::

            If you want to get more explanations for your model's predictions using SHAP values,
            like SHAP interaction values,
            you can install the shap package (https://github.com/slundberg/shap).
            Note that unlike the shap package, with ``pred_contrib`` we return a matrix with an extra
            column, where the last column is the expected value.

439
440
441
    validate_features : bool, optional (default=False)
        If True, ensure that the features used to predict match the ones used to train.
        Used only if data is pandas DataFrame.
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
    **kwargs
        Other parameters for the prediction.

    Returns
    -------
    {output_name} : {predicted_result_shape}
        The predicted values.
    X_leaves : {X_leaves_shape}
        If ``pred_leaf=True``, the predicted leaf of every tree for each sample.
    X_SHAP_values : {X_SHAP_values_shape}
        If ``pred_contrib=True``, the feature contributions for each sample.
    """
)


457
458
class LGBMModel(_LGBMModelBase):
    """Implementation of the scikit-learn API for LightGBM."""
wxchan's avatar
wxchan committed
459

460
461
462
463
464
465
466
467
    def __init__(
        self,
        boosting_type: str = 'gbdt',
        num_leaves: int = 31,
        max_depth: int = -1,
        learning_rate: float = 0.1,
        n_estimators: int = 100,
        subsample_for_bin: int = 200000,
468
        objective: Optional[Union[str, _LGBM_ScikitCustomObjectiveFunction]] = None,
469
470
471
472
473
474
475
476
477
        class_weight: Optional[Union[Dict, str]] = None,
        min_split_gain: float = 0.,
        min_child_weight: float = 1e-3,
        min_child_samples: int = 20,
        subsample: float = 1.,
        subsample_freq: int = 0,
        colsample_bytree: float = 1.,
        reg_alpha: float = 0.,
        reg_lambda: float = 0.,
478
        random_state: Optional[Union[int, np.random.RandomState, 'np.random.Generator']] = None,
479
        n_jobs: Optional[int] = None,
480
481
482
        importance_type: str = 'split',
        **kwargs
    ):
483
        r"""Construct a gradient boosting model.
wxchan's avatar
wxchan committed
484
485
486

        Parameters
        ----------
487
        boosting_type : str, optional (default='gbdt')
488
489
490
491
            'gbdt', traditional Gradient Boosting Decision Tree.
            'dart', Dropouts meet Multiple Additive Regression Trees.
            'rf', Random Forest.
        num_leaves : int, optional (default=31)
wxchan's avatar
wxchan committed
492
            Maximum tree leaves for base learners.
493
        max_depth : int, optional (default=-1)
494
            Maximum tree depth for base learners, <=0 means no limit.
495
        learning_rate : float, optional (default=0.1)
496
            Boosting learning rate.
497
498
499
            You can use ``callbacks`` parameter of ``fit`` method to shrink/adapt learning rate
            in training using ``reset_parameter`` callback.
            Note, that this will ignore the ``learning_rate`` argument in training.
500
        n_estimators : int, optional (default=100)
wxchan's avatar
wxchan committed
501
            Number of boosted trees to fit.
502
        subsample_for_bin : int, optional (default=200000)
wxchan's avatar
wxchan committed
503
            Number of samples for constructing bins.
504
        objective : str, callable or None, optional (default=None)
wxchan's avatar
wxchan committed
505
506
            Specify the learning task and the corresponding learning objective or
            a custom objective function to be used (see note below).
507
            Default: 'regression' for LGBMRegressor, 'binary' or 'multiclass' for LGBMClassifier, 'lambdarank' for LGBMRanker.
508
509
510
511
        class_weight : dict, 'balanced' or None, optional (default=None)
            Weights associated with classes in the form ``{class_label: weight}``.
            Use this parameter only for multi-class classification task;
            for binary classification task you may use ``is_unbalance`` or ``scale_pos_weight`` parameters.
512
513
514
            Note, that the usage of all these parameters will result in poor estimates of the individual class probabilities.
            You may want to consider performing probability calibration
            (https://scikit-learn.org/stable/modules/calibration.html) of your model.
515
516
517
            The 'balanced' mode uses the values of y to automatically adjust weights
            inversely proportional to class frequencies in the input data as ``n_samples / (n_classes * np.bincount(y))``.
            If None, all classes are supposed to have weight one.
518
            Note, that these weights will be multiplied with ``sample_weight`` (passed through the ``fit`` method)
519
            if ``sample_weight`` is specified.
520
        min_split_gain : float, optional (default=0.)
wxchan's avatar
wxchan committed
521
            Minimum loss reduction required to make a further partition on a leaf node of the tree.
522
        min_child_weight : float, optional (default=1e-3)
523
            Minimum sum of instance weight (Hessian) needed in a child (leaf).
524
        min_child_samples : int, optional (default=20)
525
            Minimum number of data needed in a child (leaf).
526
        subsample : float, optional (default=1.)
wxchan's avatar
wxchan committed
527
            Subsample ratio of the training instance.
528
        subsample_freq : int, optional (default=0)
Andrew Ziem's avatar
Andrew Ziem committed
529
            Frequency of subsample, <=0 means no enable.
530
        colsample_bytree : float, optional (default=1.)
wxchan's avatar
wxchan committed
531
            Subsample ratio of columns when constructing each tree.
532
        reg_alpha : float, optional (default=0.)
533
            L1 regularization term on weights.
534
        reg_lambda : float, optional (default=0.)
535
            L2 regularization term on weights.
536
        random_state : int, RandomState object or None, optional (default=None)
wxchan's avatar
wxchan committed
537
            Random number seed.
538
            If int, this number is used to seed the C++ code.
539
            If RandomState or Generator object (numpy), a random integer is picked based on its state to seed the C++ code.
540
            If None, default seeds in C++ code are used.
541
542
543
544
545
546
547
548
549
550
551
552
        n_jobs : int or None, optional (default=None)
            Number of parallel threads to use for training (can be changed at prediction time by
            passing it as an extra keyword argument).

            For better performance, it is recommended to set this to the number of physical cores
            in the CPU.

            Negative integers are interpreted as following joblib's formula (n_cpus + 1 + n_jobs), just like
            scikit-learn (so e.g. -1 means using all threads). A value of zero corresponds the default number of
            threads configured for OpenMP in the system. A value of ``None`` (the default) corresponds
            to using the number of physical cores in the system (its correct detection requires
            either the ``joblib`` or the ``psutil`` util libraries to be installed).
553
554
555

            .. versionchanged:: 4.0.0

556
        importance_type : str, optional (default='split')
557
            The type of feature importance to be filled into ``feature_importances_``.
558
559
560
561
            If 'split', result contains numbers of times the feature is used in a model.
            If 'gain', result contains total gains of splits which use the feature.
        **kwargs
            Other parameters for the model.
wxchan's avatar
wxchan committed
562
            Check http://lightgbm.readthedocs.io/en/latest/Parameters.html for more parameters.
563

Nikita Titov's avatar
Nikita Titov committed
564
565
566
            .. warning::

                \*\*kwargs is not supported in sklearn, it may cause unexpected issues.
wxchan's avatar
wxchan committed
567
568
569

        Note
        ----
570
571
        A custom objective function can be provided for the ``objective`` parameter.
        In this case, it should have the signature
572
573
574
        ``objective(y_true, y_pred) -> grad, hess``,
        ``objective(y_true, y_pred, weight) -> grad, hess``
        or ``objective(y_true, y_pred, weight, group) -> grad, hess``:
wxchan's avatar
wxchan committed
575

576
            y_true : numpy 1-D array of shape = [n_samples]
577
                The target values.
578
            y_pred : numpy 1-D array of shape = [n_samples] or numpy 2-D array of shape = [n_samples, n_classes] (for multi-class task)
579
                The predicted values.
580
581
                Predicted values are returned before any transformation,
                e.g. they are raw margin instead of probability of positive class for binary task.
582
583
            weight : numpy 1-D array of shape = [n_samples]
                The weight of samples. Weights should be non-negative.
584
            group : numpy 1-D array
585
586
587
                Group/query data.
                Only used in the learning-to-rank task.
                sum(group) = n_samples.
588
589
                For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
                where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
590
            grad : numpy 1-D array of shape = [n_samples] or numpy 2-D array of shape = [n_samples, n_classes] (for multi-class task)
591
592
                The value of the first order derivative (gradient) of the loss
                with respect to the elements of y_pred for each sample point.
593
            hess : numpy 1-D array of shape = [n_samples] or numpy 2-D array of shape = [n_samples, n_classes] (for multi-class task)
594
595
                The value of the second order derivative (Hessian) of the loss
                with respect to the elements of y_pred for each sample point.
wxchan's avatar
wxchan committed
596

597
        For multi-class task, y_pred is a numpy 2-D array of shape = [n_samples, n_classes],
598
        and grad and hess should be returned in the same format.
wxchan's avatar
wxchan committed
599
        """
wxchan's avatar
wxchan committed
600
        if not SKLEARN_INSTALLED:
601
602
            raise LightGBMError('scikit-learn is required for lightgbm.sklearn. '
                                'You must install scikit-learn and restart your session to use this module.')
wxchan's avatar
wxchan committed
603

604
        self.boosting_type = boosting_type
605
        self.objective = objective
wxchan's avatar
wxchan committed
606
607
608
609
        self.num_leaves = num_leaves
        self.max_depth = max_depth
        self.learning_rate = learning_rate
        self.n_estimators = n_estimators
wxchan's avatar
wxchan committed
610
        self.subsample_for_bin = subsample_for_bin
wxchan's avatar
wxchan committed
611
612
613
614
615
616
617
618
        self.min_split_gain = min_split_gain
        self.min_child_weight = min_child_weight
        self.min_child_samples = min_child_samples
        self.subsample = subsample
        self.subsample_freq = subsample_freq
        self.colsample_bytree = colsample_bytree
        self.reg_alpha = reg_alpha
        self.reg_lambda = reg_lambda
619
620
        self.random_state = random_state
        self.n_jobs = n_jobs
621
        self.importance_type = importance_type
622
        self._Booster: Optional[Booster] = None
623
624
        self._evals_result: _EvalResultDict = {}
        self._best_score: _LGBM_BoosterBestScoreType = {}
625
        self._best_iteration: int = -1
626
        self._other_params: Dict[str, Any] = {}
627
        self._objective = objective
628
        self.class_weight = class_weight
629
630
        self._class_weight: Optional[Union[Dict, str]] = None
        self._class_map: Optional[Dict[int, int]] = None
631
632
        self._n_features: int = -1
        self._n_features_in: int = -1
633
        self._classes: Optional[np.ndarray] = None
634
        self._n_classes: int = -1
635
        self.set_params(**kwargs)
wxchan's avatar
wxchan committed
636

637
    def _more_tags(self) -> Dict[str, Any]:
638
639
640
641
642
643
644
645
646
647
        return {
            'allow_nan': True,
            'X_types': ['2darray', 'sparse', '1dlabels'],
            '_xfail_checks': {
                'check_no_attributes_set_in_init':
                'scikit-learn incorrectly asserts that private attributes '
                'cannot be set in __init__: '
                '(see https://github.com/microsoft/LightGBM/issues/2628)'
            }
        }
Nikita Titov's avatar
Nikita Titov committed
648

649
650
651
    def __sklearn_is_fitted__(self) -> bool:
        return getattr(self, "fitted_", False)

652
    def get_params(self, deep: bool = True) -> Dict[str, Any]:
653
654
655
656
657
658
659
660
661
662
663
664
665
        """Get parameters for this estimator.

        Parameters
        ----------
        deep : bool, optional (default=True)
            If True, will return the parameters for this estimator and
            contained subobjects that are estimators.

        Returns
        -------
        params : dict
            Parameter names mapped to their values.
        """
666
        params = super().get_params(deep=deep)
667
        params.update(self._other_params)
wxchan's avatar
wxchan committed
668
669
        return params

670
    def set_params(self, **params: Any) -> "LGBMModel":
671
672
673
674
675
676
677
678
679
680
681
682
        """Set the parameters of this estimator.

        Parameters
        ----------
        **params
            Parameter names with their new values.

        Returns
        -------
        self : object
            Returns self.
        """
wxchan's avatar
wxchan committed
683
684
        for key, value in params.items():
            setattr(self, key, value)
685
686
            if hasattr(self, f"_{key}"):
                setattr(self, f"_{key}", value)
687
            self._other_params[key] = value
wxchan's avatar
wxchan committed
688
        return self
wxchan's avatar
wxchan committed
689

690
691
692
693
694
695
696
697
698
699
700
701
702
703
    def _process_params(self, stage: str) -> Dict[str, Any]:
        """Process the parameters of this estimator based on its type, parameter aliases, etc.

        Parameters
        ----------
        stage : str
            Name of the stage (can be ``fit`` or ``predict``) this method is called from.

        Returns
        -------
        processed_params : dict
            Processed parameter names mapped to their values.
        """
        assert stage in {"fit", "predict"}
704
705
706
707
708
        params = self.get_params()

        params.pop('objective', None)
        for alias in _ConfigAliases.get('objective'):
            if alias in params:
709
                obj = params.pop(alias)
710
                _log_warning(f"Found '{alias}' in params. Will use it instead of 'objective' argument")
711
712
713
714
715
716
717
718
719
720
721
722
723
                if stage == "fit":
                    self._objective = obj
        if stage == "fit":
            if self._objective is None:
                if isinstance(self, LGBMRegressor):
                    self._objective = "regression"
                elif isinstance(self, LGBMClassifier):
                    if self._n_classes > 2:
                        self._objective = "multiclass"
                    else:
                        self._objective = "binary"
                elif isinstance(self, LGBMRanker):
                    self._objective = "lambdarank"
724
                else:
725
                    raise ValueError("Unknown LGBMModel type.")
726
        if callable(self._objective):
727
            if stage == "fit":
728
729
730
                params['objective'] = _ObjectiveFunctionWrapper(self._objective)
            else:
                params['objective'] = 'None'
731
        else:
732
            params['objective'] = self._objective
733

734
        params.pop('importance_type', None)
wxchan's avatar
wxchan committed
735
        params.pop('n_estimators', None)
736
        params.pop('class_weight', None)
737

738
739
        if isinstance(params['random_state'], np.random.RandomState):
            params['random_state'] = params['random_state'].randint(np.iinfo(np.int32).max)
740
741
742
743
        elif isinstance(params['random_state'], np_random_Generator):
            params['random_state'] = int(
                params['random_state'].integers(np.iinfo(np.int32).max)
            )
744
        if self._n_classes > 2:
745
746
            for alias in _ConfigAliases.get('num_class'):
                params.pop(alias, None)
747
748
            params['num_class'] = self._n_classes
        if hasattr(self, '_eval_at'):
749
            eval_at = self._eval_at
750
            for alias in _ConfigAliases.get('eval_at'):
751
752
753
754
                if alias in params:
                    _log_warning(f"Found '{alias}' in params. Will use it instead of 'eval_at' argument")
                    eval_at = params.pop(alias)
            params['eval_at'] = eval_at
wxchan's avatar
wxchan committed
755

756
        # register default metric for consistency with callable eval_metric case
757
        original_metric = self._objective if isinstance(self._objective, str) else None
758
759
760
761
762
763
764
765
766
767
        if original_metric is None:
            # try to deduce from class instance
            if isinstance(self, LGBMRegressor):
                original_metric = "l2"
            elif isinstance(self, LGBMClassifier):
                original_metric = "multi_logloss" if self._n_classes > 2 else "binary_logloss"
            elif isinstance(self, LGBMRanker):
                original_metric = "ndcg"

        # overwrite default metric by explicitly set metric
768
        params = _choose_param_value("metric", params, original_metric)
769

770
771
772
773
774
775
        # use joblib conventions for negative n_jobs, just like scikit-learn
        # at predict time, this is handled later due to the order of parameter updates
        if stage == "fit":
            params = _choose_param_value("num_threads", params, self.n_jobs)
            params["num_threads"] = self._process_n_jobs(params["num_threads"])

776
777
        return params

778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
    def _process_n_jobs(self, n_jobs: Optional[int]) -> int:
        """Convert special values of n_jobs to their actual values according to the formulas that apply.

        Parameters
        ----------
        n_jobs : int or None
            The original value of n_jobs, potentially having special values such as 'None' or
            negative integers.

        Returns
        -------
        n_jobs : int
            The value of n_jobs with special values converted to actual number of threads.
        """
        if n_jobs is None:
            n_jobs = _LGBMCpuCount(only_physical_cores=True)
        elif n_jobs < 0:
            n_jobs = max(_LGBMCpuCount(only_physical_cores=False) + 1 + n_jobs, 1)
        return n_jobs

798
799
    def fit(
        self,
800
801
        X: _LGBM_ScikitMatrixLike,
        y: _LGBM_LabelType,
802
803
        sample_weight: Optional[_LGBM_WeightType] = None,
        init_score: Optional[_LGBM_InitScoreType] = None,
804
        group: Optional[_LGBM_GroupType] = None,
805
        eval_set: Optional[List[_LGBM_ScikitValidSet]] = None,
806
        eval_names: Optional[List[str]] = None,
807
808
809
810
        eval_sample_weight: Optional[List[_LGBM_WeightType]] = None,
        eval_class_weight: Optional[List[float]] = None,
        eval_init_score: Optional[List[_LGBM_InitScoreType]] = None,
        eval_group: Optional[List[_LGBM_GroupType]] = None,
811
        eval_metric: Optional[_LGBM_ScikitEvalMetricType] = None,
812
813
        feature_name: _LGBM_FeatureNameConfiguration = 'auto',
        categorical_feature: _LGBM_CategoricalFeatureConfiguration = 'auto',
814
        callbacks: Optional[List[Callable]] = None,
815
        init_model: Optional[Union[str, Path, Booster, "LGBMModel"]] = None
816
    ) -> "LGBMModel":
817
818
819
820
821
        """Docstring is set after definition, using a template."""
        params = self._process_params(stage="fit")

        # Do not modify original args in fit function
        # Refer to https://github.com/microsoft/LightGBM/pull/2619
822
823
824
825
826
827
828
        eval_metric_list: List[Union[str, _LGBM_ScikitCustomEvalFunction]]
        if eval_metric is None:
            eval_metric_list = []
        elif isinstance(eval_metric, list):
            eval_metric_list = copy.deepcopy(eval_metric)
        else:
            eval_metric_list = [copy.deepcopy(eval_metric)]
829
830
831
832
833

        # Separate built-in from callable evaluation metrics
        eval_metrics_callable = [_EvalFunctionWrapper(f) for f in eval_metric_list if callable(f)]
        eval_metrics_builtin = [m for m in eval_metric_list if isinstance(m, str)]

834
        # concatenate metric from params (or default if not provided in params) and eval_metric
835
836
        params['metric'] = [params['metric']] if isinstance(params['metric'], (str, type(None))) else params['metric']
        params['metric'] = [e for e in eval_metrics_builtin if e not in params['metric']] + params['metric']
837
        params['metric'] = [metric for metric in params['metric'] if metric is not None]
wxchan's avatar
wxchan committed
838

839
        if not isinstance(X, (pd_DataFrame, dt_DataTable)):
840
            _X, _y = _LGBMCheckXY(X, y, accept_sparse=True, force_all_finite=False, ensure_min_samples=2)
841
842
            if sample_weight is not None:
                sample_weight = _LGBMCheckSampleWeight(sample_weight, _X)
843
844
        else:
            _X, _y = X, y
845

846
847
848
849
        if self._class_weight is None:
            self._class_weight = self.class_weight
        if self._class_weight is not None:
            class_sample_weight = _LGBMComputeSampleWeight(self._class_weight, y)
850
851
852
853
            if sample_weight is None or len(sample_weight) == 0:
                sample_weight = class_sample_weight
            else:
                sample_weight = np.multiply(sample_weight, class_sample_weight)
854

855
        self._n_features = _X.shape[1]
856
857
        # copy for consistency
        self._n_features_in = self._n_features
858

859
860
861
        train_set = Dataset(data=_X, label=_y, weight=sample_weight, group=group,
                            init_score=init_score, categorical_feature=categorical_feature,
                            params=params)
Guolin Ke's avatar
Guolin Ke committed
862

863
        valid_sets: List[Dataset] = []
Guolin Ke's avatar
Guolin Ke committed
864
        if eval_set is not None:
865

866
            def _get_meta_data(collection, name, i):
867
868
869
870
871
872
873
                if collection is None:
                    return None
                elif isinstance(collection, list):
                    return collection[i] if len(collection) > i else None
                elif isinstance(collection, dict):
                    return collection.get(i, None)
                else:
874
                    raise TypeError(f"{name} should be dict or list")
875

Guolin Ke's avatar
Guolin Ke committed
876
877
878
            if isinstance(eval_set, tuple):
                eval_set = [eval_set]
            for i, valid_data in enumerate(eval_set):
879
                # reduce cost for prediction training data
Guolin Ke's avatar
Guolin Ke committed
880
881
882
                if valid_data[0] is X and valid_data[1] is y:
                    valid_set = train_set
                else:
883
884
885
886
887
888
                    valid_weight = _get_meta_data(eval_sample_weight, 'eval_sample_weight', i)
                    valid_class_weight = _get_meta_data(eval_class_weight, 'eval_class_weight', i)
                    if valid_class_weight is not None:
                        if isinstance(valid_class_weight, dict) and self._class_map is not None:
                            valid_class_weight = {self._class_map[k]: v for k, v in valid_class_weight.items()}
                        valid_class_sample_weight = _LGBMComputeSampleWeight(valid_class_weight, valid_data[1])
889
890
891
892
                        if valid_weight is None or len(valid_weight) == 0:
                            valid_weight = valid_class_sample_weight
                        else:
                            valid_weight = np.multiply(valid_weight, valid_class_sample_weight)
893
894
                    valid_init_score = _get_meta_data(eval_init_score, 'eval_init_score', i)
                    valid_group = _get_meta_data(eval_group, 'eval_group', i)
895
896
897
898
                    valid_set = Dataset(data=valid_data[0], label=valid_data[1], weight=valid_weight,
                                        group=valid_group, init_score=valid_init_score,
                                        categorical_feature='auto', params=params)

Guolin Ke's avatar
Guolin Ke committed
899
900
                valid_sets.append(valid_set)

901
902
903
        if isinstance(init_model, LGBMModel):
            init_model = init_model.booster_

904
905
906
        if callbacks is None:
            callbacks = []
        else:
907
            callbacks = copy.copy(callbacks)  # don't use deepcopy here to allow non-serializable objects
908

909
        evals_result: _EvalResultDict = {}
910
911
912
913
914
915
916
917
        callbacks.append(record_evaluation(evals_result))

        self._Booster = train(
            params=params,
            train_set=train_set,
            num_boost_round=self.n_estimators,
            valid_sets=valid_sets,
            valid_names=eval_names,
918
            feval=eval_metrics_callable,  # type: ignore[arg-type]
919
920
921
922
            init_model=init_model,
            feature_name=feature_name,
            callbacks=callbacks
        )
wxchan's avatar
wxchan committed
923

924
        self._evals_result = evals_result
925
        self._best_iteration = self._Booster.best_iteration
926
        self._best_score = self._Booster.best_score
wxchan's avatar
wxchan committed
927

928
929
        self.fitted_ = True

wxchan's avatar
wxchan committed
930
        # free dataset
931
        self._Booster.free_dataset()
wxchan's avatar
wxchan committed
932
        del train_set, valid_sets
wxchan's avatar
wxchan committed
933
934
        return self

935
    fit.__doc__ = _lgbmmodel_doc_fit.format(
936
937
        X_shape="numpy array, pandas DataFrame, H2O DataTable's Frame , scipy.sparse, list of lists of int or float of shape = [n_samples, n_features]",
        y_shape="numpy array, pandas DataFrame, pandas Series, list of int or float of shape = [n_samples]",
938
939
        sample_weight_shape="numpy array, pandas Series, list of int or float of shape = [n_samples] or None, optional (default=None)",
        init_score_shape="numpy array, pandas DataFrame, pandas Series, list of int or float of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task) or shape = [n_samples, n_classes] (for multi-class task) or None, optional (default=None)",
940
        group_shape="numpy array, pandas Series, list of int or float, or None, optional (default=None)",
941
942
943
        eval_sample_weight_shape="list of array (same types as ``sample_weight`` supports), or None, optional (default=None)",
        eval_init_score_shape="list of array (same types as ``init_score`` supports), or None, optional (default=None)",
        eval_group_shape="list of array (same types as ``group`` supports), or None, optional (default=None)"
944
945
    ) + "\n\n" + _lgbmmodel_doc_custom_eval_note

946
947
    def predict(
        self,
948
        X: _LGBM_ScikitMatrixLike,
949
950
951
952
953
954
955
956
        raw_score: bool = False,
        start_iteration: int = 0,
        num_iteration: Optional[int] = None,
        pred_leaf: bool = False,
        pred_contrib: bool = False,
        validate_features: bool = False,
        **kwargs: Any
    ):
957
        """Docstring is set after definition, using a template."""
958
        if not self.__sklearn_is_fitted__():
959
            raise LGBMNotFittedError("Estimator not fitted, call fit before exploiting the model.")
960
        if not isinstance(X, (pd_DataFrame, dt_DataTable)):
961
            X = _LGBMCheckArray(X, accept_sparse=True, force_all_finite=False)
962
963
964
        n_features = X.shape[1]
        if self._n_features != n_features:
            raise ValueError("Number of features of the model must "
965
966
                             f"match the input. Model n_features_ is {self._n_features} and "
                             f"input n_features is {n_features}")
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
        # retrive original params that possibly can be used in both training and prediction
        # and then overwrite them (considering aliases) with params that were passed directly in prediction
        predict_params = self._process_params(stage="predict")
        for alias in _ConfigAliases.get_by_alias(
            "data",
            "X",
            "raw_score",
            "start_iteration",
            "num_iteration",
            "pred_leaf",
            "pred_contrib",
            *kwargs.keys()
        ):
            predict_params.pop(alias, None)
        predict_params.update(kwargs)
982
983
984

        # number of threads can have values with special meaning which is only applied
        # in the scikit-learn interface, these should not reach the c++ side as-is
985
986
        predict_params = _choose_param_value("num_threads", predict_params, self.n_jobs)
        predict_params["num_threads"] = self._process_n_jobs(predict_params["num_threads"])
987

988
989
990
991
992
        return self._Booster.predict(  # type: ignore[union-attr]
            X, raw_score=raw_score, start_iteration=start_iteration, num_iteration=num_iteration,
            pred_leaf=pred_leaf, pred_contrib=pred_contrib, validate_features=validate_features,
            **predict_params
        )
wxchan's avatar
wxchan committed
993

994
995
    predict.__doc__ = _lgbmmodel_doc_predict.format(
        description="Return the predicted value for each sample.",
996
        X_shape="numpy array, pandas DataFrame, H2O DataTable's Frame , scipy.sparse, list of lists of int or float of shape = [n_samples, n_features]",
997
998
999
1000
1001
1002
        output_name="predicted_result",
        predicted_result_shape="array-like of shape = [n_samples] or shape = [n_samples, n_classes]",
        X_leaves_shape="array-like of shape = [n_samples, n_trees] or shape = [n_samples, n_trees * n_classes]",
        X_SHAP_values_shape="array-like of shape = [n_samples, n_features + 1] or shape = [n_samples, (n_features + 1) * n_classes] or list with n_classes length of such objects"
    )

1003
    @property
1004
    def n_features_(self) -> int:
1005
        """:obj:`int`: The number of features of fitted model."""
1006
        if not self.__sklearn_is_fitted__():
1007
1008
1009
            raise LGBMNotFittedError('No n_features found. Need to call fit beforehand.')
        return self._n_features

1010
    @property
1011
    def n_features_in_(self) -> int:
1012
        """:obj:`int`: The number of features of fitted model."""
1013
        if not self.__sklearn_is_fitted__():
1014
1015
1016
            raise LGBMNotFittedError('No n_features_in found. Need to call fit beforehand.')
        return self._n_features_in

1017
    @property
1018
    def best_score_(self) -> _LGBM_BoosterBestScoreType:
1019
        """:obj:`dict`: The best score of fitted model."""
1020
        if not self.__sklearn_is_fitted__():
1021
1022
1023
1024
            raise LGBMNotFittedError('No best_score found. Need to call fit beforehand.')
        return self._best_score

    @property
1025
    def best_iteration_(self) -> int:
1026
        """:obj:`int`: The best iteration of fitted model if ``early_stopping()`` callback has been specified."""
1027
        if not self.__sklearn_is_fitted__():
1028
            raise LGBMNotFittedError('No best_iteration found. Need to call fit with early_stopping callback beforehand.')
1029
1030
1031
        return self._best_iteration

    @property
1032
    def objective_(self) -> Union[str, _LGBM_ScikitCustomObjectiveFunction]:
1033
        """:obj:`str` or :obj:`callable`: The concrete objective used while fitting this model."""
1034
        if not self.__sklearn_is_fitted__():
1035
            raise LGBMNotFittedError('No objective found. Need to call fit beforehand.')
1036
        return self._objective  # type: ignore[return-value]
1037

1038
1039
1040
1041
1042
1043
    @property
    def n_estimators_(self) -> int:
        """:obj:`int`: True number of boosting iterations performed.

        This might be less than parameter ``n_estimators`` if early stopping was enabled or
        if boosting stopped early due to limits on complexity like ``min_gain_to_split``.
1044
1045
        
        .. versionadded:: 4.0.0
1046
1047
1048
        """
        if not self.__sklearn_is_fitted__():
            raise LGBMNotFittedError('No n_estimators found. Need to call fit beforehand.')
1049
        return self._Booster.current_iteration()  # type: ignore
1050
1051
1052
1053
1054
1055
1056

    @property
    def n_iter_(self) -> int:
        """:obj:`int`: True number of boosting iterations performed.

        This might be less than parameter ``n_estimators`` if early stopping was enabled or
        if boosting stopped early due to limits on complexity like ``min_gain_to_split``.
1057
1058
        
        .. versionadded:: 4.0.0
1059
1060
1061
        """
        if not self.__sklearn_is_fitted__():
            raise LGBMNotFittedError('No n_iter found. Need to call fit beforehand.')
1062
        return self._Booster.current_iteration()  # type: ignore
1063

1064
    @property
1065
    def booster_(self) -> Booster:
1066
        """Booster: The underlying Booster of this model."""
1067
        if not self.__sklearn_is_fitted__():
1068
            raise LGBMNotFittedError('No booster found. Need to call fit beforehand.')
1069
        return self._Booster  # type: ignore[return-value]
wxchan's avatar
wxchan committed
1070

1071
    @property
1072
    def evals_result_(self) -> _EvalResultDict:
1073
        """:obj:`dict`: The evaluation results if validation sets have been specified."""
1074
        if not self.__sklearn_is_fitted__():
1075
1076
            raise LGBMNotFittedError('No results found. Need to call fit with eval_set beforehand.')
        return self._evals_result
1077
1078

    @property
1079
    def feature_importances_(self) -> np.ndarray:
1080
        """:obj:`array` of shape = [n_features]: The feature importances (the higher, the more important).
1081

Nikita Titov's avatar
Nikita Titov committed
1082
1083
1084
1085
        .. note::

            ``importance_type`` attribute is passed to the function
            to configure the type of importance values to be extracted.
1086
        """
1087
        if not self.__sklearn_is_fitted__():
1088
            raise LGBMNotFittedError('No feature_importances found. Need to call fit beforehand.')
1089
        return self._Booster.feature_importance(importance_type=self.importance_type)  # type: ignore[union-attr]
wxchan's avatar
wxchan committed
1090

1091
    @property
1092
1093
    def feature_name_(self) -> List[str]:
        """:obj:`list` of shape = [n_features]: The names of features."""
1094
        if not self.__sklearn_is_fitted__():
1095
            raise LGBMNotFittedError('No feature_name found. Need to call fit beforehand.')
1096
        return self._Booster.feature_name()  # type: ignore[union-attr]
1097

wxchan's avatar
wxchan committed
1098

1099
class LGBMRegressor(_LGBMRegressorBase, LGBMModel):
1100
    """LightGBM regressor."""
wxchan's avatar
wxchan committed
1101

1102
    def fit(  # type: ignore[override]
1103
        self,
1104
1105
        X: _LGBM_ScikitMatrixLike,
        y: _LGBM_LabelType,
1106
1107
1108
        sample_weight: Optional[_LGBM_WeightType] = None,
        init_score: Optional[_LGBM_InitScoreType] = None,
        eval_set: Optional[List[_LGBM_ScikitValidSet]] = None,
1109
        eval_names: Optional[List[str]] = None,
1110
1111
        eval_sample_weight: Optional[List[_LGBM_WeightType]] = None,
        eval_init_score: Optional[List[_LGBM_InitScoreType]] = None,
1112
        eval_metric: Optional[_LGBM_ScikitEvalMetricType] = None,
1113
1114
        feature_name: _LGBM_FeatureNameConfiguration = 'auto',
        categorical_feature: _LGBM_CategoricalFeatureConfiguration = 'auto',
1115
        callbacks: Optional[List[Callable]] = None,
1116
        init_model: Optional[Union[str, Path, Booster, LGBMModel]] = None
1117
    ) -> "LGBMRegressor":
1118
        """Docstring is inherited from the LGBMModel."""
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
        super().fit(
            X,
            y,
            sample_weight=sample_weight,
            init_score=init_score,
            eval_set=eval_set,
            eval_names=eval_names,
            eval_sample_weight=eval_sample_weight,
            eval_init_score=eval_init_score,
            eval_metric=eval_metric,
            feature_name=feature_name,
            categorical_feature=categorical_feature,
            callbacks=callbacks,
            init_model=init_model
        )
Guolin Ke's avatar
Guolin Ke committed
1134
1135
        return self

1136
    _base_doc = LGBMModel.fit.__doc__.replace("self : LGBMModel", "self : LGBMRegressor")  # type: ignore
1137
1138
    _base_doc = (_base_doc[:_base_doc.find('group :')]  # type: ignore
                 + _base_doc[_base_doc.find('eval_set :'):])  # type: ignore
1139
1140
1141
1142
    _base_doc = (_base_doc[:_base_doc.find('eval_class_weight :')]
                 + _base_doc[_base_doc.find('eval_init_score :'):])
    fit.__doc__ = (_base_doc[:_base_doc.find('eval_group :')]
                   + _base_doc[_base_doc.find('eval_metric :'):])
wxchan's avatar
wxchan committed
1143

1144

1145
class LGBMClassifier(_LGBMClassifierBase, LGBMModel):
1146
    """LightGBM classifier."""
wxchan's avatar
wxchan committed
1147

1148
    def fit(  # type: ignore[override]
1149
        self,
1150
1151
        X: _LGBM_ScikitMatrixLike,
        y: _LGBM_LabelType,
1152
1153
1154
        sample_weight: Optional[_LGBM_WeightType] = None,
        init_score: Optional[_LGBM_InitScoreType] = None,
        eval_set: Optional[List[_LGBM_ScikitValidSet]] = None,
1155
        eval_names: Optional[List[str]] = None,
1156
1157
1158
        eval_sample_weight: Optional[List[_LGBM_WeightType]] = None,
        eval_class_weight: Optional[List[float]] = None,
        eval_init_score: Optional[List[_LGBM_InitScoreType]] = None,
1159
        eval_metric: Optional[_LGBM_ScikitEvalMetricType] = None,
1160
1161
        feature_name: _LGBM_FeatureNameConfiguration = 'auto',
        categorical_feature: _LGBM_CategoricalFeatureConfiguration = 'auto',
1162
        callbacks: Optional[List[Callable]] = None,
1163
        init_model: Optional[Union[str, Path, Booster, LGBMModel]] = None
1164
    ) -> "LGBMClassifier":
1165
        """Docstring is inherited from the LGBMModel."""
1166
        _LGBMAssertAllFinite(y)
1167
1168
        _LGBMCheckClassificationTargets(y)
        self._le = _LGBMLabelEncoder().fit(y)
1169
        _y = self._le.transform(y)
1170
        self._class_map = dict(zip(self._le.classes_, self._le.transform(self._le.classes_)))
1171
1172
        if isinstance(self.class_weight, dict):
            self._class_weight = {self._class_map[k]: v for k, v in self.class_weight.items()}
1173

1174
        self._classes = self._le.classes_
1175
        self._n_classes = len(self._classes)  # type: ignore[arg-type]
1176
1177
        if self.objective is None:
            self._objective = None
1178

1179
1180
        # adjust eval metrics to match whether binary or multiclass
        # classification is being performed
1181
        if not callable(eval_metric):
1182
1183
1184
1185
1186
1187
            if isinstance(eval_metric, list):
                eval_metric_list = eval_metric
            elif isinstance(eval_metric, str):
                eval_metric_list = [eval_metric]
            else:
                eval_metric_list = []
1188
            if self._n_classes > 2:
1189
                for index, metric in enumerate(eval_metric_list):
1190
                    if metric in {'logloss', 'binary_logloss'}:
1191
                        eval_metric_list[index] = "multi_logloss"
1192
                    elif metric in {'error', 'binary_error'}:
1193
                        eval_metric_list[index] = "multi_error"
1194
            else:
1195
                for index, metric in enumerate(eval_metric_list):
1196
                    if metric in {'logloss', 'multi_logloss'}:
1197
                        eval_metric_list[index] = 'binary_logloss'
1198
                    elif metric in {'error', 'multi_error'}:
1199
1200
                        eval_metric_list[index] = 'binary_error'
            eval_metric = eval_metric_list
wxchan's avatar
wxchan committed
1201

1202
        # do not modify args, as it causes errors in model selection tools
1203
        valid_sets: Optional[List[_LGBM_ScikitValidSet]] = None
wxchan's avatar
wxchan committed
1204
        if eval_set is not None:
1205
1206
            if isinstance(eval_set, tuple):
                eval_set = [eval_set]
1207
1208
            valid_sets = []
            for valid_x, valid_y in eval_set:
1209
                if valid_x is X and valid_y is y:
1210
                    valid_sets.append((valid_x, _y))
1211
                else:
1212
                    valid_sets.append((valid_x, self._le.transform(valid_y)))
1213

1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
        super().fit(
            X,
            _y,
            sample_weight=sample_weight,
            init_score=init_score,
            eval_set=valid_sets,
            eval_names=eval_names,
            eval_sample_weight=eval_sample_weight,
            eval_class_weight=eval_class_weight,
            eval_init_score=eval_init_score,
            eval_metric=eval_metric,
            feature_name=feature_name,
            categorical_feature=categorical_feature,
            callbacks=callbacks,
            init_model=init_model
        )
wxchan's avatar
wxchan committed
1230
1231
        return self

1232
    _base_doc = LGBMModel.fit.__doc__.replace("self : LGBMModel", "self : LGBMClassifier")  # type: ignore
1233
1234
    _base_doc = (_base_doc[:_base_doc.find('group :')]  # type: ignore
                 + _base_doc[_base_doc.find('eval_set :'):])  # type: ignore
1235
1236
    fit.__doc__ = (_base_doc[:_base_doc.find('eval_group :')]
                   + _base_doc[_base_doc.find('eval_metric :'):])
1237

1238
1239
    def predict(
        self,
1240
        X: _LGBM_ScikitMatrixLike,
1241
1242
1243
1244
1245
1246
1247
1248
        raw_score: bool = False,
        start_iteration: int = 0,
        num_iteration: Optional[int] = None,
        pred_leaf: bool = False,
        pred_contrib: bool = False,
        validate_features: bool = False,
        **kwargs: Any
    ):
1249
        """Docstring is inherited from the LGBMModel."""
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
        result = self.predict_proba(
            X=X,
            raw_score=raw_score,
            start_iteration=start_iteration,
            num_iteration=num_iteration,
            pred_leaf=pred_leaf,
            pred_contrib=pred_contrib,
            validate_features=validate_features,
            **kwargs
        )
1260
        if callable(self._objective) or raw_score or pred_leaf or pred_contrib:
1261
1262
1263
1264
            return result
        else:
            class_index = np.argmax(result, axis=1)
            return self._le.inverse_transform(class_index)
wxchan's avatar
wxchan committed
1265

1266
1267
    predict.__doc__ = LGBMModel.predict.__doc__

1268
1269
    def predict_proba(
        self,
1270
        X: _LGBM_ScikitMatrixLike,
1271
1272
1273
1274
1275
1276
1277
1278
        raw_score: bool = False,
        start_iteration: int = 0,
        num_iteration: Optional[int] = None,
        pred_leaf: bool = False,
        pred_contrib: bool = False,
        validate_features: bool = False,
        **kwargs: Any
    ):
1279
        """Docstring is set after definition, using a template."""
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
        result = super().predict(
            X=X,
            raw_score=raw_score,
            start_iteration=start_iteration,
            num_iteration=num_iteration,
            pred_leaf=pred_leaf,
            pred_contrib=pred_contrib,
            validate_features=validate_features,
            **kwargs
        )
1290
        if callable(self._objective) and not (raw_score or pred_leaf or pred_contrib):
1291
1292
1293
            _log_warning("Cannot compute class probabilities or labels "
                         "due to the usage of customized objective function.\n"
                         "Returning raw scores instead.")
1294
            return result
1295
        elif self._n_classes > 2 or raw_score or pred_leaf or pred_contrib:  # type: ignore [operator]
1296
            return result
wxchan's avatar
wxchan committed
1297
        else:
1298
            return np.vstack((1. - result, result)).transpose()
1299

1300
1301
    predict_proba.__doc__ = _lgbmmodel_doc_predict.format(
        description="Return the predicted probability for each class for each sample.",
1302
        X_shape="numpy array, pandas DataFrame, H2O DataTable's Frame , scipy.sparse, list of lists of int or float of shape = [n_samples, n_features]",
1303
        output_name="predicted_probability",
1304
        predicted_result_shape="array-like of shape = [n_samples] or shape = [n_samples, n_classes]",
1305
1306
1307
1308
        X_leaves_shape="array-like of shape = [n_samples, n_trees] or shape = [n_samples, n_trees * n_classes]",
        X_SHAP_values_shape="array-like of shape = [n_samples, n_features + 1] or shape = [n_samples, (n_features + 1) * n_classes] or list with n_classes length of such objects"
    )

1309
    @property
1310
    def classes_(self) -> np.ndarray:
1311
        """:obj:`array` of shape = [n_classes]: The class label array."""
1312
        if not self.__sklearn_is_fitted__():
1313
            raise LGBMNotFittedError('No classes found. Need to call fit beforehand.')
1314
        return self._classes  # type: ignore[return-value]
1315
1316

    @property
1317
    def n_classes_(self) -> int:
1318
        """:obj:`int`: The number of classes."""
1319
        if not self.__sklearn_is_fitted__():
1320
1321
            raise LGBMNotFittedError('No classes found. Need to call fit beforehand.')
        return self._n_classes
wxchan's avatar
wxchan committed
1322

wxchan's avatar
wxchan committed
1323

wxchan's avatar
wxchan committed
1324
class LGBMRanker(LGBMModel):
1325
1326
1327
1328
1329
1330
1331
1332
    """LightGBM ranker.

    .. warning::

        scikit-learn doesn't support ranking applications yet,
        therefore this class is not really compatible with the sklearn ecosystem.
        Please use this class mainly for training and applying ranking models in common sklearnish way.
    """
wxchan's avatar
wxchan committed
1333

1334
    def fit(  # type: ignore[override]
1335
        self,
1336
1337
        X: _LGBM_ScikitMatrixLike,
        y: _LGBM_LabelType,
1338
1339
        sample_weight: Optional[_LGBM_WeightType] = None,
        init_score: Optional[_LGBM_InitScoreType] = None,
1340
        group: Optional[_LGBM_GroupType] = None,
1341
        eval_set: Optional[List[_LGBM_ScikitValidSet]] = None,
1342
        eval_names: Optional[List[str]] = None,
1343
1344
1345
        eval_sample_weight: Optional[List[_LGBM_WeightType]] = None,
        eval_init_score: Optional[List[_LGBM_InitScoreType]] = None,
        eval_group: Optional[List[_LGBM_GroupType]] = None,
1346
        eval_metric: Optional[_LGBM_ScikitEvalMetricType] = None,
1347
        eval_at: Union[List[int], Tuple[int, ...]] = (1, 2, 3, 4, 5),
1348
1349
        feature_name: _LGBM_FeatureNameConfiguration = 'auto',
        categorical_feature: _LGBM_CategoricalFeatureConfiguration = 'auto',
1350
        callbacks: Optional[List[Callable]] = None,
1351
        init_model: Optional[Union[str, Path, Booster, LGBMModel]] = None
1352
    ) -> "LGBMRanker":
1353
        """Docstring is inherited from the LGBMModel."""
1354
        # check group data
Guolin Ke's avatar
Guolin Ke committed
1355
        if group is None:
1356
            raise ValueError("Should set group for ranking task")
wxchan's avatar
wxchan committed
1357
1358

        if eval_set is not None:
Guolin Ke's avatar
Guolin Ke committed
1359
            if eval_group is None:
1360
                raise ValueError("Eval_group cannot be None when eval_set is not None")
Guolin Ke's avatar
Guolin Ke committed
1361
            elif len(eval_group) != len(eval_set):
1362
                raise ValueError("Length of eval_group should be equal to eval_set")
1363
            elif (isinstance(eval_group, dict)
1364
                  and any(i not in eval_group or eval_group[i] is None for i in range(len(eval_group)))
1365
1366
                  or isinstance(eval_group, list)
                  and any(group is None for group in eval_group)):
1367
1368
                raise ValueError("Should set group for all eval datasets for ranking task; "
                                 "if you use dict, the index should start from 0")
1369

1370
        self._eval_at = eval_at
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
        super().fit(
            X,
            y,
            sample_weight=sample_weight,
            init_score=init_score,
            group=group,
            eval_set=eval_set,
            eval_names=eval_names,
            eval_sample_weight=eval_sample_weight,
            eval_init_score=eval_init_score,
            eval_group=eval_group,
            eval_metric=eval_metric,
            feature_name=feature_name,
            categorical_feature=categorical_feature,
            callbacks=callbacks,
            init_model=init_model
        )
wxchan's avatar
wxchan committed
1388
        return self
1389

1390
    _base_doc = LGBMModel.fit.__doc__.replace("self : LGBMModel", "self : LGBMRanker")  # type: ignore
1391
1392
    fit.__doc__ = (_base_doc[:_base_doc.find('eval_class_weight :')]  # type: ignore
                   + _base_doc[_base_doc.find('eval_init_score :'):])  # type: ignore
1393
    _base_doc = fit.__doc__
1394
    _before_feature_name, _feature_name, _after_feature_name = _base_doc.partition('feature_name :')
1395
    fit.__doc__ = f"""{_before_feature_name}eval_at : list or tuple of int, optional (default=(1, 2, 3, 4, 5))
1396
        The evaluation positions of the specified metric.
1397
    {_feature_name}{_after_feature_name}"""