sklearn.py 52.7 KB
Newer Older
wxchan's avatar
wxchan committed
1
# coding: utf-8
2
"""Scikit-learn wrapper interface for LightGBM."""
3
import copy
4
from inspect import signature
5
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
6

wxchan's avatar
wxchan committed
7
import numpy as np
8

9
from .basic import Booster, Dataset, LightGBMError, _ArrayLike, _choose_param_value, _ConfigAliases, _log_warning
10
from .callback import record_evaluation
11
12
13
from .compat import (SKLEARN_INSTALLED, LGBMNotFittedError, _LGBMAssertAllFinite, _LGBMCheckArray,
                     _LGBMCheckClassificationTargets, _LGBMCheckSampleWeight, _LGBMCheckXY, _LGBMClassifierBase,
                     _LGBMComputeSampleWeight, _LGBMLabelEncoder, _LGBMModelBase, _LGBMRegressorBase, dt_DataTable,
14
                     pd_DataFrame)
wxchan's avatar
wxchan committed
15
from .engine import train
16

17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
_EvalResultType = Tuple[str, float, bool]

_LGBM_ScikitCustomObjectiveFunction = Union[
    Callable[
        [np.ndarray, np.ndarray],
        Tuple[_ArrayLike, _ArrayLike]
    ],
    Callable[
        [np.ndarray, np.ndarray, np.ndarray],
        Tuple[_ArrayLike, _ArrayLike]
    ],
]
_LGBM_ScikitCustomEvalFunction = Union[
    Callable[
        [np.ndarray, np.ndarray],
        Union[_EvalResultType, List[_EvalResultType]]
    ],
    Callable[
        [np.ndarray, np.ndarray, np.ndarray],
        Union[_EvalResultType, List[_EvalResultType]]
    ],
    Callable[
        [np.ndarray, np.ndarray, np.ndarray, np.ndarray],
        Union[_EvalResultType, List[_EvalResultType]]
    ],
]

wxchan's avatar
wxchan committed
44

45
class _ObjectiveFunctionWrapper:
46
    """Proxy class for objective function."""
47

48
    def __init__(self, func: _LGBM_ScikitCustomObjectiveFunction):
49
        """Construct a proxy class.
50

51
52
        This class transforms objective function to match objective function with signature ``new_func(preds, dataset)``
        as expected by ``lightgbm.engine.train``.
53

54
55
56
        Parameters
        ----------
        func : callable
57
            Expects a callable with signature ``func(y_true, y_pred)`` or ``func(y_true, y_pred, group)``
58
59
            and returns (grad, hess):

60
                y_true : numpy 1-D array of shape = [n_samples]
61
                    The target values.
62
                y_pred : numpy 1-D array of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
63
                    The predicted values.
64
65
                    Predicted values are returned before any transformation,
                    e.g. they are raw margin instead of probability of positive class for binary task.
66
                group : numpy 1-D array
67
68
69
                    Group/query data.
                    Only used in the learning-to-rank task.
                    sum(group) = n_samples.
70
71
                    For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
                    where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
72
                grad : list, numpy 1-D array or pandas Series of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
73
74
                    The value of the first order derivative (gradient) of the loss
                    with respect to the elements of y_pred for each sample point.
75
                hess : list, numpy 1-D array or pandas Series of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
76
77
                    The value of the second order derivative (Hessian) of the loss
                    with respect to the elements of y_pred for each sample point.
wxchan's avatar
wxchan committed
78

Nikita Titov's avatar
Nikita Titov committed
79
80
81
82
83
        .. note::

            For multi-class task, the y_pred is group by class_id first, then group by row_id.
            If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i]
            and you should group grad and hess in this way as well.
84
85
        """
        self.func = func
wxchan's avatar
wxchan committed
86

87
88
89
90
91
    def __call__(self, preds, dataset):
        """Call passed function with appropriate arguments.

        Parameters
        ----------
92
        preds : numpy 1-D array of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
93
94
95
96
97
98
            The predicted values.
        dataset : Dataset
            The training dataset.

        Returns
        -------
99
        grad : list, numpy 1-D array or pandas Series of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
100
101
            The value of the first order derivative (gradient) of the loss
            with respect to the elements of preds for each sample point.
102
        hess : list, numpy 1-D array or pandas Series of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
103
104
            The value of the second order derivative (Hessian) of the loss
            with respect to the elements of preds for each sample point.
105
        """
wxchan's avatar
wxchan committed
106
        labels = dataset.get_label()
107
        argc = len(signature(self.func).parameters)
108
        if argc == 2:
109
            grad, hess = self.func(labels, preds)
110
        elif argc == 3:
111
            grad, hess = self.func(labels, preds, dataset.get_group())
112
        else:
113
            raise TypeError(f"Self-defined objective function should have 2 or 3 arguments, got {argc}")
wxchan's avatar
wxchan committed
114
115
116
117
118
119
120
121
122
123
124
        """weighted for objective"""
        weight = dataset.get_weight()
        if weight is not None:
            """only one class"""
            if len(weight) == len(grad):
                grad = np.multiply(grad, weight)
                hess = np.multiply(hess, weight)
            else:
                num_data = len(weight)
                num_class = len(grad) // num_data
                if num_class * num_data != len(grad):
125
                    raise ValueError("Length of grad and hess should equal to num_class * num_data")
126
127
                for k in range(num_class):
                    for i in range(num_data):
wxchan's avatar
wxchan committed
128
129
130
131
132
                        idx = k * num_data + i
                        grad[idx] *= weight[i]
                        hess[idx] *= weight[i]
        return grad, hess

wxchan's avatar
wxchan committed
133

134
class _EvalFunctionWrapper:
135
    """Proxy class for evaluation function."""
136

137
    def __init__(self, func: _LGBM_ScikitCustomEvalFunction):
138
        """Construct a proxy class.
139

140
141
        This class transforms evaluation function to match evaluation function with signature ``new_func(preds, dataset)``
        as expected by ``lightgbm.engine.train``.
142

143
144
145
146
147
148
149
150
151
152
        Parameters
        ----------
        func : callable
            Expects a callable with following signatures:
            ``func(y_true, y_pred)``,
            ``func(y_true, y_pred, weight)``
            or ``func(y_true, y_pred, weight, group)``
            and returns (eval_name, eval_result, is_higher_better) or
            list of (eval_name, eval_result, is_higher_better):

153
                y_true : numpy 1-D array of shape = [n_samples]
154
                    The target values.
155
                y_pred : numpy 1-D array of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
156
                    The predicted values.
157
158
                    In case of custom ``objective``, predicted values are returned before any transformation,
                    e.g. they are raw margin instead of probability of positive class for binary task in this case.
159
                weight : numpy 1-D array of shape = [n_samples]
160
                    The weight of samples.
161
                group : numpy 1-D array
162
163
164
                    Group/query data.
                    Only used in the learning-to-rank task.
                    sum(group) = n_samples.
165
166
                    For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
                    where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
167
                eval_name : str
Andrew Ziem's avatar
Andrew Ziem committed
168
                    The name of evaluation function (without whitespace).
169
170
171
172
173
                eval_result : float
                    The eval result.
                is_higher_better : bool
                    Is eval result higher better, e.g. AUC is ``is_higher_better``.

Nikita Titov's avatar
Nikita Titov committed
174
175
176
177
        .. note::

            For multi-class task, the y_pred is group by class_id first, then group by row_id.
            If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i].
178
179
        """
        self.func = func
180

181
182
    def __call__(self, preds, dataset):
        """Call passed function with appropriate arguments.
183

184
185
        Parameters
        ----------
186
        preds : numpy 1-D array of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
187
188
189
190
191
192
            The predicted values.
        dataset : Dataset
            The training dataset.

        Returns
        -------
193
        eval_name : str
Andrew Ziem's avatar
Andrew Ziem committed
194
            The name of evaluation function (without whitespace).
195
196
197
198
199
        eval_result : float
            The eval result.
        is_higher_better : bool
            Is eval result higher better, e.g. AUC is ``is_higher_better``.
        """
200
        labels = dataset.get_label()
201
        argc = len(signature(self.func).parameters)
202
        if argc == 2:
203
            return self.func(labels, preds)
204
        elif argc == 3:
205
            return self.func(labels, preds, dataset.get_weight())
206
        elif argc == 4:
207
            return self.func(labels, preds, dataset.get_weight(), dataset.get_group())
208
        else:
209
            raise TypeError(f"Self-defined eval function should have 2, 3 or 4 arguments, got {argc}")
210

wxchan's avatar
wxchan committed
211

212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
# documentation templates for LGBMModel methods are shared between the classes in
# this module and those in the ``dask`` module

_lgbmmodel_doc_fit = (
    """
    Build a gradient boosting model from the training set (X, y).

    Parameters
    ----------
    X : {X_shape}
        Input feature matrix.
    y : {y_shape}
        The target values (class labels in classification, real numbers in regression).
    sample_weight : {sample_weight_shape}
        Weights of training data.
227
    init_score : {init_score_shape}
228
229
230
231
232
233
234
235
236
        Init score of training data.
    group : {group_shape}
        Group/query data.
        Only used in the learning-to-rank task.
        sum(group) = n_samples.
        For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
        where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
    eval_set : list or None, optional (default=None)
        A list of (X, y) tuple pairs to use as validation sets.
237
    eval_names : list of str, or None, optional (default=None)
238
        Names of eval_set.
239
    eval_sample_weight : {eval_sample_weight_shape}
240
241
242
        Weights of eval data.
    eval_class_weight : list or None, optional (default=None)
        Class weights of eval data.
243
    eval_init_score : {eval_init_score_shape}
244
        Init score of eval data.
245
    eval_group : {eval_group_shape}
246
        Group data of eval data.
247
248
    eval_metric : str, callable, list or None, optional (default=None)
        If str, it should be a built-in evaluation metric to use.
249
250
251
252
        If callable, it should be a custom evaluation metric, see note below for more details.
        If list, it can be a list of built-in metrics, a list of custom evaluation metrics, or a mix of both.
        In either case, the ``metric`` from the model parameters will be evaluated and used as well.
        Default: 'l2' for LGBMRegressor, 'logloss' for LGBMClassifier, 'ndcg' for LGBMRanker.
253
    feature_name : list of str, or 'auto', optional (default='auto')
254
255
        Feature names.
        If 'auto' and data is pandas DataFrame, data columns names are used.
256
    categorical_feature : list of str or int, or 'auto', optional (default='auto')
257
258
        Categorical features.
        If list of int, interpreted as indices.
259
        If list of str, interpreted as feature names (need to specify ``feature_name`` as well).
260
261
262
263
264
        If 'auto' and data is pandas DataFrame, pandas unordered categorical columns are used.
        All values in categorical features should be less than int32 max value (2147483647).
        Large values could be memory consuming. Consider using consecutive integers starting from zero.
        All negative values in categorical features will be treated as missing values.
        The output cannot be monotonically constrained with respect to a categorical feature.
265
    callbacks : list of callable, or None, optional (default=None)
266
267
        List of callback functions that are applied at each iteration.
        See Callbacks in Python API for more information.
268
    init_model : str, pathlib.Path, Booster, LGBMModel or None, optional (default=None)
269
270
271
272
        Filename of LightGBM model, Booster instance or LGBMModel instance used for continue training.

    Returns
    -------
273
    self : LGBMModel
274
275
276
277
278
279
280
281
282
283
284
285
286
        Returns self.
    """
)

_lgbmmodel_doc_custom_eval_note = """
    Note
    ----
    Custom eval function expects a callable with following signatures:
    ``func(y_true, y_pred)``, ``func(y_true, y_pred, weight)`` or
    ``func(y_true, y_pred, weight, group)``
    and returns (eval_name, eval_result, is_higher_better) or
    list of (eval_name, eval_result, is_higher_better):

287
        y_true : numpy 1-D array of shape = [n_samples]
288
            The target values.
289
        y_pred : numpy 1-D array of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
290
            The predicted values.
291
292
            In case of custom ``objective``, predicted values are returned before any transformation,
            e.g. they are raw margin instead of probability of positive class for binary task in this case.
293
        weight : numpy 1-D array of shape = [n_samples]
294
            The weight of samples.
295
        group : numpy 1-D array
296
297
298
299
300
            Group/query data.
            Only used in the learning-to-rank task.
            sum(group) = n_samples.
            For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
            where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
301
        eval_name : str
Andrew Ziem's avatar
Andrew Ziem committed
302
            The name of evaluation function (without whitespace).
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
        eval_result : float
            The eval result.
        is_higher_better : bool
            Is eval result higher better, e.g. AUC is ``is_higher_better``.

    For multi-class task, the y_pred is group by class_id first, then group by row_id.
    If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i].
"""

_lgbmmodel_doc_predict = (
    """
    {description}

    Parameters
    ----------
    X : {X_shape}
        Input features matrix.
    raw_score : bool, optional (default=False)
        Whether to predict raw scores.
    start_iteration : int, optional (default=0)
        Start index of the iteration to predict.
        If <= 0, starts from the first iteration.
    num_iteration : int or None, optional (default=None)
        Total number of iterations used in the prediction.
        If None, if the best iteration exists and start_iteration <= 0, the best iteration is used;
        otherwise, all iterations from ``start_iteration`` are used (no limits).
        If <= 0, all iterations from ``start_iteration`` are used (no limits).
    pred_leaf : bool, optional (default=False)
        Whether to predict leaf index.
    pred_contrib : bool, optional (default=False)
        Whether to predict feature contributions.

        .. note::

            If you want to get more explanations for your model's predictions using SHAP values,
            like SHAP interaction values,
            you can install the shap package (https://github.com/slundberg/shap).
            Note that unlike the shap package, with ``pred_contrib`` we return a matrix with an extra
            column, where the last column is the expected value.

    **kwargs
        Other parameters for the prediction.

    Returns
    -------
    {output_name} : {predicted_result_shape}
        The predicted values.
    X_leaves : {X_leaves_shape}
        If ``pred_leaf=True``, the predicted leaf of every tree for each sample.
    X_SHAP_values : {X_SHAP_values_shape}
        If ``pred_contrib=True``, the feature contributions for each sample.
    """
)


358
359
class LGBMModel(_LGBMModelBase):
    """Implementation of the scikit-learn API for LightGBM."""
wxchan's avatar
wxchan committed
360

361
362
363
364
365
366
367
368
    def __init__(
        self,
        boosting_type: str = 'gbdt',
        num_leaves: int = 31,
        max_depth: int = -1,
        learning_rate: float = 0.1,
        n_estimators: int = 100,
        subsample_for_bin: int = 200000,
369
        objective: Optional[Union[str, _LGBM_ScikitCustomObjectiveFunction]] = None,
370
371
372
373
374
375
376
377
378
379
380
381
382
383
        class_weight: Optional[Union[Dict, str]] = None,
        min_split_gain: float = 0.,
        min_child_weight: float = 1e-3,
        min_child_samples: int = 20,
        subsample: float = 1.,
        subsample_freq: int = 0,
        colsample_bytree: float = 1.,
        reg_alpha: float = 0.,
        reg_lambda: float = 0.,
        random_state: Optional[Union[int, np.random.RandomState]] = None,
        n_jobs: int = -1,
        importance_type: str = 'split',
        **kwargs
    ):
384
        r"""Construct a gradient boosting model.
wxchan's avatar
wxchan committed
385
386
387

        Parameters
        ----------
388
        boosting_type : str, optional (default='gbdt')
389
390
391
392
393
            'gbdt', traditional Gradient Boosting Decision Tree.
            'dart', Dropouts meet Multiple Additive Regression Trees.
            'goss', Gradient-based One-Side Sampling.
            'rf', Random Forest.
        num_leaves : int, optional (default=31)
wxchan's avatar
wxchan committed
394
            Maximum tree leaves for base learners.
395
        max_depth : int, optional (default=-1)
396
            Maximum tree depth for base learners, <=0 means no limit.
397
        learning_rate : float, optional (default=0.1)
398
            Boosting learning rate.
399
400
401
            You can use ``callbacks`` parameter of ``fit`` method to shrink/adapt learning rate
            in training using ``reset_parameter`` callback.
            Note, that this will ignore the ``learning_rate`` argument in training.
402
        n_estimators : int, optional (default=100)
wxchan's avatar
wxchan committed
403
            Number of boosted trees to fit.
404
        subsample_for_bin : int, optional (default=200000)
wxchan's avatar
wxchan committed
405
            Number of samples for constructing bins.
406
        objective : str, callable or None, optional (default=None)
wxchan's avatar
wxchan committed
407
408
            Specify the learning task and the corresponding learning objective or
            a custom objective function to be used (see note below).
409
            Default: 'regression' for LGBMRegressor, 'binary' or 'multiclass' for LGBMClassifier, 'lambdarank' for LGBMRanker.
410
411
412
413
        class_weight : dict, 'balanced' or None, optional (default=None)
            Weights associated with classes in the form ``{class_label: weight}``.
            Use this parameter only for multi-class classification task;
            for binary classification task you may use ``is_unbalance`` or ``scale_pos_weight`` parameters.
414
415
416
            Note, that the usage of all these parameters will result in poor estimates of the individual class probabilities.
            You may want to consider performing probability calibration
            (https://scikit-learn.org/stable/modules/calibration.html) of your model.
417
418
419
            The 'balanced' mode uses the values of y to automatically adjust weights
            inversely proportional to class frequencies in the input data as ``n_samples / (n_classes * np.bincount(y))``.
            If None, all classes are supposed to have weight one.
420
            Note, that these weights will be multiplied with ``sample_weight`` (passed through the ``fit`` method)
421
            if ``sample_weight`` is specified.
422
        min_split_gain : float, optional (default=0.)
wxchan's avatar
wxchan committed
423
            Minimum loss reduction required to make a further partition on a leaf node of the tree.
424
        min_child_weight : float, optional (default=1e-3)
425
            Minimum sum of instance weight (hessian) needed in a child (leaf).
426
        min_child_samples : int, optional (default=20)
427
            Minimum number of data needed in a child (leaf).
428
        subsample : float, optional (default=1.)
wxchan's avatar
wxchan committed
429
            Subsample ratio of the training instance.
430
        subsample_freq : int, optional (default=0)
Andrew Ziem's avatar
Andrew Ziem committed
431
            Frequency of subsample, <=0 means no enable.
432
        colsample_bytree : float, optional (default=1.)
wxchan's avatar
wxchan committed
433
            Subsample ratio of columns when constructing each tree.
434
        reg_alpha : float, optional (default=0.)
435
            L1 regularization term on weights.
436
        reg_lambda : float, optional (default=0.)
437
            L2 regularization term on weights.
438
        random_state : int, RandomState object or None, optional (default=None)
wxchan's avatar
wxchan committed
439
            Random number seed.
440
441
442
            If int, this number is used to seed the C++ code.
            If RandomState object (numpy), a random integer is picked based on its state to seed the C++ code.
            If None, default seeds in C++ code are used.
443
        n_jobs : int, optional (default=-1)
444
            Number of parallel threads to use for training (can be changed at prediction time).
445
        importance_type : str, optional (default='split')
446
            The type of feature importance to be filled into ``feature_importances_``.
447
448
449
450
            If 'split', result contains numbers of times the feature is used in a model.
            If 'gain', result contains total gains of splits which use the feature.
        **kwargs
            Other parameters for the model.
wxchan's avatar
wxchan committed
451
            Check http://lightgbm.readthedocs.io/en/latest/Parameters.html for more parameters.
452

Nikita Titov's avatar
Nikita Titov committed
453
454
455
            .. warning::

                \*\*kwargs is not supported in sklearn, it may cause unexpected issues.
wxchan's avatar
wxchan committed
456
457
458

        Note
        ----
459
460
        A custom objective function can be provided for the ``objective`` parameter.
        In this case, it should have the signature
461
462
        ``objective(y_true, y_pred) -> grad, hess`` or
        ``objective(y_true, y_pred, group) -> grad, hess``:
wxchan's avatar
wxchan committed
463

464
            y_true : numpy 1-D array of shape = [n_samples]
465
                The target values.
466
            y_pred : numpy 1-D array of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
467
                The predicted values.
468
469
                Predicted values are returned before any transformation,
                e.g. they are raw margin instead of probability of positive class for binary task.
470
            group : numpy 1-D array
471
472
473
                Group/query data.
                Only used in the learning-to-rank task.
                sum(group) = n_samples.
474
475
                For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
                where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
476
            grad : list, numpy 1-D array or pandas Series of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
477
478
                The value of the first order derivative (gradient) of the loss
                with respect to the elements of y_pred for each sample point.
479
            hess : list, numpy 1-D array or pandas Series of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
480
481
                The value of the second order derivative (Hessian) of the loss
                with respect to the elements of y_pred for each sample point.
wxchan's avatar
wxchan committed
482

483
484
485
        For multi-class task, the y_pred is group by class_id first, then group by row_id.
        If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i]
        and you should group grad and hess in this way as well.
wxchan's avatar
wxchan committed
486
        """
wxchan's avatar
wxchan committed
487
        if not SKLEARN_INSTALLED:
488
489
            raise LightGBMError('scikit-learn is required for lightgbm.sklearn. '
                                'You must install scikit-learn and restart your session to use this module.')
wxchan's avatar
wxchan committed
490

491
        self.boosting_type = boosting_type
492
        self.objective = objective
wxchan's avatar
wxchan committed
493
494
495
496
        self.num_leaves = num_leaves
        self.max_depth = max_depth
        self.learning_rate = learning_rate
        self.n_estimators = n_estimators
wxchan's avatar
wxchan committed
497
        self.subsample_for_bin = subsample_for_bin
wxchan's avatar
wxchan committed
498
499
500
501
502
503
504
505
        self.min_split_gain = min_split_gain
        self.min_child_weight = min_child_weight
        self.min_child_samples = min_child_samples
        self.subsample = subsample
        self.subsample_freq = subsample_freq
        self.colsample_bytree = colsample_bytree
        self.reg_alpha = reg_alpha
        self.reg_lambda = reg_lambda
506
507
        self.random_state = random_state
        self.n_jobs = n_jobs
508
        self.importance_type = importance_type
509
        self._Booster: Optional[Booster] = None
510
511
512
        self._evals_result = None
        self._best_score = None
        self._best_iteration = None
513
        self._other_params: Dict[str, Any] = {}
514
        self._objective = objective
515
        self.class_weight = class_weight
516
517
        self._class_weight = None
        self._class_map = None
518
        self._n_features = None
519
        self._n_features_in = None
520
521
        self._classes = None
        self._n_classes = None
522
        self.set_params(**kwargs)
wxchan's avatar
wxchan committed
523

Nikita Titov's avatar
Nikita Titov committed
524
    def _more_tags(self):
525
526
527
528
529
530
531
532
533
534
        return {
            'allow_nan': True,
            'X_types': ['2darray', 'sparse', '1dlabels'],
            '_xfail_checks': {
                'check_no_attributes_set_in_init':
                'scikit-learn incorrectly asserts that private attributes '
                'cannot be set in __init__: '
                '(see https://github.com/microsoft/LightGBM/issues/2628)'
            }
        }
Nikita Titov's avatar
Nikita Titov committed
535

536
537
538
    def __sklearn_is_fitted__(self) -> bool:
        return getattr(self, "fitted_", False)

wxchan's avatar
wxchan committed
539
    def get_params(self, deep=True):
540
541
542
543
544
545
546
547
548
549
550
551
552
        """Get parameters for this estimator.

        Parameters
        ----------
        deep : bool, optional (default=True)
            If True, will return the parameters for this estimator and
            contained subobjects that are estimators.

        Returns
        -------
        params : dict
            Parameter names mapped to their values.
        """
553
        params = super().get_params(deep=deep)
554
        params.update(self._other_params)
wxchan's avatar
wxchan committed
555
556
557
        return params

    def set_params(self, **params):
558
559
560
561
562
563
564
565
566
567
568
569
        """Set the parameters of this estimator.

        Parameters
        ----------
        **params
            Parameter names with their new values.

        Returns
        -------
        self : object
            Returns self.
        """
wxchan's avatar
wxchan committed
570
571
        for key, value in params.items():
            setattr(self, key, value)
572
573
            if hasattr(self, f"_{key}"):
                setattr(self, f"_{key}", value)
574
            self._other_params[key] = value
wxchan's avatar
wxchan committed
575
        return self
wxchan's avatar
wxchan committed
576

577
578
579
580
581
582
583
584
585
586
587
588
589
590
    def _process_params(self, stage: str) -> Dict[str, Any]:
        """Process the parameters of this estimator based on its type, parameter aliases, etc.

        Parameters
        ----------
        stage : str
            Name of the stage (can be ``fit`` or ``predict``) this method is called from.

        Returns
        -------
        processed_params : dict
            Processed parameter names mapped to their values.
        """
        assert stage in {"fit", "predict"}
591
592
593
594
595
        params = self.get_params()

        params.pop('objective', None)
        for alias in _ConfigAliases.get('objective'):
            if alias in params:
596
                obj = params.pop(alias)
597
                _log_warning(f"Found '{alias}' in params. Will use it instead of 'objective' argument")
598
599
600
601
602
603
604
605
606
607
608
609
610
                if stage == "fit":
                    self._objective = obj
        if stage == "fit":
            if self._objective is None:
                if isinstance(self, LGBMRegressor):
                    self._objective = "regression"
                elif isinstance(self, LGBMClassifier):
                    if self._n_classes > 2:
                        self._objective = "multiclass"
                    else:
                        self._objective = "binary"
                elif isinstance(self, LGBMRanker):
                    self._objective = "lambdarank"
611
                else:
612
                    raise ValueError("Unknown LGBMModel type.")
613
        if callable(self._objective):
614
615
            if stage == "fit":
                self._fobj = _ObjectiveFunctionWrapper(self._objective)
616
            params['objective'] = 'None'  # objective = nullptr for unknown objective
617
        else:
618
619
            if stage == "fit":
                self._fobj = None
620
            params['objective'] = self._objective
621

622
        params.pop('importance_type', None)
wxchan's avatar
wxchan committed
623
        params.pop('n_estimators', None)
624
        params.pop('class_weight', None)
625

626
627
        if isinstance(params['random_state'], np.random.RandomState):
            params['random_state'] = params['random_state'].randint(np.iinfo(np.int32).max)
628
        if self._n_classes is not None and self._n_classes > 2:
629
630
            for alias in _ConfigAliases.get('num_class'):
                params.pop(alias, None)
631
632
            params['num_class'] = self._n_classes
        if hasattr(self, '_eval_at'):
633
            eval_at = self._eval_at
634
            for alias in _ConfigAliases.get('eval_at'):
635
636
637
638
                if alias in params:
                    _log_warning(f"Found '{alias}' in params. Will use it instead of 'eval_at' argument")
                    eval_at = params.pop(alias)
            params['eval_at'] = eval_at
wxchan's avatar
wxchan committed
639

640
        # register default metric for consistency with callable eval_metric case
641
        original_metric = self._objective if isinstance(self._objective, str) else None
642
643
644
645
646
647
648
649
650
651
        if original_metric is None:
            # try to deduce from class instance
            if isinstance(self, LGBMRegressor):
                original_metric = "l2"
            elif isinstance(self, LGBMClassifier):
                original_metric = "multi_logloss" if self._n_classes > 2 else "binary_logloss"
            elif isinstance(self, LGBMRanker):
                original_metric = "ndcg"

        # overwrite default metric by explicitly set metric
652
        params = _choose_param_value("metric", params, original_metric)
653

654
655
        return params

656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
    def fit(
        self,
        X,
        y,
        sample_weight=None,
        init_score=None,
        group=None,
        eval_set=None,
        eval_names=None,
        eval_sample_weight=None,
        eval_class_weight=None,
        eval_init_score=None,
        eval_group=None,
        eval_metric=None,
        feature_name='auto',
        categorical_feature='auto',
        callbacks=None,
        init_model=None
    ):
675
676
677
678
679
680
681
682
683
684
685
686
687
        """Docstring is set after definition, using a template."""
        params = self._process_params(stage="fit")

        # Do not modify original args in fit function
        # Refer to https://github.com/microsoft/LightGBM/pull/2619
        eval_metric_list = copy.deepcopy(eval_metric)
        if not isinstance(eval_metric_list, list):
            eval_metric_list = [eval_metric_list]

        # Separate built-in from callable evaluation metrics
        eval_metrics_callable = [_EvalFunctionWrapper(f) for f in eval_metric_list if callable(f)]
        eval_metrics_builtin = [m for m in eval_metric_list if isinstance(m, str)]

688
        # concatenate metric from params (or default if not provided in params) and eval_metric
689
690
        params['metric'] = [params['metric']] if isinstance(params['metric'], (str, type(None))) else params['metric']
        params['metric'] = [e for e in eval_metrics_builtin if e not in params['metric']] + params['metric']
691
        params['metric'] = [metric for metric in params['metric'] if metric is not None]
wxchan's avatar
wxchan committed
692

693
        if not isinstance(X, (pd_DataFrame, dt_DataTable)):
694
            _X, _y = _LGBMCheckXY(X, y, accept_sparse=True, force_all_finite=False, ensure_min_samples=2)
695
696
            if sample_weight is not None:
                sample_weight = _LGBMCheckSampleWeight(sample_weight, _X)
697
698
        else:
            _X, _y = X, y
699

700
701
702
703
        if self._class_weight is None:
            self._class_weight = self.class_weight
        if self._class_weight is not None:
            class_sample_weight = _LGBMComputeSampleWeight(self._class_weight, y)
704
705
706
707
            if sample_weight is None or len(sample_weight) == 0:
                sample_weight = class_sample_weight
            else:
                sample_weight = np.multiply(sample_weight, class_sample_weight)
708

709
        self._n_features = _X.shape[1]
710
711
        # copy for consistency
        self._n_features_in = self._n_features
712

713
714
        def _construct_dataset(X, y, sample_weight, init_score, group, params,
                               categorical_feature='auto'):
715
            return Dataset(X, label=y, weight=sample_weight, group=group,
716
717
                           init_score=init_score, params=params,
                           categorical_feature=categorical_feature)
Guolin Ke's avatar
Guolin Ke committed
718

719
720
        train_set = _construct_dataset(_X, _y, sample_weight, init_score, group, params,
                                       categorical_feature=categorical_feature)
Guolin Ke's avatar
Guolin Ke committed
721
722
723

        valid_sets = []
        if eval_set is not None:
724

725
            def _get_meta_data(collection, name, i):
726
727
728
729
730
731
732
                if collection is None:
                    return None
                elif isinstance(collection, list):
                    return collection[i] if len(collection) > i else None
                elif isinstance(collection, dict):
                    return collection.get(i, None)
                else:
733
                    raise TypeError(f"{name} should be dict or list")
734

Guolin Ke's avatar
Guolin Ke committed
735
736
737
            if isinstance(eval_set, tuple):
                eval_set = [eval_set]
            for i, valid_data in enumerate(eval_set):
738
                # reduce cost for prediction training data
Guolin Ke's avatar
Guolin Ke committed
739
740
741
                if valid_data[0] is X and valid_data[1] is y:
                    valid_set = train_set
                else:
742
743
744
745
746
747
                    valid_weight = _get_meta_data(eval_sample_weight, 'eval_sample_weight', i)
                    valid_class_weight = _get_meta_data(eval_class_weight, 'eval_class_weight', i)
                    if valid_class_weight is not None:
                        if isinstance(valid_class_weight, dict) and self._class_map is not None:
                            valid_class_weight = {self._class_map[k]: v for k, v in valid_class_weight.items()}
                        valid_class_sample_weight = _LGBMComputeSampleWeight(valid_class_weight, valid_data[1])
748
749
750
751
                        if valid_weight is None or len(valid_weight) == 0:
                            valid_weight = valid_class_sample_weight
                        else:
                            valid_weight = np.multiply(valid_weight, valid_class_sample_weight)
752
753
                    valid_init_score = _get_meta_data(eval_init_score, 'eval_init_score', i)
                    valid_group = _get_meta_data(eval_group, 'eval_group', i)
754
755
                    valid_set = _construct_dataset(valid_data[0], valid_data[1],
                                                   valid_weight, valid_init_score, valid_group, params)
Guolin Ke's avatar
Guolin Ke committed
756
757
                valid_sets.append(valid_set)

758
759
760
        if isinstance(init_model, LGBMModel):
            init_model = init_model.booster_

761
762
763
        if callbacks is None:
            callbacks = []
        else:
764
            callbacks = copy.copy(callbacks)  # don't use deepcopy here to allow non-serializable objects
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780

        evals_result = {}
        callbacks.append(record_evaluation(evals_result))

        self._Booster = train(
            params=params,
            train_set=train_set,
            num_boost_round=self.n_estimators,
            valid_sets=valid_sets,
            valid_names=eval_names,
            fobj=self._fobj,
            feval=eval_metrics_callable,
            init_model=init_model,
            feature_name=feature_name,
            callbacks=callbacks
        )
wxchan's avatar
wxchan committed
781
782

        if evals_result:
783
            self._evals_result = evals_result
784
785
        else:  # reset after previous call to fit()
            self._evals_result = None
wxchan's avatar
wxchan committed
786

787
        self._best_iteration = self._Booster.best_iteration
788
        self._best_score = self._Booster.best_score
wxchan's avatar
wxchan committed
789

790
791
        self.fitted_ = True

wxchan's avatar
wxchan committed
792
        # free dataset
793
        self._Booster.free_dataset()
wxchan's avatar
wxchan committed
794
        del train_set, valid_sets
wxchan's avatar
wxchan committed
795
796
        return self

797
798
799
800
    fit.__doc__ = _lgbmmodel_doc_fit.format(
        X_shape="array-like or sparse matrix of shape = [n_samples, n_features]",
        y_shape="array-like of shape = [n_samples]",
        sample_weight_shape="array-like of shape = [n_samples] or None, optional (default=None)",
801
        init_score_shape="array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task) or shape = [n_samples, n_classes] (for multi-class task) or None, optional (default=None)",
802
        group_shape="array-like or None, optional (default=None)",
803
804
805
        eval_sample_weight_shape="list of array, or None, optional (default=None)",
        eval_init_score_shape="list of array, or None, optional (default=None)",
        eval_group_shape="list of array, or None, optional (default=None)"
806
807
    ) + "\n\n" + _lgbmmodel_doc_custom_eval_note

808
    def predict(self, X, raw_score=False, start_iteration=0, num_iteration=None,
809
                pred_leaf=False, pred_contrib=False, **kwargs):
810
        """Docstring is set after definition, using a template."""
811
        if not self.__sklearn_is_fitted__():
812
            raise LGBMNotFittedError("Estimator not fitted, call fit before exploiting the model.")
813
        if not isinstance(X, (pd_DataFrame, dt_DataTable)):
814
            X = _LGBMCheckArray(X, accept_sparse=True, force_all_finite=False)
815
816
817
        n_features = X.shape[1]
        if self._n_features != n_features:
            raise ValueError("Number of features of the model must "
818
819
                             f"match the input. Model n_features_ is {self._n_features} and "
                             f"input n_features is {n_features}")
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
        # retrive original params that possibly can be used in both training and prediction
        # and then overwrite them (considering aliases) with params that were passed directly in prediction
        predict_params = self._process_params(stage="predict")
        for alias in _ConfigAliases.get_by_alias(
            "data",
            "X",
            "raw_score",
            "start_iteration",
            "num_iteration",
            "pred_leaf",
            "pred_contrib",
            *kwargs.keys()
        ):
            predict_params.pop(alias, None)
        predict_params.update(kwargs)
835
        return self._Booster.predict(X, raw_score=raw_score, start_iteration=start_iteration, num_iteration=num_iteration,
836
                                     pred_leaf=pred_leaf, pred_contrib=pred_contrib, **predict_params)
wxchan's avatar
wxchan committed
837

838
839
840
841
842
843
844
845
846
    predict.__doc__ = _lgbmmodel_doc_predict.format(
        description="Return the predicted value for each sample.",
        X_shape="array-like or sparse matrix of shape = [n_samples, n_features]",
        output_name="predicted_result",
        predicted_result_shape="array-like of shape = [n_samples] or shape = [n_samples, n_classes]",
        X_leaves_shape="array-like of shape = [n_samples, n_trees] or shape = [n_samples, n_trees * n_classes]",
        X_SHAP_values_shape="array-like of shape = [n_samples, n_features + 1] or shape = [n_samples, (n_features + 1) * n_classes] or list with n_classes length of such objects"
    )

847
848
    @property
    def n_features_(self):
849
        """:obj:`int`: The number of features of fitted model."""
850
        if not self.__sklearn_is_fitted__():
851
852
853
            raise LGBMNotFittedError('No n_features found. Need to call fit beforehand.')
        return self._n_features

854
855
856
    @property
    def n_features_in_(self):
        """:obj:`int`: The number of features of fitted model."""
857
        if not self.__sklearn_is_fitted__():
858
859
860
            raise LGBMNotFittedError('No n_features_in found. Need to call fit beforehand.')
        return self._n_features_in

861
862
    @property
    def best_score_(self):
863
        """:obj:`dict`: The best score of fitted model."""
864
        if not self.__sklearn_is_fitted__():
865
866
867
868
869
            raise LGBMNotFittedError('No best_score found. Need to call fit beforehand.')
        return self._best_score

    @property
    def best_iteration_(self):
870
        """:obj:`int`: The best iteration of fitted model if ``early_stopping()`` callback has been specified."""
871
        if not self.__sklearn_is_fitted__():
872
            raise LGBMNotFittedError('No best_iteration found. Need to call fit with early_stopping callback beforehand.')
873
874
875
876
        return self._best_iteration

    @property
    def objective_(self):
877
        """:obj:`str` or :obj:`callable`: The concrete objective used while fitting this model."""
878
        if not self.__sklearn_is_fitted__():
879
880
881
            raise LGBMNotFittedError('No objective found. Need to call fit beforehand.')
        return self._objective

882
883
884
885
886
887
888
889
890
    @property
    def n_estimators_(self) -> int:
        """:obj:`int`: True number of boosting iterations performed.

        This might be less than parameter ``n_estimators`` if early stopping was enabled or
        if boosting stopped early due to limits on complexity like ``min_gain_to_split``.
        """
        if not self.__sklearn_is_fitted__():
            raise LGBMNotFittedError('No n_estimators found. Need to call fit beforehand.')
891
        return self._Booster.current_iteration()  # type: ignore
892
893
894
895
896
897
898
899
900
901

    @property
    def n_iter_(self) -> int:
        """:obj:`int`: True number of boosting iterations performed.

        This might be less than parameter ``n_estimators`` if early stopping was enabled or
        if boosting stopped early due to limits on complexity like ``min_gain_to_split``.
        """
        if not self.__sklearn_is_fitted__():
            raise LGBMNotFittedError('No n_iter found. Need to call fit beforehand.')
902
        return self._Booster.current_iteration()  # type: ignore
903

904
905
    @property
    def booster_(self):
906
        """Booster: The underlying Booster of this model."""
907
        if not self.__sklearn_is_fitted__():
908
            raise LGBMNotFittedError('No booster found. Need to call fit beforehand.')
909
        return self._Booster
wxchan's avatar
wxchan committed
910

911
912
    @property
    def evals_result_(self):
913
        """:obj:`dict` or :obj:`None`: The evaluation results if validation sets have been specified."""
914
        if not self.__sklearn_is_fitted__():
915
916
            raise LGBMNotFittedError('No results found. Need to call fit with eval_set beforehand.')
        return self._evals_result
917
918

    @property
919
    def feature_importances_(self):
920
        """:obj:`array` of shape = [n_features]: The feature importances (the higher, the more important).
921

Nikita Titov's avatar
Nikita Titov committed
922
923
924
925
        .. note::

            ``importance_type`` attribute is passed to the function
            to configure the type of importance values to be extracted.
926
        """
927
        if not self.__sklearn_is_fitted__():
928
            raise LGBMNotFittedError('No feature_importances found. Need to call fit beforehand.')
929
        return self._Booster.feature_importance(importance_type=self.importance_type)
wxchan's avatar
wxchan committed
930

931
932
    @property
    def feature_name_(self):
933
        """:obj:`array` of shape = [n_features]: The names of features."""
934
        if not self.__sklearn_is_fitted__():
935
936
937
            raise LGBMNotFittedError('No feature_name found. Need to call fit beforehand.')
        return self._Booster.feature_name()

wxchan's avatar
wxchan committed
938

939
class LGBMRegressor(_LGBMRegressorBase, LGBMModel):
940
    """LightGBM regressor."""
wxchan's avatar
wxchan committed
941

942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
    def fit(
        self,
        X,
        y,
        sample_weight=None,
        init_score=None,
        eval_set=None,
        eval_names=None,
        eval_sample_weight=None,
        eval_init_score=None,
        eval_metric=None,
        feature_name='auto',
        categorical_feature='auto',
        callbacks=None,
        init_model=None
    ):
958
        """Docstring is inherited from the LGBMModel."""
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
        super().fit(
            X,
            y,
            sample_weight=sample_weight,
            init_score=init_score,
            eval_set=eval_set,
            eval_names=eval_names,
            eval_sample_weight=eval_sample_weight,
            eval_init_score=eval_init_score,
            eval_metric=eval_metric,
            feature_name=feature_name,
            categorical_feature=categorical_feature,
            callbacks=callbacks,
            init_model=init_model
        )
Guolin Ke's avatar
Guolin Ke committed
974
975
        return self

976
    _base_doc = LGBMModel.fit.__doc__.replace("self : LGBMModel", "self : LGBMRegressor")  # type: ignore
977
978
    _base_doc = (_base_doc[:_base_doc.find('group :')]  # type: ignore
                 + _base_doc[_base_doc.find('eval_set :'):])  # type: ignore
979
980
981
982
    _base_doc = (_base_doc[:_base_doc.find('eval_class_weight :')]
                 + _base_doc[_base_doc.find('eval_init_score :'):])
    fit.__doc__ = (_base_doc[:_base_doc.find('eval_group :')]
                   + _base_doc[_base_doc.find('eval_metric :'):])
wxchan's avatar
wxchan committed
983

984

985
class LGBMClassifier(_LGBMClassifierBase, LGBMModel):
986
    """LightGBM classifier."""
wxchan's avatar
wxchan committed
987

988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
    def fit(
        self,
        X,
        y,
        sample_weight=None,
        init_score=None,
        eval_set=None,
        eval_names=None,
        eval_sample_weight=None,
        eval_class_weight=None,
        eval_init_score=None,
        eval_metric=None,
        feature_name='auto',
        categorical_feature='auto',
        callbacks=None,
        init_model=None
    ):
1005
        """Docstring is inherited from the LGBMModel."""
1006
        _LGBMAssertAllFinite(y)
1007
1008
        _LGBMCheckClassificationTargets(y)
        self._le = _LGBMLabelEncoder().fit(y)
1009
        _y = self._le.transform(y)
1010
        self._class_map = dict(zip(self._le.classes_, self._le.transform(self._le.classes_)))
1011
1012
        if isinstance(self.class_weight, dict):
            self._class_weight = {self._class_map[k]: v for k, v in self.class_weight.items()}
1013

1014
1015
        self._classes = self._le.classes_
        self._n_classes = len(self._classes)
1016
1017

        if not callable(eval_metric):
1018
            if isinstance(eval_metric, (str, type(None))):
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
                eval_metric = [eval_metric]
            if self._n_classes > 2:
                for index, metric in enumerate(eval_metric):
                    if metric in {'logloss', 'binary_logloss'}:
                        eval_metric[index] = "multi_logloss"
                    elif metric in {'error', 'binary_error'}:
                        eval_metric[index] = "multi_error"
            else:
                for index, metric in enumerate(eval_metric):
                    if metric in {'logloss', 'multi_logloss'}:
                        eval_metric[index] = 'binary_logloss'
                    elif metric in {'error', 'multi_error'}:
                        eval_metric[index] = 'binary_error'
wxchan's avatar
wxchan committed
1032

1033
1034
        # do not modify args, as it causes errors in model selection tools
        valid_sets = None
wxchan's avatar
wxchan committed
1035
        if eval_set is not None:
1036
1037
            if isinstance(eval_set, tuple):
                eval_set = [eval_set]
1038
            valid_sets = [None] * len(eval_set)
1039
1040
            for i, (valid_x, valid_y) in enumerate(eval_set):
                if valid_x is X and valid_y is y:
1041
                    valid_sets[i] = (valid_x, _y)
1042
                else:
1043
                    valid_sets[i] = (valid_x, self._le.transform(valid_y))
1044

1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
        super().fit(
            X,
            _y,
            sample_weight=sample_weight,
            init_score=init_score,
            eval_set=valid_sets,
            eval_names=eval_names,
            eval_sample_weight=eval_sample_weight,
            eval_class_weight=eval_class_weight,
            eval_init_score=eval_init_score,
            eval_metric=eval_metric,
            feature_name=feature_name,
            categorical_feature=categorical_feature,
            callbacks=callbacks,
            init_model=init_model
        )
wxchan's avatar
wxchan committed
1061
1062
        return self

1063
    _base_doc = LGBMModel.fit.__doc__.replace("self : LGBMModel", "self : LGBMClassifier")  # type: ignore
1064
1065
    _base_doc = (_base_doc[:_base_doc.find('group :')]  # type: ignore
                 + _base_doc[_base_doc.find('eval_set :'):])  # type: ignore
1066
1067
    fit.__doc__ = (_base_doc[:_base_doc.find('eval_group :')]
                   + _base_doc[_base_doc.find('eval_metric :'):])
1068

1069
    def predict(self, X, raw_score=False, start_iteration=0, num_iteration=None,
1070
                pred_leaf=False, pred_contrib=False, **kwargs):
1071
        """Docstring is inherited from the LGBMModel."""
1072
        result = self.predict_proba(X, raw_score, start_iteration, num_iteration,
1073
                                    pred_leaf, pred_contrib, **kwargs)
1074
        if callable(self._objective) or raw_score or pred_leaf or pred_contrib:
1075
1076
1077
1078
            return result
        else:
            class_index = np.argmax(result, axis=1)
            return self._le.inverse_transform(class_index)
wxchan's avatar
wxchan committed
1079

1080
1081
    predict.__doc__ = LGBMModel.predict.__doc__

1082
    def predict_proba(self, X, raw_score=False, start_iteration=0, num_iteration=None,
1083
                      pred_leaf=False, pred_contrib=False, **kwargs):
1084
        """Docstring is set after definition, using a template."""
1085
        result = super().predict(X, raw_score, start_iteration, num_iteration, pred_leaf, pred_contrib, **kwargs)
1086
        if callable(self._objective) and not (raw_score or pred_leaf or pred_contrib):
1087
1088
1089
            _log_warning("Cannot compute class probabilities or labels "
                         "due to the usage of customized objective function.\n"
                         "Returning raw scores instead.")
1090
1091
            return result
        elif self._n_classes > 2 or raw_score or pred_leaf or pred_contrib:
1092
            return result
wxchan's avatar
wxchan committed
1093
        else:
1094
            return np.vstack((1. - result, result)).transpose()
1095

1096
1097
1098
1099
    predict_proba.__doc__ = _lgbmmodel_doc_predict.format(
        description="Return the predicted probability for each class for each sample.",
        X_shape="array-like or sparse matrix of shape = [n_samples, n_features]",
        output_name="predicted_probability",
1100
        predicted_result_shape="array-like of shape = [n_samples] or shape = [n_samples, n_classes]",
1101
1102
1103
1104
        X_leaves_shape="array-like of shape = [n_samples, n_trees] or shape = [n_samples, n_trees * n_classes]",
        X_SHAP_values_shape="array-like of shape = [n_samples, n_features + 1] or shape = [n_samples, (n_features + 1) * n_classes] or list with n_classes length of such objects"
    )

1105
1106
    @property
    def classes_(self):
1107
        """:obj:`array` of shape = [n_classes]: The class label array."""
1108
        if not self.__sklearn_is_fitted__():
1109
1110
            raise LGBMNotFittedError('No classes found. Need to call fit beforehand.')
        return self._classes
1111
1112
1113

    @property
    def n_classes_(self):
1114
        """:obj:`int`: The number of classes."""
1115
        if not self.__sklearn_is_fitted__():
1116
1117
            raise LGBMNotFittedError('No classes found. Need to call fit beforehand.')
        return self._n_classes
wxchan's avatar
wxchan committed
1118

wxchan's avatar
wxchan committed
1119

wxchan's avatar
wxchan committed
1120
class LGBMRanker(LGBMModel):
1121
1122
1123
1124
1125
1126
1127
1128
    """LightGBM ranker.

    .. warning::

        scikit-learn doesn't support ranking applications yet,
        therefore this class is not really compatible with the sklearn ecosystem.
        Please use this class mainly for training and applying ranking models in common sklearnish way.
    """
wxchan's avatar
wxchan committed
1129

1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
    def fit(
        self,
        X,
        y,
        sample_weight=None,
        init_score=None,
        group=None,
        eval_set=None,
        eval_names=None,
        eval_sample_weight=None,
        eval_init_score=None,
        eval_group=None,
        eval_metric=None,
        eval_at=(1, 2, 3, 4, 5),
        feature_name='auto',
        categorical_feature='auto',
        callbacks=None,
        init_model=None
    ):
1149
        """Docstring is inherited from the LGBMModel."""
1150
        # check group data
Guolin Ke's avatar
Guolin Ke committed
1151
        if group is None:
1152
            raise ValueError("Should set group for ranking task")
wxchan's avatar
wxchan committed
1153
1154

        if eval_set is not None:
Guolin Ke's avatar
Guolin Ke committed
1155
            if eval_group is None:
1156
                raise ValueError("Eval_group cannot be None when eval_set is not None")
Guolin Ke's avatar
Guolin Ke committed
1157
            elif len(eval_group) != len(eval_set):
1158
                raise ValueError("Length of eval_group should be equal to eval_set")
1159
            elif (isinstance(eval_group, dict)
1160
                  and any(i not in eval_group or eval_group[i] is None for i in range(len(eval_group)))
1161
1162
                  or isinstance(eval_group, list)
                  and any(group is None for group in eval_group)):
1163
1164
                raise ValueError("Should set group for all eval datasets for ranking task; "
                                 "if you use dict, the index should start from 0")
1165

1166
        self._eval_at = eval_at
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
        super().fit(
            X,
            y,
            sample_weight=sample_weight,
            init_score=init_score,
            group=group,
            eval_set=eval_set,
            eval_names=eval_names,
            eval_sample_weight=eval_sample_weight,
            eval_init_score=eval_init_score,
            eval_group=eval_group,
            eval_metric=eval_metric,
            feature_name=feature_name,
            categorical_feature=categorical_feature,
            callbacks=callbacks,
            init_model=init_model
        )
wxchan's avatar
wxchan committed
1184
        return self
1185

1186
    _base_doc = LGBMModel.fit.__doc__.replace("self : LGBMModel", "self : LGBMRanker")  # type: ignore
1187
1188
    fit.__doc__ = (_base_doc[:_base_doc.find('eval_class_weight :')]  # type: ignore
                   + _base_doc[_base_doc.find('eval_init_score :'):])  # type: ignore
1189
    _base_doc = fit.__doc__
1190
1191
    _before_feature_name, _feature_name, _after_feature_name = _base_doc.partition('feature_name :')
    fit.__doc__ = f"""{_before_feature_name}eval_at : iterable of int, optional (default=(1, 2, 3, 4, 5))
1192
        The evaluation positions of the specified metric.
1193
    {_feature_name}{_after_feature_name}"""