sklearn.py 55.1 KB
Newer Older
wxchan's avatar
wxchan committed
1
# coding: utf-8
2
"""Scikit-learn wrapper interface for LightGBM."""
3
import copy
4
from inspect import signature
5
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
6

wxchan's avatar
wxchan committed
7
import numpy as np
8

9
10
from .basic import (Booster, Dataset, LightGBMError, _choose_param_value, _ConfigAliases, _LGBM_EvalFunctionResultType,
                    _log_warning)
11
from .callback import record_evaluation
12
13
from .compat import (SKLEARN_INSTALLED, LGBMNotFittedError, _LGBMAssertAllFinite, _LGBMCheckArray,
                     _LGBMCheckClassificationTargets, _LGBMCheckSampleWeight, _LGBMCheckXY, _LGBMClassifierBase,
14
15
                     _LGBMComputeSampleWeight, _LGBMCpuCount, _LGBMLabelEncoder, _LGBMModelBase, _LGBMRegressorBase,
                     dt_DataTable, pd_DataFrame)
wxchan's avatar
wxchan committed
16
from .engine import train
17

18
19
20
_LGBM_ScikitCustomObjectiveFunction = Union[
    Callable[
        [np.ndarray, np.ndarray],
21
        Tuple[np.ndarray, np.ndarray]
22
23
24
    ],
    Callable[
        [np.ndarray, np.ndarray, np.ndarray],
25
        Tuple[np.ndarray, np.ndarray]
26
    ],
27
28
29
30
    Callable[
        [np.ndarray, np.ndarray, np.ndarray, np.ndarray],
        Tuple[np.ndarray, np.ndarray]
    ],
31
32
33
34
]
_LGBM_ScikitCustomEvalFunction = Union[
    Callable[
        [np.ndarray, np.ndarray],
35
        Union[_LGBM_EvalFunctionResultType, List[_LGBM_EvalFunctionResultType]]
36
37
38
    ],
    Callable[
        [np.ndarray, np.ndarray, np.ndarray],
39
        Union[_LGBM_EvalFunctionResultType, List[_LGBM_EvalFunctionResultType]]
40
41
42
    ],
    Callable[
        [np.ndarray, np.ndarray, np.ndarray, np.ndarray],
43
        Union[_LGBM_EvalFunctionResultType, List[_LGBM_EvalFunctionResultType]]
44
45
46
    ],
]

wxchan's avatar
wxchan committed
47

48
class _ObjectiveFunctionWrapper:
49
    """Proxy class for objective function."""
50

51
    def __init__(self, func: _LGBM_ScikitCustomObjectiveFunction):
52
        """Construct a proxy class.
53

54
55
        This class transforms objective function to match objective function with signature ``new_func(preds, dataset)``
        as expected by ``lightgbm.engine.train``.
56

57
58
59
        Parameters
        ----------
        func : callable
60
61
62
63
            Expects a callable with following signatures:
            ``func(y_true, y_pred)``,
            ``func(y_true, y_pred, weight)``
            or ``func(y_true, y_pred, weight, group)``
64
65
            and returns (grad, hess):

66
                y_true : numpy 1-D array of shape = [n_samples]
67
                    The target values.
68
                y_pred : numpy 1-D array of shape = [n_samples] or numpy 2-D array of shape = [n_samples, n_classes] (for multi-class task)
69
                    The predicted values.
70
71
                    Predicted values are returned before any transformation,
                    e.g. they are raw margin instead of probability of positive class for binary task.
72
73
                weight : numpy 1-D array of shape = [n_samples]
                    The weight of samples. Weights should be non-negative.
74
                group : numpy 1-D array
75
76
77
                    Group/query data.
                    Only used in the learning-to-rank task.
                    sum(group) = n_samples.
78
79
                    For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
                    where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
80
                grad : numpy 1-D array of shape = [n_samples] or numpy 2-D array of shape [n_samples, n_classes] (for multi-class task)
81
82
                    The value of the first order derivative (gradient) of the loss
                    with respect to the elements of y_pred for each sample point.
83
                hess : numpy 1-D array of shape = [n_samples] or numpy 2-D array of shape = [n_samples, n_classes] (for multi-class task)
84
85
                    The value of the second order derivative (Hessian) of the loss
                    with respect to the elements of y_pred for each sample point.
wxchan's avatar
wxchan committed
86

Nikita Titov's avatar
Nikita Titov committed
87
88
        .. note::

89
            For multi-class task, y_pred is a numpy 2-D array of shape = [n_samples, n_classes],
90
            and grad and hess should be returned in the same format.
91
92
        """
        self.func = func
wxchan's avatar
wxchan committed
93

94
    def __call__(self, preds: np.ndarray, dataset: Dataset) -> Tuple[np.ndarray, np.ndarray]:
95
96
97
98
        """Call passed function with appropriate arguments.

        Parameters
        ----------
99
        preds : numpy 1-D array of shape = [n_samples] or numpy 2-D array of shape = [n_samples, n_classes] (for multi-class task)
100
101
102
103
104
105
            The predicted values.
        dataset : Dataset
            The training dataset.

        Returns
        -------
106
        grad : numpy 1-D array of shape = [n_samples] or numpy 2-D array of shape = [n_samples, n_classes] (for multi-class task)
107
108
            The value of the first order derivative (gradient) of the loss
            with respect to the elements of preds for each sample point.
109
        hess : numpy 1-D array of shape = [n_samples] or numpy 2-D array of shape = [n_samples, n_classes] (for multi-class task)
110
111
            The value of the second order derivative (Hessian) of the loss
            with respect to the elements of preds for each sample point.
112
        """
wxchan's avatar
wxchan committed
113
        labels = dataset.get_label()
114
        argc = len(signature(self.func).parameters)
115
        if argc == 2:
116
            grad, hess = self.func(labels, preds)
117
        elif argc == 3:
118
119
120
            grad, hess = self.func(labels, preds, dataset.get_weight())
        elif argc == 4:
            grad, hess = self.func(labels, preds, dataset.get_weight(), dataset.get_group())
121
        else:
122
            raise TypeError(f"Self-defined objective function should have 2, 3 or 4 arguments, got {argc}")
wxchan's avatar
wxchan committed
123
124
        return grad, hess

wxchan's avatar
wxchan committed
125

126
class _EvalFunctionWrapper:
127
    """Proxy class for evaluation function."""
128

129
    def __init__(self, func: _LGBM_ScikitCustomEvalFunction):
130
        """Construct a proxy class.
131

132
133
        This class transforms evaluation function to match evaluation function with signature ``new_func(preds, dataset)``
        as expected by ``lightgbm.engine.train``.
134

135
136
137
138
139
140
141
142
143
144
        Parameters
        ----------
        func : callable
            Expects a callable with following signatures:
            ``func(y_true, y_pred)``,
            ``func(y_true, y_pred, weight)``
            or ``func(y_true, y_pred, weight, group)``
            and returns (eval_name, eval_result, is_higher_better) or
            list of (eval_name, eval_result, is_higher_better):

145
                y_true : numpy 1-D array of shape = [n_samples]
146
                    The target values.
147
                y_pred : numpy 1-D array of shape = [n_samples] or numpy 2-D array shape = [n_samples, n_classes] (for multi-class task)
148
                    The predicted values.
149
150
                    In case of custom ``objective``, predicted values are returned before any transformation,
                    e.g. they are raw margin instead of probability of positive class for binary task in this case.
151
                weight : numpy 1-D array of shape = [n_samples]
152
                    The weight of samples. Weights should be non-negative.
153
                group : numpy 1-D array
154
155
156
                    Group/query data.
                    Only used in the learning-to-rank task.
                    sum(group) = n_samples.
157
158
                    For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
                    where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
159
                eval_name : str
Andrew Ziem's avatar
Andrew Ziem committed
160
                    The name of evaluation function (without whitespace).
161
162
163
164
165
166
                eval_result : float
                    The eval result.
                is_higher_better : bool
                    Is eval result higher better, e.g. AUC is ``is_higher_better``.
        """
        self.func = func
167

168
    def __call__(self, preds: np.ndarray, dataset: Dataset) -> Tuple[str, float, bool]:
169
        """Call passed function with appropriate arguments.
170

171
172
        Parameters
        ----------
173
        preds : numpy 1-D array of shape = [n_samples] or numpy 2-D array of shape = [n_samples, n_classes] (for multi-class task)
174
175
176
177
178
179
            The predicted values.
        dataset : Dataset
            The training dataset.

        Returns
        -------
180
        eval_name : str
Andrew Ziem's avatar
Andrew Ziem committed
181
            The name of evaluation function (without whitespace).
182
183
184
185
186
        eval_result : float
            The eval result.
        is_higher_better : bool
            Is eval result higher better, e.g. AUC is ``is_higher_better``.
        """
187
        labels = dataset.get_label()
188
        argc = len(signature(self.func).parameters)
189
        if argc == 2:
190
            return self.func(labels, preds)
191
        elif argc == 3:
192
            return self.func(labels, preds, dataset.get_weight())
193
        elif argc == 4:
194
            return self.func(labels, preds, dataset.get_weight(), dataset.get_group())
195
        else:
196
            raise TypeError(f"Self-defined eval function should have 2, 3 or 4 arguments, got {argc}")
197

wxchan's avatar
wxchan committed
198

199
200
201
202
203
204
205
206
207
208
209
210
211
212
# documentation templates for LGBMModel methods are shared between the classes in
# this module and those in the ``dask`` module

_lgbmmodel_doc_fit = (
    """
    Build a gradient boosting model from the training set (X, y).

    Parameters
    ----------
    X : {X_shape}
        Input feature matrix.
    y : {y_shape}
        The target values (class labels in classification, real numbers in regression).
    sample_weight : {sample_weight_shape}
213
        Weights of training data. Weights should be non-negative.
214
    init_score : {init_score_shape}
215
216
217
218
219
220
221
222
223
        Init score of training data.
    group : {group_shape}
        Group/query data.
        Only used in the learning-to-rank task.
        sum(group) = n_samples.
        For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
        where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
    eval_set : list or None, optional (default=None)
        A list of (X, y) tuple pairs to use as validation sets.
224
    eval_names : list of str, or None, optional (default=None)
225
        Names of eval_set.
226
    eval_sample_weight : {eval_sample_weight_shape}
227
        Weights of eval data. Weights should be non-negative.
228
229
    eval_class_weight : list or None, optional (default=None)
        Class weights of eval data.
230
    eval_init_score : {eval_init_score_shape}
231
        Init score of eval data.
232
    eval_group : {eval_group_shape}
233
        Group data of eval data.
234
235
    eval_metric : str, callable, list or None, optional (default=None)
        If str, it should be a built-in evaluation metric to use.
236
237
238
239
        If callable, it should be a custom evaluation metric, see note below for more details.
        If list, it can be a list of built-in metrics, a list of custom evaluation metrics, or a mix of both.
        In either case, the ``metric`` from the model parameters will be evaluated and used as well.
        Default: 'l2' for LGBMRegressor, 'logloss' for LGBMClassifier, 'ndcg' for LGBMRanker.
240
    feature_name : list of str, or 'auto', optional (default='auto')
241
242
        Feature names.
        If 'auto' and data is pandas DataFrame, data columns names are used.
243
    categorical_feature : list of str or int, or 'auto', optional (default='auto')
244
245
        Categorical features.
        If list of int, interpreted as indices.
246
        If list of str, interpreted as feature names (need to specify ``feature_name`` as well).
247
        If 'auto' and data is pandas DataFrame, pandas unordered categorical columns are used.
248
        All values in categorical features will be cast to int32 and thus should be less than int32 max value (2147483647).
249
250
251
        Large values could be memory consuming. Consider using consecutive integers starting from zero.
        All negative values in categorical features will be treated as missing values.
        The output cannot be monotonically constrained with respect to a categorical feature.
252
        Floating point numbers in categorical features will be rounded towards 0.
253
    callbacks : list of callable, or None, optional (default=None)
254
255
        List of callback functions that are applied at each iteration.
        See Callbacks in Python API for more information.
256
    init_model : str, pathlib.Path, Booster, LGBMModel or None, optional (default=None)
257
258
259
260
        Filename of LightGBM model, Booster instance or LGBMModel instance used for continue training.

    Returns
    -------
261
    self : LGBMModel
262
263
264
265
266
267
268
269
270
271
272
273
274
        Returns self.
    """
)

_lgbmmodel_doc_custom_eval_note = """
    Note
    ----
    Custom eval function expects a callable with following signatures:
    ``func(y_true, y_pred)``, ``func(y_true, y_pred, weight)`` or
    ``func(y_true, y_pred, weight, group)``
    and returns (eval_name, eval_result, is_higher_better) or
    list of (eval_name, eval_result, is_higher_better):

275
        y_true : numpy 1-D array of shape = [n_samples]
276
            The target values.
277
        y_pred : numpy 1-D array of shape = [n_samples] or numpy 2-D array of shape = [n_samples, n_classes] (for multi-class task)
278
            The predicted values.
279
280
            In case of custom ``objective``, predicted values are returned before any transformation,
            e.g. they are raw margin instead of probability of positive class for binary task in this case.
281
        weight : numpy 1-D array of shape = [n_samples]
282
            The weight of samples. Weights should be non-negative.
283
        group : numpy 1-D array
284
285
286
287
288
            Group/query data.
            Only used in the learning-to-rank task.
            sum(group) = n_samples.
            For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
            where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
289
        eval_name : str
Andrew Ziem's avatar
Andrew Ziem committed
290
            The name of evaluation function (without whitespace).
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
        eval_result : float
            The eval result.
        is_higher_better : bool
            Is eval result higher better, e.g. AUC is ``is_higher_better``.
"""

_lgbmmodel_doc_predict = (
    """
    {description}

    Parameters
    ----------
    X : {X_shape}
        Input features matrix.
    raw_score : bool, optional (default=False)
        Whether to predict raw scores.
    start_iteration : int, optional (default=0)
        Start index of the iteration to predict.
        If <= 0, starts from the first iteration.
    num_iteration : int or None, optional (default=None)
        Total number of iterations used in the prediction.
        If None, if the best iteration exists and start_iteration <= 0, the best iteration is used;
        otherwise, all iterations from ``start_iteration`` are used (no limits).
        If <= 0, all iterations from ``start_iteration`` are used (no limits).
    pred_leaf : bool, optional (default=False)
        Whether to predict leaf index.
    pred_contrib : bool, optional (default=False)
        Whether to predict feature contributions.

        .. note::

            If you want to get more explanations for your model's predictions using SHAP values,
            like SHAP interaction values,
            you can install the shap package (https://github.com/slundberg/shap).
            Note that unlike the shap package, with ``pred_contrib`` we return a matrix with an extra
            column, where the last column is the expected value.

328
329
330
    validate_features : bool, optional (default=False)
        If True, ensure that the features used to predict match the ones used to train.
        Used only if data is pandas DataFrame.
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
    **kwargs
        Other parameters for the prediction.

    Returns
    -------
    {output_name} : {predicted_result_shape}
        The predicted values.
    X_leaves : {X_leaves_shape}
        If ``pred_leaf=True``, the predicted leaf of every tree for each sample.
    X_SHAP_values : {X_SHAP_values_shape}
        If ``pred_contrib=True``, the feature contributions for each sample.
    """
)


346
347
class LGBMModel(_LGBMModelBase):
    """Implementation of the scikit-learn API for LightGBM."""
wxchan's avatar
wxchan committed
348

349
350
351
352
353
354
355
356
    def __init__(
        self,
        boosting_type: str = 'gbdt',
        num_leaves: int = 31,
        max_depth: int = -1,
        learning_rate: float = 0.1,
        n_estimators: int = 100,
        subsample_for_bin: int = 200000,
357
        objective: Optional[Union[str, _LGBM_ScikitCustomObjectiveFunction]] = None,
358
359
360
361
362
363
364
365
366
367
        class_weight: Optional[Union[Dict, str]] = None,
        min_split_gain: float = 0.,
        min_child_weight: float = 1e-3,
        min_child_samples: int = 20,
        subsample: float = 1.,
        subsample_freq: int = 0,
        colsample_bytree: float = 1.,
        reg_alpha: float = 0.,
        reg_lambda: float = 0.,
        random_state: Optional[Union[int, np.random.RandomState]] = None,
368
        n_jobs: Optional[int] = None,
369
370
371
        importance_type: str = 'split',
        **kwargs
    ):
372
        r"""Construct a gradient boosting model.
wxchan's avatar
wxchan committed
373
374
375

        Parameters
        ----------
376
        boosting_type : str, optional (default='gbdt')
377
378
379
380
381
            'gbdt', traditional Gradient Boosting Decision Tree.
            'dart', Dropouts meet Multiple Additive Regression Trees.
            'goss', Gradient-based One-Side Sampling.
            'rf', Random Forest.
        num_leaves : int, optional (default=31)
wxchan's avatar
wxchan committed
382
            Maximum tree leaves for base learners.
383
        max_depth : int, optional (default=-1)
384
            Maximum tree depth for base learners, <=0 means no limit.
385
        learning_rate : float, optional (default=0.1)
386
            Boosting learning rate.
387
388
389
            You can use ``callbacks`` parameter of ``fit`` method to shrink/adapt learning rate
            in training using ``reset_parameter`` callback.
            Note, that this will ignore the ``learning_rate`` argument in training.
390
        n_estimators : int, optional (default=100)
wxchan's avatar
wxchan committed
391
            Number of boosted trees to fit.
392
        subsample_for_bin : int, optional (default=200000)
wxchan's avatar
wxchan committed
393
            Number of samples for constructing bins.
394
        objective : str, callable or None, optional (default=None)
wxchan's avatar
wxchan committed
395
396
            Specify the learning task and the corresponding learning objective or
            a custom objective function to be used (see note below).
397
            Default: 'regression' for LGBMRegressor, 'binary' or 'multiclass' for LGBMClassifier, 'lambdarank' for LGBMRanker.
398
399
400
401
        class_weight : dict, 'balanced' or None, optional (default=None)
            Weights associated with classes in the form ``{class_label: weight}``.
            Use this parameter only for multi-class classification task;
            for binary classification task you may use ``is_unbalance`` or ``scale_pos_weight`` parameters.
402
403
404
            Note, that the usage of all these parameters will result in poor estimates of the individual class probabilities.
            You may want to consider performing probability calibration
            (https://scikit-learn.org/stable/modules/calibration.html) of your model.
405
406
407
            The 'balanced' mode uses the values of y to automatically adjust weights
            inversely proportional to class frequencies in the input data as ``n_samples / (n_classes * np.bincount(y))``.
            If None, all classes are supposed to have weight one.
408
            Note, that these weights will be multiplied with ``sample_weight`` (passed through the ``fit`` method)
409
            if ``sample_weight`` is specified.
410
        min_split_gain : float, optional (default=0.)
wxchan's avatar
wxchan committed
411
            Minimum loss reduction required to make a further partition on a leaf node of the tree.
412
        min_child_weight : float, optional (default=1e-3)
413
            Minimum sum of instance weight (Hessian) needed in a child (leaf).
414
        min_child_samples : int, optional (default=20)
415
            Minimum number of data needed in a child (leaf).
416
        subsample : float, optional (default=1.)
wxchan's avatar
wxchan committed
417
            Subsample ratio of the training instance.
418
        subsample_freq : int, optional (default=0)
Andrew Ziem's avatar
Andrew Ziem committed
419
            Frequency of subsample, <=0 means no enable.
420
        colsample_bytree : float, optional (default=1.)
wxchan's avatar
wxchan committed
421
            Subsample ratio of columns when constructing each tree.
422
        reg_alpha : float, optional (default=0.)
423
            L1 regularization term on weights.
424
        reg_lambda : float, optional (default=0.)
425
            L2 regularization term on weights.
426
        random_state : int, RandomState object or None, optional (default=None)
wxchan's avatar
wxchan committed
427
            Random number seed.
428
429
430
            If int, this number is used to seed the C++ code.
            If RandomState object (numpy), a random integer is picked based on its state to seed the C++ code.
            If None, default seeds in C++ code are used.
431
432
433
434
435
436
437
438
439
440
441
442
        n_jobs : int or None, optional (default=None)
            Number of parallel threads to use for training (can be changed at prediction time by
            passing it as an extra keyword argument).

            For better performance, it is recommended to set this to the number of physical cores
            in the CPU.

            Negative integers are interpreted as following joblib's formula (n_cpus + 1 + n_jobs), just like
            scikit-learn (so e.g. -1 means using all threads). A value of zero corresponds the default number of
            threads configured for OpenMP in the system. A value of ``None`` (the default) corresponds
            to using the number of physical cores in the system (its correct detection requires
            either the ``joblib`` or the ``psutil`` util libraries to be installed).
443
        importance_type : str, optional (default='split')
444
            The type of feature importance to be filled into ``feature_importances_``.
445
446
447
448
            If 'split', result contains numbers of times the feature is used in a model.
            If 'gain', result contains total gains of splits which use the feature.
        **kwargs
            Other parameters for the model.
wxchan's avatar
wxchan committed
449
            Check http://lightgbm.readthedocs.io/en/latest/Parameters.html for more parameters.
450

Nikita Titov's avatar
Nikita Titov committed
451
452
453
            .. warning::

                \*\*kwargs is not supported in sklearn, it may cause unexpected issues.
wxchan's avatar
wxchan committed
454
455
456

        Note
        ----
457
458
        A custom objective function can be provided for the ``objective`` parameter.
        In this case, it should have the signature
459
460
461
        ``objective(y_true, y_pred) -> grad, hess``,
        ``objective(y_true, y_pred, weight) -> grad, hess``
        or ``objective(y_true, y_pred, weight, group) -> grad, hess``:
wxchan's avatar
wxchan committed
462

463
            y_true : numpy 1-D array of shape = [n_samples]
464
                The target values.
465
            y_pred : numpy 1-D array of shape = [n_samples] or numpy 2-D array of shape = [n_samples, n_classes] (for multi-class task)
466
                The predicted values.
467
468
                Predicted values are returned before any transformation,
                e.g. they are raw margin instead of probability of positive class for binary task.
469
470
            weight : numpy 1-D array of shape = [n_samples]
                The weight of samples. Weights should be non-negative.
471
            group : numpy 1-D array
472
473
474
                Group/query data.
                Only used in the learning-to-rank task.
                sum(group) = n_samples.
475
476
                For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
                where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
477
            grad : numpy 1-D array of shape = [n_samples] or numpy 2-D array of shape = [n_samples, n_classes] (for multi-class task)
478
479
                The value of the first order derivative (gradient) of the loss
                with respect to the elements of y_pred for each sample point.
480
            hess : numpy 1-D array of shape = [n_samples] or numpy 2-D array of shape = [n_samples, n_classes] (for multi-class task)
481
482
                The value of the second order derivative (Hessian) of the loss
                with respect to the elements of y_pred for each sample point.
wxchan's avatar
wxchan committed
483

484
        For multi-class task, y_pred is a numpy 2-D array of shape = [n_samples, n_classes],
485
        and grad and hess should be returned in the same format.
wxchan's avatar
wxchan committed
486
        """
wxchan's avatar
wxchan committed
487
        if not SKLEARN_INSTALLED:
488
489
            raise LightGBMError('scikit-learn is required for lightgbm.sklearn. '
                                'You must install scikit-learn and restart your session to use this module.')
wxchan's avatar
wxchan committed
490

491
        self.boosting_type = boosting_type
492
        self.objective = objective
wxchan's avatar
wxchan committed
493
494
495
496
        self.num_leaves = num_leaves
        self.max_depth = max_depth
        self.learning_rate = learning_rate
        self.n_estimators = n_estimators
wxchan's avatar
wxchan committed
497
        self.subsample_for_bin = subsample_for_bin
wxchan's avatar
wxchan committed
498
499
500
501
502
503
504
505
        self.min_split_gain = min_split_gain
        self.min_child_weight = min_child_weight
        self.min_child_samples = min_child_samples
        self.subsample = subsample
        self.subsample_freq = subsample_freq
        self.colsample_bytree = colsample_bytree
        self.reg_alpha = reg_alpha
        self.reg_lambda = reg_lambda
506
507
        self.random_state = random_state
        self.n_jobs = n_jobs
508
        self.importance_type = importance_type
509
        self._Booster: Optional[Booster] = None
510
511
512
        self._evals_result = None
        self._best_score = None
        self._best_iteration = None
513
        self._other_params: Dict[str, Any] = {}
514
        self._objective = objective
515
        self.class_weight = class_weight
516
517
        self._class_weight = None
        self._class_map = None
518
        self._n_features = None
519
        self._n_features_in = None
520
521
        self._classes = None
        self._n_classes = None
522
        self.set_params(**kwargs)
wxchan's avatar
wxchan committed
523

524
    def _more_tags(self) -> Dict[str, Any]:
525
526
527
528
529
530
531
532
533
534
        return {
            'allow_nan': True,
            'X_types': ['2darray', 'sparse', '1dlabels'],
            '_xfail_checks': {
                'check_no_attributes_set_in_init':
                'scikit-learn incorrectly asserts that private attributes '
                'cannot be set in __init__: '
                '(see https://github.com/microsoft/LightGBM/issues/2628)'
            }
        }
Nikita Titov's avatar
Nikita Titov committed
535

536
537
538
    def __sklearn_is_fitted__(self) -> bool:
        return getattr(self, "fitted_", False)

539
    def get_params(self, deep: bool = True) -> Dict[str, Any]:
540
541
542
543
544
545
546
547
548
549
550
551
552
        """Get parameters for this estimator.

        Parameters
        ----------
        deep : bool, optional (default=True)
            If True, will return the parameters for this estimator and
            contained subobjects that are estimators.

        Returns
        -------
        params : dict
            Parameter names mapped to their values.
        """
553
        params = super().get_params(deep=deep)
554
        params.update(self._other_params)
wxchan's avatar
wxchan committed
555
556
        return params

557
    def set_params(self, **params: Any) -> "LGBMModel":
558
559
560
561
562
563
564
565
566
567
568
569
        """Set the parameters of this estimator.

        Parameters
        ----------
        **params
            Parameter names with their new values.

        Returns
        -------
        self : object
            Returns self.
        """
wxchan's avatar
wxchan committed
570
571
        for key, value in params.items():
            setattr(self, key, value)
572
573
            if hasattr(self, f"_{key}"):
                setattr(self, f"_{key}", value)
574
            self._other_params[key] = value
wxchan's avatar
wxchan committed
575
        return self
wxchan's avatar
wxchan committed
576

577
578
579
580
581
582
583
584
585
586
587
588
589
590
    def _process_params(self, stage: str) -> Dict[str, Any]:
        """Process the parameters of this estimator based on its type, parameter aliases, etc.

        Parameters
        ----------
        stage : str
            Name of the stage (can be ``fit`` or ``predict``) this method is called from.

        Returns
        -------
        processed_params : dict
            Processed parameter names mapped to their values.
        """
        assert stage in {"fit", "predict"}
591
592
593
594
595
        params = self.get_params()

        params.pop('objective', None)
        for alias in _ConfigAliases.get('objective'):
            if alias in params:
596
                obj = params.pop(alias)
597
                _log_warning(f"Found '{alias}' in params. Will use it instead of 'objective' argument")
598
599
600
601
602
603
604
605
606
607
608
609
610
                if stage == "fit":
                    self._objective = obj
        if stage == "fit":
            if self._objective is None:
                if isinstance(self, LGBMRegressor):
                    self._objective = "regression"
                elif isinstance(self, LGBMClassifier):
                    if self._n_classes > 2:
                        self._objective = "multiclass"
                    else:
                        self._objective = "binary"
                elif isinstance(self, LGBMRanker):
                    self._objective = "lambdarank"
611
                else:
612
                    raise ValueError("Unknown LGBMModel type.")
613
        if callable(self._objective):
614
            if stage == "fit":
615
616
617
                params['objective'] = _ObjectiveFunctionWrapper(self._objective)
            else:
                params['objective'] = 'None'
618
        else:
619
            params['objective'] = self._objective
620

621
        params.pop('importance_type', None)
wxchan's avatar
wxchan committed
622
        params.pop('n_estimators', None)
623
        params.pop('class_weight', None)
624

625
626
        if isinstance(params['random_state'], np.random.RandomState):
            params['random_state'] = params['random_state'].randint(np.iinfo(np.int32).max)
627
        if self._n_classes is not None and self._n_classes > 2:
628
629
            for alias in _ConfigAliases.get('num_class'):
                params.pop(alias, None)
630
631
            params['num_class'] = self._n_classes
        if hasattr(self, '_eval_at'):
632
            eval_at = self._eval_at
633
            for alias in _ConfigAliases.get('eval_at'):
634
635
636
637
                if alias in params:
                    _log_warning(f"Found '{alias}' in params. Will use it instead of 'eval_at' argument")
                    eval_at = params.pop(alias)
            params['eval_at'] = eval_at
wxchan's avatar
wxchan committed
638

639
        # register default metric for consistency with callable eval_metric case
640
        original_metric = self._objective if isinstance(self._objective, str) else None
641
642
643
644
645
646
647
648
649
650
        if original_metric is None:
            # try to deduce from class instance
            if isinstance(self, LGBMRegressor):
                original_metric = "l2"
            elif isinstance(self, LGBMClassifier):
                original_metric = "multi_logloss" if self._n_classes > 2 else "binary_logloss"
            elif isinstance(self, LGBMRanker):
                original_metric = "ndcg"

        # overwrite default metric by explicitly set metric
651
        params = _choose_param_value("metric", params, original_metric)
652

653
654
655
656
657
658
        # use joblib conventions for negative n_jobs, just like scikit-learn
        # at predict time, this is handled later due to the order of parameter updates
        if stage == "fit":
            params = _choose_param_value("num_threads", params, self.n_jobs)
            params["num_threads"] = self._process_n_jobs(params["num_threads"])

659
660
        return params

661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
    def _process_n_jobs(self, n_jobs: Optional[int]) -> int:
        """Convert special values of n_jobs to their actual values according to the formulas that apply.

        Parameters
        ----------
        n_jobs : int or None
            The original value of n_jobs, potentially having special values such as 'None' or
            negative integers.

        Returns
        -------
        n_jobs : int
            The value of n_jobs with special values converted to actual number of threads.
        """
        if n_jobs is None:
            n_jobs = _LGBMCpuCount(only_physical_cores=True)
        elif n_jobs < 0:
            n_jobs = max(_LGBMCpuCount(only_physical_cores=False) + 1 + n_jobs, 1)
        return n_jobs

681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
    def fit(
        self,
        X,
        y,
        sample_weight=None,
        init_score=None,
        group=None,
        eval_set=None,
        eval_names=None,
        eval_sample_weight=None,
        eval_class_weight=None,
        eval_init_score=None,
        eval_group=None,
        eval_metric=None,
        feature_name='auto',
        categorical_feature='auto',
        callbacks=None,
        init_model=None
    ):
700
701
702
703
704
705
706
707
708
709
710
711
712
        """Docstring is set after definition, using a template."""
        params = self._process_params(stage="fit")

        # Do not modify original args in fit function
        # Refer to https://github.com/microsoft/LightGBM/pull/2619
        eval_metric_list = copy.deepcopy(eval_metric)
        if not isinstance(eval_metric_list, list):
            eval_metric_list = [eval_metric_list]

        # Separate built-in from callable evaluation metrics
        eval_metrics_callable = [_EvalFunctionWrapper(f) for f in eval_metric_list if callable(f)]
        eval_metrics_builtin = [m for m in eval_metric_list if isinstance(m, str)]

713
        # concatenate metric from params (or default if not provided in params) and eval_metric
714
715
        params['metric'] = [params['metric']] if isinstance(params['metric'], (str, type(None))) else params['metric']
        params['metric'] = [e for e in eval_metrics_builtin if e not in params['metric']] + params['metric']
716
        params['metric'] = [metric for metric in params['metric'] if metric is not None]
wxchan's avatar
wxchan committed
717

718
        if not isinstance(X, (pd_DataFrame, dt_DataTable)):
719
            _X, _y = _LGBMCheckXY(X, y, accept_sparse=True, force_all_finite=False, ensure_min_samples=2)
720
721
            if sample_weight is not None:
                sample_weight = _LGBMCheckSampleWeight(sample_weight, _X)
722
723
        else:
            _X, _y = X, y
724

725
726
727
728
        if self._class_weight is None:
            self._class_weight = self.class_weight
        if self._class_weight is not None:
            class_sample_weight = _LGBMComputeSampleWeight(self._class_weight, y)
729
730
731
732
            if sample_weight is None or len(sample_weight) == 0:
                sample_weight = class_sample_weight
            else:
                sample_weight = np.multiply(sample_weight, class_sample_weight)
733

734
        self._n_features = _X.shape[1]
735
736
        # copy for consistency
        self._n_features_in = self._n_features
737

738
739
740
        train_set = Dataset(data=_X, label=_y, weight=sample_weight, group=group,
                            init_score=init_score, categorical_feature=categorical_feature,
                            params=params)
Guolin Ke's avatar
Guolin Ke committed
741
742
743

        valid_sets = []
        if eval_set is not None:
744

745
            def _get_meta_data(collection, name, i):
746
747
748
749
750
751
752
                if collection is None:
                    return None
                elif isinstance(collection, list):
                    return collection[i] if len(collection) > i else None
                elif isinstance(collection, dict):
                    return collection.get(i, None)
                else:
753
                    raise TypeError(f"{name} should be dict or list")
754

Guolin Ke's avatar
Guolin Ke committed
755
756
757
            if isinstance(eval_set, tuple):
                eval_set = [eval_set]
            for i, valid_data in enumerate(eval_set):
758
                # reduce cost for prediction training data
Guolin Ke's avatar
Guolin Ke committed
759
760
761
                if valid_data[0] is X and valid_data[1] is y:
                    valid_set = train_set
                else:
762
763
764
765
766
767
                    valid_weight = _get_meta_data(eval_sample_weight, 'eval_sample_weight', i)
                    valid_class_weight = _get_meta_data(eval_class_weight, 'eval_class_weight', i)
                    if valid_class_weight is not None:
                        if isinstance(valid_class_weight, dict) and self._class_map is not None:
                            valid_class_weight = {self._class_map[k]: v for k, v in valid_class_weight.items()}
                        valid_class_sample_weight = _LGBMComputeSampleWeight(valid_class_weight, valid_data[1])
768
769
770
771
                        if valid_weight is None or len(valid_weight) == 0:
                            valid_weight = valid_class_sample_weight
                        else:
                            valid_weight = np.multiply(valid_weight, valid_class_sample_weight)
772
773
                    valid_init_score = _get_meta_data(eval_init_score, 'eval_init_score', i)
                    valid_group = _get_meta_data(eval_group, 'eval_group', i)
774
775
776
777
                    valid_set = Dataset(data=valid_data[0], label=valid_data[1], weight=valid_weight,
                                        group=valid_group, init_score=valid_init_score,
                                        categorical_feature='auto', params=params)

Guolin Ke's avatar
Guolin Ke committed
778
779
                valid_sets.append(valid_set)

780
781
782
        if isinstance(init_model, LGBMModel):
            init_model = init_model.booster_

783
784
785
        if callbacks is None:
            callbacks = []
        else:
786
            callbacks = copy.copy(callbacks)  # don't use deepcopy here to allow non-serializable objects
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801

        evals_result = {}
        callbacks.append(record_evaluation(evals_result))

        self._Booster = train(
            params=params,
            train_set=train_set,
            num_boost_round=self.n_estimators,
            valid_sets=valid_sets,
            valid_names=eval_names,
            feval=eval_metrics_callable,
            init_model=init_model,
            feature_name=feature_name,
            callbacks=callbacks
        )
wxchan's avatar
wxchan committed
802

803
        self._evals_result = evals_result
804
        self._best_iteration = self._Booster.best_iteration
805
        self._best_score = self._Booster.best_score
wxchan's avatar
wxchan committed
806

807
808
        self.fitted_ = True

wxchan's avatar
wxchan committed
809
        # free dataset
810
        self._Booster.free_dataset()
wxchan's avatar
wxchan committed
811
        del train_set, valid_sets
wxchan's avatar
wxchan committed
812
813
        return self

814
815
816
817
    fit.__doc__ = _lgbmmodel_doc_fit.format(
        X_shape="array-like or sparse matrix of shape = [n_samples, n_features]",
        y_shape="array-like of shape = [n_samples]",
        sample_weight_shape="array-like of shape = [n_samples] or None, optional (default=None)",
818
        init_score_shape="array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task) or shape = [n_samples, n_classes] (for multi-class task) or None, optional (default=None)",
819
        group_shape="array-like or None, optional (default=None)",
820
821
822
        eval_sample_weight_shape="list of array, or None, optional (default=None)",
        eval_init_score_shape="list of array, or None, optional (default=None)",
        eval_group_shape="list of array, or None, optional (default=None)"
823
824
    ) + "\n\n" + _lgbmmodel_doc_custom_eval_note

825
826
827
828
829
830
831
832
833
834
835
    def predict(
        self,
        X,
        raw_score: bool = False,
        start_iteration: int = 0,
        num_iteration: Optional[int] = None,
        pred_leaf: bool = False,
        pred_contrib: bool = False,
        validate_features: bool = False,
        **kwargs: Any
    ):
836
        """Docstring is set after definition, using a template."""
837
        if not self.__sklearn_is_fitted__():
838
            raise LGBMNotFittedError("Estimator not fitted, call fit before exploiting the model.")
839
        if not isinstance(X, (pd_DataFrame, dt_DataTable)):
840
            X = _LGBMCheckArray(X, accept_sparse=True, force_all_finite=False)
841
842
843
        n_features = X.shape[1]
        if self._n_features != n_features:
            raise ValueError("Number of features of the model must "
844
845
                             f"match the input. Model n_features_ is {self._n_features} and "
                             f"input n_features is {n_features}")
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
        # retrive original params that possibly can be used in both training and prediction
        # and then overwrite them (considering aliases) with params that were passed directly in prediction
        predict_params = self._process_params(stage="predict")
        for alias in _ConfigAliases.get_by_alias(
            "data",
            "X",
            "raw_score",
            "start_iteration",
            "num_iteration",
            "pred_leaf",
            "pred_contrib",
            *kwargs.keys()
        ):
            predict_params.pop(alias, None)
        predict_params.update(kwargs)
861
862
863

        # number of threads can have values with special meaning which is only applied
        # in the scikit-learn interface, these should not reach the c++ side as-is
864
865
        predict_params = _choose_param_value("num_threads", predict_params, self.n_jobs)
        predict_params["num_threads"] = self._process_n_jobs(predict_params["num_threads"])
866

867
        return self._Booster.predict(X, raw_score=raw_score, start_iteration=start_iteration, num_iteration=num_iteration,
868
869
                                     pred_leaf=pred_leaf, pred_contrib=pred_contrib, validate_features=validate_features,
                                     **predict_params)
wxchan's avatar
wxchan committed
870

871
872
873
874
875
876
877
878
879
    predict.__doc__ = _lgbmmodel_doc_predict.format(
        description="Return the predicted value for each sample.",
        X_shape="array-like or sparse matrix of shape = [n_samples, n_features]",
        output_name="predicted_result",
        predicted_result_shape="array-like of shape = [n_samples] or shape = [n_samples, n_classes]",
        X_leaves_shape="array-like of shape = [n_samples, n_trees] or shape = [n_samples, n_trees * n_classes]",
        X_SHAP_values_shape="array-like of shape = [n_samples, n_features + 1] or shape = [n_samples, (n_features + 1) * n_classes] or list with n_classes length of such objects"
    )

880
    @property
881
    def n_features_(self) -> int:
882
        """:obj:`int`: The number of features of fitted model."""
883
        if not self.__sklearn_is_fitted__():
884
885
886
            raise LGBMNotFittedError('No n_features found. Need to call fit beforehand.')
        return self._n_features

887
    @property
888
    def n_features_in_(self) -> int:
889
        """:obj:`int`: The number of features of fitted model."""
890
        if not self.__sklearn_is_fitted__():
891
892
893
            raise LGBMNotFittedError('No n_features_in found. Need to call fit beforehand.')
        return self._n_features_in

894
895
    @property
    def best_score_(self):
896
        """:obj:`dict`: The best score of fitted model."""
897
        if not self.__sklearn_is_fitted__():
898
899
900
901
            raise LGBMNotFittedError('No best_score found. Need to call fit beforehand.')
        return self._best_score

    @property
902
    def best_iteration_(self) -> int:
903
        """:obj:`int`: The best iteration of fitted model if ``early_stopping()`` callback has been specified."""
904
        if not self.__sklearn_is_fitted__():
905
            raise LGBMNotFittedError('No best_iteration found. Need to call fit with early_stopping callback beforehand.')
906
907
908
        return self._best_iteration

    @property
909
    def objective_(self) -> Union[str, _LGBM_ScikitCustomObjectiveFunction]:
910
        """:obj:`str` or :obj:`callable`: The concrete objective used while fitting this model."""
911
        if not self.__sklearn_is_fitted__():
912
913
914
            raise LGBMNotFittedError('No objective found. Need to call fit beforehand.')
        return self._objective

915
916
917
918
919
920
921
922
923
    @property
    def n_estimators_(self) -> int:
        """:obj:`int`: True number of boosting iterations performed.

        This might be less than parameter ``n_estimators`` if early stopping was enabled or
        if boosting stopped early due to limits on complexity like ``min_gain_to_split``.
        """
        if not self.__sklearn_is_fitted__():
            raise LGBMNotFittedError('No n_estimators found. Need to call fit beforehand.')
924
        return self._Booster.current_iteration()  # type: ignore
925
926
927
928
929
930
931
932
933
934

    @property
    def n_iter_(self) -> int:
        """:obj:`int`: True number of boosting iterations performed.

        This might be less than parameter ``n_estimators`` if early stopping was enabled or
        if boosting stopped early due to limits on complexity like ``min_gain_to_split``.
        """
        if not self.__sklearn_is_fitted__():
            raise LGBMNotFittedError('No n_iter found. Need to call fit beforehand.')
935
        return self._Booster.current_iteration()  # type: ignore
936

937
938
    @property
    def booster_(self):
939
        """Booster: The underlying Booster of this model."""
940
        if not self.__sklearn_is_fitted__():
941
            raise LGBMNotFittedError('No booster found. Need to call fit beforehand.')
942
        return self._Booster
wxchan's avatar
wxchan committed
943

944
945
    @property
    def evals_result_(self):
946
        """:obj:`dict`: The evaluation results if validation sets have been specified."""
947
        if not self.__sklearn_is_fitted__():
948
949
            raise LGBMNotFittedError('No results found. Need to call fit with eval_set beforehand.')
        return self._evals_result
950
951

    @property
952
    def feature_importances_(self):
953
        """:obj:`array` of shape = [n_features]: The feature importances (the higher, the more important).
954

Nikita Titov's avatar
Nikita Titov committed
955
956
957
958
        .. note::

            ``importance_type`` attribute is passed to the function
            to configure the type of importance values to be extracted.
959
        """
960
        if not self.__sklearn_is_fitted__():
961
            raise LGBMNotFittedError('No feature_importances found. Need to call fit beforehand.')
962
        return self._Booster.feature_importance(importance_type=self.importance_type)
wxchan's avatar
wxchan committed
963

964
965
    @property
    def feature_name_(self):
966
        """:obj:`array` of shape = [n_features]: The names of features."""
967
        if not self.__sklearn_is_fitted__():
968
969
970
            raise LGBMNotFittedError('No feature_name found. Need to call fit beforehand.')
        return self._Booster.feature_name()

wxchan's avatar
wxchan committed
971

972
class LGBMRegressor(_LGBMRegressorBase, LGBMModel):
973
    """LightGBM regressor."""
wxchan's avatar
wxchan committed
974

975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
    def fit(
        self,
        X,
        y,
        sample_weight=None,
        init_score=None,
        eval_set=None,
        eval_names=None,
        eval_sample_weight=None,
        eval_init_score=None,
        eval_metric=None,
        feature_name='auto',
        categorical_feature='auto',
        callbacks=None,
        init_model=None
    ):
991
        """Docstring is inherited from the LGBMModel."""
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
        super().fit(
            X,
            y,
            sample_weight=sample_weight,
            init_score=init_score,
            eval_set=eval_set,
            eval_names=eval_names,
            eval_sample_weight=eval_sample_weight,
            eval_init_score=eval_init_score,
            eval_metric=eval_metric,
            feature_name=feature_name,
            categorical_feature=categorical_feature,
            callbacks=callbacks,
            init_model=init_model
        )
Guolin Ke's avatar
Guolin Ke committed
1007
1008
        return self

1009
    _base_doc = LGBMModel.fit.__doc__.replace("self : LGBMModel", "self : LGBMRegressor")  # type: ignore
1010
1011
    _base_doc = (_base_doc[:_base_doc.find('group :')]  # type: ignore
                 + _base_doc[_base_doc.find('eval_set :'):])  # type: ignore
1012
1013
1014
1015
    _base_doc = (_base_doc[:_base_doc.find('eval_class_weight :')]
                 + _base_doc[_base_doc.find('eval_init_score :'):])
    fit.__doc__ = (_base_doc[:_base_doc.find('eval_group :')]
                   + _base_doc[_base_doc.find('eval_metric :'):])
wxchan's avatar
wxchan committed
1016

1017

1018
class LGBMClassifier(_LGBMClassifierBase, LGBMModel):
1019
    """LightGBM classifier."""
wxchan's avatar
wxchan committed
1020

1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
    def fit(
        self,
        X,
        y,
        sample_weight=None,
        init_score=None,
        eval_set=None,
        eval_names=None,
        eval_sample_weight=None,
        eval_class_weight=None,
        eval_init_score=None,
        eval_metric=None,
        feature_name='auto',
        categorical_feature='auto',
        callbacks=None,
        init_model=None
    ):
1038
        """Docstring is inherited from the LGBMModel."""
1039
        _LGBMAssertAllFinite(y)
1040
1041
        _LGBMCheckClassificationTargets(y)
        self._le = _LGBMLabelEncoder().fit(y)
1042
        _y = self._le.transform(y)
1043
        self._class_map = dict(zip(self._le.classes_, self._le.transform(self._le.classes_)))
1044
1045
        if isinstance(self.class_weight, dict):
            self._class_weight = {self._class_map[k]: v for k, v in self.class_weight.items()}
1046

1047
1048
        self._classes = self._le.classes_
        self._n_classes = len(self._classes)
1049
1050

        if not callable(eval_metric):
1051
            if isinstance(eval_metric, (str, type(None))):
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
                eval_metric = [eval_metric]
            if self._n_classes > 2:
                for index, metric in enumerate(eval_metric):
                    if metric in {'logloss', 'binary_logloss'}:
                        eval_metric[index] = "multi_logloss"
                    elif metric in {'error', 'binary_error'}:
                        eval_metric[index] = "multi_error"
            else:
                for index, metric in enumerate(eval_metric):
                    if metric in {'logloss', 'multi_logloss'}:
                        eval_metric[index] = 'binary_logloss'
                    elif metric in {'error', 'multi_error'}:
                        eval_metric[index] = 'binary_error'
wxchan's avatar
wxchan committed
1065

1066
1067
        # do not modify args, as it causes errors in model selection tools
        valid_sets = None
wxchan's avatar
wxchan committed
1068
        if eval_set is not None:
1069
1070
            if isinstance(eval_set, tuple):
                eval_set = [eval_set]
1071
            valid_sets = [None] * len(eval_set)
1072
1073
            for i, (valid_x, valid_y) in enumerate(eval_set):
                if valid_x is X and valid_y is y:
1074
                    valid_sets[i] = (valid_x, _y)
1075
                else:
1076
                    valid_sets[i] = (valid_x, self._le.transform(valid_y))
1077

1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
        super().fit(
            X,
            _y,
            sample_weight=sample_weight,
            init_score=init_score,
            eval_set=valid_sets,
            eval_names=eval_names,
            eval_sample_weight=eval_sample_weight,
            eval_class_weight=eval_class_weight,
            eval_init_score=eval_init_score,
            eval_metric=eval_metric,
            feature_name=feature_name,
            categorical_feature=categorical_feature,
            callbacks=callbacks,
            init_model=init_model
        )
wxchan's avatar
wxchan committed
1094
1095
        return self

1096
    _base_doc = LGBMModel.fit.__doc__.replace("self : LGBMModel", "self : LGBMClassifier")  # type: ignore
1097
1098
    _base_doc = (_base_doc[:_base_doc.find('group :')]  # type: ignore
                 + _base_doc[_base_doc.find('eval_set :'):])  # type: ignore
1099
1100
    fit.__doc__ = (_base_doc[:_base_doc.find('eval_group :')]
                   + _base_doc[_base_doc.find('eval_metric :'):])
1101

1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
    def predict(
        self,
        X,
        raw_score: bool = False,
        start_iteration: int = 0,
        num_iteration: Optional[int] = None,
        pred_leaf: bool = False,
        pred_contrib: bool = False,
        validate_features: bool = False,
        **kwargs: Any
    ):
1113
        """Docstring is inherited from the LGBMModel."""
1114
        result = self.predict_proba(X, raw_score, start_iteration, num_iteration,
1115
1116
                                    pred_leaf, pred_contrib, validate_features,
                                    **kwargs)
1117
        if callable(self._objective) or raw_score or pred_leaf or pred_contrib:
1118
1119
1120
1121
            return result
        else:
            class_index = np.argmax(result, axis=1)
            return self._le.inverse_transform(class_index)
wxchan's avatar
wxchan committed
1122

1123
1124
    predict.__doc__ = LGBMModel.predict.__doc__

1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
    def predict_proba(
        self,
        X,
        raw_score: bool = False,
        start_iteration: int = 0,
        num_iteration: Optional[int] = None,
        pred_leaf: bool = False,
        pred_contrib: bool = False,
        validate_features: bool = False,
        **kwargs: Any
    ):
1136
        """Docstring is set after definition, using a template."""
1137
        result = super().predict(X, raw_score, start_iteration, num_iteration, pred_leaf, pred_contrib, validate_features, **kwargs)
1138
        if callable(self._objective) and not (raw_score or pred_leaf or pred_contrib):
1139
1140
1141
            _log_warning("Cannot compute class probabilities or labels "
                         "due to the usage of customized objective function.\n"
                         "Returning raw scores instead.")
1142
1143
            return result
        elif self._n_classes > 2 or raw_score or pred_leaf or pred_contrib:
1144
            return result
wxchan's avatar
wxchan committed
1145
        else:
1146
            return np.vstack((1. - result, result)).transpose()
1147

1148
1149
1150
1151
    predict_proba.__doc__ = _lgbmmodel_doc_predict.format(
        description="Return the predicted probability for each class for each sample.",
        X_shape="array-like or sparse matrix of shape = [n_samples, n_features]",
        output_name="predicted_probability",
1152
        predicted_result_shape="array-like of shape = [n_samples] or shape = [n_samples, n_classes]",
1153
1154
1155
1156
        X_leaves_shape="array-like of shape = [n_samples, n_trees] or shape = [n_samples, n_trees * n_classes]",
        X_SHAP_values_shape="array-like of shape = [n_samples, n_features + 1] or shape = [n_samples, (n_features + 1) * n_classes] or list with n_classes length of such objects"
    )

1157
1158
    @property
    def classes_(self):
1159
        """:obj:`array` of shape = [n_classes]: The class label array."""
1160
        if not self.__sklearn_is_fitted__():
1161
1162
            raise LGBMNotFittedError('No classes found. Need to call fit beforehand.')
        return self._classes
1163
1164

    @property
1165
    def n_classes_(self) -> int:
1166
        """:obj:`int`: The number of classes."""
1167
        if not self.__sklearn_is_fitted__():
1168
1169
            raise LGBMNotFittedError('No classes found. Need to call fit beforehand.')
        return self._n_classes
wxchan's avatar
wxchan committed
1170

wxchan's avatar
wxchan committed
1171

wxchan's avatar
wxchan committed
1172
class LGBMRanker(LGBMModel):
1173
1174
1175
1176
1177
1178
1179
1180
    """LightGBM ranker.

    .. warning::

        scikit-learn doesn't support ranking applications yet,
        therefore this class is not really compatible with the sklearn ecosystem.
        Please use this class mainly for training and applying ranking models in common sklearnish way.
    """
wxchan's avatar
wxchan committed
1181

1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
    def fit(
        self,
        X,
        y,
        sample_weight=None,
        init_score=None,
        group=None,
        eval_set=None,
        eval_names=None,
        eval_sample_weight=None,
        eval_init_score=None,
        eval_group=None,
        eval_metric=None,
        eval_at=(1, 2, 3, 4, 5),
        feature_name='auto',
        categorical_feature='auto',
        callbacks=None,
        init_model=None
    ):
1201
        """Docstring is inherited from the LGBMModel."""
1202
        # check group data
Guolin Ke's avatar
Guolin Ke committed
1203
        if group is None:
1204
            raise ValueError("Should set group for ranking task")
wxchan's avatar
wxchan committed
1205
1206

        if eval_set is not None:
Guolin Ke's avatar
Guolin Ke committed
1207
            if eval_group is None:
1208
                raise ValueError("Eval_group cannot be None when eval_set is not None")
Guolin Ke's avatar
Guolin Ke committed
1209
            elif len(eval_group) != len(eval_set):
1210
                raise ValueError("Length of eval_group should be equal to eval_set")
1211
            elif (isinstance(eval_group, dict)
1212
                  and any(i not in eval_group or eval_group[i] is None for i in range(len(eval_group)))
1213
1214
                  or isinstance(eval_group, list)
                  and any(group is None for group in eval_group)):
1215
1216
                raise ValueError("Should set group for all eval datasets for ranking task; "
                                 "if you use dict, the index should start from 0")
1217

1218
        self._eval_at = eval_at
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
        super().fit(
            X,
            y,
            sample_weight=sample_weight,
            init_score=init_score,
            group=group,
            eval_set=eval_set,
            eval_names=eval_names,
            eval_sample_weight=eval_sample_weight,
            eval_init_score=eval_init_score,
            eval_group=eval_group,
            eval_metric=eval_metric,
            feature_name=feature_name,
            categorical_feature=categorical_feature,
            callbacks=callbacks,
            init_model=init_model
        )
wxchan's avatar
wxchan committed
1236
        return self
1237

1238
    _base_doc = LGBMModel.fit.__doc__.replace("self : LGBMModel", "self : LGBMRanker")  # type: ignore
1239
1240
    fit.__doc__ = (_base_doc[:_base_doc.find('eval_class_weight :')]  # type: ignore
                   + _base_doc[_base_doc.find('eval_init_score :'):])  # type: ignore
1241
    _base_doc = fit.__doc__
1242
1243
    _before_feature_name, _feature_name, _after_feature_name = _base_doc.partition('feature_name :')
    fit.__doc__ = f"""{_before_feature_name}eval_at : iterable of int, optional (default=(1, 2, 3, 4, 5))
1244
        The evaluation positions of the specified metric.
1245
    {_feature_name}{_after_feature_name}"""