sklearn.py 44.3 KB
Newer Older
wxchan's avatar
wxchan committed
1
# coding: utf-8
2
# pylint: disable = invalid-name, W0105, C0111, C0301
3
"""Scikit-learn wrapper interface for LightGBM."""
wxchan's avatar
wxchan committed
4
from __future__ import absolute_import
5

wxchan's avatar
wxchan committed
6
import numpy as np
7

8
from .basic import Dataset, LightGBMError, _ConfigAliases
9
from .compat import (SKLEARN_INSTALLED, _LGBMClassifierBase,
10
11
                     LGBMNotFittedError, _LGBMLabelEncoder, _LGBMModelBase,
                     _LGBMRegressorBase, _LGBMCheckXY, _LGBMCheckArray, _LGBMCheckConsistentLength,
12
                     _LGBMAssertAllFinite, _LGBMCheckClassificationTargets, _LGBMComputeSampleWeight,
13
                     argc_, range_, zip_, string_type, DataFrame, DataTable)
wxchan's avatar
wxchan committed
14
from .engine import train
15

wxchan's avatar
wxchan committed
16

17
18
class _ObjectiveFunctionWrapper(object):
    """Proxy class for objective function."""
19

20
21
    def __init__(self, func):
        """Construct a proxy class.
22

23
24
        This class transforms objective function to match objective function with signature ``new_func(preds, dataset)``
        as expected by ``lightgbm.engine.train``.
25

26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
        Parameters
        ----------
        func : callable
            Expects a callable with signature ``func(y_true, y_pred)`` or ``func(y_true, y_pred, group)
            and returns (grad, hess):

                y_true : array-like of shape = [n_samples]
                    The target values.
                y_pred : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
                    The predicted values.
                group : array-like
                    Group/query data, used for ranking task.
                grad : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
                    The value of the first order derivative (gradient) for each sample point.
                hess : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
                    The value of the second order derivative (Hessian) for each sample point.
wxchan's avatar
wxchan committed
42

Nikita Titov's avatar
Nikita Titov committed
43
44
45
46
47
        .. note::

            For multi-class task, the y_pred is group by class_id first, then group by row_id.
            If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i]
            and you should group grad and hess in this way as well.
48
49
        """
        self.func = func
wxchan's avatar
wxchan committed
50

51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
    def __call__(self, preds, dataset):
        """Call passed function with appropriate arguments.

        Parameters
        ----------
        preds : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
            The predicted values.
        dataset : Dataset
            The training dataset.

        Returns
        -------
        grad : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
            The value of the first order derivative (gradient) for each sample point.
        hess : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
            The value of the second order derivative (Hessian) for each sample point.
        """
wxchan's avatar
wxchan committed
68
        labels = dataset.get_label()
69
        argc = argc_(self.func)
70
        if argc == 2:
71
            grad, hess = self.func(labels, preds)
72
        elif argc == 3:
73
            grad, hess = self.func(labels, preds, dataset.get_group())
74
        else:
wxchan's avatar
wxchan committed
75
            raise TypeError("Self-defined objective function should have 2 or 3 arguments, got %d" % argc)
wxchan's avatar
wxchan committed
76
77
78
79
80
81
82
83
84
85
86
        """weighted for objective"""
        weight = dataset.get_weight()
        if weight is not None:
            """only one class"""
            if len(weight) == len(grad):
                grad = np.multiply(grad, weight)
                hess = np.multiply(hess, weight)
            else:
                num_data = len(weight)
                num_class = len(grad) // num_data
                if num_class * num_data != len(grad):
87
                    raise ValueError("Length of grad and hess should equal to num_class * num_data")
wxchan's avatar
wxchan committed
88
89
                for k in range_(num_class):
                    for i in range_(num_data):
wxchan's avatar
wxchan committed
90
91
92
93
94
                        idx = k * num_data + i
                        grad[idx] *= weight[i]
                        hess[idx] *= weight[i]
        return grad, hess

wxchan's avatar
wxchan committed
95

96
97
class _EvalFunctionWrapper(object):
    """Proxy class for evaluation function."""
98

99
100
    def __init__(self, func):
        """Construct a proxy class.
101

102
103
        This class transforms evaluation function to match evaluation function with signature ``new_func(preds, dataset)``
        as expected by ``lightgbm.engine.train``.
104

105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
        Parameters
        ----------
        func : callable
            Expects a callable with following signatures:
            ``func(y_true, y_pred)``,
            ``func(y_true, y_pred, weight)``
            or ``func(y_true, y_pred, weight, group)``
            and returns (eval_name, eval_result, is_higher_better) or
            list of (eval_name, eval_result, is_higher_better):

                y_true : array-like of shape = [n_samples]
                    The target values.
                y_pred : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
                    The predicted values.
                weight : array-like of shape = [n_samples]
                    The weight of samples.
                group : array-like
                    Group/query data, used for ranking task.
                eval_name : string
124
                    The name of evaluation function (without whitespaces).
125
126
127
128
129
                eval_result : float
                    The eval result.
                is_higher_better : bool
                    Is eval result higher better, e.g. AUC is ``is_higher_better``.

Nikita Titov's avatar
Nikita Titov committed
130
131
132
133
        .. note::

            For multi-class task, the y_pred is group by class_id first, then group by row_id.
            If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i].
134
135
        """
        self.func = func
136

137
138
    def __call__(self, preds, dataset):
        """Call passed function with appropriate arguments.
139

140
141
142
143
144
145
146
147
148
149
        Parameters
        ----------
        preds : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
            The predicted values.
        dataset : Dataset
            The training dataset.

        Returns
        -------
        eval_name : string
150
            The name of evaluation function (without whitespaces).
151
152
153
154
155
        eval_result : float
            The eval result.
        is_higher_better : bool
            Is eval result higher better, e.g. AUC is ``is_higher_better``.
        """
156
        labels = dataset.get_label()
157
        argc = argc_(self.func)
158
        if argc == 2:
159
            return self.func(labels, preds)
160
        elif argc == 3:
161
            return self.func(labels, preds, dataset.get_weight())
162
        elif argc == 4:
163
            return self.func(labels, preds, dataset.get_weight(), dataset.get_group())
164
        else:
wxchan's avatar
wxchan committed
165
            raise TypeError("Self-defined eval function should have 2, 3 or 4 arguments, got %d" % argc)
166

wxchan's avatar
wxchan committed
167

168
169
class LGBMModel(_LGBMModelBase):
    """Implementation of the scikit-learn API for LightGBM."""
wxchan's avatar
wxchan committed
170

171
    def __init__(self, boosting_type='gbdt', num_leaves=31, max_depth=-1,
172
                 learning_rate=0.1, n_estimators=100,
173
                 subsample_for_bin=200000, objective=None, class_weight=None,
174
                 min_split_gain=0., min_child_weight=1e-3, min_child_samples=20,
175
                 subsample=1., subsample_freq=0, colsample_bytree=1.,
176
                 reg_alpha=0., reg_lambda=0., random_state=None,
177
                 n_jobs=-1, silent=True, importance_type='split', **kwargs):
178
        r"""Construct a gradient boosting model.
wxchan's avatar
wxchan committed
179
180
181

        Parameters
        ----------
182
        boosting_type : string, optional (default='gbdt')
183
184
185
186
187
            'gbdt', traditional Gradient Boosting Decision Tree.
            'dart', Dropouts meet Multiple Additive Regression Trees.
            'goss', Gradient-based One-Side Sampling.
            'rf', Random Forest.
        num_leaves : int, optional (default=31)
wxchan's avatar
wxchan committed
188
            Maximum tree leaves for base learners.
189
        max_depth : int, optional (default=-1)
190
            Maximum tree depth for base learners, <=0 means no limit.
191
        learning_rate : float, optional (default=0.1)
192
            Boosting learning rate.
193
194
195
            You can use ``callbacks`` parameter of ``fit`` method to shrink/adapt learning rate
            in training using ``reset_parameter`` callback.
            Note, that this will ignore the ``learning_rate`` argument in training.
196
        n_estimators : int, optional (default=100)
wxchan's avatar
wxchan committed
197
            Number of boosted trees to fit.
198
        subsample_for_bin : int, optional (default=200000)
wxchan's avatar
wxchan committed
199
            Number of samples for constructing bins.
200
        objective : string, callable or None, optional (default=None)
wxchan's avatar
wxchan committed
201
202
            Specify the learning task and the corresponding learning objective or
            a custom objective function to be used (see note below).
203
            Default: 'regression' for LGBMRegressor, 'binary' or 'multiclass' for LGBMClassifier, 'lambdarank' for LGBMRanker.
204
205
206
207
        class_weight : dict, 'balanced' or None, optional (default=None)
            Weights associated with classes in the form ``{class_label: weight}``.
            Use this parameter only for multi-class classification task;
            for binary classification task you may use ``is_unbalance`` or ``scale_pos_weight`` parameters.
208
209
210
            Note, that the usage of all these parameters will result in poor estimates of the individual class probabilities.
            You may want to consider performing probability calibration
            (https://scikit-learn.org/stable/modules/calibration.html) of your model.
211
212
213
            The 'balanced' mode uses the values of y to automatically adjust weights
            inversely proportional to class frequencies in the input data as ``n_samples / (n_classes * np.bincount(y))``.
            If None, all classes are supposed to have weight one.
214
            Note, that these weights will be multiplied with ``sample_weight`` (passed through the ``fit`` method)
215
            if ``sample_weight`` is specified.
216
        min_split_gain : float, optional (default=0.)
wxchan's avatar
wxchan committed
217
            Minimum loss reduction required to make a further partition on a leaf node of the tree.
218
        min_child_weight : float, optional (default=1e-3)
219
            Minimum sum of instance weight (hessian) needed in a child (leaf).
220
        min_child_samples : int, optional (default=20)
221
            Minimum number of data needed in a child (leaf).
222
        subsample : float, optional (default=1.)
wxchan's avatar
wxchan committed
223
            Subsample ratio of the training instance.
224
        subsample_freq : int, optional (default=0)
225
226
            Frequence of subsample, <=0 means no enable.
        colsample_bytree : float, optional (default=1.)
wxchan's avatar
wxchan committed
227
            Subsample ratio of columns when constructing each tree.
228
        reg_alpha : float, optional (default=0.)
229
            L1 regularization term on weights.
230
        reg_lambda : float, optional (default=0.)
231
            L2 regularization term on weights.
232
        random_state : int or None, optional (default=None)
wxchan's avatar
wxchan committed
233
            Random number seed.
234
            If None, default seeds in C++ code will be used.
235
        n_jobs : int, optional (default=-1)
236
            Number of parallel threads.
237
        silent : bool, optional (default=True)
wxchan's avatar
wxchan committed
238
            Whether to print messages while running boosting.
239
        importance_type : string, optional (default='split')
240
            The type of feature importance to be filled into ``feature_importances_``.
241
242
243
244
            If 'split', result contains numbers of times the feature is used in a model.
            If 'gain', result contains total gains of splits which use the feature.
        **kwargs
            Other parameters for the model.
wxchan's avatar
wxchan committed
245
            Check http://lightgbm.readthedocs.io/en/latest/Parameters.html for more parameters.
246

Nikita Titov's avatar
Nikita Titov committed
247
248
249
            .. warning::

                \*\*kwargs is not supported in sklearn, it may cause unexpected issues.
wxchan's avatar
wxchan committed
250

251
252
253
254
255
256
257
258
259
        Attributes
        ----------
        n_features_ : int
            The number of features of fitted model.
        classes_ : array of shape = [n_classes]
            The class label array (only for classification problem).
        n_classes_ : int
            The number of classes (only for classification problem).
        best_score_ : dict or None
260
            The best score of fitted model.
261
        best_iteration_ : int or None
262
            The best iteration of fitted model if ``early_stopping_rounds`` has been specified.
263
264
265
266
267
        objective_ : string or callable
            The concrete objective used while fitting this model.
        booster_ : Booster
            The underlying Booster of this model.
        evals_result_ : dict or None
268
            The evaluation results if ``early_stopping_rounds`` has been specified.
269
270
271
        feature_importances_ : array of shape = [n_features]
            The feature importances (the higher, the more important the feature).

wxchan's avatar
wxchan committed
272
273
        Note
        ----
274
275
        A custom objective function can be provided for the ``objective`` parameter.
        In this case, it should have the signature
276
277
        ``objective(y_true, y_pred) -> grad, hess`` or
        ``objective(y_true, y_pred, group) -> grad, hess``:
wxchan's avatar
wxchan committed
278

Nikita Titov's avatar
Nikita Titov committed
279
            y_true : array-like of shape = [n_samples]
280
                The target values.
Nikita Titov's avatar
Nikita Titov committed
281
            y_pred : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
282
                The predicted values.
Nikita Titov's avatar
Nikita Titov committed
283
            group : array-like
284
                Group/query data, used for ranking task.
Nikita Titov's avatar
Nikita Titov committed
285
            grad : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
286
                The value of the first order derivative (gradient) for each sample point.
Nikita Titov's avatar
Nikita Titov committed
287
            hess : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
288
                The value of the second order derivative (Hessian) for each sample point.
wxchan's avatar
wxchan committed
289

290
291
292
        For multi-class task, the y_pred is group by class_id first, then group by row_id.
        If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i]
        and you should group grad and hess in this way as well.
wxchan's avatar
wxchan committed
293
        """
wxchan's avatar
wxchan committed
294
        if not SKLEARN_INSTALLED:
295
            raise LightGBMError('Scikit-learn is required for this module')
wxchan's avatar
wxchan committed
296

297
        self.boosting_type = boosting_type
298
        self.objective = objective
wxchan's avatar
wxchan committed
299
300
301
302
        self.num_leaves = num_leaves
        self.max_depth = max_depth
        self.learning_rate = learning_rate
        self.n_estimators = n_estimators
wxchan's avatar
wxchan committed
303
        self.subsample_for_bin = subsample_for_bin
wxchan's avatar
wxchan committed
304
305
306
307
308
309
310
311
        self.min_split_gain = min_split_gain
        self.min_child_weight = min_child_weight
        self.min_child_samples = min_child_samples
        self.subsample = subsample
        self.subsample_freq = subsample_freq
        self.colsample_bytree = colsample_bytree
        self.reg_alpha = reg_alpha
        self.reg_lambda = reg_lambda
312
313
        self.random_state = random_state
        self.n_jobs = n_jobs
wxchan's avatar
wxchan committed
314
        self.silent = silent
315
        self.importance_type = importance_type
wxchan's avatar
wxchan committed
316
        self._Booster = None
317
318
319
320
        self._evals_result = None
        self._best_score = None
        self._best_iteration = None
        self._other_params = {}
321
        self._objective = objective
322
        self.class_weight = class_weight
323
324
        self._class_weight = None
        self._class_map = None
325
326
327
        self._n_features = None
        self._classes = None
        self._n_classes = None
328
        self.set_params(**kwargs)
wxchan's avatar
wxchan committed
329

Nikita Titov's avatar
Nikita Titov committed
330
331
332
333
    def _more_tags(self):
        return {'allow_nan': True,
                'X_types': ['2darray', 'sparse', '1dlabels']}

wxchan's avatar
wxchan committed
334
    def get_params(self, deep=True):
335
336
337
338
339
340
341
342
343
344
345
346
347
        """Get parameters for this estimator.

        Parameters
        ----------
        deep : bool, optional (default=True)
            If True, will return the parameters for this estimator and
            contained subobjects that are estimators.

        Returns
        -------
        params : dict
            Parameter names mapped to their values.
        """
wxchan's avatar
wxchan committed
348
        params = super(LGBMModel, self).get_params(deep=deep)
349
        params.update(self._other_params)
wxchan's avatar
wxchan committed
350
351
352
353
        return params

    # minor change to support `**kwargs`
    def set_params(self, **params):
354
355
356
357
358
359
360
361
362
363
364
365
        """Set the parameters of this estimator.

        Parameters
        ----------
        **params
            Parameter names with their new values.

        Returns
        -------
        self : object
            Returns self.
        """
wxchan's avatar
wxchan committed
366
367
        for key, value in params.items():
            setattr(self, key, value)
368
369
            if hasattr(self, '_' + key):
                setattr(self, '_' + key, value)
370
            self._other_params[key] = value
wxchan's avatar
wxchan committed
371
        return self
wxchan's avatar
wxchan committed
372

Guolin Ke's avatar
Guolin Ke committed
373
    def fit(self, X, y,
374
            sample_weight=None, init_score=None, group=None,
375
            eval_set=None, eval_names=None, eval_sample_weight=None,
376
377
378
            eval_class_weight=None, eval_init_score=None, eval_group=None,
            eval_metric=None, early_stopping_rounds=None, verbose=True,
            feature_name='auto', categorical_feature='auto', callbacks=None):
379
        """Build a gradient boosting model from the training set (X, y).
wxchan's avatar
wxchan committed
380
381
382

        Parameters
        ----------
383
384
385
386
387
388
389
390
        X : array-like or sparse matrix of shape = [n_samples, n_features]
            Input feature matrix.
        y : array-like of shape = [n_samples]
            The target values (class labels in classification, real numbers in regression).
        sample_weight : array-like of shape = [n_samples] or None, optional (default=None)
            Weights of training data.
        init_score : array-like of shape = [n_samples] or None, optional (default=None)
            Init score of training data.
391
        group : array-like or None, optional (default=None)
392
393
            Group data of training data.
        eval_set : list or None, optional (default=None)
394
            A list of (X, y) tuple pairs to use as validation sets.
395
        eval_names : list of strings or None, optional (default=None)
396
397
398
            Names of eval_set.
        eval_sample_weight : list of arrays or None, optional (default=None)
            Weights of eval data.
399
400
        eval_class_weight : list or None, optional (default=None)
            Class weights of eval data.
401
402
403
404
405
406
        eval_init_score : list of arrays or None, optional (default=None)
            Init score of eval data.
        eval_group : list of arrays or None, optional (default=None)
            Group data of eval data.
        eval_metric : string, list of strings, callable or None, optional (default=None)
            If string, it should be a built-in evaluation metric to use.
407
            If callable, it should be a custom evaluation metric, see note below for more details.
Misha Lisovyi's avatar
Misha Lisovyi committed
408
            In either case, the ``metric`` from the model parameters will be evaluated and used as well.
409
            Default: 'l2' for LGBMRegressor, 'logloss' for LGBMClassifier, 'ndcg' for LGBMRanker.
410
411
        early_stopping_rounds : int or None, optional (default=None)
            Activates early stopping. The model will train until the validation score stops improving.
412
            Validation score needs to improve at least every ``early_stopping_rounds`` round(s)
413
            to continue training.
414
415
            Requires at least one validation data and one metric.
            If there's more than one, will check all of them. But the training data is ignored anyway.
416
417
            To check only the first metric, set the ``first_metric_only`` parameter to ``True``
            in additional parameters ``**kwargs`` of the model constructor.
418
419
420
421
422
423
        verbose : bool or int, optional (default=True)
            Requires at least one evaluation data.
            If True, the eval metric on the eval set is printed at each boosting stage.
            If int, the eval metric on the eval set is printed at every ``verbose`` boosting stage.
            The last boosting stage or the boosting stage found by using ``early_stopping_rounds`` is also printed.

Nikita Titov's avatar
Nikita Titov committed
424
425
            .. rubric:: Example

426
427
428
            With ``verbose`` = 4 and at least one item in ``eval_set``,
            an evaluation metric is printed every 4 (instead of 1) boosting stages.

429
        feature_name : list of strings or 'auto', optional (default='auto')
430
431
            Feature names.
            If 'auto' and data is pandas DataFrame, data columns names are used.
432
        categorical_feature : list of strings or int, or 'auto', optional (default='auto')
433
434
            Categorical features.
            If list of int, interpreted as indices.
435
            If list of strings, interpreted as feature names (need to specify ``feature_name`` as well).
436
            If 'auto' and data is pandas DataFrame, pandas unordered categorical columns are used.
437
            All values in categorical features should be less than int32 max value (2147483647).
438
            Large values could be memory consuming. Consider using consecutive integers starting from zero.
439
            All negative values in categorical features will be treated as missing values.
440
            The output cannot be monotonically constrained with respect to a categorical feature.
441
        callbacks : list of callback functions or None, optional (default=None)
442
            List of callback functions that are applied at each iteration.
443
            See Callbacks in Python API for more information.
444

445
446
447
448
449
        Returns
        -------
        self : object
            Returns self.

450
451
        Note
        ----
452
        Custom eval function expects a callable with following signatures:
453
        ``func(y_true, y_pred)``, ``func(y_true, y_pred, weight)`` or
454
        ``func(y_true, y_pred, weight, group)``
455
456
        and returns (eval_name, eval_result, is_higher_better) or
        list of (eval_name, eval_result, is_higher_better):
457

Nikita Titov's avatar
Nikita Titov committed
458
            y_true : array-like of shape = [n_samples]
459
                The target values.
460
            y_pred : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
461
                The predicted values.
Nikita Titov's avatar
Nikita Titov committed
462
            weight : array-like of shape = [n_samples]
463
                The weight of samples.
Nikita Titov's avatar
Nikita Titov committed
464
            group : array-like
465
                Group/query data, used for ranking task.
Nikita Titov's avatar
Nikita Titov committed
466
            eval_name : string
467
                The name of evaluation function (without whitespaces).
Nikita Titov's avatar
Nikita Titov committed
468
            eval_result : float
469
                The eval result.
470
471
            is_higher_better : bool
                Is eval result higher better, e.g. AUC is ``is_higher_better``.
472

473
474
        For multi-class task, the y_pred is group by class_id first, then group by row_id.
        If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i].
wxchan's avatar
wxchan committed
475
        """
476
477
478
479
480
481
482
483
484
485
        if self._objective is None:
            if isinstance(self, LGBMRegressor):
                self._objective = "regression"
            elif isinstance(self, LGBMClassifier):
                self._objective = "binary"
            elif isinstance(self, LGBMRanker):
                self._objective = "lambdarank"
            else:
                raise ValueError("Unknown LGBMModel type.")
        if callable(self._objective):
486
            self._fobj = _ObjectiveFunctionWrapper(self._objective)
487
488
        else:
            self._fobj = None
wxchan's avatar
wxchan committed
489
490
        evals_result = {}
        params = self.get_params()
wxchan's avatar
wxchan committed
491
        # user can set verbose with kwargs, it has higher priority
492
        if not any(verbose_alias in params for verbose_alias in _ConfigAliases.get("verbosity")) and self.silent:
493
            params['verbose'] = -1
wxchan's avatar
wxchan committed
494
        params.pop('silent', None)
495
        params.pop('importance_type', None)
wxchan's avatar
wxchan committed
496
        params.pop('n_estimators', None)
497
        params.pop('class_weight', None)
498
499
        for alias in _ConfigAliases.get('objective'):
            params.pop(alias, None)
500
        if self._n_classes is not None and self._n_classes > 2:
501
502
            for alias in _ConfigAliases.get('num_class'):
                params.pop(alias, None)
503
504
            params['num_class'] = self._n_classes
        if hasattr(self, '_eval_at'):
505
506
            for alias in _ConfigAliases.get('eval_at'):
                params.pop(alias, None)
507
            params['eval_at'] = self._eval_at
508
509
        params['objective'] = self._objective
        if self._fobj:
wxchan's avatar
wxchan committed
510
            params['objective'] = 'None'  # objective = nullptr for unknown objective
wxchan's avatar
wxchan committed
511
512

        if callable(eval_metric):
513
            feval = _EvalFunctionWrapper(eval_metric)
wxchan's avatar
wxchan committed
514
515
        else:
            feval = None
516
517
518
519
520
521
522
523
524
525
526
            # register default metric for consistency with callable eval_metric case
            original_metric = self._objective if isinstance(self._objective, string_type) else None
            if original_metric is None:
                # try to deduce from class instance
                if isinstance(self, LGBMRegressor):
                    original_metric = "l2"
                elif isinstance(self, LGBMClassifier):
                    original_metric = "multi_logloss" if self._n_classes > 2 else "binary_logloss"
                elif isinstance(self, LGBMRanker):
                    original_metric = "ndcg"
            # overwrite default metric by explicitly set metric
527
            for metric_alias in _ConfigAliases.get("metric"):
528
529
530
531
532
                if metric_alias in params:
                    original_metric = params.pop(metric_alias)
            # concatenate metric from params (or default if not provided in params) and eval_metric
            original_metric = [original_metric] if isinstance(original_metric, (string_type, type(None))) else original_metric
            eval_metric = [eval_metric] if isinstance(eval_metric, (string_type, type(None))) else eval_metric
533
534
            params['metric'] = [e for e in eval_metric if e not in original_metric] + original_metric
            params['metric'] = [metric for metric in params['metric'] if metric is not None]
wxchan's avatar
wxchan committed
535

536
        if not isinstance(X, (DataFrame, DataTable)):
537
538
539
540
            _X, _y = _LGBMCheckXY(X, y, accept_sparse=True, force_all_finite=False, ensure_min_samples=2)
            _LGBMCheckConsistentLength(_X, _y, sample_weight)
        else:
            _X, _y = X, y
541

542
543
544
545
        if self._class_weight is None:
            self._class_weight = self.class_weight
        if self._class_weight is not None:
            class_sample_weight = _LGBMComputeSampleWeight(self._class_weight, y)
546
547
548
549
            if sample_weight is None or len(sample_weight) == 0:
                sample_weight = class_sample_weight
            else:
                sample_weight = np.multiply(sample_weight, class_sample_weight)
550

551
        self._n_features = _X.shape[1]
552

Guolin Ke's avatar
Guolin Ke committed
553
        def _construct_dataset(X, y, sample_weight, init_score, group, params):
554
            ret = Dataset(X, label=y, weight=sample_weight, group=group, params=params)
Nikita Titov's avatar
Nikita Titov committed
555
            return ret.set_init_score(init_score)
Guolin Ke's avatar
Guolin Ke committed
556

557
        train_set = _construct_dataset(_X, _y, sample_weight, init_score, group, params)
Guolin Ke's avatar
Guolin Ke committed
558
559
560

        valid_sets = []
        if eval_set is not None:
561

562
            def _get_meta_data(collection, name, i):
563
564
565
566
567
568
569
                if collection is None:
                    return None
                elif isinstance(collection, list):
                    return collection[i] if len(collection) > i else None
                elif isinstance(collection, dict):
                    return collection.get(i, None)
                else:
570
                    raise TypeError('{} should be dict or list'.format(name))
571

Guolin Ke's avatar
Guolin Ke committed
572
573
574
            if isinstance(eval_set, tuple):
                eval_set = [eval_set]
            for i, valid_data in enumerate(eval_set):
575
                # reduce cost for prediction training data
Guolin Ke's avatar
Guolin Ke committed
576
577
578
                if valid_data[0] is X and valid_data[1] is y:
                    valid_set = train_set
                else:
579
580
581
582
583
584
                    valid_weight = _get_meta_data(eval_sample_weight, 'eval_sample_weight', i)
                    valid_class_weight = _get_meta_data(eval_class_weight, 'eval_class_weight', i)
                    if valid_class_weight is not None:
                        if isinstance(valid_class_weight, dict) and self._class_map is not None:
                            valid_class_weight = {self._class_map[k]: v for k, v in valid_class_weight.items()}
                        valid_class_sample_weight = _LGBMComputeSampleWeight(valid_class_weight, valid_data[1])
585
586
587
588
                        if valid_weight is None or len(valid_weight) == 0:
                            valid_weight = valid_class_sample_weight
                        else:
                            valid_weight = np.multiply(valid_weight, valid_class_sample_weight)
589
590
                    valid_init_score = _get_meta_data(eval_init_score, 'eval_init_score', i)
                    valid_group = _get_meta_data(eval_group, 'eval_group', i)
591
592
                    valid_set = _construct_dataset(valid_data[0], valid_data[1],
                                                   valid_weight, valid_init_score, valid_group, params)
Guolin Ke's avatar
Guolin Ke committed
593
594
595
                valid_sets.append(valid_set)

        self._Booster = train(params, train_set,
596
                              self.n_estimators, valid_sets=valid_sets, valid_names=eval_names,
wxchan's avatar
wxchan committed
597
                              early_stopping_rounds=early_stopping_rounds,
598
                              evals_result=evals_result, fobj=self._fobj, feval=feval,
Guolin Ke's avatar
Guolin Ke committed
599
                              verbose_eval=verbose, feature_name=feature_name,
600
                              categorical_feature=categorical_feature,
601
                              callbacks=callbacks)
wxchan's avatar
wxchan committed
602
603

        if evals_result:
604
            self._evals_result = evals_result
wxchan's avatar
wxchan committed
605
606

        if early_stopping_rounds is not None:
607
            self._best_iteration = self._Booster.best_iteration
608
609

        self._best_score = self._Booster.best_score
wxchan's avatar
wxchan committed
610
611
612
613

        # free dataset
        self.booster_.free_dataset()
        del train_set, valid_sets
wxchan's avatar
wxchan committed
614
615
        return self

616
    def predict(self, X, raw_score=False, num_iteration=None,
617
                pred_leaf=False, pred_contrib=False, **kwargs):
618
        """Return the predicted value for each sample.
wxchan's avatar
wxchan committed
619
620
621

        Parameters
        ----------
622
        X : array-like or sparse matrix of shape = [n_samples, n_features]
wxchan's avatar
wxchan committed
623
            Input features matrix.
624
625
        raw_score : bool, optional (default=False)
            Whether to predict raw scores.
626
        num_iteration : int or None, optional (default=None)
627
            Limit number of iterations in the prediction.
628
629
            If None, if the best iteration exists, it is used; otherwise, all trees are used.
            If <= 0, all trees are used (no limits).
630
631
632
633
        pred_leaf : bool, optional (default=False)
            Whether to predict leaf index.
        pred_contrib : bool, optional (default=False)
            Whether to predict feature contributions.
634

Nikita Titov's avatar
Nikita Titov committed
635
636
637
638
639
640
641
            .. note::

                If you want to get more explanations for your model's predictions using SHAP values,
                like SHAP interaction values,
                you can install the shap package (https://github.com/slundberg/shap).
                Note that unlike the shap package, with ``pred_contrib`` we return a matrix with an extra
                column, where the last column is the expected value.
642

643
644
        **kwargs
            Other parameters for the prediction.
wxchan's avatar
wxchan committed
645
646
647

        Returns
        -------
648
649
        predicted_result : array-like of shape = [n_samples] or shape = [n_samples, n_classes]
            The predicted values.
650
        X_leaves : array-like of shape = [n_samples, n_trees] or shape = [n_samples, n_trees * n_classes]
Nikita Titov's avatar
Nikita Titov committed
651
            If ``pred_leaf=True``, the predicted leaf of every tree for each sample.
652
653
        X_SHAP_values : array-like of shape = [n_samples, n_features + 1] or shape = [n_samples, (n_features + 1) * n_classes]
            If ``pred_contrib=True``, the feature contributions for each sample.
wxchan's avatar
wxchan committed
654
        """
655
656
        if self._n_features is None:
            raise LGBMNotFittedError("Estimator not fitted, call `fit` before exploiting the model.")
657
        if not isinstance(X, (DataFrame, DataTable)):
658
            X = _LGBMCheckArray(X, accept_sparse=True, force_all_finite=False)
659
660
661
662
663
664
        n_features = X.shape[1]
        if self._n_features != n_features:
            raise ValueError("Number of features of the model must "
                             "match the input. Model n_features_ is %s and "
                             "input n_features is %s "
                             % (self._n_features, n_features))
665
666
        return self.booster_.predict(X, raw_score=raw_score, num_iteration=num_iteration,
                                     pred_leaf=pred_leaf, pred_contrib=pred_contrib, **kwargs)
wxchan's avatar
wxchan committed
667

668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
    @property
    def n_features_(self):
        """Get the number of features of fitted model."""
        if self._n_features is None:
            raise LGBMNotFittedError('No n_features found. Need to call fit beforehand.')
        return self._n_features

    @property
    def best_score_(self):
        """Get the best score of fitted model."""
        if self._n_features is None:
            raise LGBMNotFittedError('No best_score found. Need to call fit beforehand.')
        return self._best_score

    @property
    def best_iteration_(self):
        """Get the best iteration of fitted model."""
        if self._n_features is None:
            raise LGBMNotFittedError('No best_iteration found. Need to call fit with early_stopping_rounds beforehand.')
        return self._best_iteration

    @property
    def objective_(self):
        """Get the concrete objective used while fitting this model."""
        if self._n_features is None:
            raise LGBMNotFittedError('No objective found. Need to call fit beforehand.')
        return self._objective

696
697
698
699
    @property
    def booster_(self):
        """Get the underlying lightgbm Booster of this model."""
        if self._Booster is None:
700
            raise LGBMNotFittedError('No booster found. Need to call fit beforehand.')
701
        return self._Booster
wxchan's avatar
wxchan committed
702

703
704
705
    @property
    def evals_result_(self):
        """Get the evaluation results."""
706
707
708
        if self._n_features is None:
            raise LGBMNotFittedError('No results found. Need to call fit with eval_set beforehand.')
        return self._evals_result
709
710

    @property
711
    def feature_importances_(self):
712
        """Get feature importances.
713

Nikita Titov's avatar
Nikita Titov committed
714
715
716
717
718
719
        .. note::

            Feature importance in sklearn interface used to normalize to 1,
            it's deprecated after 2.0.4 and is the same as Booster.feature_importance() now.
            ``importance_type`` attribute is passed to the function
            to configure the type of importance values to be extracted.
720
        """
721
722
        if self._n_features is None:
            raise LGBMNotFittedError('No feature_importances found. Need to call fit beforehand.')
723
        return self.booster_.feature_importance(importance_type=self.importance_type)
wxchan's avatar
wxchan committed
724

wxchan's avatar
wxchan committed
725

726
727
class LGBMRegressor(LGBMModel, _LGBMRegressorBase):
    """LightGBM regressor."""
wxchan's avatar
wxchan committed
728

Guolin Ke's avatar
Guolin Ke committed
729
730
    def fit(self, X, y,
            sample_weight=None, init_score=None,
731
            eval_set=None, eval_names=None, eval_sample_weight=None,
732
            eval_init_score=None, eval_metric=None, early_stopping_rounds=None,
733
            verbose=True, feature_name='auto', categorical_feature='auto', callbacks=None):
734
        """Docstring is inherited from the LGBMModel."""
735
736
        super(LGBMRegressor, self).fit(X, y, sample_weight=sample_weight,
                                       init_score=init_score, eval_set=eval_set,
737
                                       eval_names=eval_names,
738
739
740
741
742
                                       eval_sample_weight=eval_sample_weight,
                                       eval_init_score=eval_init_score,
                                       eval_metric=eval_metric,
                                       early_stopping_rounds=early_stopping_rounds,
                                       verbose=verbose, feature_name=feature_name,
743
                                       categorical_feature=categorical_feature,
Guolin Ke's avatar
Guolin Ke committed
744
                                       callbacks=callbacks)
Guolin Ke's avatar
Guolin Ke committed
745
746
        return self

747
748
749
    _base_doc = LGBMModel.fit.__doc__
    fit.__doc__ = (_base_doc[:_base_doc.find('eval_class_weight :')]
                   + _base_doc[_base_doc.find('eval_init_score :'):])
wxchan's avatar
wxchan committed
750

751
752
753

class LGBMClassifier(LGBMModel, _LGBMClassifierBase):
    """LightGBM classifier."""
wxchan's avatar
wxchan committed
754

Guolin Ke's avatar
Guolin Ke committed
755
756
    def fit(self, X, y,
            sample_weight=None, init_score=None,
757
            eval_set=None, eval_names=None, eval_sample_weight=None,
758
            eval_class_weight=None, eval_init_score=None, eval_metric=None,
wxchan's avatar
wxchan committed
759
            early_stopping_rounds=None, verbose=True,
760
            feature_name='auto', categorical_feature='auto', callbacks=None):
761
        """Docstring is inherited from the LGBMModel."""
762
        _LGBMAssertAllFinite(y)
763
764
        _LGBMCheckClassificationTargets(y)
        self._le = _LGBMLabelEncoder().fit(y)
765
        _y = self._le.transform(y)
766
767
768
        self._class_map = dict(zip_(self._le.classes_, self._le.transform(self._le.classes_)))
        if isinstance(self.class_weight, dict):
            self._class_weight = {self._class_map[k]: v for k, v in self.class_weight.items()}
769

770
771
772
        self._classes = self._le.classes_
        self._n_classes = len(self._classes)
        if self._n_classes > 2:
wxchan's avatar
wxchan committed
773
            # Switch to using a multiclass objective in the underlying LGBM instance
774
775
            ova_aliases = ("multiclassova", "multiclass_ova", "ova", "ovr")
            if self._objective not in ova_aliases and not callable(self._objective):
776
                self._objective = "multiclass"
777
            if eval_metric in ('logloss', 'binary_logloss'):
wxchan's avatar
wxchan committed
778
                eval_metric = "multi_logloss"
779
            elif eval_metric in ('error', 'binary_error'):
wxchan's avatar
wxchan committed
780
781
                eval_metric = "multi_error"
        else:
782
            if eval_metric in ('logloss', 'multi_logloss'):
wxchan's avatar
wxchan committed
783
                eval_metric = 'binary_logloss'
784
            elif eval_metric in ('error', 'multi_error'):
wxchan's avatar
wxchan committed
785
                eval_metric = 'binary_error'
wxchan's avatar
wxchan committed
786
787

        if eval_set is not None:
788
789
790
791
792
793
794
            if isinstance(eval_set, tuple):
                eval_set = [eval_set]
            for i, (valid_x, valid_y) in enumerate(eval_set):
                if valid_x is X and valid_y is y:
                    eval_set[i] = (valid_x, _y)
                else:
                    eval_set[i] = (valid_x, self._le.transform(valid_y))
795

796
        super(LGBMClassifier, self).fit(X, _y, sample_weight=sample_weight,
797
                                        init_score=init_score, eval_set=eval_set,
798
                                        eval_names=eval_names,
799
                                        eval_sample_weight=eval_sample_weight,
800
                                        eval_class_weight=eval_class_weight,
801
802
803
804
                                        eval_init_score=eval_init_score,
                                        eval_metric=eval_metric,
                                        early_stopping_rounds=early_stopping_rounds,
                                        verbose=verbose, feature_name=feature_name,
805
                                        categorical_feature=categorical_feature,
806
                                        callbacks=callbacks)
wxchan's avatar
wxchan committed
807
808
        return self

809
    fit.__doc__ = LGBMModel.fit.__doc__
810

811
    def predict(self, X, raw_score=False, num_iteration=None,
812
                pred_leaf=False, pred_contrib=False, **kwargs):
813
        """Docstring is inherited from the LGBMModel."""
814
815
816
817
818
819
820
        result = self.predict_proba(X, raw_score, num_iteration,
                                    pred_leaf, pred_contrib, **kwargs)
        if raw_score or pred_leaf or pred_contrib:
            return result
        else:
            class_index = np.argmax(result, axis=1)
            return self._le.inverse_transform(class_index)
wxchan's avatar
wxchan committed
821

822
823
    predict.__doc__ = LGBMModel.predict.__doc__

824
    def predict_proba(self, X, raw_score=False, num_iteration=None,
825
                      pred_leaf=False, pred_contrib=False, **kwargs):
826
        """Return the predicted probability for each class for each sample.
wxchan's avatar
wxchan committed
827
828
829

        Parameters
        ----------
830
        X : array-like or sparse matrix of shape = [n_samples, n_features]
wxchan's avatar
wxchan committed
831
            Input features matrix.
832
833
        raw_score : bool, optional (default=False)
            Whether to predict raw scores.
834
        num_iteration : int or None, optional (default=None)
835
            Limit number of iterations in the prediction.
836
837
            If None, if the best iteration exists, it is used; otherwise, all trees are used.
            If <= 0, all trees are used (no limits).
838
839
840
841
        pred_leaf : bool, optional (default=False)
            Whether to predict leaf index.
        pred_contrib : bool, optional (default=False)
            Whether to predict feature contributions.
842

Nikita Titov's avatar
Nikita Titov committed
843
844
845
846
847
848
849
            .. note::

                If you want to get more explanations for your model's predictions using SHAP values,
                like SHAP interaction values,
                you can install the shap package (https://github.com/slundberg/shap).
                Note that unlike the shap package, with ``pred_contrib`` we return a matrix with an extra
                column, where the last column is the expected value.
850

851
852
        **kwargs
            Other parameters for the prediction.
wxchan's avatar
wxchan committed
853
854
855

        Returns
        -------
856
857
        predicted_probability : array-like of shape = [n_samples, n_classes]
            The predicted probability for each class for each sample.
858
        X_leaves : array-like of shape = [n_samples, n_trees * n_classes]
859
            If ``pred_leaf=True``, the predicted leaf of every tree for each sample.
860
        X_SHAP_values : array-like of shape = [n_samples, (n_features + 1) * n_classes]
861
            If ``pred_contrib=True``, the feature contributions for each sample.
wxchan's avatar
wxchan committed
862
        """
863
864
        result = super(LGBMClassifier, self).predict(X, raw_score, num_iteration,
                                                     pred_leaf, pred_contrib, **kwargs)
865
        if self._n_classes > 2 or raw_score or pred_leaf or pred_contrib:
866
            return result
wxchan's avatar
wxchan committed
867
        else:
868
            return np.vstack((1. - result, result)).transpose()
869
870
871

    @property
    def classes_(self):
872
873
874
875
        """Get the class label array."""
        if self._classes is None:
            raise LGBMNotFittedError('No classes found. Need to call fit beforehand.')
        return self._classes
876
877
878

    @property
    def n_classes_(self):
879
880
881
882
        """Get the number of classes."""
        if self._n_classes is None:
            raise LGBMNotFittedError('No classes found. Need to call fit beforehand.')
        return self._n_classes
wxchan's avatar
wxchan committed
883

wxchan's avatar
wxchan committed
884

wxchan's avatar
wxchan committed
885
class LGBMRanker(LGBMModel):
886
    """LightGBM ranker."""
wxchan's avatar
wxchan committed
887

Guolin Ke's avatar
Guolin Ke committed
888
    def fit(self, X, y,
889
            sample_weight=None, init_score=None, group=None,
890
            eval_set=None, eval_names=None, eval_sample_weight=None,
891
            eval_init_score=None, eval_group=None, eval_metric=None,
892
893
            eval_at=[1], early_stopping_rounds=None, verbose=True,
            feature_name='auto', categorical_feature='auto', callbacks=None):
894
        """Docstring is inherited from the LGBMModel."""
895
        # check group data
Guolin Ke's avatar
Guolin Ke committed
896
        if group is None:
897
            raise ValueError("Should set group for ranking task")
wxchan's avatar
wxchan committed
898
899

        if eval_set is not None:
Guolin Ke's avatar
Guolin Ke committed
900
            if eval_group is None:
901
                raise ValueError("Eval_group cannot be None when eval_set is not None")
Guolin Ke's avatar
Guolin Ke committed
902
            elif len(eval_group) != len(eval_set):
903
                raise ValueError("Length of eval_group should be equal to eval_set")
904
905
906
907
            elif (isinstance(eval_group, dict)
                  and any(i not in eval_group or eval_group[i] is None for i in range_(len(eval_group)))
                  or isinstance(eval_group, list)
                  and any(group is None for group in eval_group)):
908
909
                raise ValueError("Should set group for all eval datasets for ranking task; "
                                 "if you use dict, the index should start from 0")
910

911
        self._eval_at = eval_at
912
913
        super(LGBMRanker, self).fit(X, y, sample_weight=sample_weight,
                                    init_score=init_score, group=group,
914
915
                                    eval_set=eval_set, eval_names=eval_names,
                                    eval_sample_weight=eval_sample_weight,
916
917
918
919
                                    eval_init_score=eval_init_score, eval_group=eval_group,
                                    eval_metric=eval_metric,
                                    early_stopping_rounds=early_stopping_rounds,
                                    verbose=verbose, feature_name=feature_name,
920
                                    categorical_feature=categorical_feature,
921
                                    callbacks=callbacks)
wxchan's avatar
wxchan committed
922
        return self
923

924
925
926
927
    _base_doc = LGBMModel.fit.__doc__
    fit.__doc__ = (_base_doc[:_base_doc.find('eval_class_weight :')]
                   + _base_doc[_base_doc.find('eval_init_score :'):])
    _base_doc = fit.__doc__
928
929
    _before_early_stop, _early_stop, _after_early_stop = _base_doc.partition('early_stopping_rounds :')
    fit.__doc__ = (_before_early_stop
930
                   + 'eval_at : list of int, optional (default=[1])\n'
931
932
                   + ' ' * 12 + 'The evaluation positions of the specified metric.\n'
                   + ' ' * 8 + _early_stop + _after_early_stop)