sklearn.py 45.3 KB
Newer Older
wxchan's avatar
wxchan committed
1
# coding: utf-8
2
"""Scikit-learn wrapper interface for LightGBM."""
wxchan's avatar
wxchan committed
3
from __future__ import absolute_import
4

5
6
import warnings

wxchan's avatar
wxchan committed
7
import numpy as np
8

9
from .basic import Dataset, LightGBMError, _ConfigAliases
10
from .compat import (SKLEARN_INSTALLED, SKLEARN_VERSION, _LGBMClassifierBase,
11
12
                     LGBMNotFittedError, _LGBMLabelEncoder, _LGBMModelBase,
                     _LGBMRegressorBase, _LGBMCheckXY, _LGBMCheckArray, _LGBMCheckConsistentLength,
13
                     _LGBMAssertAllFinite, _LGBMCheckClassificationTargets, _LGBMComputeSampleWeight,
14
                     argc_, range_, zip_, string_type, DataFrame, DataTable)
wxchan's avatar
wxchan committed
15
from .engine import train
16

wxchan's avatar
wxchan committed
17

18
19
class _ObjectiveFunctionWrapper(object):
    """Proxy class for objective function."""
20

21
22
    def __init__(self, func):
        """Construct a proxy class.
23

24
25
        This class transforms objective function to match objective function with signature ``new_func(preds, dataset)``
        as expected by ``lightgbm.engine.train``.
26

27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
        Parameters
        ----------
        func : callable
            Expects a callable with signature ``func(y_true, y_pred)`` or ``func(y_true, y_pred, group)
            and returns (grad, hess):

                y_true : array-like of shape = [n_samples]
                    The target values.
                y_pred : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
                    The predicted values.
                group : array-like
                    Group/query data, used for ranking task.
                grad : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
                    The value of the first order derivative (gradient) for each sample point.
                hess : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
                    The value of the second order derivative (Hessian) for each sample point.
wxchan's avatar
wxchan committed
43

Nikita Titov's avatar
Nikita Titov committed
44
45
46
47
48
        .. note::

            For multi-class task, the y_pred is group by class_id first, then group by row_id.
            If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i]
            and you should group grad and hess in this way as well.
49
50
        """
        self.func = func
wxchan's avatar
wxchan committed
51

52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
    def __call__(self, preds, dataset):
        """Call passed function with appropriate arguments.

        Parameters
        ----------
        preds : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
            The predicted values.
        dataset : Dataset
            The training dataset.

        Returns
        -------
        grad : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
            The value of the first order derivative (gradient) for each sample point.
        hess : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
            The value of the second order derivative (Hessian) for each sample point.
        """
wxchan's avatar
wxchan committed
69
        labels = dataset.get_label()
70
        argc = argc_(self.func)
71
        if argc == 2:
72
            grad, hess = self.func(labels, preds)
73
        elif argc == 3:
74
            grad, hess = self.func(labels, preds, dataset.get_group())
75
        else:
wxchan's avatar
wxchan committed
76
            raise TypeError("Self-defined objective function should have 2 or 3 arguments, got %d" % argc)
wxchan's avatar
wxchan committed
77
78
79
80
81
82
83
84
85
86
87
        """weighted for objective"""
        weight = dataset.get_weight()
        if weight is not None:
            """only one class"""
            if len(weight) == len(grad):
                grad = np.multiply(grad, weight)
                hess = np.multiply(hess, weight)
            else:
                num_data = len(weight)
                num_class = len(grad) // num_data
                if num_class * num_data != len(grad):
88
                    raise ValueError("Length of grad and hess should equal to num_class * num_data")
wxchan's avatar
wxchan committed
89
90
                for k in range_(num_class):
                    for i in range_(num_data):
wxchan's avatar
wxchan committed
91
92
93
94
95
                        idx = k * num_data + i
                        grad[idx] *= weight[i]
                        hess[idx] *= weight[i]
        return grad, hess

wxchan's avatar
wxchan committed
96

97
98
class _EvalFunctionWrapper(object):
    """Proxy class for evaluation function."""
99

100
101
    def __init__(self, func):
        """Construct a proxy class.
102

103
104
        This class transforms evaluation function to match evaluation function with signature ``new_func(preds, dataset)``
        as expected by ``lightgbm.engine.train``.
105

106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
        Parameters
        ----------
        func : callable
            Expects a callable with following signatures:
            ``func(y_true, y_pred)``,
            ``func(y_true, y_pred, weight)``
            or ``func(y_true, y_pred, weight, group)``
            and returns (eval_name, eval_result, is_higher_better) or
            list of (eval_name, eval_result, is_higher_better):

                y_true : array-like of shape = [n_samples]
                    The target values.
                y_pred : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
                    The predicted values.
                weight : array-like of shape = [n_samples]
                    The weight of samples.
                group : array-like
                    Group/query data, used for ranking task.
                eval_name : string
125
                    The name of evaluation function (without whitespaces).
126
127
128
129
130
                eval_result : float
                    The eval result.
                is_higher_better : bool
                    Is eval result higher better, e.g. AUC is ``is_higher_better``.

Nikita Titov's avatar
Nikita Titov committed
131
132
133
134
        .. note::

            For multi-class task, the y_pred is group by class_id first, then group by row_id.
            If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i].
135
136
        """
        self.func = func
137

138
139
    def __call__(self, preds, dataset):
        """Call passed function with appropriate arguments.
140

141
142
143
144
145
146
147
148
149
150
        Parameters
        ----------
        preds : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
            The predicted values.
        dataset : Dataset
            The training dataset.

        Returns
        -------
        eval_name : string
151
            The name of evaluation function (without whitespaces).
152
153
154
155
156
        eval_result : float
            The eval result.
        is_higher_better : bool
            Is eval result higher better, e.g. AUC is ``is_higher_better``.
        """
157
        labels = dataset.get_label()
158
        argc = argc_(self.func)
159
        if argc == 2:
160
            return self.func(labels, preds)
161
        elif argc == 3:
162
            return self.func(labels, preds, dataset.get_weight())
163
        elif argc == 4:
164
            return self.func(labels, preds, dataset.get_weight(), dataset.get_group())
165
        else:
wxchan's avatar
wxchan committed
166
            raise TypeError("Self-defined eval function should have 2, 3 or 4 arguments, got %d" % argc)
167

wxchan's avatar
wxchan committed
168

169
170
class LGBMModel(_LGBMModelBase):
    """Implementation of the scikit-learn API for LightGBM."""
wxchan's avatar
wxchan committed
171

172
    def __init__(self, boosting_type='gbdt', num_leaves=31, max_depth=-1,
173
                 learning_rate=0.1, n_estimators=100,
174
                 subsample_for_bin=200000, objective=None, class_weight=None,
175
                 min_split_gain=0., min_child_weight=1e-3, min_child_samples=20,
176
                 subsample=1., subsample_freq=0, colsample_bytree=1.,
177
                 reg_alpha=0., reg_lambda=0., random_state=None,
178
                 n_jobs=-1, silent=True, importance_type='split', **kwargs):
179
        r"""Construct a gradient boosting model.
wxchan's avatar
wxchan committed
180
181
182

        Parameters
        ----------
183
        boosting_type : string, optional (default='gbdt')
184
185
186
187
188
            'gbdt', traditional Gradient Boosting Decision Tree.
            'dart', Dropouts meet Multiple Additive Regression Trees.
            'goss', Gradient-based One-Side Sampling.
            'rf', Random Forest.
        num_leaves : int, optional (default=31)
wxchan's avatar
wxchan committed
189
            Maximum tree leaves for base learners.
190
        max_depth : int, optional (default=-1)
191
            Maximum tree depth for base learners, <=0 means no limit.
192
        learning_rate : float, optional (default=0.1)
193
            Boosting learning rate.
194
195
196
            You can use ``callbacks`` parameter of ``fit`` method to shrink/adapt learning rate
            in training using ``reset_parameter`` callback.
            Note, that this will ignore the ``learning_rate`` argument in training.
197
        n_estimators : int, optional (default=100)
wxchan's avatar
wxchan committed
198
            Number of boosted trees to fit.
199
        subsample_for_bin : int, optional (default=200000)
wxchan's avatar
wxchan committed
200
            Number of samples for constructing bins.
201
        objective : string, callable or None, optional (default=None)
wxchan's avatar
wxchan committed
202
203
            Specify the learning task and the corresponding learning objective or
            a custom objective function to be used (see note below).
204
            Default: 'regression' for LGBMRegressor, 'binary' or 'multiclass' for LGBMClassifier, 'lambdarank' for LGBMRanker.
205
206
207
208
        class_weight : dict, 'balanced' or None, optional (default=None)
            Weights associated with classes in the form ``{class_label: weight}``.
            Use this parameter only for multi-class classification task;
            for binary classification task you may use ``is_unbalance`` or ``scale_pos_weight`` parameters.
209
210
211
            Note, that the usage of all these parameters will result in poor estimates of the individual class probabilities.
            You may want to consider performing probability calibration
            (https://scikit-learn.org/stable/modules/calibration.html) of your model.
212
213
214
            The 'balanced' mode uses the values of y to automatically adjust weights
            inversely proportional to class frequencies in the input data as ``n_samples / (n_classes * np.bincount(y))``.
            If None, all classes are supposed to have weight one.
215
            Note, that these weights will be multiplied with ``sample_weight`` (passed through the ``fit`` method)
216
            if ``sample_weight`` is specified.
217
        min_split_gain : float, optional (default=0.)
wxchan's avatar
wxchan committed
218
            Minimum loss reduction required to make a further partition on a leaf node of the tree.
219
        min_child_weight : float, optional (default=1e-3)
220
            Minimum sum of instance weight (hessian) needed in a child (leaf).
221
        min_child_samples : int, optional (default=20)
222
            Minimum number of data needed in a child (leaf).
223
        subsample : float, optional (default=1.)
wxchan's avatar
wxchan committed
224
            Subsample ratio of the training instance.
225
        subsample_freq : int, optional (default=0)
226
227
            Frequence of subsample, <=0 means no enable.
        colsample_bytree : float, optional (default=1.)
wxchan's avatar
wxchan committed
228
            Subsample ratio of columns when constructing each tree.
229
        reg_alpha : float, optional (default=0.)
230
            L1 regularization term on weights.
231
        reg_lambda : float, optional (default=0.)
232
            L2 regularization term on weights.
233
        random_state : int or None, optional (default=None)
wxchan's avatar
wxchan committed
234
            Random number seed.
235
            If None, default seeds in C++ code will be used.
236
        n_jobs : int, optional (default=-1)
237
            Number of parallel threads.
238
        silent : bool, optional (default=True)
wxchan's avatar
wxchan committed
239
            Whether to print messages while running boosting.
240
        importance_type : string, optional (default='split')
241
            The type of feature importance to be filled into ``feature_importances_``.
242
243
244
245
            If 'split', result contains numbers of times the feature is used in a model.
            If 'gain', result contains total gains of splits which use the feature.
        **kwargs
            Other parameters for the model.
wxchan's avatar
wxchan committed
246
            Check http://lightgbm.readthedocs.io/en/latest/Parameters.html for more parameters.
247

Nikita Titov's avatar
Nikita Titov committed
248
249
250
            .. warning::

                \*\*kwargs is not supported in sklearn, it may cause unexpected issues.
wxchan's avatar
wxchan committed
251

252
253
254
255
256
257
258
259
260
        Attributes
        ----------
        n_features_ : int
            The number of features of fitted model.
        classes_ : array of shape = [n_classes]
            The class label array (only for classification problem).
        n_classes_ : int
            The number of classes (only for classification problem).
        best_score_ : dict or None
261
            The best score of fitted model.
262
        best_iteration_ : int or None
263
            The best iteration of fitted model if ``early_stopping_rounds`` has been specified.
264
265
266
267
268
        objective_ : string or callable
            The concrete objective used while fitting this model.
        booster_ : Booster
            The underlying Booster of this model.
        evals_result_ : dict or None
269
            The evaluation results if ``early_stopping_rounds`` has been specified.
270
271
272
        feature_importances_ : array of shape = [n_features]
            The feature importances (the higher, the more important the feature).

wxchan's avatar
wxchan committed
273
274
        Note
        ----
275
276
        A custom objective function can be provided for the ``objective`` parameter.
        In this case, it should have the signature
277
278
        ``objective(y_true, y_pred) -> grad, hess`` or
        ``objective(y_true, y_pred, group) -> grad, hess``:
wxchan's avatar
wxchan committed
279

Nikita Titov's avatar
Nikita Titov committed
280
            y_true : array-like of shape = [n_samples]
281
                The target values.
Nikita Titov's avatar
Nikita Titov committed
282
            y_pred : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
283
                The predicted values.
Nikita Titov's avatar
Nikita Titov committed
284
            group : array-like
285
                Group/query data, used for ranking task.
Nikita Titov's avatar
Nikita Titov committed
286
            grad : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
287
                The value of the first order derivative (gradient) for each sample point.
Nikita Titov's avatar
Nikita Titov committed
288
            hess : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
289
                The value of the second order derivative (Hessian) for each sample point.
wxchan's avatar
wxchan committed
290

291
292
293
        For multi-class task, the y_pred is group by class_id first, then group by row_id.
        If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i]
        and you should group grad and hess in this way as well.
wxchan's avatar
wxchan committed
294
        """
wxchan's avatar
wxchan committed
295
        if not SKLEARN_INSTALLED:
296
            raise LightGBMError('Scikit-learn is required for this module')
297
298
299
        elif SKLEARN_VERSION > '0.21.3':
            raise RuntimeError("The last supported version of scikit-learn is 0.21.3.\n"
                               "Found version: {0}.".format(SKLEARN_VERSION))
wxchan's avatar
wxchan committed
300

301
        self.boosting_type = boosting_type
302
        self.objective = objective
wxchan's avatar
wxchan committed
303
304
305
306
        self.num_leaves = num_leaves
        self.max_depth = max_depth
        self.learning_rate = learning_rate
        self.n_estimators = n_estimators
wxchan's avatar
wxchan committed
307
        self.subsample_for_bin = subsample_for_bin
wxchan's avatar
wxchan committed
308
309
310
311
312
313
314
315
        self.min_split_gain = min_split_gain
        self.min_child_weight = min_child_weight
        self.min_child_samples = min_child_samples
        self.subsample = subsample
        self.subsample_freq = subsample_freq
        self.colsample_bytree = colsample_bytree
        self.reg_alpha = reg_alpha
        self.reg_lambda = reg_lambda
316
317
        self.random_state = random_state
        self.n_jobs = n_jobs
wxchan's avatar
wxchan committed
318
        self.silent = silent
319
        self.importance_type = importance_type
wxchan's avatar
wxchan committed
320
        self._Booster = None
321
322
323
324
        self._evals_result = None
        self._best_score = None
        self._best_iteration = None
        self._other_params = {}
325
        self._objective = objective
326
        self.class_weight = class_weight
327
328
        self._class_weight = None
        self._class_map = None
329
330
331
        self._n_features = None
        self._classes = None
        self._n_classes = None
332
        self.set_params(**kwargs)
wxchan's avatar
wxchan committed
333

Nikita Titov's avatar
Nikita Titov committed
334
335
336
337
    def _more_tags(self):
        return {'allow_nan': True,
                'X_types': ['2darray', 'sparse', '1dlabels']}

wxchan's avatar
wxchan committed
338
    def get_params(self, deep=True):
339
340
341
342
343
344
345
346
347
348
349
350
351
        """Get parameters for this estimator.

        Parameters
        ----------
        deep : bool, optional (default=True)
            If True, will return the parameters for this estimator and
            contained subobjects that are estimators.

        Returns
        -------
        params : dict
            Parameter names mapped to their values.
        """
wxchan's avatar
wxchan committed
352
        params = super(LGBMModel, self).get_params(deep=deep)
353
        params.update(self._other_params)
wxchan's avatar
wxchan committed
354
355
356
        return params

    def set_params(self, **params):
357
358
359
360
361
362
363
364
365
366
367
368
        """Set the parameters of this estimator.

        Parameters
        ----------
        **params
            Parameter names with their new values.

        Returns
        -------
        self : object
            Returns self.
        """
wxchan's avatar
wxchan committed
369
370
        for key, value in params.items():
            setattr(self, key, value)
371
372
            if hasattr(self, '_' + key):
                setattr(self, '_' + key, value)
373
            self._other_params[key] = value
wxchan's avatar
wxchan committed
374
        return self
wxchan's avatar
wxchan committed
375

Guolin Ke's avatar
Guolin Ke committed
376
    def fit(self, X, y,
377
            sample_weight=None, init_score=None, group=None,
378
            eval_set=None, eval_names=None, eval_sample_weight=None,
379
380
            eval_class_weight=None, eval_init_score=None, eval_group=None,
            eval_metric=None, early_stopping_rounds=None, verbose=True,
381
382
            feature_name='auto', categorical_feature='auto',
            callbacks=None, init_model=None):
383
        """Build a gradient boosting model from the training set (X, y).
wxchan's avatar
wxchan committed
384
385
386

        Parameters
        ----------
387
388
389
390
391
392
393
394
        X : array-like or sparse matrix of shape = [n_samples, n_features]
            Input feature matrix.
        y : array-like of shape = [n_samples]
            The target values (class labels in classification, real numbers in regression).
        sample_weight : array-like of shape = [n_samples] or None, optional (default=None)
            Weights of training data.
        init_score : array-like of shape = [n_samples] or None, optional (default=None)
            Init score of training data.
395
        group : array-like or None, optional (default=None)
396
397
            Group data of training data.
        eval_set : list or None, optional (default=None)
398
            A list of (X, y) tuple pairs to use as validation sets.
399
        eval_names : list of strings or None, optional (default=None)
400
401
402
            Names of eval_set.
        eval_sample_weight : list of arrays or None, optional (default=None)
            Weights of eval data.
403
404
        eval_class_weight : list or None, optional (default=None)
            Class weights of eval data.
405
406
407
408
409
410
        eval_init_score : list of arrays or None, optional (default=None)
            Init score of eval data.
        eval_group : list of arrays or None, optional (default=None)
            Group data of eval data.
        eval_metric : string, list of strings, callable or None, optional (default=None)
            If string, it should be a built-in evaluation metric to use.
411
            If callable, it should be a custom evaluation metric, see note below for more details.
Misha Lisovyi's avatar
Misha Lisovyi committed
412
            In either case, the ``metric`` from the model parameters will be evaluated and used as well.
413
            Default: 'l2' for LGBMRegressor, 'logloss' for LGBMClassifier, 'ndcg' for LGBMRanker.
414
415
        early_stopping_rounds : int or None, optional (default=None)
            Activates early stopping. The model will train until the validation score stops improving.
416
            Validation score needs to improve at least every ``early_stopping_rounds`` round(s)
417
            to continue training.
418
419
            Requires at least one validation data and one metric.
            If there's more than one, will check all of them. But the training data is ignored anyway.
420
421
            To check only the first metric, set the ``first_metric_only`` parameter to ``True``
            in additional parameters ``**kwargs`` of the model constructor.
422
423
424
425
426
427
        verbose : bool or int, optional (default=True)
            Requires at least one evaluation data.
            If True, the eval metric on the eval set is printed at each boosting stage.
            If int, the eval metric on the eval set is printed at every ``verbose`` boosting stage.
            The last boosting stage or the boosting stage found by using ``early_stopping_rounds`` is also printed.

Nikita Titov's avatar
Nikita Titov committed
428
429
            .. rubric:: Example

430
431
432
            With ``verbose`` = 4 and at least one item in ``eval_set``,
            an evaluation metric is printed every 4 (instead of 1) boosting stages.

433
        feature_name : list of strings or 'auto', optional (default='auto')
434
435
            Feature names.
            If 'auto' and data is pandas DataFrame, data columns names are used.
436
        categorical_feature : list of strings or int, or 'auto', optional (default='auto')
437
438
            Categorical features.
            If list of int, interpreted as indices.
439
            If list of strings, interpreted as feature names (need to specify ``feature_name`` as well).
440
            If 'auto' and data is pandas DataFrame, pandas unordered categorical columns are used.
441
            All values in categorical features should be less than int32 max value (2147483647).
442
            Large values could be memory consuming. Consider using consecutive integers starting from zero.
443
            All negative values in categorical features will be treated as missing values.
444
            The output cannot be monotonically constrained with respect to a categorical feature.
445
        callbacks : list of callback functions or None, optional (default=None)
446
            List of callback functions that are applied at each iteration.
447
            See Callbacks in Python API for more information.
448
449
        init_model : string, Booster, LGBMModel or None, optional (default=None)
            Filename of LightGBM model, Booster instance or LGBMModel instance used for continue training.
450

451
452
453
454
455
        Returns
        -------
        self : object
            Returns self.

456
457
        Note
        ----
458
        Custom eval function expects a callable with following signatures:
459
        ``func(y_true, y_pred)``, ``func(y_true, y_pred, weight)`` or
460
        ``func(y_true, y_pred, weight, group)``
461
462
        and returns (eval_name, eval_result, is_higher_better) or
        list of (eval_name, eval_result, is_higher_better):
463

Nikita Titov's avatar
Nikita Titov committed
464
            y_true : array-like of shape = [n_samples]
465
                The target values.
466
            y_pred : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
467
                The predicted values.
Nikita Titov's avatar
Nikita Titov committed
468
            weight : array-like of shape = [n_samples]
469
                The weight of samples.
Nikita Titov's avatar
Nikita Titov committed
470
            group : array-like
471
                Group/query data, used for ranking task.
Nikita Titov's avatar
Nikita Titov committed
472
            eval_name : string
473
                The name of evaluation function (without whitespaces).
Nikita Titov's avatar
Nikita Titov committed
474
            eval_result : float
475
                The eval result.
476
477
            is_higher_better : bool
                Is eval result higher better, e.g. AUC is ``is_higher_better``.
478

479
480
        For multi-class task, the y_pred is group by class_id first, then group by row_id.
        If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i].
wxchan's avatar
wxchan committed
481
        """
482
483
484
485
486
487
488
489
490
491
        if self._objective is None:
            if isinstance(self, LGBMRegressor):
                self._objective = "regression"
            elif isinstance(self, LGBMClassifier):
                self._objective = "binary"
            elif isinstance(self, LGBMRanker):
                self._objective = "lambdarank"
            else:
                raise ValueError("Unknown LGBMModel type.")
        if callable(self._objective):
492
            self._fobj = _ObjectiveFunctionWrapper(self._objective)
493
494
        else:
            self._fobj = None
wxchan's avatar
wxchan committed
495
496
        evals_result = {}
        params = self.get_params()
wxchan's avatar
wxchan committed
497
        # user can set verbose with kwargs, it has higher priority
498
        if not any(verbose_alias in params for verbose_alias in _ConfigAliases.get("verbosity")) and self.silent:
499
            params['verbose'] = -1
wxchan's avatar
wxchan committed
500
        params.pop('silent', None)
501
        params.pop('importance_type', None)
wxchan's avatar
wxchan committed
502
        params.pop('n_estimators', None)
503
        params.pop('class_weight', None)
504
505
        for alias in _ConfigAliases.get('objective'):
            params.pop(alias, None)
506
        if self._n_classes is not None and self._n_classes > 2:
507
508
            for alias in _ConfigAliases.get('num_class'):
                params.pop(alias, None)
509
510
            params['num_class'] = self._n_classes
        if hasattr(self, '_eval_at'):
511
512
            for alias in _ConfigAliases.get('eval_at'):
                params.pop(alias, None)
513
            params['eval_at'] = self._eval_at
514
515
        params['objective'] = self._objective
        if self._fobj:
wxchan's avatar
wxchan committed
516
            params['objective'] = 'None'  # objective = nullptr for unknown objective
wxchan's avatar
wxchan committed
517
518

        if callable(eval_metric):
519
            feval = _EvalFunctionWrapper(eval_metric)
wxchan's avatar
wxchan committed
520
521
        else:
            feval = None
522
523
524
525
526
527
528
529
530
531
532
            # register default metric for consistency with callable eval_metric case
            original_metric = self._objective if isinstance(self._objective, string_type) else None
            if original_metric is None:
                # try to deduce from class instance
                if isinstance(self, LGBMRegressor):
                    original_metric = "l2"
                elif isinstance(self, LGBMClassifier):
                    original_metric = "multi_logloss" if self._n_classes > 2 else "binary_logloss"
                elif isinstance(self, LGBMRanker):
                    original_metric = "ndcg"
            # overwrite default metric by explicitly set metric
533
            for metric_alias in _ConfigAliases.get("metric"):
534
535
536
537
538
                if metric_alias in params:
                    original_metric = params.pop(metric_alias)
            # concatenate metric from params (or default if not provided in params) and eval_metric
            original_metric = [original_metric] if isinstance(original_metric, (string_type, type(None))) else original_metric
            eval_metric = [eval_metric] if isinstance(eval_metric, (string_type, type(None))) else eval_metric
539
540
            params['metric'] = [e for e in eval_metric if e not in original_metric] + original_metric
            params['metric'] = [metric for metric in params['metric'] if metric is not None]
wxchan's avatar
wxchan committed
541

542
        if not isinstance(X, (DataFrame, DataTable)):
543
544
545
546
            _X, _y = _LGBMCheckXY(X, y, accept_sparse=True, force_all_finite=False, ensure_min_samples=2)
            _LGBMCheckConsistentLength(_X, _y, sample_weight)
        else:
            _X, _y = X, y
547

548
549
550
551
        if self._class_weight is None:
            self._class_weight = self.class_weight
        if self._class_weight is not None:
            class_sample_weight = _LGBMComputeSampleWeight(self._class_weight, y)
552
553
554
555
            if sample_weight is None or len(sample_weight) == 0:
                sample_weight = class_sample_weight
            else:
                sample_weight = np.multiply(sample_weight, class_sample_weight)
556

557
        self._n_features = _X.shape[1]
558

Guolin Ke's avatar
Guolin Ke committed
559
        def _construct_dataset(X, y, sample_weight, init_score, group, params):
560
561
            return Dataset(X, label=y, weight=sample_weight, group=group,
                           init_score=init_score, params=params)
Guolin Ke's avatar
Guolin Ke committed
562

563
        train_set = _construct_dataset(_X, _y, sample_weight, init_score, group, params)
Guolin Ke's avatar
Guolin Ke committed
564
565
566

        valid_sets = []
        if eval_set is not None:
567

568
            def _get_meta_data(collection, name, i):
569
570
571
572
573
574
575
                if collection is None:
                    return None
                elif isinstance(collection, list):
                    return collection[i] if len(collection) > i else None
                elif isinstance(collection, dict):
                    return collection.get(i, None)
                else:
576
                    raise TypeError('{} should be dict or list'.format(name))
577

Guolin Ke's avatar
Guolin Ke committed
578
579
580
            if isinstance(eval_set, tuple):
                eval_set = [eval_set]
            for i, valid_data in enumerate(eval_set):
581
                # reduce cost for prediction training data
Guolin Ke's avatar
Guolin Ke committed
582
583
584
                if valid_data[0] is X and valid_data[1] is y:
                    valid_set = train_set
                else:
585
586
587
588
589
590
                    valid_weight = _get_meta_data(eval_sample_weight, 'eval_sample_weight', i)
                    valid_class_weight = _get_meta_data(eval_class_weight, 'eval_class_weight', i)
                    if valid_class_weight is not None:
                        if isinstance(valid_class_weight, dict) and self._class_map is not None:
                            valid_class_weight = {self._class_map[k]: v for k, v in valid_class_weight.items()}
                        valid_class_sample_weight = _LGBMComputeSampleWeight(valid_class_weight, valid_data[1])
591
592
593
594
                        if valid_weight is None or len(valid_weight) == 0:
                            valid_weight = valid_class_sample_weight
                        else:
                            valid_weight = np.multiply(valid_weight, valid_class_sample_weight)
595
596
                    valid_init_score = _get_meta_data(eval_init_score, 'eval_init_score', i)
                    valid_group = _get_meta_data(eval_group, 'eval_group', i)
597
598
                    valid_set = _construct_dataset(valid_data[0], valid_data[1],
                                                   valid_weight, valid_init_score, valid_group, params)
Guolin Ke's avatar
Guolin Ke committed
599
600
                valid_sets.append(valid_set)

601
602
603
        if isinstance(init_model, LGBMModel):
            init_model = init_model.booster_

Guolin Ke's avatar
Guolin Ke committed
604
        self._Booster = train(params, train_set,
605
                              self.n_estimators, valid_sets=valid_sets, valid_names=eval_names,
wxchan's avatar
wxchan committed
606
                              early_stopping_rounds=early_stopping_rounds,
607
                              evals_result=evals_result, fobj=self._fobj, feval=feval,
Guolin Ke's avatar
Guolin Ke committed
608
                              verbose_eval=verbose, feature_name=feature_name,
609
                              categorical_feature=categorical_feature,
610
                              callbacks=callbacks, init_model=init_model)
wxchan's avatar
wxchan committed
611
612

        if evals_result:
613
            self._evals_result = evals_result
wxchan's avatar
wxchan committed
614
615

        if early_stopping_rounds is not None:
616
            self._best_iteration = self._Booster.best_iteration
617
618

        self._best_score = self._Booster.best_score
wxchan's avatar
wxchan committed
619
620

        # free dataset
621
        self._Booster.free_dataset()
wxchan's avatar
wxchan committed
622
        del train_set, valid_sets
wxchan's avatar
wxchan committed
623
624
        return self

625
    def predict(self, X, raw_score=False, num_iteration=None,
626
                pred_leaf=False, pred_contrib=False, **kwargs):
627
        """Return the predicted value for each sample.
wxchan's avatar
wxchan committed
628
629
630

        Parameters
        ----------
631
        X : array-like or sparse matrix of shape = [n_samples, n_features]
wxchan's avatar
wxchan committed
632
            Input features matrix.
633
634
        raw_score : bool, optional (default=False)
            Whether to predict raw scores.
635
        num_iteration : int or None, optional (default=None)
636
            Limit number of iterations in the prediction.
637
638
            If None, if the best iteration exists, it is used; otherwise, all trees are used.
            If <= 0, all trees are used (no limits).
639
640
641
642
        pred_leaf : bool, optional (default=False)
            Whether to predict leaf index.
        pred_contrib : bool, optional (default=False)
            Whether to predict feature contributions.
643

Nikita Titov's avatar
Nikita Titov committed
644
645
646
647
648
649
650
            .. note::

                If you want to get more explanations for your model's predictions using SHAP values,
                like SHAP interaction values,
                you can install the shap package (https://github.com/slundberg/shap).
                Note that unlike the shap package, with ``pred_contrib`` we return a matrix with an extra
                column, where the last column is the expected value.
651

652
653
        **kwargs
            Other parameters for the prediction.
wxchan's avatar
wxchan committed
654
655
656

        Returns
        -------
657
658
        predicted_result : array-like of shape = [n_samples] or shape = [n_samples, n_classes]
            The predicted values.
659
        X_leaves : array-like of shape = [n_samples, n_trees] or shape = [n_samples, n_trees * n_classes]
Nikita Titov's avatar
Nikita Titov committed
660
            If ``pred_leaf=True``, the predicted leaf of every tree for each sample.
661
662
        X_SHAP_values : array-like of shape = [n_samples, n_features + 1] or shape = [n_samples, (n_features + 1) * n_classes]
            If ``pred_contrib=True``, the feature contributions for each sample.
wxchan's avatar
wxchan committed
663
        """
664
665
        if self._n_features is None:
            raise LGBMNotFittedError("Estimator not fitted, call `fit` before exploiting the model.")
666
        if not isinstance(X, (DataFrame, DataTable)):
667
            X = _LGBMCheckArray(X, accept_sparse=True, force_all_finite=False)
668
669
670
671
672
673
        n_features = X.shape[1]
        if self._n_features != n_features:
            raise ValueError("Number of features of the model must "
                             "match the input. Model n_features_ is %s and "
                             "input n_features is %s "
                             % (self._n_features, n_features))
674
        return self._Booster.predict(X, raw_score=raw_score, num_iteration=num_iteration,
675
                                     pred_leaf=pred_leaf, pred_contrib=pred_contrib, **kwargs)
wxchan's avatar
wxchan committed
676

677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
    @property
    def n_features_(self):
        """Get the number of features of fitted model."""
        if self._n_features is None:
            raise LGBMNotFittedError('No n_features found. Need to call fit beforehand.')
        return self._n_features

    @property
    def best_score_(self):
        """Get the best score of fitted model."""
        if self._n_features is None:
            raise LGBMNotFittedError('No best_score found. Need to call fit beforehand.')
        return self._best_score

    @property
    def best_iteration_(self):
        """Get the best iteration of fitted model."""
        if self._n_features is None:
            raise LGBMNotFittedError('No best_iteration found. Need to call fit with early_stopping_rounds beforehand.')
        return self._best_iteration

    @property
    def objective_(self):
        """Get the concrete objective used while fitting this model."""
        if self._n_features is None:
            raise LGBMNotFittedError('No objective found. Need to call fit beforehand.')
        return self._objective

705
706
707
708
    @property
    def booster_(self):
        """Get the underlying lightgbm Booster of this model."""
        if self._Booster is None:
709
            raise LGBMNotFittedError('No booster found. Need to call fit beforehand.')
710
        return self._Booster
wxchan's avatar
wxchan committed
711

712
713
714
    @property
    def evals_result_(self):
        """Get the evaluation results."""
715
716
717
        if self._n_features is None:
            raise LGBMNotFittedError('No results found. Need to call fit with eval_set beforehand.')
        return self._evals_result
718
719

    @property
720
    def feature_importances_(self):
721
        """Get feature importances.
722

Nikita Titov's avatar
Nikita Titov committed
723
724
725
726
        .. note::

            ``importance_type`` attribute is passed to the function
            to configure the type of importance values to be extracted.
727
        """
728
729
        if self._n_features is None:
            raise LGBMNotFittedError('No feature_importances found. Need to call fit beforehand.')
730
        return self._Booster.feature_importance(importance_type=self.importance_type)
wxchan's avatar
wxchan committed
731

wxchan's avatar
wxchan committed
732

733
734
class LGBMRegressor(LGBMModel, _LGBMRegressorBase):
    """LightGBM regressor."""
wxchan's avatar
wxchan committed
735

Guolin Ke's avatar
Guolin Ke committed
736
737
    def fit(self, X, y,
            sample_weight=None, init_score=None,
738
            eval_set=None, eval_names=None, eval_sample_weight=None,
739
            eval_init_score=None, eval_metric=None, early_stopping_rounds=None,
740
741
            verbose=True, feature_name='auto', categorical_feature='auto',
            callbacks=None, init_model=None):
742
        """Docstring is inherited from the LGBMModel."""
743
744
        super(LGBMRegressor, self).fit(X, y, sample_weight=sample_weight,
                                       init_score=init_score, eval_set=eval_set,
745
                                       eval_names=eval_names,
746
747
748
749
750
                                       eval_sample_weight=eval_sample_weight,
                                       eval_init_score=eval_init_score,
                                       eval_metric=eval_metric,
                                       early_stopping_rounds=early_stopping_rounds,
                                       verbose=verbose, feature_name=feature_name,
751
                                       categorical_feature=categorical_feature,
752
                                       callbacks=callbacks, init_model=init_model)
Guolin Ke's avatar
Guolin Ke committed
753
754
        return self

755
756
757
    _base_doc = LGBMModel.fit.__doc__
    fit.__doc__ = (_base_doc[:_base_doc.find('eval_class_weight :')]
                   + _base_doc[_base_doc.find('eval_init_score :'):])
wxchan's avatar
wxchan committed
758

759
760
761

class LGBMClassifier(LGBMModel, _LGBMClassifierBase):
    """LightGBM classifier."""
wxchan's avatar
wxchan committed
762

Guolin Ke's avatar
Guolin Ke committed
763
764
    def fit(self, X, y,
            sample_weight=None, init_score=None,
765
            eval_set=None, eval_names=None, eval_sample_weight=None,
766
            eval_class_weight=None, eval_init_score=None, eval_metric=None,
wxchan's avatar
wxchan committed
767
            early_stopping_rounds=None, verbose=True,
768
769
            feature_name='auto', categorical_feature='auto',
            callbacks=None, init_model=None):
770
        """Docstring is inherited from the LGBMModel."""
771
        _LGBMAssertAllFinite(y)
772
773
        _LGBMCheckClassificationTargets(y)
        self._le = _LGBMLabelEncoder().fit(y)
774
        _y = self._le.transform(y)
775
776
777
        self._class_map = dict(zip_(self._le.classes_, self._le.transform(self._le.classes_)))
        if isinstance(self.class_weight, dict):
            self._class_weight = {self._class_map[k]: v for k, v in self.class_weight.items()}
778

779
780
781
        self._classes = self._le.classes_
        self._n_classes = len(self._classes)
        if self._n_classes > 2:
wxchan's avatar
wxchan committed
782
            # Switch to using a multiclass objective in the underlying LGBM instance
783
            ova_aliases = {"multiclassova", "multiclass_ova", "ova", "ovr"}
784
            if self._objective not in ova_aliases and not callable(self._objective):
785
                self._objective = "multiclass"
786
            if eval_metric in {'logloss', 'binary_logloss'}:
wxchan's avatar
wxchan committed
787
                eval_metric = "multi_logloss"
788
            elif eval_metric in {'error', 'binary_error'}:
wxchan's avatar
wxchan committed
789
790
                eval_metric = "multi_error"
        else:
791
            if eval_metric in {'logloss', 'multi_logloss'}:
wxchan's avatar
wxchan committed
792
                eval_metric = 'binary_logloss'
793
            elif eval_metric in {'error', 'multi_error'}:
wxchan's avatar
wxchan committed
794
                eval_metric = 'binary_error'
wxchan's avatar
wxchan committed
795

796
797
        # do not modify args, as it causes errors in model selection tools
        valid_sets = None
wxchan's avatar
wxchan committed
798
        if eval_set is not None:
799
800
            if isinstance(eval_set, tuple):
                eval_set = [eval_set]
801
            valid_sets = [None] * len(eval_set)
802
803
            for i, (valid_x, valid_y) in enumerate(eval_set):
                if valid_x is X and valid_y is y:
804
                    valid_sets[i] = (valid_x, _y)
805
                else:
806
                    valid_sets[i] = (valid_x, self._le.transform(valid_y))
807

808
        super(LGBMClassifier, self).fit(X, _y, sample_weight=sample_weight,
809
                                        init_score=init_score, eval_set=valid_sets,
810
                                        eval_names=eval_names,
811
                                        eval_sample_weight=eval_sample_weight,
812
                                        eval_class_weight=eval_class_weight,
813
814
815
816
                                        eval_init_score=eval_init_score,
                                        eval_metric=eval_metric,
                                        early_stopping_rounds=early_stopping_rounds,
                                        verbose=verbose, feature_name=feature_name,
817
                                        categorical_feature=categorical_feature,
818
                                        callbacks=callbacks, init_model=init_model)
wxchan's avatar
wxchan committed
819
820
        return self

821
    fit.__doc__ = LGBMModel.fit.__doc__
822

823
    def predict(self, X, raw_score=False, num_iteration=None,
824
                pred_leaf=False, pred_contrib=False, **kwargs):
825
        """Docstring is inherited from the LGBMModel."""
826
827
        result = self.predict_proba(X, raw_score, num_iteration,
                                    pred_leaf, pred_contrib, **kwargs)
828
        if callable(self._objective) or raw_score or pred_leaf or pred_contrib:
829
830
831
832
            return result
        else:
            class_index = np.argmax(result, axis=1)
            return self._le.inverse_transform(class_index)
wxchan's avatar
wxchan committed
833

834
835
    predict.__doc__ = LGBMModel.predict.__doc__

836
    def predict_proba(self, X, raw_score=False, num_iteration=None,
837
                      pred_leaf=False, pred_contrib=False, **kwargs):
838
        """Return the predicted probability for each class for each sample.
wxchan's avatar
wxchan committed
839
840
841

        Parameters
        ----------
842
        X : array-like or sparse matrix of shape = [n_samples, n_features]
wxchan's avatar
wxchan committed
843
            Input features matrix.
844
845
        raw_score : bool, optional (default=False)
            Whether to predict raw scores.
846
        num_iteration : int or None, optional (default=None)
847
            Limit number of iterations in the prediction.
848
849
            If None, if the best iteration exists, it is used; otherwise, all trees are used.
            If <= 0, all trees are used (no limits).
850
851
852
853
        pred_leaf : bool, optional (default=False)
            Whether to predict leaf index.
        pred_contrib : bool, optional (default=False)
            Whether to predict feature contributions.
854

Nikita Titov's avatar
Nikita Titov committed
855
856
857
858
859
860
861
            .. note::

                If you want to get more explanations for your model's predictions using SHAP values,
                like SHAP interaction values,
                you can install the shap package (https://github.com/slundberg/shap).
                Note that unlike the shap package, with ``pred_contrib`` we return a matrix with an extra
                column, where the last column is the expected value.
862

863
864
        **kwargs
            Other parameters for the prediction.
wxchan's avatar
wxchan committed
865
866
867

        Returns
        -------
868
869
        predicted_probability : array-like of shape = [n_samples, n_classes]
            The predicted probability for each class for each sample.
870
        X_leaves : array-like of shape = [n_samples, n_trees * n_classes]
871
            If ``pred_leaf=True``, the predicted leaf of every tree for each sample.
872
        X_SHAP_values : array-like of shape = [n_samples, (n_features + 1) * n_classes]
873
            If ``pred_contrib=True``, the feature contributions for each sample.
wxchan's avatar
wxchan committed
874
        """
875
876
        result = super(LGBMClassifier, self).predict(X, raw_score, num_iteration,
                                                     pred_leaf, pred_contrib, **kwargs)
877
878
879
880
881
882
        if callable(self._objective) and not (raw_score or pred_leaf or pred_contrib):
            warnings.warn("Cannot compute class probabilities or labels "
                          "due to the usage of customized objective function.\n"
                          "Returning raw scores instead.")
            return result
        elif self._n_classes > 2 or raw_score or pred_leaf or pred_contrib:
883
            return result
wxchan's avatar
wxchan committed
884
        else:
885
            return np.vstack((1. - result, result)).transpose()
886
887
888

    @property
    def classes_(self):
889
890
891
892
        """Get the class label array."""
        if self._classes is None:
            raise LGBMNotFittedError('No classes found. Need to call fit beforehand.')
        return self._classes
893
894
895

    @property
    def n_classes_(self):
896
897
898
899
        """Get the number of classes."""
        if self._n_classes is None:
            raise LGBMNotFittedError('No classes found. Need to call fit beforehand.')
        return self._n_classes
wxchan's avatar
wxchan committed
900

wxchan's avatar
wxchan committed
901

wxchan's avatar
wxchan committed
902
class LGBMRanker(LGBMModel):
903
    """LightGBM ranker."""
wxchan's avatar
wxchan committed
904

Guolin Ke's avatar
Guolin Ke committed
905
    def fit(self, X, y,
906
            sample_weight=None, init_score=None, group=None,
907
            eval_set=None, eval_names=None, eval_sample_weight=None,
908
            eval_init_score=None, eval_group=None, eval_metric=None,
909
            eval_at=[1, 2, 3, 4, 5], early_stopping_rounds=None, verbose=True,
910
911
            feature_name='auto', categorical_feature='auto',
            callbacks=None, init_model=None):
912
        """Docstring is inherited from the LGBMModel."""
913
        # check group data
Guolin Ke's avatar
Guolin Ke committed
914
        if group is None:
915
            raise ValueError("Should set group for ranking task")
wxchan's avatar
wxchan committed
916
917

        if eval_set is not None:
Guolin Ke's avatar
Guolin Ke committed
918
            if eval_group is None:
919
                raise ValueError("Eval_group cannot be None when eval_set is not None")
Guolin Ke's avatar
Guolin Ke committed
920
            elif len(eval_group) != len(eval_set):
921
                raise ValueError("Length of eval_group should be equal to eval_set")
922
923
924
925
            elif (isinstance(eval_group, dict)
                  and any(i not in eval_group or eval_group[i] is None for i in range_(len(eval_group)))
                  or isinstance(eval_group, list)
                  and any(group is None for group in eval_group)):
926
927
                raise ValueError("Should set group for all eval datasets for ranking task; "
                                 "if you use dict, the index should start from 0")
928

929
        self._eval_at = eval_at
930
931
        super(LGBMRanker, self).fit(X, y, sample_weight=sample_weight,
                                    init_score=init_score, group=group,
932
933
                                    eval_set=eval_set, eval_names=eval_names,
                                    eval_sample_weight=eval_sample_weight,
934
935
936
937
                                    eval_init_score=eval_init_score, eval_group=eval_group,
                                    eval_metric=eval_metric,
                                    early_stopping_rounds=early_stopping_rounds,
                                    verbose=verbose, feature_name=feature_name,
938
                                    categorical_feature=categorical_feature,
939
                                    callbacks=callbacks, init_model=init_model)
wxchan's avatar
wxchan committed
940
        return self
941

942
943
944
945
    _base_doc = LGBMModel.fit.__doc__
    fit.__doc__ = (_base_doc[:_base_doc.find('eval_class_weight :')]
                   + _base_doc[_base_doc.find('eval_init_score :'):])
    _base_doc = fit.__doc__
946
947
    _before_early_stop, _early_stop, _after_early_stop = _base_doc.partition('early_stopping_rounds :')
    fit.__doc__ = (_before_early_stop
948
                   + 'eval_at : list of int, optional (default=[1, 2, 3, 4, 5])\n'
949
950
                   + ' ' * 12 + 'The evaluation positions of the specified metric.\n'
                   + ' ' * 8 + _early_stop + _after_early_stop)