sklearn.py 43.8 KB
Newer Older
wxchan's avatar
wxchan committed
1
# coding: utf-8
2
# pylint: disable = invalid-name, W0105, C0111, C0301
3
"""Scikit-learn wrapper interface for LightGBM."""
wxchan's avatar
wxchan committed
4
from __future__ import absolute_import
5

wxchan's avatar
wxchan committed
6
import numpy as np
7

wxchan's avatar
wxchan committed
8
from .basic import Dataset, LightGBMError
9
from .compat import (SKLEARN_INSTALLED, _LGBMClassifierBase,
10
11
                     LGBMNotFittedError, _LGBMLabelEncoder, _LGBMModelBase,
                     _LGBMRegressorBase, _LGBMCheckXY, _LGBMCheckArray, _LGBMCheckConsistentLength,
12
                     _LGBMAssertAllFinite, _LGBMCheckClassificationTargets, _LGBMComputeSampleWeight,
13
                     argc_, range_, zip_, string_type, DataFrame, DataTable)
wxchan's avatar
wxchan committed
14
from .engine import train
15

wxchan's avatar
wxchan committed
16

17
18
class _ObjectiveFunctionWrapper(object):
    """Proxy class for objective function."""
19

20
21
    def __init__(self, func):
        """Construct a proxy class.
22

23
24
        This class transforms objective function to match objective function with signature ``new_func(preds, dataset)``
        as expected by ``lightgbm.engine.train``.
25

26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
        Parameters
        ----------
        func : callable
            Expects a callable with signature ``func(y_true, y_pred)`` or ``func(y_true, y_pred, group)
            and returns (grad, hess):

                y_true : array-like of shape = [n_samples]
                    The target values.
                y_pred : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
                    The predicted values.
                group : array-like
                    Group/query data, used for ranking task.
                grad : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
                    The value of the first order derivative (gradient) for each sample point.
                hess : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
                    The value of the second order derivative (Hessian) for each sample point.
wxchan's avatar
wxchan committed
42

43
44
45
46
47
48
49
        Note
        ----
        For multi-class task, the y_pred is group by class_id first, then group by row_id.
        If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i]
        and you should group grad and hess in this way as well.
        """
        self.func = func
wxchan's avatar
wxchan committed
50

51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
    def __call__(self, preds, dataset):
        """Call passed function with appropriate arguments.

        Parameters
        ----------
        preds : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
            The predicted values.
        dataset : Dataset
            The training dataset.

        Returns
        -------
        grad : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
            The value of the first order derivative (gradient) for each sample point.
        hess : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
            The value of the second order derivative (Hessian) for each sample point.
        """
wxchan's avatar
wxchan committed
68
        labels = dataset.get_label()
69
        argc = argc_(self.func)
70
        if argc == 2:
71
            grad, hess = self.func(labels, preds)
72
        elif argc == 3:
73
            grad, hess = self.func(labels, preds, dataset.get_group())
74
        else:
wxchan's avatar
wxchan committed
75
            raise TypeError("Self-defined objective function should have 2 or 3 arguments, got %d" % argc)
wxchan's avatar
wxchan committed
76
77
78
79
80
81
82
83
84
85
86
        """weighted for objective"""
        weight = dataset.get_weight()
        if weight is not None:
            """only one class"""
            if len(weight) == len(grad):
                grad = np.multiply(grad, weight)
                hess = np.multiply(hess, weight)
            else:
                num_data = len(weight)
                num_class = len(grad) // num_data
                if num_class * num_data != len(grad):
87
                    raise ValueError("Length of grad and hess should equal to num_class * num_data")
wxchan's avatar
wxchan committed
88
89
                for k in range_(num_class):
                    for i in range_(num_data):
wxchan's avatar
wxchan committed
90
91
92
93
94
                        idx = k * num_data + i
                        grad[idx] *= weight[i]
                        hess[idx] *= weight[i]
        return grad, hess

wxchan's avatar
wxchan committed
95

96
97
class _EvalFunctionWrapper(object):
    """Proxy class for evaluation function."""
98

99
100
    def __init__(self, func):
        """Construct a proxy class.
101

102
103
        This class transforms evaluation function to match evaluation function with signature ``new_func(preds, dataset)``
        as expected by ``lightgbm.engine.train``.
104

105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
        Parameters
        ----------
        func : callable
            Expects a callable with following signatures:
            ``func(y_true, y_pred)``,
            ``func(y_true, y_pred, weight)``
            or ``func(y_true, y_pred, weight, group)``
            and returns (eval_name, eval_result, is_higher_better) or
            list of (eval_name, eval_result, is_higher_better):

                y_true : array-like of shape = [n_samples]
                    The target values.
                y_pred : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
                    The predicted values.
                weight : array-like of shape = [n_samples]
                    The weight of samples.
                group : array-like
                    Group/query data, used for ranking task.
                eval_name : string
                    The name of evaluation function.
                eval_result : float
                    The eval result.
                is_higher_better : bool
                    Is eval result higher better, e.g. AUC is ``is_higher_better``.

        Note
        ----
        For multi-class task, the y_pred is group by class_id first, then group by row_id.
        If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i].
        """
        self.func = func
136

137
138
    def __call__(self, preds, dataset):
        """Call passed function with appropriate arguments.
139

140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
        Parameters
        ----------
        preds : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
            The predicted values.
        dataset : Dataset
            The training dataset.

        Returns
        -------
        eval_name : string
            The name of evaluation function.
        eval_result : float
            The eval result.
        is_higher_better : bool
            Is eval result higher better, e.g. AUC is ``is_higher_better``.
        """
156
        labels = dataset.get_label()
157
        argc = argc_(self.func)
158
        if argc == 2:
159
            return self.func(labels, preds)
160
        elif argc == 3:
161
            return self.func(labels, preds, dataset.get_weight())
162
        elif argc == 4:
163
            return self.func(labels, preds, dataset.get_weight(), dataset.get_group())
164
        else:
wxchan's avatar
wxchan committed
165
            raise TypeError("Self-defined eval function should have 2, 3 or 4 arguments, got %d" % argc)
166

wxchan's avatar
wxchan committed
167

168
169
class LGBMModel(_LGBMModelBase):
    """Implementation of the scikit-learn API for LightGBM."""
wxchan's avatar
wxchan committed
170

171
    def __init__(self, boosting_type='gbdt', num_leaves=31, max_depth=-1,
172
                 learning_rate=0.1, n_estimators=100,
173
                 subsample_for_bin=200000, objective=None, class_weight=None,
174
                 min_split_gain=0., min_child_weight=1e-3, min_child_samples=20,
175
                 subsample=1., subsample_freq=0, colsample_bytree=1.,
176
                 reg_alpha=0., reg_lambda=0., random_state=None,
177
                 n_jobs=-1, silent=True, importance_type='split', **kwargs):
178
        r"""Construct a gradient boosting model.
wxchan's avatar
wxchan committed
179
180
181

        Parameters
        ----------
182
        boosting_type : string, optional (default='gbdt')
183
184
185
186
187
            'gbdt', traditional Gradient Boosting Decision Tree.
            'dart', Dropouts meet Multiple Additive Regression Trees.
            'goss', Gradient-based One-Side Sampling.
            'rf', Random Forest.
        num_leaves : int, optional (default=31)
wxchan's avatar
wxchan committed
188
            Maximum tree leaves for base learners.
189
        max_depth : int, optional (default=-1)
190
            Maximum tree depth for base learners, <=0 means no limit.
191
        learning_rate : float, optional (default=0.1)
192
            Boosting learning rate.
193
194
195
            You can use ``callbacks`` parameter of ``fit`` method to shrink/adapt learning rate
            in training using ``reset_parameter`` callback.
            Note, that this will ignore the ``learning_rate`` argument in training.
196
        n_estimators : int, optional (default=100)
wxchan's avatar
wxchan committed
197
            Number of boosted trees to fit.
198
        subsample_for_bin : int, optional (default=200000)
wxchan's avatar
wxchan committed
199
            Number of samples for constructing bins.
200
        objective : string, callable or None, optional (default=None)
wxchan's avatar
wxchan committed
201
202
            Specify the learning task and the corresponding learning objective or
            a custom objective function to be used (see note below).
203
            Default: 'regression' for LGBMRegressor, 'binary' or 'multiclass' for LGBMClassifier, 'lambdarank' for LGBMRanker.
204
205
206
207
        class_weight : dict, 'balanced' or None, optional (default=None)
            Weights associated with classes in the form ``{class_label: weight}``.
            Use this parameter only for multi-class classification task;
            for binary classification task you may use ``is_unbalance`` or ``scale_pos_weight`` parameters.
208
209
210
            Note, that the usage of all these parameters will result in poor estimates of the individual class probabilities.
            You may want to consider performing probability calibration
            (https://scikit-learn.org/stable/modules/calibration.html) of your model.
211
212
213
            The 'balanced' mode uses the values of y to automatically adjust weights
            inversely proportional to class frequencies in the input data as ``n_samples / (n_classes * np.bincount(y))``.
            If None, all classes are supposed to have weight one.
214
            Note, that these weights will be multiplied with ``sample_weight`` (passed through the ``fit`` method)
215
            if ``sample_weight`` is specified.
216
        min_split_gain : float, optional (default=0.)
wxchan's avatar
wxchan committed
217
            Minimum loss reduction required to make a further partition on a leaf node of the tree.
218
        min_child_weight : float, optional (default=1e-3)
219
            Minimum sum of instance weight (hessian) needed in a child (leaf).
220
        min_child_samples : int, optional (default=20)
221
            Minimum number of data needed in a child (leaf).
222
        subsample : float, optional (default=1.)
wxchan's avatar
wxchan committed
223
            Subsample ratio of the training instance.
224
        subsample_freq : int, optional (default=0)
225
226
            Frequence of subsample, <=0 means no enable.
        colsample_bytree : float, optional (default=1.)
wxchan's avatar
wxchan committed
227
            Subsample ratio of columns when constructing each tree.
228
        reg_alpha : float, optional (default=0.)
229
            L1 regularization term on weights.
230
        reg_lambda : float, optional (default=0.)
231
            L2 regularization term on weights.
232
        random_state : int or None, optional (default=None)
wxchan's avatar
wxchan committed
233
            Random number seed.
234
            If None, default seeds in C++ code will be used.
235
        n_jobs : int, optional (default=-1)
236
            Number of parallel threads.
237
        silent : bool, optional (default=True)
wxchan's avatar
wxchan committed
238
            Whether to print messages while running boosting.
239
        importance_type : string, optional (default='split')
240
            The type of feature importance to be filled into ``feature_importances_``.
241
242
243
244
            If 'split', result contains numbers of times the feature is used in a model.
            If 'gain', result contains total gains of splits which use the feature.
        **kwargs
            Other parameters for the model.
wxchan's avatar
wxchan committed
245
            Check http://lightgbm.readthedocs.io/en/latest/Parameters.html for more parameters.
246
247
248

            Note
            ----
249
            \*\*kwargs is not supported in sklearn, it may cause unexpected issues.
wxchan's avatar
wxchan committed
250

251
252
253
254
255
256
257
258
259
        Attributes
        ----------
        n_features_ : int
            The number of features of fitted model.
        classes_ : array of shape = [n_classes]
            The class label array (only for classification problem).
        n_classes_ : int
            The number of classes (only for classification problem).
        best_score_ : dict or None
260
            The best score of fitted model.
261
        best_iteration_ : int or None
262
            The best iteration of fitted model if ``early_stopping_rounds`` has been specified.
263
264
265
266
267
        objective_ : string or callable
            The concrete objective used while fitting this model.
        booster_ : Booster
            The underlying Booster of this model.
        evals_result_ : dict or None
268
            The evaluation results if ``early_stopping_rounds`` has been specified.
269
270
271
        feature_importances_ : array of shape = [n_features]
            The feature importances (the higher, the more important the feature).

wxchan's avatar
wxchan committed
272
273
        Note
        ----
274
275
        A custom objective function can be provided for the ``objective`` parameter.
        In this case, it should have the signature
276
277
        ``objective(y_true, y_pred) -> grad, hess`` or
        ``objective(y_true, y_pred, group) -> grad, hess``:
wxchan's avatar
wxchan committed
278

Nikita Titov's avatar
Nikita Titov committed
279
            y_true : array-like of shape = [n_samples]
280
                The target values.
Nikita Titov's avatar
Nikita Titov committed
281
            y_pred : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
282
                The predicted values.
Nikita Titov's avatar
Nikita Titov committed
283
            group : array-like
284
                Group/query data, used for ranking task.
Nikita Titov's avatar
Nikita Titov committed
285
            grad : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
286
                The value of the first order derivative (gradient) for each sample point.
Nikita Titov's avatar
Nikita Titov committed
287
            hess : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
288
                The value of the second order derivative (Hessian) for each sample point.
wxchan's avatar
wxchan committed
289

290
291
292
        For multi-class task, the y_pred is group by class_id first, then group by row_id.
        If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i]
        and you should group grad and hess in this way as well.
wxchan's avatar
wxchan committed
293
        """
wxchan's avatar
wxchan committed
294
        if not SKLEARN_INSTALLED:
295
            raise LightGBMError('Scikit-learn is required for this module')
wxchan's avatar
wxchan committed
296

297
        self.boosting_type = boosting_type
298
        self.objective = objective
wxchan's avatar
wxchan committed
299
300
301
302
        self.num_leaves = num_leaves
        self.max_depth = max_depth
        self.learning_rate = learning_rate
        self.n_estimators = n_estimators
wxchan's avatar
wxchan committed
303
        self.subsample_for_bin = subsample_for_bin
wxchan's avatar
wxchan committed
304
305
306
307
308
309
310
311
        self.min_split_gain = min_split_gain
        self.min_child_weight = min_child_weight
        self.min_child_samples = min_child_samples
        self.subsample = subsample
        self.subsample_freq = subsample_freq
        self.colsample_bytree = colsample_bytree
        self.reg_alpha = reg_alpha
        self.reg_lambda = reg_lambda
312
313
        self.random_state = random_state
        self.n_jobs = n_jobs
wxchan's avatar
wxchan committed
314
        self.silent = silent
315
        self.importance_type = importance_type
wxchan's avatar
wxchan committed
316
        self._Booster = None
317
318
319
320
        self._evals_result = None
        self._best_score = None
        self._best_iteration = None
        self._other_params = {}
321
        self._objective = objective
322
        self.class_weight = class_weight
323
324
        self._class_weight = None
        self._class_map = None
325
326
327
        self._n_features = None
        self._classes = None
        self._n_classes = None
328
        self.set_params(**kwargs)
wxchan's avatar
wxchan committed
329

Nikita Titov's avatar
Nikita Titov committed
330
331
332
333
    def _more_tags(self):
        return {'allow_nan': True,
                'X_types': ['2darray', 'sparse', '1dlabels']}

wxchan's avatar
wxchan committed
334
    def get_params(self, deep=True):
335
336
337
338
339
340
341
342
343
344
345
346
347
        """Get parameters for this estimator.

        Parameters
        ----------
        deep : bool, optional (default=True)
            If True, will return the parameters for this estimator and
            contained subobjects that are estimators.

        Returns
        -------
        params : dict
            Parameter names mapped to their values.
        """
wxchan's avatar
wxchan committed
348
        params = super(LGBMModel, self).get_params(deep=deep)
349
        params.update(self._other_params)
wxchan's avatar
wxchan committed
350
351
352
353
        return params

    # minor change to support `**kwargs`
    def set_params(self, **params):
354
355
356
357
358
359
360
361
362
363
364
365
        """Set the parameters of this estimator.

        Parameters
        ----------
        **params
            Parameter names with their new values.

        Returns
        -------
        self : object
            Returns self.
        """
wxchan's avatar
wxchan committed
366
367
        for key, value in params.items():
            setattr(self, key, value)
368
369
            if hasattr(self, '_' + key):
                setattr(self, '_' + key, value)
370
            self._other_params[key] = value
wxchan's avatar
wxchan committed
371
        return self
wxchan's avatar
wxchan committed
372

Guolin Ke's avatar
Guolin Ke committed
373
    def fit(self, X, y,
374
            sample_weight=None, init_score=None, group=None,
375
            eval_set=None, eval_names=None, eval_sample_weight=None,
376
377
378
            eval_class_weight=None, eval_init_score=None, eval_group=None,
            eval_metric=None, early_stopping_rounds=None, verbose=True,
            feature_name='auto', categorical_feature='auto', callbacks=None):
379
        """Build a gradient boosting model from the training set (X, y).
wxchan's avatar
wxchan committed
380
381
382

        Parameters
        ----------
383
384
385
386
387
388
389
390
        X : array-like or sparse matrix of shape = [n_samples, n_features]
            Input feature matrix.
        y : array-like of shape = [n_samples]
            The target values (class labels in classification, real numbers in regression).
        sample_weight : array-like of shape = [n_samples] or None, optional (default=None)
            Weights of training data.
        init_score : array-like of shape = [n_samples] or None, optional (default=None)
            Init score of training data.
391
        group : array-like or None, optional (default=None)
392
393
            Group data of training data.
        eval_set : list or None, optional (default=None)
394
            A list of (X, y) tuple pairs to use as validation sets.
395
        eval_names : list of strings or None, optional (default=None)
396
397
398
            Names of eval_set.
        eval_sample_weight : list of arrays or None, optional (default=None)
            Weights of eval data.
399
400
        eval_class_weight : list or None, optional (default=None)
            Class weights of eval data.
401
402
403
404
405
406
        eval_init_score : list of arrays or None, optional (default=None)
            Init score of eval data.
        eval_group : list of arrays or None, optional (default=None)
            Group data of eval data.
        eval_metric : string, list of strings, callable or None, optional (default=None)
            If string, it should be a built-in evaluation metric to use.
407
            If callable, it should be a custom evaluation metric, see note below for more details.
Misha Lisovyi's avatar
Misha Lisovyi committed
408
            In either case, the ``metric`` from the model parameters will be evaluated and used as well.
409
            Default: 'l2' for LGBMRegressor, 'logloss' for LGBMClassifier, 'ndcg' for LGBMRanker.
410
411
        early_stopping_rounds : int or None, optional (default=None)
            Activates early stopping. The model will train until the validation score stops improving.
412
            Validation score needs to improve at least every ``early_stopping_rounds`` round(s)
413
            to continue training.
414
415
            Requires at least one validation data and one metric.
            If there's more than one, will check all of them. But the training data is ignored anyway.
416
417
            To check only the first metric, set the ``first_metric_only`` parameter to ``True``
            in additional parameters ``**kwargs`` of the model constructor.
418
419
420
421
422
423
424
425
426
427
428
        verbose : bool or int, optional (default=True)
            Requires at least one evaluation data.
            If True, the eval metric on the eval set is printed at each boosting stage.
            If int, the eval metric on the eval set is printed at every ``verbose`` boosting stage.
            The last boosting stage or the boosting stage found by using ``early_stopping_rounds`` is also printed.

            Example
            -------
            With ``verbose`` = 4 and at least one item in ``eval_set``,
            an evaluation metric is printed every 4 (instead of 1) boosting stages.

429
        feature_name : list of strings or 'auto', optional (default='auto')
430
431
            Feature names.
            If 'auto' and data is pandas DataFrame, data columns names are used.
432
        categorical_feature : list of strings or int, or 'auto', optional (default='auto')
433
434
            Categorical features.
            If list of int, interpreted as indices.
435
            If list of strings, interpreted as feature names (need to specify ``feature_name`` as well).
436
            If 'auto' and data is pandas DataFrame, pandas unordered categorical columns are used.
437
            All values in categorical features should be less than int32 max value (2147483647).
438
            Large values could be memory consuming. Consider using consecutive integers starting from zero.
439
            All negative values in categorical features will be treated as missing values.
440
            The output cannot be monotonically constrained with respect to a categorical feature.
441
        callbacks : list of callback functions or None, optional (default=None)
442
            List of callback functions that are applied at each iteration.
443
            See Callbacks in Python API for more information.
444

445
446
447
448
449
        Returns
        -------
        self : object
            Returns self.

450
451
        Note
        ----
452
        Custom eval function expects a callable with following signatures:
453
        ``func(y_true, y_pred)``, ``func(y_true, y_pred, weight)`` or
454
        ``func(y_true, y_pred, weight, group)``
455
456
        and returns (eval_name, eval_result, is_higher_better) or
        list of (eval_name, eval_result, is_higher_better):
457

Nikita Titov's avatar
Nikita Titov committed
458
            y_true : array-like of shape = [n_samples]
459
                The target values.
460
            y_pred : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
461
                The predicted values.
Nikita Titov's avatar
Nikita Titov committed
462
            weight : array-like of shape = [n_samples]
463
                The weight of samples.
Nikita Titov's avatar
Nikita Titov committed
464
            group : array-like
465
                Group/query data, used for ranking task.
Nikita Titov's avatar
Nikita Titov committed
466
            eval_name : string
467
                The name of evaluation function.
Nikita Titov's avatar
Nikita Titov committed
468
            eval_result : float
469
                The eval result.
470
471
            is_higher_better : bool
                Is eval result higher better, e.g. AUC is ``is_higher_better``.
472

473
474
        For multi-class task, the y_pred is group by class_id first, then group by row_id.
        If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i].
wxchan's avatar
wxchan committed
475
        """
476
477
478
479
480
481
482
483
484
485
        if self._objective is None:
            if isinstance(self, LGBMRegressor):
                self._objective = "regression"
            elif isinstance(self, LGBMClassifier):
                self._objective = "binary"
            elif isinstance(self, LGBMRanker):
                self._objective = "lambdarank"
            else:
                raise ValueError("Unknown LGBMModel type.")
        if callable(self._objective):
486
            self._fobj = _ObjectiveFunctionWrapper(self._objective)
487
488
        else:
            self._fobj = None
wxchan's avatar
wxchan committed
489
490
        evals_result = {}
        params = self.get_params()
wxchan's avatar
wxchan committed
491
        # user can set verbose with kwargs, it has higher priority
492
        if not any(verbose_alias in params for verbose_alias in ('verbose', 'verbosity')) and self.silent:
493
            params['verbose'] = -1
wxchan's avatar
wxchan committed
494
        params.pop('silent', None)
495
        params.pop('importance_type', None)
wxchan's avatar
wxchan committed
496
        params.pop('n_estimators', None)
497
        params.pop('class_weight', None)
498
499
500
        if self._n_classes is not None and self._n_classes > 2:
            params['num_class'] = self._n_classes
        if hasattr(self, '_eval_at'):
501
            params['eval_at'] = self._eval_at
502
503
        params['objective'] = self._objective
        if self._fobj:
wxchan's avatar
wxchan committed
504
            params['objective'] = 'None'  # objective = nullptr for unknown objective
wxchan's avatar
wxchan committed
505
506

        if callable(eval_metric):
507
            feval = _EvalFunctionWrapper(eval_metric)
wxchan's avatar
wxchan committed
508
509
        else:
            feval = None
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
            # register default metric for consistency with callable eval_metric case
            original_metric = self._objective if isinstance(self._objective, string_type) else None
            if original_metric is None:
                # try to deduce from class instance
                if isinstance(self, LGBMRegressor):
                    original_metric = "l2"
                elif isinstance(self, LGBMClassifier):
                    original_metric = "multi_logloss" if self._n_classes > 2 else "binary_logloss"
                elif isinstance(self, LGBMRanker):
                    original_metric = "ndcg"
            # overwrite default metric by explicitly set metric
            for metric_alias in ['metric', 'metrics', 'metric_types']:
                if metric_alias in params:
                    original_metric = params.pop(metric_alias)
            # concatenate metric from params (or default if not provided in params) and eval_metric
            original_metric = [original_metric] if isinstance(original_metric, (string_type, type(None))) else original_metric
            eval_metric = [eval_metric] if isinstance(eval_metric, (string_type, type(None))) else eval_metric
            params['metric'] = set(original_metric + eval_metric)
wxchan's avatar
wxchan committed
528

529
        if not isinstance(X, (DataFrame, DataTable)):
530
531
532
533
            _X, _y = _LGBMCheckXY(X, y, accept_sparse=True, force_all_finite=False, ensure_min_samples=2)
            _LGBMCheckConsistentLength(_X, _y, sample_weight)
        else:
            _X, _y = X, y
534

535
536
537
538
        if self._class_weight is None:
            self._class_weight = self.class_weight
        if self._class_weight is not None:
            class_sample_weight = _LGBMComputeSampleWeight(self._class_weight, y)
539
540
541
542
            if sample_weight is None or len(sample_weight) == 0:
                sample_weight = class_sample_weight
            else:
                sample_weight = np.multiply(sample_weight, class_sample_weight)
543

544
        self._n_features = _X.shape[1]
545

Guolin Ke's avatar
Guolin Ke committed
546
        def _construct_dataset(X, y, sample_weight, init_score, group, params):
547
            ret = Dataset(X, label=y, weight=sample_weight, group=group, params=params)
Nikita Titov's avatar
Nikita Titov committed
548
            return ret.set_init_score(init_score)
Guolin Ke's avatar
Guolin Ke committed
549

550
        train_set = _construct_dataset(_X, _y, sample_weight, init_score, group, params)
Guolin Ke's avatar
Guolin Ke committed
551
552
553

        valid_sets = []
        if eval_set is not None:
554

555
            def _get_meta_data(collection, name, i):
556
557
558
559
560
561
562
                if collection is None:
                    return None
                elif isinstance(collection, list):
                    return collection[i] if len(collection) > i else None
                elif isinstance(collection, dict):
                    return collection.get(i, None)
                else:
563
                    raise TypeError('{} should be dict or list'.format(name))
564

Guolin Ke's avatar
Guolin Ke committed
565
566
567
            if isinstance(eval_set, tuple):
                eval_set = [eval_set]
            for i, valid_data in enumerate(eval_set):
568
                # reduce cost for prediction training data
Guolin Ke's avatar
Guolin Ke committed
569
570
571
                if valid_data[0] is X and valid_data[1] is y:
                    valid_set = train_set
                else:
572
573
574
575
576
577
                    valid_weight = _get_meta_data(eval_sample_weight, 'eval_sample_weight', i)
                    valid_class_weight = _get_meta_data(eval_class_weight, 'eval_class_weight', i)
                    if valid_class_weight is not None:
                        if isinstance(valid_class_weight, dict) and self._class_map is not None:
                            valid_class_weight = {self._class_map[k]: v for k, v in valid_class_weight.items()}
                        valid_class_sample_weight = _LGBMComputeSampleWeight(valid_class_weight, valid_data[1])
578
579
580
581
                        if valid_weight is None or len(valid_weight) == 0:
                            valid_weight = valid_class_sample_weight
                        else:
                            valid_weight = np.multiply(valid_weight, valid_class_sample_weight)
582
583
                    valid_init_score = _get_meta_data(eval_init_score, 'eval_init_score', i)
                    valid_group = _get_meta_data(eval_group, 'eval_group', i)
584
585
                    valid_set = _construct_dataset(valid_data[0], valid_data[1],
                                                   valid_weight, valid_init_score, valid_group, params)
Guolin Ke's avatar
Guolin Ke committed
586
587
588
                valid_sets.append(valid_set)

        self._Booster = train(params, train_set,
589
                              self.n_estimators, valid_sets=valid_sets, valid_names=eval_names,
wxchan's avatar
wxchan committed
590
                              early_stopping_rounds=early_stopping_rounds,
591
                              evals_result=evals_result, fobj=self._fobj, feval=feval,
Guolin Ke's avatar
Guolin Ke committed
592
                              verbose_eval=verbose, feature_name=feature_name,
593
                              categorical_feature=categorical_feature,
594
                              callbacks=callbacks)
wxchan's avatar
wxchan committed
595
596

        if evals_result:
597
            self._evals_result = evals_result
wxchan's avatar
wxchan committed
598
599

        if early_stopping_rounds is not None:
600
            self._best_iteration = self._Booster.best_iteration
601
602

        self._best_score = self._Booster.best_score
wxchan's avatar
wxchan committed
603
604
605
606

        # free dataset
        self.booster_.free_dataset()
        del train_set, valid_sets
wxchan's avatar
wxchan committed
607
608
        return self

609
    def predict(self, X, raw_score=False, num_iteration=None,
610
                pred_leaf=False, pred_contrib=False, **kwargs):
611
        """Return the predicted value for each sample.
wxchan's avatar
wxchan committed
612
613
614

        Parameters
        ----------
615
        X : array-like or sparse matrix of shape = [n_samples, n_features]
wxchan's avatar
wxchan committed
616
            Input features matrix.
617
618
        raw_score : bool, optional (default=False)
            Whether to predict raw scores.
619
        num_iteration : int or None, optional (default=None)
620
            Limit number of iterations in the prediction.
621
622
            If None, if the best iteration exists, it is used; otherwise, all trees are used.
            If <= 0, all trees are used (no limits).
623
624
625
626
        pred_leaf : bool, optional (default=False)
            Whether to predict leaf index.
        pred_contrib : bool, optional (default=False)
            Whether to predict feature contributions.
627
628
629

            Note
            ----
630
            If you want to get more explanations for your model's predictions using SHAP values,
631
            like SHAP interaction values,
632
633
634
            you can install the shap package (https://github.com/slundberg/shap).
            Note that unlike the shap package, with ``pred_contrib`` we return a matrix with an extra
            column, where the last column is the expected value.
635

636
637
        **kwargs
            Other parameters for the prediction.
wxchan's avatar
wxchan committed
638
639
640

        Returns
        -------
641
642
        predicted_result : array-like of shape = [n_samples] or shape = [n_samples, n_classes]
            The predicted values.
643
        X_leaves : array-like of shape = [n_samples, n_trees] or shape = [n_samples, n_trees * n_classes]
Nikita Titov's avatar
Nikita Titov committed
644
            If ``pred_leaf=True``, the predicted leaf of every tree for each sample.
645
646
        X_SHAP_values : array-like of shape = [n_samples, n_features + 1] or shape = [n_samples, (n_features + 1) * n_classes]
            If ``pred_contrib=True``, the feature contributions for each sample.
wxchan's avatar
wxchan committed
647
        """
648
649
        if self._n_features is None:
            raise LGBMNotFittedError("Estimator not fitted, call `fit` before exploiting the model.")
650
        if not isinstance(X, (DataFrame, DataTable)):
651
            X = _LGBMCheckArray(X, accept_sparse=True, force_all_finite=False)
652
653
654
655
656
657
        n_features = X.shape[1]
        if self._n_features != n_features:
            raise ValueError("Number of features of the model must "
                             "match the input. Model n_features_ is %s and "
                             "input n_features is %s "
                             % (self._n_features, n_features))
658
659
        return self.booster_.predict(X, raw_score=raw_score, num_iteration=num_iteration,
                                     pred_leaf=pred_leaf, pred_contrib=pred_contrib, **kwargs)
wxchan's avatar
wxchan committed
660

661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
    @property
    def n_features_(self):
        """Get the number of features of fitted model."""
        if self._n_features is None:
            raise LGBMNotFittedError('No n_features found. Need to call fit beforehand.')
        return self._n_features

    @property
    def best_score_(self):
        """Get the best score of fitted model."""
        if self._n_features is None:
            raise LGBMNotFittedError('No best_score found. Need to call fit beforehand.')
        return self._best_score

    @property
    def best_iteration_(self):
        """Get the best iteration of fitted model."""
        if self._n_features is None:
            raise LGBMNotFittedError('No best_iteration found. Need to call fit with early_stopping_rounds beforehand.')
        return self._best_iteration

    @property
    def objective_(self):
        """Get the concrete objective used while fitting this model."""
        if self._n_features is None:
            raise LGBMNotFittedError('No objective found. Need to call fit beforehand.')
        return self._objective

689
690
691
692
    @property
    def booster_(self):
        """Get the underlying lightgbm Booster of this model."""
        if self._Booster is None:
693
            raise LGBMNotFittedError('No booster found. Need to call fit beforehand.')
694
        return self._Booster
wxchan's avatar
wxchan committed
695

696
697
698
    @property
    def evals_result_(self):
        """Get the evaluation results."""
699
700
701
        if self._n_features is None:
            raise LGBMNotFittedError('No results found. Need to call fit with eval_set beforehand.')
        return self._evals_result
702
703

    @property
704
    def feature_importances_(self):
705
        """Get feature importances.
706

707
708
709
        Note
        ----
        Feature importance in sklearn interface used to normalize to 1,
710
711
712
        it's deprecated after 2.0.4 and is the same as Booster.feature_importance() now.
        ``importance_type`` attribute is passed to the function
        to configure the type of importance values to be extracted.
713
        """
714
715
        if self._n_features is None:
            raise LGBMNotFittedError('No feature_importances found. Need to call fit beforehand.')
716
        return self.booster_.feature_importance(importance_type=self.importance_type)
wxchan's avatar
wxchan committed
717

wxchan's avatar
wxchan committed
718

719
720
class LGBMRegressor(LGBMModel, _LGBMRegressorBase):
    """LightGBM regressor."""
wxchan's avatar
wxchan committed
721

Guolin Ke's avatar
Guolin Ke committed
722
723
    def fit(self, X, y,
            sample_weight=None, init_score=None,
724
            eval_set=None, eval_names=None, eval_sample_weight=None,
725
            eval_init_score=None, eval_metric=None, early_stopping_rounds=None,
726
            verbose=True, feature_name='auto', categorical_feature='auto', callbacks=None):
727
        """Docstring is inherited from the LGBMModel."""
728
729
        super(LGBMRegressor, self).fit(X, y, sample_weight=sample_weight,
                                       init_score=init_score, eval_set=eval_set,
730
                                       eval_names=eval_names,
731
732
733
734
735
                                       eval_sample_weight=eval_sample_weight,
                                       eval_init_score=eval_init_score,
                                       eval_metric=eval_metric,
                                       early_stopping_rounds=early_stopping_rounds,
                                       verbose=verbose, feature_name=feature_name,
736
                                       categorical_feature=categorical_feature,
Guolin Ke's avatar
Guolin Ke committed
737
                                       callbacks=callbacks)
Guolin Ke's avatar
Guolin Ke committed
738
739
        return self

740
741
742
    _base_doc = LGBMModel.fit.__doc__
    fit.__doc__ = (_base_doc[:_base_doc.find('eval_class_weight :')]
                   + _base_doc[_base_doc.find('eval_init_score :'):])
wxchan's avatar
wxchan committed
743

744
745
746

class LGBMClassifier(LGBMModel, _LGBMClassifierBase):
    """LightGBM classifier."""
wxchan's avatar
wxchan committed
747

Guolin Ke's avatar
Guolin Ke committed
748
749
    def fit(self, X, y,
            sample_weight=None, init_score=None,
750
            eval_set=None, eval_names=None, eval_sample_weight=None,
751
            eval_class_weight=None, eval_init_score=None, eval_metric=None,
wxchan's avatar
wxchan committed
752
            early_stopping_rounds=None, verbose=True,
753
            feature_name='auto', categorical_feature='auto', callbacks=None):
754
        """Docstring is inherited from the LGBMModel."""
755
        _LGBMAssertAllFinite(y)
756
757
        _LGBMCheckClassificationTargets(y)
        self._le = _LGBMLabelEncoder().fit(y)
758
        _y = self._le.transform(y)
759
760
761
        self._class_map = dict(zip_(self._le.classes_, self._le.transform(self._le.classes_)))
        if isinstance(self.class_weight, dict):
            self._class_weight = {self._class_map[k]: v for k, v in self.class_weight.items()}
762

763
764
765
        self._classes = self._le.classes_
        self._n_classes = len(self._classes)
        if self._n_classes > 2:
wxchan's avatar
wxchan committed
766
            # Switch to using a multiclass objective in the underlying LGBM instance
767
768
            ova_aliases = ("multiclassova", "multiclass_ova", "ova", "ovr")
            if self._objective not in ova_aliases and not callable(self._objective):
769
                self._objective = "multiclass"
770
            if eval_metric in ('logloss', 'binary_logloss'):
wxchan's avatar
wxchan committed
771
                eval_metric = "multi_logloss"
772
            elif eval_metric in ('error', 'binary_error'):
wxchan's avatar
wxchan committed
773
774
                eval_metric = "multi_error"
        else:
775
            if eval_metric in ('logloss', 'multi_logloss'):
wxchan's avatar
wxchan committed
776
                eval_metric = 'binary_logloss'
777
            elif eval_metric in ('error', 'multi_error'):
wxchan's avatar
wxchan committed
778
                eval_metric = 'binary_error'
wxchan's avatar
wxchan committed
779
780

        if eval_set is not None:
781
782
783
784
785
786
787
            if isinstance(eval_set, tuple):
                eval_set = [eval_set]
            for i, (valid_x, valid_y) in enumerate(eval_set):
                if valid_x is X and valid_y is y:
                    eval_set[i] = (valid_x, _y)
                else:
                    eval_set[i] = (valid_x, self._le.transform(valid_y))
788

789
        super(LGBMClassifier, self).fit(X, _y, sample_weight=sample_weight,
790
                                        init_score=init_score, eval_set=eval_set,
791
                                        eval_names=eval_names,
792
                                        eval_sample_weight=eval_sample_weight,
793
                                        eval_class_weight=eval_class_weight,
794
795
796
797
                                        eval_init_score=eval_init_score,
                                        eval_metric=eval_metric,
                                        early_stopping_rounds=early_stopping_rounds,
                                        verbose=verbose, feature_name=feature_name,
798
                                        categorical_feature=categorical_feature,
799
                                        callbacks=callbacks)
wxchan's avatar
wxchan committed
800
801
        return self

802
    fit.__doc__ = LGBMModel.fit.__doc__
803

804
    def predict(self, X, raw_score=False, num_iteration=None,
805
                pred_leaf=False, pred_contrib=False, **kwargs):
806
        """Docstring is inherited from the LGBMModel."""
807
808
809
810
811
812
813
        result = self.predict_proba(X, raw_score, num_iteration,
                                    pred_leaf, pred_contrib, **kwargs)
        if raw_score or pred_leaf or pred_contrib:
            return result
        else:
            class_index = np.argmax(result, axis=1)
            return self._le.inverse_transform(class_index)
wxchan's avatar
wxchan committed
814

815
816
    predict.__doc__ = LGBMModel.predict.__doc__

817
    def predict_proba(self, X, raw_score=False, num_iteration=None,
818
                      pred_leaf=False, pred_contrib=False, **kwargs):
819
        """Return the predicted probability for each class for each sample.
wxchan's avatar
wxchan committed
820
821
822

        Parameters
        ----------
823
        X : array-like or sparse matrix of shape = [n_samples, n_features]
wxchan's avatar
wxchan committed
824
            Input features matrix.
825
826
        raw_score : bool, optional (default=False)
            Whether to predict raw scores.
827
        num_iteration : int or None, optional (default=None)
828
            Limit number of iterations in the prediction.
829
830
            If None, if the best iteration exists, it is used; otherwise, all trees are used.
            If <= 0, all trees are used (no limits).
831
832
833
834
        pred_leaf : bool, optional (default=False)
            Whether to predict leaf index.
        pred_contrib : bool, optional (default=False)
            Whether to predict feature contributions.
835
836
837

            Note
            ----
838
            If you want to get more explanations for your model's predictions using SHAP values,
839
            like SHAP interaction values,
840
841
842
            you can install the shap package (https://github.com/slundberg/shap).
            Note that unlike the shap package, with ``pred_contrib`` we return a matrix with an extra
            column, where the last column is the expected value.
843

844
845
        **kwargs
            Other parameters for the prediction.
wxchan's avatar
wxchan committed
846
847
848

        Returns
        -------
849
850
        predicted_probability : array-like of shape = [n_samples, n_classes]
            The predicted probability for each class for each sample.
851
        X_leaves : array-like of shape = [n_samples, n_trees * n_classes]
852
            If ``pred_leaf=True``, the predicted leaf of every tree for each sample.
853
        X_SHAP_values : array-like of shape = [n_samples, (n_features + 1) * n_classes]
854
            If ``pred_contrib=True``, the feature contributions for each sample.
wxchan's avatar
wxchan committed
855
        """
856
857
        result = super(LGBMClassifier, self).predict(X, raw_score, num_iteration,
                                                     pred_leaf, pred_contrib, **kwargs)
858
        if self._n_classes > 2 or raw_score or pred_leaf or pred_contrib:
859
            return result
wxchan's avatar
wxchan committed
860
        else:
861
            return np.vstack((1. - result, result)).transpose()
862
863
864

    @property
    def classes_(self):
865
866
867
868
        """Get the class label array."""
        if self._classes is None:
            raise LGBMNotFittedError('No classes found. Need to call fit beforehand.')
        return self._classes
869
870
871

    @property
    def n_classes_(self):
872
873
874
875
        """Get the number of classes."""
        if self._n_classes is None:
            raise LGBMNotFittedError('No classes found. Need to call fit beforehand.')
        return self._n_classes
wxchan's avatar
wxchan committed
876

wxchan's avatar
wxchan committed
877

wxchan's avatar
wxchan committed
878
class LGBMRanker(LGBMModel):
879
    """LightGBM ranker."""
wxchan's avatar
wxchan committed
880

Guolin Ke's avatar
Guolin Ke committed
881
    def fit(self, X, y,
882
            sample_weight=None, init_score=None, group=None,
883
            eval_set=None, eval_names=None, eval_sample_weight=None,
884
            eval_init_score=None, eval_group=None, eval_metric=None,
885
886
            eval_at=[1], early_stopping_rounds=None, verbose=True,
            feature_name='auto', categorical_feature='auto', callbacks=None):
887
        """Docstring is inherited from the LGBMModel."""
888
        # check group data
Guolin Ke's avatar
Guolin Ke committed
889
        if group is None:
890
            raise ValueError("Should set group for ranking task")
wxchan's avatar
wxchan committed
891
892

        if eval_set is not None:
Guolin Ke's avatar
Guolin Ke committed
893
            if eval_group is None:
894
                raise ValueError("Eval_group cannot be None when eval_set is not None")
Guolin Ke's avatar
Guolin Ke committed
895
            elif len(eval_group) != len(eval_set):
896
                raise ValueError("Length of eval_group should be equal to eval_set")
897
898
899
900
            elif (isinstance(eval_group, dict)
                  and any(i not in eval_group or eval_group[i] is None for i in range_(len(eval_group)))
                  or isinstance(eval_group, list)
                  and any(group is None for group in eval_group)):
901
902
                raise ValueError("Should set group for all eval datasets for ranking task; "
                                 "if you use dict, the index should start from 0")
903

904
        self._eval_at = eval_at
905
906
        super(LGBMRanker, self).fit(X, y, sample_weight=sample_weight,
                                    init_score=init_score, group=group,
907
908
                                    eval_set=eval_set, eval_names=eval_names,
                                    eval_sample_weight=eval_sample_weight,
909
910
911
912
                                    eval_init_score=eval_init_score, eval_group=eval_group,
                                    eval_metric=eval_metric,
                                    early_stopping_rounds=early_stopping_rounds,
                                    verbose=verbose, feature_name=feature_name,
913
                                    categorical_feature=categorical_feature,
914
                                    callbacks=callbacks)
wxchan's avatar
wxchan committed
915
        return self
916

917
918
919
920
    _base_doc = LGBMModel.fit.__doc__
    fit.__doc__ = (_base_doc[:_base_doc.find('eval_class_weight :')]
                   + _base_doc[_base_doc.find('eval_init_score :'):])
    _base_doc = fit.__doc__
921
922
    _before_early_stop, _early_stop, _after_early_stop = _base_doc.partition('early_stopping_rounds :')
    fit.__doc__ = (_before_early_stop
923
                   + 'eval_at : list of int, optional (default=[1])\n'
924
925
                   + ' ' * 12 + 'The evaluation positions of the specified metric.\n'
                   + ' ' * 8 + _early_stop + _after_early_stop)