sklearn.py 45.1 KB
Newer Older
wxchan's avatar
wxchan committed
1
# coding: utf-8
2
"""Scikit-learn wrapper interface for LightGBM."""
wxchan's avatar
wxchan committed
3
from __future__ import absolute_import
4

5
6
import warnings

wxchan's avatar
wxchan committed
7
import numpy as np
8

9
from .basic import Dataset, LightGBMError, _ConfigAliases
10
from .compat import (SKLEARN_INSTALLED, _LGBMClassifierBase,
11
12
                     LGBMNotFittedError, _LGBMLabelEncoder, _LGBMModelBase,
                     _LGBMRegressorBase, _LGBMCheckXY, _LGBMCheckArray, _LGBMCheckConsistentLength,
13
                     _LGBMAssertAllFinite, _LGBMCheckClassificationTargets, _LGBMComputeSampleWeight,
14
                     argc_, range_, zip_, string_type, DataFrame, DataTable)
wxchan's avatar
wxchan committed
15
from .engine import train
16

wxchan's avatar
wxchan committed
17

18
19
class _ObjectiveFunctionWrapper(object):
    """Proxy class for objective function."""
20

21
22
    def __init__(self, func):
        """Construct a proxy class.
23

24
25
        This class transforms objective function to match objective function with signature ``new_func(preds, dataset)``
        as expected by ``lightgbm.engine.train``.
26

27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
        Parameters
        ----------
        func : callable
            Expects a callable with signature ``func(y_true, y_pred)`` or ``func(y_true, y_pred, group)
            and returns (grad, hess):

                y_true : array-like of shape = [n_samples]
                    The target values.
                y_pred : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
                    The predicted values.
                group : array-like
                    Group/query data, used for ranking task.
                grad : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
                    The value of the first order derivative (gradient) for each sample point.
                hess : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
                    The value of the second order derivative (Hessian) for each sample point.
wxchan's avatar
wxchan committed
43

Nikita Titov's avatar
Nikita Titov committed
44
45
46
47
48
        .. note::

            For multi-class task, the y_pred is group by class_id first, then group by row_id.
            If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i]
            and you should group grad and hess in this way as well.
49
50
        """
        self.func = func
wxchan's avatar
wxchan committed
51

52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
    def __call__(self, preds, dataset):
        """Call passed function with appropriate arguments.

        Parameters
        ----------
        preds : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
            The predicted values.
        dataset : Dataset
            The training dataset.

        Returns
        -------
        grad : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
            The value of the first order derivative (gradient) for each sample point.
        hess : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
            The value of the second order derivative (Hessian) for each sample point.
        """
wxchan's avatar
wxchan committed
69
        labels = dataset.get_label()
70
        argc = argc_(self.func)
71
        if argc == 2:
72
            grad, hess = self.func(labels, preds)
73
        elif argc == 3:
74
            grad, hess = self.func(labels, preds, dataset.get_group())
75
        else:
wxchan's avatar
wxchan committed
76
            raise TypeError("Self-defined objective function should have 2 or 3 arguments, got %d" % argc)
wxchan's avatar
wxchan committed
77
78
79
80
81
82
83
84
85
86
87
        """weighted for objective"""
        weight = dataset.get_weight()
        if weight is not None:
            """only one class"""
            if len(weight) == len(grad):
                grad = np.multiply(grad, weight)
                hess = np.multiply(hess, weight)
            else:
                num_data = len(weight)
                num_class = len(grad) // num_data
                if num_class * num_data != len(grad):
88
                    raise ValueError("Length of grad and hess should equal to num_class * num_data")
wxchan's avatar
wxchan committed
89
90
                for k in range_(num_class):
                    for i in range_(num_data):
wxchan's avatar
wxchan committed
91
92
93
94
95
                        idx = k * num_data + i
                        grad[idx] *= weight[i]
                        hess[idx] *= weight[i]
        return grad, hess

wxchan's avatar
wxchan committed
96

97
98
class _EvalFunctionWrapper(object):
    """Proxy class for evaluation function."""
99

100
101
    def __init__(self, func):
        """Construct a proxy class.
102

103
104
        This class transforms evaluation function to match evaluation function with signature ``new_func(preds, dataset)``
        as expected by ``lightgbm.engine.train``.
105

106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
        Parameters
        ----------
        func : callable
            Expects a callable with following signatures:
            ``func(y_true, y_pred)``,
            ``func(y_true, y_pred, weight)``
            or ``func(y_true, y_pred, weight, group)``
            and returns (eval_name, eval_result, is_higher_better) or
            list of (eval_name, eval_result, is_higher_better):

                y_true : array-like of shape = [n_samples]
                    The target values.
                y_pred : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
                    The predicted values.
                weight : array-like of shape = [n_samples]
                    The weight of samples.
                group : array-like
                    Group/query data, used for ranking task.
                eval_name : string
125
                    The name of evaluation function (without whitespaces).
126
127
128
129
130
                eval_result : float
                    The eval result.
                is_higher_better : bool
                    Is eval result higher better, e.g. AUC is ``is_higher_better``.

Nikita Titov's avatar
Nikita Titov committed
131
132
133
134
        .. note::

            For multi-class task, the y_pred is group by class_id first, then group by row_id.
            If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i].
135
136
        """
        self.func = func
137

138
139
    def __call__(self, preds, dataset):
        """Call passed function with appropriate arguments.
140

141
142
143
144
145
146
147
148
149
150
        Parameters
        ----------
        preds : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
            The predicted values.
        dataset : Dataset
            The training dataset.

        Returns
        -------
        eval_name : string
151
            The name of evaluation function (without whitespaces).
152
153
154
155
156
        eval_result : float
            The eval result.
        is_higher_better : bool
            Is eval result higher better, e.g. AUC is ``is_higher_better``.
        """
157
        labels = dataset.get_label()
158
        argc = argc_(self.func)
159
        if argc == 2:
160
            return self.func(labels, preds)
161
        elif argc == 3:
162
            return self.func(labels, preds, dataset.get_weight())
163
        elif argc == 4:
164
            return self.func(labels, preds, dataset.get_weight(), dataset.get_group())
165
        else:
wxchan's avatar
wxchan committed
166
            raise TypeError("Self-defined eval function should have 2, 3 or 4 arguments, got %d" % argc)
167

wxchan's avatar
wxchan committed
168

169
170
class LGBMModel(_LGBMModelBase):
    """Implementation of the scikit-learn API for LightGBM."""
wxchan's avatar
wxchan committed
171

172
    def __init__(self, boosting_type='gbdt', num_leaves=31, max_depth=-1,
173
                 learning_rate=0.1, n_estimators=100,
174
                 subsample_for_bin=200000, objective=None, class_weight=None,
175
                 min_split_gain=0., min_child_weight=1e-3, min_child_samples=20,
176
                 subsample=1., subsample_freq=0, colsample_bytree=1.,
177
                 reg_alpha=0., reg_lambda=0., random_state=None,
178
                 n_jobs=-1, silent=True, importance_type='split', **kwargs):
179
        r"""Construct a gradient boosting model.
wxchan's avatar
wxchan committed
180
181
182

        Parameters
        ----------
183
        boosting_type : string, optional (default='gbdt')
184
185
186
187
188
            'gbdt', traditional Gradient Boosting Decision Tree.
            'dart', Dropouts meet Multiple Additive Regression Trees.
            'goss', Gradient-based One-Side Sampling.
            'rf', Random Forest.
        num_leaves : int, optional (default=31)
wxchan's avatar
wxchan committed
189
            Maximum tree leaves for base learners.
190
        max_depth : int, optional (default=-1)
191
            Maximum tree depth for base learners, <=0 means no limit.
192
        learning_rate : float, optional (default=0.1)
193
            Boosting learning rate.
194
195
196
            You can use ``callbacks`` parameter of ``fit`` method to shrink/adapt learning rate
            in training using ``reset_parameter`` callback.
            Note, that this will ignore the ``learning_rate`` argument in training.
197
        n_estimators : int, optional (default=100)
wxchan's avatar
wxchan committed
198
            Number of boosted trees to fit.
199
        subsample_for_bin : int, optional (default=200000)
wxchan's avatar
wxchan committed
200
            Number of samples for constructing bins.
201
        objective : string, callable or None, optional (default=None)
wxchan's avatar
wxchan committed
202
203
            Specify the learning task and the corresponding learning objective or
            a custom objective function to be used (see note below).
204
            Default: 'regression' for LGBMRegressor, 'binary' or 'multiclass' for LGBMClassifier, 'lambdarank' for LGBMRanker.
205
206
207
208
        class_weight : dict, 'balanced' or None, optional (default=None)
            Weights associated with classes in the form ``{class_label: weight}``.
            Use this parameter only for multi-class classification task;
            for binary classification task you may use ``is_unbalance`` or ``scale_pos_weight`` parameters.
209
210
211
            Note, that the usage of all these parameters will result in poor estimates of the individual class probabilities.
            You may want to consider performing probability calibration
            (https://scikit-learn.org/stable/modules/calibration.html) of your model.
212
213
214
            The 'balanced' mode uses the values of y to automatically adjust weights
            inversely proportional to class frequencies in the input data as ``n_samples / (n_classes * np.bincount(y))``.
            If None, all classes are supposed to have weight one.
215
            Note, that these weights will be multiplied with ``sample_weight`` (passed through the ``fit`` method)
216
            if ``sample_weight`` is specified.
217
        min_split_gain : float, optional (default=0.)
wxchan's avatar
wxchan committed
218
            Minimum loss reduction required to make a further partition on a leaf node of the tree.
219
        min_child_weight : float, optional (default=1e-3)
220
            Minimum sum of instance weight (hessian) needed in a child (leaf).
221
        min_child_samples : int, optional (default=20)
222
            Minimum number of data needed in a child (leaf).
223
        subsample : float, optional (default=1.)
wxchan's avatar
wxchan committed
224
            Subsample ratio of the training instance.
225
        subsample_freq : int, optional (default=0)
226
227
            Frequence of subsample, <=0 means no enable.
        colsample_bytree : float, optional (default=1.)
wxchan's avatar
wxchan committed
228
            Subsample ratio of columns when constructing each tree.
229
        reg_alpha : float, optional (default=0.)
230
            L1 regularization term on weights.
231
        reg_lambda : float, optional (default=0.)
232
            L2 regularization term on weights.
233
        random_state : int or None, optional (default=None)
wxchan's avatar
wxchan committed
234
            Random number seed.
235
            If None, default seeds in C++ code will be used.
236
        n_jobs : int, optional (default=-1)
237
            Number of parallel threads.
238
        silent : bool, optional (default=True)
wxchan's avatar
wxchan committed
239
            Whether to print messages while running boosting.
240
        importance_type : string, optional (default='split')
241
            The type of feature importance to be filled into ``feature_importances_``.
242
243
244
245
            If 'split', result contains numbers of times the feature is used in a model.
            If 'gain', result contains total gains of splits which use the feature.
        **kwargs
            Other parameters for the model.
wxchan's avatar
wxchan committed
246
            Check http://lightgbm.readthedocs.io/en/latest/Parameters.html for more parameters.
247

Nikita Titov's avatar
Nikita Titov committed
248
249
250
            .. warning::

                \*\*kwargs is not supported in sklearn, it may cause unexpected issues.
wxchan's avatar
wxchan committed
251

252
253
254
255
256
257
258
259
260
        Attributes
        ----------
        n_features_ : int
            The number of features of fitted model.
        classes_ : array of shape = [n_classes]
            The class label array (only for classification problem).
        n_classes_ : int
            The number of classes (only for classification problem).
        best_score_ : dict or None
261
            The best score of fitted model.
262
        best_iteration_ : int or None
263
            The best iteration of fitted model if ``early_stopping_rounds`` has been specified.
264
265
266
267
268
        objective_ : string or callable
            The concrete objective used while fitting this model.
        booster_ : Booster
            The underlying Booster of this model.
        evals_result_ : dict or None
269
            The evaluation results if ``early_stopping_rounds`` has been specified.
270
271
272
        feature_importances_ : array of shape = [n_features]
            The feature importances (the higher, the more important the feature).

wxchan's avatar
wxchan committed
273
274
        Note
        ----
275
276
        A custom objective function can be provided for the ``objective`` parameter.
        In this case, it should have the signature
277
278
        ``objective(y_true, y_pred) -> grad, hess`` or
        ``objective(y_true, y_pred, group) -> grad, hess``:
wxchan's avatar
wxchan committed
279

Nikita Titov's avatar
Nikita Titov committed
280
            y_true : array-like of shape = [n_samples]
281
                The target values.
Nikita Titov's avatar
Nikita Titov committed
282
            y_pred : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
283
                The predicted values.
Nikita Titov's avatar
Nikita Titov committed
284
            group : array-like
285
                Group/query data, used for ranking task.
Nikita Titov's avatar
Nikita Titov committed
286
            grad : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
287
                The value of the first order derivative (gradient) for each sample point.
Nikita Titov's avatar
Nikita Titov committed
288
            hess : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
289
                The value of the second order derivative (Hessian) for each sample point.
wxchan's avatar
wxchan committed
290

291
292
293
        For multi-class task, the y_pred is group by class_id first, then group by row_id.
        If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i]
        and you should group grad and hess in this way as well.
wxchan's avatar
wxchan committed
294
        """
wxchan's avatar
wxchan committed
295
        if not SKLEARN_INSTALLED:
296
            raise LightGBMError('Scikit-learn is required for this module')
wxchan's avatar
wxchan committed
297

298
        self.boosting_type = boosting_type
299
        self.objective = objective
wxchan's avatar
wxchan committed
300
301
302
303
        self.num_leaves = num_leaves
        self.max_depth = max_depth
        self.learning_rate = learning_rate
        self.n_estimators = n_estimators
wxchan's avatar
wxchan committed
304
        self.subsample_for_bin = subsample_for_bin
wxchan's avatar
wxchan committed
305
306
307
308
309
310
311
312
        self.min_split_gain = min_split_gain
        self.min_child_weight = min_child_weight
        self.min_child_samples = min_child_samples
        self.subsample = subsample
        self.subsample_freq = subsample_freq
        self.colsample_bytree = colsample_bytree
        self.reg_alpha = reg_alpha
        self.reg_lambda = reg_lambda
313
314
        self.random_state = random_state
        self.n_jobs = n_jobs
wxchan's avatar
wxchan committed
315
        self.silent = silent
316
        self.importance_type = importance_type
wxchan's avatar
wxchan committed
317
        self._Booster = None
318
319
320
321
        self._evals_result = None
        self._best_score = None
        self._best_iteration = None
        self._other_params = {}
322
        self._objective = objective
323
        self.class_weight = class_weight
324
325
        self._class_weight = None
        self._class_map = None
326
327
328
        self._n_features = None
        self._classes = None
        self._n_classes = None
329
        self.set_params(**kwargs)
wxchan's avatar
wxchan committed
330

Nikita Titov's avatar
Nikita Titov committed
331
332
333
334
    def _more_tags(self):
        return {'allow_nan': True,
                'X_types': ['2darray', 'sparse', '1dlabels']}

wxchan's avatar
wxchan committed
335
    def get_params(self, deep=True):
336
337
338
339
340
341
342
343
344
345
346
347
348
        """Get parameters for this estimator.

        Parameters
        ----------
        deep : bool, optional (default=True)
            If True, will return the parameters for this estimator and
            contained subobjects that are estimators.

        Returns
        -------
        params : dict
            Parameter names mapped to their values.
        """
wxchan's avatar
wxchan committed
349
        params = super(LGBMModel, self).get_params(deep=deep)
350
        params.update(self._other_params)
wxchan's avatar
wxchan committed
351
352
353
        return params

    def set_params(self, **params):
354
355
356
357
358
359
360
361
362
363
364
365
        """Set the parameters of this estimator.

        Parameters
        ----------
        **params
            Parameter names with their new values.

        Returns
        -------
        self : object
            Returns self.
        """
wxchan's avatar
wxchan committed
366
367
        for key, value in params.items():
            setattr(self, key, value)
368
369
            if hasattr(self, '_' + key):
                setattr(self, '_' + key, value)
370
            self._other_params[key] = value
wxchan's avatar
wxchan committed
371
        return self
wxchan's avatar
wxchan committed
372

Guolin Ke's avatar
Guolin Ke committed
373
    def fit(self, X, y,
374
            sample_weight=None, init_score=None, group=None,
375
            eval_set=None, eval_names=None, eval_sample_weight=None,
376
377
            eval_class_weight=None, eval_init_score=None, eval_group=None,
            eval_metric=None, early_stopping_rounds=None, verbose=True,
378
379
            feature_name='auto', categorical_feature='auto',
            callbacks=None, init_model=None):
380
        """Build a gradient boosting model from the training set (X, y).
wxchan's avatar
wxchan committed
381
382
383

        Parameters
        ----------
384
385
386
387
388
389
390
391
        X : array-like or sparse matrix of shape = [n_samples, n_features]
            Input feature matrix.
        y : array-like of shape = [n_samples]
            The target values (class labels in classification, real numbers in regression).
        sample_weight : array-like of shape = [n_samples] or None, optional (default=None)
            Weights of training data.
        init_score : array-like of shape = [n_samples] or None, optional (default=None)
            Init score of training data.
392
        group : array-like or None, optional (default=None)
393
394
            Group data of training data.
        eval_set : list or None, optional (default=None)
395
            A list of (X, y) tuple pairs to use as validation sets.
396
        eval_names : list of strings or None, optional (default=None)
397
398
399
            Names of eval_set.
        eval_sample_weight : list of arrays or None, optional (default=None)
            Weights of eval data.
400
401
        eval_class_weight : list or None, optional (default=None)
            Class weights of eval data.
402
403
404
405
406
407
        eval_init_score : list of arrays or None, optional (default=None)
            Init score of eval data.
        eval_group : list of arrays or None, optional (default=None)
            Group data of eval data.
        eval_metric : string, list of strings, callable or None, optional (default=None)
            If string, it should be a built-in evaluation metric to use.
408
            If callable, it should be a custom evaluation metric, see note below for more details.
Misha Lisovyi's avatar
Misha Lisovyi committed
409
            In either case, the ``metric`` from the model parameters will be evaluated and used as well.
410
            Default: 'l2' for LGBMRegressor, 'logloss' for LGBMClassifier, 'ndcg' for LGBMRanker.
411
412
        early_stopping_rounds : int or None, optional (default=None)
            Activates early stopping. The model will train until the validation score stops improving.
413
            Validation score needs to improve at least every ``early_stopping_rounds`` round(s)
414
            to continue training.
415
416
            Requires at least one validation data and one metric.
            If there's more than one, will check all of them. But the training data is ignored anyway.
417
418
            To check only the first metric, set the ``first_metric_only`` parameter to ``True``
            in additional parameters ``**kwargs`` of the model constructor.
419
420
421
422
423
424
        verbose : bool or int, optional (default=True)
            Requires at least one evaluation data.
            If True, the eval metric on the eval set is printed at each boosting stage.
            If int, the eval metric on the eval set is printed at every ``verbose`` boosting stage.
            The last boosting stage or the boosting stage found by using ``early_stopping_rounds`` is also printed.

Nikita Titov's avatar
Nikita Titov committed
425
426
            .. rubric:: Example

427
428
429
            With ``verbose`` = 4 and at least one item in ``eval_set``,
            an evaluation metric is printed every 4 (instead of 1) boosting stages.

430
        feature_name : list of strings or 'auto', optional (default='auto')
431
432
            Feature names.
            If 'auto' and data is pandas DataFrame, data columns names are used.
433
        categorical_feature : list of strings or int, or 'auto', optional (default='auto')
434
435
            Categorical features.
            If list of int, interpreted as indices.
436
            If list of strings, interpreted as feature names (need to specify ``feature_name`` as well).
437
            If 'auto' and data is pandas DataFrame, pandas unordered categorical columns are used.
438
            All values in categorical features should be less than int32 max value (2147483647).
439
            Large values could be memory consuming. Consider using consecutive integers starting from zero.
440
            All negative values in categorical features will be treated as missing values.
441
            The output cannot be monotonically constrained with respect to a categorical feature.
442
        callbacks : list of callback functions or None, optional (default=None)
443
            List of callback functions that are applied at each iteration.
444
            See Callbacks in Python API for more information.
445
446
        init_model : string, Booster, LGBMModel or None, optional (default=None)
            Filename of LightGBM model, Booster instance or LGBMModel instance used for continue training.
447

448
449
450
451
452
        Returns
        -------
        self : object
            Returns self.

453
454
        Note
        ----
455
        Custom eval function expects a callable with following signatures:
456
        ``func(y_true, y_pred)``, ``func(y_true, y_pred, weight)`` or
457
        ``func(y_true, y_pred, weight, group)``
458
459
        and returns (eval_name, eval_result, is_higher_better) or
        list of (eval_name, eval_result, is_higher_better):
460

Nikita Titov's avatar
Nikita Titov committed
461
            y_true : array-like of shape = [n_samples]
462
                The target values.
463
            y_pred : array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
464
                The predicted values.
Nikita Titov's avatar
Nikita Titov committed
465
            weight : array-like of shape = [n_samples]
466
                The weight of samples.
Nikita Titov's avatar
Nikita Titov committed
467
            group : array-like
468
                Group/query data, used for ranking task.
Nikita Titov's avatar
Nikita Titov committed
469
            eval_name : string
470
                The name of evaluation function (without whitespaces).
Nikita Titov's avatar
Nikita Titov committed
471
            eval_result : float
472
                The eval result.
473
474
            is_higher_better : bool
                Is eval result higher better, e.g. AUC is ``is_higher_better``.
475

476
477
        For multi-class task, the y_pred is group by class_id first, then group by row_id.
        If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i].
wxchan's avatar
wxchan committed
478
        """
479
480
481
482
483
484
485
486
487
488
        if self._objective is None:
            if isinstance(self, LGBMRegressor):
                self._objective = "regression"
            elif isinstance(self, LGBMClassifier):
                self._objective = "binary"
            elif isinstance(self, LGBMRanker):
                self._objective = "lambdarank"
            else:
                raise ValueError("Unknown LGBMModel type.")
        if callable(self._objective):
489
            self._fobj = _ObjectiveFunctionWrapper(self._objective)
490
491
        else:
            self._fobj = None
wxchan's avatar
wxchan committed
492
493
        evals_result = {}
        params = self.get_params()
wxchan's avatar
wxchan committed
494
        # user can set verbose with kwargs, it has higher priority
495
        if not any(verbose_alias in params for verbose_alias in _ConfigAliases.get("verbosity")) and self.silent:
496
            params['verbose'] = -1
wxchan's avatar
wxchan committed
497
        params.pop('silent', None)
498
        params.pop('importance_type', None)
wxchan's avatar
wxchan committed
499
        params.pop('n_estimators', None)
500
        params.pop('class_weight', None)
501
502
        for alias in _ConfigAliases.get('objective'):
            params.pop(alias, None)
503
        if self._n_classes is not None and self._n_classes > 2:
504
505
            for alias in _ConfigAliases.get('num_class'):
                params.pop(alias, None)
506
507
            params['num_class'] = self._n_classes
        if hasattr(self, '_eval_at'):
508
509
            for alias in _ConfigAliases.get('eval_at'):
                params.pop(alias, None)
510
            params['eval_at'] = self._eval_at
511
512
        params['objective'] = self._objective
        if self._fobj:
wxchan's avatar
wxchan committed
513
            params['objective'] = 'None'  # objective = nullptr for unknown objective
wxchan's avatar
wxchan committed
514
515

        if callable(eval_metric):
516
            feval = _EvalFunctionWrapper(eval_metric)
wxchan's avatar
wxchan committed
517
518
        else:
            feval = None
519
520
521
522
523
524
525
526
527
528
529
            # register default metric for consistency with callable eval_metric case
            original_metric = self._objective if isinstance(self._objective, string_type) else None
            if original_metric is None:
                # try to deduce from class instance
                if isinstance(self, LGBMRegressor):
                    original_metric = "l2"
                elif isinstance(self, LGBMClassifier):
                    original_metric = "multi_logloss" if self._n_classes > 2 else "binary_logloss"
                elif isinstance(self, LGBMRanker):
                    original_metric = "ndcg"
            # overwrite default metric by explicitly set metric
530
            for metric_alias in _ConfigAliases.get("metric"):
531
532
533
534
535
                if metric_alias in params:
                    original_metric = params.pop(metric_alias)
            # concatenate metric from params (or default if not provided in params) and eval_metric
            original_metric = [original_metric] if isinstance(original_metric, (string_type, type(None))) else original_metric
            eval_metric = [eval_metric] if isinstance(eval_metric, (string_type, type(None))) else eval_metric
536
537
            params['metric'] = [e for e in eval_metric if e not in original_metric] + original_metric
            params['metric'] = [metric for metric in params['metric'] if metric is not None]
wxchan's avatar
wxchan committed
538

539
        if not isinstance(X, (DataFrame, DataTable)):
540
541
542
543
            _X, _y = _LGBMCheckXY(X, y, accept_sparse=True, force_all_finite=False, ensure_min_samples=2)
            _LGBMCheckConsistentLength(_X, _y, sample_weight)
        else:
            _X, _y = X, y
544

545
546
547
548
        if self._class_weight is None:
            self._class_weight = self.class_weight
        if self._class_weight is not None:
            class_sample_weight = _LGBMComputeSampleWeight(self._class_weight, y)
549
550
551
552
            if sample_weight is None or len(sample_weight) == 0:
                sample_weight = class_sample_weight
            else:
                sample_weight = np.multiply(sample_weight, class_sample_weight)
553

554
        self._n_features = _X.shape[1]
555

Guolin Ke's avatar
Guolin Ke committed
556
        def _construct_dataset(X, y, sample_weight, init_score, group, params):
557
558
            return Dataset(X, label=y, weight=sample_weight, group=group,
                           init_score=init_score, params=params)
Guolin Ke's avatar
Guolin Ke committed
559

560
        train_set = _construct_dataset(_X, _y, sample_weight, init_score, group, params)
Guolin Ke's avatar
Guolin Ke committed
561
562
563

        valid_sets = []
        if eval_set is not None:
564

565
            def _get_meta_data(collection, name, i):
566
567
568
569
570
571
572
                if collection is None:
                    return None
                elif isinstance(collection, list):
                    return collection[i] if len(collection) > i else None
                elif isinstance(collection, dict):
                    return collection.get(i, None)
                else:
573
                    raise TypeError('{} should be dict or list'.format(name))
574

Guolin Ke's avatar
Guolin Ke committed
575
576
577
            if isinstance(eval_set, tuple):
                eval_set = [eval_set]
            for i, valid_data in enumerate(eval_set):
578
                # reduce cost for prediction training data
Guolin Ke's avatar
Guolin Ke committed
579
580
581
                if valid_data[0] is X and valid_data[1] is y:
                    valid_set = train_set
                else:
582
583
584
585
586
587
                    valid_weight = _get_meta_data(eval_sample_weight, 'eval_sample_weight', i)
                    valid_class_weight = _get_meta_data(eval_class_weight, 'eval_class_weight', i)
                    if valid_class_weight is not None:
                        if isinstance(valid_class_weight, dict) and self._class_map is not None:
                            valid_class_weight = {self._class_map[k]: v for k, v in valid_class_weight.items()}
                        valid_class_sample_weight = _LGBMComputeSampleWeight(valid_class_weight, valid_data[1])
588
589
590
591
                        if valid_weight is None or len(valid_weight) == 0:
                            valid_weight = valid_class_sample_weight
                        else:
                            valid_weight = np.multiply(valid_weight, valid_class_sample_weight)
592
593
                    valid_init_score = _get_meta_data(eval_init_score, 'eval_init_score', i)
                    valid_group = _get_meta_data(eval_group, 'eval_group', i)
594
595
                    valid_set = _construct_dataset(valid_data[0], valid_data[1],
                                                   valid_weight, valid_init_score, valid_group, params)
Guolin Ke's avatar
Guolin Ke committed
596
597
                valid_sets.append(valid_set)

598
599
600
        if isinstance(init_model, LGBMModel):
            init_model = init_model.booster_

Guolin Ke's avatar
Guolin Ke committed
601
        self._Booster = train(params, train_set,
602
                              self.n_estimators, valid_sets=valid_sets, valid_names=eval_names,
wxchan's avatar
wxchan committed
603
                              early_stopping_rounds=early_stopping_rounds,
604
                              evals_result=evals_result, fobj=self._fobj, feval=feval,
Guolin Ke's avatar
Guolin Ke committed
605
                              verbose_eval=verbose, feature_name=feature_name,
606
                              categorical_feature=categorical_feature,
607
                              callbacks=callbacks, init_model=init_model)
wxchan's avatar
wxchan committed
608
609

        if evals_result:
610
            self._evals_result = evals_result
wxchan's avatar
wxchan committed
611
612

        if early_stopping_rounds is not None:
613
            self._best_iteration = self._Booster.best_iteration
614
615

        self._best_score = self._Booster.best_score
wxchan's avatar
wxchan committed
616
617

        # free dataset
618
        self._Booster.free_dataset()
wxchan's avatar
wxchan committed
619
        del train_set, valid_sets
wxchan's avatar
wxchan committed
620
621
        return self

622
    def predict(self, X, raw_score=False, num_iteration=None,
623
                pred_leaf=False, pred_contrib=False, **kwargs):
624
        """Return the predicted value for each sample.
wxchan's avatar
wxchan committed
625
626
627

        Parameters
        ----------
628
        X : array-like or sparse matrix of shape = [n_samples, n_features]
wxchan's avatar
wxchan committed
629
            Input features matrix.
630
631
        raw_score : bool, optional (default=False)
            Whether to predict raw scores.
632
        num_iteration : int or None, optional (default=None)
633
            Limit number of iterations in the prediction.
634
635
            If None, if the best iteration exists, it is used; otherwise, all trees are used.
            If <= 0, all trees are used (no limits).
636
637
638
639
        pred_leaf : bool, optional (default=False)
            Whether to predict leaf index.
        pred_contrib : bool, optional (default=False)
            Whether to predict feature contributions.
640

Nikita Titov's avatar
Nikita Titov committed
641
642
643
644
645
646
647
            .. note::

                If you want to get more explanations for your model's predictions using SHAP values,
                like SHAP interaction values,
                you can install the shap package (https://github.com/slundberg/shap).
                Note that unlike the shap package, with ``pred_contrib`` we return a matrix with an extra
                column, where the last column is the expected value.
648

649
650
        **kwargs
            Other parameters for the prediction.
wxchan's avatar
wxchan committed
651
652
653

        Returns
        -------
654
655
        predicted_result : array-like of shape = [n_samples] or shape = [n_samples, n_classes]
            The predicted values.
656
        X_leaves : array-like of shape = [n_samples, n_trees] or shape = [n_samples, n_trees * n_classes]
Nikita Titov's avatar
Nikita Titov committed
657
            If ``pred_leaf=True``, the predicted leaf of every tree for each sample.
658
659
        X_SHAP_values : array-like of shape = [n_samples, n_features + 1] or shape = [n_samples, (n_features + 1) * n_classes]
            If ``pred_contrib=True``, the feature contributions for each sample.
wxchan's avatar
wxchan committed
660
        """
661
662
        if self._n_features is None:
            raise LGBMNotFittedError("Estimator not fitted, call `fit` before exploiting the model.")
663
        if not isinstance(X, (DataFrame, DataTable)):
664
            X = _LGBMCheckArray(X, accept_sparse=True, force_all_finite=False)
665
666
667
668
669
670
        n_features = X.shape[1]
        if self._n_features != n_features:
            raise ValueError("Number of features of the model must "
                             "match the input. Model n_features_ is %s and "
                             "input n_features is %s "
                             % (self._n_features, n_features))
671
        return self._Booster.predict(X, raw_score=raw_score, num_iteration=num_iteration,
672
                                     pred_leaf=pred_leaf, pred_contrib=pred_contrib, **kwargs)
wxchan's avatar
wxchan committed
673

674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
    @property
    def n_features_(self):
        """Get the number of features of fitted model."""
        if self._n_features is None:
            raise LGBMNotFittedError('No n_features found. Need to call fit beforehand.')
        return self._n_features

    @property
    def best_score_(self):
        """Get the best score of fitted model."""
        if self._n_features is None:
            raise LGBMNotFittedError('No best_score found. Need to call fit beforehand.')
        return self._best_score

    @property
    def best_iteration_(self):
        """Get the best iteration of fitted model."""
        if self._n_features is None:
            raise LGBMNotFittedError('No best_iteration found. Need to call fit with early_stopping_rounds beforehand.')
        return self._best_iteration

    @property
    def objective_(self):
        """Get the concrete objective used while fitting this model."""
        if self._n_features is None:
            raise LGBMNotFittedError('No objective found. Need to call fit beforehand.')
        return self._objective

702
703
704
705
    @property
    def booster_(self):
        """Get the underlying lightgbm Booster of this model."""
        if self._Booster is None:
706
            raise LGBMNotFittedError('No booster found. Need to call fit beforehand.')
707
        return self._Booster
wxchan's avatar
wxchan committed
708

709
710
711
    @property
    def evals_result_(self):
        """Get the evaluation results."""
712
713
714
        if self._n_features is None:
            raise LGBMNotFittedError('No results found. Need to call fit with eval_set beforehand.')
        return self._evals_result
715
716

    @property
717
    def feature_importances_(self):
718
        """Get feature importances.
719

Nikita Titov's avatar
Nikita Titov committed
720
721
722
723
        .. note::

            ``importance_type`` attribute is passed to the function
            to configure the type of importance values to be extracted.
724
        """
725
726
        if self._n_features is None:
            raise LGBMNotFittedError('No feature_importances found. Need to call fit beforehand.')
727
        return self._Booster.feature_importance(importance_type=self.importance_type)
wxchan's avatar
wxchan committed
728

wxchan's avatar
wxchan committed
729

730
731
class LGBMRegressor(LGBMModel, _LGBMRegressorBase):
    """LightGBM regressor."""
wxchan's avatar
wxchan committed
732

Guolin Ke's avatar
Guolin Ke committed
733
734
    def fit(self, X, y,
            sample_weight=None, init_score=None,
735
            eval_set=None, eval_names=None, eval_sample_weight=None,
736
            eval_init_score=None, eval_metric=None, early_stopping_rounds=None,
737
738
            verbose=True, feature_name='auto', categorical_feature='auto',
            callbacks=None, init_model=None):
739
        """Docstring is inherited from the LGBMModel."""
740
741
        super(LGBMRegressor, self).fit(X, y, sample_weight=sample_weight,
                                       init_score=init_score, eval_set=eval_set,
742
                                       eval_names=eval_names,
743
744
745
746
747
                                       eval_sample_weight=eval_sample_weight,
                                       eval_init_score=eval_init_score,
                                       eval_metric=eval_metric,
                                       early_stopping_rounds=early_stopping_rounds,
                                       verbose=verbose, feature_name=feature_name,
748
                                       categorical_feature=categorical_feature,
749
                                       callbacks=callbacks, init_model=init_model)
Guolin Ke's avatar
Guolin Ke committed
750
751
        return self

752
753
754
    _base_doc = LGBMModel.fit.__doc__
    fit.__doc__ = (_base_doc[:_base_doc.find('eval_class_weight :')]
                   + _base_doc[_base_doc.find('eval_init_score :'):])
wxchan's avatar
wxchan committed
755

756
757
758

class LGBMClassifier(LGBMModel, _LGBMClassifierBase):
    """LightGBM classifier."""
wxchan's avatar
wxchan committed
759

Guolin Ke's avatar
Guolin Ke committed
760
761
    def fit(self, X, y,
            sample_weight=None, init_score=None,
762
            eval_set=None, eval_names=None, eval_sample_weight=None,
763
            eval_class_weight=None, eval_init_score=None, eval_metric=None,
wxchan's avatar
wxchan committed
764
            early_stopping_rounds=None, verbose=True,
765
766
            feature_name='auto', categorical_feature='auto',
            callbacks=None, init_model=None):
767
        """Docstring is inherited from the LGBMModel."""
768
        _LGBMAssertAllFinite(y)
769
770
        _LGBMCheckClassificationTargets(y)
        self._le = _LGBMLabelEncoder().fit(y)
771
        _y = self._le.transform(y)
772
773
774
        self._class_map = dict(zip_(self._le.classes_, self._le.transform(self._le.classes_)))
        if isinstance(self.class_weight, dict):
            self._class_weight = {self._class_map[k]: v for k, v in self.class_weight.items()}
775

776
777
778
        self._classes = self._le.classes_
        self._n_classes = len(self._classes)
        if self._n_classes > 2:
wxchan's avatar
wxchan committed
779
            # Switch to using a multiclass objective in the underlying LGBM instance
780
            ova_aliases = {"multiclassova", "multiclass_ova", "ova", "ovr"}
781
            if self._objective not in ova_aliases and not callable(self._objective):
782
                self._objective = "multiclass"
783
            if eval_metric in {'logloss', 'binary_logloss'}:
wxchan's avatar
wxchan committed
784
                eval_metric = "multi_logloss"
785
            elif eval_metric in {'error', 'binary_error'}:
wxchan's avatar
wxchan committed
786
787
                eval_metric = "multi_error"
        else:
788
            if eval_metric in {'logloss', 'multi_logloss'}:
wxchan's avatar
wxchan committed
789
                eval_metric = 'binary_logloss'
790
            elif eval_metric in {'error', 'multi_error'}:
wxchan's avatar
wxchan committed
791
                eval_metric = 'binary_error'
wxchan's avatar
wxchan committed
792

793
794
        # do not modify args, as it causes errors in model selection tools
        valid_sets = None
wxchan's avatar
wxchan committed
795
        if eval_set is not None:
796
797
            if isinstance(eval_set, tuple):
                eval_set = [eval_set]
798
            valid_sets = [None] * len(eval_set)
799
800
            for i, (valid_x, valid_y) in enumerate(eval_set):
                if valid_x is X and valid_y is y:
801
                    valid_sets[i] = (valid_x, _y)
802
                else:
803
                    valid_sets[i] = (valid_x, self._le.transform(valid_y))
804

805
        super(LGBMClassifier, self).fit(X, _y, sample_weight=sample_weight,
806
                                        init_score=init_score, eval_set=valid_sets,
807
                                        eval_names=eval_names,
808
                                        eval_sample_weight=eval_sample_weight,
809
                                        eval_class_weight=eval_class_weight,
810
811
812
813
                                        eval_init_score=eval_init_score,
                                        eval_metric=eval_metric,
                                        early_stopping_rounds=early_stopping_rounds,
                                        verbose=verbose, feature_name=feature_name,
814
                                        categorical_feature=categorical_feature,
815
                                        callbacks=callbacks, init_model=init_model)
wxchan's avatar
wxchan committed
816
817
        return self

818
    fit.__doc__ = LGBMModel.fit.__doc__
819

820
    def predict(self, X, raw_score=False, num_iteration=None,
821
                pred_leaf=False, pred_contrib=False, **kwargs):
822
        """Docstring is inherited from the LGBMModel."""
823
824
        result = self.predict_proba(X, raw_score, num_iteration,
                                    pred_leaf, pred_contrib, **kwargs)
825
        if callable(self._objective) or raw_score or pred_leaf or pred_contrib:
826
827
828
829
            return result
        else:
            class_index = np.argmax(result, axis=1)
            return self._le.inverse_transform(class_index)
wxchan's avatar
wxchan committed
830

831
832
    predict.__doc__ = LGBMModel.predict.__doc__

833
    def predict_proba(self, X, raw_score=False, num_iteration=None,
834
                      pred_leaf=False, pred_contrib=False, **kwargs):
835
        """Return the predicted probability for each class for each sample.
wxchan's avatar
wxchan committed
836
837
838

        Parameters
        ----------
839
        X : array-like or sparse matrix of shape = [n_samples, n_features]
wxchan's avatar
wxchan committed
840
            Input features matrix.
841
842
        raw_score : bool, optional (default=False)
            Whether to predict raw scores.
843
        num_iteration : int or None, optional (default=None)
844
            Limit number of iterations in the prediction.
845
846
            If None, if the best iteration exists, it is used; otherwise, all trees are used.
            If <= 0, all trees are used (no limits).
847
848
849
850
        pred_leaf : bool, optional (default=False)
            Whether to predict leaf index.
        pred_contrib : bool, optional (default=False)
            Whether to predict feature contributions.
851

Nikita Titov's avatar
Nikita Titov committed
852
853
854
855
856
857
858
            .. note::

                If you want to get more explanations for your model's predictions using SHAP values,
                like SHAP interaction values,
                you can install the shap package (https://github.com/slundberg/shap).
                Note that unlike the shap package, with ``pred_contrib`` we return a matrix with an extra
                column, where the last column is the expected value.
859

860
861
        **kwargs
            Other parameters for the prediction.
wxchan's avatar
wxchan committed
862
863
864

        Returns
        -------
865
866
        predicted_probability : array-like of shape = [n_samples, n_classes]
            The predicted probability for each class for each sample.
867
        X_leaves : array-like of shape = [n_samples, n_trees * n_classes]
868
            If ``pred_leaf=True``, the predicted leaf of every tree for each sample.
869
        X_SHAP_values : array-like of shape = [n_samples, (n_features + 1) * n_classes]
870
            If ``pred_contrib=True``, the feature contributions for each sample.
wxchan's avatar
wxchan committed
871
        """
872
873
        result = super(LGBMClassifier, self).predict(X, raw_score, num_iteration,
                                                     pred_leaf, pred_contrib, **kwargs)
874
875
876
877
878
879
        if callable(self._objective) and not (raw_score or pred_leaf or pred_contrib):
            warnings.warn("Cannot compute class probabilities or labels "
                          "due to the usage of customized objective function.\n"
                          "Returning raw scores instead.")
            return result
        elif self._n_classes > 2 or raw_score or pred_leaf or pred_contrib:
880
            return result
wxchan's avatar
wxchan committed
881
        else:
882
            return np.vstack((1. - result, result)).transpose()
883
884
885

    @property
    def classes_(self):
886
887
888
889
        """Get the class label array."""
        if self._classes is None:
            raise LGBMNotFittedError('No classes found. Need to call fit beforehand.')
        return self._classes
890
891
892

    @property
    def n_classes_(self):
893
894
895
896
        """Get the number of classes."""
        if self._n_classes is None:
            raise LGBMNotFittedError('No classes found. Need to call fit beforehand.')
        return self._n_classes
wxchan's avatar
wxchan committed
897

wxchan's avatar
wxchan committed
898

wxchan's avatar
wxchan committed
899
class LGBMRanker(LGBMModel):
900
    """LightGBM ranker."""
wxchan's avatar
wxchan committed
901

Guolin Ke's avatar
Guolin Ke committed
902
    def fit(self, X, y,
903
            sample_weight=None, init_score=None, group=None,
904
            eval_set=None, eval_names=None, eval_sample_weight=None,
905
            eval_init_score=None, eval_group=None, eval_metric=None,
906
            eval_at=[1, 2, 3, 4, 5], early_stopping_rounds=None, verbose=True,
907
908
            feature_name='auto', categorical_feature='auto',
            callbacks=None, init_model=None):
909
        """Docstring is inherited from the LGBMModel."""
910
        # check group data
Guolin Ke's avatar
Guolin Ke committed
911
        if group is None:
912
            raise ValueError("Should set group for ranking task")
wxchan's avatar
wxchan committed
913
914

        if eval_set is not None:
Guolin Ke's avatar
Guolin Ke committed
915
            if eval_group is None:
916
                raise ValueError("Eval_group cannot be None when eval_set is not None")
Guolin Ke's avatar
Guolin Ke committed
917
            elif len(eval_group) != len(eval_set):
918
                raise ValueError("Length of eval_group should be equal to eval_set")
919
920
921
922
            elif (isinstance(eval_group, dict)
                  and any(i not in eval_group or eval_group[i] is None for i in range_(len(eval_group)))
                  or isinstance(eval_group, list)
                  and any(group is None for group in eval_group)):
923
924
                raise ValueError("Should set group for all eval datasets for ranking task; "
                                 "if you use dict, the index should start from 0")
925

926
        self._eval_at = eval_at
927
928
        super(LGBMRanker, self).fit(X, y, sample_weight=sample_weight,
                                    init_score=init_score, group=group,
929
930
                                    eval_set=eval_set, eval_names=eval_names,
                                    eval_sample_weight=eval_sample_weight,
931
932
933
934
                                    eval_init_score=eval_init_score, eval_group=eval_group,
                                    eval_metric=eval_metric,
                                    early_stopping_rounds=early_stopping_rounds,
                                    verbose=verbose, feature_name=feature_name,
935
                                    categorical_feature=categorical_feature,
936
                                    callbacks=callbacks, init_model=init_model)
wxchan's avatar
wxchan committed
937
        return self
938

939
940
941
942
    _base_doc = LGBMModel.fit.__doc__
    fit.__doc__ = (_base_doc[:_base_doc.find('eval_class_weight :')]
                   + _base_doc[_base_doc.find('eval_init_score :'):])
    _base_doc = fit.__doc__
943
944
    _before_early_stop, _early_stop, _after_early_stop = _base_doc.partition('early_stopping_rounds :')
    fit.__doc__ = (_before_early_stop
945
                   + 'eval_at : list of int, optional (default=[1, 2, 3, 4, 5])\n'
946
947
                   + ' ' * 12 + 'The evaluation positions of the specified metric.\n'
                   + ' ' * 8 + _early_stop + _after_early_stop)