sklearn.py 34.4 KB
Newer Older
wxchan's avatar
wxchan committed
1
# coding: utf-8
2
# pylint: disable = invalid-name, W0105, C0111, C0301
wxchan's avatar
wxchan committed
3
4
"""Scikit-Learn Wrapper interface for LightGBM."""
from __future__ import absolute_import
5

wxchan's avatar
wxchan committed
6
import numpy as np
7
import warnings
8
9
10
11
12
try:
    import pandas as pd
    _IS_PANDAS_INSTALLED = True
except ImportError:
    _IS_PANDAS_INSTALLED = False
13

wxchan's avatar
wxchan committed
14
from .basic import Dataset, LightGBMError
15
from .compat import (SKLEARN_INSTALLED, _LGBMClassifierBase,
16
17
                     LGBMNotFittedError, _LGBMLabelEncoder, _LGBMModelBase,
                     _LGBMRegressorBase, _LGBMCheckXY, _LGBMCheckArray, _LGBMCheckConsistentLength,
18
                     _LGBMCheckClassificationTargets, argc_, range_, LGBMDeprecationWarning)
wxchan's avatar
wxchan committed
19
from .engine import train
20

wxchan's avatar
wxchan committed
21

22
def _objective_function_wrapper(func):
wxchan's avatar
wxchan committed
23
    """Decorate an objective function
24
25
26
27
    Note: for multi-class task, the y_pred is group by class_id first, then group by row_id.
          If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i]
          and you should group grad and hess in this way as well.

wxchan's avatar
wxchan committed
28
29
30
    Parameters
    ----------
    func: callable
31
        Expects a callable with signature ``func(y_true, y_pred)`` or ``func(y_true, y_pred, group):
32
33
34
35
36
37
            y_true: array-like of shape = [n_samples]
                The target values.
            y_pred: array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class)
                The predicted values.
            group: array-like
                Group/query data, used for ranking task.
wxchan's avatar
wxchan committed
38
39
40
41
42
43
44

    Returns
    -------
    new_func: callable
        The new objective function as expected by ``lightgbm.engine.train``.
        The signature is ``new_func(preds, dataset)``:

45
46
        preds: array-like of shape = [n_samples] or shape = [n_samples * n_classes]
            The predicted values.
wxchan's avatar
wxchan committed
47
48
        dataset: ``dataset``
            The training set from which the labels will be extracted using
49
            ``dataset.get_label()``.
wxchan's avatar
wxchan committed
50
51
52
53
    """
    def inner(preds, dataset):
        """internal function"""
        labels = dataset.get_label()
wxchan's avatar
wxchan committed
54
        argc = argc_(func)
55
56
57
58
59
        if argc == 2:
            grad, hess = func(labels, preds)
        elif argc == 3:
            grad, hess = func(labels, preds, dataset.get_group())
        else:
wxchan's avatar
wxchan committed
60
            raise TypeError("Self-defined objective function should have 2 or 3 arguments, got %d" % argc)
wxchan's avatar
wxchan committed
61
62
63
64
65
66
67
68
69
70
71
        """weighted for objective"""
        weight = dataset.get_weight()
        if weight is not None:
            """only one class"""
            if len(weight) == len(grad):
                grad = np.multiply(grad, weight)
                hess = np.multiply(hess, weight)
            else:
                num_data = len(weight)
                num_class = len(grad) // num_data
                if num_class * num_data != len(grad):
72
                    raise ValueError("Length of grad and hess should equal to num_class * num_data")
wxchan's avatar
wxchan committed
73
74
                for k in range_(num_class):
                    for i in range_(num_data):
wxchan's avatar
wxchan committed
75
76
77
78
79
80
                        idx = k * num_data + i
                        grad[idx] *= weight[i]
                        hess[idx] *= weight[i]
        return grad, hess
    return inner

wxchan's avatar
wxchan committed
81

82
83
def _eval_function_wrapper(func):
    """Decorate an eval function
84
85
86
    Note: for multi-class task, the y_pred is group by class_id first, then group by row_id.
          If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i].

87
88
89
    Parameters
    ----------
    func: callable
90
91
92
93
94
        Expects a callable with following functions:
            ``func(y_true, y_pred)``,
            ``func(y_true, y_pred, weight)``
         or ``func(y_true, y_pred, weight, group)``
            and return (eval_name->str, eval_result->float, is_bigger_better->Bool):
95

96
97
98
99
100
101
102
103
            y_true: array-like of shape = [n_samples]
                The target values.
            y_pred: array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class)
                The predicted values.
            weight: array_like of shape = [n_samples]
                The weight of samples.
            group: array-like
                Group/query data, used for ranking task.
104
105
106
107
108
109
110

    Returns
    -------
    new_func: callable
        The new eval function as expected by ``lightgbm.engine.train``.
        The signature is ``new_func(preds, dataset)``:

111
112
        preds: array-like of shape = [n_samples] or shape = [n_samples * n_classes]
            The predicted values.
113
114
        dataset: ``dataset``
            The training set from which the labels will be extracted using
115
            ``dataset.get_label()``.
116
117
118
119
    """
    def inner(preds, dataset):
        """internal function"""
        labels = dataset.get_label()
wxchan's avatar
wxchan committed
120
        argc = argc_(func)
121
122
123
124
125
126
127
        if argc == 2:
            return func(labels, preds)
        elif argc == 3:
            return func(labels, preds, dataset.get_weight())
        elif argc == 4:
            return func(labels, preds, dataset.get_weight(), dataset.get_group())
        else:
wxchan's avatar
wxchan committed
128
            raise TypeError("Self-defined eval function should have 2, 3 or 4 arguments, got %d" % argc)
129
130
    return inner

wxchan's avatar
wxchan committed
131

132
133
class LGBMModel(_LGBMModelBase):
    """Implementation of the scikit-learn API for LightGBM."""
wxchan's avatar
wxchan committed
134

135
    def __init__(self, boosting_type="gbdt", num_leaves=31, max_depth=-1,
wxchan's avatar
wxchan committed
136
                 learning_rate=0.1, n_estimators=10, max_bin=255,
137
138
                 subsample_for_bin=200000, objective=None,
                 min_split_gain=0., min_child_weight=1e-3, min_child_samples=20,
139
                 subsample=1., subsample_freq=1, colsample_bytree=1.,
140
                 reg_alpha=0., reg_lambda=0., random_state=None,
141
                 n_jobs=-1, silent=True, **kwargs):
142
        """Construct a gradient boosting model.
wxchan's avatar
wxchan committed
143
144
145

        Parameters
        ----------
146
147
148
149
150
151
        boosting_type : string, optional (default="gbdt")
            'gbdt', traditional Gradient Boosting Decision Tree.
            'dart', Dropouts meet Multiple Additive Regression Trees.
            'goss', Gradient-based One-Side Sampling.
            'rf', Random Forest.
        num_leaves : int, optional (default=31)
wxchan's avatar
wxchan committed
152
            Maximum tree leaves for base learners.
153
        max_depth : int, optional (default=-1)
wxchan's avatar
wxchan committed
154
            Maximum tree depth for base learners, -1 means no limit.
155
        learning_rate : float, optional (default=0.1)
156
            Boosting learning rate.
157
        n_estimators : int, optional (default=10)
wxchan's avatar
wxchan committed
158
            Number of boosted trees to fit.
159
        max_bin : int, optional (default=255)
160
            Number of bucketed bins for feature values.
161
        subsample_for_bin : int, optional (default=50000)
wxchan's avatar
wxchan committed
162
            Number of samples for constructing bins.
163
        objective : string, callable or None, optional (default=None)
wxchan's avatar
wxchan committed
164
165
            Specify the learning task and the corresponding learning objective or
            a custom objective function to be used (see note below).
166
            default: 'regression' for LGBMRegressor, 'binary' or 'multiclass' for LGBMClassifier, 'lambdarank' for LGBMRanker.
167
        min_split_gain : float, optional (default=0.)
wxchan's avatar
wxchan committed
168
            Minimum loss reduction required to make a further partition on a leaf node of the tree.
169
        min_child_weight : float, optional (default=1e-3)
170
            Minimum sum of instance weight(hessian) needed in a child(leaf).
171
        min_child_samples : int, optional (default=20)
172
            Minimum number of data need in a child(leaf).
173
        subsample : float, optional (default=1.)
wxchan's avatar
wxchan committed
174
            Subsample ratio of the training instance.
175
176
177
        subsample_freq : int, optional (default=1)
            Frequence of subsample, <=0 means no enable.
        colsample_bytree : float, optional (default=1.)
wxchan's avatar
wxchan committed
178
            Subsample ratio of columns when constructing each tree.
179
        reg_alpha : float, optional (default=0.)
180
            L1 regularization term on weights.
181
        reg_lambda : float, optional (default=0.)
182
            L2 regularization term on weights.
183
        random_state : int or None, optional (default=None)
wxchan's avatar
wxchan committed
184
            Random number seed.
185
            Will use default seeds in c++ code if set to None.
186
        n_jobs : int, optional (default=-1)
187
            Number of parallel threads.
188
        silent : bool, optional (default=True)
wxchan's avatar
wxchan committed
189
            Whether to print messages while running boosting.
wxchan's avatar
wxchan committed
190
191
        **kwargs : other parameters
            Check http://lightgbm.readthedocs.io/en/latest/Parameters.html for more parameters.
192
193
194

            Note
            ----
195
            \*\*kwargs is not supported in sklearn, it may cause unexpected issues.
wxchan's avatar
wxchan committed
196

197
198
199
200
201
202
203
204
205
        Attributes
        ----------
        n_features_ : int
            The number of features of fitted model.
        classes_ : array of shape = [n_classes]
            The class label array (only for classification problem).
        n_classes_ : int
            The number of classes (only for classification problem).
        best_score_ : dict or None
206
            The best score of fitted model.
207
        best_iteration_ : int or None
208
            The best iteration of fitted model if ``early_stopping_rounds`` has been specified.
209
210
211
212
213
        objective_ : string or callable
            The concrete objective used while fitting this model.
        booster_ : Booster
            The underlying Booster of this model.
        evals_result_ : dict or None
214
            The evaluation results if ``early_stopping_rounds`` has been specified.
215
216
217
        feature_importances_ : array of shape = [n_features]
            The feature importances (the higher, the more important the feature).

wxchan's avatar
wxchan committed
218
219
220
221
        Note
        ----
        A custom objective function can be provided for the ``objective``
        parameter. In this case, it should have the signature
222
223
        ``objective(y_true, y_pred) -> grad, hess`` or
        ``objective(y_true, y_pred, group) -> grad, hess``:
wxchan's avatar
wxchan committed
224

225
            y_true: array-like of shape = [n_samples]
226
                The target values.
227
            y_pred: array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
228
                The predicted values.
229
230
231
            group: array-like
                Group/query data, used for ranking task.
            grad: array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
wxchan's avatar
wxchan committed
232
                The value of the gradient for each sample point.
233
            hess: array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class task)
234
                The value of the second derivative for each sample point.
wxchan's avatar
wxchan committed
235

236
237
238
        For multi-class task, the y_pred is group by class_id first, then group by row_id.
        If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i]
        and you should group grad and hess in this way as well.
wxchan's avatar
wxchan committed
239
        """
wxchan's avatar
wxchan committed
240
        if not SKLEARN_INSTALLED:
241
            raise LightGBMError('Scikit-learn is required for this module')
wxchan's avatar
wxchan committed
242

243
        self.boosting_type = boosting_type
244
        self.objective = objective
wxchan's avatar
wxchan committed
245
246
247
248
249
        self.num_leaves = num_leaves
        self.max_depth = max_depth
        self.learning_rate = learning_rate
        self.n_estimators = n_estimators
        self.max_bin = max_bin
wxchan's avatar
wxchan committed
250
        self.subsample_for_bin = subsample_for_bin
wxchan's avatar
wxchan committed
251
252
253
254
255
256
257
258
        self.min_split_gain = min_split_gain
        self.min_child_weight = min_child_weight
        self.min_child_samples = min_child_samples
        self.subsample = subsample
        self.subsample_freq = subsample_freq
        self.colsample_bytree = colsample_bytree
        self.reg_alpha = reg_alpha
        self.reg_lambda = reg_lambda
259
260
        self.random_state = random_state
        self.n_jobs = n_jobs
wxchan's avatar
wxchan committed
261
        self.silent = silent
wxchan's avatar
wxchan committed
262
        self._Booster = None
263
264
265
266
        self._evals_result = None
        self._best_score = None
        self._best_iteration = None
        self._other_params = {}
267
        self._objective = objective
268
269
270
        self._n_features = None
        self._classes = None
        self._n_classes = None
271
        self.set_params(**kwargs)
wxchan's avatar
wxchan committed
272
273
274

    def get_params(self, deep=True):
        params = super(LGBMModel, self).get_params(deep=deep)
275
        params.update(self._other_params)
wxchan's avatar
wxchan committed
276
277
278
279
280
281
        return params

    # minor change to support `**kwargs`
    def set_params(self, **params):
        for key, value in params.items():
            setattr(self, key, value)
282
283
            if hasattr(self, '_' + key):
                setattr(self, '_' + key, value)
284
            self._other_params[key] = value
wxchan's avatar
wxchan committed
285
        return self
wxchan's avatar
wxchan committed
286

Guolin Ke's avatar
Guolin Ke committed
287
    def fit(self, X, y,
288
            sample_weight=None, init_score=None, group=None,
289
            eval_set=None, eval_names=None, eval_sample_weight=None,
290
291
292
293
            eval_init_score=None, eval_group=None, eval_metric=None,
            early_stopping_rounds=None, verbose=True, feature_name='auto',
            categorical_feature='auto', callbacks=None):
        """Build a gradient boosting model from the training set (X, y).
wxchan's avatar
wxchan committed
294
295
296

        Parameters
        ----------
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
        X : array-like or sparse matrix of shape = [n_samples, n_features]
            Input feature matrix.
        y : array-like of shape = [n_samples]
            The target values (class labels in classification, real numbers in regression).
        sample_weight : array-like of shape = [n_samples] or None, optional (default=None)
            Weights of training data.
        init_score : array-like of shape = [n_samples] or None, optional (default=None)
            Init score of training data.
        group : array-like of shape = [n_samples] or None, optional (default=None)
            Group data of training data.
        eval_set : list or None, optional (default=None)
            A list of (X, y) tuple pairs to use as a validation sets for early-stopping.
        eval_names: list of strings or None, optional (default=None)
            Names of eval_set.
        eval_sample_weight : list of arrays or None, optional (default=None)
            Weights of eval data.
        eval_init_score : list of arrays or None, optional (default=None)
            Init score of eval data.
        eval_group : list of arrays or None, optional (default=None)
            Group data of eval data.
        eval_metric : string, list of strings, callable or None, optional (default=None)
            If string, it should be a built-in evaluation metric to use.
            If callable, it should be a custom evaluation metric, see note for more details.
        early_stopping_rounds : int or None, optional (default=None)
            Activates early stopping. The model will train until the validation score stops improving.
322
            Validation error needs to decrease at least every ``early_stopping_rounds`` round(s)
323
324
325
326
327
328
329
330
331
            to continue training.
        verbose : bool, optional (default=True)
            If True and an evaluation set is used, writes the evaluation progress.
        feature_name : list of strings or 'auto', optional (default="auto")
            Feature names.
            If 'auto' and data is pandas DataFrame, data columns names are used.
        categorical_feature : list of strings or int, or 'auto', optional (default="auto")
            Categorical features.
            If list of int, interpreted as indices.
332
            If list of strings, interpreted as feature names (need to specify ``feature_name`` as well).
333
334
            If 'auto' and data is pandas DataFrame, pandas categorical columns are used.
        callbacks : list of callback functions or None, optional (default=None)
335
            List of callback functions that are applied at each iteration.
336
            See Callbacks in Python API for more information.
337

338
339
340
341
342
        Returns
        -------
        self : object
            Returns self.

343
344
        Note
        ----
wxchan's avatar
wxchan committed
345
        Custom eval function expects a callable with following functions:
346
347
348
349
350
351
352
353
354
355
356
357
358
        ``func(y_true, y_pred)``, ``func(y_true, y_pred, weight)`` or
        ``func(y_true, y_pred, weight, group)``.
        Returns (eval_name, eval_result, is_bigger_better) or
        list of (eval_name, eval_result, is_bigger_better)

            y_true: array-like of shape = [n_samples]
                The target values.
            y_pred: array-like of shape = [n_samples] or shape = [n_samples * n_classes] (for multi-class)
                The predicted values.
            weight: array-like of shape = [n_samples]
                The weight of samples.
            group: array-like
                Group/query data, used for ranking task.
359
            eval_name: str
360
                The name of evaluation.
361
            eval_result: float
362
                The eval result.
363
            is_bigger_better: bool
364
                Is eval result bigger better, e.g. AUC is bigger_better.
365

366
367
        For multi-class task, the y_pred is group by class_id first, then group by row_id.
        If you want to get i-th row y_pred in j-th class, the access way is y_pred[j * num_data + i].
wxchan's avatar
wxchan committed
368
        """
369
370
371
372
373
374
375
376
377
378
379
380
381
        if self._objective is None:
            if isinstance(self, LGBMRegressor):
                self._objective = "regression"
            elif isinstance(self, LGBMClassifier):
                self._objective = "binary"
            elif isinstance(self, LGBMRanker):
                self._objective = "lambdarank"
            else:
                raise ValueError("Unknown LGBMModel type.")
        if callable(self._objective):
            self._fobj = _objective_function_wrapper(self._objective)
        else:
            self._fobj = None
wxchan's avatar
wxchan committed
382
383
        evals_result = {}
        params = self.get_params()
384
385
386
        # sklearn interface has another naming convention
        params.setdefault('seed', params.pop('random_state'))
        params.setdefault('nthread', params.pop('n_jobs'))
wxchan's avatar
wxchan committed
387
388
389
390
391
        # user can set verbose with kwargs, it has higher priority
        if 'verbose' not in params and self.silent:
            params['verbose'] = -1
        params.pop('silent', None)
        params.pop('n_estimators', None)
392
393
394
395
396
397
        if self._n_classes is not None and self._n_classes > 2:
            params['num_class'] = self._n_classes
        if hasattr(self, '_eval_at'):
            params['ndcg_eval_at'] = self._eval_at
        params['objective'] = self._objective
        if self._fobj:
wxchan's avatar
wxchan committed
398
            params['objective'] = 'None'  # objective = nullptr for unknown objective
wxchan's avatar
wxchan committed
399
400

        if callable(eval_metric):
401
            feval = _eval_function_wrapper(eval_metric)
wxchan's avatar
wxchan committed
402
403
        else:
            feval = None
404
            params['metric'] = eval_metric
wxchan's avatar
wxchan committed
405

406
407
408
409
        if not _IS_PANDAS_INSTALLED or not isinstance(X, pd.DataFrame):
            X, y = _LGBMCheckXY(X, y, accept_sparse=True, force_all_finite=False, ensure_min_samples=2)
            _LGBMCheckConsistentLength(X, y, sample_weight)

410
411
        self._n_features = X.shape[1]

Guolin Ke's avatar
Guolin Ke committed
412
        def _construct_dataset(X, y, sample_weight, init_score, group, params):
Guolin Ke's avatar
Guolin Ke committed
413
            ret = Dataset(X, label=y, max_bin=self.max_bin, weight=sample_weight, group=group, params=params)
Guolin Ke's avatar
Guolin Ke committed
414
415
416
            ret.set_init_score(init_score)
            return ret

Guolin Ke's avatar
Guolin Ke committed
417
        train_set = _construct_dataset(X, y, sample_weight, init_score, group, params)
Guolin Ke's avatar
Guolin Ke committed
418
419
420
421
422
423

        valid_sets = []
        if eval_set is not None:
            if isinstance(eval_set, tuple):
                eval_set = [eval_set]
            for i, valid_data in enumerate(eval_set):
424
                # reduce cost for prediction training data
Guolin Ke's avatar
Guolin Ke committed
425
426
427
                if valid_data[0] is X and valid_data[1] is y:
                    valid_set = train_set
                else:
Tsukasa OMOTO's avatar
Tsukasa OMOTO committed
428
429
430
431
                    def get_meta_data(collection, i):
                        if collection is None:
                            return None
                        elif isinstance(collection, list):
432
                            return collection[i] if len(collection) > i else None
Tsukasa OMOTO's avatar
Tsukasa OMOTO committed
433
434
435
436
437
438
439
                        elif isinstance(collection, dict):
                            return collection.get(i, None)
                        else:
                            raise TypeError('eval_sample_weight, eval_init_score, and eval_group should be dict or list')
                    valid_weight = get_meta_data(eval_sample_weight, i)
                    valid_init_score = get_meta_data(eval_init_score, i)
                    valid_group = get_meta_data(eval_group, i)
Guolin Ke's avatar
Guolin Ke committed
440
                    valid_set = _construct_dataset(valid_data[0], valid_data[1], valid_weight, valid_init_score, valid_group, params)
Guolin Ke's avatar
Guolin Ke committed
441
442
443
                valid_sets.append(valid_set)

        self._Booster = train(params, train_set,
444
                              self.n_estimators, valid_sets=valid_sets, valid_names=eval_names,
wxchan's avatar
wxchan committed
445
                              early_stopping_rounds=early_stopping_rounds,
446
                              evals_result=evals_result, fobj=self._fobj, feval=feval,
Guolin Ke's avatar
Guolin Ke committed
447
                              verbose_eval=verbose, feature_name=feature_name,
448
                              categorical_feature=categorical_feature,
449
                              callbacks=callbacks)
wxchan's avatar
wxchan committed
450
451

        if evals_result:
452
            self._evals_result = evals_result
wxchan's avatar
wxchan committed
453
454

        if early_stopping_rounds is not None:
455
            self._best_iteration = self._Booster.best_iteration
456
457

        self._best_score = self._Booster.best_score
wxchan's avatar
wxchan committed
458
459
460
461

        # free dataset
        self.booster_.free_dataset()
        del train_set, valid_sets
wxchan's avatar
wxchan committed
462
463
        return self

464
    def predict(self, X, raw_score=False, num_iteration=0):
465
        """Return the predicted value for each sample.
wxchan's avatar
wxchan committed
466
467
468

        Parameters
        ----------
469
        X : array-like or sparse matrix of shape = [n_samples, n_features]
wxchan's avatar
wxchan committed
470
            Input features matrix.
471
472
473
        raw_score : bool, optional (default=False)
            Whether to predict raw scores.
        num_iteration : int, optional (default=0)
wxchan's avatar
wxchan committed
474
475
476
477
            Limit number of iterations in the prediction; defaults to 0 (use all trees).

        Returns
        -------
478
479
        predicted_result : array-like of shape = [n_samples] or shape = [n_samples, n_classes]
            The predicted values.
wxchan's avatar
wxchan committed
480
        """
481
482
        if self._n_features is None:
            raise LGBMNotFittedError("Estimator not fitted, call `fit` before exploiting the model.")
483
484
        if not _IS_PANDAS_INSTALLED or not isinstance(X, pd.DataFrame):
            X = _LGBMCheckArray(X, accept_sparse=True, force_all_finite=False)
485
486
487
488
489
490
        n_features = X.shape[1]
        if self._n_features != n_features:
            raise ValueError("Number of features of the model must "
                             "match the input. Model n_features_ is %s and "
                             "input n_features is %s "
                             % (self._n_features, n_features))
491
        return self.booster_.predict(X, raw_score=raw_score, num_iteration=num_iteration)
wxchan's avatar
wxchan committed
492
493

    def apply(self, X, num_iteration=0):
494
        """Return the predicted leaf every tree for each sample.
wxchan's avatar
wxchan committed
495
496
497

        Parameters
        ----------
498
        X : array-like or sparse matrix of shape = [n_samples, n_features]
wxchan's avatar
wxchan committed
499
            Input features matrix.
500
        num_iteration : int, optional (default=0)
wxchan's avatar
wxchan committed
501
            Limit number of iterations in the prediction; defaults to 0 (use all trees).
wxchan's avatar
wxchan committed
502
503
504

        Returns
        -------
505
506
        X_leaves : array-like of shape = [n_samples, n_trees]
            The predicted leaf every tree for each sample.
wxchan's avatar
wxchan committed
507
        """
508
509
        if self._n_features is None:
            raise LGBMNotFittedError("Estimator not fitted, call `fit` before exploiting the model.")
510
511
        if not _IS_PANDAS_INSTALLED or not isinstance(X, pd.DataFrame):
            X = _LGBMCheckArray(X, accept_sparse=True, force_all_finite=False)
512
513
514
515
516
517
        n_features = X.shape[1]
        if self._n_features != n_features:
            raise ValueError("Number of features of the model must "
                             "match the input. Model n_features_ is %s and "
                             "input n_features is %s "
                             % (self._n_features, n_features))
518
        return self.booster_.predict(X, pred_leaf=True, num_iteration=num_iteration)
wxchan's avatar
wxchan committed
519

520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
    @property
    def n_features_(self):
        """Get the number of features of fitted model."""
        if self._n_features is None:
            raise LGBMNotFittedError('No n_features found. Need to call fit beforehand.')
        return self._n_features

    @property
    def best_score_(self):
        """Get the best score of fitted model."""
        if self._n_features is None:
            raise LGBMNotFittedError('No best_score found. Need to call fit beforehand.')
        return self._best_score

    @property
    def best_iteration_(self):
        """Get the best iteration of fitted model."""
        if self._n_features is None:
            raise LGBMNotFittedError('No best_iteration found. Need to call fit with early_stopping_rounds beforehand.')
        return self._best_iteration

    @property
    def objective_(self):
        """Get the concrete objective used while fitting this model."""
        if self._n_features is None:
            raise LGBMNotFittedError('No objective found. Need to call fit beforehand.')
        return self._objective

548
549
550
551
    @property
    def booster_(self):
        """Get the underlying lightgbm Booster of this model."""
        if self._Booster is None:
552
            raise LGBMNotFittedError('No booster found. Need to call fit beforehand.')
553
        return self._Booster
wxchan's avatar
wxchan committed
554

555
556
557
    @property
    def evals_result_(self):
        """Get the evaluation results."""
558
559
560
        if self._n_features is None:
            raise LGBMNotFittedError('No results found. Need to call fit with eval_set beforehand.')
        return self._evals_result
561
562

    @property
563
    def feature_importances_(self):
564
        """Get feature importances.
565

566
567
568
        Note
        ----
        Feature importance in sklearn interface used to normalize to 1,
569
        it's deprecated after 2.0.4 and same as Booster.feature_importance() now.
570
        """
571
572
        if self._n_features is None:
            raise LGBMNotFittedError('No feature_importances found. Need to call fit beforehand.')
573
        return self.booster_.feature_importance()
wxchan's avatar
wxchan committed
574

wxchan's avatar
wxchan committed
575

576
577
class LGBMRegressor(LGBMModel, _LGBMRegressorBase):
    """LightGBM regressor."""
wxchan's avatar
wxchan committed
578

Guolin Ke's avatar
Guolin Ke committed
579
580
    def fit(self, X, y,
            sample_weight=None, init_score=None,
581
            eval_set=None, eval_names=None, eval_sample_weight=None,
582
583
            eval_init_score=None, eval_metric="l2", early_stopping_rounds=None,
            verbose=True, feature_name='auto', categorical_feature='auto', callbacks=None):
584
585
586

        super(LGBMRegressor, self).fit(X, y, sample_weight=sample_weight,
                                       init_score=init_score, eval_set=eval_set,
587
                                       eval_names=eval_names,
588
589
590
591
592
                                       eval_sample_weight=eval_sample_weight,
                                       eval_init_score=eval_init_score,
                                       eval_metric=eval_metric,
                                       early_stopping_rounds=early_stopping_rounds,
                                       verbose=verbose, feature_name=feature_name,
593
                                       categorical_feature=categorical_feature,
Guolin Ke's avatar
Guolin Ke committed
594
                                       callbacks=callbacks)
Guolin Ke's avatar
Guolin Ke committed
595
596
        return self

597
598
599
600
    base_doc = LGBMModel.fit.__doc__
    fit.__doc__ = (base_doc[:base_doc.find('eval_metric :')] +
                   'eval_metric : string, list of strings, callable or None, optional (default="l2")\n' +
                   base_doc[base_doc.find('            If string, it should be a built-in evaluation metric to use.'):])
wxchan's avatar
wxchan committed
601

602
603
604

class LGBMClassifier(LGBMModel, _LGBMClassifierBase):
    """LightGBM classifier."""
wxchan's avatar
wxchan committed
605

Guolin Ke's avatar
Guolin Ke committed
606
607
    def fit(self, X, y,
            sample_weight=None, init_score=None,
608
            eval_set=None, eval_names=None, eval_sample_weight=None,
609
            eval_init_score=None, eval_metric="logloss",
wxchan's avatar
wxchan committed
610
            early_stopping_rounds=None, verbose=True,
611
612
613
            feature_name='auto', categorical_feature='auto', callbacks=None):
        _LGBMCheckClassificationTargets(y)
        self._le = _LGBMLabelEncoder().fit(y)
614
        _y = self._le.transform(y)
615

616
617
618
        self._classes = self._le.classes_
        self._n_classes = len(self._classes)
        if self._n_classes > 2:
wxchan's avatar
wxchan committed
619
            # Switch to using a multiclass objective in the underlying LGBM instance
620
621
            if self._objective != "multiclassova" and not callable(self._objective):
                self._objective = "multiclass"
wxchan's avatar
wxchan committed
622
            if eval_metric == 'logloss' or eval_metric == 'binary_logloss':
wxchan's avatar
wxchan committed
623
                eval_metric = "multi_logloss"
wxchan's avatar
wxchan committed
624
625
626
627
628
629
630
            elif eval_metric == 'error' or eval_metric == 'binary_error':
                eval_metric = "multi_error"
        else:
            if eval_metric == 'logloss' or eval_metric == 'multi_logloss':
                eval_metric = 'binary_logloss'
            elif eval_metric == 'error' or eval_metric == 'multi_error':
                eval_metric = 'binary_error'
wxchan's avatar
wxchan committed
631
632

        if eval_set is not None:
633
634
635
636
637
638
639
            if isinstance(eval_set, tuple):
                eval_set = [eval_set]
            for i, (valid_x, valid_y) in enumerate(eval_set):
                if valid_x is X and valid_y is y:
                    eval_set[i] = (valid_x, _y)
                else:
                    eval_set[i] = (valid_x, self._le.transform(valid_y))
640

641
        super(LGBMClassifier, self).fit(X, _y, sample_weight=sample_weight,
642
                                        init_score=init_score, eval_set=eval_set,
643
                                        eval_names=eval_names,
644
645
646
647
648
                                        eval_sample_weight=eval_sample_weight,
                                        eval_init_score=eval_init_score,
                                        eval_metric=eval_metric,
                                        early_stopping_rounds=early_stopping_rounds,
                                        verbose=verbose, feature_name=feature_name,
649
                                        categorical_feature=categorical_feature,
650
                                        callbacks=callbacks)
wxchan's avatar
wxchan committed
651
652
        return self

653
654
655
656
657
    base_doc = LGBMModel.fit.__doc__
    fit.__doc__ = (base_doc[:base_doc.find('eval_metric :')] +
                   'eval_metric : string, list of strings, callable or None, optional (default="logloss")\n' +
                   base_doc[base_doc.find('            If string, it should be a built-in evaluation metric to use.'):])

658
659
660
661
    def predict(self, X, raw_score=False, num_iteration=0):
        class_probs = self.predict_proba(X, raw_score, num_iteration)
        class_index = np.argmax(class_probs, axis=1)
        return self._le.inverse_transform(class_index)
wxchan's avatar
wxchan committed
662

663
    def predict_proba(self, X, raw_score=False, num_iteration=0):
664
        """Return the predicted probability for each class for each sample.
wxchan's avatar
wxchan committed
665
666
667

        Parameters
        ----------
668
        X : array-like or sparse matrix of shape = [n_samples, n_features]
wxchan's avatar
wxchan committed
669
            Input features matrix.
670
671
672
        raw_score : bool, optional (default=False)
            Whether to predict raw scores.
        num_iteration : int, optional (default=0)
wxchan's avatar
wxchan committed
673
674
675
676
            Limit number of iterations in the prediction; defaults to 0 (use all trees).

        Returns
        -------
677
678
        predicted_probability : array-like of shape = [n_samples, n_classes]
            The predicted probability for each class for each sample.
wxchan's avatar
wxchan committed
679
        """
680
681
        if self._n_features is None:
            raise LGBMNotFittedError("Estimator not fitted, call `fit` before exploiting the model.")
682
683
        if not _IS_PANDAS_INSTALLED or not isinstance(X, pd.DataFrame):
            X = _LGBMCheckArray(X, accept_sparse=True, force_all_finite=False)
684
685
686
687
688
689
        n_features = X.shape[1]
        if self._n_features != n_features:
            raise ValueError("Number of features of the model must "
                             "match the input. Model n_features_ is %s and "
                             "input n_features is %s "
                             % (self._n_features, n_features))
690
        class_probs = self.booster_.predict(X, raw_score=raw_score, num_iteration=num_iteration)
691
        if self._n_classes > 2:
wxchan's avatar
wxchan committed
692
693
            return class_probs
        else:
694
695
696
697
            return np.vstack((1. - class_probs, class_probs)).transpose()

    @property
    def classes_(self):
698
699
700
701
        """Get the class label array."""
        if self._classes is None:
            raise LGBMNotFittedError('No classes found. Need to call fit beforehand.')
        return self._classes
702
703
704

    @property
    def n_classes_(self):
705
706
707
708
        """Get the number of classes."""
        if self._n_classes is None:
            raise LGBMNotFittedError('No classes found. Need to call fit beforehand.')
        return self._n_classes
wxchan's avatar
wxchan committed
709

wxchan's avatar
wxchan committed
710

wxchan's avatar
wxchan committed
711
class LGBMRanker(LGBMModel):
712
    """LightGBM ranker."""
wxchan's avatar
wxchan committed
713

Guolin Ke's avatar
Guolin Ke committed
714
    def fit(self, X, y,
715
            sample_weight=None, init_score=None, group=None,
716
            eval_set=None, eval_names=None, eval_sample_weight=None,
717
718
719
720
            eval_init_score=None, eval_group=None, eval_metric='ndcg',
            eval_at=[1], early_stopping_rounds=None, verbose=True,
            feature_name='auto', categorical_feature='auto', callbacks=None):
        # check group data
Guolin Ke's avatar
Guolin Ke committed
721
        if group is None:
722
            raise ValueError("Should set group for ranking task")
wxchan's avatar
wxchan committed
723
724

        if eval_set is not None:
Guolin Ke's avatar
Guolin Ke committed
725
            if eval_group is None:
726
                raise ValueError("Eval_group cannot be None when eval_set is not None")
Guolin Ke's avatar
Guolin Ke committed
727
            elif len(eval_group) != len(eval_set):
728
                raise ValueError("Length of eval_group should be equal to eval_set")
wxchan's avatar
wxchan committed
729
            elif (isinstance(eval_group, dict) and any(i not in eval_group or eval_group[i] is None for i in range_(len(eval_group)))) \
wxchan's avatar
wxchan committed
730
                    or (isinstance(eval_group, list) and any(group is None for group in eval_group)):
731
732
                raise ValueError("Should set group for all eval datasets for ranking task; "
                                 "if you use dict, the index should start from 0")
733

734
        self._eval_at = eval_at
735
736
        super(LGBMRanker, self).fit(X, y, sample_weight=sample_weight,
                                    init_score=init_score, group=group,
737
738
                                    eval_set=eval_set, eval_names=eval_names,
                                    eval_sample_weight=eval_sample_weight,
739
740
741
742
                                    eval_init_score=eval_init_score, eval_group=eval_group,
                                    eval_metric=eval_metric,
                                    early_stopping_rounds=early_stopping_rounds,
                                    verbose=verbose, feature_name=feature_name,
743
                                    categorical_feature=categorical_feature,
744
                                    callbacks=callbacks)
wxchan's avatar
wxchan committed
745
        return self
746
747
748
749
750
751
752
753

    base_doc = LGBMModel.fit.__doc__
    fit.__doc__ = (base_doc[:base_doc.find('eval_metric :')] +
                   'eval_metric : string, list of strings, callable or None, optional (default="ndcg")\n' +
                   base_doc[base_doc.find('            If string, it should be a built-in evaluation metric to use.'):base_doc.find('early_stopping_rounds :')] +
                   'eval_at : list of int, optional (default=[1])\n'
                   '            The evaluation positions of NDCG.\n' +
                   base_doc[base_doc.find('        early_stopping_rounds :'):])