sklearn.py 31.3 KB
Newer Older
wxchan's avatar
wxchan committed
1
# coding: utf-8
2
# pylint: disable = invalid-name, W0105, C0111, C0301
wxchan's avatar
wxchan committed
3
4
"""Scikit-Learn Wrapper interface for LightGBM."""
from __future__ import absolute_import
5

wxchan's avatar
wxchan committed
6
import numpy as np
7

wxchan's avatar
wxchan committed
8
9
10
11
from .basic import Dataset, LightGBMError
from .compat import (SKLEARN_INSTALLED, LGBMClassifierBase, LGBMDeprecated,
                     LGBMLabelEncoder, LGBMModelBase, LGBMRegressorBase, argc_,
                     range_)
wxchan's avatar
wxchan committed
12
from .engine import train
13

wxchan's avatar
wxchan committed
14

15
def _objective_function_wrapper(func):
wxchan's avatar
wxchan committed
16
17
18
19
20
21
22
    """Decorate an objective function
    Note: for multi-class task, the y_pred is group by class_id first, then group by row_id
          if you want to get i-th row y_pred in j-th class, the access way is y_pred[j*num_data+i]
          and you should group grad and hess in this way as well
    Parameters
    ----------
    func: callable
23
24
25
        Expects a callable with signature ``func(y_true, y_pred)`` or ``func(y_true, y_pred, group):
            y_true: array_like of shape [n_samples]
                The target values
26
            y_pred: array_like of shape [n_samples] or shape[n_samples * n_class] (for multi-class)
27
28
29
                The predicted values
            group: array_like
                group/query data, used for ranking task
wxchan's avatar
wxchan committed
30
31
32
33
34
35
36

    Returns
    -------
    new_func: callable
        The new objective function as expected by ``lightgbm.engine.train``.
        The signature is ``new_func(preds, dataset)``:

37
        preds: array_like, shape [n_samples] or shape[n_samples * n_class]
wxchan's avatar
wxchan committed
38
39
40
41
42
43
44
45
            The predicted values
        dataset: ``dataset``
            The training set from which the labels will be extracted using
            ``dataset.get_label()``
    """
    def inner(preds, dataset):
        """internal function"""
        labels = dataset.get_label()
wxchan's avatar
wxchan committed
46
        argc = argc_(func)
47
48
49
50
51
        if argc == 2:
            grad, hess = func(labels, preds)
        elif argc == 3:
            grad, hess = func(labels, preds, dataset.get_group())
        else:
wxchan's avatar
wxchan committed
52
            raise TypeError("Self-defined objective function should have 2 or 3 arguments, got %d" % argc)
wxchan's avatar
wxchan committed
53
54
55
56
57
58
59
60
61
62
63
        """weighted for objective"""
        weight = dataset.get_weight()
        if weight is not None:
            """only one class"""
            if len(weight) == len(grad):
                grad = np.multiply(grad, weight)
                hess = np.multiply(hess, weight)
            else:
                num_data = len(weight)
                num_class = len(grad) // num_data
                if num_class * num_data != len(grad):
64
                    raise ValueError("Length of grad and hess should equal to num_class * num_data")
wxchan's avatar
wxchan committed
65
66
                for k in range_(num_class):
                    for i in range_(num_data):
wxchan's avatar
wxchan committed
67
68
69
70
71
72
                        idx = k * num_data + i
                        grad[idx] *= weight[i]
                        hess[idx] *= weight[i]
        return grad, hess
    return inner

wxchan's avatar
wxchan committed
73

74
75
76
77
78
79
80
def _eval_function_wrapper(func):
    """Decorate an eval function
    Note: for multi-class task, the y_pred is group by class_id first, then group by row_id
          if you want to get i-th row y_pred in j-th class, the access way is y_pred[j*num_data+i]
    Parameters
    ----------
    func: callable
81
82
83
84
85
        Expects a callable with following functions:
            ``func(y_true, y_pred)``,
            ``func(y_true, y_pred, weight)``
         or ``func(y_true, y_pred, weight, group)``
            and return (eval_name->str, eval_result->float, is_bigger_better->Bool):
86
87
88

            y_true: array_like of shape [n_samples]
                The target values
89
            y_pred: array_like of shape [n_samples] or shape[n_samples * n_class] (for multi-class)
90
91
92
93
94
95
96
97
98
99
100
101
                The predicted values
            weight: array_like of shape [n_samples]
                The weight of samples
            group: array_like
                group/query data, used for ranking task

    Returns
    -------
    new_func: callable
        The new eval function as expected by ``lightgbm.engine.train``.
        The signature is ``new_func(preds, dataset)``:

102
        preds: array_like, shape [n_samples] or shape[n_samples * n_class]
103
104
105
106
107
108
109
110
            The predicted values
        dataset: ``dataset``
            The training set from which the labels will be extracted using
            ``dataset.get_label()``
    """
    def inner(preds, dataset):
        """internal function"""
        labels = dataset.get_label()
wxchan's avatar
wxchan committed
111
        argc = argc_(func)
112
113
114
115
116
117
118
        if argc == 2:
            return func(labels, preds)
        elif argc == 3:
            return func(labels, preds, dataset.get_weight())
        elif argc == 4:
            return func(labels, preds, dataset.get_weight(), dataset.get_group())
        else:
wxchan's avatar
wxchan committed
119
            raise TypeError("Self-defined eval function should have 2, 3 or 4 arguments, got %d" % argc)
120
121
    return inner

wxchan's avatar
wxchan committed
122

wxchan's avatar
wxchan committed
123
124
class LGBMModel(LGBMModelBase):

125
    def __init__(self, boosting_type="gbdt", num_leaves=31, max_depth=-1,
wxchan's avatar
wxchan committed
126
                 learning_rate=0.1, n_estimators=10, max_bin=255,
wxchan's avatar
wxchan committed
127
128
                 subsample_for_bin=50000, objective="regression",
                 min_split_gain=0, min_child_weight=5, min_child_samples=10,
wxchan's avatar
wxchan committed
129
130
                 subsample=1, subsample_freq=1, colsample_bytree=1,
                 reg_alpha=0, reg_lambda=0, scale_pos_weight=1,
wxchan's avatar
wxchan committed
131
                 is_unbalance=False, seed=0, nthread=-1, silent=True,
132
                 sigmoid=1.0, huber_delta=1.0, gaussian_eta=1.0, fair_c=1.0,
133
                 poisson_max_delta_step=0.7,
134
                 max_position=20, label_gain=None,
135
136
                 drop_rate=0.1, skip_drop=0.5, max_drop=50,
                 uniform_drop=False, xgboost_dart_mode=False):
wxchan's avatar
wxchan committed
137
138
139
140
141
        """
        Implementation of the Scikit-Learn API for LightGBM.

        Parameters
        ----------
142
143
144
        boosting_type : string
            gbdt, traditional Gradient Boosting Decision Tree
            dart, Dropouts meet Multiple Additive Regression Trees
wxchan's avatar
wxchan committed
145
146
147
148
149
150
151
152
        num_leaves : int
            Maximum tree leaves for base learners.
        max_depth : int
            Maximum tree depth for base learners, -1 means no limit.
        learning_rate : float
            Boosting learning rate
        n_estimators : int
            Number of boosted trees to fit.
Guolin Ke's avatar
Guolin Ke committed
153
154
        max_bin : int
            Number of bucketed bin for feature values
wxchan's avatar
wxchan committed
155
156
        subsample_for_bin : int
            Number of samples for constructing bins.
wxchan's avatar
wxchan committed
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
        objective : string or callable
            Specify the learning task and the corresponding learning objective or
            a custom objective function to be used (see note below).
            default: binary for LGBMClassifier, lambdarank for LGBMRanker
        min_split_gain : float
            Minimum loss reduction required to make a further partition on a leaf node of the tree.
        min_child_weight : int
            Minimum sum of instance weight(hessian) needed in a child(leaf)
        min_child_samples : int
            Minimum number of data need in a child(leaf)
        subsample : float
            Subsample ratio of the training instance.
        subsample_freq : int
            frequence of subsample, <=0 means no enable
        colsample_bytree : float
            Subsample ratio of columns when constructing each tree.
        reg_alpha : float
            L1 regularization term on weights
        reg_lambda : float
            L2 regularization term on weights
        scale_pos_weight : float
            Balancing of positive and negative weights.
        is_unbalance : bool
            Is unbalance for binary classification
        seed : int
            Random number seed.
wxchan's avatar
wxchan committed
183
184
185
186
187
188
        nthread : int
            Number of parallel threads
        silent : boolean
            Whether to print messages while running boosting.
        sigmoid : float
            Only used in binary classification and lambdarank. Parameter for sigmoid function.
Tsukasa OMOTO's avatar
Tsukasa OMOTO committed
189
190
        huber_delta : float
            Only used in regression. Parameter for Huber loss function.
191
192
193
        gaussian_eta : float
            Only used in regression. Parameter for L1 and Huber loss function.
            It is used to control the width of Gaussian function to approximate hessian.
Tsukasa OMOTO's avatar
Tsukasa OMOTO committed
194
195
        fair_c : float
            Only used in regression. Parameter for Fair loss function.
196
197
        poisson_max_delta_step : float
            parameter used to safeguard optimization in Poisson regression.
wxchan's avatar
wxchan committed
198
199
200
201
202
203
        max_position : int
            Only used in lambdarank, will optimize NDCG at this position.
        label_gain : list of float
            Only used in lambdarank, relevant gain for labels.
            For example, the gain of label 2 is 3 if using default label gains.
            None (default) means use default value of CLI version: {0,1,3,7,15,31,63,...}.
204
205
206
207
208
209
210
211
212
213
        drop_rate : float
            Only used when boosting_type='dart'. Probablity to select dropping trees.
        skip_drop : float
            Only used when boosting_type='dart'. Probablity to skip dropping trees.
        max_drop : int
            Only used when boosting_type='dart'. Max number of dropped trees in one iteration.
        uniform_drop : bool
            Only used when boosting_type='dart'. If true, drop trees uniformly, else drop according to weights.
        xgboost_dart_mode : bool
            Only used when boosting_type='dart'. Whether use xgboost dart mode.
wxchan's avatar
wxchan committed
214
215
216
217
218

        Note
        ----
        A custom objective function can be provided for the ``objective``
        parameter. In this case, it should have the signature
219
        ``objective(y_true, y_pred) -> grad, hess``
wxchan's avatar
wxchan committed
220
221
222
223
            or ``objective(y_true, y_pred, group) -> grad, hess``:

            y_true: array_like of shape [n_samples]
                The target values
224
            y_pred: array_like of shape [n_samples] or shape[n_samples * n_class]
wxchan's avatar
wxchan committed
225
226
227
                The predicted values
            group: array_like
                group/query data, used for ranking task
228
            grad: array_like of shape [n_samples] or shape[n_samples * n_class]
wxchan's avatar
wxchan committed
229
                The value of the gradient for each sample point.
230
            hess: array_like of shape [n_samples] or shape[n_samples * n_class]
wxchan's avatar
wxchan committed
231
232
233
234
235
236
                The value of the second derivative for each sample point

        for multi-class task, the y_pred is group by class_id first, then group by row_id
            if you want to get i-th row y_pred in j-th class, the access way is y_pred[j*num_data+i]
            and you should group grad and hess in this way as well
        """
wxchan's avatar
wxchan committed
237
        if not SKLEARN_INSTALLED:
238
            raise LightGBMError('Scikit-learn is required for this module')
wxchan's avatar
wxchan committed
239

240
        self.boosting_type = boosting_type
wxchan's avatar
wxchan committed
241
242
243
244
245
        self.num_leaves = num_leaves
        self.max_depth = max_depth
        self.learning_rate = learning_rate
        self.n_estimators = n_estimators
        self.max_bin = max_bin
wxchan's avatar
wxchan committed
246
        self.subsample_for_bin = subsample_for_bin
wxchan's avatar
wxchan committed
247
248
249
250
251
252
253
254
255
256
257
258
        self.objective = objective
        self.min_split_gain = min_split_gain
        self.min_child_weight = min_child_weight
        self.min_child_samples = min_child_samples
        self.subsample = subsample
        self.subsample_freq = subsample_freq
        self.colsample_bytree = colsample_bytree
        self.reg_alpha = reg_alpha
        self.reg_lambda = reg_lambda
        self.scale_pos_weight = scale_pos_weight
        self.is_unbalance = is_unbalance
        self.seed = seed
wxchan's avatar
wxchan committed
259
260
261
        self.nthread = nthread
        self.silent = silent
        self.sigmoid = sigmoid
Tsukasa OMOTO's avatar
Tsukasa OMOTO committed
262
        self.huber_delta = huber_delta
263
        self.gaussian_eta = gaussian_eta
Tsukasa OMOTO's avatar
Tsukasa OMOTO committed
264
        self.fair_c = fair_c
265
        self.poisson_max_delta_step = poisson_max_delta_step
wxchan's avatar
wxchan committed
266
267
        self.max_position = max_position
        self.label_gain = label_gain
268
269
270
271
272
        self.drop_rate = drop_rate
        self.skip_drop = skip_drop
        self.max_drop = max_drop
        self.uniform_drop = uniform_drop
        self.xgboost_dart_mode = xgboost_dart_mode
wxchan's avatar
wxchan committed
273
        self._Booster = None
274
        self.evals_result = None
275
        self.best_iteration = -1
wxchan's avatar
wxchan committed
276
        if callable(self.objective):
277
            self.fobj = _objective_function_wrapper(self.objective)
wxchan's avatar
wxchan committed
278
279
280
        else:
            self.fobj = None

Guolin Ke's avatar
Guolin Ke committed
281
    def fit(self, X, y,
282
283
            sample_weight=None, init_score=None, group=None,
            eval_set=None, eval_sample_weight=None,
Guolin Ke's avatar
Guolin Ke committed
284
285
            eval_init_score=None, eval_group=None,
            eval_metric=None,
wxchan's avatar
wxchan committed
286
            early_stopping_rounds=None, verbose=True,
Guolin Ke's avatar
Guolin Ke committed
287
            feature_name='auto',
288
            callbacks=None):
wxchan's avatar
wxchan committed
289
290
291
292
293
294
295
296
297
        """
        Fit the gradient boosting model

        Parameters
        ----------
        X : array_like
            Feature matrix
        y : array_like
            Labels
Guolin Ke's avatar
Guolin Ke committed
298
299
300
301
302
303
        sample_weight : array_like
            weight of training data
        init_score : array_like
            init score of training data
        group : array_like
            group data of training data
wxchan's avatar
wxchan committed
304
305
        eval_set : list, optional
            A list of (X, y) tuple pairs to use as a validation set for early-stopping
Guolin Ke's avatar
Guolin Ke committed
306
307
308
309
310
311
        eval_sample_weight : List of array
            weight of eval data
        eval_init_score : List of array
            init score of eval data
        eval_group : List of array
            group data of eval data
wxchan's avatar
wxchan committed
312
313
        eval_metric : str, list of str, callable, optional
            If a str, should be a built-in evaluation metric to use.
314
            If callable, a custom evaluation metric, see note for more details.
wxchan's avatar
wxchan committed
315
316
317
        early_stopping_rounds : int
        verbose : bool
            If `verbose` and an evaluation set is used, writes the evaluation
wxchan's avatar
wxchan committed
318
        feature_name : list of str, or 'auto'
319
            Feature names
wxchan's avatar
wxchan committed
320
            If 'auto' and data is pandas DataFrame, use data columns name
321
322
323
        callbacks : list of callback functions
            List of callback functions that are applied at each iteration.
            See Callbacks in Python-API.md for more information.
324
325
326

        Note
        ----
wxchan's avatar
wxchan committed
327
328
329
330
331
        Custom eval function expects a callable with following functions:
            ``func(y_true, y_pred)``, ``func(y_true, y_pred, weight)``
                or ``func(y_true, y_pred, weight, group)``.
            return (eval_name, eval_result, is_bigger_better)
                or list of (eval_name, eval_result, is_bigger_better)
332
333
334

            y_true: array_like of shape [n_samples]
                The target values
wxchan's avatar
wxchan committed
335
            y_pred: array_like of shape [n_samples] or shape[n_samples * n_class] (for multi-class)
336
337
338
339
340
341
342
343
344
345
346
347
348
                The predicted values
            weight: array_like of shape [n_samples]
                The weight of samples
            group: array_like
                group/query data, used for ranking task
            eval_name: str
                name of evaluation
            eval_result: float
                eval result
            is_bigger_better: bool
                is eval result bigger better, e.g. AUC is bigger_better.
        for multi-class task, the y_pred is group by class_id first, then group by row_id
          if you want to get i-th row y_pred in j-th class, the access way is y_pred[j*num_data+i]
wxchan's avatar
wxchan committed
349
350
351
        """
        evals_result = {}
        params = self.get_params()
352
353
354
355
356
        params['verbose'] = -1 if self.silent else 1
        if hasattr(self, 'n_classes_') and self.n_classes_ > 2:
            params['num_class'] = self.n_classes_
        if hasattr(self, 'eval_at'):
            params['ndcg_eval_at'] = self.eval_at
wxchan's avatar
wxchan committed
357
        if self.fobj:
wxchan's avatar
wxchan committed
358
            params['objective'] = 'None'  # objective = nullptr for unknown objective
wxchan's avatar
wxchan committed
359
        if 'label_gain' in params and params['label_gain'] is None:
wxchan's avatar
wxchan committed
360
            del params['label_gain']  # use default of cli version
wxchan's avatar
wxchan committed
361
362

        if callable(eval_metric):
363
            feval = _eval_function_wrapper(eval_metric)
wxchan's avatar
wxchan committed
364
365
        else:
            feval = None
366
            params['metric'] = eval_metric
wxchan's avatar
wxchan committed
367

Guolin Ke's avatar
Guolin Ke committed
368
        def _construct_dataset(X, y, sample_weight, init_score, group, params):
Guolin Ke's avatar
Guolin Ke committed
369
            ret = Dataset(X, label=y, max_bin=self.max_bin, weight=sample_weight, group=group, params=params)
Guolin Ke's avatar
Guolin Ke committed
370
371
372
            ret.set_init_score(init_score)
            return ret

Guolin Ke's avatar
Guolin Ke committed
373
        train_set = _construct_dataset(X, y, sample_weight, init_score, group, params)
Guolin Ke's avatar
Guolin Ke committed
374
375
376
377
378
379
380
381
382
383

        valid_sets = []
        if eval_set is not None:
            if isinstance(eval_set, tuple):
                eval_set = [eval_set]
            for i, valid_data in enumerate(eval_set):
                """reduce cost for prediction training data"""
                if valid_data[0] is X and valid_data[1] is y:
                    valid_set = train_set
                else:
Tsukasa OMOTO's avatar
Tsukasa OMOTO committed
384
385
386
387
                    def get_meta_data(collection, i):
                        if collection is None:
                            return None
                        elif isinstance(collection, list):
388
                            return collection[i] if len(collection) > i else None
Tsukasa OMOTO's avatar
Tsukasa OMOTO committed
389
390
391
392
393
394
395
                        elif isinstance(collection, dict):
                            return collection.get(i, None)
                        else:
                            raise TypeError('eval_sample_weight, eval_init_score, and eval_group should be dict or list')
                    valid_weight = get_meta_data(eval_sample_weight, i)
                    valid_init_score = get_meta_data(eval_init_score, i)
                    valid_group = get_meta_data(eval_group, i)
Guolin Ke's avatar
Guolin Ke committed
396
                    valid_set = _construct_dataset(valid_data[0], valid_data[1], valid_weight, valid_init_score, valid_group, params)
Guolin Ke's avatar
Guolin Ke committed
397
398
399
400
                valid_sets.append(valid_set)

        self._Booster = train(params, train_set,
                              self.n_estimators, valid_sets=valid_sets,
wxchan's avatar
wxchan committed
401
402
                              early_stopping_rounds=early_stopping_rounds,
                              evals_result=evals_result, fobj=self.fobj, feval=feval,
Guolin Ke's avatar
Guolin Ke committed
403
                              verbose_eval=verbose, feature_name=feature_name,
404
                              callbacks=callbacks)
wxchan's avatar
wxchan committed
405
406

        if evals_result:
407
            self.evals_result = evals_result
wxchan's avatar
wxchan committed
408
409
410
411
412

        if early_stopping_rounds is not None:
            self.best_iteration = self._Booster.best_iteration
        return self

413
    def predict(self, X, raw_score=False, num_iteration=0):
wxchan's avatar
wxchan committed
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
        """
        Return the predicted value for each sample.

        Parameters
        ----------
        X : array_like, shape=[n_samples, n_features]
            Input features matrix.

        num_iteration : int
            Limit number of iterations in the prediction; defaults to 0 (use all trees).

        Returns
        -------
        predicted_result : array_like, shape=[n_samples] or [n_samples, n_classes]
        """
429
        return self.booster_.predict(X, raw_score=raw_score, num_iteration=num_iteration)
wxchan's avatar
wxchan committed
430
431

    def apply(self, X, num_iteration=0):
wxchan's avatar
wxchan committed
432
433
        """
        Return the predicted leaf every tree for each sample.
wxchan's avatar
wxchan committed
434
435
436
437
438
439

        Parameters
        ----------
        X : array_like, shape=[n_samples, n_features]
            Input features matrix.

wxchan's avatar
wxchan committed
440
441
        num_iteration : int
            Limit number of iterations in the prediction; defaults to 0 (use all trees).
wxchan's avatar
wxchan committed
442
443
444
445
446

        Returns
        -------
        X_leaves : array_like, shape=[n_samples, n_trees]
        """
447
        return self.booster_.predict(X, pred_leaf=True, num_iteration=num_iteration)
wxchan's avatar
wxchan committed
448

449
450
451
452
453
454
    @property
    def booster_(self):
        """Get the underlying lightgbm Booster of this model."""
        if self._Booster is None:
            raise LightGBMError('No booster found. Need to call fit beforehand.')
        return self._Booster
wxchan's avatar
wxchan committed
455

456
457
458
459
460
461
462
463
    @property
    def evals_result_(self):
        """Get the evaluation results."""
        if self.evals_result is None:
            raise LightGBMError('No results found. Need to call fit with eval set beforehand.')
        return self.evals_result

    @property
464
    def feature_importances_(self):
465
466
467
        """Get normailized feature importances."""
        importace_array = self.booster_.feature_importance().astype(np.float32)
        return importace_array / importace_array.sum()
wxchan's avatar
wxchan committed
468

wxchan's avatar
wxchan committed
469
    @LGBMDeprecated('Use attribute booster_ instead.')
470
471
    def booster(self):
        return self.booster_
wxchan's avatar
wxchan committed
472

473
    @LGBMDeprecated('Use attribute feature_importances_ instead.')
474
    def feature_importance(self):
475
        return self.feature_importances_
wxchan's avatar
wxchan committed
476

wxchan's avatar
wxchan committed
477

wxchan's avatar
wxchan committed
478
479
class LGBMRegressor(LGBMModel, LGBMRegressorBase):

Tsukasa OMOTO's avatar
Tsukasa OMOTO committed
480
481
482
483
484
485
486
    def __init__(self, boosting_type="gbdt", num_leaves=31, max_depth=-1,
                 learning_rate=0.1, n_estimators=10, max_bin=255,
                 subsample_for_bin=50000, objective="regression",
                 min_split_gain=0, min_child_weight=5, min_child_samples=10,
                 subsample=1, subsample_freq=1, colsample_bytree=1,
                 reg_alpha=0, reg_lambda=0,
                 seed=0, nthread=-1, silent=True,
487
                 huber_delta=1.0, gaussian_eta=1.0, fair_c=1.0,
488
                 poisson_max_delta_step=0.7,
Tsukasa OMOTO's avatar
Tsukasa OMOTO committed
489
490
491
492
493
494
495
496
497
498
499
                 drop_rate=0.1, skip_drop=0.5, max_drop=50,
                 uniform_drop=False, xgboost_dart_mode=False):
        super(LGBMRegressor, self).__init__(boosting_type=boosting_type, num_leaves=num_leaves,
                                            max_depth=max_depth, learning_rate=learning_rate,
                                            n_estimators=n_estimators, max_bin=max_bin,
                                            subsample_for_bin=subsample_for_bin, objective=objective,
                                            min_split_gain=min_split_gain, min_child_weight=min_child_weight,
                                            min_child_samples=min_child_samples, subsample=subsample,
                                            subsample_freq=subsample_freq, colsample_bytree=colsample_bytree,
                                            reg_alpha=reg_alpha, reg_lambda=reg_lambda,
                                            seed=seed, nthread=nthread, silent=silent,
500
                                            huber_delta=huber_delta, gaussian_eta=gaussian_eta, fair_c=fair_c,
501
                                            poisson_max_delta_step=poisson_max_delta_step,
Tsukasa OMOTO's avatar
Tsukasa OMOTO committed
502
503
504
                                            drop_rate=drop_rate, skip_drop=skip_drop, max_drop=max_drop,
                                            uniform_drop=uniform_drop, xgboost_dart_mode=xgboost_dart_mode)

Guolin Ke's avatar
Guolin Ke committed
505
506
    def fit(self, X, y,
            sample_weight=None, init_score=None,
507
            eval_set=None, eval_sample_weight=None,
Guolin Ke's avatar
Guolin Ke committed
508
            eval_init_score=None,
wxchan's avatar
wxchan committed
509
            eval_metric="l2",
Guolin Ke's avatar
Guolin Ke committed
510
            early_stopping_rounds=None, verbose=True,
Guolin Ke's avatar
Guolin Ke committed
511
            feature_name='auto', callbacks=None):
512
513
514
515
516
517
518
519

        super(LGBMRegressor, self).fit(X, y, sample_weight=sample_weight,
                                       init_score=init_score, eval_set=eval_set,
                                       eval_sample_weight=eval_sample_weight,
                                       eval_init_score=eval_init_score,
                                       eval_metric=eval_metric,
                                       early_stopping_rounds=early_stopping_rounds,
                                       verbose=verbose, feature_name=feature_name,
Guolin Ke's avatar
Guolin Ke committed
520
                                       callbacks=callbacks)
Guolin Ke's avatar
Guolin Ke committed
521
522
        return self

wxchan's avatar
wxchan committed
523

wxchan's avatar
wxchan committed
524
525
class LGBMClassifier(LGBMModel, LGBMClassifierBase):

526
    def __init__(self, boosting_type="gbdt", num_leaves=31, max_depth=-1,
Guolin Ke's avatar
Guolin Ke committed
527
                 learning_rate=0.1, n_estimators=10, max_bin=255,
wxchan's avatar
wxchan committed
528
529
                 subsample_for_bin=50000, objective="binary",
                 min_split_gain=0, min_child_weight=5, min_child_samples=10,
Guolin Ke's avatar
Guolin Ke committed
530
531
                 subsample=1, subsample_freq=1, colsample_bytree=1,
                 reg_alpha=0, reg_lambda=0, scale_pos_weight=1,
wxchan's avatar
wxchan committed
532
533
                 is_unbalance=False, seed=0, nthread=-1,
                 silent=True, sigmoid=1.0,
534
535
                 drop_rate=0.1, skip_drop=0.5, max_drop=50,
                 uniform_drop=False, xgboost_dart_mode=False):
536
        self.classes, self.n_classes = None, None
537
538
539
        super(LGBMClassifier, self).__init__(boosting_type=boosting_type, num_leaves=num_leaves,
                                             max_depth=max_depth, learning_rate=learning_rate,
                                             n_estimators=n_estimators, max_bin=max_bin,
wxchan's avatar
wxchan committed
540
                                             subsample_for_bin=subsample_for_bin, objective=objective,
541
542
543
544
                                             min_split_gain=min_split_gain, min_child_weight=min_child_weight,
                                             min_child_samples=min_child_samples, subsample=subsample,
                                             subsample_freq=subsample_freq, colsample_bytree=colsample_bytree,
                                             reg_alpha=reg_alpha, reg_lambda=reg_lambda,
wxchan's avatar
wxchan committed
545
546
                                             scale_pos_weight=scale_pos_weight, is_unbalance=is_unbalance,
                                             seed=seed, nthread=nthread, silent=silent, sigmoid=sigmoid,
547
548
                                             drop_rate=drop_rate, skip_drop=skip_drop, max_drop=max_drop,
                                             uniform_drop=uniform_drop, xgboost_dart_mode=xgboost_dart_mode)
Guolin Ke's avatar
Guolin Ke committed
549
550
551

    def fit(self, X, y,
            sample_weight=None, init_score=None,
552
            eval_set=None, eval_sample_weight=None,
Guolin Ke's avatar
Guolin Ke committed
553
            eval_init_score=None,
wxchan's avatar
wxchan committed
554
            eval_metric="binary_logloss",
wxchan's avatar
wxchan committed
555
            early_stopping_rounds=None, verbose=True,
Guolin Ke's avatar
Guolin Ke committed
556
            feature_name='auto',
557
            callbacks=None):
558
559
560
        self._le = LGBMLabelEncoder().fit(y)
        y = self._le.transform(y)

561
562
563
        self.classes = self._le.classes_
        self.n_classes = len(self.classes_)
        if self.n_classes > 2:
wxchan's avatar
wxchan committed
564
565
            # Switch to using a multiclass objective in the underlying LGBM instance
            self.objective = "multiclass"
wxchan's avatar
wxchan committed
566
            if eval_metric == "binary_logloss":
wxchan's avatar
wxchan committed
567
                eval_metric = "multi_logloss"
wxchan's avatar
wxchan committed
568
569

        if eval_set is not None:
570
571
572
573
574
575
576
577
578
            eval_set = [(x[0], self._le.transform(x[1])) for x in eval_set]

        super(LGBMClassifier, self).fit(X, y, sample_weight=sample_weight,
                                        init_score=init_score, eval_set=eval_set,
                                        eval_sample_weight=eval_sample_weight,
                                        eval_init_score=eval_init_score,
                                        eval_metric=eval_metric,
                                        early_stopping_rounds=early_stopping_rounds,
                                        verbose=verbose, feature_name=feature_name,
579
                                        callbacks=callbacks)
wxchan's avatar
wxchan committed
580
581
        return self

582
583
584
585
    def predict(self, X, raw_score=False, num_iteration=0):
        class_probs = self.predict_proba(X, raw_score, num_iteration)
        class_index = np.argmax(class_probs, axis=1)
        return self._le.inverse_transform(class_index)
wxchan's avatar
wxchan committed
586

587
    def predict_proba(self, X, raw_score=False, num_iteration=0):
wxchan's avatar
wxchan committed
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
        """
        Return the predicted probability for each class for each sample.

        Parameters
        ----------
        X : array_like, shape=[n_samples, n_features]
            Input features matrix.

        num_iteration : int
            Limit number of iterations in the prediction; defaults to 0 (use all trees).

        Returns
        -------
        predicted_probability : array_like, shape=[n_samples, n_classes]
        """
603
604
        class_probs = self.booster_.predict(X, raw_score=raw_score, num_iteration=num_iteration)
        if self.n_classes > 2:
wxchan's avatar
wxchan committed
605
606
            return class_probs
        else:
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
            return np.vstack((1. - class_probs, class_probs)).transpose()

    @property
    def classes_(self):
        """Get class label array."""
        if self.classes is None:
            raise LightGBMError('No classes found. Need to call fit beforehand.')
        return self.classes

    @property
    def n_classes_(self):
        """Get number of classes"""
        if self.n_classes is None:
            raise LightGBMError('No classes found. Need to call fit beforehand.')
        return self.n_classes
wxchan's avatar
wxchan committed
622

wxchan's avatar
wxchan committed
623

wxchan's avatar
wxchan committed
624
625
class LGBMRanker(LGBMModel):

626
    def __init__(self, boosting_type="gbdt", num_leaves=31, max_depth=-1,
Guolin Ke's avatar
Guolin Ke committed
627
                 learning_rate=0.1, n_estimators=10, max_bin=255,
wxchan's avatar
wxchan committed
628
629
                 subsample_for_bin=50000, objective="lambdarank",
                 min_split_gain=0, min_child_weight=5, min_child_samples=10,
Guolin Ke's avatar
Guolin Ke committed
630
631
                 subsample=1, subsample_freq=1, colsample_bytree=1,
                 reg_alpha=0, reg_lambda=0, scale_pos_weight=1,
wxchan's avatar
wxchan committed
632
633
                 is_unbalance=False, seed=0, nthread=-1, silent=True,
                 sigmoid=1.0, max_position=20, label_gain=None,
634
635
636
637
638
                 drop_rate=0.1, skip_drop=0.5, max_drop=50,
                 uniform_drop=False, xgboost_dart_mode=False):
        super(LGBMRanker, self).__init__(boosting_type=boosting_type, num_leaves=num_leaves,
                                         max_depth=max_depth, learning_rate=learning_rate,
                                         n_estimators=n_estimators, max_bin=max_bin,
wxchan's avatar
wxchan committed
639
                                         subsample_for_bin=subsample_for_bin, objective=objective,
640
641
642
643
                                         min_split_gain=min_split_gain, min_child_weight=min_child_weight,
                                         min_child_samples=min_child_samples, subsample=subsample,
                                         subsample_freq=subsample_freq, colsample_bytree=colsample_bytree,
                                         reg_alpha=reg_alpha, reg_lambda=reg_lambda,
wxchan's avatar
wxchan committed
644
645
646
                                         scale_pos_weight=scale_pos_weight, is_unbalance=is_unbalance,
                                         seed=seed, nthread=nthread, silent=silent,
                                         sigmoid=sigmoid, max_position=max_position, label_gain=label_gain,
647
648
                                         drop_rate=drop_rate, skip_drop=skip_drop, max_drop=max_drop,
                                         uniform_drop=uniform_drop, xgboost_dart_mode=xgboost_dart_mode)
Guolin Ke's avatar
Guolin Ke committed
649
650

    def fit(self, X, y,
651
652
            sample_weight=None, init_score=None, group=None,
            eval_set=None, eval_sample_weight=None,
Guolin Ke's avatar
Guolin Ke committed
653
            eval_init_score=None, eval_group=None,
wxchan's avatar
wxchan committed
654
            eval_metric='ndcg', eval_at=1,
wxchan's avatar
wxchan committed
655
            early_stopping_rounds=None, verbose=True,
Guolin Ke's avatar
Guolin Ke committed
656
            feature_name='auto',
657
            callbacks=None):
Guolin Ke's avatar
Guolin Ke committed
658
        """
wxchan's avatar
wxchan committed
659
        Most arguments like common methods except following:
Guolin Ke's avatar
Guolin Ke committed
660
661
662
663

        eval_at : list of int
            The evaulation positions of NDCG
        """
wxchan's avatar
wxchan committed
664
665

        """check group data"""
Guolin Ke's avatar
Guolin Ke committed
666
        if group is None:
667
            raise ValueError("Should set group for ranking task")
wxchan's avatar
wxchan committed
668
669

        if eval_set is not None:
Guolin Ke's avatar
Guolin Ke committed
670
            if eval_group is None:
671
                raise ValueError("Eval_group cannot be None when eval_set is not None")
Guolin Ke's avatar
Guolin Ke committed
672
            elif len(eval_group) != len(eval_set):
673
                raise ValueError("Length of eval_group should equal to eval_set")
wxchan's avatar
wxchan committed
674
            elif (isinstance(eval_group, dict) and any(i not in eval_group or eval_group[i] is None for i in range_(len(eval_group)))) \
wxchan's avatar
wxchan committed
675
                    or (isinstance(eval_group, list) and any(group is None for group in eval_group)):
676
                raise ValueError("Should set group for all eval dataset for ranking task; if you use dict, the index should start from 0")
677

Guolin Ke's avatar
Guolin Ke committed
678
        if eval_at is not None:
679
680
681
682
683
684
685
686
            self.eval_at = eval_at
        super(LGBMRanker, self).fit(X, y, sample_weight=sample_weight,
                                    init_score=init_score, group=group,
                                    eval_set=eval_set, eval_sample_weight=eval_sample_weight,
                                    eval_init_score=eval_init_score, eval_group=eval_group,
                                    eval_metric=eval_metric,
                                    early_stopping_rounds=early_stopping_rounds,
                                    verbose=verbose, feature_name=feature_name,
687
                                    callbacks=callbacks)
wxchan's avatar
wxchan committed
688
        return self