engine.py 16.1 KB
Newer Older
wxchan's avatar
wxchan committed
1
2
3
4
5
# coding: utf-8
# pylint: disable = invalid-name, W0105
"""Training Library containing training routines of LightGBM."""
from __future__ import absolute_import

wxchan's avatar
wxchan committed
6
7
import collections
from operator import attrgetter
wxchan's avatar
wxchan committed
8
import numpy as np
Guolin Ke's avatar
Guolin Ke committed
9
from .basic import LightGBMError, _InnerPredictor, Dataset, Booster, is_str
wxchan's avatar
wxchan committed
10
11
from . import callback

Guolin Ke's avatar
Guolin Ke committed
12
13
def train(params, train_set, num_boost_round=100,
          valid_sets=None, valid_names=None,
wxchan's avatar
wxchan committed
14
          fobj=None, feval=None, init_model=None,
Guolin Ke's avatar
Guolin Ke committed
15
          feature_name=None, categorical_feature=None,
wxchan's avatar
wxchan committed
16
17
          early_stopping_rounds=None, evals_result=None,
          verbose_eval=True, learning_rates=None, callbacks=None):
wxchan's avatar
wxchan committed
18
19
    """
    Train with given parameters.
wxchan's avatar
wxchan committed
20
21
22
23

    Parameters
    ----------
    params : dict
24
        Parameters for training.
Guolin Ke's avatar
Guolin Ke committed
25
    train_set : Dataset
wxchan's avatar
wxchan committed
26
27
28
        Data to be trained.
    num_boost_round: int
        Number of boosting iterations.
Guolin Ke's avatar
Guolin Ke committed
29
    valid_sets: list of Datasets
wxchan's avatar
wxchan committed
30
31
        List of data to be evaluated during training
    valid_names: list of string
Guolin Ke's avatar
Guolin Ke committed
32
        Names of valid_sets
wxchan's avatar
wxchan committed
33
34
35
36
37
38
39
    fobj : function
        Customized objective function.
    feval : function
        Customized evaluation function.
        Note: should return (eval_name, eval_result, is_higher_better) of list of this
    init_model : file name of lightgbm model or 'Booster' instance
        model used for continued train
Guolin Ke's avatar
Guolin Ke committed
40
    feature_name : list of str
41
42
        Feature names
    categorical_feature : list of str or int
wxchan's avatar
wxchan committed
43
44
        Categorical features,
        type int represents index,
45
        type str represents feature names (need to specify feature_name as well)
wxchan's avatar
wxchan committed
46
47
48
49
50
51
52
    early_stopping_rounds: int
        Activates early stopping.
        Requires at least one validation data and one metric
        If there's more than one, will check all of them
        Returns the model with (best_iter + early_stopping_rounds)
        If early stopping occurs, the model will add 'best_iteration' field
    evals_result: dict or None
Guolin Ke's avatar
Guolin Ke committed
53
        This dictionary used to store all evaluation results of all the items in valid_sets.
wxchan's avatar
wxchan committed
54
55
56
        Example: with a valid_sets containing [valid_set, train_set]
                 and valid_names containing ['eval', 'train']
                 and a paramater containing ('metric':'logloss')
wxchan's avatar
wxchan committed
57
58
59
60
61
        Returns: {'train': {'logloss': ['0.48253', '0.35953', ...]},
                  'eval': {'logloss': ['0.480385', '0.357756', ...]}}
        passed with None means no using this function
    verbose_eval : bool or int
        Requires at least one item in evals.
wxchan's avatar
wxchan committed
62
63
64
65
66
67
68
69
        If `verbose_eval` is True,
            the eval metric on the valid set is printed at each boosting stage.
        If `verbose_eval` is int,
            the eval metric on the valid set is printed at every `verbose_eval` boosting stage.
        The last boosting stage
            or the boosting stage found by using `early_stopping_rounds` is also printed.
        Example: with verbose_eval=4 and at least one item in evals,
            an evaluation metric is printed every 4 (instead of 1) boosting stages.
wxchan's avatar
wxchan committed
70
    learning_rates: list or function
wxchan's avatar
wxchan committed
71
        List of learning rate for each boosting round
72
73
        or a customized function that calculates learning_rate
        in terms of current number of round (e.g. yields learning rate decay)
wxchan's avatar
wxchan committed
74
        - list l: learning_rate = l[current_round]
75
        - function f: learning_rate = f(current_round)
wxchan's avatar
wxchan committed
76
77
78
79
80
81
82
83
84
    callbacks : list of callback functions
        List of callback functions that are applied at end of each iteration.

    Returns
    -------
    booster : a trained booster model
    """
    """create predictor first"""
    if is_str(init_model):
Guolin Ke's avatar
Guolin Ke committed
85
        predictor = _InnerPredictor(model_file=init_model)
wxchan's avatar
wxchan committed
86
    elif isinstance(init_model, Booster):
Guolin Ke's avatar
Guolin Ke committed
87
        predictor = init_model._to_predictor()
wxchan's avatar
wxchan committed
88
89
    else:
        predictor = None
90
    init_iteration = predictor.num_total_iteration if predictor is not None else 0
Guolin Ke's avatar
Guolin Ke committed
91
92
    """check dataset"""
    if not isinstance(train_set, Dataset):
93
        raise TypeError("Traninig only accepts Dataset object")
Guolin Ke's avatar
Guolin Ke committed
94

95
    train_set._update_params(params)
Guolin Ke's avatar
Guolin Ke committed
96
97
98
99
    train_set._set_predictor(predictor)
    train_set.set_feature_name(feature_name)
    train_set.set_categorical_feature(categorical_feature)

wxchan's avatar
wxchan committed
100
101
    is_valid_contain_train = False
    train_data_name = "training"
Guolin Ke's avatar
Guolin Ke committed
102
    reduced_valid_sets = []
wxchan's avatar
wxchan committed
103
    name_valid_sets = []
104
    if valid_sets is not None:
Guolin Ke's avatar
Guolin Ke committed
105
106
        if isinstance(valid_sets, Dataset):
            valid_sets = [valid_sets]
wxchan's avatar
wxchan committed
107
108
        if isinstance(valid_names, str):
            valid_names = [valid_names]
Guolin Ke's avatar
Guolin Ke committed
109
        for i, valid_data in enumerate(valid_sets):
wxchan's avatar
wxchan committed
110
            """reduce cost for prediction training data"""
Guolin Ke's avatar
Guolin Ke committed
111
            if valid_data is train_set:
wxchan's avatar
wxchan committed
112
113
114
115
                is_valid_contain_train = True
                if valid_names is not None:
                    train_data_name = valid_names[i]
                continue
Guolin Ke's avatar
Guolin Ke committed
116
            if not isinstance(valid_data, Dataset):
117
                raise TypeError("Traninig only accepts Dataset object")
Guolin Ke's avatar
Guolin Ke committed
118
119
            valid_data.set_reference(train_set)
            reduced_valid_sets.append(valid_data)
120
            if valid_names is not None and len(valid_names) > i:
wxchan's avatar
wxchan committed
121
122
123
                name_valid_sets.append(valid_names[i])
            else:
                name_valid_sets.append('valid_'+str(i))
124
125
        for valid_data in valid_sets:
            valid_data._update_params(params)
wxchan's avatar
wxchan committed
126
    """process callbacks"""
127
    if callbacks is None:
wxchan's avatar
wxchan committed
128
129
130
131
132
        callbacks = set()
    else:
        for i, cb in enumerate(callbacks):
            cb.__dict__.setdefault('order', i - len(callbacks))
        callbacks = set(callbacks)
wxchan's avatar
wxchan committed
133
134

    # Most of legacy advanced options becomes callbacks
wxchan's avatar
wxchan committed
135
136
    if verbose_eval is True:
        callbacks.add(callback.print_evaluation())
137
    elif isinstance(verbose_eval, int):
wxchan's avatar
wxchan committed
138
        callbacks.add(callback.print_evaluation(verbose_eval))
wxchan's avatar
wxchan committed
139

140
    if early_stopping_rounds is not None:
141
        callbacks.add(callback.early_stopping(early_stopping_rounds, verbose=bool(verbose_eval)))
142

wxchan's avatar
wxchan committed
143
    if learning_rates is not None:
144
        callbacks.add(callback.reset_parameter(learning_rate=learning_rates))
wxchan's avatar
wxchan committed
145
146

    if evals_result is not None:
wxchan's avatar
wxchan committed
147
148
149
150
151
152
        callbacks.add(callback.record_evaluation(evals_result))

    callbacks_before_iter = {cb for cb in callbacks if getattr(cb, 'before_iteration', False)}
    callbacks_after_iter = callbacks - callbacks_before_iter
    callbacks_before_iter = sorted(callbacks_before_iter, key=attrgetter('order'))
    callbacks_after_iter = sorted(callbacks_after_iter, key=attrgetter('order'))
wxchan's avatar
wxchan committed
153
154
155
156
157

    """construct booster"""
    booster = Booster(params=params, train_set=train_set)
    if is_valid_contain_train:
        booster.set_train_data_name(train_data_name)
Guolin Ke's avatar
Guolin Ke committed
158
    for valid_set, name_valid_set in zip(reduced_valid_sets, name_valid_sets):
wxchan's avatar
wxchan committed
159
        booster.add_valid(valid_set, name_valid_set)
wxchan's avatar
wxchan committed
160

wxchan's avatar
wxchan committed
161
    """start training"""
162
    for i in range(init_iteration, init_iteration + num_boost_round):
wxchan's avatar
wxchan committed
163
164
165
166
        for cb in callbacks_before_iter:
            cb(callback.CallbackEnv(model=booster,
                                    cvfolds=None,
                                    iteration=i,
167
168
                                    begin_iteration=init_iteration,
                                    end_iteration=init_iteration + num_boost_round,
wxchan's avatar
wxchan committed
169
170
171
172
173
174
                                    evaluation_result_list=None))

        booster.update(fobj=fobj)

        evaluation_result_list = []
        # check evaluation result.
175
        if valid_sets is not None:
wxchan's avatar
wxchan committed
176
177
178
179
180
181
182
183
            if is_valid_contain_train:
                evaluation_result_list.extend(booster.eval_train(feval))
            evaluation_result_list.extend(booster.eval_valid(feval))
        try:
            for cb in callbacks_after_iter:
                cb(callback.CallbackEnv(model=booster,
                                        cvfolds=None,
                                        iteration=i,
184
185
                                        begin_iteration=init_iteration,
                                        end_iteration=init_iteration + num_boost_round,
wxchan's avatar
wxchan committed
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
                                        evaluation_result_list=evaluation_result_list))
        except callback.EarlyStopException:
            break
    if booster.attr('best_iteration') is not None:
        booster.best_iteration = int(booster.attr('best_iteration')) + 1
    else:
        booster.best_iteration = num_boost_round
    return booster


class CVBooster(object):
    """"Auxiliary datastruct to hold one fold of CV."""
    def __init__(self, train_set, valid_test, params):
        """"Initialize the CVBooster"""
        self.train_set = train_set
        self.valid_test = valid_test
        self.booster = Booster(params=params, train_set=train_set)
        self.booster.add_valid(valid_test, 'valid')

    def update(self, fobj):
        """"Update the boosters for one iteration"""
        self.booster.update(fobj=fobj)

    def eval(self, feval):
        """"Evaluate the CVBooster for one iteration."""
        return self.booster.eval_valid(feval)

try:
214
    from sklearn.model_selection import StratifiedKFold
wxchan's avatar
wxchan committed
215
216
    SKLEARN_StratifiedKFold = True
except ImportError:
217
218
219
220
221
    try:
        from sklearn.cross_validation import StratifiedKFold
        SKLEARN_StratifiedKFold = True
    except ImportError:
        SKLEARN_StratifiedKFold = False
wxchan's avatar
wxchan committed
222

wxchan's avatar
wxchan committed
223
def _make_n_folds(full_data, nfold, params, seed, fpreproc=None, stratified=False, shuffle=True):
wxchan's avatar
wxchan committed
224
225
226
227
228
229
    """
    Make an n-fold list of CVBooster from random indices.
    """
    np.random.seed(seed)
    if stratified:
        if SKLEARN_StratifiedKFold:
wxchan's avatar
wxchan committed
230
            sfk = StratifiedKFold(n_splits=nfold, shuffle=shuffle, random_state=seed)
wxchan's avatar
wxchan committed
231
232
            idset = [x[1] for x in sfk.split(X=full_data.get_label(), y=full_data.get_label())]
        else:
233
            raise LightGBMError('Scikit-learn is required for stratified cv')
wxchan's avatar
wxchan committed
234
    else:
Guolin Ke's avatar
Guolin Ke committed
235
        full_data.construct()
wxchan's avatar
wxchan committed
236
237
        if shuffle:
            randidx = np.random.permutation(full_data.num_data())
wxchan's avatar
wxchan committed
238
239
240
241
242
243
244
245
246
        kstep = int(len(randidx) / nfold)
        idset = [randidx[(i * kstep): min(len(randidx), (i + 1) * kstep)] for i in range(nfold)]

    ret = []
    for k in range(nfold):
        train_set = full_data.subset(np.concatenate([idset[i] for i in range(nfold) if k != i]))
        valid_set = full_data.subset(idset[k])
        # run preprocessing on the data set if needed
        if fpreproc is not None:
wxchan's avatar
wxchan committed
247
            train_set, valid_set, tparam = fpreproc(train_set, valid_set, params.copy())
wxchan's avatar
wxchan committed
248
        else:
wxchan's avatar
wxchan committed
249
            tparam = params
wxchan's avatar
wxchan committed
250
251
252
253
254
255
256
        ret.append(CVBooster(train_set, valid_set, tparam))
    return ret

def _agg_cv_result(raw_results):
    """
    Aggregate cross-validation results.
    """
wxchan's avatar
wxchan committed
257
    cvmap = collections.defaultdict(list)
wxchan's avatar
wxchan committed
258
259
260
    metric_type = {}
    for one_result in raw_results:
        for one_line in one_result:
wxchan's avatar
wxchan committed
261
262
263
            metric_type[one_line[1]] = one_line[3]
            cvmap[one_line[1]].append(one_line[2])
    return [('cv_agg', k, np.mean(v), metric_type[k], np.std(v)) for k, v in cvmap.items()]
wxchan's avatar
wxchan committed
264

Guolin Ke's avatar
Guolin Ke committed
265
def cv(params, train_set, num_boost_round=10, nfold=5, stratified=False,
wxchan's avatar
wxchan committed
266
       shuffle=True, metrics=None, fobj=None, feval=None, init_model=None,
Guolin Ke's avatar
Guolin Ke committed
267
268
269
       feature_name=None, categorical_feature=None,
       early_stopping_rounds=None, fpreproc=None,
       verbose_eval=None, show_stdv=True, seed=0,
wxchan's avatar
wxchan committed
270
       callbacks=None):
wxchan's avatar
wxchan committed
271
272
    """
    Cross-validation with given paramaters.
wxchan's avatar
wxchan committed
273
274
275
276
277

    Parameters
    ----------
    params : dict
        Booster params.
Guolin Ke's avatar
Guolin Ke committed
278
    train_set : Dataset
wxchan's avatar
wxchan committed
279
280
281
282
283
284
285
        Data to be trained.
    num_boost_round : int
        Number of boosting iterations.
    nfold : int
        Number of folds in CV.
    stratified : bool
        Perform stratified sampling.
wxchan's avatar
wxchan committed
286
287
    shuffle: bool
        Whether shuffle before split data
wxchan's avatar
wxchan committed
288
289
290
291
292
293
294
295
    folds : a KFold or StratifiedKFold instance
        Sklearn KFolds or StratifiedKFolds.
    metrics : string or list of strings
        Evaluation metrics to be watched in CV.
    fobj : function
        Custom objective function.
    feval : function
        Custom evaluation function.
Guolin Ke's avatar
Guolin Ke committed
296
297
    init_model : file name of lightgbm model or 'Booster' instance
        model used for continued train
Guolin Ke's avatar
Guolin Ke committed
298
    feature_name : list of str
299
300
        Feature names
    categorical_feature : list of str or int
wxchan's avatar
wxchan committed
301
        Categorical features, type int represents index,
302
        type str represents feature names (need to specify feature_name as well)
wxchan's avatar
wxchan committed
303
    early_stopping_rounds: int
wxchan's avatar
wxchan committed
304
        Activates early stopping. CV error needs to decrease at least
wxchan's avatar
wxchan committed
305
306
307
        every <early_stopping_rounds> round(s) to continue.
        Last entry in evaluation history is the one from best iteration.
    fpreproc : function
wxchan's avatar
wxchan committed
308
309
        Preprocessing function that takes (dtrain, dtest, param)
        and returns transformed versions of those.
wxchan's avatar
wxchan committed
310
    verbose_eval : bool, int, or None, default None
wxchan's avatar
wxchan committed
311
312
313
314
315
        Whether to display the progress.
        If None, progress will be displayed when np.ndarray is returned.
        If True, progress will be displayed at boosting stage.
        If an integer is given,
            progress will be displayed at every given `verbose_eval` boosting stage.
wxchan's avatar
wxchan committed
316
317
318
319
320
321
322
323
324
325
326
327
    show_stdv : bool, default True
        Whether to display the standard deviation in progress.
        Results are not affected, and always contains std.
    seed : int
        Seed used to generate the folds (passed to numpy.random.seed).
    callbacks : list of callback functions
        List of callback functions that are applied at end of each iteration.

    Returns
    -------
    evaluation history : list(string)
    """
Guolin Ke's avatar
Guolin Ke committed
328
    if not isinstance(train_set, Dataset):
329
        raise TypeError("Traninig only accepts Dataset object")
Guolin Ke's avatar
Guolin Ke committed
330
331
332
333
334
335
336

    if is_str(init_model):
        predictor = _InnerPredictor(model_file=init_model)
    elif isinstance(init_model, Booster):
        predictor = init_model._to_predictor()
    else:
        predictor = None
337
    train_set._update_params(params)
Guolin Ke's avatar
Guolin Ke committed
338
339
340
341
    train_set._set_predictor(predictor)
    train_set.set_feature_name(feature_name)
    train_set.set_categorical_feature(categorical_feature)

342
343
344
345
    if metrics:
        params.setdefault('metric', [])
        if is_str(metrics):
            params['metric'].append(metrics)
wxchan's avatar
wxchan committed
346
        else:
347
            params['metric'].extend(metrics)
wxchan's avatar
wxchan committed
348

wxchan's avatar
wxchan committed
349
    results = collections.defaultdict(list)
wxchan's avatar
wxchan committed
350
    cvfolds = _make_n_folds(train_set, nfold, params, seed, fpreproc, stratified, shuffle)
wxchan's avatar
wxchan committed
351
352

    # setup callbacks
353
    if callbacks is None:
wxchan's avatar
wxchan committed
354
355
356
357
358
        callbacks = set()
    else:
        for i, cb in enumerate(callbacks):
            cb.__dict__.setdefault('order', i - len(callbacks))
        callbacks = set(callbacks)
359
    if early_stopping_rounds is not None:
360
        callbacks.add(callback.early_stopping(early_stopping_rounds, verbose=False))
wxchan's avatar
wxchan committed
361
362
    if verbose_eval is True:
        callbacks.add(callback.print_evaluation(show_stdv=show_stdv))
363
    elif isinstance(verbose_eval, int):
wxchan's avatar
wxchan committed
364
        callbacks.add(callback.print_evaluation(verbose_eval, show_stdv=show_stdv))
wxchan's avatar
wxchan committed
365

wxchan's avatar
wxchan committed
366
367
368
369
    callbacks_before_iter = {cb for cb in callbacks if getattr(cb, 'before_iteration', False)}
    callbacks_after_iter = callbacks - callbacks_before_iter
    callbacks_before_iter = sorted(callbacks_before_iter, key=attrgetter('order'))
    callbacks_after_iter = sorted(callbacks_after_iter, key=attrgetter('order'))
wxchan's avatar
wxchan committed
370
371
372
373
374
375
376
377
378
379
380
381
382
383

    for i in range(num_boost_round):
        for cb in callbacks_before_iter:
            cb(callback.CallbackEnv(model=None,
                                    cvfolds=cvfolds,
                                    iteration=i,
                                    begin_iteration=0,
                                    end_iteration=num_boost_round,
                                    evaluation_result_list=None))
        for fold in cvfolds:
            fold.update(fobj)
        res = _agg_cv_result([f.eval(feval) for f in cvfolds])
        for _, key, mean, _, std in res:
            results[key + '-mean'].append(mean)
wxchan's avatar
wxchan committed
384
            results[key + '-stdv'].append(std)
wxchan's avatar
wxchan committed
385
386
387
388
389
390
391
392
393
394
        try:
            for cb in callbacks_after_iter:
                cb(callback.CallbackEnv(model=None,
                                        cvfolds=cvfolds,
                                        iteration=i,
                                        begin_iteration=0,
                                        end_iteration=num_boost_round,
                                        evaluation_result_list=res))
        except callback.EarlyStopException as e:
            for k in results:
wxchan's avatar
wxchan committed
395
                results[k] = results[k][:e.best_iteration + 1]
wxchan's avatar
wxchan committed
396
            break
wxchan's avatar
wxchan committed
397
    return dict(results)