engine.py 18.5 KB
Newer Older
wxchan's avatar
wxchan committed
1
2
3
4
5
# coding: utf-8
# pylint: disable = invalid-name, W0105
"""Training Library containing training routines of LightGBM."""
from __future__ import absolute_import

wxchan's avatar
wxchan committed
6
7
import collections
from operator import attrgetter
wxchan's avatar
wxchan committed
8
9
10
11
12
import numpy as np
from .basic import LightGBMError, Predictor, Dataset, Booster, is_str
from . import callback

def _construct_dataset(X_y, reference=None,
13
                       params=None, other_fields=None,
Guolin Ke's avatar
Guolin Ke committed
14
                       feature_name=None, categorical_feature=None,
wxchan's avatar
wxchan committed
15
16
17
18
19
20
21
22
23
24
                       predictor=None):
    if 'max_bin' in params:
        max_bin = int(params['max_bin'])
    else:
        max_bin = 255
    weight = None
    group = None
    init_score = None
    if other_fields is not None:
        if not isinstance(other_fields, dict):
wxchan's avatar
wxchan committed
25
            raise TypeError("type of other filed data should be dict")
26
27
28
        weight = other_fields.get('weight', None)
        group = other_fields.get('group', None)
        init_score = other_fields.get('init_score', None)
wxchan's avatar
wxchan committed
29
30
31
32
33
    if is_str(X_y):
        data = X_y
        label = None
    else:
        if len(X_y) != 2:
wxchan's avatar
wxchan committed
34
            raise TypeError("should pass (data, label) tuple for dataset")
wxchan's avatar
wxchan committed
35
36
37
38
39
        data = X_y[0]
        label = X_y[1]
    if reference is None:
        ret = Dataset(data, label=label, max_bin=max_bin,
                      weight=weight, group=group,
Guolin Ke's avatar
Guolin Ke committed
40
41
42
43
                      predictor=predictor,
                      feature_name=feature_name,
                      categorical_feature=categorical_feature,
                      params=params)
wxchan's avatar
wxchan committed
44
45
46
47
48
49
50
51
52
53
54
    else:
        ret = reference.create_valid(data, label=label, weight=weight,
                                     group=group, params=params)
    if init_score is not None:
        ret.set_init_score(init_score)
    return ret

def train(params, train_data, num_boost_round=100,
          valid_datas=None, valid_names=None,
          fobj=None, feval=None, init_model=None,
          train_fields=None, valid_fields=None,
Guolin Ke's avatar
Guolin Ke committed
55
          feature_name=None, categorical_feature=None,
wxchan's avatar
wxchan committed
56
57
58
59
60
61
62
          early_stopping_rounds=None, evals_result=None,
          verbose_eval=True, learning_rates=None, callbacks=None):
    """Train with given parameters.

    Parameters
    ----------
    params : dict
63
        Parameters for training.
wxchan's avatar
wxchan committed
64
65
66
67
    train_data : Dataset, tuple (X, y) or filename of data
        Data to be trained.
    num_boost_round: int
        Number of boosting iterations.
68
    valid_datas: list of Datasets, tuples (valid_X, valid_y) or filenames of data
wxchan's avatar
wxchan committed
69
70
        List of data to be evaluated during training
    valid_names: list of string
71
        Names of valid_datas
wxchan's avatar
wxchan committed
72
73
74
75
76
77
78
79
    fobj : function
        Customized objective function.
    feval : function
        Customized evaluation function.
        Note: should return (eval_name, eval_result, is_higher_better) of list of this
    init_model : file name of lightgbm model or 'Booster' instance
        model used for continued train
    train_fields : dict
80
81
        Other data file in training data. e.g. train_fields['weight'] is weight data
        Support fields: weight, group, init_score
wxchan's avatar
wxchan committed
82
    valid_fields : dict
83
        Other data file in training data. \
wxchan's avatar
wxchan committed
84
        e.g. valid_fields[0]['weight'] is weight data for first valid data
85
        Support fields: weight, group, init_score
Guolin Ke's avatar
Guolin Ke committed
86
    feature_name : list of str
87
88
89
90
        Feature names
    categorical_feature : list of str or int
        Categorical features, type int represents index, \
        type str represents feature names (need to specify feature_name as well)
wxchan's avatar
wxchan committed
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
    early_stopping_rounds: int
        Activates early stopping.
        Requires at least one validation data and one metric
        If there's more than one, will check all of them
        Returns the model with (best_iter + early_stopping_rounds)
        If early stopping occurs, the model will add 'best_iteration' field
    evals_result: dict or None
        This dictionary used to store all evaluation results of all the items in valid_datas.
        Example: with a valid_datas containing [valid_set, train_set] \
        and valid_names containing ['eval', 'train'] and a paramater containing ('metric':'logloss')
        Returns: {'train': {'logloss': ['0.48253', '0.35953', ...]},
                  'eval': {'logloss': ['0.480385', '0.357756', ...]}}
        passed with None means no using this function
    verbose_eval : bool or int
        Requires at least one item in evals.
106
        If `verbose_eval` is True then the evaluation metric on the validation set is \
wxchan's avatar
wxchan committed
107
        printed at each boosting stage.
108
109
        If `verbose_eval` is an integer then the evaluation metric on the validation set \
        is printed at every given `verbose_eval` boosting stage. The last boosting stage \
wxchan's avatar
wxchan committed
110
        / the boosting stage found by using `early_stopping_rounds` is also printed.
111
        Example: with verbose_eval=4 and at least one item in evals, an evaluation metric \
wxchan's avatar
wxchan committed
112
113
        is printed every 4 boosting stages, instead of every boosting stage.
    learning_rates: list or function
114
115
116
117
        List of learning rate for each boosting round \
        or a customized function that calculates learning_rate in terms of \
        current number of round and the total number of boosting round \
        (e.g. yields learning rate decay)
wxchan's avatar
wxchan committed
118
        - list l: learning_rate = l[current_round]
wxchan's avatar
wxchan committed
119
120
        - function f: learning_rate = f(current_round, total_boost_round) \
        or learning_rate = f(current_round)
wxchan's avatar
wxchan committed
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
    callbacks : list of callback functions
        List of callback functions that are applied at end of each iteration.

    Returns
    -------
    booster : a trained booster model
    """
    """create predictor first"""
    if is_str(init_model):
        predictor = Predictor(model_file=init_model)
    elif isinstance(init_model, Booster):
        predictor = init_model.to_predictor()
    elif isinstance(init_model, Predictor):
        predictor = init_model
    else:
        predictor = None
137
    init_iteration = predictor.num_total_iteration if predictor else 0
wxchan's avatar
wxchan committed
138
139
140
    """create dataset"""
    if isinstance(train_data, Dataset):
        train_set = train_data
141
142
143
        if train_fields is not None:
            for field, data in train_fields.items():
                train_set.set_field(field, data)
wxchan's avatar
wxchan committed
144
    else:
145
146
        train_set = _construct_dataset(train_data, None, params,
                                       other_fields=train_fields,
Guolin Ke's avatar
Guolin Ke committed
147
148
149
                                       feature_name=feature_name,
                                       categorical_feature=categorical_feature,
                                       predictor=predictor)
wxchan's avatar
wxchan committed
150
151
152
153
    is_valid_contain_train = False
    train_data_name = "training"
    valid_sets = []
    name_valid_sets = []
wxchan's avatar
wxchan committed
154
    if valid_datas:
wxchan's avatar
wxchan committed
155
156
157
158
159
        if isinstance(valid_datas, (Dataset, tuple)):
            valid_datas = [valid_datas]
        if isinstance(valid_names, str):
            valid_names = [valid_names]
        for i, valid_data in enumerate(valid_datas):
160
            other_fields = None if valid_fields is None else valid_fields.get(i, None)
wxchan's avatar
wxchan committed
161
            """reduce cost for prediction training data"""
162
            if valid_data[0] is train_data[0] and valid_data[1] is train_data[1]:
wxchan's avatar
wxchan committed
163
164
165
166
167
168
                is_valid_contain_train = True
                if valid_names is not None:
                    train_data_name = valid_names[i]
                continue
            if isinstance(valid_data, Dataset):
                valid_set = valid_data
169
170
171
                if other_fields is not None:
                    for field, data in other_fields.items():
                        valid_set.set_field(field, data)
wxchan's avatar
wxchan committed
172
173
174
175
176
            else:
                valid_set = _construct_dataset(
                    valid_data,
                    train_set,
                    params,
Guolin Ke's avatar
Guolin Ke committed
177
178
179
180
                    other_fields=other_fields,
                    feature_name=feature_name,
                    categorical_feature=categorical_feature,
                    predictor=predictor)
wxchan's avatar
wxchan committed
181
            valid_sets.append(valid_set)
182
            if valid_names is not None and len(valid_names) > i:
wxchan's avatar
wxchan committed
183
184
185
                name_valid_sets.append(valid_names[i])
            else:
                name_valid_sets.append('valid_'+str(i))
wxchan's avatar
wxchan committed
186

wxchan's avatar
wxchan committed
187
    """process callbacks"""
wxchan's avatar
wxchan committed
188
189
190
191
192
193
    if not callbacks:
        callbacks = set()
    else:
        for i, cb in enumerate(callbacks):
            cb.__dict__.setdefault('order', i - len(callbacks))
        callbacks = set(callbacks)
wxchan's avatar
wxchan committed
194
195

    # Most of legacy advanced options becomes callbacks
wxchan's avatar
wxchan committed
196
197
    if verbose_eval is True:
        callbacks.add(callback.print_evaluation())
198
    elif isinstance(verbose_eval, int):
wxchan's avatar
wxchan committed
199
        callbacks.add(callback.print_evaluation(verbose_eval))
wxchan's avatar
wxchan committed
200

wxchan's avatar
wxchan committed
201
202
203
    if early_stopping_rounds:
        callbacks.add(callback.early_stop(early_stopping_rounds,
                                          verbose=bool(verbose_eval)))
204

wxchan's avatar
wxchan committed
205
    if learning_rates is not None:
wxchan's avatar
wxchan committed
206
        callbacks.add(callback.reset_learning_rate(learning_rates))
wxchan's avatar
wxchan committed
207
208

    if evals_result is not None:
wxchan's avatar
wxchan committed
209
210
211
212
213
214
        callbacks.add(callback.record_evaluation(evals_result))

    callbacks_before_iter = {cb for cb in callbacks if getattr(cb, 'before_iteration', False)}
    callbacks_after_iter = callbacks - callbacks_before_iter
    callbacks_before_iter = sorted(callbacks_before_iter, key=attrgetter('order'))
    callbacks_after_iter = sorted(callbacks_after_iter, key=attrgetter('order'))
wxchan's avatar
wxchan committed
215
216
217
218
219
220
221

    """construct booster"""
    booster = Booster(params=params, train_set=train_set)
    if is_valid_contain_train:
        booster.set_train_data_name(train_data_name)
    for valid_set, name_valid_set in zip(valid_sets, name_valid_sets):
        booster.add_valid(valid_set, name_valid_set)
wxchan's avatar
wxchan committed
222

wxchan's avatar
wxchan committed
223
    """start training"""
224
    for i in range(init_iteration, init_iteration + num_boost_round):
wxchan's avatar
wxchan committed
225
226
227
228
        for cb in callbacks_before_iter:
            cb(callback.CallbackEnv(model=booster,
                                    cvfolds=None,
                                    iteration=i,
229
230
                                    begin_iteration=init_iteration,
                                    end_iteration=init_iteration + num_boost_round,
wxchan's avatar
wxchan committed
231
232
233
234
235
236
                                    evaluation_result_list=None))

        booster.update(fobj=fobj)

        evaluation_result_list = []
        # check evaluation result.
237
        if valid_sets:
wxchan's avatar
wxchan committed
238
239
240
241
242
243
244
245
            if is_valid_contain_train:
                evaluation_result_list.extend(booster.eval_train(feval))
            evaluation_result_list.extend(booster.eval_valid(feval))
        try:
            for cb in callbacks_after_iter:
                cb(callback.CallbackEnv(model=booster,
                                        cvfolds=None,
                                        iteration=i,
246
247
                                        begin_iteration=init_iteration,
                                        end_iteration=init_iteration + num_boost_round,
wxchan's avatar
wxchan committed
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
                                        evaluation_result_list=evaluation_result_list))
        except callback.EarlyStopException:
            break
    if booster.attr('best_iteration') is not None:
        booster.best_iteration = int(booster.attr('best_iteration')) + 1
    else:
        booster.best_iteration = num_boost_round
    return booster


class CVBooster(object):
    """"Auxiliary datastruct to hold one fold of CV."""
    def __init__(self, train_set, valid_test, params):
        """"Initialize the CVBooster"""
        self.train_set = train_set
        self.valid_test = valid_test
        self.booster = Booster(params=params, train_set=train_set)
        self.booster.add_valid(valid_test, 'valid')

    def update(self, fobj):
        """"Update the boosters for one iteration"""
        self.booster.update(fobj=fobj)

    def eval(self, feval):
        """"Evaluate the CVBooster for one iteration."""
        return self.booster.eval_valid(feval)

try:
276
    from sklearn.model_selection import StratifiedKFold
wxchan's avatar
wxchan committed
277
278
    SKLEARN_StratifiedKFold = True
except ImportError:
279
280
281
282
283
    try:
        from sklearn.cross_validation import StratifiedKFold
        SKLEARN_StratifiedKFold = True
    except ImportError:
        SKLEARN_StratifiedKFold = False
wxchan's avatar
wxchan committed
284

wxchan's avatar
wxchan committed
285
def _make_n_folds(full_data, nfold, params, seed, fpreproc=None, stratified=False):
wxchan's avatar
wxchan committed
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
    """
    Make an n-fold list of CVBooster from random indices.
    """
    np.random.seed(seed)
    if stratified:
        if SKLEARN_StratifiedKFold:
            sfk = StratifiedKFold(n_splits=nfold, shuffle=True, random_state=seed)
            idset = [x[1] for x in sfk.split(X=full_data.get_label(), y=full_data.get_label())]
        else:
            raise LightGBMError('sklearn needs to be installed in order to use stratified cv')
    else:
        randidx = np.random.permutation(full_data.num_data())
        kstep = int(len(randidx) / nfold)
        idset = [randidx[(i * kstep): min(len(randidx), (i + 1) * kstep)] for i in range(nfold)]

    ret = []
    for k in range(nfold):
        train_set = full_data.subset(np.concatenate([idset[i] for i in range(nfold) if k != i]))
        valid_set = full_data.subset(idset[k])
        # run preprocessing on the data set if needed
        if fpreproc is not None:
wxchan's avatar
wxchan committed
307
            train_set, valid_set, tparam = fpreproc(train_set, valid_set, params.copy())
wxchan's avatar
wxchan committed
308
        else:
wxchan's avatar
wxchan committed
309
            tparam = params
wxchan's avatar
wxchan committed
310
311
312
313
314
315
316
        ret.append(CVBooster(train_set, valid_set, tparam))
    return ret

def _agg_cv_result(raw_results):
    """
    Aggregate cross-validation results.
    """
wxchan's avatar
wxchan committed
317
    cvmap = collections.defaultdict(list)
wxchan's avatar
wxchan committed
318
319
320
    metric_type = {}
    for one_result in raw_results:
        for one_line in one_result:
wxchan's avatar
wxchan committed
321
322
323
            metric_type[one_line[1]] = one_line[3]
            cvmap[one_line[1]].append(one_line[2])
    return [('cv_agg', k, np.mean(v), metric_type[k], np.std(v)) for k, v in cvmap.items()]
wxchan's avatar
wxchan committed
324
325

def cv(params, train_data, num_boost_round=10, nfold=5, stratified=False,
Guolin Ke's avatar
Guolin Ke committed
326
327
328
329
       metrics=(), fobj=None, feval=None, train_fields=None,
       feature_name=None, categorical_feature=None,
       early_stopping_rounds=None, fpreproc=None,
       verbose_eval=None, show_stdv=True, seed=0,
wxchan's avatar
wxchan committed
330
331
332
333
334
335
336
       callbacks=None):
    """Cross-validation with given paramaters.

    Parameters
    ----------
    params : dict
        Booster params.
wxchan's avatar
wxchan committed
337
    train_data : tuple (X, y) or filename of data
wxchan's avatar
wxchan committed
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
        Data to be trained.
    num_boost_round : int
        Number of boosting iterations.
    nfold : int
        Number of folds in CV.
    stratified : bool
        Perform stratified sampling.
    folds : a KFold or StratifiedKFold instance
        Sklearn KFolds or StratifiedKFolds.
    metrics : string or list of strings
        Evaluation metrics to be watched in CV.
    fobj : function
        Custom objective function.
    feval : function
        Custom evaluation function.
    train_fields : dict
354
355
        Other data file in training data. e.g. train_fields['weight'] is weight data
        Support fields: weight, group, init_score
Guolin Ke's avatar
Guolin Ke committed
356
    feature_name : list of str
357
358
359
360
        Feature names
    categorical_feature : list of str or int
        Categorical features, type int represents index, \
        type str represents feature names (need to specify feature_name as well)
wxchan's avatar
wxchan committed
361
    early_stopping_rounds: int
362
        Activates early stopping. CV error needs to decrease at least \
wxchan's avatar
wxchan committed
363
364
365
        every <early_stopping_rounds> round(s) to continue.
        Last entry in evaluation history is the one from best iteration.
    fpreproc : function
366
        Preprocessing function that takes (dtrain, dtest, param) and returns \
wxchan's avatar
wxchan committed
367
368
        transformed versions of those.
    verbose_eval : bool, int, or None, default None
369
370
371
        Whether to display the progress. If None, progress will be displayed \
        when np.ndarray is returned. If True, progress will be displayed at \
        boosting stage. If an integer is given, progress will be displayed \
wxchan's avatar
wxchan committed
372
373
374
375
376
377
378
379
380
381
382
383
384
        at every given `verbose_eval` boosting stage.
    show_stdv : bool, default True
        Whether to display the standard deviation in progress.
        Results are not affected, and always contains std.
    seed : int
        Seed used to generate the folds (passed to numpy.random.seed).
    callbacks : list of callback functions
        List of callback functions that are applied at end of each iteration.

    Returns
    -------
    evaluation history : list(string)
    """
385
386
387
388
    if metrics:
        params.setdefault('metric', [])
        if is_str(metrics):
            params['metric'].append(metrics)
wxchan's avatar
wxchan committed
389
        else:
390
            params['metric'].extend(metrics)
wxchan's avatar
wxchan committed
391

392
    train_set = _construct_dataset(train_data, None, params,
Guolin Ke's avatar
Guolin Ke committed
393
394
395
								   other_fields=train_fields,
                                   feature_name=feature_name,
                                   categorical_feature=categorical_feature)
wxchan's avatar
wxchan committed
396

wxchan's avatar
wxchan committed
397
    results = collections.defaultdict(list)
wxchan's avatar
wxchan committed
398
399
400
    cvfolds = _make_n_folds(train_set, nfold, params, seed, fpreproc, stratified)

    # setup callbacks
wxchan's avatar
wxchan committed
401
402
403
404
405
406
407
408
409
410
    if not callbacks:
        callbacks = set()
    else:
        for i, cb in enumerate(callbacks):
            cb.__dict__.setdefault('order', i - len(callbacks))
        callbacks = set(callbacks)
    if early_stopping_rounds:
        callbacks.add(callback.early_stop(early_stopping_rounds, verbose=False))
    if verbose_eval is True:
        callbacks.add(callback.print_evaluation(show_stdv=show_stdv))
411
    elif isinstance(verbose_eval, int):
wxchan's avatar
wxchan committed
412
        callbacks.add(callback.print_evaluation(verbose_eval, show_stdv=show_stdv))
wxchan's avatar
wxchan committed
413

wxchan's avatar
wxchan committed
414
415
416
417
    callbacks_before_iter = {cb for cb in callbacks if getattr(cb, 'before_iteration', False)}
    callbacks_after_iter = callbacks - callbacks_before_iter
    callbacks_before_iter = sorted(callbacks_before_iter, key=attrgetter('order'))
    callbacks_after_iter = sorted(callbacks_after_iter, key=attrgetter('order'))
wxchan's avatar
wxchan committed
418
419
420
421
422
423
424
425
426
427
428
429
430
431

    for i in range(num_boost_round):
        for cb in callbacks_before_iter:
            cb(callback.CallbackEnv(model=None,
                                    cvfolds=cvfolds,
                                    iteration=i,
                                    begin_iteration=0,
                                    end_iteration=num_boost_round,
                                    evaluation_result_list=None))
        for fold in cvfolds:
            fold.update(fobj)
        res = _agg_cv_result([f.eval(feval) for f in cvfolds])
        for _, key, mean, _, std in res:
            results[key + '-mean'].append(mean)
wxchan's avatar
wxchan committed
432
            results[key + '-stdv'].append(std)
wxchan's avatar
wxchan committed
433
434
435
436
437
438
439
440
441
442
        try:
            for cb in callbacks_after_iter:
                cb(callback.CallbackEnv(model=None,
                                        cvfolds=cvfolds,
                                        iteration=i,
                                        begin_iteration=0,
                                        end_iteration=num_boost_round,
                                        evaluation_result_list=res))
        except callback.EarlyStopException as e:
            for k in results:
wxchan's avatar
wxchan committed
443
                results[k] = results[k][:e.best_iteration + 1]
wxchan's avatar
wxchan committed
444
            break
wxchan's avatar
wxchan committed
445
    return dict(results)