engine.py 16.2 KB
Newer Older
wxchan's avatar
wxchan committed
1
2
3
4
5
# coding: utf-8
# pylint: disable = invalid-name, W0105
"""Training Library containing training routines of LightGBM."""
from __future__ import absolute_import

wxchan's avatar
wxchan committed
6
7
import collections
from operator import attrgetter
wxchan's avatar
wxchan committed
8
import numpy as np
Guolin Ke's avatar
Guolin Ke committed
9
from .basic import LightGBMError, _InnerPredictor, Dataset, Booster, is_str
wxchan's avatar
wxchan committed
10
11
from . import callback

Guolin Ke's avatar
Guolin Ke committed
12
13
def train(params, train_set, num_boost_round=100,
          valid_sets=None, valid_names=None,
wxchan's avatar
wxchan committed
14
          fobj=None, feval=None, init_model=None,
Guolin Ke's avatar
Guolin Ke committed
15
          feature_name=None, categorical_feature=None,
wxchan's avatar
wxchan committed
16
17
          early_stopping_rounds=None, evals_result=None,
          verbose_eval=True, learning_rates=None, callbacks=None):
wxchan's avatar
wxchan committed
18
19
    """
    Train with given parameters.
wxchan's avatar
wxchan committed
20
21
22
23

    Parameters
    ----------
    params : dict
24
        Parameters for training.
Guolin Ke's avatar
Guolin Ke committed
25
    train_set : Dataset
wxchan's avatar
wxchan committed
26
27
28
        Data to be trained.
    num_boost_round: int
        Number of boosting iterations.
Guolin Ke's avatar
Guolin Ke committed
29
    valid_sets: list of Datasets
wxchan's avatar
wxchan committed
30
31
        List of data to be evaluated during training
    valid_names: list of string
Guolin Ke's avatar
Guolin Ke committed
32
        Names of valid_sets
wxchan's avatar
wxchan committed
33
34
35
36
37
38
39
    fobj : function
        Customized objective function.
    feval : function
        Customized evaluation function.
        Note: should return (eval_name, eval_result, is_higher_better) of list of this
    init_model : file name of lightgbm model or 'Booster' instance
        model used for continued train
Guolin Ke's avatar
Guolin Ke committed
40
    feature_name : list of str
41
42
        Feature names
    categorical_feature : list of str or int
wxchan's avatar
wxchan committed
43
44
        Categorical features,
        type int represents index,
45
        type str represents feature names (need to specify feature_name as well)
wxchan's avatar
wxchan committed
46
47
48
49
50
51
52
    early_stopping_rounds: int
        Activates early stopping.
        Requires at least one validation data and one metric
        If there's more than one, will check all of them
        Returns the model with (best_iter + early_stopping_rounds)
        If early stopping occurs, the model will add 'best_iteration' field
    evals_result: dict or None
Guolin Ke's avatar
Guolin Ke committed
53
        This dictionary used to store all evaluation results of all the items in valid_sets.
wxchan's avatar
wxchan committed
54
55
56
        Example: with a valid_sets containing [valid_set, train_set]
                 and valid_names containing ['eval', 'train']
                 and a paramater containing ('metric':'logloss')
wxchan's avatar
wxchan committed
57
58
59
60
61
        Returns: {'train': {'logloss': ['0.48253', '0.35953', ...]},
                  'eval': {'logloss': ['0.480385', '0.357756', ...]}}
        passed with None means no using this function
    verbose_eval : bool or int
        Requires at least one item in evals.
wxchan's avatar
wxchan committed
62
63
64
65
66
67
68
69
        If `verbose_eval` is True,
            the eval metric on the valid set is printed at each boosting stage.
        If `verbose_eval` is int,
            the eval metric on the valid set is printed at every `verbose_eval` boosting stage.
        The last boosting stage
            or the boosting stage found by using `early_stopping_rounds` is also printed.
        Example: with verbose_eval=4 and at least one item in evals,
            an evaluation metric is printed every 4 (instead of 1) boosting stages.
wxchan's avatar
wxchan committed
70
    learning_rates: list or function
wxchan's avatar
wxchan committed
71
        List of learning rate for each boosting round
72
73
        or a customized function that calculates learning_rate
        in terms of current number of round (e.g. yields learning rate decay)
wxchan's avatar
wxchan committed
74
        - list l: learning_rate = l[current_round]
75
        - function f: learning_rate = f(current_round)
wxchan's avatar
wxchan committed
76
    callbacks : list of callback functions
77
78
        List of callback functions that are applied at each iteration.
        See Callbacks in Python-API.md for more information.
wxchan's avatar
wxchan committed
79
80
81
82
83
84
85

    Returns
    -------
    booster : a trained booster model
    """
    """create predictor first"""
    if is_str(init_model):
Guolin Ke's avatar
Guolin Ke committed
86
        predictor = _InnerPredictor(model_file=init_model)
wxchan's avatar
wxchan committed
87
    elif isinstance(init_model, Booster):
Guolin Ke's avatar
Guolin Ke committed
88
        predictor = init_model._to_predictor()
wxchan's avatar
wxchan committed
89
90
    else:
        predictor = None
91
    init_iteration = predictor.num_total_iteration if predictor is not None else 0
Guolin Ke's avatar
Guolin Ke committed
92
93
    """check dataset"""
    if not isinstance(train_set, Dataset):
94
        raise TypeError("Traninig only accepts Dataset object")
Guolin Ke's avatar
Guolin Ke committed
95

96
    train_set._update_params(params)
Guolin Ke's avatar
Guolin Ke committed
97
98
99
100
    train_set._set_predictor(predictor)
    train_set.set_feature_name(feature_name)
    train_set.set_categorical_feature(categorical_feature)

wxchan's avatar
wxchan committed
101
102
    is_valid_contain_train = False
    train_data_name = "training"
Guolin Ke's avatar
Guolin Ke committed
103
    reduced_valid_sets = []
wxchan's avatar
wxchan committed
104
    name_valid_sets = []
105
    if valid_sets is not None:
Guolin Ke's avatar
Guolin Ke committed
106
107
        if isinstance(valid_sets, Dataset):
            valid_sets = [valid_sets]
wxchan's avatar
wxchan committed
108
109
        if isinstance(valid_names, str):
            valid_names = [valid_names]
Guolin Ke's avatar
Guolin Ke committed
110
        for i, valid_data in enumerate(valid_sets):
wxchan's avatar
wxchan committed
111
            """reduce cost for prediction training data"""
Guolin Ke's avatar
Guolin Ke committed
112
            if valid_data is train_set:
wxchan's avatar
wxchan committed
113
114
115
116
                is_valid_contain_train = True
                if valid_names is not None:
                    train_data_name = valid_names[i]
                continue
Guolin Ke's avatar
Guolin Ke committed
117
            if not isinstance(valid_data, Dataset):
118
                raise TypeError("Traninig only accepts Dataset object")
Guolin Ke's avatar
Guolin Ke committed
119
120
            valid_data.set_reference(train_set)
            reduced_valid_sets.append(valid_data)
121
            if valid_names is not None and len(valid_names) > i:
wxchan's avatar
wxchan committed
122
123
124
                name_valid_sets.append(valid_names[i])
            else:
                name_valid_sets.append('valid_'+str(i))
125
126
        for valid_data in valid_sets:
            valid_data._update_params(params)
wxchan's avatar
wxchan committed
127
    """process callbacks"""
128
    if callbacks is None:
wxchan's avatar
wxchan committed
129
130
131
132
133
        callbacks = set()
    else:
        for i, cb in enumerate(callbacks):
            cb.__dict__.setdefault('order', i - len(callbacks))
        callbacks = set(callbacks)
wxchan's avatar
wxchan committed
134
135

    # Most of legacy advanced options becomes callbacks
wxchan's avatar
wxchan committed
136
137
    if verbose_eval is True:
        callbacks.add(callback.print_evaluation())
138
    elif isinstance(verbose_eval, int):
wxchan's avatar
wxchan committed
139
        callbacks.add(callback.print_evaluation(verbose_eval))
wxchan's avatar
wxchan committed
140

141
    if early_stopping_rounds is not None:
142
        callbacks.add(callback.early_stopping(early_stopping_rounds, verbose=bool(verbose_eval)))
143

wxchan's avatar
wxchan committed
144
    if learning_rates is not None:
145
        callbacks.add(callback.reset_parameter(learning_rate=learning_rates))
wxchan's avatar
wxchan committed
146
147

    if evals_result is not None:
wxchan's avatar
wxchan committed
148
149
150
151
152
153
        callbacks.add(callback.record_evaluation(evals_result))

    callbacks_before_iter = {cb for cb in callbacks if getattr(cb, 'before_iteration', False)}
    callbacks_after_iter = callbacks - callbacks_before_iter
    callbacks_before_iter = sorted(callbacks_before_iter, key=attrgetter('order'))
    callbacks_after_iter = sorted(callbacks_after_iter, key=attrgetter('order'))
wxchan's avatar
wxchan committed
154
155
156
157
158

    """construct booster"""
    booster = Booster(params=params, train_set=train_set)
    if is_valid_contain_train:
        booster.set_train_data_name(train_data_name)
Guolin Ke's avatar
Guolin Ke committed
159
    for valid_set, name_valid_set in zip(reduced_valid_sets, name_valid_sets):
wxchan's avatar
wxchan committed
160
        booster.add_valid(valid_set, name_valid_set)
wxchan's avatar
wxchan committed
161

wxchan's avatar
wxchan committed
162
    """start training"""
163
    for i in range(init_iteration, init_iteration + num_boost_round):
wxchan's avatar
wxchan committed
164
165
166
167
        for cb in callbacks_before_iter:
            cb(callback.CallbackEnv(model=booster,
                                    cvfolds=None,
                                    iteration=i,
168
169
                                    begin_iteration=init_iteration,
                                    end_iteration=init_iteration + num_boost_round,
wxchan's avatar
wxchan committed
170
171
172
173
174
175
                                    evaluation_result_list=None))

        booster.update(fobj=fobj)

        evaluation_result_list = []
        # check evaluation result.
176
        if valid_sets is not None:
wxchan's avatar
wxchan committed
177
178
179
180
181
182
183
184
            if is_valid_contain_train:
                evaluation_result_list.extend(booster.eval_train(feval))
            evaluation_result_list.extend(booster.eval_valid(feval))
        try:
            for cb in callbacks_after_iter:
                cb(callback.CallbackEnv(model=booster,
                                        cvfolds=None,
                                        iteration=i,
185
186
                                        begin_iteration=init_iteration,
                                        end_iteration=init_iteration + num_boost_round,
wxchan's avatar
wxchan committed
187
188
189
190
191
192
                                        evaluation_result_list=evaluation_result_list))
        except callback.EarlyStopException:
            break
    if booster.attr('best_iteration') is not None:
        booster.best_iteration = int(booster.attr('best_iteration')) + 1
    else:
193
        booster.best_iteration = -1
wxchan's avatar
wxchan committed
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
    return booster


class CVBooster(object):
    """"Auxiliary datastruct to hold one fold of CV."""
    def __init__(self, train_set, valid_test, params):
        """"Initialize the CVBooster"""
        self.train_set = train_set
        self.valid_test = valid_test
        self.booster = Booster(params=params, train_set=train_set)
        self.booster.add_valid(valid_test, 'valid')

    def update(self, fobj):
        """"Update the boosters for one iteration"""
        self.booster.update(fobj=fobj)

    def eval(self, feval):
        """"Evaluate the CVBooster for one iteration."""
        return self.booster.eval_valid(feval)

try:
215
    from sklearn.model_selection import StratifiedKFold
wxchan's avatar
wxchan committed
216
217
    SKLEARN_StratifiedKFold = True
except ImportError:
218
219
220
221
222
    try:
        from sklearn.cross_validation import StratifiedKFold
        SKLEARN_StratifiedKFold = True
    except ImportError:
        SKLEARN_StratifiedKFold = False
wxchan's avatar
wxchan committed
223

wxchan's avatar
wxchan committed
224
def _make_n_folds(full_data, nfold, params, seed, fpreproc=None, stratified=False, shuffle=True):
wxchan's avatar
wxchan committed
225
226
227
228
229
230
    """
    Make an n-fold list of CVBooster from random indices.
    """
    np.random.seed(seed)
    if stratified:
        if SKLEARN_StratifiedKFold:
wxchan's avatar
wxchan committed
231
            sfk = StratifiedKFold(n_splits=nfold, shuffle=shuffle, random_state=seed)
wxchan's avatar
wxchan committed
232
233
            idset = [x[1] for x in sfk.split(X=full_data.get_label(), y=full_data.get_label())]
        else:
234
            raise LightGBMError('Scikit-learn is required for stratified cv')
wxchan's avatar
wxchan committed
235
    else:
Guolin Ke's avatar
Guolin Ke committed
236
        full_data.construct()
wxchan's avatar
wxchan committed
237
238
        if shuffle:
            randidx = np.random.permutation(full_data.num_data())
wxchan's avatar
wxchan committed
239
240
241
242
243
244
245
246
247
        kstep = int(len(randidx) / nfold)
        idset = [randidx[(i * kstep): min(len(randidx), (i + 1) * kstep)] for i in range(nfold)]

    ret = []
    for k in range(nfold):
        train_set = full_data.subset(np.concatenate([idset[i] for i in range(nfold) if k != i]))
        valid_set = full_data.subset(idset[k])
        # run preprocessing on the data set if needed
        if fpreproc is not None:
wxchan's avatar
wxchan committed
248
            train_set, valid_set, tparam = fpreproc(train_set, valid_set, params.copy())
wxchan's avatar
wxchan committed
249
        else:
wxchan's avatar
wxchan committed
250
            tparam = params
wxchan's avatar
wxchan committed
251
252
253
254
255
256
257
        ret.append(CVBooster(train_set, valid_set, tparam))
    return ret

def _agg_cv_result(raw_results):
    """
    Aggregate cross-validation results.
    """
wxchan's avatar
wxchan committed
258
    cvmap = collections.defaultdict(list)
wxchan's avatar
wxchan committed
259
260
261
    metric_type = {}
    for one_result in raw_results:
        for one_line in one_result:
wxchan's avatar
wxchan committed
262
263
264
            metric_type[one_line[1]] = one_line[3]
            cvmap[one_line[1]].append(one_line[2])
    return [('cv_agg', k, np.mean(v), metric_type[k], np.std(v)) for k, v in cvmap.items()]
wxchan's avatar
wxchan committed
265

Guolin Ke's avatar
Guolin Ke committed
266
def cv(params, train_set, num_boost_round=10, nfold=5, stratified=False,
wxchan's avatar
wxchan committed
267
       shuffle=True, metrics=None, fobj=None, feval=None, init_model=None,
Guolin Ke's avatar
Guolin Ke committed
268
269
270
       feature_name=None, categorical_feature=None,
       early_stopping_rounds=None, fpreproc=None,
       verbose_eval=None, show_stdv=True, seed=0,
wxchan's avatar
wxchan committed
271
       callbacks=None):
wxchan's avatar
wxchan committed
272
273
    """
    Cross-validation with given paramaters.
wxchan's avatar
wxchan committed
274
275
276
277
278

    Parameters
    ----------
    params : dict
        Booster params.
Guolin Ke's avatar
Guolin Ke committed
279
    train_set : Dataset
wxchan's avatar
wxchan committed
280
281
282
283
284
285
286
        Data to be trained.
    num_boost_round : int
        Number of boosting iterations.
    nfold : int
        Number of folds in CV.
    stratified : bool
        Perform stratified sampling.
wxchan's avatar
wxchan committed
287
288
    shuffle: bool
        Whether shuffle before split data
wxchan's avatar
wxchan committed
289
290
291
292
293
294
295
296
    folds : a KFold or StratifiedKFold instance
        Sklearn KFolds or StratifiedKFolds.
    metrics : string or list of strings
        Evaluation metrics to be watched in CV.
    fobj : function
        Custom objective function.
    feval : function
        Custom evaluation function.
Guolin Ke's avatar
Guolin Ke committed
297
298
    init_model : file name of lightgbm model or 'Booster' instance
        model used for continued train
Guolin Ke's avatar
Guolin Ke committed
299
    feature_name : list of str
300
301
        Feature names
    categorical_feature : list of str or int
wxchan's avatar
wxchan committed
302
        Categorical features, type int represents index,
303
        type str represents feature names (need to specify feature_name as well)
wxchan's avatar
wxchan committed
304
    early_stopping_rounds: int
wxchan's avatar
wxchan committed
305
        Activates early stopping. CV error needs to decrease at least
wxchan's avatar
wxchan committed
306
307
308
        every <early_stopping_rounds> round(s) to continue.
        Last entry in evaluation history is the one from best iteration.
    fpreproc : function
wxchan's avatar
wxchan committed
309
310
        Preprocessing function that takes (dtrain, dtest, param)
        and returns transformed versions of those.
wxchan's avatar
wxchan committed
311
    verbose_eval : bool, int, or None, default None
wxchan's avatar
wxchan committed
312
313
314
315
316
        Whether to display the progress.
        If None, progress will be displayed when np.ndarray is returned.
        If True, progress will be displayed at boosting stage.
        If an integer is given,
            progress will be displayed at every given `verbose_eval` boosting stage.
wxchan's avatar
wxchan committed
317
318
319
320
321
322
    show_stdv : bool, default True
        Whether to display the standard deviation in progress.
        Results are not affected, and always contains std.
    seed : int
        Seed used to generate the folds (passed to numpy.random.seed).
    callbacks : list of callback functions
323
324
        List of callback functions that are applied at each iteration.
        See Callbacks in Python-API.md for more information.
wxchan's avatar
wxchan committed
325
326
327
328
329

    Returns
    -------
    evaluation history : list(string)
    """
Guolin Ke's avatar
Guolin Ke committed
330
    if not isinstance(train_set, Dataset):
331
        raise TypeError("Traninig only accepts Dataset object")
Guolin Ke's avatar
Guolin Ke committed
332
333
334
335
336
337
338

    if is_str(init_model):
        predictor = _InnerPredictor(model_file=init_model)
    elif isinstance(init_model, Booster):
        predictor = init_model._to_predictor()
    else:
        predictor = None
339
    train_set._update_params(params)
Guolin Ke's avatar
Guolin Ke committed
340
341
342
343
    train_set._set_predictor(predictor)
    train_set.set_feature_name(feature_name)
    train_set.set_categorical_feature(categorical_feature)

344
345
346
347
    if metrics:
        params.setdefault('metric', [])
        if is_str(metrics):
            params['metric'].append(metrics)
wxchan's avatar
wxchan committed
348
        else:
349
            params['metric'].extend(metrics)
wxchan's avatar
wxchan committed
350

wxchan's avatar
wxchan committed
351
    results = collections.defaultdict(list)
wxchan's avatar
wxchan committed
352
    cvfolds = _make_n_folds(train_set, nfold, params, seed, fpreproc, stratified, shuffle)
wxchan's avatar
wxchan committed
353
354

    # setup callbacks
355
    if callbacks is None:
wxchan's avatar
wxchan committed
356
357
358
359
360
        callbacks = set()
    else:
        for i, cb in enumerate(callbacks):
            cb.__dict__.setdefault('order', i - len(callbacks))
        callbacks = set(callbacks)
361
    if early_stopping_rounds is not None:
362
        callbacks.add(callback.early_stopping(early_stopping_rounds, verbose=False))
wxchan's avatar
wxchan committed
363
364
    if verbose_eval is True:
        callbacks.add(callback.print_evaluation(show_stdv=show_stdv))
365
    elif isinstance(verbose_eval, int):
wxchan's avatar
wxchan committed
366
        callbacks.add(callback.print_evaluation(verbose_eval, show_stdv=show_stdv))
wxchan's avatar
wxchan committed
367

wxchan's avatar
wxchan committed
368
369
370
371
    callbacks_before_iter = {cb for cb in callbacks if getattr(cb, 'before_iteration', False)}
    callbacks_after_iter = callbacks - callbacks_before_iter
    callbacks_before_iter = sorted(callbacks_before_iter, key=attrgetter('order'))
    callbacks_after_iter = sorted(callbacks_after_iter, key=attrgetter('order'))
wxchan's avatar
wxchan committed
372
373
374
375
376
377
378
379
380
381
382
383
384
385

    for i in range(num_boost_round):
        for cb in callbacks_before_iter:
            cb(callback.CallbackEnv(model=None,
                                    cvfolds=cvfolds,
                                    iteration=i,
                                    begin_iteration=0,
                                    end_iteration=num_boost_round,
                                    evaluation_result_list=None))
        for fold in cvfolds:
            fold.update(fobj)
        res = _agg_cv_result([f.eval(feval) for f in cvfolds])
        for _, key, mean, _, std in res:
            results[key + '-mean'].append(mean)
wxchan's avatar
wxchan committed
386
            results[key + '-stdv'].append(std)
wxchan's avatar
wxchan committed
387
388
389
390
391
392
393
394
395
396
        try:
            for cb in callbacks_after_iter:
                cb(callback.CallbackEnv(model=None,
                                        cvfolds=cvfolds,
                                        iteration=i,
                                        begin_iteration=0,
                                        end_iteration=num_boost_round,
                                        evaluation_result_list=res))
        except callback.EarlyStopException as e:
            for k in results:
wxchan's avatar
wxchan committed
397
                results[k] = results[k][:e.best_iteration + 1]
wxchan's avatar
wxchan committed
398
            break
wxchan's avatar
wxchan committed
399
    return dict(results)