basic.py 109 KB
Newer Older
wxchan's avatar
wxchan committed
1
# coding: utf-8
2
3
# pylint: disable = invalid-name, C0111, C0301
# pylint: disable = R0912, R0913, R0914, W0105, W0201, W0212
4
"""Wrapper for C API of LightGBM."""
wxchan's avatar
wxchan committed
5
6
from __future__ import absolute_import

7
import copy
wxchan's avatar
wxchan committed
8
import ctypes
9
import os
wxchan's avatar
wxchan committed
10
import warnings
wxchan's avatar
wxchan committed
11
from tempfile import NamedTemporaryFile
wxchan's avatar
wxchan committed
12
13
14
15

import numpy as np
import scipy.sparse

16
from .compat import (PANDAS_INSTALLED, DataFrame, Series, is_dtype_sparse,
17
                     DataTable,
18
19
                     decode_string, string_type,
                     integer_types, numeric_types,
20
                     json, json_default_with_numpy,
21
                     range_, zip_)
wxchan's avatar
wxchan committed
22
23
from .libpath import find_lib_path

wxchan's avatar
wxchan committed
24

wxchan's avatar
wxchan committed
25
def _load_lib():
26
    """Load LightGBM library."""
wxchan's avatar
wxchan committed
27
28
    lib_path = find_lib_path()
    if len(lib_path) == 0:
29
        return None
wxchan's avatar
wxchan committed
30
31
32
33
    lib = ctypes.cdll.LoadLibrary(lib_path[0])
    lib.LGBM_GetLastError.restype = ctypes.c_char_p
    return lib

wxchan's avatar
wxchan committed
34

wxchan's avatar
wxchan committed
35
36
_LIB = _load_lib()

wxchan's avatar
wxchan committed
37

wxchan's avatar
wxchan committed
38
def _safe_call(ret):
39
40
    """Check the return value from C API call.

wxchan's avatar
wxchan committed
41
42
43
    Parameters
    ----------
    ret : int
44
        The return value from C API calls.
wxchan's avatar
wxchan committed
45
46
    """
    if ret != 0:
47
        raise LightGBMError(decode_string(_LIB.LGBM_GetLastError()))
wxchan's avatar
wxchan committed
48

wxchan's avatar
wxchan committed
49

wxchan's avatar
wxchan committed
50
def is_numeric(obj):
51
    """Check whether object is a number or not, include numpy number, etc."""
wxchan's avatar
wxchan committed
52
53
54
    try:
        float(obj)
        return True
wxchan's avatar
wxchan committed
55
56
57
    except (TypeError, ValueError):
        # TypeError: obj is not a string or a number
        # ValueError: invalid literal
wxchan's avatar
wxchan committed
58
59
        return False

wxchan's avatar
wxchan committed
60

wxchan's avatar
wxchan committed
61
def is_numpy_1d_array(data):
62
    """Check whether data is a numpy 1-D array."""
63
    return isinstance(data, np.ndarray) and len(data.shape) == 1
wxchan's avatar
wxchan committed
64

wxchan's avatar
wxchan committed
65

wxchan's avatar
wxchan committed
66
def is_1d_list(data):
67
68
    """Check whether data is a 1-D list."""
    return isinstance(data, list) and (not data or is_numeric(data[0]))
wxchan's avatar
wxchan committed
69

wxchan's avatar
wxchan committed
70

71
def list_to_1d_numpy(data, dtype=np.float32, name='list'):
72
    """Convert data to numpy 1-D array."""
wxchan's avatar
wxchan committed
73
74
75
76
77
78
79
    if is_numpy_1d_array(data):
        if data.dtype == dtype:
            return data
        else:
            return data.astype(dtype=dtype, copy=False)
    elif is_1d_list(data):
        return np.array(data, dtype=dtype, copy=False)
80
    elif isinstance(data, Series):
81
82
        if _get_bad_pandas_dtypes([data.dtypes]):
            raise ValueError('Series.dtypes must be int, float or bool')
83
        return np.array(data, dtype=dtype, copy=False)  # SparseArray should be supported as well
wxchan's avatar
wxchan committed
84
    else:
85
86
        raise TypeError("Wrong type({0}) for {1}.\n"
                        "It should be list, numpy 1-D array or pandas Series".format(type(data).__name__, name))
wxchan's avatar
wxchan committed
87

wxchan's avatar
wxchan committed
88

wxchan's avatar
wxchan committed
89
def cfloat32_array_to_numpy(cptr, length):
90
    """Convert a ctypes float pointer array to a numpy array."""
wxchan's avatar
wxchan committed
91
    if isinstance(cptr, ctypes.POINTER(ctypes.c_float)):
92
        return np.fromiter(cptr, dtype=np.float32, count=length)
wxchan's avatar
wxchan committed
93
    else:
94
        raise RuntimeError('Expected float pointer')
wxchan's avatar
wxchan committed
95

Guolin Ke's avatar
Guolin Ke committed
96

Guolin Ke's avatar
Guolin Ke committed
97
def cfloat64_array_to_numpy(cptr, length):
98
    """Convert a ctypes double pointer array to a numpy array."""
Guolin Ke's avatar
Guolin Ke committed
99
100
101
102
103
    if isinstance(cptr, ctypes.POINTER(ctypes.c_double)):
        return np.fromiter(cptr, dtype=np.float64, count=length)
    else:
        raise RuntimeError('Expected double pointer')

wxchan's avatar
wxchan committed
104

wxchan's avatar
wxchan committed
105
def cint32_array_to_numpy(cptr, length):
106
    """Convert a ctypes int pointer array to a numpy array."""
wxchan's avatar
wxchan committed
107
    if isinstance(cptr, ctypes.POINTER(ctypes.c_int32)):
108
        return np.fromiter(cptr, dtype=np.int32, count=length)
wxchan's avatar
wxchan committed
109
    else:
110
        raise RuntimeError('Expected int pointer')
wxchan's avatar
wxchan committed
111

wxchan's avatar
wxchan committed
112

113
114
115
116
117
118
119
120
def cint8_array_to_numpy(cptr, length):
    """Convert a ctypes int pointer array to a numpy array."""
    if isinstance(cptr, ctypes.POINTER(ctypes.c_int8)):
        return np.fromiter(cptr, dtype=np.int8, count=length)
    else:
        raise RuntimeError('Expected int pointer')


wxchan's avatar
wxchan committed
121
def c_str(string):
122
    """Convert a Python string to C string."""
wxchan's avatar
wxchan committed
123
124
    return ctypes.c_char_p(string.encode('utf-8'))

wxchan's avatar
wxchan committed
125

wxchan's avatar
wxchan committed
126
def c_array(ctype, values):
127
    """Convert a Python array to C array."""
wxchan's avatar
wxchan committed
128
129
    return (ctype * len(values))(*values)

wxchan's avatar
wxchan committed
130

wxchan's avatar
wxchan committed
131
def param_dict_to_str(data):
132
    """Convert Python dictionary to string, which is passed to C API."""
133
    if data is None or not data:
wxchan's avatar
wxchan committed
134
135
136
        return ""
    pairs = []
    for key, val in data.items():
137
        if isinstance(val, (list, tuple, set)) or is_numpy_1d_array(val):
wxchan's avatar
wxchan committed
138
            pairs.append(str(key) + '=' + ','.join(map(str, val)))
wxchan's avatar
wxchan committed
139
        elif isinstance(val, string_type) or isinstance(val, numeric_types) or is_numeric(val):
wxchan's avatar
wxchan committed
140
            pairs.append(str(key) + '=' + str(val))
141
        elif val is not None:
142
            raise TypeError('Unknown type of parameter:%s, got:%s'
wxchan's avatar
wxchan committed
143
144
                            % (key, type(val).__name__))
    return ' '.join(pairs)
145

wxchan's avatar
wxchan committed
146

147
class _TempFile(object):
148
149
150
151
    def __enter__(self):
        with NamedTemporaryFile(prefix="lightgbm_tmp_", delete=True) as f:
            self.name = f.name
        return self
wxchan's avatar
wxchan committed
152

153
154
155
    def __exit__(self, exc_type, exc_val, exc_tb):
        if os.path.isfile(self.name):
            os.remove(self.name)
wxchan's avatar
wxchan committed
156

157
158
159
160
    def readlines(self):
        with open(self.name, "r+") as f:
            ret = f.readlines()
        return ret
wxchan's avatar
wxchan committed
161

162
163
    def writelines(self, lines):
        with open(self.name, "w+") as f:
164
            f.writelines(lines)
165

wxchan's avatar
wxchan committed
166

167
class LightGBMError(Exception):
168
169
    """Error thrown by LightGBM."""

170
171
172
    pass


173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
class _ConfigAliases(object):
    aliases = {"boosting": {"boosting",
                            "boosting_type",
                            "boost"},
               "categorical_feature": {"categorical_feature",
                                       "cat_feature",
                                       "categorical_column",
                                       "cat_column"},
               "early_stopping_round": {"early_stopping_round",
                                        "early_stopping_rounds",
                                        "early_stopping",
                                        "n_iter_no_change"},
               "eval_at": {"eval_at",
                           "ndcg_eval_at",
                           "ndcg_at",
                           "map_eval_at",
                           "map_at"},
               "header": {"header",
                          "has_header"},
               "machines": {"machines",
                            "workers",
                            "nodes"},
               "metric": {"metric",
                          "metrics",
                          "metric_types"},
               "num_class": {"num_class",
                             "num_classes"},
               "num_iterations": {"num_iterations",
                                  "num_iteration",
                                  "n_iter",
                                  "num_tree",
                                  "num_trees",
                                  "num_round",
                                  "num_rounds",
                                  "num_boost_round",
                                  "n_estimators"},
               "objective": {"objective",
                             "objective_type",
                             "app",
                             "application"},
               "verbosity": {"verbosity",
                             "verbose"}}

    @classmethod
    def get(cls, *args):
        ret = set()
        for i in args:
            ret |= cls.aliases.get(i, set())
        return ret


224
225
MAX_INT32 = (1 << 31) - 1

226
"""Macro definition of data type in C API of LightGBM"""
wxchan's avatar
wxchan committed
227
228
229
230
C_API_DTYPE_FLOAT32 = 0
C_API_DTYPE_FLOAT64 = 1
C_API_DTYPE_INT32 = 2
C_API_DTYPE_INT64 = 3
231
C_API_DTYPE_INT8 = 4
Guolin Ke's avatar
Guolin Ke committed
232

233
"""Matrix is row major in Python"""
wxchan's avatar
wxchan committed
234
235
C_API_IS_ROW_MAJOR = 1

236
"""Macro definition of prediction type in C API of LightGBM"""
wxchan's avatar
wxchan committed
237
238
239
C_API_PREDICT_NORMAL = 0
C_API_PREDICT_RAW_SCORE = 1
C_API_PREDICT_LEAF_INDEX = 2
240
C_API_PREDICT_CONTRIB = 3
wxchan's avatar
wxchan committed
241

242
"""Data type of data field"""
wxchan's avatar
wxchan committed
243
244
FIELD_TYPE_MAPPER = {"label": C_API_DTYPE_FLOAT32,
                     "weight": C_API_DTYPE_FLOAT32,
Guolin Ke's avatar
Guolin Ke committed
245
                     "init_score": C_API_DTYPE_FLOAT64,
246
247
248
                     "group": C_API_DTYPE_INT32,
                     "feature_penalty": C_API_DTYPE_FLOAT64,
                     "monotone_constraints": C_API_DTYPE_INT8}
wxchan's avatar
wxchan committed
249

wxchan's avatar
wxchan committed
250

251
def convert_from_sliced_object(data):
252
    """Fix the memory of multi-dimensional sliced object."""
253
    if isinstance(data, np.ndarray) and isinstance(data.base, np.ndarray):
254
        if not data.flags.c_contiguous:
255
256
            warnings.warn("Usage of np.ndarray subset (sliced data) is not recommended "
                          "due to it will double the peak memory cost in LightGBM.")
257
258
259
260
            return np.copy(data)
    return data


wxchan's avatar
wxchan committed
261
def c_float_array(data):
262
    """Get pointer of float numpy array / list."""
wxchan's avatar
wxchan committed
263
264
265
    if is_1d_list(data):
        data = np.array(data, copy=False)
    if is_numpy_1d_array(data):
266
267
        data = convert_from_sliced_object(data)
        assert data.flags.c_contiguous
wxchan's avatar
wxchan committed
268
269
270
271
272
273
274
        if data.dtype == np.float32:
            ptr_data = data.ctypes.data_as(ctypes.POINTER(ctypes.c_float))
            type_data = C_API_DTYPE_FLOAT32
        elif data.dtype == np.float64:
            ptr_data = data.ctypes.data_as(ctypes.POINTER(ctypes.c_double))
            type_data = C_API_DTYPE_FLOAT64
        else:
275
            raise TypeError("Expected np.float32 or np.float64, met type({})"
wxchan's avatar
wxchan committed
276
277
                            .format(data.dtype))
    else:
278
        raise TypeError("Unknown type({})".format(type(data).__name__))
279
    return (ptr_data, type_data, data)  # return `data` to avoid the temporary copy is freed
wxchan's avatar
wxchan committed
280

wxchan's avatar
wxchan committed
281

wxchan's avatar
wxchan committed
282
def c_int_array(data):
283
    """Get pointer of int numpy array / list."""
wxchan's avatar
wxchan committed
284
285
286
    if is_1d_list(data):
        data = np.array(data, copy=False)
    if is_numpy_1d_array(data):
287
288
        data = convert_from_sliced_object(data)
        assert data.flags.c_contiguous
wxchan's avatar
wxchan committed
289
290
291
292
293
294
295
        if data.dtype == np.int32:
            ptr_data = data.ctypes.data_as(ctypes.POINTER(ctypes.c_int32))
            type_data = C_API_DTYPE_INT32
        elif data.dtype == np.int64:
            ptr_data = data.ctypes.data_as(ctypes.POINTER(ctypes.c_int64))
            type_data = C_API_DTYPE_INT64
        else:
296
            raise TypeError("Expected np.int32 or np.int64, met type({})"
wxchan's avatar
wxchan committed
297
298
                            .format(data.dtype))
    else:
299
        raise TypeError("Unknown type({})".format(type(data).__name__))
300
    return (ptr_data, type_data, data)  # return `data` to avoid the temporary copy is freed
wxchan's avatar
wxchan committed
301

wxchan's avatar
wxchan committed
302

303
304
305
306
307
308
309
310
311
312
313
def _get_bad_pandas_dtypes(dtypes):
    pandas_dtype_mapper = {'int8': 'int', 'int16': 'int', 'int32': 'int',
                           'int64': 'int', 'uint8': 'int', 'uint16': 'int',
                           'uint32': 'int', 'uint64': 'int', 'bool': 'int',
                           'float16': 'float', 'float32': 'float', 'float64': 'float'}
    bad_indices = [i for i, dtype in enumerate(dtypes) if (dtype.name not in pandas_dtype_mapper
                                                           and (not is_dtype_sparse(dtype)
                                                                or dtype.subtype.name not in pandas_dtype_mapper))]
    return bad_indices


314
def _data_from_pandas(data, feature_name, categorical_feature, pandas_categorical):
315
    if isinstance(data, DataFrame):
316
317
        if len(data.shape) != 2 or data.shape[0] < 1:
            raise ValueError('Input data must be 2 dimensional and non empty.')
318
319
        if feature_name == 'auto' or feature_name is None:
            data = data.rename(columns=str)
320
321
        cat_cols = list(data.select_dtypes(include=['category']).columns)
        cat_cols_not_ordered = [col for col in cat_cols if not data[col].cat.ordered]
322
323
324
325
326
        if pandas_categorical is None:  # train dataset
            pandas_categorical = [list(data[col].cat.categories) for col in cat_cols]
        else:
            if len(cat_cols) != len(pandas_categorical):
                raise ValueError('train and valid dataset categorical_feature do not match.')
327
            for col, category in zip_(cat_cols, pandas_categorical):
328
329
                if list(data[col].cat.categories) != list(category):
                    data[col] = data[col].cat.set_categories(category)
330
        if len(cat_cols):  # cat_cols is list
331
            data = data.copy()  # not alter origin DataFrame
332
            data[cat_cols] = data[cat_cols].apply(lambda x: x.cat.codes).replace({-1: np.nan})
333
334
335
        if categorical_feature is not None:
            if feature_name is None:
                feature_name = list(data.columns)
336
            if categorical_feature == 'auto':  # use cat cols from DataFrame
337
                categorical_feature = cat_cols_not_ordered
338
339
            else:  # use cat cols specified by user
                categorical_feature = list(categorical_feature)
340
341
        if feature_name == 'auto':
            feature_name = list(data.columns)
342
343
        bad_indices = _get_bad_pandas_dtypes(data.dtypes)
        if bad_indices:
344
            raise ValueError("DataFrame.dtypes for data must be int, float or bool.\n"
345
                             "Did not expect the data types in the following fields: "
346
                             + ', '.join(data.columns[bad_indices]))
347
348
349
        data = data.values
        if data.dtype != np.float32 and data.dtype != np.float64:
            data = data.astype(np.float32)
350
351
352
353
354
355
    else:
        if feature_name == 'auto':
            feature_name = None
        if categorical_feature == 'auto':
            categorical_feature = None
    return data, feature_name, categorical_feature, pandas_categorical
356
357
358
359
360
361


def _label_from_pandas(label):
    if isinstance(label, DataFrame):
        if len(label.columns) > 1:
            raise ValueError('DataFrame for label cannot have multiple columns')
362
        if _get_bad_pandas_dtypes(label.dtypes):
363
            raise ValueError('DataFrame.dtypes for label must be int, float or bool')
364
        label = np.ravel(label.values.astype(np.float32, copy=False))
365
366
367
    return label


368
369
370
371
372
373
374
375
376
377
378
def _dump_pandas_categorical(pandas_categorical, file_name=None):
    pandas_str = ('\npandas_categorical:'
                  + json.dumps(pandas_categorical, default=json_default_with_numpy)
                  + '\n')
    if file_name is not None:
        with open(file_name, 'a') as f:
            f.write(pandas_str)
    return pandas_str


def _load_pandas_categorical(file_name=None, model_str=None):
379
380
    pandas_key = 'pandas_categorical:'
    offset = -len(pandas_key)
381
    if file_name is not None:
382
383
384
385
386
387
388
389
390
391
392
393
394
        max_offset = -os.path.getsize(file_name)
        with open(file_name, 'rb') as f:
            while True:
                if offset < max_offset:
                    offset = max_offset
                f.seek(offset, os.SEEK_END)
                lines = f.readlines()
                if len(lines) >= 2:
                    break
                offset *= 2
        last_line = decode_string(lines[-1]).strip()
        if not last_line.startswith(pandas_key):
            last_line = decode_string(lines[-2]).strip()
395
    elif model_str is not None:
396
397
398
399
400
401
        idx = model_str.rfind('\n', 0, offset)
        last_line = model_str[idx:].strip()
    if last_line.startswith(pandas_key):
        return json.loads(last_line[len(pandas_key):])
    else:
        return None
402
403


Guolin Ke's avatar
Guolin Ke committed
404
class _InnerPredictor(object):
405
406
407
408
409
    """_InnerPredictor of LightGBM.

    Not exposed to user.
    Used only for prediction, usually used for continued training.

Nikita Titov's avatar
Nikita Titov committed
410
411
412
    .. note::

        Can be converted from Booster, but cannot be converted to Booster.
Guolin Ke's avatar
Guolin Ke committed
413
    """
414

415
    def __init__(self, model_file=None, booster_handle=None, pred_parameter=None):
416
        """Initialize the _InnerPredictor.
wxchan's avatar
wxchan committed
417
418
419

        Parameters
        ----------
420
        model_file : string or None, optional (default=None)
wxchan's avatar
wxchan committed
421
            Path to the model file.
422
423
424
425
        booster_handle : object or None, optional (default=None)
            Handle of Booster.
        pred_parameter: dict or None, optional (default=None)
            Other parameters for the prediciton.
wxchan's avatar
wxchan committed
426
427
428
429
430
        """
        self.handle = ctypes.c_void_p()
        self.__is_manage_handle = True
        if model_file is not None:
            """Prediction task"""
Guolin Ke's avatar
Guolin Ke committed
431
            out_num_iterations = ctypes.c_int(0)
wxchan's avatar
wxchan committed
432
433
434
435
            _safe_call(_LIB.LGBM_BoosterCreateFromModelfile(
                c_str(model_file),
                ctypes.byref(out_num_iterations),
                ctypes.byref(self.handle)))
Guolin Ke's avatar
Guolin Ke committed
436
            out_num_class = ctypes.c_int(0)
wxchan's avatar
wxchan committed
437
438
439
440
            _safe_call(_LIB.LGBM_BoosterGetNumClasses(
                self.handle,
                ctypes.byref(out_num_class)))
            self.num_class = out_num_class.value
441
            self.num_total_iteration = out_num_iterations.value
442
            self.pandas_categorical = _load_pandas_categorical(file_name=model_file)
wxchan's avatar
wxchan committed
443
        elif booster_handle is not None:
Guolin Ke's avatar
Guolin Ke committed
444
            self.__is_manage_handle = False
wxchan's avatar
wxchan committed
445
            self.handle = booster_handle
Guolin Ke's avatar
Guolin Ke committed
446
            out_num_class = ctypes.c_int(0)
wxchan's avatar
wxchan committed
447
448
449
450
            _safe_call(_LIB.LGBM_BoosterGetNumClasses(
                self.handle,
                ctypes.byref(out_num_class)))
            self.num_class = out_num_class.value
Guolin Ke's avatar
Guolin Ke committed
451
            out_num_iterations = ctypes.c_int(0)
wxchan's avatar
wxchan committed
452
453
454
            _safe_call(_LIB.LGBM_BoosterGetCurrentIteration(
                self.handle,
                ctypes.byref(out_num_iterations)))
455
            self.num_total_iteration = out_num_iterations.value
456
            self.pandas_categorical = None
wxchan's avatar
wxchan committed
457
        else:
458
            raise TypeError('Need model_file or booster_handle to create a predictor')
wxchan's avatar
wxchan committed
459

460
461
        pred_parameter = {} if pred_parameter is None else pred_parameter
        self.pred_parameter = param_dict_to_str(pred_parameter)
cbecker's avatar
cbecker committed
462

wxchan's avatar
wxchan committed
463
    def __del__(self):
464
465
466
467
468
        try:
            if self.__is_manage_handle:
                _safe_call(_LIB.LGBM_BoosterFree(self.handle))
        except AttributeError:
            pass
wxchan's avatar
wxchan committed
469

470
471
472
473
474
    def __getstate__(self):
        this = self.__dict__.copy()
        this.pop('handle', None)
        return this

wxchan's avatar
wxchan committed
475
    def predict(self, data, num_iteration=-1,
476
                raw_score=False, pred_leaf=False, pred_contrib=False, data_has_header=False,
wxchan's avatar
wxchan committed
477
                is_reshape=True):
478
        """Predict logic.
wxchan's avatar
wxchan committed
479
480
481

        Parameters
        ----------
482
        data : string, numpy array, pandas DataFrame, H2O DataTable's Frame or scipy.sparse
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
            Data source for prediction.
            When data type is string, it represents the path of txt file.
        num_iteration : int, optional (default=-1)
            Iteration used for prediction.
        raw_score : bool, optional (default=False)
            Whether to predict raw scores.
        pred_leaf : bool, optional (default=False)
            Whether to predict leaf index.
        pred_contrib : bool, optional (default=False)
            Whether to predict feature contributions.
        data_has_header : bool, optional (default=False)
            Whether data has header.
            Used only for txt data.
        is_reshape : bool, optional (default=True)
            Whether to reshape to (nrow, ncol).
wxchan's avatar
wxchan committed
498
499
500

        Returns
        -------
501
502
        result : numpy array
            Prediction result.
wxchan's avatar
wxchan committed
503
        """
wxchan's avatar
wxchan committed
504
        if isinstance(data, Dataset):
505
            raise TypeError("Cannot use Dataset instance for prediction, please use raw data instead")
506
        data = _data_from_pandas(data, None, None, self.pandas_categorical)[0]
wxchan's avatar
wxchan committed
507
508
509
510
511
        predict_type = C_API_PREDICT_NORMAL
        if raw_score:
            predict_type = C_API_PREDICT_RAW_SCORE
        if pred_leaf:
            predict_type = C_API_PREDICT_LEAF_INDEX
512
513
        if pred_contrib:
            predict_type = C_API_PREDICT_CONTRIB
wxchan's avatar
wxchan committed
514
        int_data_has_header = 1 if data_has_header else 0
515
516
        if num_iteration > self.num_total_iteration:
            num_iteration = self.num_total_iteration
cbecker's avatar
cbecker committed
517

wxchan's avatar
wxchan committed
518
        if isinstance(data, string_type):
519
            with _TempFile() as f:
wxchan's avatar
wxchan committed
520
521
522
                _safe_call(_LIB.LGBM_BoosterPredictForFile(
                    self.handle,
                    c_str(data),
Guolin Ke's avatar
Guolin Ke committed
523
524
525
                    ctypes.c_int(int_data_has_header),
                    ctypes.c_int(predict_type),
                    ctypes.c_int(num_iteration),
526
                    c_str(self.pred_parameter),
wxchan's avatar
wxchan committed
527
528
                    c_str(f.name)))
                lines = f.readlines()
529
530
                nrow = len(lines)
                preds = [float(token) for line in lines for token in line.split('\t')]
Guolin Ke's avatar
Guolin Ke committed
531
                preds = np.array(preds, dtype=np.float64, copy=False)
wxchan's avatar
wxchan committed
532
        elif isinstance(data, scipy.sparse.csr_matrix):
533
            preds, nrow = self.__pred_for_csr(data, num_iteration, predict_type)
Guolin Ke's avatar
Guolin Ke committed
534
        elif isinstance(data, scipy.sparse.csc_matrix):
535
            preds, nrow = self.__pred_for_csc(data, num_iteration, predict_type)
wxchan's avatar
wxchan committed
536
        elif isinstance(data, np.ndarray):
537
            preds, nrow = self.__pred_for_np2d(data, num_iteration, predict_type)
538
539
540
        elif isinstance(data, list):
            try:
                data = np.array(data)
541
            except BaseException:
542
                raise ValueError('Cannot convert data list to numpy array.')
543
            preds, nrow = self.__pred_for_np2d(data, num_iteration, predict_type)
544
545
        elif isinstance(data, DataTable):
            preds, nrow = self.__pred_for_np2d(data.to_numpy(), num_iteration, predict_type)
wxchan's avatar
wxchan committed
546
547
        else:
            try:
548
                warnings.warn('Converting data to scipy sparse matrix.')
wxchan's avatar
wxchan committed
549
                csr = scipy.sparse.csr_matrix(data)
550
            except BaseException:
551
                raise TypeError('Cannot predict data for type {}'.format(type(data).__name__))
552
            preds, nrow = self.__pred_for_csr(csr, num_iteration, predict_type)
wxchan's avatar
wxchan committed
553
554
        if pred_leaf:
            preds = preds.astype(np.int32)
555
        if is_reshape and preds.size != nrow:
wxchan's avatar
wxchan committed
556
            if preds.size % nrow == 0:
557
                preds = preds.reshape(nrow, -1)
wxchan's avatar
wxchan committed
558
            else:
559
                raise ValueError('Length of predict result (%d) cannot be divide nrow (%d)'
wxchan's avatar
wxchan committed
560
561
562
563
                                 % (preds.size, nrow))
        return preds

    def __get_num_preds(self, num_iteration, nrow, predict_type):
564
        """Get size of prediction result."""
565
566
567
568
569
        if nrow > MAX_INT32:
            raise LightGBMError('LightGBM cannot perform prediction for data'
                                'with number of rows greater than MAX_INT32 (%d).\n'
                                'You can split your data into chunks'
                                'and then concatenate predictions for them' % MAX_INT32)
Guolin Ke's avatar
Guolin Ke committed
570
571
572
        n_preds = ctypes.c_int64(0)
        _safe_call(_LIB.LGBM_BoosterCalcNumPredict(
            self.handle,
Guolin Ke's avatar
Guolin Ke committed
573
574
575
            ctypes.c_int(nrow),
            ctypes.c_int(predict_type),
            ctypes.c_int(num_iteration),
Guolin Ke's avatar
Guolin Ke committed
576
577
            ctypes.byref(n_preds)))
        return n_preds.value
wxchan's avatar
wxchan committed
578
579

    def __pred_for_np2d(self, mat, num_iteration, predict_type):
580
        """Predict for a 2-D numpy matrix."""
wxchan's avatar
wxchan committed
581
        if len(mat.shape) != 2:
582
            raise ValueError('Input numpy.ndarray or list must be 2 dimensional')
wxchan's avatar
wxchan committed
583

584
585
586
        def inner_predict(mat, num_iteration, predict_type, preds=None):
            if mat.dtype == np.float32 or mat.dtype == np.float64:
                data = np.array(mat.reshape(mat.size), dtype=mat.dtype, copy=False)
587
            else:  # change non-float data to float data, need to copy
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
                data = np.array(mat.reshape(mat.size), dtype=np.float32)
            ptr_data, type_ptr_data, _ = c_float_array(data)
            n_preds = self.__get_num_preds(num_iteration, mat.shape[0], predict_type)
            if preds is None:
                preds = np.zeros(n_preds, dtype=np.float64)
            elif len(preds.shape) != 1 or len(preds) != n_preds:
                raise ValueError("Wrong length of pre-allocated predict array")
            out_num_preds = ctypes.c_int64(0)
            _safe_call(_LIB.LGBM_BoosterPredictForMat(
                self.handle,
                ptr_data,
                ctypes.c_int(type_ptr_data),
                ctypes.c_int(mat.shape[0]),
                ctypes.c_int(mat.shape[1]),
                ctypes.c_int(C_API_IS_ROW_MAJOR),
                ctypes.c_int(predict_type),
                ctypes.c_int(num_iteration),
                c_str(self.pred_parameter),
                ctypes.byref(out_num_preds),
                preds.ctypes.data_as(ctypes.POINTER(ctypes.c_double))))
            if n_preds != out_num_preds.value:
                raise ValueError("Wrong length for predict results")
            return preds, mat.shape[0]

        nrow = mat.shape[0]
        if nrow > MAX_INT32:
            sections = np.arange(start=MAX_INT32, stop=nrow, step=MAX_INT32)
            # __get_num_preds() cannot work with nrow > MAX_INT32, so calculate overall number of predictions piecemeal
            n_preds = [self.__get_num_preds(num_iteration, i, predict_type) for i in np.diff([0] + list(sections) + [nrow])]
            n_preds_sections = np.array([0] + n_preds, dtype=np.intp).cumsum()
            preds = np.zeros(sum(n_preds), dtype=np.float64)
619
620
            for chunk, (start_idx_pred, end_idx_pred) in zip_(np.array_split(mat, sections),
                                                              zip_(n_preds_sections, n_preds_sections[1:])):
621
622
623
                # avoid memory consumption by arrays concatenation operations
                inner_predict(chunk, num_iteration, predict_type, preds[start_idx_pred:end_idx_pred])
            return preds, nrow
wxchan's avatar
wxchan committed
624
        else:
625
            return inner_predict(mat, num_iteration, predict_type)
wxchan's avatar
wxchan committed
626
627

    def __pred_for_csr(self, csr, num_iteration, predict_type):
628
        """Predict for a CSR data."""
629
630
631
632
633
634
635
636
637
638
639
640
        def inner_predict(csr, num_iteration, predict_type, preds=None):
            nrow = len(csr.indptr) - 1
            n_preds = self.__get_num_preds(num_iteration, nrow, predict_type)
            if preds is None:
                preds = np.zeros(n_preds, dtype=np.float64)
            elif len(preds.shape) != 1 or len(preds) != n_preds:
                raise ValueError("Wrong length of pre-allocated predict array")
            out_num_preds = ctypes.c_int64(0)

            ptr_indptr, type_ptr_indptr, __ = c_int_array(csr.indptr)
            ptr_data, type_ptr_data, _ = c_float_array(csr.data)

641
642
643
            assert csr.shape[1] <= MAX_INT32
            csr.indices = csr.indices.astype(np.int32, copy=False)

644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
            _safe_call(_LIB.LGBM_BoosterPredictForCSR(
                self.handle,
                ptr_indptr,
                ctypes.c_int32(type_ptr_indptr),
                csr.indices.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
                ptr_data,
                ctypes.c_int(type_ptr_data),
                ctypes.c_int64(len(csr.indptr)),
                ctypes.c_int64(len(csr.data)),
                ctypes.c_int64(csr.shape[1]),
                ctypes.c_int(predict_type),
                ctypes.c_int(num_iteration),
                c_str(self.pred_parameter),
                ctypes.byref(out_num_preds),
                preds.ctypes.data_as(ctypes.POINTER(ctypes.c_double))))
            if n_preds != out_num_preds.value:
                raise ValueError("Wrong length for predict results")
            return preds, nrow
wxchan's avatar
wxchan committed
662

663
664
665
666
667
668
669
670
671
672
673
674
675
676
        nrow = len(csr.indptr) - 1
        if nrow > MAX_INT32:
            sections = [0] + list(np.arange(start=MAX_INT32, stop=nrow, step=MAX_INT32)) + [nrow]
            # __get_num_preds() cannot work with nrow > MAX_INT32, so calculate overall number of predictions piecemeal
            n_preds = [self.__get_num_preds(num_iteration, i, predict_type) for i in np.diff(sections)]
            n_preds_sections = np.array([0] + n_preds, dtype=np.intp).cumsum()
            preds = np.zeros(sum(n_preds), dtype=np.float64)
            for (start_idx, end_idx), (start_idx_pred, end_idx_pred) in zip_(zip_(sections, sections[1:]),
                                                                             zip_(n_preds_sections, n_preds_sections[1:])):
                # avoid memory consumption by arrays concatenation operations
                inner_predict(csr[start_idx:end_idx], num_iteration, predict_type, preds[start_idx_pred:end_idx_pred])
            return preds, nrow
        else:
            return inner_predict(csr, num_iteration, predict_type)
Guolin Ke's avatar
Guolin Ke committed
677
678

    def __pred_for_csc(self, csc, num_iteration, predict_type):
679
        """Predict for a CSC data."""
Guolin Ke's avatar
Guolin Ke committed
680
        nrow = csc.shape[0]
681
682
        if nrow > MAX_INT32:
            return self.__pred_for_csr(csc.tocsr(), num_iteration, predict_type)
Guolin Ke's avatar
Guolin Ke committed
683
684
685
686
        n_preds = self.__get_num_preds(num_iteration, nrow, predict_type)
        preds = np.zeros(n_preds, dtype=np.float64)
        out_num_preds = ctypes.c_int64(0)

687
688
        ptr_indptr, type_ptr_indptr, __ = c_int_array(csc.indptr)
        ptr_data, type_ptr_data, _ = c_float_array(csc.data)
Guolin Ke's avatar
Guolin Ke committed
689

690
691
692
        assert csc.shape[0] <= MAX_INT32
        csc.indices = csc.indices.astype(np.int32, copy=False)

Guolin Ke's avatar
Guolin Ke committed
693
694
695
        _safe_call(_LIB.LGBM_BoosterPredictForCSC(
            self.handle,
            ptr_indptr,
Guolin Ke's avatar
Guolin Ke committed
696
            ctypes.c_int32(type_ptr_indptr),
Guolin Ke's avatar
Guolin Ke committed
697
698
            csc.indices.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
            ptr_data,
Guolin Ke's avatar
Guolin Ke committed
699
700
701
702
703
704
            ctypes.c_int(type_ptr_data),
            ctypes.c_int64(len(csc.indptr)),
            ctypes.c_int64(len(csc.data)),
            ctypes.c_int64(csc.shape[0]),
            ctypes.c_int(predict_type),
            ctypes.c_int(num_iteration),
705
            c_str(self.pred_parameter),
Guolin Ke's avatar
Guolin Ke committed
706
            ctypes.byref(out_num_preds),
wxchan's avatar
wxchan committed
707
            preds.ctypes.data_as(ctypes.POINTER(ctypes.c_double))))
wxchan's avatar
wxchan committed
708
        if n_preds != out_num_preds.value:
709
            raise ValueError("Wrong length for predict results")
wxchan's avatar
wxchan committed
710
711
        return preds, nrow

wxchan's avatar
wxchan committed
712

wxchan's avatar
wxchan committed
713
714
class Dataset(object):
    """Dataset in LightGBM."""
715

716
    def __init__(self, data, label=None, reference=None,
717
                 weight=None, group=None, init_score=None, silent=False,
718
                 feature_name='auto', categorical_feature='auto', params=None,
wxchan's avatar
wxchan committed
719
                 free_raw_data=True):
720
        """Initialize Dataset.
721

wxchan's avatar
wxchan committed
722
723
        Parameters
        ----------
724
        data : string, numpy array, pandas DataFrame, H2O DataTable's Frame, scipy.sparse or list of numpy arrays
wxchan's avatar
wxchan committed
725
            Data source of Dataset.
726
            If string, it represents the path to txt file.
727
        label : list, numpy 1-D array, pandas Series / one-column DataFrame or None, optional (default=None)
728
729
730
            Label of the data.
        reference : Dataset or None, optional (default=None)
            If this is Dataset for validation, training data should be used as reference.
731
        weight : list, numpy 1-D array, pandas Series or None, optional (default=None)
wxchan's avatar
wxchan committed
732
            Weight for each instance.
733
        group : list, numpy 1-D array, pandas Series or None, optional (default=None)
734
            Group/query size for Dataset.
735
        init_score : list, numpy 1-D array, pandas Series or None, optional (default=None)
736
            Init score for Dataset.
737
738
739
740
741
742
743
744
745
        silent : bool, optional (default=False)
            Whether to print messages during construction.
        feature_name : list of strings or 'auto', optional (default="auto")
            Feature names.
            If 'auto' and data is pandas DataFrame, data columns names are used.
        categorical_feature : list of strings or int, or 'auto', optional (default="auto")
            Categorical features.
            If list of int, interpreted as indices.
            If list of strings, interpreted as feature names (need to specify ``feature_name`` as well).
746
            If 'auto' and data is pandas DataFrame, pandas unordered categorical columns are used.
747
            All values in categorical features should be less than int32 max value (2147483647).
748
            Large values could be memory consuming. Consider using consecutive integers starting from zero.
749
            All negative values in categorical features will be treated as missing values.
750
            The output cannot be monotonically constrained with respect to a categorical feature.
Nikita Titov's avatar
Nikita Titov committed
751
        params : dict or None, optional (default=None)
752
            Other parameters for Dataset.
Nikita Titov's avatar
Nikita Titov committed
753
        free_raw_data : bool, optional (default=True)
754
            If True, raw data is freed after constructing inner Dataset.
wxchan's avatar
wxchan committed
755
        """
wxchan's avatar
wxchan committed
756
757
758
759
760
761
        self.handle = None
        self.data = data
        self.label = label
        self.reference = reference
        self.weight = weight
        self.group = group
762
        self.init_score = init_score
wxchan's avatar
wxchan committed
763
764
        self.silent = silent
        self.feature_name = feature_name
765
        self.categorical_feature = categorical_feature
766
        self.params = copy.deepcopy(params)
wxchan's avatar
wxchan committed
767
768
        self.free_raw_data = free_raw_data
        self.used_indices = None
769
        self.need_slice = True
wxchan's avatar
wxchan committed
770
        self._predictor = None
771
        self.pandas_categorical = None
772
        self.params_back_up = None
773
774
        self.feature_penalty = None
        self.monotone_constraints = None
wxchan's avatar
wxchan committed
775
776

    def __del__(self):
777
778
779
780
        try:
            self._free_handle()
        except AttributeError:
            pass
781
782

    def _free_handle(self):
783
        if self.handle is not None:
784
            _safe_call(_LIB.LGBM_DatasetFree(self.handle))
785
            self.handle = None
Guolin Ke's avatar
Guolin Ke committed
786
787
788
        self.need_slice = True
        if self.used_indices is not None:
            self.data = None
Nikita Titov's avatar
Nikita Titov committed
789
        return self
wxchan's avatar
wxchan committed
790

Guolin Ke's avatar
Guolin Ke committed
791
792
793
794
    def _set_init_score_by_predictor(self, predictor, data, used_indices=None):
        data_has_header = False
        if isinstance(data, string_type):
            # check data has header or not
795
            data_has_header = any(self.params.get(alias, False) for alias in _ConfigAliases.get("header"))
Guolin Ke's avatar
Guolin Ke committed
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
        init_score = predictor.predict(data,
                                       raw_score=True,
                                       data_has_header=data_has_header,
                                       is_reshape=False)
        num_data = self.num_data()
        if used_indices is not None:
            assert not self.need_slice
            if isinstance(data, string_type):
                sub_init_score = np.zeros(num_data * predictor.num_class, dtype=np.float32)
                assert num_data == len(used_indices)
                for i in range_(len(used_indices)):
                    for j in range_(predictor.num_class):
                        sub_init_score[i * redictor.num_class + j] = init_score[used_indices[i] * redictor.num_class + j]
                init_score = sub_init_score
        if predictor.num_class > 1:
            # need to regroup init_score
            new_init_score = np.zeros(init_score.size, dtype=np.float32)
            for i in range_(num_data):
                for j in range_(predictor.num_class):
                    new_init_score[j * num_data + i] = init_score[i * predictor.num_class + j]
            init_score = new_init_score
        self.set_init_score(init_score)

819
    def _lazy_init(self, data, label=None, reference=None,
820
                   weight=None, group=None, init_score=None, predictor=None,
wxchan's avatar
wxchan committed
821
                   silent=False, feature_name='auto',
822
                   categorical_feature='auto', params=None):
wxchan's avatar
wxchan committed
823
824
        if data is None:
            self.handle = None
Nikita Titov's avatar
Nikita Titov committed
825
            return self
Guolin Ke's avatar
Guolin Ke committed
826
827
828
        if reference is not None:
            self.pandas_categorical = reference.pandas_categorical
            categorical_feature = reference.categorical_feature
829
830
831
832
        data, feature_name, categorical_feature, self.pandas_categorical = _data_from_pandas(data,
                                                                                             feature_name,
                                                                                             categorical_feature,
                                                                                             self.pandas_categorical)
wxchan's avatar
wxchan committed
833
        label = _label_from_pandas(label)
Guolin Ke's avatar
Guolin Ke committed
834

835
        # process for args
wxchan's avatar
wxchan committed
836
        params = {} if params is None else params
837
838
839
        args_names = (getattr(self.__class__, '_lazy_init')
                      .__code__
                      .co_varnames[:getattr(self.__class__, '_lazy_init').__code__.co_argcount])
840
841
        for key, _ in params.items():
            if key in args_names:
842
843
844
                warnings.warn('{0} keyword has been found in `params` and will be ignored.\n'
                              'Please use {0} argument of the Dataset constructor to pass this parameter.'
                              .format(key))
845
        # user can set verbose with params, it has higher priority
846
        if not any(verbose_alias in params for verbose_alias in _ConfigAliases.get("verbosity")) and silent:
847
            params["verbose"] = -1
848
        # get categorical features
849
850
851
852
853
854
855
856
857
858
859
860
861
        if categorical_feature is not None:
            categorical_indices = set()
            feature_dict = {}
            if feature_name is not None:
                feature_dict = {name: i for i, name in enumerate(feature_name)}
            for name in categorical_feature:
                if isinstance(name, string_type) and name in feature_dict:
                    categorical_indices.add(feature_dict[name])
                elif isinstance(name, integer_types):
                    categorical_indices.add(name)
                else:
                    raise TypeError("Wrong type({}) or unknown name({}) in categorical_feature"
                                    .format(type(name).__name__, name))
862
            if categorical_indices:
863
864
865
866
                for cat_alias in _ConfigAliases.get("categorical_feature"):
                    if cat_alias in params:
                        warnings.warn('{} in param dict is overridden.'.format(cat_alias))
                        params.pop(cat_alias, None)
867
                params['categorical_column'] = sorted(categorical_indices)
868

wxchan's avatar
wxchan committed
869
        params_str = param_dict_to_str(params)
870
        # process for reference dataset
wxchan's avatar
wxchan committed
871
        ref_dataset = None
wxchan's avatar
wxchan committed
872
        if isinstance(reference, Dataset):
873
            ref_dataset = reference.construct().handle
wxchan's avatar
wxchan committed
874
875
        elif reference is not None:
            raise TypeError('Reference dataset should be None or dataset instance')
876
        # start construct data
wxchan's avatar
wxchan committed
877
        if isinstance(data, string_type):
wxchan's avatar
wxchan committed
878
879
880
881
882
883
884
885
            self.handle = ctypes.c_void_p()
            _safe_call(_LIB.LGBM_DatasetCreateFromFile(
                c_str(data),
                c_str(params_str),
                ref_dataset,
                ctypes.byref(self.handle)))
        elif isinstance(data, scipy.sparse.csr_matrix):
            self.__init_from_csr(data, params_str, ref_dataset)
Guolin Ke's avatar
Guolin Ke committed
886
887
        elif isinstance(data, scipy.sparse.csc_matrix):
            self.__init_from_csc(data, params_str, ref_dataset)
wxchan's avatar
wxchan committed
888
889
        elif isinstance(data, np.ndarray):
            self.__init_from_np2d(data, params_str, ref_dataset)
890
891
        elif isinstance(data, list) and len(data) > 0 and all(isinstance(x, np.ndarray) for x in data):
            self.__init_from_list_np2d(data, params_str, ref_dataset)
892
893
        elif isinstance(data, DataTable):
            self.__init_from_np2d(data.to_numpy(), params_str, ref_dataset)
wxchan's avatar
wxchan committed
894
895
896
897
        else:
            try:
                csr = scipy.sparse.csr_matrix(data)
                self.__init_from_csr(csr, params_str, ref_dataset)
898
            except BaseException:
wxchan's avatar
wxchan committed
899
                raise TypeError('Cannot initialize Dataset from {}'.format(type(data).__name__))
wxchan's avatar
wxchan committed
900
901
902
        if label is not None:
            self.set_label(label)
        if self.get_label() is None:
903
            raise ValueError("Label should not be None")
wxchan's avatar
wxchan committed
904
905
906
907
        if weight is not None:
            self.set_weight(weight)
        if group is not None:
            self.set_group(group)
908
909
910
        if isinstance(predictor, _InnerPredictor):
            if self._predictor is None and init_score is not None:
                warnings.warn("The init_score will be overridden by the prediction of init_model.")
Guolin Ke's avatar
Guolin Ke committed
911
            self._set_init_score_by_predictor(predictor, data)
912
913
        elif init_score is not None:
            self.set_init_score(init_score)
Guolin Ke's avatar
Guolin Ke committed
914
915
        elif predictor is not None:
            raise TypeError('Wrong predictor type {}'.format(type(predictor).__name__))
Guolin Ke's avatar
Guolin Ke committed
916
        # set feature names
Nikita Titov's avatar
Nikita Titov committed
917
        return self.set_feature_name(feature_name)
wxchan's avatar
wxchan committed
918
919

    def __init_from_np2d(self, mat, params_str, ref_dataset):
920
        """Initialize data from a 2-D numpy matrix."""
wxchan's avatar
wxchan committed
921
922
923
924
925
926
        if len(mat.shape) != 2:
            raise ValueError('Input numpy.ndarray must be 2 dimensional')

        self.handle = ctypes.c_void_p()
        if mat.dtype == np.float32 or mat.dtype == np.float64:
            data = np.array(mat.reshape(mat.size), dtype=mat.dtype, copy=False)
927
        else:  # change non-float data to float data, need to copy
wxchan's avatar
wxchan committed
928
929
            data = np.array(mat.reshape(mat.size), dtype=np.float32)

930
        ptr_data, type_ptr_data, _ = c_float_array(data)
wxchan's avatar
wxchan committed
931
932
        _safe_call(_LIB.LGBM_DatasetCreateFromMat(
            ptr_data,
Guolin Ke's avatar
Guolin Ke committed
933
934
935
936
            ctypes.c_int(type_ptr_data),
            ctypes.c_int(mat.shape[0]),
            ctypes.c_int(mat.shape[1]),
            ctypes.c_int(C_API_IS_ROW_MAJOR),
wxchan's avatar
wxchan committed
937
938
939
            c_str(params_str),
            ref_dataset,
            ctypes.byref(self.handle)))
Nikita Titov's avatar
Nikita Titov committed
940
        return self
wxchan's avatar
wxchan committed
941

942
    def __init_from_list_np2d(self, mats, params_str, ref_dataset):
943
        """Initialize data from a list of 2-D numpy matrices."""
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
        ncol = mats[0].shape[1]
        nrow = np.zeros((len(mats),), np.int32)
        if mats[0].dtype == np.float64:
            ptr_data = (ctypes.POINTER(ctypes.c_double) * len(mats))()
        else:
            ptr_data = (ctypes.POINTER(ctypes.c_float) * len(mats))()

        holders = []
        type_ptr_data = None

        for i, mat in enumerate(mats):
            if len(mat.shape) != 2:
                raise ValueError('Input numpy.ndarray must be 2 dimensional')

            if mat.shape[1] != ncol:
                raise ValueError('Input arrays must have same number of columns')

            nrow[i] = mat.shape[0]

            if mat.dtype == np.float32 or mat.dtype == np.float64:
                mats[i] = np.array(mat.reshape(mat.size), dtype=mat.dtype, copy=False)
965
            else:  # change non-float data to float data, need to copy
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
                mats[i] = np.array(mat.reshape(mat.size), dtype=np.float32)

            chunk_ptr_data, chunk_type_ptr_data, holder = c_float_array(mats[i])
            if type_ptr_data is not None and chunk_type_ptr_data != type_ptr_data:
                raise ValueError('Input chunks must have same type')
            ptr_data[i] = chunk_ptr_data
            type_ptr_data = chunk_type_ptr_data
            holders.append(holder)

        self.handle = ctypes.c_void_p()
        _safe_call(_LIB.LGBM_DatasetCreateFromMats(
            ctypes.c_int(len(mats)),
            ctypes.cast(ptr_data, ctypes.POINTER(ctypes.POINTER(ctypes.c_double))),
            ctypes.c_int(type_ptr_data),
            nrow.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
            ctypes.c_int(ncol),
            ctypes.c_int(C_API_IS_ROW_MAJOR),
            c_str(params_str),
            ref_dataset,
            ctypes.byref(self.handle)))
Nikita Titov's avatar
Nikita Titov committed
986
        return self
987

wxchan's avatar
wxchan committed
988
    def __init_from_csr(self, csr, params_str, ref_dataset):
989
        """Initialize data from a CSR matrix."""
wxchan's avatar
wxchan committed
990
        if len(csr.indices) != len(csr.data):
991
            raise ValueError('Length mismatch: {} vs {}'.format(len(csr.indices), len(csr.data)))
wxchan's avatar
wxchan committed
992
993
        self.handle = ctypes.c_void_p()

994
995
        ptr_indptr, type_ptr_indptr, __ = c_int_array(csr.indptr)
        ptr_data, type_ptr_data, _ = c_float_array(csr.data)
wxchan's avatar
wxchan committed
996

997
998
999
        assert csr.shape[1] <= MAX_INT32
        csr.indices = csr.indices.astype(np.int32, copy=False)

wxchan's avatar
wxchan committed
1000
1001
        _safe_call(_LIB.LGBM_DatasetCreateFromCSR(
            ptr_indptr,
Guolin Ke's avatar
Guolin Ke committed
1002
            ctypes.c_int(type_ptr_indptr),
wxchan's avatar
wxchan committed
1003
1004
            csr.indices.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
            ptr_data,
Guolin Ke's avatar
Guolin Ke committed
1005
1006
1007
1008
            ctypes.c_int(type_ptr_data),
            ctypes.c_int64(len(csr.indptr)),
            ctypes.c_int64(len(csr.data)),
            ctypes.c_int64(csr.shape[1]),
wxchan's avatar
wxchan committed
1009
1010
1011
            c_str(params_str),
            ref_dataset,
            ctypes.byref(self.handle)))
Nikita Titov's avatar
Nikita Titov committed
1012
        return self
wxchan's avatar
wxchan committed
1013

Guolin Ke's avatar
Guolin Ke committed
1014
    def __init_from_csc(self, csc, params_str, ref_dataset):
1015
        """Initialize data from a CSC matrix."""
Guolin Ke's avatar
Guolin Ke committed
1016
1017
1018
1019
        if len(csc.indices) != len(csc.data):
            raise ValueError('Length mismatch: {} vs {}'.format(len(csc.indices), len(csc.data)))
        self.handle = ctypes.c_void_p()

1020
1021
        ptr_indptr, type_ptr_indptr, __ = c_int_array(csc.indptr)
        ptr_data, type_ptr_data, _ = c_float_array(csc.data)
Guolin Ke's avatar
Guolin Ke committed
1022

1023
1024
1025
        assert csc.shape[0] <= MAX_INT32
        csc.indices = csc.indices.astype(np.int32, copy=False)

Guolin Ke's avatar
Guolin Ke committed
1026
1027
        _safe_call(_LIB.LGBM_DatasetCreateFromCSC(
            ptr_indptr,
Guolin Ke's avatar
Guolin Ke committed
1028
            ctypes.c_int(type_ptr_indptr),
Guolin Ke's avatar
Guolin Ke committed
1029
1030
            csc.indices.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
            ptr_data,
Guolin Ke's avatar
Guolin Ke committed
1031
1032
1033
1034
            ctypes.c_int(type_ptr_data),
            ctypes.c_int64(len(csc.indptr)),
            ctypes.c_int64(len(csc.data)),
            ctypes.c_int64(csc.shape[0]),
Guolin Ke's avatar
Guolin Ke committed
1035
1036
1037
            c_str(params_str),
            ref_dataset,
            ctypes.byref(self.handle)))
Nikita Titov's avatar
Nikita Titov committed
1038
        return self
Guolin Ke's avatar
Guolin Ke committed
1039

wxchan's avatar
wxchan committed
1040
    def construct(self):
1041
1042
1043
1044
1045
        """Lazy init.

        Returns
        -------
        self : Dataset
Nikita Titov's avatar
Nikita Titov committed
1046
            Constructed Dataset object.
1047
        """
1048
        if self.handle is None:
wxchan's avatar
wxchan committed
1049
1050
            if self.reference is not None:
                if self.used_indices is None:
1051
                    # create valid
1052
                    self._lazy_init(self.data, label=self.label, reference=self.reference,
1053
1054
                                    weight=self.weight, group=self.group,
                                    init_score=self.init_score, predictor=self._predictor,
1055
                                    silent=self.silent, feature_name=self.feature_name, params=self.params)
wxchan's avatar
wxchan committed
1056
                else:
1057
                    # construct subset
wxchan's avatar
wxchan committed
1058
                    used_indices = list_to_1d_numpy(self.used_indices, np.int32, name='used_indices')
1059
                    assert used_indices.flags.c_contiguous
Guolin Ke's avatar
Guolin Ke committed
1060
                    if self.reference.group is not None:
1061
                        group_info = np.array(self.reference.group).astype(np.int32, copy=False)
1062
1063
                        _, self.group = np.unique(np.repeat(range_(len(group_info)), repeats=group_info)[self.used_indices],
                                                  return_counts=True)
1064
                    self.handle = ctypes.c_void_p()
wxchan's avatar
wxchan committed
1065
1066
                    params_str = param_dict_to_str(self.params)
                    _safe_call(_LIB.LGBM_DatasetGetSubset(
1067
                        self.reference.construct().handle,
wxchan's avatar
wxchan committed
1068
                        used_indices.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
Guolin Ke's avatar
Guolin Ke committed
1069
                        ctypes.c_int(used_indices.shape[0]),
wxchan's avatar
wxchan committed
1070
1071
                        c_str(params_str),
                        ctypes.byref(self.handle)))
Guolin Ke's avatar
Guolin Ke committed
1072
1073
                    if not self.free_raw_data:
                        self.get_data()
Guolin Ke's avatar
Guolin Ke committed
1074
1075
                    if self.group is not None:
                        self.set_group(self.group)
wxchan's avatar
wxchan committed
1076
1077
                    if self.get_label() is None:
                        raise ValueError("Label should not be None.")
Guolin Ke's avatar
Guolin Ke committed
1078
1079
1080
                    if isinstance(self._predictor, _InnerPredictor) and self._predictor is not self.reference._predictor:
                        self.get_data()
                        self._set_init_score_by_predictor(self._predictor, self.data, used_indices)
wxchan's avatar
wxchan committed
1081
            else:
1082
                # create train
1083
                self._lazy_init(self.data, label=self.label,
1084
1085
1086
                                weight=self.weight, group=self.group,
                                init_score=self.init_score, predictor=self._predictor,
                                silent=self.silent, feature_name=self.feature_name,
1087
                                categorical_feature=self.categorical_feature, params=self.params)
wxchan's avatar
wxchan committed
1088
1089
1090
            if self.free_raw_data:
                self.data = None
        return self
wxchan's avatar
wxchan committed
1091

wxchan's avatar
wxchan committed
1092
    def create_valid(self, data, label=None, weight=None, group=None,
1093
                     init_score=None, silent=False, params=None):
1094
        """Create validation data align with current Dataset.
wxchan's avatar
wxchan committed
1095
1096
1097

        Parameters
        ----------
1098
        data : string, numpy array, pandas DataFrame, H2O DataTable's Frame, scipy.sparse or list of numpy arrays
wxchan's avatar
wxchan committed
1099
            Data source of Dataset.
1100
            If string, it represents the path to txt file.
1101
        label : list, numpy 1-D array, pandas Series / one-column DataFrame or None, optional (default=None)
1102
1103
            Label of the data.
        weight : list, numpy 1-D array, pandas Series or None, optional (default=None)
wxchan's avatar
wxchan committed
1104
            Weight for each instance.
1105
        group : list, numpy 1-D array, pandas Series or None, optional (default=None)
1106
            Group/query size for Dataset.
1107
        init_score : list, numpy 1-D array, pandas Series or None, optional (default=None)
1108
            Init score for Dataset.
1109
1110
        silent : bool, optional (default=False)
            Whether to print messages during construction.
Nikita Titov's avatar
Nikita Titov committed
1111
        params : dict or None, optional (default=None)
1112
            Other parameters for validation Dataset.
1113
1114
1115

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
1116
1117
        valid : Dataset
            Validation Dataset with reference to self.
wxchan's avatar
wxchan committed
1118
        """
1119
        ret = Dataset(data, label=label, reference=self,
1120
1121
                      weight=weight, group=group, init_score=init_score,
                      silent=silent, params=params, free_raw_data=self.free_raw_data)
wxchan's avatar
wxchan committed
1122
        ret._predictor = self._predictor
1123
        ret.pandas_categorical = self.pandas_categorical
wxchan's avatar
wxchan committed
1124
        return ret
wxchan's avatar
wxchan committed
1125

wxchan's avatar
wxchan committed
1126
    def subset(self, used_indices, params=None):
1127
        """Get subset of current Dataset.
wxchan's avatar
wxchan committed
1128
1129
1130
1131

        Parameters
        ----------
        used_indices : list of int
1132
            Indices used to create the subset.
Nikita Titov's avatar
Nikita Titov committed
1133
        params : dict or None, optional (default=None)
1134
            These parameters will be passed to Dataset constructor.
1135
1136
1137
1138
1139

        Returns
        -------
        subset : Dataset
            Subset of the current Dataset.
wxchan's avatar
wxchan committed
1140
        """
wxchan's avatar
wxchan committed
1141
1142
        if params is None:
            params = self.params
wxchan's avatar
wxchan committed
1143
        ret = Dataset(None, reference=self, feature_name=self.feature_name,
1144
1145
                      categorical_feature=self.categorical_feature, params=params,
                      free_raw_data=self.free_raw_data)
wxchan's avatar
wxchan committed
1146
        ret._predictor = self._predictor
1147
        ret.pandas_categorical = self.pandas_categorical
1148
        ret.used_indices = sorted(used_indices)
wxchan's avatar
wxchan committed
1149
1150
1151
        return ret

    def save_binary(self, filename):
1152
        """Save Dataset to a binary file.
wxchan's avatar
wxchan committed
1153
1154
1155
1156
1157

        Parameters
        ----------
        filename : string
            Name of the output file.
Nikita Titov's avatar
Nikita Titov committed
1158
1159
1160
1161
1162

        Returns
        -------
        self : Dataset
            Returns self.
wxchan's avatar
wxchan committed
1163
1164
1165
1166
        """
        _safe_call(_LIB.LGBM_DatasetSaveBinary(
            self.construct().handle,
            c_str(filename)))
Nikita Titov's avatar
Nikita Titov committed
1167
        return self
wxchan's avatar
wxchan committed
1168
1169

    def _update_params(self, params):
1170
1171
        if self.handle is not None and params is not None:
            _safe_call(_LIB.LGBM_DatasetUpdateParam(self.handle, c_str(param_dict_to_str(params))))
wxchan's avatar
wxchan committed
1172
        if not self.params:
1173
            self.params = copy.deepcopy(params)
wxchan's avatar
wxchan committed
1174
        else:
1175
            self.params_back_up = copy.deepcopy(self.params)
wxchan's avatar
wxchan committed
1176
            self.params.update(params)
Nikita Titov's avatar
Nikita Titov committed
1177
        return self
wxchan's avatar
wxchan committed
1178

1179
1180
1181
    def _reverse_update_params(self):
        self.params = copy.deepcopy(self.params_back_up)
        self.params_back_up = None
1182
1183
        if self.handle is not None and self.params is not None:
            _safe_call(_LIB.LGBM_DatasetUpdateParam(self.handle, c_str(param_dict_to_str(self.params))))
Nikita Titov's avatar
Nikita Titov committed
1184
        return self
1185

wxchan's avatar
wxchan committed
1186
    def set_field(self, field_name, data):
wxchan's avatar
wxchan committed
1187
        """Set property into the Dataset.
wxchan's avatar
wxchan committed
1188
1189
1190

        Parameters
        ----------
Nikita Titov's avatar
Nikita Titov committed
1191
        field_name : string
1192
            The field name of the information.
1193
        data : list, numpy 1-D array, pandas Series or None
1194
            The array of data to be set.
Nikita Titov's avatar
Nikita Titov committed
1195
1196
1197
1198
1199

        Returns
        -------
        self : Dataset
            Dataset with set property.
wxchan's avatar
wxchan committed
1200
        """
1201
1202
        if self.handle is None:
            raise Exception("Cannot set %s before construct dataset" % field_name)
wxchan's avatar
wxchan committed
1203
        if data is None:
1204
            # set to None
wxchan's avatar
wxchan committed
1205
1206
1207
1208
            _safe_call(_LIB.LGBM_DatasetSetField(
                self.handle,
                c_str(field_name),
                None,
Guolin Ke's avatar
Guolin Ke committed
1209
1210
                ctypes.c_int(0),
                ctypes.c_int(FIELD_TYPE_MAPPER[field_name])))
Nikita Titov's avatar
Nikita Titov committed
1211
            return self
Guolin Ke's avatar
Guolin Ke committed
1212
1213
1214
1215
1216
        dtype = np.float32
        if field_name == 'group':
            dtype = np.int32
        elif field_name == 'init_score':
            dtype = np.float64
1217
        data = list_to_1d_numpy(data, dtype, name=field_name)
1218
1219
        if data.dtype == np.float32 or data.dtype == np.float64:
            ptr_data, type_data, _ = c_float_array(data)
wxchan's avatar
wxchan committed
1220
        elif data.dtype == np.int32:
1221
            ptr_data, type_data, _ = c_int_array(data)
wxchan's avatar
wxchan committed
1222
        else:
Nikita Titov's avatar
Nikita Titov committed
1223
            raise TypeError("Expected np.float32/64 or np.int32, met type({})".format(data.dtype))
wxchan's avatar
wxchan committed
1224
        if type_data != FIELD_TYPE_MAPPER[field_name]:
1225
            raise TypeError("Input type error for set_field")
wxchan's avatar
wxchan committed
1226
1227
1228
1229
        _safe_call(_LIB.LGBM_DatasetSetField(
            self.handle,
            c_str(field_name),
            ptr_data,
Guolin Ke's avatar
Guolin Ke committed
1230
1231
            ctypes.c_int(len(data)),
            ctypes.c_int(type_data)))
Nikita Titov's avatar
Nikita Titov committed
1232
        return self
wxchan's avatar
wxchan committed
1233

wxchan's avatar
wxchan committed
1234
1235
    def get_field(self, field_name):
        """Get property from the Dataset.
wxchan's avatar
wxchan committed
1236
1237
1238

        Parameters
        ----------
Nikita Titov's avatar
Nikita Titov committed
1239
        field_name : string
1240
            The field name of the information.
wxchan's avatar
wxchan committed
1241
1242
1243

        Returns
        -------
1244
1245
        info : numpy array
            A numpy array with information from the Dataset.
Guolin Ke's avatar
Guolin Ke committed
1246
        """
1247
        if self.handle is None:
1248
            raise Exception("Cannot get %s before construct Dataset" % field_name)
Guolin Ke's avatar
Guolin Ke committed
1249
1250
        tmp_out_len = ctypes.c_int()
        out_type = ctypes.c_int()
wxchan's avatar
wxchan committed
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
        ret = ctypes.POINTER(ctypes.c_void_p)()
        _safe_call(_LIB.LGBM_DatasetGetField(
            self.handle,
            c_str(field_name),
            ctypes.byref(tmp_out_len),
            ctypes.byref(ret),
            ctypes.byref(out_type)))
        if out_type.value != FIELD_TYPE_MAPPER[field_name]:
            raise TypeError("Return type error for get_field")
        if tmp_out_len.value == 0:
            return None
        if out_type.value == C_API_DTYPE_INT32:
            return cint32_array_to_numpy(ctypes.cast(ret, ctypes.POINTER(ctypes.c_int32)), tmp_out_len.value)
        elif out_type.value == C_API_DTYPE_FLOAT32:
            return cfloat32_array_to_numpy(ctypes.cast(ret, ctypes.POINTER(ctypes.c_float)), tmp_out_len.value)
Guolin Ke's avatar
Guolin Ke committed
1266
1267
        elif out_type.value == C_API_DTYPE_FLOAT64:
            return cfloat64_array_to_numpy(ctypes.cast(ret, ctypes.POINTER(ctypes.c_double)), tmp_out_len.value)
1268
1269
        elif out_type.value == C_API_DTYPE_INT8:
            return cint8_array_to_numpy(ctypes.cast(ret, ctypes.POINTER(ctypes.c_int8)), tmp_out_len.value)
1270
        else:
wxchan's avatar
wxchan committed
1271
            raise TypeError("Unknown type")
Guolin Ke's avatar
Guolin Ke committed
1272

1273
    def set_categorical_feature(self, categorical_feature):
1274
        """Set categorical features.
1275
1276
1277

        Parameters
        ----------
1278
1279
        categorical_feature : list of int or strings
            Names or indices of categorical features.
Nikita Titov's avatar
Nikita Titov committed
1280
1281
1282
1283
1284

        Returns
        -------
        self : Dataset
            Dataset with set categorical features.
1285
1286
        """
        if self.categorical_feature == categorical_feature:
Nikita Titov's avatar
Nikita Titov committed
1287
            return self
1288
        if self.data is not None:
1289
1290
            if self.categorical_feature is None:
                self.categorical_feature = categorical_feature
Nikita Titov's avatar
Nikita Titov committed
1291
                return self._free_handle()
1292
1293
            elif categorical_feature == 'auto':
                warnings.warn('Using categorical_feature in Dataset.')
Nikita Titov's avatar
Nikita Titov committed
1294
                return self
1295
            else:
1296
1297
                warnings.warn('categorical_feature in Dataset is overridden.\n'
                              'New categorical_feature is {}'.format(sorted(list(categorical_feature))))
1298
                self.categorical_feature = categorical_feature
Nikita Titov's avatar
Nikita Titov committed
1299
                return self._free_handle()
1300
        else:
1301
1302
            raise LightGBMError("Cannot set categorical feature after freed raw data, "
                                "set free_raw_data=False when construct Dataset to avoid this.")
1303

Guolin Ke's avatar
Guolin Ke committed
1304
    def _set_predictor(self, predictor):
1305
1306
1307
1308
        """Set predictor for continued training.

        It is not recommended for user to call this function.
        Please use init_model argument in engine.train() or engine.cv() instead.
Guolin Ke's avatar
Guolin Ke committed
1309
1310
        """
        if predictor is self._predictor:
Nikita Titov's avatar
Nikita Titov committed
1311
            return self
1312
1313
1314
        if self.data is not None or (self.used_indices is not None
                                     and self.reference is not None
                                     and self.reference.data is not None):
Guolin Ke's avatar
Guolin Ke committed
1315
            self._predictor = predictor
Nikita Titov's avatar
Nikita Titov committed
1316
            return self._free_handle()
Guolin Ke's avatar
Guolin Ke committed
1317
        else:
1318
1319
            raise LightGBMError("Cannot set predictor after freed raw data, "
                                "set free_raw_data=False when construct Dataset to avoid this.")
Guolin Ke's avatar
Guolin Ke committed
1320
1321

    def set_reference(self, reference):
1322
        """Set reference Dataset.
Guolin Ke's avatar
Guolin Ke committed
1323
1324
1325
1326

        Parameters
        ----------
        reference : Dataset
1327
            Reference that is used as a template to construct the current Dataset.
Nikita Titov's avatar
Nikita Titov committed
1328
1329
1330
1331
1332

        Returns
        -------
        self : Dataset
            Dataset with set reference.
Guolin Ke's avatar
Guolin Ke committed
1333
        """
1334
1335
1336
        self.set_categorical_feature(reference.categorical_feature) \
            .set_feature_name(reference.feature_name) \
            ._set_predictor(reference._predictor)
1337
1338
        # we're done if self and reference share a common upstrem reference
        if self.get_ref_chain().intersection(reference.get_ref_chain()):
Nikita Titov's avatar
Nikita Titov committed
1339
            return self
Guolin Ke's avatar
Guolin Ke committed
1340
1341
        if self.data is not None:
            self.reference = reference
Nikita Titov's avatar
Nikita Titov committed
1342
            return self._free_handle()
Guolin Ke's avatar
Guolin Ke committed
1343
        else:
1344
1345
            raise LightGBMError("Cannot set reference after freed raw data, "
                                "set free_raw_data=False when construct Dataset to avoid this.")
Guolin Ke's avatar
Guolin Ke committed
1346
1347

    def set_feature_name(self, feature_name):
1348
        """Set feature name.
Guolin Ke's avatar
Guolin Ke committed
1349
1350
1351

        Parameters
        ----------
1352
1353
        feature_name : list of strings
            Feature names.
Nikita Titov's avatar
Nikita Titov committed
1354
1355
1356
1357
1358

        Returns
        -------
        self : Dataset
            Dataset with set feature name.
Guolin Ke's avatar
Guolin Ke committed
1359
        """
1360
1361
        if feature_name != 'auto':
            self.feature_name = feature_name
1362
        if self.handle is not None and feature_name is not None and feature_name != 'auto':
wxchan's avatar
wxchan committed
1363
            if len(feature_name) != self.num_feature():
1364
1365
                raise ValueError("Length of feature_name({}) and num_feature({}) don't match"
                                 .format(len(feature_name), self.num_feature()))
1366
            c_feature_name = [c_str(name) for name in feature_name]
wxchan's avatar
wxchan committed
1367
1368
1369
            _safe_call(_LIB.LGBM_DatasetSetFeatureNames(
                self.handle,
                c_array(ctypes.c_char_p, c_feature_name),
Guolin Ke's avatar
Guolin Ke committed
1370
                ctypes.c_int(len(feature_name))))
Nikita Titov's avatar
Nikita Titov committed
1371
        return self
Guolin Ke's avatar
Guolin Ke committed
1372
1373

    def set_label(self, label):
1374
        """Set label of Dataset.
Guolin Ke's avatar
Guolin Ke committed
1375
1376
1377

        Parameters
        ----------
1378
        label : list, numpy 1-D array, pandas Series / one-column DataFrame or None
1379
            The label information to be set into Dataset.
Nikita Titov's avatar
Nikita Titov committed
1380
1381
1382
1383
1384

        Returns
        -------
        self : Dataset
            Dataset with set label.
Guolin Ke's avatar
Guolin Ke committed
1385
1386
        """
        self.label = label
1387
        if self.handle is not None:
1388
            label = list_to_1d_numpy(_label_from_pandas(label), name='label')
wxchan's avatar
wxchan committed
1389
            self.set_field('label', label)
1390
            self.label = self.get_field('label')  # original values can be modified at cpp side
Nikita Titov's avatar
Nikita Titov committed
1391
        return self
Guolin Ke's avatar
Guolin Ke committed
1392
1393

    def set_weight(self, weight):
1394
        """Set weight of each instance.
Guolin Ke's avatar
Guolin Ke committed
1395
1396
1397

        Parameters
        ----------
1398
        weight : list, numpy 1-D array, pandas Series or None
1399
            Weight to be set for each data point.
Nikita Titov's avatar
Nikita Titov committed
1400
1401
1402
1403
1404

        Returns
        -------
        self : Dataset
            Dataset with set weight.
Guolin Ke's avatar
Guolin Ke committed
1405
        """
1406
1407
        if weight is not None and np.all(weight == 1):
            weight = None
Guolin Ke's avatar
Guolin Ke committed
1408
        self.weight = weight
1409
        if self.handle is not None and weight is not None:
wxchan's avatar
wxchan committed
1410
1411
            weight = list_to_1d_numpy(weight, name='weight')
            self.set_field('weight', weight)
1412
            self.weight = self.get_field('weight')  # original values can be modified at cpp side
Nikita Titov's avatar
Nikita Titov committed
1413
        return self
Guolin Ke's avatar
Guolin Ke committed
1414
1415

    def set_init_score(self, init_score):
1416
        """Set init score of Booster to start from.
Guolin Ke's avatar
Guolin Ke committed
1417
1418
1419

        Parameters
        ----------
1420
        init_score : list, numpy 1-D array, pandas Series or None
1421
            Init score for Booster.
Nikita Titov's avatar
Nikita Titov committed
1422
1423
1424
1425
1426

        Returns
        -------
        self : Dataset
            Dataset with set init score.
Guolin Ke's avatar
Guolin Ke committed
1427
1428
        """
        self.init_score = init_score
1429
        if self.handle is not None and init_score is not None:
Guolin Ke's avatar
Guolin Ke committed
1430
            init_score = list_to_1d_numpy(init_score, np.float64, name='init_score')
wxchan's avatar
wxchan committed
1431
            self.set_field('init_score', init_score)
1432
            self.init_score = self.get_field('init_score')  # original values can be modified at cpp side
Nikita Titov's avatar
Nikita Titov committed
1433
        return self
Guolin Ke's avatar
Guolin Ke committed
1434
1435

    def set_group(self, group):
1436
        """Set group size of Dataset (used for ranking).
Guolin Ke's avatar
Guolin Ke committed
1437
1438
1439

        Parameters
        ----------
1440
        group : list, numpy 1-D array, pandas Series or None
1441
            Group size of each group.
Nikita Titov's avatar
Nikita Titov committed
1442
1443
1444
1445
1446

        Returns
        -------
        self : Dataset
            Dataset with set group.
Guolin Ke's avatar
Guolin Ke committed
1447
1448
        """
        self.group = group
1449
        if self.handle is not None and group is not None:
wxchan's avatar
wxchan committed
1450
1451
            group = list_to_1d_numpy(group, np.int32, name='group')
            self.set_field('group', group)
Nikita Titov's avatar
Nikita Titov committed
1452
        return self
Guolin Ke's avatar
Guolin Ke committed
1453
1454

    def get_label(self):
1455
        """Get the label of the Dataset.
Guolin Ke's avatar
Guolin Ke committed
1456
1457
1458

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
1459
        label : numpy array or None
1460
            The label information from the Dataset.
Guolin Ke's avatar
Guolin Ke committed
1461
        """
1462
        if self.label is None:
wxchan's avatar
wxchan committed
1463
            self.label = self.get_field('label')
Guolin Ke's avatar
Guolin Ke committed
1464
1465
1466
        return self.label

    def get_weight(self):
1467
        """Get the weight of the Dataset.
Guolin Ke's avatar
Guolin Ke committed
1468
1469
1470

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
1471
        weight : numpy array or None
1472
            Weight for each data point from the Dataset.
Guolin Ke's avatar
Guolin Ke committed
1473
        """
1474
        if self.weight is None:
wxchan's avatar
wxchan committed
1475
            self.weight = self.get_field('weight')
Guolin Ke's avatar
Guolin Ke committed
1476
1477
        return self.weight

1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
    def get_feature_penalty(self):
        """Get the feature penalty of the Dataset.

        Returns
        -------
        feature_penalty : numpy array or None
            Feature penalty for each feature in the Dataset.
        """
        if self.feature_penalty is None:
            self.feature_penalty = self.get_field('feature_penalty')
        return self.feature_penalty

    def get_monotone_constraints(self):
        """Get the monotone constraints of the Dataset.

        Returns
        -------
        monotone_constraints : numpy array or None
            Monotone constraints: -1, 0 or 1, for each feature in the Dataset.
        """
        if self.monotone_constraints is None:
            self.monotone_constraints = self.get_field('monotone_constraints')
        return self.monotone_constraints

Guolin Ke's avatar
Guolin Ke committed
1502
    def get_init_score(self):
1503
        """Get the initial score of the Dataset.
Guolin Ke's avatar
Guolin Ke committed
1504
1505
1506

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
1507
        init_score : numpy array or None
1508
            Init score of Booster.
Guolin Ke's avatar
Guolin Ke committed
1509
        """
1510
        if self.init_score is None:
wxchan's avatar
wxchan committed
1511
            self.init_score = self.get_field('init_score')
Guolin Ke's avatar
Guolin Ke committed
1512
1513
        return self.init_score

1514
1515
1516
1517
1518
    def get_data(self):
        """Get the raw data of the Dataset.

        Returns
        -------
1519
        data : string, numpy array, pandas DataFrame, H2O DataTable's Frame, scipy.sparse, list of numpy arrays or None
1520
1521
1522
1523
            Raw data used in the Dataset construction.
        """
        if self.handle is None:
            raise Exception("Cannot get data before construct Dataset")
Guolin Ke's avatar
Guolin Ke committed
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
        if self.need_slice and self.used_indices is not None and self.reference is not None:
            self.data = self.reference.data
            if self.data is not None:
                if isinstance(self.data, np.ndarray) or scipy.sparse.issparse(self.data):
                    self.data = self.data[self.used_indices, :]
                elif isinstance(self.data, DataFrame):
                    self.data = self.data.iloc[self.used_indices].copy()
                elif isinstance(self.data, DataTable):
                    self.data = self.data[self.used_indices, :]
                else:
                    warnings.warn("Cannot subset {} type of raw data.\n"
                                  "Returning original raw data".format(type(self.data).__name__))
1536
            self.need_slice = False
Guolin Ke's avatar
Guolin Ke committed
1537
1538
1539
        if self.data is None:
            raise LightGBMError("Cannot call `get_data` after freed raw data, "
                                "set free_raw_data=False when construct Dataset to avoid this.")
1540
1541
        return self.data

Guolin Ke's avatar
Guolin Ke committed
1542
    def get_group(self):
1543
        """Get the group of the Dataset.
Guolin Ke's avatar
Guolin Ke committed
1544
1545
1546

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
1547
        group : numpy array or None
1548
            Group size of each group.
Guolin Ke's avatar
Guolin Ke committed
1549
        """
1550
        if self.group is None:
wxchan's avatar
wxchan committed
1551
            self.group = self.get_field('group')
Guolin Ke's avatar
Guolin Ke committed
1552
1553
            if self.group is not None:
                # group data from LightGBM is boundaries data, need to convert to group size
Nikita Titov's avatar
Nikita Titov committed
1554
                self.group = np.diff(self.group)
Guolin Ke's avatar
Guolin Ke committed
1555
1556
1557
        return self.group

    def num_data(self):
1558
        """Get the number of rows in the Dataset.
Guolin Ke's avatar
Guolin Ke committed
1559
1560
1561

        Returns
        -------
1562
1563
        number_of_rows : int
            The number of rows in the Dataset.
Guolin Ke's avatar
Guolin Ke committed
1564
        """
1565
        if self.handle is not None:
Guolin Ke's avatar
Guolin Ke committed
1566
            ret = ctypes.c_int()
wxchan's avatar
wxchan committed
1567
1568
1569
            _safe_call(_LIB.LGBM_DatasetGetNumData(self.handle,
                                                   ctypes.byref(ret)))
            return ret.value
Guolin Ke's avatar
Guolin Ke committed
1570
        else:
1571
            raise LightGBMError("Cannot get num_data before construct dataset")
Guolin Ke's avatar
Guolin Ke committed
1572
1573

    def num_feature(self):
1574
        """Get the number of columns (features) in the Dataset.
Guolin Ke's avatar
Guolin Ke committed
1575
1576
1577

        Returns
        -------
1578
1579
        number_of_columns : int
            The number of columns (features) in the Dataset.
Guolin Ke's avatar
Guolin Ke committed
1580
        """
1581
        if self.handle is not None:
Guolin Ke's avatar
Guolin Ke committed
1582
            ret = ctypes.c_int()
wxchan's avatar
wxchan committed
1583
1584
1585
            _safe_call(_LIB.LGBM_DatasetGetNumFeature(self.handle,
                                                      ctypes.byref(ret)))
            return ret.value
Guolin Ke's avatar
Guolin Ke committed
1586
        else:
1587
            raise LightGBMError("Cannot get num_feature before construct dataset")
Guolin Ke's avatar
Guolin Ke committed
1588

1589
    def get_ref_chain(self, ref_limit=100):
1590
1591
1592
1593
1594
        """Get a chain of Dataset objects.

        Starts with r, then goes to r.reference (if exists),
        then to r.reference.reference, etc.
        until we hit ``ref_limit`` or a reference loop.
1595
1596
1597
1598
1599

        Parameters
        ----------
        ref_limit : int, optional (default=100)
            The limit number of references.
1600
1601
1602

        Returns
        -------
1603
1604
1605
        ref_chain : set of Dataset
            Chain of references of the Datasets.
        """
1606
        head = self
1607
        ref_chain = set()
1608
1609
        while len(ref_chain) < ref_limit:
            if isinstance(head, Dataset):
1610
                ref_chain.add(head)
1611
1612
1613
1614
1615
1616
                if (head.reference is not None) and (head.reference not in ref_chain):
                    head = head.reference
                else:
                    break
            else:
                break
Nikita Titov's avatar
Nikita Titov committed
1617
        return ref_chain
1618

1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
    def add_features_from(self, other):
        """Add features from other Dataset to the current Dataset.

        Both Datasets must be constructed before calling this method.

        Parameters
        ----------
        other : Dataset
            The Dataset to take features from.

        Returns
        -------
        self : Dataset
            Dataset with the new features added.
        """
        if self.handle is None or other.handle is None:
            raise ValueError('Both source and target Datasets must be constructed before adding features')
        _safe_call(_LIB.LGBM_DatasetAddFeaturesFrom(self.handle, other.handle))
        return self

1639
    def _dump_text(self, filename):
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
        """Save Dataset to a text file.

        This format cannot be loaded back in by LightGBM, but is useful for debugging purposes.

        Parameters
        ----------
        filename : string
            Name of the output file.

        Returns
        -------
        self : Dataset
            Returns self.
        """
        _safe_call(_LIB.LGBM_DatasetDumpText(
            self.construct().handle,
            c_str(filename)))
        return self

wxchan's avatar
wxchan committed
1659

wxchan's avatar
wxchan committed
1660
class Booster(object):
1661
    """Booster in LightGBM."""
1662

1663
    def __init__(self, params=None, train_set=None, model_file=None, model_str=None, silent=False):
1664
        """Initialize the Booster.
wxchan's avatar
wxchan committed
1665
1666
1667

        Parameters
        ----------
Nikita Titov's avatar
Nikita Titov committed
1668
        params : dict or None, optional (default=None)
1669
1670
1671
1672
            Parameters for Booster.
        train_set : Dataset or None, optional (default=None)
            Training dataset.
        model_file : string or None, optional (default=None)
wxchan's avatar
wxchan committed
1673
            Path to the model file.
1674
1675
        model_str : string or None, optional (default=None)
            Model will be loaded from this string.
1676
1677
        silent : bool, optional (default=False)
            Whether to print messages during construction.
wxchan's avatar
wxchan committed
1678
        """
1679
        self.handle = None
1680
        self.network = False
wxchan's avatar
wxchan committed
1681
        self.__need_reload_eval_info = True
1682
        self._train_data_name = "training"
wxchan's avatar
wxchan committed
1683
        self.__attr = {}
1684
        self.__set_objective_to_none = False
wxchan's avatar
wxchan committed
1685
        self.best_iteration = -1
wxchan's avatar
wxchan committed
1686
        self.best_score = {}
1687
        params = {} if params is None else copy.deepcopy(params)
1688
        # user can set verbose with params, it has higher priority
1689
        if not any(verbose_alias in params for verbose_alias in _ConfigAliases.get("verbosity")) and silent:
1690
            params["verbose"] = -1
wxchan's avatar
wxchan committed
1691
        if train_set is not None:
1692
            # Training task
wxchan's avatar
wxchan committed
1693
            if not isinstance(train_set, Dataset):
1694
1695
                raise TypeError('Training data should be Dataset instance, met {}'
                                .format(type(train_set).__name__))
wxchan's avatar
wxchan committed
1696
            params_str = param_dict_to_str(params)
1697
            # set network if necessary
1698
            for alias in _ConfigAliases.get("machines"):
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
                if alias in params:
                    machines = params[alias]
                    if isinstance(machines, string_type):
                        num_machines = len(machines.split(','))
                    elif isinstance(machines, (list, set)):
                        num_machines = len(machines)
                        machines = ','.join(machines)
                    else:
                        raise ValueError("Invalid machines in params.")
                    self.set_network(machines,
                                     local_listen_port=params.get("local_listen_port", 12400),
                                     listen_time_out=params.get("listen_time_out", 120),
                                     num_machines=params.get("num_machines", num_machines))
                    break
1713
            # construct booster object
1714
            self.handle = ctypes.c_void_p()
wxchan's avatar
wxchan committed
1715
            _safe_call(_LIB.LGBM_BoosterCreate(
wxchan's avatar
wxchan committed
1716
                train_set.construct().handle,
wxchan's avatar
wxchan committed
1717
1718
                c_str(params_str),
                ctypes.byref(self.handle)))
1719
            # save reference to data
wxchan's avatar
wxchan committed
1720
1721
1722
1723
            self.train_set = train_set
            self.valid_sets = []
            self.name_valid_sets = []
            self.__num_dataset = 1
Guolin Ke's avatar
Guolin Ke committed
1724
1725
            self.__init_predictor = train_set._predictor
            if self.__init_predictor is not None:
wxchan's avatar
wxchan committed
1726
1727
                _safe_call(_LIB.LGBM_BoosterMerge(
                    self.handle,
Guolin Ke's avatar
Guolin Ke committed
1728
                    self.__init_predictor.handle))
Guolin Ke's avatar
Guolin Ke committed
1729
            out_num_class = ctypes.c_int(0)
wxchan's avatar
wxchan committed
1730
1731
1732
1733
            _safe_call(_LIB.LGBM_BoosterGetNumClasses(
                self.handle,
                ctypes.byref(out_num_class)))
            self.__num_class = out_num_class.value
1734
            # buffer for inner predict
wxchan's avatar
wxchan committed
1735
1736
1737
            self.__inner_predict_buffer = [None]
            self.__is_predicted_cur_iter = [False]
            self.__get_eval_info()
1738
            self.pandas_categorical = train_set.pandas_categorical
wxchan's avatar
wxchan committed
1739
        elif model_file is not None:
1740
            # Prediction task
Guolin Ke's avatar
Guolin Ke committed
1741
            out_num_iterations = ctypes.c_int(0)
1742
            self.handle = ctypes.c_void_p()
wxchan's avatar
wxchan committed
1743
1744
1745
1746
            _safe_call(_LIB.LGBM_BoosterCreateFromModelfile(
                c_str(model_file),
                ctypes.byref(out_num_iterations),
                ctypes.byref(self.handle)))
Guolin Ke's avatar
Guolin Ke committed
1747
            out_num_class = ctypes.c_int(0)
wxchan's avatar
wxchan committed
1748
1749
1750
1751
            _safe_call(_LIB.LGBM_BoosterGetNumClasses(
                self.handle,
                ctypes.byref(out_num_class)))
            self.__num_class = out_num_class.value
1752
            self.pandas_categorical = _load_pandas_categorical(file_name=model_file)
1753
1754
        elif model_str is not None:
            self.model_from_string(model_str, not silent)
wxchan's avatar
wxchan committed
1755
        else:
1756
1757
            raise TypeError('Need at least one training dataset or model file or model string '
                            'to create Booster instance')
1758
        self.params = params
wxchan's avatar
wxchan committed
1759
1760

    def __del__(self):
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
        try:
            if self.network:
                self.free_network()
        except AttributeError:
            pass
        try:
            if self.handle is not None:
                _safe_call(_LIB.LGBM_BoosterFree(self.handle))
        except AttributeError:
            pass
wxchan's avatar
wxchan committed
1771

wxchan's avatar
wxchan committed
1772
1773
1774
1775
    def __copy__(self):
        return self.__deepcopy__(None)

    def __deepcopy__(self, _):
1776
        model_str = self.model_to_string(num_iteration=-1)
1777
        booster = Booster(model_str=model_str)
1778
        return booster
wxchan's avatar
wxchan committed
1779
1780
1781
1782
1783
1784
1785

    def __getstate__(self):
        this = self.__dict__.copy()
        handle = this['handle']
        this.pop('train_set', None)
        this.pop('valid_sets', None)
        if handle is not None:
1786
            this["handle"] = self.model_to_string(num_iteration=-1)
wxchan's avatar
wxchan committed
1787
1788
1789
        return this

    def __setstate__(self, state):
1790
1791
        model_str = state.get('handle', None)
        if model_str is not None:
wxchan's avatar
wxchan committed
1792
            handle = ctypes.c_void_p()
Guolin Ke's avatar
Guolin Ke committed
1793
            out_num_iterations = ctypes.c_int(0)
1794
1795
1796
1797
            _safe_call(_LIB.LGBM_BoosterLoadModelFromString(
                c_str(model_str),
                ctypes.byref(out_num_iterations),
                ctypes.byref(handle)))
wxchan's avatar
wxchan committed
1798
1799
1800
            state['handle'] = handle
        self.__dict__.update(state)

wxchan's avatar
wxchan committed
1801
    def free_dataset(self):
Nikita Titov's avatar
Nikita Titov committed
1802
1803
1804
1805
1806
1807
1808
        """Free Booster's Datasets.

        Returns
        -------
        self : Booster
            Booster without Datasets.
        """
wxchan's avatar
wxchan committed
1809
1810
        self.__dict__.pop('train_set', None)
        self.__dict__.pop('valid_sets', None)
1811
        self.__num_dataset = 0
Nikita Titov's avatar
Nikita Titov committed
1812
        return self
wxchan's avatar
wxchan committed
1813

1814
1815
1816
    def _free_buffer(self):
        self.__inner_predict_buffer = []
        self.__is_predicted_cur_iter = []
Nikita Titov's avatar
Nikita Titov committed
1817
        return self
1818

1819
1820
1821
1822
1823
1824
    def set_network(self, machines, local_listen_port=12400,
                    listen_time_out=120, num_machines=1):
        """Set the network configuration.

        Parameters
        ----------
Nikita Titov's avatar
Nikita Titov committed
1825
        machines : list, set or string
1826
            Names of machines.
Nikita Titov's avatar
Nikita Titov committed
1827
        local_listen_port : int, optional (default=12400)
1828
            TCP listen port for local machines.
Nikita Titov's avatar
Nikita Titov committed
1829
        listen_time_out : int, optional (default=120)
1830
            Socket time-out in minutes.
Nikita Titov's avatar
Nikita Titov committed
1831
        num_machines : int, optional (default=1)
1832
            The number of machines for parallel learning application.
Nikita Titov's avatar
Nikita Titov committed
1833
1834
1835
1836
1837

        Returns
        -------
        self : Booster
            Booster with set network.
1838
1839
1840
1841
1842
1843
        """
        _safe_call(_LIB.LGBM_NetworkInit(c_str(machines),
                                         ctypes.c_int(local_listen_port),
                                         ctypes.c_int(listen_time_out),
                                         ctypes.c_int(num_machines)))
        self.network = True
Nikita Titov's avatar
Nikita Titov committed
1844
        return self
1845
1846

    def free_network(self):
Nikita Titov's avatar
Nikita Titov committed
1847
1848
1849
1850
1851
1852
1853
        """Free Booster's network.

        Returns
        -------
        self : Booster
            Booster with freed network.
        """
1854
1855
        _safe_call(_LIB.LGBM_NetworkFree())
        self.network = False
Nikita Titov's avatar
Nikita Titov committed
1856
        return self
1857

wxchan's avatar
wxchan committed
1858
    def set_train_data_name(self, name):
1859
1860
1861
1862
        """Set the name to the training Dataset.

        Parameters
        ----------
Nikita Titov's avatar
Nikita Titov committed
1863
1864
1865
1866
1867
1868
1869
        name : string
            Name for the training Dataset.

        Returns
        -------
        self : Booster
            Booster with set training Dataset name.
1870
        """
1871
        self._train_data_name = name
Nikita Titov's avatar
Nikita Titov committed
1872
        return self
wxchan's avatar
wxchan committed
1873
1874

    def add_valid(self, data, name):
1875
        """Add validation data.
wxchan's avatar
wxchan committed
1876
1877
1878
1879

        Parameters
        ----------
        data : Dataset
1880
1881
1882
            Validation data.
        name : string
            Name of validation data.
Nikita Titov's avatar
Nikita Titov committed
1883
1884
1885
1886
1887

        Returns
        -------
        self : Booster
            Booster with set validation data.
wxchan's avatar
wxchan committed
1888
        """
Guolin Ke's avatar
Guolin Ke committed
1889
        if not isinstance(data, Dataset):
1890
1891
            raise TypeError('Validation data should be Dataset instance, met {}'
                            .format(type(data).__name__))
Guolin Ke's avatar
Guolin Ke committed
1892
        if data._predictor is not self.__init_predictor:
1893
1894
            raise LightGBMError("Add validation data failed, "
                                "you should use same predictor for these data")
wxchan's avatar
wxchan committed
1895
1896
        _safe_call(_LIB.LGBM_BoosterAddValidData(
            self.handle,
wxchan's avatar
wxchan committed
1897
            data.construct().handle))
wxchan's avatar
wxchan committed
1898
1899
1900
1901
1902
        self.valid_sets.append(data)
        self.name_valid_sets.append(name)
        self.__num_dataset += 1
        self.__inner_predict_buffer.append(None)
        self.__is_predicted_cur_iter.append(False)
Nikita Titov's avatar
Nikita Titov committed
1903
        return self
wxchan's avatar
wxchan committed
1904
1905

    def reset_parameter(self, params):
1906
        """Reset parameters of Booster.
wxchan's avatar
wxchan committed
1907
1908
1909
1910

        Parameters
        ----------
        params : dict
1911
            New parameters for Booster.
Nikita Titov's avatar
Nikita Titov committed
1912
1913
1914
1915
1916

        Returns
        -------
        self : Booster
            Booster with new parameters.
wxchan's avatar
wxchan committed
1917
        """
1918
        if any(metric_alias in params for metric_alias in _ConfigAliases.get("metric")):
wxchan's avatar
wxchan committed
1919
1920
1921
1922
1923
1924
            self.__need_reload_eval_info = True
        params_str = param_dict_to_str(params)
        if params_str:
            _safe_call(_LIB.LGBM_BoosterResetParameter(
                self.handle,
                c_str(params_str)))
Guolin Ke's avatar
Guolin Ke committed
1925
        self.params.update(params)
Nikita Titov's avatar
Nikita Titov committed
1926
        return self
wxchan's avatar
wxchan committed
1927
1928

    def update(self, train_set=None, fobj=None):
Nikita Titov's avatar
Nikita Titov committed
1929
        """Update Booster for one iteration.
1930

wxchan's avatar
wxchan committed
1931
1932
        Parameters
        ----------
1933
1934
1935
1936
        train_set : Dataset or None, optional (default=None)
            Training data.
            If None, last training data is used.
        fobj : callable or None, optional (default=None)
wxchan's avatar
wxchan committed
1937
            Customized objective function.
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
            Should accept two parameters: preds, train_data,
            and return (grad, hess).

                preds : list or numpy 1-D array
                    The predicted values.
                train_data : Dataset
                    The training dataset.
                grad : list or numpy 1-D array
                    The value of the first order derivative (gradient) for each sample point.
                hess : list or numpy 1-D array
                    The value of the second order derivative (Hessian) for each sample point.
wxchan's avatar
wxchan committed
1949

1950
1951
            For multi-class task, the preds is group by class_id first, then group by row_id.
            If you want to get i-th row preds in j-th class, the access way is score[j * num_data + i]
1952
1953
            and you should group grad and hess in this way as well.

wxchan's avatar
wxchan committed
1954
1955
        Returns
        -------
1956
1957
        is_finished : bool
            Whether the update was successfully finished.
wxchan's avatar
wxchan committed
1958
        """
1959
        # need reset training data
wxchan's avatar
wxchan committed
1960
        if train_set is not None and train_set is not self.train_set:
Guolin Ke's avatar
Guolin Ke committed
1961
            if not isinstance(train_set, Dataset):
1962
1963
                raise TypeError('Training data should be Dataset instance, met {}'
                                .format(type(train_set).__name__))
Guolin Ke's avatar
Guolin Ke committed
1964
            if train_set._predictor is not self.__init_predictor:
1965
1966
                raise LightGBMError("Replace training data failed, "
                                    "you should use same predictor for these data")
wxchan's avatar
wxchan committed
1967
1968
1969
            self.train_set = train_set
            _safe_call(_LIB.LGBM_BoosterResetTrainingData(
                self.handle,
wxchan's avatar
wxchan committed
1970
                self.train_set.construct().handle))
wxchan's avatar
wxchan committed
1971
1972
1973
            self.__inner_predict_buffer[0] = None
        is_finished = ctypes.c_int(0)
        if fobj is None:
1974
            if self.__set_objective_to_none:
1975
                raise LightGBMError('Cannot update due to null objective function.')
wxchan's avatar
wxchan committed
1976
1977
1978
            _safe_call(_LIB.LGBM_BoosterUpdateOneIter(
                self.handle,
                ctypes.byref(is_finished)))
wxchan's avatar
wxchan committed
1979
            self.__is_predicted_cur_iter = [False for _ in range_(self.__num_dataset)]
wxchan's avatar
wxchan committed
1980
1981
            return is_finished.value == 1
        else:
1982
            if not self.__set_objective_to_none:
Nikita Titov's avatar
Nikita Titov committed
1983
                self.reset_parameter({"objective": "none"}).__set_objective_to_none = True
wxchan's avatar
wxchan committed
1984
1985
1986
1987
            grad, hess = fobj(self.__inner_predict(0), self.train_set)
            return self.__boost(grad, hess)

    def __boost(self, grad, hess):
1988
        """Boost Booster for one iteration with customized gradient statistics.
Nikita Titov's avatar
Nikita Titov committed
1989

Nikita Titov's avatar
Nikita Titov committed
1990
1991
1992
1993
1994
        .. note::

            For multi-class task, the score is group by class_id first, then group by row_id.
            If you want to get i-th row score in j-th class, the access way is score[j * num_data + i]
            and you should group grad and hess in this way as well.
1995

wxchan's avatar
wxchan committed
1996
1997
        Parameters
        ----------
1998
        grad : list or numpy 1-D array
Nikita Titov's avatar
Nikita Titov committed
1999
            The first order derivative (gradient).
2000
        hess : list or numpy 1-D array
Nikita Titov's avatar
Nikita Titov committed
2001
            The second order derivative (Hessian).
wxchan's avatar
wxchan committed
2002
2003
2004

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
2005
2006
        is_finished : bool
            Whether the boost was successfully finished.
wxchan's avatar
wxchan committed
2007
        """
2008
2009
        grad = list_to_1d_numpy(grad, name='gradient')
        hess = list_to_1d_numpy(hess, name='hessian')
2010
2011
        assert grad.flags.c_contiguous
        assert hess.flags.c_contiguous
wxchan's avatar
wxchan committed
2012
        if len(grad) != len(hess):
2013
2014
            raise ValueError("Lengths of gradient({}) and hessian({}) don't match"
                             .format(len(grad), len(hess)))
wxchan's avatar
wxchan committed
2015
2016
2017
2018
2019
2020
        is_finished = ctypes.c_int(0)
        _safe_call(_LIB.LGBM_BoosterUpdateOneIterCustom(
            self.handle,
            grad.ctypes.data_as(ctypes.POINTER(ctypes.c_float)),
            hess.ctypes.data_as(ctypes.POINTER(ctypes.c_float)),
            ctypes.byref(is_finished)))
wxchan's avatar
wxchan committed
2021
        self.__is_predicted_cur_iter = [False for _ in range_(self.__num_dataset)]
wxchan's avatar
wxchan committed
2022
2023
2024
        return is_finished.value == 1

    def rollback_one_iter(self):
Nikita Titov's avatar
Nikita Titov committed
2025
2026
2027
2028
2029
2030
2031
        """Rollback one iteration.

        Returns
        -------
        self : Booster
            Booster with rolled back one iteration.
        """
wxchan's avatar
wxchan committed
2032
2033
        _safe_call(_LIB.LGBM_BoosterRollbackOneIter(
            self.handle))
wxchan's avatar
wxchan committed
2034
        self.__is_predicted_cur_iter = [False for _ in range_(self.__num_dataset)]
Nikita Titov's avatar
Nikita Titov committed
2035
        return self
wxchan's avatar
wxchan committed
2036
2037

    def current_iteration(self):
2038
2039
2040
2041
2042
2043
2044
        """Get the index of the current iteration.

        Returns
        -------
        cur_iter : int
            The index of the current iteration.
        """
Guolin Ke's avatar
Guolin Ke committed
2045
        out_cur_iter = ctypes.c_int(0)
wxchan's avatar
wxchan committed
2046
2047
2048
2049
2050
        _safe_call(_LIB.LGBM_BoosterGetCurrentIteration(
            self.handle,
            ctypes.byref(out_cur_iter)))
        return out_cur_iter.value

2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
    def num_model_per_iteration(self):
        """Get number of models per iteration.

        Returns
        -------
        model_per_iter : int
            The number of models per iteration.
        """
        model_per_iter = ctypes.c_int(0)
        _safe_call(_LIB.LGBM_BoosterNumModelPerIteration(
            self.handle,
            ctypes.byref(model_per_iter)))
        return model_per_iter.value

    def num_trees(self):
        """Get number of weak sub-models.

        Returns
        -------
        num_trees : int
            The number of weak sub-models.
        """
        num_trees = ctypes.c_int(0)
        _safe_call(_LIB.LGBM_BoosterNumberOfTotalModel(
            self.handle,
            ctypes.byref(num_trees)))
        return num_trees.value

wxchan's avatar
wxchan committed
2079
    def eval(self, data, name, feval=None):
2080
        """Evaluate for data.
wxchan's avatar
wxchan committed
2081
2082
2083

        Parameters
        ----------
2084
2085
2086
2087
2088
        data : Dataset
            Data for the evaluating.
        name : string
            Name of the data.
        feval : callable or None, optional (default=None)
2089
            Customized evaluation function.
2090
            Should accept two parameters: preds, eval_data,
2091
            and return (eval_name, eval_result, is_higher_better) or list of such tuples.
2092
2093
2094
2095
2096
2097

                preds : list or numpy 1-D array
                    The predicted values.
                eval_data : Dataset
                    The evaluation dataset.
                eval_name : string
2098
                    The name of evaluation function (without whitespaces).
2099
2100
2101
2102
2103
                eval_result : float
                    The eval result.
                is_higher_better : bool
                    Is eval result higher better, e.g. AUC is ``is_higher_better``.

2104
2105
            For multi-class task, the preds is group by class_id first, then group by row_id.
            If you want to get i-th row preds in j-th class, the access way is preds[j * num_data + i].
2106

wxchan's avatar
wxchan committed
2107
2108
        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
2109
        result : list
2110
            List with evaluation results.
wxchan's avatar
wxchan committed
2111
        """
Guolin Ke's avatar
Guolin Ke committed
2112
2113
        if not isinstance(data, Dataset):
            raise TypeError("Can only eval for Dataset instance")
wxchan's avatar
wxchan committed
2114
2115
2116
2117
        data_idx = -1
        if data is self.train_set:
            data_idx = 0
        else:
wxchan's avatar
wxchan committed
2118
            for i in range_(len(self.valid_sets)):
wxchan's avatar
wxchan committed
2119
2120
2121
                if data is self.valid_sets[i]:
                    data_idx = i + 1
                    break
2122
        # need to push new valid data
wxchan's avatar
wxchan committed
2123
2124
2125
2126
2127
2128
2129
        if data_idx == -1:
            self.add_valid(data, name)
            data_idx = self.__num_dataset - 1

        return self.__inner_eval(name, data_idx, feval)

    def eval_train(self, feval=None):
2130
        """Evaluate for training data.
wxchan's avatar
wxchan committed
2131
2132
2133

        Parameters
        ----------
2134
        feval : callable or None, optional (default=None)
2135
            Customized evaluation function.
2136
2137
            Should accept two parameters: preds, train_data,
            and return (eval_name, eval_result, is_higher_better) or list of such tuples.
2138
2139
2140
2141
2142
2143

                preds : list or numpy 1-D array
                    The predicted values.
                train_data : Dataset
                    The training dataset.
                eval_name : string
2144
                    The name of evaluation function (without whitespaces).
2145
2146
2147
2148
2149
                eval_result : float
                    The eval result.
                is_higher_better : bool
                    Is eval result higher better, e.g. AUC is ``is_higher_better``.

2150
2151
            For multi-class task, the preds is group by class_id first, then group by row_id.
            If you want to get i-th row preds in j-th class, the access way is preds[j * num_data + i].
wxchan's avatar
wxchan committed
2152
2153
2154

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
2155
        result : list
2156
            List with evaluation results.
wxchan's avatar
wxchan committed
2157
        """
2158
        return self.__inner_eval(self._train_data_name, 0, feval)
wxchan's avatar
wxchan committed
2159
2160

    def eval_valid(self, feval=None):
2161
        """Evaluate for validation data.
wxchan's avatar
wxchan committed
2162
2163
2164

        Parameters
        ----------
2165
        feval : callable or None, optional (default=None)
2166
            Customized evaluation function.
2167
            Should accept two parameters: preds, valid_data,
2168
            and return (eval_name, eval_result, is_higher_better) or list of such tuples.
2169
2170
2171
2172
2173
2174

                preds : list or numpy 1-D array
                    The predicted values.
                valid_data : Dataset
                    The validation dataset.
                eval_name : string
2175
                    The name of evaluation function (without whitespaces).
2176
2177
2178
2179
2180
                eval_result : float
                    The eval result.
                is_higher_better : bool
                    Is eval result higher better, e.g. AUC is ``is_higher_better``.

2181
2182
            For multi-class task, the preds is group by class_id first, then group by row_id.
            If you want to get i-th row preds in j-th class, the access way is preds[j * num_data + i].
wxchan's avatar
wxchan committed
2183
2184
2185

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
2186
        result : list
2187
            List with evaluation results.
wxchan's avatar
wxchan committed
2188
        """
wxchan's avatar
wxchan committed
2189
        return [item for i in range_(1, self.__num_dataset)
wxchan's avatar
wxchan committed
2190
                for item in self.__inner_eval(self.name_valid_sets[i - 1], i, feval)]
wxchan's avatar
wxchan committed
2191

2192
    def save_model(self, filename, num_iteration=None, start_iteration=0):
2193
        """Save Booster to file.
wxchan's avatar
wxchan committed
2194
2195
2196

        Parameters
        ----------
2197
2198
        filename : string
            Filename to save Booster.
2199
2200
2201
2202
        num_iteration : int or None, optional (default=None)
            Index of the iteration that should be saved.
            If None, if the best iteration exists, it is saved; otherwise, all iterations are saved.
            If <= 0, all iterations are saved.
Nikita Titov's avatar
Nikita Titov committed
2203
        start_iteration : int, optional (default=0)
2204
            Start index of the iteration that should be saved.
Nikita Titov's avatar
Nikita Titov committed
2205
2206
2207
2208
2209

        Returns
        -------
        self : Booster
            Returns self.
wxchan's avatar
wxchan committed
2210
        """
2211
        if num_iteration is None:
2212
            num_iteration = self.best_iteration
wxchan's avatar
wxchan committed
2213
2214
        _safe_call(_LIB.LGBM_BoosterSaveModel(
            self.handle,
2215
            ctypes.c_int(start_iteration),
Guolin Ke's avatar
Guolin Ke committed
2216
            ctypes.c_int(num_iteration),
wxchan's avatar
wxchan committed
2217
            c_str(filename)))
2218
        _dump_pandas_categorical(self.pandas_categorical, filename)
Nikita Titov's avatar
Nikita Titov committed
2219
        return self
wxchan's avatar
wxchan committed
2220

2221
    def shuffle_models(self, start_iteration=0, end_iteration=-1):
2222
        """Shuffle models.
Nikita Titov's avatar
Nikita Titov committed
2223

2224
2225
2226
        Parameters
        ----------
        start_iteration : int, optional (default=0)
2227
            The first iteration that will be shuffled.
2228
2229
        end_iteration : int, optional (default=-1)
            The last iteration that will be shuffled.
2230
            If <= 0, means the last available iteration.
2231

Nikita Titov's avatar
Nikita Titov committed
2232
2233
2234
2235
        Returns
        -------
        self : Booster
            Booster with shuffled models.
2236
        """
2237
2238
        _safe_call(_LIB.LGBM_BoosterShuffleModels(
            self.handle,
Guolin Ke's avatar
Guolin Ke committed
2239
2240
            ctypes.c_int(start_iteration),
            ctypes.c_int(end_iteration)))
Nikita Titov's avatar
Nikita Titov committed
2241
        return self
2242
2243
2244
2245
2246
2247

    def model_from_string(self, model_str, verbose=True):
        """Load Booster from a string.

        Parameters
        ----------
Nikita Titov's avatar
Nikita Titov committed
2248
        model_str : string
2249
            Model will be loaded from this string.
Nikita Titov's avatar
Nikita Titov committed
2250
2251
        verbose : bool, optional (default=True)
            Whether to print messages while loading model.
2252
2253
2254

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
2255
        self : Booster
2256
2257
            Loaded Booster object.
        """
2258
2259
2260
2261
        if self.handle is not None:
            _safe_call(_LIB.LGBM_BoosterFree(self.handle))
        self._free_buffer()
        self.handle = ctypes.c_void_p()
2262
2263
2264
2265
2266
2267
2268
2269
2270
        out_num_iterations = ctypes.c_int(0)
        _safe_call(_LIB.LGBM_BoosterLoadModelFromString(
            c_str(model_str),
            ctypes.byref(out_num_iterations),
            ctypes.byref(self.handle)))
        out_num_class = ctypes.c_int(0)
        _safe_call(_LIB.LGBM_BoosterGetNumClasses(
            self.handle,
            ctypes.byref(out_num_class)))
2271
        if verbose:
Nikita Titov's avatar
Nikita Titov committed
2272
            print('Finished loading model, total used %d iterations' % int(out_num_iterations.value))
2273
        self.__num_class = out_num_class.value
2274
        self.pandas_categorical = _load_pandas_categorical(model_str=model_str)
2275
2276
2277
2278
        return self

    def model_to_string(self, num_iteration=None, start_iteration=0):
        """Save Booster to string.
2279

2280
2281
2282
2283
2284
2285
        Parameters
        ----------
        num_iteration : int or None, optional (default=None)
            Index of the iteration that should be saved.
            If None, if the best iteration exists, it is saved; otherwise, all iterations are saved.
            If <= 0, all iterations are saved.
Nikita Titov's avatar
Nikita Titov committed
2286
        start_iteration : int, optional (default=0)
2287
2288
2289
2290
            Start index of the iteration that should be saved.

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
2291
        str_repr : string
2292
2293
            String representation of Booster.
        """
2294
        if num_iteration is None:
2295
2296
            num_iteration = self.best_iteration
        buffer_len = 1 << 20
2297
        tmp_out_len = ctypes.c_int64(0)
2298
2299
2300
2301
        string_buffer = ctypes.create_string_buffer(buffer_len)
        ptr_string_buffer = ctypes.c_char_p(*[ctypes.addressof(string_buffer)])
        _safe_call(_LIB.LGBM_BoosterSaveModelToString(
            self.handle,
2302
            ctypes.c_int(start_iteration),
2303
            ctypes.c_int(num_iteration),
2304
            ctypes.c_int64(buffer_len),
2305
2306
2307
            ctypes.byref(tmp_out_len),
            ptr_string_buffer))
        actual_len = tmp_out_len.value
2308
        # if buffer length is not long enough, re-allocate a buffer
2309
2310
2311
2312
2313
        if actual_len > buffer_len:
            string_buffer = ctypes.create_string_buffer(actual_len)
            ptr_string_buffer = ctypes.c_char_p(*[ctypes.addressof(string_buffer)])
            _safe_call(_LIB.LGBM_BoosterSaveModelToString(
                self.handle,
2314
                ctypes.c_int(start_iteration),
2315
                ctypes.c_int(num_iteration),
2316
                ctypes.c_int64(actual_len),
2317
2318
                ctypes.byref(tmp_out_len),
                ptr_string_buffer))
2319
2320
2321
        ret = string_buffer.value.decode()
        ret += _dump_pandas_categorical(self.pandas_categorical)
        return ret
2322

2323
    def dump_model(self, num_iteration=None, start_iteration=0):
Nikita Titov's avatar
Nikita Titov committed
2324
        """Dump Booster to JSON format.
wxchan's avatar
wxchan committed
2325

2326
2327
        Parameters
        ----------
2328
2329
2330
2331
        num_iteration : int or None, optional (default=None)
            Index of the iteration that should be dumped.
            If None, if the best iteration exists, it is dumped; otherwise, all iterations are dumped.
            If <= 0, all iterations are dumped.
Nikita Titov's avatar
Nikita Titov committed
2332
        start_iteration : int, optional (default=0)
2333
            Start index of the iteration that should be dumped.
2334

wxchan's avatar
wxchan committed
2335
2336
        Returns
        -------
2337
        json_repr : dict
Nikita Titov's avatar
Nikita Titov committed
2338
            JSON format of Booster.
wxchan's avatar
wxchan committed
2339
        """
2340
        if num_iteration is None:
2341
            num_iteration = self.best_iteration
wxchan's avatar
wxchan committed
2342
        buffer_len = 1 << 20
2343
        tmp_out_len = ctypes.c_int64(0)
wxchan's avatar
wxchan committed
2344
2345
2346
2347
        string_buffer = ctypes.create_string_buffer(buffer_len)
        ptr_string_buffer = ctypes.c_char_p(*[ctypes.addressof(string_buffer)])
        _safe_call(_LIB.LGBM_BoosterDumpModel(
            self.handle,
2348
            ctypes.c_int(start_iteration),
Guolin Ke's avatar
Guolin Ke committed
2349
            ctypes.c_int(num_iteration),
2350
            ctypes.c_int64(buffer_len),
wxchan's avatar
wxchan committed
2351
            ctypes.byref(tmp_out_len),
Guolin Ke's avatar
Guolin Ke committed
2352
            ptr_string_buffer))
wxchan's avatar
wxchan committed
2353
        actual_len = tmp_out_len.value
2354
        # if buffer length is not long enough, reallocate a buffer
wxchan's avatar
wxchan committed
2355
2356
2357
2358
2359
        if actual_len > buffer_len:
            string_buffer = ctypes.create_string_buffer(actual_len)
            ptr_string_buffer = ctypes.c_char_p(*[ctypes.addressof(string_buffer)])
            _safe_call(_LIB.LGBM_BoosterDumpModel(
                self.handle,
2360
                ctypes.c_int(start_iteration),
Guolin Ke's avatar
Guolin Ke committed
2361
                ctypes.c_int(num_iteration),
2362
                ctypes.c_int64(actual_len),
wxchan's avatar
wxchan committed
2363
                ctypes.byref(tmp_out_len),
Guolin Ke's avatar
Guolin Ke committed
2364
                ptr_string_buffer))
2365
2366
2367
2368
        ret = json.loads(string_buffer.value.decode())
        ret['pandas_categorical'] = json.loads(json.dumps(self.pandas_categorical,
                                                          default=json_default_with_numpy))
        return ret
wxchan's avatar
wxchan committed
2369

2370
2371
    def predict(self, data, num_iteration=None,
                raw_score=False, pred_leaf=False, pred_contrib=False,
2372
                data_has_header=False, is_reshape=True, **kwargs):
2373
        """Make a prediction.
wxchan's avatar
wxchan committed
2374
2375
2376

        Parameters
        ----------
2377
        data : string, numpy array, pandas DataFrame, H2O DataTable's Frame or scipy.sparse
2378
2379
            Data source for prediction.
            If string, it represents the path to txt file.
2380
2381
2382
2383
        num_iteration : int or None, optional (default=None)
            Limit number of iterations in the prediction.
            If None, if the best iteration exists, it is used; otherwise, all iterations are used.
            If <= 0, all iterations are used (no limits).
2384
2385
2386
2387
        raw_score : bool, optional (default=False)
            Whether to predict raw scores.
        pred_leaf : bool, optional (default=False)
            Whether to predict leaf index.
2388
2389
        pred_contrib : bool, optional (default=False)
            Whether to predict feature contributions.
2390

Nikita Titov's avatar
Nikita Titov committed
2391
2392
2393
2394
2395
2396
2397
            .. note::

                If you want to get more explanations for your model's predictions using SHAP values,
                like SHAP interaction values,
                you can install the shap package (https://github.com/slundberg/shap).
                Note that unlike the shap package, with ``pred_contrib`` we return a matrix with an extra
                column, where the last column is the expected value.
2398

2399
2400
2401
2402
2403
        data_has_header : bool, optional (default=False)
            Whether the data has header.
            Used only if data is string.
        is_reshape : bool, optional (default=True)
            If True, result is reshaped to [nrow, ncol].
2404
2405
        **kwargs
            Other parameters for the prediction.
wxchan's avatar
wxchan committed
2406
2407
2408

        Returns
        -------
2409
2410
        result : numpy array
            Prediction result.
wxchan's avatar
wxchan committed
2411
        """
2412
        predictor = self._to_predictor(copy.deepcopy(kwargs))
2413
        if num_iteration is None:
2414
            num_iteration = self.best_iteration
2415
2416
2417
        return predictor.predict(data, num_iteration,
                                 raw_score, pred_leaf, pred_contrib,
                                 data_has_header, is_reshape)
wxchan's avatar
wxchan committed
2418

2419
    def refit(self, data, label, decay_rate=0.9, **kwargs):
Guolin Ke's avatar
Guolin Ke committed
2420
2421
2422
2423
        """Refit the existing Booster by new data.

        Parameters
        ----------
2424
        data : string, numpy array, pandas DataFrame, H2O DataTable's Frame or scipy.sparse
Guolin Ke's avatar
Guolin Ke committed
2425
2426
            Data source for refit.
            If string, it represents the path to txt file.
2427
        label : list, numpy 1-D array or pandas Series / one-column DataFrame
Guolin Ke's avatar
Guolin Ke committed
2428
2429
            Label for refit.
        decay_rate : float, optional (default=0.9)
2430
2431
            Decay rate of refit,
            will use ``leaf_output = decay_rate * old_leaf_output + (1.0 - decay_rate) * new_leaf_output`` to refit trees.
2432
2433
        **kwargs
            Other parameters for refit.
2434
            These parameters will be passed to ``predict`` method.
Guolin Ke's avatar
Guolin Ke committed
2435
2436
2437
2438
2439
2440

        Returns
        -------
        result : Booster
            Refitted Booster.
        """
2441
2442
        if self.__set_objective_to_none:
            raise LightGBMError('Cannot refit due to null objective function.')
2443
        predictor = self._to_predictor(copy.deepcopy(kwargs))
2444
        leaf_preds = predictor.predict(data, -1, pred_leaf=True)
2445
        nrow, ncol = leaf_preds.shape
2446
        train_set = Dataset(data, label, silent=True)
2447
2448
2449
        new_params = copy.deepcopy(self.params)
        new_params['refit_decay_rate'] = decay_rate
        new_booster = Booster(new_params, train_set, silent=True)
Guolin Ke's avatar
Guolin Ke committed
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
        # Copy models
        _safe_call(_LIB.LGBM_BoosterMerge(
            new_booster.handle,
            predictor.handle))
        leaf_preds = leaf_preds.reshape(-1)
        ptr_data, type_ptr_data, _ = c_int_array(leaf_preds)
        _safe_call(_LIB.LGBM_BoosterRefit(
            new_booster.handle,
            ptr_data,
            ctypes.c_int(nrow),
            ctypes.c_int(ncol)))
2461
2462
        new_booster.network = self.network
        new_booster.__attr = self.__attr.copy()
Guolin Ke's avatar
Guolin Ke committed
2463
2464
        return new_booster

2465
    def get_leaf_output(self, tree_id, leaf_id):
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
        """Get the output of a leaf.

        Parameters
        ----------
        tree_id : int
            The index of the tree.
        leaf_id : int
            The index of the leaf in the tree.

        Returns
        -------
        result : float
            The output of the leaf.
        """
2480
2481
2482
2483
2484
2485
2486
2487
        ret = ctypes.c_double(0)
        _safe_call(_LIB.LGBM_BoosterGetLeafValue(
            self.handle,
            ctypes.c_int(tree_id),
            ctypes.c_int(leaf_id),
            ctypes.byref(ret)))
        return ret.value

2488
    def _to_predictor(self, pred_parameter=None):
2489
        """Convert to predictor."""
2490
        predictor = _InnerPredictor(booster_handle=self.handle, pred_parameter=pred_parameter)
2491
        predictor.pandas_categorical = self.pandas_categorical
wxchan's avatar
wxchan committed
2492
2493
        return predictor

2494
    def num_feature(self):
2495
2496
2497
2498
2499
2500
2501
        """Get number of features.

        Returns
        -------
        num_feature : int
            The number of features.
        """
2502
2503
2504
2505
2506
2507
        out_num_feature = ctypes.c_int(0)
        _safe_call(_LIB.LGBM_BoosterGetNumFeature(
            self.handle,
            ctypes.byref(out_num_feature)))
        return out_num_feature.value

wxchan's avatar
wxchan committed
2508
    def feature_name(self):
2509
        """Get names of features.
wxchan's avatar
wxchan committed
2510
2511
2512

        Returns
        -------
2513
2514
        result : list
            List with names of features.
wxchan's avatar
wxchan committed
2515
        """
2516
        num_feature = self.num_feature()
2517
        # Get name of features
wxchan's avatar
wxchan committed
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
        tmp_out_len = ctypes.c_int(0)
        string_buffers = [ctypes.create_string_buffer(255) for i in range_(num_feature)]
        ptr_string_buffers = (ctypes.c_char_p * num_feature)(*map(ctypes.addressof, string_buffers))
        _safe_call(_LIB.LGBM_BoosterGetFeatureNames(
            self.handle,
            ctypes.byref(tmp_out_len),
            ptr_string_buffers))
        if num_feature != tmp_out_len.value:
            raise ValueError("Length of feature names doesn't equal with num_feature")
        return [string_buffers[i].value.decode() for i in range_(num_feature)]

2529
    def feature_importance(self, importance_type='split', iteration=None):
2530
        """Get feature importances.
2531

2532
2533
        Parameters
        ----------
2534
2535
2536
2537
        importance_type : string, optional (default="split")
            How the importance is calculated.
            If "split", result contains numbers of times the feature is used in a model.
            If "gain", result contains total gains of splits which use the feature.
2538
2539
2540
2541
        iteration : int or None, optional (default=None)
            Limit number of iterations in the feature importance calculation.
            If None, if the best iteration exists, it is used; otherwise, all trees are used.
            If <= 0, all trees are used (no limits).
2542

2543
2544
        Returns
        -------
2545
2546
        result : numpy array
            Array with feature importances.
2547
        """
2548
2549
        if iteration is None:
            iteration = self.best_iteration
2550
2551
2552
2553
2554
2555
        if importance_type == "split":
            importance_type_int = 0
        elif importance_type == "gain":
            importance_type_int = 1
        else:
            importance_type_int = -1
Nikita Titov's avatar
Nikita Titov committed
2556
        result = np.zeros(self.num_feature(), dtype=np.float64)
2557
2558
2559
2560
2561
2562
        _safe_call(_LIB.LGBM_BoosterFeatureImportance(
            self.handle,
            ctypes.c_int(iteration),
            ctypes.c_int(importance_type_int),
            result.ctypes.data_as(ctypes.POINTER(ctypes.c_double))))
        if importance_type_int == 0:
2563
            return result.astype(np.int32)
2564
2565
        else:
            return result
2566

2567
2568
2569
2570
2571
2572
2573
2574
2575
    def get_split_value_histogram(self, feature, bins=None, xgboost_style=False):
        """Get split value histogram for the specified feature.

        Parameters
        ----------
        feature : int or string
            The feature name or index the histogram is calculated for.
            If int, interpreted as index.
            If string, interpreted as name.
2576

Nikita Titov's avatar
Nikita Titov committed
2577
2578
2579
            .. warning::

                Categorical features are not supported.
2580

2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
        bins : int, string or None, optional (default=None)
            The maximum number of bins.
            If None, or int and > number of unique split values and ``xgboost_style=True``,
            the number of bins equals number of unique split values.
            If string, it should be one from the list of the supported values by ``numpy.histogram()`` function.
        xgboost_style : bool, optional (default=False)
            Whether the returned result should be in the same form as it is in XGBoost.
            If False, the returned value is tuple of 2 numpy arrays as it is in ``numpy.histogram()`` function.
            If True, the returned value is matrix, in which the first column is the right edges of non-empty bins
            and the second one is the histogram values.

        Returns
        -------
        result_tuple : tuple of 2 numpy arrays
            If ``xgboost_style=False``, the values of the histogram of used splitting values for the specified feature
            and the bin edges.
        result_array_like : numpy array or pandas DataFrame (if pandas is installed)
            If ``xgboost_style=True``, the histogram of used splitting values for the specified feature.
        """
        def add(root):
            """Recursively add thresholds."""
            if 'split_index' in root:  # non-leaf
                if feature_names is not None and isinstance(feature, string_type):
                    split_feature = feature_names[root['split_feature']]
                else:
                    split_feature = root['split_feature']
                if split_feature == feature:
2608
2609
2610
2611
                    if isinstance(root['threshold'], string_type):
                        raise LightGBMError('Cannot compute split value histogram for the categorical feature')
                    else:
                        values.append(root['threshold'])
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
                add(root['left_child'])
                add(root['right_child'])

        model = self.dump_model()
        feature_names = model.get('feature_names')
        tree_infos = model['tree_info']
        values = []
        for tree_info in tree_infos:
            add(tree_info['tree_structure'])

        if bins is None or isinstance(bins, integer_types) and xgboost_style:
            n_unique = len(np.unique(values))
            bins = max(min(n_unique, bins) if bins is not None else n_unique, 1)
        hist, bin_edges = np.histogram(values, bins=bins)
        if xgboost_style:
            ret = np.column_stack((bin_edges[1:], hist))
            ret = ret[ret[:, 1] > 0]
            if PANDAS_INSTALLED:
                return DataFrame(ret, columns=['SplitValue', 'Count'])
            else:
                return ret
        else:
            return hist, bin_edges

wxchan's avatar
wxchan committed
2636
    def __inner_eval(self, data_name, data_idx, feval=None):
2637
        """Evaluate training or validation data."""
wxchan's avatar
wxchan committed
2638
        if data_idx >= self.__num_dataset:
2639
            raise ValueError("Data_idx should be smaller than number of dataset")
wxchan's avatar
wxchan committed
2640
2641
2642
        self.__get_eval_info()
        ret = []
        if self.__num_inner_eval > 0:
2643
            result = np.zeros(self.__num_inner_eval, dtype=np.float64)
Guolin Ke's avatar
Guolin Ke committed
2644
            tmp_out_len = ctypes.c_int(0)
wxchan's avatar
wxchan committed
2645
2646
            _safe_call(_LIB.LGBM_BoosterGetEval(
                self.handle,
Guolin Ke's avatar
Guolin Ke committed
2647
                ctypes.c_int(data_idx),
wxchan's avatar
wxchan committed
2648
                ctypes.byref(tmp_out_len),
Guolin Ke's avatar
Guolin Ke committed
2649
                result.ctypes.data_as(ctypes.POINTER(ctypes.c_double))))
wxchan's avatar
wxchan committed
2650
            if tmp_out_len.value != self.__num_inner_eval:
2651
                raise ValueError("Wrong length of eval results")
wxchan's avatar
wxchan committed
2652
            for i in range_(self.__num_inner_eval):
2653
2654
                ret.append((data_name, self.__name_inner_eval[i],
                            result[i], self.__higher_better_inner_eval[i]))
wxchan's avatar
wxchan committed
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
        if feval is not None:
            if data_idx == 0:
                cur_data = self.train_set
            else:
                cur_data = self.valid_sets[data_idx - 1]
            feval_ret = feval(self.__inner_predict(data_idx), cur_data)
            if isinstance(feval_ret, list):
                for eval_name, val, is_higher_better in feval_ret:
                    ret.append((data_name, eval_name, val, is_higher_better))
            else:
                eval_name, val, is_higher_better = feval_ret
                ret.append((data_name, eval_name, val, is_higher_better))
        return ret

    def __inner_predict(self, data_idx):
2670
        """Predict for training and validation dataset."""
wxchan's avatar
wxchan committed
2671
        if data_idx >= self.__num_dataset:
2672
            raise ValueError("Data_idx should be smaller than number of dataset")
wxchan's avatar
wxchan committed
2673
2674
2675
2676
2677
        if self.__inner_predict_buffer[data_idx] is None:
            if data_idx == 0:
                n_preds = self.train_set.num_data() * self.__num_class
            else:
                n_preds = self.valid_sets[data_idx - 1].num_data() * self.__num_class
2678
            self.__inner_predict_buffer[data_idx] = np.zeros(n_preds, dtype=np.float64)
2679
        # avoid to predict many time in one iteration
wxchan's avatar
wxchan committed
2680
2681
        if not self.__is_predicted_cur_iter[data_idx]:
            tmp_out_len = ctypes.c_int64(0)
Guolin Ke's avatar
Guolin Ke committed
2682
            data_ptr = self.__inner_predict_buffer[data_idx].ctypes.data_as(ctypes.POINTER(ctypes.c_double))
wxchan's avatar
wxchan committed
2683
2684
            _safe_call(_LIB.LGBM_BoosterGetPredict(
                self.handle,
Guolin Ke's avatar
Guolin Ke committed
2685
                ctypes.c_int(data_idx),
wxchan's avatar
wxchan committed
2686
2687
2688
                ctypes.byref(tmp_out_len),
                data_ptr))
            if tmp_out_len.value != len(self.__inner_predict_buffer[data_idx]):
2689
                raise ValueError("Wrong length of predict results for data %d" % (data_idx))
wxchan's avatar
wxchan committed
2690
2691
2692
2693
            self.__is_predicted_cur_iter[data_idx] = True
        return self.__inner_predict_buffer[data_idx]

    def __get_eval_info(self):
2694
        """Get inner evaluation count and names."""
wxchan's avatar
wxchan committed
2695
2696
        if self.__need_reload_eval_info:
            self.__need_reload_eval_info = False
Guolin Ke's avatar
Guolin Ke committed
2697
            out_num_eval = ctypes.c_int(0)
2698
            # Get num of inner evals
wxchan's avatar
wxchan committed
2699
2700
2701
2702
2703
            _safe_call(_LIB.LGBM_BoosterGetEvalCounts(
                self.handle,
                ctypes.byref(out_num_eval)))
            self.__num_inner_eval = out_num_eval.value
            if self.__num_inner_eval > 0:
2704
                # Get name of evals
Guolin Ke's avatar
Guolin Ke committed
2705
                tmp_out_len = ctypes.c_int(0)
wxchan's avatar
wxchan committed
2706
                string_buffers = [ctypes.create_string_buffer(255) for i in range_(self.__num_inner_eval)]
wxchan's avatar
wxchan committed
2707
                ptr_string_buffers = (ctypes.c_char_p * self.__num_inner_eval)(*map(ctypes.addressof, string_buffers))
wxchan's avatar
wxchan committed
2708
2709
2710
2711
2712
                _safe_call(_LIB.LGBM_BoosterGetEvalNames(
                    self.handle,
                    ctypes.byref(tmp_out_len),
                    ptr_string_buffers))
                if self.__num_inner_eval != tmp_out_len.value:
2713
                    raise ValueError("Length of eval names doesn't equal with num_evals")
2714
                self.__name_inner_eval = \
wxchan's avatar
wxchan committed
2715
                    [string_buffers[i].value.decode() for i in range_(self.__num_inner_eval)]
2716
                self.__higher_better_inner_eval = \
2717
                    [name.startswith(('auc', 'ndcg@', 'map@')) for name in self.__name_inner_eval]
2718

wxchan's avatar
wxchan committed
2719
    def attr(self, key):
2720
        """Get attribute string from the Booster.
wxchan's avatar
wxchan committed
2721
2722
2723

        Parameters
        ----------
2724
2725
        key : string
            The name of the attribute.
wxchan's avatar
wxchan committed
2726
2727
2728

        Returns
        -------
2729
2730
        value : string or None
            The attribute value.
Nikita Titov's avatar
Nikita Titov committed
2731
            Returns None if attribute does not exist.
wxchan's avatar
wxchan committed
2732
        """
2733
        return self.__attr.get(key, None)
wxchan's avatar
wxchan committed
2734
2735

    def set_attr(self, **kwargs):
2736
        """Set attributes to the Booster.
wxchan's avatar
wxchan committed
2737
2738
2739
2740

        Parameters
        ----------
        **kwargs
2741
2742
            The attributes to set.
            Setting a value to None deletes an attribute.
Nikita Titov's avatar
Nikita Titov committed
2743
2744
2745
2746

        Returns
        -------
        self : Booster
2747
            Booster with set attributes.
wxchan's avatar
wxchan committed
2748
2749
2750
        """
        for key, value in kwargs.items():
            if value is not None:
wxchan's avatar
wxchan committed
2751
                if not isinstance(value, string_type):
Nikita Titov's avatar
Nikita Titov committed
2752
                    raise ValueError("Only string values are accepted")
wxchan's avatar
wxchan committed
2753
2754
2755
                self.__attr[key] = value
            else:
                self.__attr.pop(key, None)
Nikita Titov's avatar
Nikita Titov committed
2756
        return self