basic.py 164 KB
Newer Older
wxchan's avatar
wxchan committed
1
# coding: utf-8
2
"""Wrapper for C API of LightGBM."""
3
import abc
wxchan's avatar
wxchan committed
4
import ctypes
5
import json
wxchan's avatar
wxchan committed
6
import warnings
7
from collections import OrderedDict
8
from copy import deepcopy
9
10
from functools import wraps
from logging import Logger
11
12
13
from os import SEEK_END
from os.path import getsize
from pathlib import Path
14
from tempfile import NamedTemporaryFile
15
from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Union
wxchan's avatar
wxchan committed
16
17
18
19

import numpy as np
import scipy.sparse

20
from .compat import PANDAS_INSTALLED, concat, dt_DataTable, pd_CategoricalDtype, pd_DataFrame, pd_Series
wxchan's avatar
wxchan committed
21
22
from .libpath import find_lib_path

23
24
25
26
27
28
29
30
31
32
33
34
ZERO_THRESHOLD = 1e-35


def _get_sample_count(total_nrow: int, params: str):
    sample_cnt = ctypes.c_int(0)
    _safe_call(_LIB.LGBM_GetSampleCount(
        ctypes.c_int32(total_nrow),
        c_str(params),
        ctypes.byref(sample_cnt),
    ))
    return sample_cnt.value

wxchan's avatar
wxchan committed
35

36
class _DummyLogger:
37
    def info(self, msg: str) -> None:
38
39
        print(msg)

40
    def warning(self, msg: str) -> None:
41
42
43
        warnings.warn(msg, stacklevel=3)


44
45
46
_LOGGER: Any = _DummyLogger()
_INFO_METHOD_NAME = "info"
_WARNING_METHOD_NAME = "warning"
47
48


49
50
51
def register_logger(
    logger: Any, info_method_name: str = "info", warning_method_name: str = "warning"
) -> None:
52
53
54
55
    """Register custom logger.

    Parameters
    ----------
56
    logger : Any
57
        Custom logger.
58
59
60
61
    info_method_name : str, optional (default="info")
        Method used to log info messages.
    warning_method_name : str, optional (default="warning")
        Method used to log warning messages.
62
    """
63
64
65
66
67
68
69
70
71
    def _has_method(logger: Any, method_name: str) -> bool:
        return callable(getattr(logger, method_name, None))

    if not _has_method(logger, info_method_name) or not _has_method(logger, warning_method_name):
        raise TypeError(
            f"Logger must provide '{info_method_name}' and '{warning_method_name}' method"
        )

    global _LOGGER, _INFO_METHOD_NAME, _WARNING_METHOD_NAME
72
    _LOGGER = logger
73
74
    _INFO_METHOD_NAME = info_method_name
    _WARNING_METHOD_NAME = warning_method_name
75
76


77
def _normalize_native_string(func: Callable[[str], None]) -> Callable[[str], None]:
78
    """Join log messages from native library which come by chunks."""
79
    msg_normalized: List[str] = []
80
81

    @wraps(func)
82
    def wrapper(msg: str) -> None:
83
84
85
86
87
88
89
90
91
92
93
        nonlocal msg_normalized
        if msg.strip() == '':
            msg = ''.join(msg_normalized)
            msg_normalized = []
            return func(msg)
        else:
            msg_normalized.append(msg)

    return wrapper


94
def _log_info(msg: str) -> None:
95
    getattr(_LOGGER, _INFO_METHOD_NAME)(msg)
96
97


98
def _log_warning(msg: str) -> None:
99
    getattr(_LOGGER, _WARNING_METHOD_NAME)(msg)
100
101
102


@_normalize_native_string
103
def _log_native(msg: str) -> None:
104
    getattr(_LOGGER, _INFO_METHOD_NAME)(msg)
105
106


107
def _log_callback(msg: bytes) -> None:
108
    """Redirect logs from native library into Python."""
109
    _log_native(str(msg.decode('utf-8')))
110
111


wxchan's avatar
wxchan committed
112
def _load_lib():
113
    """Load LightGBM library."""
wxchan's avatar
wxchan committed
114
115
    lib_path = find_lib_path()
    if len(lib_path) == 0:
116
        return None
wxchan's avatar
wxchan committed
117
118
    lib = ctypes.cdll.LoadLibrary(lib_path[0])
    lib.LGBM_GetLastError.restype = ctypes.c_char_p
119
120
121
    callback = ctypes.CFUNCTYPE(None, ctypes.c_char_p)
    lib.callback = callback(_log_callback)
    if lib.LGBM_RegisterLogCallback(lib.callback) != 0:
122
        raise LightGBMError(lib.LGBM_GetLastError().decode('utf-8'))
wxchan's avatar
wxchan committed
123
124
    return lib

wxchan's avatar
wxchan committed
125

wxchan's avatar
wxchan committed
126
127
_LIB = _load_lib()

wxchan's avatar
wxchan committed
128

129
NUMERIC_TYPES = (int, float, bool)
130
_ArrayLike = Union[List, np.ndarray, pd_Series]
131
132


133
def _safe_call(ret: int) -> None:
134
135
    """Check the return value from C API call.

wxchan's avatar
wxchan committed
136
137
138
    Parameters
    ----------
    ret : int
139
        The return value from C API calls.
wxchan's avatar
wxchan committed
140
141
    """
    if ret != 0:
142
        raise LightGBMError(_LIB.LGBM_GetLastError().decode('utf-8'))
wxchan's avatar
wxchan committed
143

wxchan's avatar
wxchan committed
144

wxchan's avatar
wxchan committed
145
def is_numeric(obj):
146
    """Check whether object is a number or not, include numpy number, etc."""
wxchan's avatar
wxchan committed
147
148
149
    try:
        float(obj)
        return True
wxchan's avatar
wxchan committed
150
151
152
    except (TypeError, ValueError):
        # TypeError: obj is not a string or a number
        # ValueError: invalid literal
wxchan's avatar
wxchan committed
153
154
        return False

wxchan's avatar
wxchan committed
155

wxchan's avatar
wxchan committed
156
def is_numpy_1d_array(data):
157
    """Check whether data is a numpy 1-D array."""
158
    return isinstance(data, np.ndarray) and len(data.shape) == 1
wxchan's avatar
wxchan committed
159

wxchan's avatar
wxchan committed
160

161
162
163
164
165
166
167
168
def is_numpy_column_array(data):
    """Check whether data is a column numpy array."""
    if not isinstance(data, np.ndarray):
        return False
    shape = data.shape
    return len(shape) == 2 and shape[1] == 1


169
170
def cast_numpy_array_to_dtype(array, dtype):
    """Cast numpy array to given dtype."""
171
172
173
174
175
    if array.dtype == dtype:
        return array
    return array.astype(dtype=dtype, copy=False)


wxchan's avatar
wxchan committed
176
def is_1d_list(data):
177
178
    """Check whether data is a 1-D list."""
    return isinstance(data, list) and (not data or is_numeric(data[0]))
wxchan's avatar
wxchan committed
179

wxchan's avatar
wxchan committed
180

181
182
183
184
185
186
187
188
189
190
def _is_1d_collection(data: Any) -> bool:
    """Check whether data is a 1-D collection."""
    return (
        is_numpy_1d_array(data)
        or is_numpy_column_array(data)
        or is_1d_list(data)
        or isinstance(data, pd_Series)
    )


191
def list_to_1d_numpy(data, dtype=np.float32, name='list'):
192
    """Convert data to numpy 1-D array."""
wxchan's avatar
wxchan committed
193
    if is_numpy_1d_array(data):
194
        return cast_numpy_array_to_dtype(data, dtype)
195
196
197
    elif is_numpy_column_array(data):
        _log_warning('Converting column-vector to 1d array')
        array = data.ravel()
198
        return cast_numpy_array_to_dtype(array, dtype)
wxchan's avatar
wxchan committed
199
200
    elif is_1d_list(data):
        return np.array(data, dtype=dtype, copy=False)
201
    elif isinstance(data, pd_Series):
202
        _check_for_bad_pandas_dtypes(data.to_frame().dtypes)
203
        return np.array(data, dtype=dtype, copy=False)  # SparseArray should be supported as well
wxchan's avatar
wxchan committed
204
    else:
205
206
        raise TypeError(f"Wrong type({type(data).__name__}) for {name}.\n"
                        "It should be list, numpy 1-D array or pandas Series")
wxchan's avatar
wxchan committed
207

wxchan's avatar
wxchan committed
208

209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
def _is_numpy_2d_array(data: Any) -> bool:
    """Check whether data is a numpy 2-D array."""
    return isinstance(data, np.ndarray) and len(data.shape) == 2 and data.shape[1] > 1


def _is_2d_list(data: Any) -> bool:
    """Check whether data is a 2-D list."""
    return isinstance(data, list) and len(data) > 0 and is_1d_list(data[0])


def _is_2d_collection(data: Any) -> bool:
    """Check whether data is a 2-D collection."""
    return (
        _is_numpy_2d_array(data)
        or _is_2d_list(data)
        or isinstance(data, pd_DataFrame)
    )


def _data_to_2d_numpy(data: Any, dtype: type = np.float32, name: str = 'list') -> np.ndarray:
    """Convert data to numpy 2-D array."""
    if _is_numpy_2d_array(data):
        return cast_numpy_array_to_dtype(data, dtype)
    if _is_2d_list(data):
        return np.array(data, dtype=dtype)
    if isinstance(data, pd_DataFrame):
235
        _check_for_bad_pandas_dtypes(data.dtypes)
236
237
238
239
240
        return cast_numpy_array_to_dtype(data.values, dtype)
    raise TypeError(f"Wrong type({type(data).__name__}) for {name}.\n"
                    "It should be list of lists, numpy 2-D array or pandas DataFrame")


wxchan's avatar
wxchan committed
241
def cfloat32_array_to_numpy(cptr, length):
242
    """Convert a ctypes float pointer array to a numpy array."""
wxchan's avatar
wxchan committed
243
    if isinstance(cptr, ctypes.POINTER(ctypes.c_float)):
244
        return np.ctypeslib.as_array(cptr, shape=(length,)).copy()
wxchan's avatar
wxchan committed
245
    else:
246
        raise RuntimeError('Expected float pointer')
wxchan's avatar
wxchan committed
247

Guolin Ke's avatar
Guolin Ke committed
248

Guolin Ke's avatar
Guolin Ke committed
249
def cfloat64_array_to_numpy(cptr, length):
250
    """Convert a ctypes double pointer array to a numpy array."""
Guolin Ke's avatar
Guolin Ke committed
251
    if isinstance(cptr, ctypes.POINTER(ctypes.c_double)):
252
        return np.ctypeslib.as_array(cptr, shape=(length,)).copy()
Guolin Ke's avatar
Guolin Ke committed
253
254
255
    else:
        raise RuntimeError('Expected double pointer')

wxchan's avatar
wxchan committed
256

wxchan's avatar
wxchan committed
257
def cint32_array_to_numpy(cptr, length):
258
    """Convert a ctypes int pointer array to a numpy array."""
wxchan's avatar
wxchan committed
259
    if isinstance(cptr, ctypes.POINTER(ctypes.c_int32)):
260
        return np.ctypeslib.as_array(cptr, shape=(length,)).copy()
wxchan's avatar
wxchan committed
261
    else:
262
263
264
265
266
267
        raise RuntimeError('Expected int32 pointer')


def cint64_array_to_numpy(cptr, length):
    """Convert a ctypes int pointer array to a numpy array."""
    if isinstance(cptr, ctypes.POINTER(ctypes.c_int64)):
268
        return np.ctypeslib.as_array(cptr, shape=(length,)).copy()
269
270
    else:
        raise RuntimeError('Expected int64 pointer')
wxchan's avatar
wxchan committed
271

wxchan's avatar
wxchan committed
272

wxchan's avatar
wxchan committed
273
def c_str(string):
274
    """Convert a Python string to C string."""
wxchan's avatar
wxchan committed
275
276
    return ctypes.c_char_p(string.encode('utf-8'))

wxchan's avatar
wxchan committed
277

wxchan's avatar
wxchan committed
278
def c_array(ctype, values):
279
    """Convert a Python array to C array."""
wxchan's avatar
wxchan committed
280
281
    return (ctype * len(values))(*values)

wxchan's avatar
wxchan committed
282

283
284
285
286
287
288
289
290
291
292
def json_default_with_numpy(obj):
    """Convert numpy classes to JSON serializable objects."""
    if isinstance(obj, (np.integer, np.floating, np.bool_)):
        return obj.item()
    elif isinstance(obj, np.ndarray):
        return obj.tolist()
    else:
        return obj


wxchan's avatar
wxchan committed
293
def param_dict_to_str(data):
294
    """Convert Python dictionary to string, which is passed to C API."""
295
    if data is None or not data:
wxchan's avatar
wxchan committed
296
297
298
        return ""
    pairs = []
    for key, val in data.items():
299
        if isinstance(val, (list, tuple, set)) or is_numpy_1d_array(val):
300
301
            def to_string(x):
                if isinstance(x, list):
302
                    return f"[{','.join(map(str, x))}]"
303
304
                else:
                    return str(x)
305
            pairs.append(f"{key}={','.join(map(to_string, val))}")
306
        elif isinstance(val, (str, Path, NUMERIC_TYPES)) or is_numeric(val):
307
            pairs.append(f"{key}={val}")
308
        elif val is not None:
309
            raise TypeError(f'Unknown type of parameter:{key}, got:{type(val).__name__}')
wxchan's avatar
wxchan committed
310
    return ' '.join(pairs)
311

wxchan's avatar
wxchan committed
312

313
class _TempFile:
314
315
    """Proxy class to workaround errors on Windows."""

316
317
318
    def __enter__(self):
        with NamedTemporaryFile(prefix="lightgbm_tmp_", delete=True) as f:
            self.name = f.name
319
            self.path = Path(self.name)
320
        return self
wxchan's avatar
wxchan committed
321

322
    def __exit__(self, exc_type, exc_val, exc_tb):
323
324
        if self.path.is_file():
            self.path.unlink()
325

wxchan's avatar
wxchan committed
326

327
class LightGBMError(Exception):
328
329
    """Error thrown by LightGBM."""

330
331
332
    pass


333
334
335
336
337
338
339
340
# DeprecationWarning is not shown by default, so let's create our own with higher level
class LGBMDeprecationWarning(UserWarning):
    """Custom deprecation warning."""

    pass


class _ConfigAliases:
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
    # lazy evaluation to allow import without dynamic library, e.g., for docs generation
    aliases = None

    @staticmethod
    def _get_all_param_aliases() -> Dict[str, Set[str]]:
        buffer_len = 1 << 20
        tmp_out_len = ctypes.c_int64(0)
        string_buffer = ctypes.create_string_buffer(buffer_len)
        ptr_string_buffer = ctypes.c_char_p(*[ctypes.addressof(string_buffer)])
        _safe_call(_LIB.LGBM_DumpParamAliases(
            ctypes.c_int64(buffer_len),
            ctypes.byref(tmp_out_len),
            ptr_string_buffer))
        actual_len = tmp_out_len.value
        # if buffer length is not long enough, re-allocate a buffer
        if actual_len > buffer_len:
            string_buffer = ctypes.create_string_buffer(actual_len)
            ptr_string_buffer = ctypes.c_char_p(*[ctypes.addressof(string_buffer)])
            _safe_call(_LIB.LGBM_DumpParamAliases(
                ctypes.c_int64(actual_len),
                ctypes.byref(tmp_out_len),
                ptr_string_buffer))
        aliases = json.loads(
            string_buffer.value.decode('utf-8'),
            object_hook=lambda obj: {k: set(v) | {k} for k, v in obj.items()}
        )
        return aliases
368
369

    @classmethod
370
371
372
    def get(cls, *args) -> Set[str]:
        if cls.aliases is None:
            cls.aliases = cls._get_all_param_aliases()
373
374
        ret = set()
        for i in args:
375
            ret |= cls.aliases.get(i, {i})
376
377
        return ret

378
    @classmethod
379
380
381
    def get_by_alias(cls, *args) -> Set[str]:
        if cls.aliases is None:
            cls.aliases = cls._get_all_param_aliases()
382
383
384
385
386
387
388
389
        ret = set(args)
        for arg in args:
            for aliases in cls.aliases.values():
                if arg in aliases:
                    ret |= aliases
                    break
        return ret

390

391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
def _choose_param_value(main_param_name: str, params: Dict[str, Any], default_value: Any) -> Dict[str, Any]:
    """Get a single parameter value, accounting for aliases.

    Parameters
    ----------
    main_param_name : str
        Name of the main parameter to get a value for. One of the keys of ``_ConfigAliases``.
    params : dict
        Dictionary of LightGBM parameters.
    default_value : Any
        Default value to use for the parameter, if none is found in ``params``.

    Returns
    -------
    params : dict
        A ``params`` dict with exactly one value for ``main_param_name``, and all aliases ``main_param_name`` removed.
        If both ``main_param_name`` and one or more aliases for it are found, the value of ``main_param_name`` will be preferred.
    """
    # avoid side effects on passed-in parameters
    params = deepcopy(params)

    # find a value, and remove other aliases with .pop()
    # prefer the value of 'main_param_name' if it exists, otherwise search the aliases
    found_value = None
    if main_param_name in params.keys():
        found_value = params[main_param_name]

    for param in _ConfigAliases.get(main_param_name):
        val = params.pop(param, None)
        if found_value is None and val is not None:
            found_value = val

    if found_value is not None:
        params[main_param_name] = found_value
    else:
        params[main_param_name] = default_value

    return params


431
432
MAX_INT32 = (1 << 31) - 1

433
"""Macro definition of data type in C API of LightGBM"""
wxchan's avatar
wxchan committed
434
435
436
437
C_API_DTYPE_FLOAT32 = 0
C_API_DTYPE_FLOAT64 = 1
C_API_DTYPE_INT32 = 2
C_API_DTYPE_INT64 = 3
Guolin Ke's avatar
Guolin Ke committed
438

439
"""Matrix is row major in Python"""
wxchan's avatar
wxchan committed
440
441
C_API_IS_ROW_MAJOR = 1

442
"""Macro definition of prediction type in C API of LightGBM"""
wxchan's avatar
wxchan committed
443
444
445
C_API_PREDICT_NORMAL = 0
C_API_PREDICT_RAW_SCORE = 1
C_API_PREDICT_LEAF_INDEX = 2
446
C_API_PREDICT_CONTRIB = 3
wxchan's avatar
wxchan committed
447

448
449
450
451
"""Macro definition of sparse matrix type"""
C_API_MATRIX_TYPE_CSR = 0
C_API_MATRIX_TYPE_CSC = 1

452
453
454
455
"""Macro definition of feature importance type"""
C_API_FEATURE_IMPORTANCE_SPLIT = 0
C_API_FEATURE_IMPORTANCE_GAIN = 1

456
"""Data type of data field"""
wxchan's avatar
wxchan committed
457
458
FIELD_TYPE_MAPPER = {"label": C_API_DTYPE_FLOAT32,
                     "weight": C_API_DTYPE_FLOAT32,
Guolin Ke's avatar
Guolin Ke committed
459
                     "init_score": C_API_DTYPE_FLOAT64,
460
                     "group": C_API_DTYPE_INT32}
wxchan's avatar
wxchan committed
461

462
463
464
465
"""String name to int feature importance type mapper"""
FEATURE_IMPORTANCE_TYPE_MAPPER = {"split": C_API_FEATURE_IMPORTANCE_SPLIT,
                                  "gain": C_API_FEATURE_IMPORTANCE_GAIN}

wxchan's avatar
wxchan committed
466

467
def convert_from_sliced_object(data):
468
    """Fix the memory of multi-dimensional sliced object."""
469
    if isinstance(data, np.ndarray) and isinstance(data.base, np.ndarray):
470
        if not data.flags.c_contiguous:
471
472
            _log_warning("Usage of np.ndarray subset (sliced data) is not recommended "
                         "due to it will double the peak memory cost in LightGBM.")
473
474
475
476
            return np.copy(data)
    return data


wxchan's avatar
wxchan committed
477
def c_float_array(data):
478
    """Get pointer of float numpy array / list."""
wxchan's avatar
wxchan committed
479
480
481
    if is_1d_list(data):
        data = np.array(data, copy=False)
    if is_numpy_1d_array(data):
482
483
        data = convert_from_sliced_object(data)
        assert data.flags.c_contiguous
wxchan's avatar
wxchan committed
484
485
486
487
488
489
490
        if data.dtype == np.float32:
            ptr_data = data.ctypes.data_as(ctypes.POINTER(ctypes.c_float))
            type_data = C_API_DTYPE_FLOAT32
        elif data.dtype == np.float64:
            ptr_data = data.ctypes.data_as(ctypes.POINTER(ctypes.c_double))
            type_data = C_API_DTYPE_FLOAT64
        else:
491
            raise TypeError(f"Expected np.float32 or np.float64, met type({data.dtype})")
wxchan's avatar
wxchan committed
492
    else:
493
        raise TypeError(f"Unknown type({type(data).__name__})")
494
    return (ptr_data, type_data, data)  # return `data` to avoid the temporary copy is freed
wxchan's avatar
wxchan committed
495

wxchan's avatar
wxchan committed
496

wxchan's avatar
wxchan committed
497
def c_int_array(data):
498
    """Get pointer of int numpy array / list."""
wxchan's avatar
wxchan committed
499
500
501
    if is_1d_list(data):
        data = np.array(data, copy=False)
    if is_numpy_1d_array(data):
502
503
        data = convert_from_sliced_object(data)
        assert data.flags.c_contiguous
wxchan's avatar
wxchan committed
504
505
506
507
508
509
510
        if data.dtype == np.int32:
            ptr_data = data.ctypes.data_as(ctypes.POINTER(ctypes.c_int32))
            type_data = C_API_DTYPE_INT32
        elif data.dtype == np.int64:
            ptr_data = data.ctypes.data_as(ctypes.POINTER(ctypes.c_int64))
            type_data = C_API_DTYPE_INT64
        else:
511
            raise TypeError(f"Expected np.int32 or np.int64, met type({data.dtype})")
wxchan's avatar
wxchan committed
512
    else:
513
        raise TypeError(f"Unknown type({type(data).__name__})")
514
    return (ptr_data, type_data, data)  # return `data` to avoid the temporary copy is freed
wxchan's avatar
wxchan committed
515

wxchan's avatar
wxchan committed
516

517
def _check_for_bad_pandas_dtypes(pandas_dtypes_series):
518
519
520
521
522
523
524
525
    float128 = getattr(np, 'float128', type(None))

    def is_allowed_numpy_dtype(dtype):
        return (
            issubclass(dtype, (np.integer, np.floating, np.bool_))
            and not issubclass(dtype, (np.timedelta64, float128))
        )

526
527
528
529
530
531
532
533
    bad_pandas_dtypes = [
        f'{column_name}: {pandas_dtype}'
        for column_name, pandas_dtype in pandas_dtypes_series.iteritems()
        if not is_allowed_numpy_dtype(pandas_dtype.type)
    ]
    if bad_pandas_dtypes:
        raise ValueError('pandas dtypes must be int, float or bool.\n'
                         f'Fields with bad pandas dtypes: {", ".join(bad_pandas_dtypes)}')
534
535


536
def _data_from_pandas(data, feature_name, categorical_feature, pandas_categorical):
537
    if isinstance(data, pd_DataFrame):
538
539
        if len(data.shape) != 2 or data.shape[0] < 1:
            raise ValueError('Input data must be 2 dimensional and non empty.')
540
541
        if feature_name == 'auto' or feature_name is None:
            data = data.rename(columns=str)
542
        cat_cols = [col for col, dtype in zip(data.columns, data.dtypes) if isinstance(dtype, pd_CategoricalDtype)]
543
        cat_cols_not_ordered = [col for col in cat_cols if not data[col].cat.ordered]
544
545
546
547
548
        if pandas_categorical is None:  # train dataset
            pandas_categorical = [list(data[col].cat.categories) for col in cat_cols]
        else:
            if len(cat_cols) != len(pandas_categorical):
                raise ValueError('train and valid dataset categorical_feature do not match.')
549
            for col, category in zip(cat_cols, pandas_categorical):
550
551
                if list(data[col].cat.categories) != list(category):
                    data[col] = data[col].cat.set_categories(category)
552
        if len(cat_cols):  # cat_cols is list
553
            data = data.copy()  # not alter origin DataFrame
554
            data[cat_cols] = data[cat_cols].apply(lambda x: x.cat.codes).replace({-1: np.nan})
555
556
557
        if categorical_feature is not None:
            if feature_name is None:
                feature_name = list(data.columns)
558
            if categorical_feature == 'auto':  # use cat cols from DataFrame
559
                categorical_feature = cat_cols_not_ordered
560
561
            else:  # use cat cols specified by user
                categorical_feature = list(categorical_feature)
562
563
        if feature_name == 'auto':
            feature_name = list(data.columns)
564
        _check_for_bad_pandas_dtypes(data.dtypes)
565
566
567
568
        df_dtypes = [dtype.type for dtype in data.dtypes]
        df_dtypes.append(np.float32)  # so that the target dtype considers floats
        target_dtype = np.find_common_type(df_dtypes, [])
        data = data.astype(target_dtype, copy=False).values
569
570
571
572
573
574
    else:
        if feature_name == 'auto':
            feature_name = None
        if categorical_feature == 'auto':
            categorical_feature = None
    return data, feature_name, categorical_feature, pandas_categorical
575
576
577


def _label_from_pandas(label):
578
    if isinstance(label, pd_DataFrame):
579
580
        if len(label.columns) > 1:
            raise ValueError('DataFrame for label cannot have multiple columns')
581
        _check_for_bad_pandas_dtypes(label.dtypes)
582
        label = np.ravel(label.values.astype(np.float32, copy=False))
583
584
585
    return label


586
def _dump_pandas_categorical(pandas_categorical, file_name=None):
587
588
    categorical_json = json.dumps(pandas_categorical, default=json_default_with_numpy)
    pandas_str = f'\npandas_categorical:{categorical_json}\n'
589
590
591
592
593
594
595
    if file_name is not None:
        with open(file_name, 'a') as f:
            f.write(pandas_str)
    return pandas_str


def _load_pandas_categorical(file_name=None, model_str=None):
596
597
    pandas_key = 'pandas_categorical:'
    offset = -len(pandas_key)
598
    if file_name is not None:
599
        max_offset = -getsize(file_name)
600
601
602
603
        with open(file_name, 'rb') as f:
            while True:
                if offset < max_offset:
                    offset = max_offset
604
                f.seek(offset, SEEK_END)
605
606
607
608
                lines = f.readlines()
                if len(lines) >= 2:
                    break
                offset *= 2
609
        last_line = lines[-1].decode('utf-8').strip()
610
        if not last_line.startswith(pandas_key):
611
            last_line = lines[-2].decode('utf-8').strip()
612
    elif model_str is not None:
613
614
615
616
617
618
        idx = model_str.rfind('\n', 0, offset)
        last_line = model_str[idx:].strip()
    if last_line.startswith(pandas_key):
        return json.loads(last_line[len(pandas_key):])
    else:
        return None
619
620


621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
class Sequence(abc.ABC):
    """
    Generic data access interface.

    Object should support the following operations:

    .. code-block::

        # Get total row number.
        >>> len(seq)
        # Random access by row index. Used for data sampling.
        >>> seq[10]
        # Range data access. Used to read data in batch when constructing Dataset.
        >>> seq[0:100]
        # Optionally specify batch_size to control range data read size.
        >>> seq.batch_size

    - With random access, **data sampling does not need to go through all data**.
    - With range data access, there's **no need to read all data into memory thus reduce memory usage**.

641
642
    .. versionadded:: 3.3.0

643
644
645
646
647
648
649
650
651
    Attributes
    ----------
    batch_size : int
        Default size of a batch.
    """

    batch_size = 4096  # Defaults to read 4K rows in each batch.

    @abc.abstractmethod
652
    def __getitem__(self, idx: Union[int, slice, List[int]]) -> np.ndarray:
653
654
655
656
657
658
659
        """Return data for given row index.

        A basic implementation should look like this:

        .. code-block:: python

            if isinstance(idx, numbers.Integral):
660
                return self._get_one_line(idx)
661
            elif isinstance(idx, slice):
662
663
                return np.stack([self._get_one_line(i) for i in range(idx.start, idx.stop)])
            elif isinstance(idx, list):
664
                # Only required if using ``Dataset.subset()``.
665
                return np.array([self._get_one_line(i) for i in idx])
666
            else:
667
                raise TypeError(f"Sequence index must be integer, slice or list, got {type(idx).__name__}")
668
669
670

        Parameters
        ----------
671
        idx : int, slice[int], list[int]
672
673
674
675
            Item index.

        Returns
        -------
676
        result : numpy 1-D array or numpy 2-D array
677
            1-D array if idx is int, 2-D array if idx is slice or list.
678
679
680
681
682
683
684
685
686
        """
        raise NotImplementedError("Sub-classes of lightgbm.Sequence must implement __getitem__()")

    @abc.abstractmethod
    def __len__(self) -> int:
        """Return row count of this sequence."""
        raise NotImplementedError("Sub-classes of lightgbm.Sequence must implement __len__()")


687
class _InnerPredictor:
688
689
690
691
692
    """_InnerPredictor of LightGBM.

    Not exposed to user.
    Used only for prediction, usually used for continued training.

Nikita Titov's avatar
Nikita Titov committed
693
694
695
    .. note::

        Can be converted from Booster, but cannot be converted to Booster.
Guolin Ke's avatar
Guolin Ke committed
696
    """
697

698
    def __init__(self, model_file=None, booster_handle=None, pred_parameter=None):
699
        """Initialize the _InnerPredictor.
wxchan's avatar
wxchan committed
700
701
702

        Parameters
        ----------
703
        model_file : str, pathlib.Path or None, optional (default=None)
wxchan's avatar
wxchan committed
704
            Path to the model file.
705
706
707
        booster_handle : object or None, optional (default=None)
            Handle of Booster.
        pred_parameter: dict or None, optional (default=None)
708
            Other parameters for the prediction.
wxchan's avatar
wxchan committed
709
710
711
712
713
        """
        self.handle = ctypes.c_void_p()
        self.__is_manage_handle = True
        if model_file is not None:
            """Prediction task"""
Guolin Ke's avatar
Guolin Ke committed
714
            out_num_iterations = ctypes.c_int(0)
wxchan's avatar
wxchan committed
715
            _safe_call(_LIB.LGBM_BoosterCreateFromModelfile(
716
                c_str(str(model_file)),
wxchan's avatar
wxchan committed
717
718
                ctypes.byref(out_num_iterations),
                ctypes.byref(self.handle)))
Guolin Ke's avatar
Guolin Ke committed
719
            out_num_class = ctypes.c_int(0)
wxchan's avatar
wxchan committed
720
721
722
723
            _safe_call(_LIB.LGBM_BoosterGetNumClasses(
                self.handle,
                ctypes.byref(out_num_class)))
            self.num_class = out_num_class.value
724
            self.num_total_iteration = out_num_iterations.value
725
            self.pandas_categorical = _load_pandas_categorical(file_name=model_file)
wxchan's avatar
wxchan committed
726
        elif booster_handle is not None:
Guolin Ke's avatar
Guolin Ke committed
727
            self.__is_manage_handle = False
wxchan's avatar
wxchan committed
728
            self.handle = booster_handle
Guolin Ke's avatar
Guolin Ke committed
729
            out_num_class = ctypes.c_int(0)
wxchan's avatar
wxchan committed
730
731
732
733
            _safe_call(_LIB.LGBM_BoosterGetNumClasses(
                self.handle,
                ctypes.byref(out_num_class)))
            self.num_class = out_num_class.value
734
            self.num_total_iteration = self.current_iteration()
735
            self.pandas_categorical = None
wxchan's avatar
wxchan committed
736
        else:
737
            raise TypeError('Need model_file or booster_handle to create a predictor')
wxchan's avatar
wxchan committed
738

739
740
        pred_parameter = {} if pred_parameter is None else pred_parameter
        self.pred_parameter = param_dict_to_str(pred_parameter)
cbecker's avatar
cbecker committed
741

wxchan's avatar
wxchan committed
742
    def __del__(self):
743
744
745
746
747
        try:
            if self.__is_manage_handle:
                _safe_call(_LIB.LGBM_BoosterFree(self.handle))
        except AttributeError:
            pass
wxchan's avatar
wxchan committed
748

749
750
751
752
753
    def __getstate__(self):
        this = self.__dict__.copy()
        this.pop('handle', None)
        return this

754
    def predict(self, data, start_iteration=0, num_iteration=-1,
755
                raw_score=False, pred_leaf=False, pred_contrib=False, data_has_header=False):
756
        """Predict logic.
wxchan's avatar
wxchan committed
757
758
759

        Parameters
        ----------
760
        data : str, pathlib.Path, numpy array, pandas DataFrame, H2O DataTable's Frame or scipy.sparse
761
            Data source for prediction.
762
            If str or pathlib.Path, it represents the path to a text file (CSV, TSV, or LibSVM).
763
764
        start_iteration : int, optional (default=0)
            Start index of the iteration to predict.
765
766
767
768
769
770
771
772
773
774
775
        num_iteration : int, optional (default=-1)
            Iteration used for prediction.
        raw_score : bool, optional (default=False)
            Whether to predict raw scores.
        pred_leaf : bool, optional (default=False)
            Whether to predict leaf index.
        pred_contrib : bool, optional (default=False)
            Whether to predict feature contributions.
        data_has_header : bool, optional (default=False)
            Whether data has header.
            Used only for txt data.
wxchan's avatar
wxchan committed
776
777
778

        Returns
        -------
779
        result : numpy array, scipy.sparse or list of scipy.sparse
780
            Prediction result.
781
            Can be sparse or a list of sparse objects (each element represents predictions for one class) for feature contributions (when ``pred_contrib=True``).
wxchan's avatar
wxchan committed
782
        """
wxchan's avatar
wxchan committed
783
        if isinstance(data, Dataset):
784
            raise TypeError("Cannot use Dataset instance for prediction, please use raw data instead")
785
        data = _data_from_pandas(data, None, None, self.pandas_categorical)[0]
wxchan's avatar
wxchan committed
786
787
788
789
790
        predict_type = C_API_PREDICT_NORMAL
        if raw_score:
            predict_type = C_API_PREDICT_RAW_SCORE
        if pred_leaf:
            predict_type = C_API_PREDICT_LEAF_INDEX
791
792
        if pred_contrib:
            predict_type = C_API_PREDICT_CONTRIB
wxchan's avatar
wxchan committed
793
        int_data_has_header = 1 if data_has_header else 0
cbecker's avatar
cbecker committed
794

795
        if isinstance(data, (str, Path)):
796
            with _TempFile() as f:
wxchan's avatar
wxchan committed
797
798
                _safe_call(_LIB.LGBM_BoosterPredictForFile(
                    self.handle,
799
                    c_str(str(data)),
Guolin Ke's avatar
Guolin Ke committed
800
801
                    ctypes.c_int(int_data_has_header),
                    ctypes.c_int(predict_type),
802
                    ctypes.c_int(start_iteration),
Guolin Ke's avatar
Guolin Ke committed
803
                    ctypes.c_int(num_iteration),
804
                    c_str(self.pred_parameter),
wxchan's avatar
wxchan committed
805
                    c_str(f.name)))
806
807
                preds = np.loadtxt(f.name, dtype=np.float64)
                nrow = preds.shape[0]
wxchan's avatar
wxchan committed
808
        elif isinstance(data, scipy.sparse.csr_matrix):
809
            preds, nrow = self.__pred_for_csr(data, start_iteration, num_iteration, predict_type)
Guolin Ke's avatar
Guolin Ke committed
810
        elif isinstance(data, scipy.sparse.csc_matrix):
811
            preds, nrow = self.__pred_for_csc(data, start_iteration, num_iteration, predict_type)
wxchan's avatar
wxchan committed
812
        elif isinstance(data, np.ndarray):
813
            preds, nrow = self.__pred_for_np2d(data, start_iteration, num_iteration, predict_type)
814
815
816
        elif isinstance(data, list):
            try:
                data = np.array(data)
817
            except BaseException:
818
                raise ValueError('Cannot convert data list to numpy array.')
819
            preds, nrow = self.__pred_for_np2d(data, start_iteration, num_iteration, predict_type)
820
        elif isinstance(data, dt_DataTable):
821
            preds, nrow = self.__pred_for_np2d(data.to_numpy(), start_iteration, num_iteration, predict_type)
wxchan's avatar
wxchan committed
822
823
        else:
            try:
824
                _log_warning('Converting data to scipy sparse matrix.')
wxchan's avatar
wxchan committed
825
                csr = scipy.sparse.csr_matrix(data)
826
            except BaseException:
827
                raise TypeError(f'Cannot predict data for type {type(data).__name__}')
828
            preds, nrow = self.__pred_for_csr(csr, start_iteration, num_iteration, predict_type)
wxchan's avatar
wxchan committed
829
830
        if pred_leaf:
            preds = preds.astype(np.int32)
831
        is_sparse = scipy.sparse.issparse(preds) or isinstance(preds, list)
832
        if not is_sparse and preds.size != nrow:
wxchan's avatar
wxchan committed
833
            if preds.size % nrow == 0:
834
                preds = preds.reshape(nrow, -1)
wxchan's avatar
wxchan committed
835
            else:
836
                raise ValueError(f'Length of predict result ({preds.size}) cannot be divide nrow ({nrow})')
wxchan's avatar
wxchan committed
837
838
        return preds

839
    def __get_num_preds(self, start_iteration, num_iteration, nrow, predict_type):
840
        """Get size of prediction result."""
841
        if nrow > MAX_INT32:
842
            raise LightGBMError('LightGBM cannot perform prediction for data '
843
                                f'with number of rows greater than MAX_INT32 ({MAX_INT32}).\n'
844
                                'You can split your data into chunks '
845
                                'and then concatenate predictions for them')
Guolin Ke's avatar
Guolin Ke committed
846
847
848
        n_preds = ctypes.c_int64(0)
        _safe_call(_LIB.LGBM_BoosterCalcNumPredict(
            self.handle,
Guolin Ke's avatar
Guolin Ke committed
849
850
            ctypes.c_int(nrow),
            ctypes.c_int(predict_type),
851
            ctypes.c_int(start_iteration),
Guolin Ke's avatar
Guolin Ke committed
852
            ctypes.c_int(num_iteration),
Guolin Ke's avatar
Guolin Ke committed
853
854
            ctypes.byref(n_preds)))
        return n_preds.value
wxchan's avatar
wxchan committed
855

856
    def __pred_for_np2d(self, mat, start_iteration, num_iteration, predict_type):
857
        """Predict for a 2-D numpy matrix."""
wxchan's avatar
wxchan committed
858
        if len(mat.shape) != 2:
859
            raise ValueError('Input numpy.ndarray or list must be 2 dimensional')
wxchan's avatar
wxchan committed
860

861
        def inner_predict(mat, start_iteration, num_iteration, predict_type, preds=None):
862
863
            if mat.dtype == np.float32 or mat.dtype == np.float64:
                data = np.array(mat.reshape(mat.size), dtype=mat.dtype, copy=False)
864
            else:  # change non-float data to float data, need to copy
865
866
                data = np.array(mat.reshape(mat.size), dtype=np.float32)
            ptr_data, type_ptr_data, _ = c_float_array(data)
867
            n_preds = self.__get_num_preds(start_iteration, num_iteration, mat.shape[0], predict_type)
868
            if preds is None:
869
                preds = np.empty(n_preds, dtype=np.float64)
870
871
872
873
874
875
876
            elif len(preds.shape) != 1 or len(preds) != n_preds:
                raise ValueError("Wrong length of pre-allocated predict array")
            out_num_preds = ctypes.c_int64(0)
            _safe_call(_LIB.LGBM_BoosterPredictForMat(
                self.handle,
                ptr_data,
                ctypes.c_int(type_ptr_data),
877
878
                ctypes.c_int32(mat.shape[0]),
                ctypes.c_int32(mat.shape[1]),
879
880
                ctypes.c_int(C_API_IS_ROW_MAJOR),
                ctypes.c_int(predict_type),
881
                ctypes.c_int(start_iteration),
882
883
884
885
886
887
888
889
890
891
892
893
                ctypes.c_int(num_iteration),
                c_str(self.pred_parameter),
                ctypes.byref(out_num_preds),
                preds.ctypes.data_as(ctypes.POINTER(ctypes.c_double))))
            if n_preds != out_num_preds.value:
                raise ValueError("Wrong length for predict results")
            return preds, mat.shape[0]

        nrow = mat.shape[0]
        if nrow > MAX_INT32:
            sections = np.arange(start=MAX_INT32, stop=nrow, step=MAX_INT32)
            # __get_num_preds() cannot work with nrow > MAX_INT32, so calculate overall number of predictions piecemeal
894
            n_preds = [self.__get_num_preds(start_iteration, num_iteration, i, predict_type) for i in np.diff([0] + list(sections) + [nrow])]
895
            n_preds_sections = np.array([0] + n_preds, dtype=np.intp).cumsum()
896
            preds = np.empty(sum(n_preds), dtype=np.float64)
897
898
            for chunk, (start_idx_pred, end_idx_pred) in zip(np.array_split(mat, sections),
                                                             zip(n_preds_sections, n_preds_sections[1:])):
899
                # avoid memory consumption by arrays concatenation operations
900
                inner_predict(chunk, start_iteration, num_iteration, predict_type, preds[start_idx_pred:end_idx_pred])
901
            return preds, nrow
wxchan's avatar
wxchan committed
902
        else:
903
            return inner_predict(mat, start_iteration, num_iteration, predict_type)
wxchan's avatar
wxchan committed
904

905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
    def __create_sparse_native(self, cs, out_shape, out_ptr_indptr, out_ptr_indices, out_ptr_data,
                               indptr_type, data_type, is_csr=True):
        # create numpy array from output arrays
        data_indices_len = out_shape[0]
        indptr_len = out_shape[1]
        if indptr_type == C_API_DTYPE_INT32:
            out_indptr = cint32_array_to_numpy(out_ptr_indptr, indptr_len)
        elif indptr_type == C_API_DTYPE_INT64:
            out_indptr = cint64_array_to_numpy(out_ptr_indptr, indptr_len)
        else:
            raise TypeError("Expected int32 or int64 type for indptr")
        if data_type == C_API_DTYPE_FLOAT32:
            out_data = cfloat32_array_to_numpy(out_ptr_data, data_indices_len)
        elif data_type == C_API_DTYPE_FLOAT64:
            out_data = cfloat64_array_to_numpy(out_ptr_data, data_indices_len)
        else:
            raise TypeError("Expected float32 or float64 type for data")
        out_indices = cint32_array_to_numpy(out_ptr_indices, data_indices_len)
        # break up indptr based on number of rows (note more than one matrix in multiclass case)
        per_class_indptr_shape = cs.indptr.shape[0]
        # for CSC there is extra column added
        if not is_csr:
            per_class_indptr_shape += 1
        out_indptr_arrays = np.split(out_indptr, out_indptr.shape[0] / per_class_indptr_shape)
        # reformat output into a csr or csc matrix or list of csr or csc matrices
        cs_output_matrices = []
        offset = 0
        for cs_indptr in out_indptr_arrays:
            matrix_indptr_len = cs_indptr[cs_indptr.shape[0] - 1]
            cs_indices = out_indices[offset + cs_indptr[0]:offset + matrix_indptr_len]
            cs_data = out_data[offset + cs_indptr[0]:offset + matrix_indptr_len]
            offset += matrix_indptr_len
            # same shape as input csr or csc matrix except extra column for expected value
            cs_shape = [cs.shape[0], cs.shape[1] + 1]
            # note: make sure we copy data as it will be deallocated next
            if is_csr:
                cs_output_matrices.append(scipy.sparse.csr_matrix((cs_data, cs_indices, cs_indptr), cs_shape))
            else:
                cs_output_matrices.append(scipy.sparse.csc_matrix((cs_data, cs_indices, cs_indptr), cs_shape))
        # free the temporary native indptr, indices, and data
        _safe_call(_LIB.LGBM_BoosterFreePredictSparse(out_ptr_indptr, out_ptr_indices, out_ptr_data,
                                                      ctypes.c_int(indptr_type), ctypes.c_int(data_type)))
        if len(cs_output_matrices) == 1:
            return cs_output_matrices[0]
        return cs_output_matrices

951
    def __pred_for_csr(self, csr, start_iteration, num_iteration, predict_type):
952
        """Predict for a CSR data."""
953
        def inner_predict(csr, start_iteration, num_iteration, predict_type, preds=None):
954
            nrow = len(csr.indptr) - 1
955
            n_preds = self.__get_num_preds(start_iteration, num_iteration, nrow, predict_type)
956
            if preds is None:
957
                preds = np.empty(n_preds, dtype=np.float64)
958
959
960
961
962
963
964
            elif len(preds.shape) != 1 or len(preds) != n_preds:
                raise ValueError("Wrong length of pre-allocated predict array")
            out_num_preds = ctypes.c_int64(0)

            ptr_indptr, type_ptr_indptr, __ = c_int_array(csr.indptr)
            ptr_data, type_ptr_data, _ = c_float_array(csr.data)

965
            assert csr.shape[1] <= MAX_INT32
966
            csr_indices = csr.indices.astype(np.int32, copy=False)
967

968
969
970
            _safe_call(_LIB.LGBM_BoosterPredictForCSR(
                self.handle,
                ptr_indptr,
971
                ctypes.c_int(type_ptr_indptr),
972
                csr_indices.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
973
974
975
976
977
978
                ptr_data,
                ctypes.c_int(type_ptr_data),
                ctypes.c_int64(len(csr.indptr)),
                ctypes.c_int64(len(csr.data)),
                ctypes.c_int64(csr.shape[1]),
                ctypes.c_int(predict_type),
979
                ctypes.c_int(start_iteration),
980
981
982
983
984
985
986
                ctypes.c_int(num_iteration),
                c_str(self.pred_parameter),
                ctypes.byref(out_num_preds),
                preds.ctypes.data_as(ctypes.POINTER(ctypes.c_double))))
            if n_preds != out_num_preds.value:
                raise ValueError("Wrong length for predict results")
            return preds, nrow
wxchan's avatar
wxchan committed
987

988
        def inner_predict_sparse(csr, start_iteration, num_iteration, predict_type):
989
990
991
992
993
994
995
996
997
998
999
1000
1001
            ptr_indptr, type_ptr_indptr, __ = c_int_array(csr.indptr)
            ptr_data, type_ptr_data, _ = c_float_array(csr.data)
            csr_indices = csr.indices.astype(np.int32, copy=False)
            matrix_type = C_API_MATRIX_TYPE_CSR
            if type_ptr_indptr == C_API_DTYPE_INT32:
                out_ptr_indptr = ctypes.POINTER(ctypes.c_int32)()
            else:
                out_ptr_indptr = ctypes.POINTER(ctypes.c_int64)()
            out_ptr_indices = ctypes.POINTER(ctypes.c_int32)()
            if type_ptr_data == C_API_DTYPE_FLOAT32:
                out_ptr_data = ctypes.POINTER(ctypes.c_float)()
            else:
                out_ptr_data = ctypes.POINTER(ctypes.c_double)()
1002
            out_shape = np.empty(2, dtype=np.int64)
1003
1004
1005
            _safe_call(_LIB.LGBM_BoosterPredictSparseOutput(
                self.handle,
                ptr_indptr,
1006
                ctypes.c_int(type_ptr_indptr),
1007
1008
1009
1010
1011
1012
1013
                csr_indices.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
                ptr_data,
                ctypes.c_int(type_ptr_data),
                ctypes.c_int64(len(csr.indptr)),
                ctypes.c_int64(len(csr.data)),
                ctypes.c_int64(csr.shape[1]),
                ctypes.c_int(predict_type),
1014
                ctypes.c_int(start_iteration),
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
                ctypes.c_int(num_iteration),
                c_str(self.pred_parameter),
                ctypes.c_int(matrix_type),
                out_shape.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)),
                ctypes.byref(out_ptr_indptr),
                ctypes.byref(out_ptr_indices),
                ctypes.byref(out_ptr_data)))
            matrices = self.__create_sparse_native(csr, out_shape, out_ptr_indptr, out_ptr_indices, out_ptr_data,
                                                   type_ptr_indptr, type_ptr_data, is_csr=True)
            nrow = len(csr.indptr) - 1
            return matrices, nrow

        if predict_type == C_API_PREDICT_CONTRIB:
1028
            return inner_predict_sparse(csr, start_iteration, num_iteration, predict_type)
1029
1030
1031
1032
        nrow = len(csr.indptr) - 1
        if nrow > MAX_INT32:
            sections = [0] + list(np.arange(start=MAX_INT32, stop=nrow, step=MAX_INT32)) + [nrow]
            # __get_num_preds() cannot work with nrow > MAX_INT32, so calculate overall number of predictions piecemeal
1033
            n_preds = [self.__get_num_preds(start_iteration, num_iteration, i, predict_type) for i in np.diff(sections)]
1034
            n_preds_sections = np.array([0] + n_preds, dtype=np.intp).cumsum()
1035
            preds = np.empty(sum(n_preds), dtype=np.float64)
1036
1037
            for (start_idx, end_idx), (start_idx_pred, end_idx_pred) in zip(zip(sections, sections[1:]),
                                                                            zip(n_preds_sections, n_preds_sections[1:])):
1038
                # avoid memory consumption by arrays concatenation operations
1039
                inner_predict(csr[start_idx:end_idx], start_iteration, num_iteration, predict_type, preds[start_idx_pred:end_idx_pred])
1040
1041
            return preds, nrow
        else:
1042
            return inner_predict(csr, start_iteration, num_iteration, predict_type)
Guolin Ke's avatar
Guolin Ke committed
1043

1044
    def __pred_for_csc(self, csc, start_iteration, num_iteration, predict_type):
1045
        """Predict for a CSC data."""
1046
        def inner_predict_sparse(csc, start_iteration, num_iteration, predict_type):
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
            ptr_indptr, type_ptr_indptr, __ = c_int_array(csc.indptr)
            ptr_data, type_ptr_data, _ = c_float_array(csc.data)
            csc_indices = csc.indices.astype(np.int32, copy=False)
            matrix_type = C_API_MATRIX_TYPE_CSC
            if type_ptr_indptr == C_API_DTYPE_INT32:
                out_ptr_indptr = ctypes.POINTER(ctypes.c_int32)()
            else:
                out_ptr_indptr = ctypes.POINTER(ctypes.c_int64)()
            out_ptr_indices = ctypes.POINTER(ctypes.c_int32)()
            if type_ptr_data == C_API_DTYPE_FLOAT32:
                out_ptr_data = ctypes.POINTER(ctypes.c_float)()
            else:
                out_ptr_data = ctypes.POINTER(ctypes.c_double)()
1060
            out_shape = np.empty(2, dtype=np.int64)
1061
1062
1063
            _safe_call(_LIB.LGBM_BoosterPredictSparseOutput(
                self.handle,
                ptr_indptr,
1064
                ctypes.c_int(type_ptr_indptr),
1065
1066
1067
1068
1069
1070
1071
                csc_indices.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
                ptr_data,
                ctypes.c_int(type_ptr_data),
                ctypes.c_int64(len(csc.indptr)),
                ctypes.c_int64(len(csc.data)),
                ctypes.c_int64(csc.shape[0]),
                ctypes.c_int(predict_type),
1072
                ctypes.c_int(start_iteration),
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
                ctypes.c_int(num_iteration),
                c_str(self.pred_parameter),
                ctypes.c_int(matrix_type),
                out_shape.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)),
                ctypes.byref(out_ptr_indptr),
                ctypes.byref(out_ptr_indices),
                ctypes.byref(out_ptr_data)))
            matrices = self.__create_sparse_native(csc, out_shape, out_ptr_indptr, out_ptr_indices, out_ptr_data,
                                                   type_ptr_indptr, type_ptr_data, is_csr=False)
            nrow = csc.shape[0]
            return matrices, nrow

Guolin Ke's avatar
Guolin Ke committed
1085
        nrow = csc.shape[0]
1086
        if nrow > MAX_INT32:
1087
            return self.__pred_for_csr(csc.tocsr(), start_iteration, num_iteration, predict_type)
1088
        if predict_type == C_API_PREDICT_CONTRIB:
1089
1090
            return inner_predict_sparse(csc, start_iteration, num_iteration, predict_type)
        n_preds = self.__get_num_preds(start_iteration, num_iteration, nrow, predict_type)
1091
        preds = np.empty(n_preds, dtype=np.float64)
Guolin Ke's avatar
Guolin Ke committed
1092
1093
        out_num_preds = ctypes.c_int64(0)

1094
1095
        ptr_indptr, type_ptr_indptr, __ = c_int_array(csc.indptr)
        ptr_data, type_ptr_data, _ = c_float_array(csc.data)
Guolin Ke's avatar
Guolin Ke committed
1096

1097
        assert csc.shape[0] <= MAX_INT32
1098
        csc_indices = csc.indices.astype(np.int32, copy=False)
1099

Guolin Ke's avatar
Guolin Ke committed
1100
1101
1102
        _safe_call(_LIB.LGBM_BoosterPredictForCSC(
            self.handle,
            ptr_indptr,
1103
            ctypes.c_int(type_ptr_indptr),
1104
            csc_indices.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
Guolin Ke's avatar
Guolin Ke committed
1105
            ptr_data,
Guolin Ke's avatar
Guolin Ke committed
1106
1107
1108
1109
1110
            ctypes.c_int(type_ptr_data),
            ctypes.c_int64(len(csc.indptr)),
            ctypes.c_int64(len(csc.data)),
            ctypes.c_int64(csc.shape[0]),
            ctypes.c_int(predict_type),
1111
            ctypes.c_int(start_iteration),
Guolin Ke's avatar
Guolin Ke committed
1112
            ctypes.c_int(num_iteration),
1113
            c_str(self.pred_parameter),
Guolin Ke's avatar
Guolin Ke committed
1114
            ctypes.byref(out_num_preds),
wxchan's avatar
wxchan committed
1115
            preds.ctypes.data_as(ctypes.POINTER(ctypes.c_double))))
wxchan's avatar
wxchan committed
1116
        if n_preds != out_num_preds.value:
1117
            raise ValueError("Wrong length for predict results")
wxchan's avatar
wxchan committed
1118
1119
        return preds, nrow

1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
    def current_iteration(self):
        """Get the index of the current iteration.

        Returns
        -------
        cur_iter : int
            The index of the current iteration.
        """
        out_cur_iter = ctypes.c_int(0)
        _safe_call(_LIB.LGBM_BoosterGetCurrentIteration(
            self.handle,
            ctypes.byref(out_cur_iter)))
        return out_cur_iter.value

wxchan's avatar
wxchan committed
1134

1135
class Dataset:
wxchan's avatar
wxchan committed
1136
    """Dataset in LightGBM."""
1137

1138
    def __init__(self, data, label=None, reference=None,
1139
                 weight=None, group=None, init_score=None,
1140
                 feature_name='auto', categorical_feature='auto', params=None,
wxchan's avatar
wxchan committed
1141
                 free_raw_data=True):
1142
        """Initialize Dataset.
1143

wxchan's avatar
wxchan committed
1144
1145
        Parameters
        ----------
1146
        data : str, pathlib.Path, numpy array, pandas DataFrame, H2O DataTable's Frame, scipy.sparse, Sequence, list of Sequence or list of numpy array
wxchan's avatar
wxchan committed
1147
            Data source of Dataset.
1148
            If str or pathlib.Path, it represents the path to a text file (CSV, TSV, or LibSVM) or a LightGBM Dataset binary file.
1149
        label : list, numpy 1-D array, pandas Series / one-column DataFrame or None, optional (default=None)
1150
1151
1152
            Label of the data.
        reference : Dataset or None, optional (default=None)
            If this is Dataset for validation, training data should be used as reference.
1153
        weight : list, numpy 1-D array, pandas Series or None, optional (default=None)
1154
            Weight for each instance. Weights should be non-negative.
1155
        group : list, numpy 1-D array, pandas Series or None, optional (default=None)
1156
1157
1158
            Group/query data.
            Only used in the learning-to-rank task.
            sum(group) = n_samples.
1159
1160
            For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
            where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
1161
        init_score : list, list of lists (for multi-class task), numpy array, pandas Series, pandas DataFrame (for multi-class task), or None, optional (default=None)
1162
            Init score for Dataset.
1163
        feature_name : list of str, or 'auto', optional (default="auto")
1164
1165
            Feature names.
            If 'auto' and data is pandas DataFrame, data columns names are used.
1166
        categorical_feature : list of str or int, or 'auto', optional (default="auto")
1167
1168
            Categorical features.
            If list of int, interpreted as indices.
1169
            If list of str, interpreted as feature names (need to specify ``feature_name`` as well).
1170
            If 'auto' and data is pandas DataFrame, pandas unordered categorical columns are used.
1171
            All values in categorical features will be cast to int32 and thus should be less than int32 max value (2147483647).
1172
            Large values could be memory consuming. Consider using consecutive integers starting from zero.
1173
            All negative values in categorical features will be treated as missing values.
1174
            The output cannot be monotonically constrained with respect to a categorical feature.
1175
            Floating point numbers in categorical features will be rounded towards 0.
Nikita Titov's avatar
Nikita Titov committed
1176
        params : dict or None, optional (default=None)
1177
            Other parameters for Dataset.
Nikita Titov's avatar
Nikita Titov committed
1178
        free_raw_data : bool, optional (default=True)
1179
            If True, raw data is freed after constructing inner Dataset.
wxchan's avatar
wxchan committed
1180
        """
wxchan's avatar
wxchan committed
1181
1182
1183
1184
1185
1186
        self.handle = None
        self.data = data
        self.label = label
        self.reference = reference
        self.weight = weight
        self.group = group
1187
        self.init_score = init_score
wxchan's avatar
wxchan committed
1188
        self.feature_name = feature_name
1189
        self.categorical_feature = categorical_feature
1190
        self.params = deepcopy(params)
wxchan's avatar
wxchan committed
1191
1192
        self.free_raw_data = free_raw_data
        self.used_indices = None
1193
        self.need_slice = True
wxchan's avatar
wxchan committed
1194
        self._predictor = None
1195
        self.pandas_categorical = None
1196
        self.params_back_up = None
1197
1198
        self.feature_penalty = None
        self.monotone_constraints = None
1199
        self.version = 0
1200
        self._start_row = 0  # Used when pushing rows one by one.
wxchan's avatar
wxchan committed
1201
1202

    def __del__(self):
1203
1204
1205
1206
        try:
            self._free_handle()
        except AttributeError:
            pass
1207

1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
    def _create_sample_indices(self, total_nrow: int) -> np.ndarray:
        """Get an array of randomly chosen indices from this ``Dataset``.

        Indices are sampled without replacement.

        Parameters
        ----------
        total_nrow : int
            Total number of rows to sample from.
            If this value is greater than the value of parameter ``bin_construct_sample_cnt``, only ``bin_construct_sample_cnt`` indices will be used.
            If Dataset has multiple input data, this should be the sum of rows of every file.

        Returns
        -------
        indices : numpy array
            Indices for sampled data.
        """
        param_str = param_dict_to_str(self.get_params())
        sample_cnt = _get_sample_count(total_nrow, param_str)
        indices = np.empty(sample_cnt, dtype=np.int32)
        ptr_data, _, _ = c_int_array(indices)
        actual_sample_cnt = ctypes.c_int32(0)

        _safe_call(_LIB.LGBM_SampleIndices(
            ctypes.c_int32(total_nrow),
            c_str(param_str),
            ptr_data,
            ctypes.byref(actual_sample_cnt),
        ))
1237
1238
        assert sample_cnt == actual_sample_cnt.value
        return indices
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273

    def _init_from_ref_dataset(self, total_nrow: int, ref_dataset: 'Dataset') -> 'Dataset':
        """Create dataset from a reference dataset.

        Parameters
        ----------
        total_nrow : int
            Number of rows expected to add to dataset.
        ref_dataset : Dataset
            Reference dataset to extract meta from.

        Returns
        -------
        self : Dataset
            Constructed Dataset object.
        """
        self.handle = ctypes.c_void_p()
        _safe_call(_LIB.LGBM_DatasetCreateByReference(
            ref_dataset,
            ctypes.c_int64(total_nrow),
            ctypes.byref(self.handle),
        ))
        return self

    def _init_from_sample(
        self,
        sample_data: List[np.ndarray],
        sample_indices: List[np.ndarray],
        sample_cnt: int,
        total_nrow: int,
    ) -> "Dataset":
        """Create Dataset from sampled data structures.

        Parameters
        ----------
1274
        sample_data : list of numpy array
1275
            Sample data for each column.
1276
        sample_indices : list of numpy array
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
            Sample data row index for each column.
        sample_cnt : int
            Number of samples.
        total_nrow : int
            Total number of rows for all input files.

        Returns
        -------
        self : Dataset
            Constructed Dataset object.
        """
        ncol = len(sample_indices)
        assert len(sample_data) == ncol, "#sample data column != #column indices"

        for i in range(ncol):
            if sample_data[i].dtype != np.double:
                raise ValueError(f"sample_data[{i}] type {sample_data[i].dtype} is not double")
            if sample_indices[i].dtype != np.int32:
                raise ValueError(f"sample_indices[{i}] type {sample_indices[i].dtype} is not int32")

        # c type: double**
        # each double* element points to start of each column of sample data.
        sample_col_ptr = (ctypes.POINTER(ctypes.c_double) * ncol)()
        # c type int**
        # each int* points to start of indices for each column
        indices_col_ptr = (ctypes.POINTER(ctypes.c_int32) * ncol)()
        for i in range(ncol):
            sample_col_ptr[i] = c_float_array(sample_data[i])[0]
            indices_col_ptr[i] = c_int_array(sample_indices[i])[0]

        num_per_col = np.array([len(d) for d in sample_indices], dtype=np.int32)
        num_per_col_ptr, _, _ = c_int_array(num_per_col)

        self.handle = ctypes.c_void_p()
        params_str = param_dict_to_str(self.get_params())
        _safe_call(_LIB.LGBM_DatasetCreateFromSampledColumn(
            ctypes.cast(sample_col_ptr, ctypes.POINTER(ctypes.POINTER(ctypes.c_double))),
            ctypes.cast(indices_col_ptr, ctypes.POINTER(ctypes.POINTER(ctypes.c_int32))),
            ctypes.c_int32(ncol),
            num_per_col_ptr,
            ctypes.c_int32(sample_cnt),
            ctypes.c_int32(total_nrow),
            c_str(params_str),
            ctypes.byref(self.handle),
        ))
        return self

    def _push_rows(self, data: np.ndarray) -> 'Dataset':
        """Add rows to Dataset.

        Parameters
        ----------
        data : numpy 1-D array
            New data to add to the Dataset.

        Returns
        -------
        self : Dataset
            Dataset object.
        """
        nrow, ncol = data.shape
        data = data.reshape(data.size)
        data_ptr, data_type, _ = c_float_array(data)

        _safe_call(_LIB.LGBM_DatasetPushRows(
            self.handle,
            data_ptr,
            data_type,
            ctypes.c_int32(nrow),
            ctypes.c_int32(ncol),
            ctypes.c_int32(self._start_row),
        ))
        self._start_row += nrow
        return self

1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
    def get_params(self):
        """Get the used parameters in the Dataset.

        Returns
        -------
        params : dict or None
            The used parameters in this Dataset object.
        """
        if self.params is not None:
            # no min_data, nthreads and verbose in this function
            dataset_params = _ConfigAliases.get("bin_construct_sample_cnt",
                                                "categorical_feature",
                                                "data_random_seed",
                                                "enable_bundle",
                                                "feature_pre_filter",
                                                "forcedbins_filename",
                                                "group_column",
                                                "header",
                                                "ignore_column",
                                                "is_enable_sparse",
                                                "label_column",
1373
                                                "linear_tree",
1374
1375
1376
1377
                                                "max_bin",
                                                "max_bin_by_feature",
                                                "min_data_in_bin",
                                                "pre_partition",
Nikita Titov's avatar
Nikita Titov committed
1378
                                                "precise_float_parser",
1379
1380
1381
1382
1383
1384
                                                "two_round",
                                                "use_missing",
                                                "weight_column",
                                                "zero_as_missing")
            return {k: v for k, v in self.params.items() if k in dataset_params}

1385
    def _free_handle(self):
1386
        if self.handle is not None:
1387
            _safe_call(_LIB.LGBM_DatasetFree(self.handle))
1388
            self.handle = None
Guolin Ke's avatar
Guolin Ke committed
1389
1390
1391
        self.need_slice = True
        if self.used_indices is not None:
            self.data = None
Nikita Titov's avatar
Nikita Titov committed
1392
        return self
wxchan's avatar
wxchan committed
1393

Guolin Ke's avatar
Guolin Ke committed
1394
1395
    def _set_init_score_by_predictor(self, predictor, data, used_indices=None):
        data_has_header = False
1396
        if isinstance(data, (str, Path)):
Guolin Ke's avatar
Guolin Ke committed
1397
            # check data has header or not
1398
            data_has_header = any(self.params.get(alias, False) for alias in _ConfigAliases.get("header"))
Guolin Ke's avatar
Guolin Ke committed
1399
        num_data = self.num_data()
1400
1401
1402
        if predictor is not None:
            init_score = predictor.predict(data,
                                           raw_score=True,
1403
1404
                                           data_has_header=data_has_header)
            init_score = init_score.ravel()
1405
1406
            if used_indices is not None:
                assert not self.need_slice
1407
                if isinstance(data, (str, Path)):
1408
                    sub_init_score = np.empty(num_data * predictor.num_class, dtype=np.float64)
1409
                    assert num_data == len(used_indices)
1410
1411
                    for i in range(len(used_indices)):
                        for j in range(predictor.num_class):
1412
1413
1414
1415
                            sub_init_score[i * predictor.num_class + j] = init_score[used_indices[i] * predictor.num_class + j]
                    init_score = sub_init_score
            if predictor.num_class > 1:
                # need to regroup init_score
1416
                new_init_score = np.empty(init_score.size, dtype=np.float64)
1417
1418
                for i in range(num_data):
                    for j in range(predictor.num_class):
1419
1420
1421
                        new_init_score[j * num_data + i] = init_score[i * predictor.num_class + j]
                init_score = new_init_score
        elif self.init_score is not None:
1422
            init_score = np.zeros(self.init_score.shape, dtype=np.float64)
1423
1424
        else:
            return self
Guolin Ke's avatar
Guolin Ke committed
1425
1426
        self.set_init_score(init_score)

1427
    def _lazy_init(self, data, label=None, reference=None,
1428
                   weight=None, group=None, init_score=None, predictor=None,
1429
                   feature_name='auto', categorical_feature='auto', params=None):
wxchan's avatar
wxchan committed
1430
1431
        if data is None:
            self.handle = None
Nikita Titov's avatar
Nikita Titov committed
1432
            return self
Guolin Ke's avatar
Guolin Ke committed
1433
1434
1435
        if reference is not None:
            self.pandas_categorical = reference.pandas_categorical
            categorical_feature = reference.categorical_feature
1436
1437
1438
1439
        data, feature_name, categorical_feature, self.pandas_categorical = _data_from_pandas(data,
                                                                                             feature_name,
                                                                                             categorical_feature,
                                                                                             self.pandas_categorical)
wxchan's avatar
wxchan committed
1440
        label = _label_from_pandas(label)
Guolin Ke's avatar
Guolin Ke committed
1441

1442
        # process for args
wxchan's avatar
wxchan committed
1443
        params = {} if params is None else params
1444
1445
1446
        args_names = (getattr(self.__class__, '_lazy_init')
                      .__code__
                      .co_varnames[:getattr(self.__class__, '_lazy_init').__code__.co_argcount])
1447
        for key in params.keys():
1448
            if key in args_names:
1449
1450
                _log_warning(f'{key} keyword has been found in `params` and will be ignored.\n'
                             f'Please use {key} argument of the Dataset constructor to pass this parameter.')
1451
        # get categorical features
1452
1453
1454
1455
1456
1457
        if categorical_feature is not None:
            categorical_indices = set()
            feature_dict = {}
            if feature_name is not None:
                feature_dict = {name: i for i, name in enumerate(feature_name)}
            for name in categorical_feature:
1458
                if isinstance(name, str) and name in feature_dict:
1459
                    categorical_indices.add(feature_dict[name])
1460
                elif isinstance(name, int):
1461
1462
                    categorical_indices.add(name)
                else:
1463
                    raise TypeError(f"Wrong type({type(name).__name__}) or unknown name({name}) in categorical_feature")
1464
            if categorical_indices:
1465
1466
                for cat_alias in _ConfigAliases.get("categorical_feature"):
                    if cat_alias in params:
1467
1468
1469
                        # If the params[cat_alias] is equal to categorical_indices, do not report the warning.
                        if not(isinstance(params[cat_alias], list) and set(params[cat_alias]) == categorical_indices):
                            _log_warning(f'{cat_alias} in param dict is overridden.')
1470
                        params.pop(cat_alias, None)
1471
                params['categorical_column'] = sorted(categorical_indices)
1472

wxchan's avatar
wxchan committed
1473
        params_str = param_dict_to_str(params)
1474
        self.params = params
1475
        # process for reference dataset
wxchan's avatar
wxchan committed
1476
        ref_dataset = None
wxchan's avatar
wxchan committed
1477
        if isinstance(reference, Dataset):
1478
            ref_dataset = reference.construct().handle
wxchan's avatar
wxchan committed
1479
1480
        elif reference is not None:
            raise TypeError('Reference dataset should be None or dataset instance')
1481
        # start construct data
1482
        if isinstance(data, (str, Path)):
wxchan's avatar
wxchan committed
1483
1484
            self.handle = ctypes.c_void_p()
            _safe_call(_LIB.LGBM_DatasetCreateFromFile(
1485
                c_str(str(data)),
wxchan's avatar
wxchan committed
1486
1487
1488
1489
1490
                c_str(params_str),
                ref_dataset,
                ctypes.byref(self.handle)))
        elif isinstance(data, scipy.sparse.csr_matrix):
            self.__init_from_csr(data, params_str, ref_dataset)
Guolin Ke's avatar
Guolin Ke committed
1491
1492
        elif isinstance(data, scipy.sparse.csc_matrix):
            self.__init_from_csc(data, params_str, ref_dataset)
wxchan's avatar
wxchan committed
1493
1494
        elif isinstance(data, np.ndarray):
            self.__init_from_np2d(data, params_str, ref_dataset)
1495
1496
1497
1498
1499
1500
1501
1502
1503
        elif isinstance(data, list) and len(data) > 0:
            if all(isinstance(x, np.ndarray) for x in data):
                self.__init_from_list_np2d(data, params_str, ref_dataset)
            elif all(isinstance(x, Sequence) for x in data):
                self.__init_from_seqs(data, ref_dataset)
            else:
                raise TypeError('Data list can only be of ndarray or Sequence')
        elif isinstance(data, Sequence):
            self.__init_from_seqs([data], ref_dataset)
1504
        elif isinstance(data, dt_DataTable):
1505
            self.__init_from_np2d(data.to_numpy(), params_str, ref_dataset)
wxchan's avatar
wxchan committed
1506
1507
1508
1509
        else:
            try:
                csr = scipy.sparse.csr_matrix(data)
                self.__init_from_csr(csr, params_str, ref_dataset)
1510
            except BaseException:
1511
                raise TypeError(f'Cannot initialize Dataset from {type(data).__name__}')
wxchan's avatar
wxchan committed
1512
1513
1514
        if label is not None:
            self.set_label(label)
        if self.get_label() is None:
1515
            raise ValueError("Label should not be None")
wxchan's avatar
wxchan committed
1516
1517
1518
1519
        if weight is not None:
            self.set_weight(weight)
        if group is not None:
            self.set_group(group)
1520
1521
        if isinstance(predictor, _InnerPredictor):
            if self._predictor is None and init_score is not None:
1522
                _log_warning("The init_score will be overridden by the prediction of init_model.")
Guolin Ke's avatar
Guolin Ke committed
1523
            self._set_init_score_by_predictor(predictor, data)
1524
1525
        elif init_score is not None:
            self.set_init_score(init_score)
Guolin Ke's avatar
Guolin Ke committed
1526
        elif predictor is not None:
1527
            raise TypeError(f'Wrong predictor type {type(predictor).__name__}')
Guolin Ke's avatar
Guolin Ke committed
1528
        # set feature names
Nikita Titov's avatar
Nikita Titov committed
1529
        return self.set_feature_name(feature_name)
wxchan's avatar
wxchan committed
1530

1531
1532
    @staticmethod
    def _yield_row_from_seqlist(seqs: List[Sequence], indices: Iterable[int]):
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
        offset = 0
        seq_id = 0
        seq = seqs[seq_id]
        for row_id in indices:
            assert row_id >= offset, "sample indices are expected to be monotonic"
            while row_id >= offset + len(seq):
                offset += len(seq)
                seq_id += 1
                seq = seqs[seq_id]
            id_in_seq = row_id - offset
            row = seq[id_in_seq]
            yield row if row.flags['OWNDATA'] else row.copy()

    def __sample(self, seqs: List[Sequence], total_nrow: int) -> Tuple[List[np.ndarray], List[np.ndarray]]:
        """Sample data from seqs.

        Mimics behavior in c_api.cpp:LGBM_DatasetCreateFromMats()

        Returns
        -------
            sampled_rows, sampled_row_indices
        """
        indices = self._create_sample_indices(total_nrow)

        # Select sampled rows, transpose to column order.
1558
        sampled = np.array([row for row in self._yield_row_from_seqlist(seqs, indices)])
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
        sampled = sampled.T

        filtered = []
        filtered_idx = []
        sampled_row_range = np.arange(len(indices), dtype=np.int32)
        for col in sampled:
            col_predicate = (np.abs(col) > ZERO_THRESHOLD) | np.isnan(col)
            filtered_col = col[col_predicate]
            filtered_row_idx = sampled_row_range[col_predicate]

            filtered.append(filtered_col)
            filtered_idx.append(filtered_row_idx)

        return filtered, filtered_idx

    def __init_from_seqs(self, seqs: List[Sequence], ref_dataset: Optional['Dataset'] = None):
        """
        Initialize data from list of Sequence objects.

        Sequence: Generic Data Access Object
            Supports random access and access by batch if properly defined by user

        Data scheme uniformity are trusted, not checked
        """
        total_nrow = sum(len(seq) for seq in seqs)

        # create validation dataset from ref_dataset
        if ref_dataset is not None:
            self._init_from_ref_dataset(total_nrow, ref_dataset)
        else:
            param_str = param_dict_to_str(self.get_params())
            sample_cnt = _get_sample_count(total_nrow, param_str)

            sample_data, col_indices = self.__sample(seqs, total_nrow)
            self._init_from_sample(sample_data, col_indices, sample_cnt, total_nrow)

        for seq in seqs:
            nrow = len(seq)
            batch_size = getattr(seq, 'batch_size', None) or Sequence.batch_size
            for start in range(0, nrow, batch_size):
                end = min(start + batch_size, nrow)
                self._push_rows(seq[start:end])
        return self

wxchan's avatar
wxchan committed
1603
    def __init_from_np2d(self, mat, params_str, ref_dataset):
1604
        """Initialize data from a 2-D numpy matrix."""
wxchan's avatar
wxchan committed
1605
1606
1607
1608
1609
1610
        if len(mat.shape) != 2:
            raise ValueError('Input numpy.ndarray must be 2 dimensional')

        self.handle = ctypes.c_void_p()
        if mat.dtype == np.float32 or mat.dtype == np.float64:
            data = np.array(mat.reshape(mat.size), dtype=mat.dtype, copy=False)
1611
        else:  # change non-float data to float data, need to copy
wxchan's avatar
wxchan committed
1612
1613
            data = np.array(mat.reshape(mat.size), dtype=np.float32)

1614
        ptr_data, type_ptr_data, _ = c_float_array(data)
wxchan's avatar
wxchan committed
1615
1616
        _safe_call(_LIB.LGBM_DatasetCreateFromMat(
            ptr_data,
Guolin Ke's avatar
Guolin Ke committed
1617
            ctypes.c_int(type_ptr_data),
1618
1619
            ctypes.c_int32(mat.shape[0]),
            ctypes.c_int32(mat.shape[1]),
Guolin Ke's avatar
Guolin Ke committed
1620
            ctypes.c_int(C_API_IS_ROW_MAJOR),
wxchan's avatar
wxchan committed
1621
1622
1623
            c_str(params_str),
            ref_dataset,
            ctypes.byref(self.handle)))
Nikita Titov's avatar
Nikita Titov committed
1624
        return self
wxchan's avatar
wxchan committed
1625

1626
    def __init_from_list_np2d(self, mats, params_str, ref_dataset):
1627
        """Initialize data from a list of 2-D numpy matrices."""
1628
        ncol = mats[0].shape[1]
1629
        nrow = np.empty((len(mats),), np.int32)
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
        if mats[0].dtype == np.float64:
            ptr_data = (ctypes.POINTER(ctypes.c_double) * len(mats))()
        else:
            ptr_data = (ctypes.POINTER(ctypes.c_float) * len(mats))()

        holders = []
        type_ptr_data = None

        for i, mat in enumerate(mats):
            if len(mat.shape) != 2:
                raise ValueError('Input numpy.ndarray must be 2 dimensional')

            if mat.shape[1] != ncol:
                raise ValueError('Input arrays must have same number of columns')

            nrow[i] = mat.shape[0]

            if mat.dtype == np.float32 or mat.dtype == np.float64:
                mats[i] = np.array(mat.reshape(mat.size), dtype=mat.dtype, copy=False)
1649
            else:  # change non-float data to float data, need to copy
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
                mats[i] = np.array(mat.reshape(mat.size), dtype=np.float32)

            chunk_ptr_data, chunk_type_ptr_data, holder = c_float_array(mats[i])
            if type_ptr_data is not None and chunk_type_ptr_data != type_ptr_data:
                raise ValueError('Input chunks must have same type')
            ptr_data[i] = chunk_ptr_data
            type_ptr_data = chunk_type_ptr_data
            holders.append(holder)

        self.handle = ctypes.c_void_p()
        _safe_call(_LIB.LGBM_DatasetCreateFromMats(
1661
            ctypes.c_int32(len(mats)),
1662
1663
1664
            ctypes.cast(ptr_data, ctypes.POINTER(ctypes.POINTER(ctypes.c_double))),
            ctypes.c_int(type_ptr_data),
            nrow.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
1665
            ctypes.c_int32(ncol),
1666
1667
1668
1669
            ctypes.c_int(C_API_IS_ROW_MAJOR),
            c_str(params_str),
            ref_dataset,
            ctypes.byref(self.handle)))
Nikita Titov's avatar
Nikita Titov committed
1670
        return self
1671

wxchan's avatar
wxchan committed
1672
    def __init_from_csr(self, csr, params_str, ref_dataset):
1673
        """Initialize data from a CSR matrix."""
wxchan's avatar
wxchan committed
1674
        if len(csr.indices) != len(csr.data):
1675
            raise ValueError(f'Length mismatch: {len(csr.indices)} vs {len(csr.data)}')
wxchan's avatar
wxchan committed
1676
1677
        self.handle = ctypes.c_void_p()

1678
1679
        ptr_indptr, type_ptr_indptr, __ = c_int_array(csr.indptr)
        ptr_data, type_ptr_data, _ = c_float_array(csr.data)
wxchan's avatar
wxchan committed
1680

1681
        assert csr.shape[1] <= MAX_INT32
1682
        csr_indices = csr.indices.astype(np.int32, copy=False)
1683

wxchan's avatar
wxchan committed
1684
1685
        _safe_call(_LIB.LGBM_DatasetCreateFromCSR(
            ptr_indptr,
Guolin Ke's avatar
Guolin Ke committed
1686
            ctypes.c_int(type_ptr_indptr),
1687
            csr_indices.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
wxchan's avatar
wxchan committed
1688
            ptr_data,
Guolin Ke's avatar
Guolin Ke committed
1689
1690
1691
1692
            ctypes.c_int(type_ptr_data),
            ctypes.c_int64(len(csr.indptr)),
            ctypes.c_int64(len(csr.data)),
            ctypes.c_int64(csr.shape[1]),
wxchan's avatar
wxchan committed
1693
1694
1695
            c_str(params_str),
            ref_dataset,
            ctypes.byref(self.handle)))
Nikita Titov's avatar
Nikita Titov committed
1696
        return self
wxchan's avatar
wxchan committed
1697

Guolin Ke's avatar
Guolin Ke committed
1698
    def __init_from_csc(self, csc, params_str, ref_dataset):
1699
        """Initialize data from a CSC matrix."""
Guolin Ke's avatar
Guolin Ke committed
1700
        if len(csc.indices) != len(csc.data):
1701
            raise ValueError(f'Length mismatch: {len(csc.indices)} vs {len(csc.data)}')
Guolin Ke's avatar
Guolin Ke committed
1702
1703
        self.handle = ctypes.c_void_p()

1704
1705
        ptr_indptr, type_ptr_indptr, __ = c_int_array(csc.indptr)
        ptr_data, type_ptr_data, _ = c_float_array(csc.data)
Guolin Ke's avatar
Guolin Ke committed
1706

1707
        assert csc.shape[0] <= MAX_INT32
1708
        csc_indices = csc.indices.astype(np.int32, copy=False)
1709

Guolin Ke's avatar
Guolin Ke committed
1710
1711
        _safe_call(_LIB.LGBM_DatasetCreateFromCSC(
            ptr_indptr,
Guolin Ke's avatar
Guolin Ke committed
1712
            ctypes.c_int(type_ptr_indptr),
1713
            csc_indices.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
Guolin Ke's avatar
Guolin Ke committed
1714
            ptr_data,
Guolin Ke's avatar
Guolin Ke committed
1715
1716
1717
1718
            ctypes.c_int(type_ptr_data),
            ctypes.c_int64(len(csc.indptr)),
            ctypes.c_int64(len(csc.data)),
            ctypes.c_int64(csc.shape[0]),
Guolin Ke's avatar
Guolin Ke committed
1719
1720
1721
            c_str(params_str),
            ref_dataset,
            ctypes.byref(self.handle)))
Nikita Titov's avatar
Nikita Titov committed
1722
        return self
Guolin Ke's avatar
Guolin Ke committed
1723

1724
    @staticmethod
1725
1726
1727
1728
1729
1730
    def _compare_params_for_warning(
        params: Optional[Dict[str, Any]],
        other_params: Optional[Dict[str, Any]],
        ignore_keys: Set[str]
    ) -> bool:
        """Compare two dictionaries with params ignoring some keys.
1731

1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
        It is only for the warning purpose.

        Parameters
        ----------
        params : dict or None
            One dictionary with parameters to compare.
        other_params : dict or None
            Another dictionary with parameters to compare.
        ignore_keys : set
            Keys that should be ignored during comparing two dictionaries.
1742
1743
1744

        Returns
        -------
1745
1746
        compare_result : bool
          Returns whether two dictionaries with params are equal.
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
        """
        if params is None:
            params = {}
        if other_params is None:
            other_params = {}
        for k in other_params:
            if k not in ignore_keys:
                if k not in params or params[k] != other_params[k]:
                    return False
        for k in params:
            if k not in ignore_keys:
                if k not in other_params or params[k] != other_params[k]:
                    return False
        return True

wxchan's avatar
wxchan committed
1762
    def construct(self):
1763
1764
1765
1766
1767
        """Lazy init.

        Returns
        -------
        self : Dataset
Nikita Titov's avatar
Nikita Titov committed
1768
            Constructed Dataset object.
1769
        """
1770
        if self.handle is None:
wxchan's avatar
wxchan committed
1771
            if self.reference is not None:
1772
                reference_params = self.reference.get_params()
1773
1774
                params = self.get_params()
                if params != reference_params:
1775
1776
1777
1778
1779
                    if not self._compare_params_for_warning(
                        params=params,
                        other_params=reference_params,
                        ignore_keys=_ConfigAliases.get("categorical_feature")
                    ):
1780
                        _log_warning('Overriding the parameters from Reference Dataset.')
1781
                    self._update_params(reference_params)
wxchan's avatar
wxchan committed
1782
                if self.used_indices is None:
1783
                    # create valid
1784
                    self._lazy_init(self.data, label=self.label, reference=self.reference,
1785
1786
                                    weight=self.weight, group=self.group,
                                    init_score=self.init_score, predictor=self._predictor,
1787
                                    feature_name=self.feature_name, params=self.params)
wxchan's avatar
wxchan committed
1788
                else:
1789
                    # construct subset
wxchan's avatar
wxchan committed
1790
                    used_indices = list_to_1d_numpy(self.used_indices, np.int32, name='used_indices')
1791
                    assert used_indices.flags.c_contiguous
Guolin Ke's avatar
Guolin Ke committed
1792
                    if self.reference.group is not None:
1793
                        group_info = np.array(self.reference.group).astype(np.int32, copy=False)
1794
                        _, self.group = np.unique(np.repeat(range(len(group_info)), repeats=group_info)[self.used_indices],
1795
                                                  return_counts=True)
1796
                    self.handle = ctypes.c_void_p()
wxchan's avatar
wxchan committed
1797
1798
                    params_str = param_dict_to_str(self.params)
                    _safe_call(_LIB.LGBM_DatasetGetSubset(
1799
                        self.reference.construct().handle,
wxchan's avatar
wxchan committed
1800
                        used_indices.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
1801
                        ctypes.c_int32(used_indices.shape[0]),
wxchan's avatar
wxchan committed
1802
1803
                        c_str(params_str),
                        ctypes.byref(self.handle)))
Guolin Ke's avatar
Guolin Ke committed
1804
1805
                    if not self.free_raw_data:
                        self.get_data()
Guolin Ke's avatar
Guolin Ke committed
1806
1807
                    if self.group is not None:
                        self.set_group(self.group)
wxchan's avatar
wxchan committed
1808
1809
                    if self.get_label() is None:
                        raise ValueError("Label should not be None.")
Guolin Ke's avatar
Guolin Ke committed
1810
1811
1812
                    if isinstance(self._predictor, _InnerPredictor) and self._predictor is not self.reference._predictor:
                        self.get_data()
                        self._set_init_score_by_predictor(self._predictor, self.data, used_indices)
wxchan's avatar
wxchan committed
1813
            else:
1814
                # create train
1815
                self._lazy_init(self.data, label=self.label,
1816
1817
                                weight=self.weight, group=self.group,
                                init_score=self.init_score, predictor=self._predictor,
1818
                                feature_name=self.feature_name, categorical_feature=self.categorical_feature, params=self.params)
wxchan's avatar
wxchan committed
1819
1820
1821
            if self.free_raw_data:
                self.data = None
        return self
wxchan's avatar
wxchan committed
1822

1823
    def create_valid(self, data, label=None, weight=None, group=None, init_score=None, params=None):
1824
        """Create validation data align with current Dataset.
wxchan's avatar
wxchan committed
1825
1826
1827

        Parameters
        ----------
1828
        data : str, pathlib.Path, numpy array, pandas DataFrame, H2O DataTable's Frame, scipy.sparse, Sequence, list of Sequence or list of numpy array
wxchan's avatar
wxchan committed
1829
            Data source of Dataset.
1830
            If str or pathlib.Path, it represents the path to a text file (CSV, TSV, or LibSVM) or a LightGBM Dataset binary file.
1831
        label : list, numpy 1-D array, pandas Series / one-column DataFrame or None, optional (default=None)
1832
1833
            Label of the data.
        weight : list, numpy 1-D array, pandas Series or None, optional (default=None)
1834
            Weight for each instance. Weights should be non-negative.
1835
        group : list, numpy 1-D array, pandas Series or None, optional (default=None)
1836
1837
1838
            Group/query data.
            Only used in the learning-to-rank task.
            sum(group) = n_samples.
1839
1840
            For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
            where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
1841
        init_score : list, list of lists (for multi-class task), numpy array, pandas Series, pandas DataFrame (for multi-class task), or None, optional (default=None)
1842
            Init score for Dataset.
Nikita Titov's avatar
Nikita Titov committed
1843
        params : dict or None, optional (default=None)
1844
            Other parameters for validation Dataset.
1845
1846
1847

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
1848
1849
        valid : Dataset
            Validation Dataset with reference to self.
wxchan's avatar
wxchan committed
1850
        """
1851
        ret = Dataset(data, label=label, reference=self,
1852
                      weight=weight, group=group, init_score=init_score,
1853
                      params=params, free_raw_data=self.free_raw_data)
wxchan's avatar
wxchan committed
1854
        ret._predictor = self._predictor
1855
        ret.pandas_categorical = self.pandas_categorical
wxchan's avatar
wxchan committed
1856
        return ret
wxchan's avatar
wxchan committed
1857

wxchan's avatar
wxchan committed
1858
    def subset(self, used_indices, params=None):
1859
        """Get subset of current Dataset.
wxchan's avatar
wxchan committed
1860
1861
1862
1863

        Parameters
        ----------
        used_indices : list of int
1864
            Indices used to create the subset.
Nikita Titov's avatar
Nikita Titov committed
1865
        params : dict or None, optional (default=None)
1866
            These parameters will be passed to Dataset constructor.
1867
1868
1869
1870
1871

        Returns
        -------
        subset : Dataset
            Subset of the current Dataset.
wxchan's avatar
wxchan committed
1872
        """
wxchan's avatar
wxchan committed
1873
1874
        if params is None:
            params = self.params
wxchan's avatar
wxchan committed
1875
        ret = Dataset(None, reference=self, feature_name=self.feature_name,
1876
1877
                      categorical_feature=self.categorical_feature, params=params,
                      free_raw_data=self.free_raw_data)
wxchan's avatar
wxchan committed
1878
        ret._predictor = self._predictor
1879
        ret.pandas_categorical = self.pandas_categorical
1880
        ret.used_indices = sorted(used_indices)
wxchan's avatar
wxchan committed
1881
1882
1883
        return ret

    def save_binary(self, filename):
1884
        """Save Dataset to a binary file.
wxchan's avatar
wxchan committed
1885

1886
1887
1888
1889
1890
        .. note::

            Please note that `init_score` is not saved in binary file.
            If you need it, please set it again after loading Dataset.

wxchan's avatar
wxchan committed
1891
1892
        Parameters
        ----------
1893
        filename : str or pathlib.Path
wxchan's avatar
wxchan committed
1894
            Name of the output file.
Nikita Titov's avatar
Nikita Titov committed
1895
1896
1897
1898
1899

        Returns
        -------
        self : Dataset
            Returns self.
wxchan's avatar
wxchan committed
1900
1901
1902
        """
        _safe_call(_LIB.LGBM_DatasetSaveBinary(
            self.construct().handle,
1903
            c_str(str(filename))))
Nikita Titov's avatar
Nikita Titov committed
1904
        return self
wxchan's avatar
wxchan committed
1905
1906

    def _update_params(self, params):
1907
1908
        if not params:
            return self
1909
        params = deepcopy(params)
1910
1911
1912
1913
1914

        def update():
            if not self.params:
                self.params = params
            else:
1915
                self.params_back_up = deepcopy(self.params)
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
                self.params.update(params)

        if self.handle is None:
            update()
        elif params is not None:
            ret = _LIB.LGBM_DatasetUpdateParamChecking(
                c_str(param_dict_to_str(self.params)),
                c_str(param_dict_to_str(params)))
            if ret != 0:
                # could be updated if data is not freed
                if self.data is not None:
                    update()
                    self._free_handle()
                else:
1930
                    raise LightGBMError(_LIB.LGBM_GetLastError().decode('utf-8'))
Nikita Titov's avatar
Nikita Titov committed
1931
        return self
wxchan's avatar
wxchan committed
1932

1933
    def _reverse_update_params(self):
1934
        if self.handle is None:
1935
            self.params = deepcopy(self.params_back_up)
1936
            self.params_back_up = None
Nikita Titov's avatar
Nikita Titov committed
1937
        return self
1938

wxchan's avatar
wxchan committed
1939
    def set_field(self, field_name, data):
wxchan's avatar
wxchan committed
1940
        """Set property into the Dataset.
wxchan's avatar
wxchan committed
1941
1942
1943

        Parameters
        ----------
1944
        field_name : str
1945
            The field name of the information.
1946
1947
        data : list, list of lists (for multi-class task), numpy array, pandas Series, pandas DataFrame (for multi-class task), or None
            The data to be set.
Nikita Titov's avatar
Nikita Titov committed
1948
1949
1950
1951
1952

        Returns
        -------
        self : Dataset
            Dataset with set property.
wxchan's avatar
wxchan committed
1953
        """
1954
        if self.handle is None:
1955
            raise Exception(f"Cannot set {field_name} before construct dataset")
wxchan's avatar
wxchan committed
1956
        if data is None:
1957
            # set to None
wxchan's avatar
wxchan committed
1958
1959
1960
1961
            _safe_call(_LIB.LGBM_DatasetSetField(
                self.handle,
                c_str(field_name),
                None,
Guolin Ke's avatar
Guolin Ke committed
1962
1963
                ctypes.c_int(0),
                ctypes.c_int(FIELD_TYPE_MAPPER[field_name])))
Nikita Titov's avatar
Nikita Titov committed
1964
            return self
1965
        if field_name == 'init_score':
Guolin Ke's avatar
Guolin Ke committed
1966
            dtype = np.float64
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
            if _is_1d_collection(data):
                data = list_to_1d_numpy(data, dtype, name=field_name)
            elif _is_2d_collection(data):
                data = _data_to_2d_numpy(data, dtype, name=field_name)
                data = data.ravel(order='F')
            else:
                raise TypeError(
                    'init_score must be list, numpy 1-D array or pandas Series.\n'
                    'In multiclass classification init_score can also be a list of lists, numpy 2-D array or pandas DataFrame.'
                )
        else:
            dtype = np.int32 if field_name == 'group' else np.float32
            data = list_to_1d_numpy(data, dtype, name=field_name)

1981
1982
        if data.dtype == np.float32 or data.dtype == np.float64:
            ptr_data, type_data, _ = c_float_array(data)
wxchan's avatar
wxchan committed
1983
        elif data.dtype == np.int32:
1984
            ptr_data, type_data, _ = c_int_array(data)
wxchan's avatar
wxchan committed
1985
        else:
1986
            raise TypeError(f"Expected np.float32/64 or np.int32, met type({data.dtype})")
wxchan's avatar
wxchan committed
1987
        if type_data != FIELD_TYPE_MAPPER[field_name]:
1988
            raise TypeError("Input type error for set_field")
wxchan's avatar
wxchan committed
1989
1990
1991
1992
        _safe_call(_LIB.LGBM_DatasetSetField(
            self.handle,
            c_str(field_name),
            ptr_data,
Guolin Ke's avatar
Guolin Ke committed
1993
1994
            ctypes.c_int(len(data)),
            ctypes.c_int(type_data)))
1995
        self.version += 1
Nikita Titov's avatar
Nikita Titov committed
1996
        return self
wxchan's avatar
wxchan committed
1997

wxchan's avatar
wxchan committed
1998
1999
    def get_field(self, field_name):
        """Get property from the Dataset.
wxchan's avatar
wxchan committed
2000
2001
2002

        Parameters
        ----------
2003
        field_name : str
2004
            The field name of the information.
wxchan's avatar
wxchan committed
2005
2006
2007

        Returns
        -------
2008
        info : numpy array or None
2009
            A numpy array with information from the Dataset.
Guolin Ke's avatar
Guolin Ke committed
2010
        """
2011
        if self.handle is None:
2012
            raise Exception(f"Cannot get {field_name} before construct Dataset")
2013
2014
        tmp_out_len = ctypes.c_int(0)
        out_type = ctypes.c_int(0)
wxchan's avatar
wxchan committed
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
        ret = ctypes.POINTER(ctypes.c_void_p)()
        _safe_call(_LIB.LGBM_DatasetGetField(
            self.handle,
            c_str(field_name),
            ctypes.byref(tmp_out_len),
            ctypes.byref(ret),
            ctypes.byref(out_type)))
        if out_type.value != FIELD_TYPE_MAPPER[field_name]:
            raise TypeError("Return type error for get_field")
        if tmp_out_len.value == 0:
            return None
        if out_type.value == C_API_DTYPE_INT32:
2027
            arr = cint32_array_to_numpy(ctypes.cast(ret, ctypes.POINTER(ctypes.c_int32)), tmp_out_len.value)
wxchan's avatar
wxchan committed
2028
        elif out_type.value == C_API_DTYPE_FLOAT32:
2029
            arr = cfloat32_array_to_numpy(ctypes.cast(ret, ctypes.POINTER(ctypes.c_float)), tmp_out_len.value)
Guolin Ke's avatar
Guolin Ke committed
2030
        elif out_type.value == C_API_DTYPE_FLOAT64:
2031
            arr = cfloat64_array_to_numpy(ctypes.cast(ret, ctypes.POINTER(ctypes.c_double)), tmp_out_len.value)
2032
        else:
wxchan's avatar
wxchan committed
2033
            raise TypeError("Unknown type")
2034
2035
2036
2037
2038
2039
        if field_name == 'init_score':
            num_data = self.num_data()
            num_classes = arr.size // num_data
            if num_classes > 1:
                arr = arr.reshape((num_data, num_classes), order='F')
        return arr
Guolin Ke's avatar
Guolin Ke committed
2040

2041
    def set_categorical_feature(self, categorical_feature):
2042
        """Set categorical features.
2043
2044
2045

        Parameters
        ----------
2046
        categorical_feature : list of int or str
2047
            Names or indices of categorical features.
Nikita Titov's avatar
Nikita Titov committed
2048
2049
2050
2051
2052

        Returns
        -------
        self : Dataset
            Dataset with set categorical features.
2053
2054
        """
        if self.categorical_feature == categorical_feature:
Nikita Titov's avatar
Nikita Titov committed
2055
            return self
2056
        if self.data is not None:
2057
2058
            if self.categorical_feature is None:
                self.categorical_feature = categorical_feature
Nikita Titov's avatar
Nikita Titov committed
2059
                return self._free_handle()
2060
            elif categorical_feature == 'auto':
Nikita Titov's avatar
Nikita Titov committed
2061
                return self
2062
            else:
2063
2064
2065
                if self.categorical_feature != 'auto':
                    _log_warning('categorical_feature in Dataset is overridden.\n'
                                 f'New categorical_feature is {sorted(list(categorical_feature))}')
2066
                self.categorical_feature = categorical_feature
Nikita Titov's avatar
Nikita Titov committed
2067
                return self._free_handle()
2068
        else:
2069
2070
            raise LightGBMError("Cannot set categorical feature after freed raw data, "
                                "set free_raw_data=False when construct Dataset to avoid this.")
2071

Guolin Ke's avatar
Guolin Ke committed
2072
    def _set_predictor(self, predictor):
2073
2074
2075
2076
        """Set predictor for continued training.

        It is not recommended for user to call this function.
        Please use init_model argument in engine.train() or engine.cv() instead.
Guolin Ke's avatar
Guolin Ke committed
2077
        """
2078
        if predictor is self._predictor and (predictor is None or predictor.current_iteration() == self._predictor.current_iteration()):
Nikita Titov's avatar
Nikita Titov committed
2079
            return self
2080
        if self.handle is None:
Guolin Ke's avatar
Guolin Ke committed
2081
            self._predictor = predictor
2082
2083
2084
2085
2086
2087
        elif self.data is not None:
            self._predictor = predictor
            self._set_init_score_by_predictor(self._predictor, self.data)
        elif self.used_indices is not None and self.reference is not None and self.reference.data is not None:
            self._predictor = predictor
            self._set_init_score_by_predictor(self._predictor, self.reference.data, self.used_indices)
Guolin Ke's avatar
Guolin Ke committed
2088
        else:
2089
2090
            raise LightGBMError("Cannot set predictor after freed raw data, "
                                "set free_raw_data=False when construct Dataset to avoid this.")
2091
        return self
Guolin Ke's avatar
Guolin Ke committed
2092
2093

    def set_reference(self, reference):
2094
        """Set reference Dataset.
Guolin Ke's avatar
Guolin Ke committed
2095
2096
2097
2098

        Parameters
        ----------
        reference : Dataset
2099
            Reference that is used as a template to construct the current Dataset.
Nikita Titov's avatar
Nikita Titov committed
2100
2101
2102
2103
2104

        Returns
        -------
        self : Dataset
            Dataset with set reference.
Guolin Ke's avatar
Guolin Ke committed
2105
        """
2106
2107
2108
        self.set_categorical_feature(reference.categorical_feature) \
            .set_feature_name(reference.feature_name) \
            ._set_predictor(reference._predictor)
2109
        # we're done if self and reference share a common upstream reference
2110
        if self.get_ref_chain().intersection(reference.get_ref_chain()):
Nikita Titov's avatar
Nikita Titov committed
2111
            return self
Guolin Ke's avatar
Guolin Ke committed
2112
2113
        if self.data is not None:
            self.reference = reference
Nikita Titov's avatar
Nikita Titov committed
2114
            return self._free_handle()
Guolin Ke's avatar
Guolin Ke committed
2115
        else:
2116
2117
            raise LightGBMError("Cannot set reference after freed raw data, "
                                "set free_raw_data=False when construct Dataset to avoid this.")
Guolin Ke's avatar
Guolin Ke committed
2118
2119

    def set_feature_name(self, feature_name):
2120
        """Set feature name.
Guolin Ke's avatar
Guolin Ke committed
2121
2122
2123

        Parameters
        ----------
2124
        feature_name : list of str
2125
            Feature names.
Nikita Titov's avatar
Nikita Titov committed
2126
2127
2128
2129
2130

        Returns
        -------
        self : Dataset
            Dataset with set feature name.
Guolin Ke's avatar
Guolin Ke committed
2131
        """
2132
2133
        if feature_name != 'auto':
            self.feature_name = feature_name
2134
        if self.handle is not None and feature_name is not None and feature_name != 'auto':
wxchan's avatar
wxchan committed
2135
            if len(feature_name) != self.num_feature():
2136
                raise ValueError(f"Length of feature_name({len(feature_name)}) and num_feature({self.num_feature()}) don't match")
2137
            c_feature_name = [c_str(name) for name in feature_name]
wxchan's avatar
wxchan committed
2138
2139
2140
            _safe_call(_LIB.LGBM_DatasetSetFeatureNames(
                self.handle,
                c_array(ctypes.c_char_p, c_feature_name),
Guolin Ke's avatar
Guolin Ke committed
2141
                ctypes.c_int(len(feature_name))))
Nikita Titov's avatar
Nikita Titov committed
2142
        return self
Guolin Ke's avatar
Guolin Ke committed
2143
2144

    def set_label(self, label):
2145
        """Set label of Dataset.
Guolin Ke's avatar
Guolin Ke committed
2146
2147
2148

        Parameters
        ----------
2149
        label : list, numpy 1-D array, pandas Series / one-column DataFrame or None
2150
            The label information to be set into Dataset.
Nikita Titov's avatar
Nikita Titov committed
2151
2152
2153
2154
2155

        Returns
        -------
        self : Dataset
            Dataset with set label.
Guolin Ke's avatar
Guolin Ke committed
2156
2157
        """
        self.label = label
2158
        if self.handle is not None:
2159
            label = list_to_1d_numpy(_label_from_pandas(label), name='label')
wxchan's avatar
wxchan committed
2160
            self.set_field('label', label)
2161
            self.label = self.get_field('label')  # original values can be modified at cpp side
Nikita Titov's avatar
Nikita Titov committed
2162
        return self
Guolin Ke's avatar
Guolin Ke committed
2163
2164

    def set_weight(self, weight):
2165
        """Set weight of each instance.
Guolin Ke's avatar
Guolin Ke committed
2166
2167
2168

        Parameters
        ----------
2169
        weight : list, numpy 1-D array, pandas Series or None
2170
            Weight to be set for each data point. Weights should be non-negative.
Nikita Titov's avatar
Nikita Titov committed
2171
2172
2173
2174
2175

        Returns
        -------
        self : Dataset
            Dataset with set weight.
Guolin Ke's avatar
Guolin Ke committed
2176
        """
2177
2178
        if weight is not None and np.all(weight == 1):
            weight = None
Guolin Ke's avatar
Guolin Ke committed
2179
        self.weight = weight
2180
        if self.handle is not None and weight is not None:
wxchan's avatar
wxchan committed
2181
2182
            weight = list_to_1d_numpy(weight, name='weight')
            self.set_field('weight', weight)
2183
            self.weight = self.get_field('weight')  # original values can be modified at cpp side
Nikita Titov's avatar
Nikita Titov committed
2184
        return self
Guolin Ke's avatar
Guolin Ke committed
2185
2186

    def set_init_score(self, init_score):
2187
        """Set init score of Booster to start from.
Guolin Ke's avatar
Guolin Ke committed
2188
2189
2190

        Parameters
        ----------
2191
        init_score : list, list of lists (for multi-class task), numpy array, pandas Series, pandas DataFrame (for multi-class task), or None
2192
            Init score for Booster.
Nikita Titov's avatar
Nikita Titov committed
2193
2194
2195
2196
2197

        Returns
        -------
        self : Dataset
            Dataset with set init score.
Guolin Ke's avatar
Guolin Ke committed
2198
2199
        """
        self.init_score = init_score
2200
        if self.handle is not None and init_score is not None:
wxchan's avatar
wxchan committed
2201
            self.set_field('init_score', init_score)
2202
            self.init_score = self.get_field('init_score')  # original values can be modified at cpp side
Nikita Titov's avatar
Nikita Titov committed
2203
        return self
Guolin Ke's avatar
Guolin Ke committed
2204
2205

    def set_group(self, group):
2206
        """Set group size of Dataset (used for ranking).
Guolin Ke's avatar
Guolin Ke committed
2207
2208
2209

        Parameters
        ----------
2210
        group : list, numpy 1-D array, pandas Series or None
2211
2212
2213
            Group/query data.
            Only used in the learning-to-rank task.
            sum(group) = n_samples.
2214
2215
            For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
            where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
Nikita Titov's avatar
Nikita Titov committed
2216
2217
2218
2219
2220

        Returns
        -------
        self : Dataset
            Dataset with set group.
Guolin Ke's avatar
Guolin Ke committed
2221
2222
        """
        self.group = group
2223
        if self.handle is not None and group is not None:
wxchan's avatar
wxchan committed
2224
2225
            group = list_to_1d_numpy(group, np.int32, name='group')
            self.set_field('group', group)
Nikita Titov's avatar
Nikita Titov committed
2226
        return self
Guolin Ke's avatar
Guolin Ke committed
2227

2228
2229
2230
2231
2232
    def get_feature_name(self):
        """Get the names of columns (features) in the Dataset.

        Returns
        -------
2233
        feature_names : list of str
2234
2235
2236
2237
2238
2239
2240
2241
            The names of columns (features) in the Dataset.
        """
        if self.handle is None:
            raise LightGBMError("Cannot get feature_name before construct dataset")
        num_feature = self.num_feature()
        tmp_out_len = ctypes.c_int(0)
        reserved_string_buffer_size = 255
        required_string_buffer_size = ctypes.c_size_t(0)
2242
        string_buffers = [ctypes.create_string_buffer(reserved_string_buffer_size) for _ in range(num_feature)]
2243
2244
2245
        ptr_string_buffers = (ctypes.c_char_p * num_feature)(*map(ctypes.addressof, string_buffers))
        _safe_call(_LIB.LGBM_DatasetGetFeatureNames(
            self.handle,
2246
            ctypes.c_int(num_feature),
2247
            ctypes.byref(tmp_out_len),
2248
            ctypes.c_size_t(reserved_string_buffer_size),
2249
2250
2251
2252
            ctypes.byref(required_string_buffer_size),
            ptr_string_buffers))
        if num_feature != tmp_out_len.value:
            raise ValueError("Length of feature names doesn't equal with num_feature")
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
        actual_string_buffer_size = required_string_buffer_size.value
        # if buffer length is not long enough, reallocate buffers
        if reserved_string_buffer_size < actual_string_buffer_size:
            string_buffers = [ctypes.create_string_buffer(actual_string_buffer_size) for _ in range(num_feature)]
            ptr_string_buffers = (ctypes.c_char_p * num_feature)(*map(ctypes.addressof, string_buffers))
            _safe_call(_LIB.LGBM_DatasetGetFeatureNames(
                self.handle,
                ctypes.c_int(num_feature),
                ctypes.byref(tmp_out_len),
                ctypes.c_size_t(actual_string_buffer_size),
                ctypes.byref(required_string_buffer_size),
                ptr_string_buffers))
2265
        return [string_buffers[i].value.decode('utf-8') for i in range(num_feature)]
2266

Guolin Ke's avatar
Guolin Ke committed
2267
    def get_label(self):
2268
        """Get the label of the Dataset.
Guolin Ke's avatar
Guolin Ke committed
2269
2270
2271

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
2272
        label : numpy array or None
2273
            The label information from the Dataset.
Guolin Ke's avatar
Guolin Ke committed
2274
        """
2275
        if self.label is None:
wxchan's avatar
wxchan committed
2276
            self.label = self.get_field('label')
Guolin Ke's avatar
Guolin Ke committed
2277
2278
2279
        return self.label

    def get_weight(self):
2280
        """Get the weight of the Dataset.
Guolin Ke's avatar
Guolin Ke committed
2281
2282
2283

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
2284
        weight : numpy array or None
2285
            Weight for each data point from the Dataset. Weights should be non-negative.
Guolin Ke's avatar
Guolin Ke committed
2286
        """
2287
        if self.weight is None:
wxchan's avatar
wxchan committed
2288
            self.weight = self.get_field('weight')
Guolin Ke's avatar
Guolin Ke committed
2289
2290
2291
        return self.weight

    def get_init_score(self):
2292
        """Get the initial score of the Dataset.
Guolin Ke's avatar
Guolin Ke committed
2293
2294
2295

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
2296
        init_score : numpy array or None
2297
            Init score of Booster.
Guolin Ke's avatar
Guolin Ke committed
2298
        """
2299
        if self.init_score is None:
wxchan's avatar
wxchan committed
2300
            self.init_score = self.get_field('init_score')
Guolin Ke's avatar
Guolin Ke committed
2301
2302
        return self.init_score

2303
2304
2305
2306
2307
    def get_data(self):
        """Get the raw data of the Dataset.

        Returns
        -------
2308
        data : str, pathlib.Path, numpy array, pandas DataFrame, H2O DataTable's Frame, scipy.sparse, Sequence, list of Sequence or list of numpy array or None
2309
2310
2311
2312
            Raw data used in the Dataset construction.
        """
        if self.handle is None:
            raise Exception("Cannot get data before construct Dataset")
Guolin Ke's avatar
Guolin Ke committed
2313
2314
2315
2316
2317
        if self.need_slice and self.used_indices is not None and self.reference is not None:
            self.data = self.reference.data
            if self.data is not None:
                if isinstance(self.data, np.ndarray) or scipy.sparse.issparse(self.data):
                    self.data = self.data[self.used_indices, :]
2318
                elif isinstance(self.data, pd_DataFrame):
Guolin Ke's avatar
Guolin Ke committed
2319
                    self.data = self.data.iloc[self.used_indices].copy()
2320
                elif isinstance(self.data, dt_DataTable):
Guolin Ke's avatar
Guolin Ke committed
2321
                    self.data = self.data[self.used_indices, :]
2322
2323
2324
2325
                elif isinstance(self.data, Sequence):
                    self.data = self.data[self.used_indices]
                elif isinstance(self.data, list) and len(self.data) > 0 and all(isinstance(x, Sequence) for x in self.data):
                    self.data = np.array([row for row in self._yield_row_from_seqlist(self.data, self.used_indices)])
Guolin Ke's avatar
Guolin Ke committed
2326
                else:
2327
2328
                    _log_warning(f"Cannot subset {type(self.data).__name__} type of raw data.\n"
                                 "Returning original raw data")
2329
            self.need_slice = False
Guolin Ke's avatar
Guolin Ke committed
2330
2331
2332
        if self.data is None:
            raise LightGBMError("Cannot call `get_data` after freed raw data, "
                                "set free_raw_data=False when construct Dataset to avoid this.")
2333
2334
        return self.data

Guolin Ke's avatar
Guolin Ke committed
2335
    def get_group(self):
2336
        """Get the group of the Dataset.
Guolin Ke's avatar
Guolin Ke committed
2337
2338
2339

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
2340
        group : numpy array or None
2341
2342
2343
            Group/query data.
            Only used in the learning-to-rank task.
            sum(group) = n_samples.
2344
2345
            For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
            where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
Guolin Ke's avatar
Guolin Ke committed
2346
        """
2347
        if self.group is None:
wxchan's avatar
wxchan committed
2348
            self.group = self.get_field('group')
Guolin Ke's avatar
Guolin Ke committed
2349
2350
            if self.group is not None:
                # group data from LightGBM is boundaries data, need to convert to group size
Nikita Titov's avatar
Nikita Titov committed
2351
                self.group = np.diff(self.group)
Guolin Ke's avatar
Guolin Ke committed
2352
2353
2354
        return self.group

    def num_data(self):
2355
        """Get the number of rows in the Dataset.
Guolin Ke's avatar
Guolin Ke committed
2356
2357
2358

        Returns
        -------
2359
2360
        number_of_rows : int
            The number of rows in the Dataset.
Guolin Ke's avatar
Guolin Ke committed
2361
        """
2362
        if self.handle is not None:
2363
            ret = ctypes.c_int(0)
wxchan's avatar
wxchan committed
2364
2365
2366
            _safe_call(_LIB.LGBM_DatasetGetNumData(self.handle,
                                                   ctypes.byref(ret)))
            return ret.value
Guolin Ke's avatar
Guolin Ke committed
2367
        else:
2368
            raise LightGBMError("Cannot get num_data before construct dataset")
Guolin Ke's avatar
Guolin Ke committed
2369
2370

    def num_feature(self):
2371
        """Get the number of columns (features) in the Dataset.
Guolin Ke's avatar
Guolin Ke committed
2372
2373
2374

        Returns
        -------
2375
2376
        number_of_columns : int
            The number of columns (features) in the Dataset.
Guolin Ke's avatar
Guolin Ke committed
2377
        """
2378
        if self.handle is not None:
2379
            ret = ctypes.c_int(0)
wxchan's avatar
wxchan committed
2380
2381
2382
            _safe_call(_LIB.LGBM_DatasetGetNumFeature(self.handle,
                                                      ctypes.byref(ret)))
            return ret.value
Guolin Ke's avatar
Guolin Ke committed
2383
        else:
2384
            raise LightGBMError("Cannot get num_feature before construct dataset")
Guolin Ke's avatar
Guolin Ke committed
2385

2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
    def feature_num_bin(self, feature: int) -> int:
        """Get the number of bins for a feature.

        Parameters
        ----------
        feature : int
            Index of the feature.

        Returns
        -------
        number_of_bins : int
            The number of constructed bins for the feature in the Dataset.
        """
        if self.handle is not None:
            ret = ctypes.c_int(0)
            _safe_call(_LIB.LGBM_DatasetGetFeatureNumBin(self.handle,
                                                         ctypes.c_int(feature),
                                                         ctypes.byref(ret)))
            return ret.value
        else:
            raise LightGBMError("Cannot get feature_num_bin before construct dataset")

2408
    def get_ref_chain(self, ref_limit=100):
2409
2410
2411
2412
2413
        """Get a chain of Dataset objects.

        Starts with r, then goes to r.reference (if exists),
        then to r.reference.reference, etc.
        until we hit ``ref_limit`` or a reference loop.
2414
2415
2416
2417
2418

        Parameters
        ----------
        ref_limit : int, optional (default=100)
            The limit number of references.
2419
2420
2421

        Returns
        -------
2422
2423
2424
        ref_chain : set of Dataset
            Chain of references of the Datasets.
        """
2425
        head = self
2426
        ref_chain = set()
2427
2428
        while len(ref_chain) < ref_limit:
            if isinstance(head, Dataset):
2429
                ref_chain.add(head)
2430
2431
2432
2433
2434
2435
                if (head.reference is not None) and (head.reference not in ref_chain):
                    head = head.reference
                else:
                    break
            else:
                break
Nikita Titov's avatar
Nikita Titov committed
2436
        return ref_chain
2437

2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
    def add_features_from(self, other):
        """Add features from other Dataset to the current Dataset.

        Both Datasets must be constructed before calling this method.

        Parameters
        ----------
        other : Dataset
            The Dataset to take features from.

        Returns
        -------
        self : Dataset
            Dataset with the new features added.
        """
        if self.handle is None or other.handle is None:
            raise ValueError('Both source and target Datasets must be constructed before adding features')
        _safe_call(_LIB.LGBM_DatasetAddFeaturesFrom(self.handle, other.handle))
Guolin Ke's avatar
Guolin Ke committed
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
        was_none = self.data is None
        old_self_data_type = type(self.data).__name__
        if other.data is None:
            self.data = None
        elif self.data is not None:
            if isinstance(self.data, np.ndarray):
                if isinstance(other.data, np.ndarray):
                    self.data = np.hstack((self.data, other.data))
                elif scipy.sparse.issparse(other.data):
                    self.data = np.hstack((self.data, other.data.toarray()))
2466
                elif isinstance(other.data, pd_DataFrame):
Guolin Ke's avatar
Guolin Ke committed
2467
                    self.data = np.hstack((self.data, other.data.values))
2468
                elif isinstance(other.data, dt_DataTable):
Guolin Ke's avatar
Guolin Ke committed
2469
2470
2471
2472
2473
2474
2475
                    self.data = np.hstack((self.data, other.data.to_numpy()))
                else:
                    self.data = None
            elif scipy.sparse.issparse(self.data):
                sparse_format = self.data.getformat()
                if isinstance(other.data, np.ndarray) or scipy.sparse.issparse(other.data):
                    self.data = scipy.sparse.hstack((self.data, other.data), format=sparse_format)
2476
                elif isinstance(other.data, pd_DataFrame):
Guolin Ke's avatar
Guolin Ke committed
2477
                    self.data = scipy.sparse.hstack((self.data, other.data.values), format=sparse_format)
2478
                elif isinstance(other.data, dt_DataTable):
Guolin Ke's avatar
Guolin Ke committed
2479
2480
2481
                    self.data = scipy.sparse.hstack((self.data, other.data.to_numpy()), format=sparse_format)
                else:
                    self.data = None
2482
            elif isinstance(self.data, pd_DataFrame):
Guolin Ke's avatar
Guolin Ke committed
2483
2484
                if not PANDAS_INSTALLED:
                    raise LightGBMError("Cannot add features to DataFrame type of raw data "
2485
2486
                                        "without pandas installed. "
                                        "Install pandas and restart your session.")
Guolin Ke's avatar
Guolin Ke committed
2487
                if isinstance(other.data, np.ndarray):
2488
                    self.data = concat((self.data, pd_DataFrame(other.data)),
Guolin Ke's avatar
Guolin Ke committed
2489
2490
                                       axis=1, ignore_index=True)
                elif scipy.sparse.issparse(other.data):
2491
                    self.data = concat((self.data, pd_DataFrame(other.data.toarray())),
Guolin Ke's avatar
Guolin Ke committed
2492
                                       axis=1, ignore_index=True)
2493
                elif isinstance(other.data, pd_DataFrame):
Guolin Ke's avatar
Guolin Ke committed
2494
2495
                    self.data = concat((self.data, other.data),
                                       axis=1, ignore_index=True)
2496
2497
                elif isinstance(other.data, dt_DataTable):
                    self.data = concat((self.data, pd_DataFrame(other.data.to_numpy())),
Guolin Ke's avatar
Guolin Ke committed
2498
2499
2500
                                       axis=1, ignore_index=True)
                else:
                    self.data = None
2501
            elif isinstance(self.data, dt_DataTable):
Guolin Ke's avatar
Guolin Ke committed
2502
                if isinstance(other.data, np.ndarray):
2503
                    self.data = dt_DataTable(np.hstack((self.data.to_numpy(), other.data)))
Guolin Ke's avatar
Guolin Ke committed
2504
                elif scipy.sparse.issparse(other.data):
2505
2506
2507
2508
2509
                    self.data = dt_DataTable(np.hstack((self.data.to_numpy(), other.data.toarray())))
                elif isinstance(other.data, pd_DataFrame):
                    self.data = dt_DataTable(np.hstack((self.data.to_numpy(), other.data.values)))
                elif isinstance(other.data, dt_DataTable):
                    self.data = dt_DataTable(np.hstack((self.data.to_numpy(), other.data.to_numpy())))
Guolin Ke's avatar
Guolin Ke committed
2510
2511
2512
2513
2514
                else:
                    self.data = None
            else:
                self.data = None
        if self.data is None:
2515
2516
            err_msg = (f"Cannot add features from {type(other.data).__name__} type of raw data to "
                       f"{old_self_data_type} type of raw data.\n")
Guolin Ke's avatar
Guolin Ke committed
2517
2518
            err_msg += ("Set free_raw_data=False when construct Dataset to avoid this"
                        if was_none else "Freeing raw data")
2519
            _log_warning(err_msg)
Guolin Ke's avatar
Guolin Ke committed
2520
        self.feature_name = self.get_feature_name()
2521
2522
        _log_warning("Reseting categorical features.\n"
                     "You can set new categorical features via ``set_categorical_feature`` method")
Guolin Ke's avatar
Guolin Ke committed
2523
2524
        self.categorical_feature = "auto"
        self.pandas_categorical = None
2525
2526
        return self

2527
    def _dump_text(self, filename):
2528
2529
2530
2531
2532
2533
        """Save Dataset to a text file.

        This format cannot be loaded back in by LightGBM, but is useful for debugging purposes.

        Parameters
        ----------
2534
        filename : str or pathlib.Path
2535
2536
2537
2538
2539
2540
2541
2542
2543
            Name of the output file.

        Returns
        -------
        self : Dataset
            Returns self.
        """
        _safe_call(_LIB.LGBM_DatasetDumpText(
            self.construct().handle,
2544
            c_str(str(filename))))
2545
2546
        return self

wxchan's avatar
wxchan committed
2547

2548
class Booster:
2549
    """Booster in LightGBM."""
2550

2551
    def __init__(self, params=None, train_set=None, model_file=None, model_str=None):
2552
        """Initialize the Booster.
wxchan's avatar
wxchan committed
2553
2554
2555

        Parameters
        ----------
Nikita Titov's avatar
Nikita Titov committed
2556
        params : dict or None, optional (default=None)
2557
2558
2559
            Parameters for Booster.
        train_set : Dataset or None, optional (default=None)
            Training dataset.
2560
        model_file : str, pathlib.Path or None, optional (default=None)
wxchan's avatar
wxchan committed
2561
            Path to the model file.
2562
        model_str : str or None, optional (default=None)
2563
            Model will be loaded from this string.
wxchan's avatar
wxchan committed
2564
        """
2565
        self.handle = None
2566
        self.network = False
wxchan's avatar
wxchan committed
2567
        self.__need_reload_eval_info = True
2568
        self._train_data_name = "training"
wxchan's avatar
wxchan committed
2569
        self.__attr = {}
2570
        self.__set_objective_to_none = False
wxchan's avatar
wxchan committed
2571
        self.best_iteration = -1
wxchan's avatar
wxchan committed
2572
        self.best_score = {}
2573
        params = {} if params is None else deepcopy(params)
wxchan's avatar
wxchan committed
2574
        if train_set is not None:
2575
            # Training task
wxchan's avatar
wxchan committed
2576
            if not isinstance(train_set, Dataset):
2577
                raise TypeError(f'Training data should be Dataset instance, met {type(train_set).__name__}')
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
            params = _choose_param_value(
                main_param_name="machines",
                params=params,
                default_value=None
            )
            # if "machines" is given, assume user wants to do distributed learning, and set up network
            if params["machines"] is None:
                params.pop("machines", None)
            else:
                machines = params["machines"]
                if isinstance(machines, str):
                    num_machines_from_machine_list = len(machines.split(','))
                elif isinstance(machines, (list, set)):
                    num_machines_from_machine_list = len(machines)
                    machines = ','.join(machines)
                else:
                    raise ValueError("Invalid machines in params.")

                params = _choose_param_value(
                    main_param_name="num_machines",
                    params=params,
                    default_value=num_machines_from_machine_list
                )
                params = _choose_param_value(
                    main_param_name="local_listen_port",
                    params=params,
                    default_value=12400
                )
                self.set_network(
                    machines=machines,
                    local_listen_port=params["local_listen_port"],
                    listen_time_out=params.get("time_out", 120),
                    num_machines=params["num_machines"]
                )
2612
            # construct booster object
2613
2614
2615
2616
            train_set.construct()
            # copy the parameters from train_set
            params.update(train_set.get_params())
            params_str = param_dict_to_str(params)
2617
            self.handle = ctypes.c_void_p()
wxchan's avatar
wxchan committed
2618
            _safe_call(_LIB.LGBM_BoosterCreate(
2619
                train_set.handle,
wxchan's avatar
wxchan committed
2620
2621
                c_str(params_str),
                ctypes.byref(self.handle)))
2622
            # save reference to data
wxchan's avatar
wxchan committed
2623
2624
2625
2626
            self.train_set = train_set
            self.valid_sets = []
            self.name_valid_sets = []
            self.__num_dataset = 1
Guolin Ke's avatar
Guolin Ke committed
2627
2628
            self.__init_predictor = train_set._predictor
            if self.__init_predictor is not None:
wxchan's avatar
wxchan committed
2629
2630
                _safe_call(_LIB.LGBM_BoosterMerge(
                    self.handle,
Guolin Ke's avatar
Guolin Ke committed
2631
                    self.__init_predictor.handle))
Guolin Ke's avatar
Guolin Ke committed
2632
            out_num_class = ctypes.c_int(0)
wxchan's avatar
wxchan committed
2633
2634
2635
2636
            _safe_call(_LIB.LGBM_BoosterGetNumClasses(
                self.handle,
                ctypes.byref(out_num_class)))
            self.__num_class = out_num_class.value
2637
            # buffer for inner predict
wxchan's avatar
wxchan committed
2638
2639
2640
            self.__inner_predict_buffer = [None]
            self.__is_predicted_cur_iter = [False]
            self.__get_eval_info()
2641
            self.pandas_categorical = train_set.pandas_categorical
2642
            self.train_set_version = train_set.version
wxchan's avatar
wxchan committed
2643
        elif model_file is not None:
2644
            # Prediction task
Guolin Ke's avatar
Guolin Ke committed
2645
            out_num_iterations = ctypes.c_int(0)
2646
            self.handle = ctypes.c_void_p()
wxchan's avatar
wxchan committed
2647
            _safe_call(_LIB.LGBM_BoosterCreateFromModelfile(
2648
                c_str(str(model_file)),
wxchan's avatar
wxchan committed
2649
2650
                ctypes.byref(out_num_iterations),
                ctypes.byref(self.handle)))
Guolin Ke's avatar
Guolin Ke committed
2651
            out_num_class = ctypes.c_int(0)
wxchan's avatar
wxchan committed
2652
2653
2654
2655
            _safe_call(_LIB.LGBM_BoosterGetNumClasses(
                self.handle,
                ctypes.byref(out_num_class)))
            self.__num_class = out_num_class.value
2656
            self.pandas_categorical = _load_pandas_categorical(file_name=model_file)
2657
        elif model_str is not None:
2658
            self.model_from_string(model_str)
wxchan's avatar
wxchan committed
2659
        else:
2660
2661
            raise TypeError('Need at least one training dataset or model file or model string '
                            'to create Booster instance')
2662
        self.params = params
wxchan's avatar
wxchan committed
2663
2664

    def __del__(self):
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
        try:
            if self.network:
                self.free_network()
        except AttributeError:
            pass
        try:
            if self.handle is not None:
                _safe_call(_LIB.LGBM_BoosterFree(self.handle))
        except AttributeError:
            pass
wxchan's avatar
wxchan committed
2675

wxchan's avatar
wxchan committed
2676
2677
2678
2679
    def __copy__(self):
        return self.__deepcopy__(None)

    def __deepcopy__(self, _):
2680
        model_str = self.model_to_string(num_iteration=-1)
2681
        booster = Booster(model_str=model_str)
2682
        return booster
wxchan's avatar
wxchan committed
2683
2684
2685
2686
2687
2688
2689

    def __getstate__(self):
        this = self.__dict__.copy()
        handle = this['handle']
        this.pop('train_set', None)
        this.pop('valid_sets', None)
        if handle is not None:
2690
            this["handle"] = self.model_to_string(num_iteration=-1)
wxchan's avatar
wxchan committed
2691
2692
2693
        return this

    def __setstate__(self, state):
2694
2695
        model_str = state.get('handle', None)
        if model_str is not None:
wxchan's avatar
wxchan committed
2696
            handle = ctypes.c_void_p()
Guolin Ke's avatar
Guolin Ke committed
2697
            out_num_iterations = ctypes.c_int(0)
2698
2699
2700
2701
            _safe_call(_LIB.LGBM_BoosterLoadModelFromString(
                c_str(model_str),
                ctypes.byref(out_num_iterations),
                ctypes.byref(handle)))
wxchan's avatar
wxchan committed
2702
2703
2704
            state['handle'] = handle
        self.__dict__.update(state)

wxchan's avatar
wxchan committed
2705
    def free_dataset(self):
Nikita Titov's avatar
Nikita Titov committed
2706
2707
2708
2709
2710
2711
2712
        """Free Booster's Datasets.

        Returns
        -------
        self : Booster
            Booster without Datasets.
        """
wxchan's avatar
wxchan committed
2713
2714
        self.__dict__.pop('train_set', None)
        self.__dict__.pop('valid_sets', None)
2715
        self.__num_dataset = 0
Nikita Titov's avatar
Nikita Titov committed
2716
        return self
wxchan's avatar
wxchan committed
2717

2718
2719
2720
    def _free_buffer(self):
        self.__inner_predict_buffer = []
        self.__is_predicted_cur_iter = []
Nikita Titov's avatar
Nikita Titov committed
2721
        return self
2722

2723
2724
2725
2726
2727
2728
2729
    def set_network(
        self,
        machines: Union[List[str], Set[str], str],
        local_listen_port: int = 12400,
        listen_time_out: int = 120,
        num_machines: int = 1
    ) -> "Booster":
2730
2731
2732
2733
        """Set the network configuration.

        Parameters
        ----------
2734
        machines : list, set or str
2735
            Names of machines.
Nikita Titov's avatar
Nikita Titov committed
2736
        local_listen_port : int, optional (default=12400)
2737
            TCP listen port for local machines.
Nikita Titov's avatar
Nikita Titov committed
2738
        listen_time_out : int, optional (default=120)
2739
            Socket time-out in minutes.
Nikita Titov's avatar
Nikita Titov committed
2740
        num_machines : int, optional (default=1)
2741
            The number of machines for distributed learning application.
Nikita Titov's avatar
Nikita Titov committed
2742
2743
2744
2745
2746

        Returns
        -------
        self : Booster
            Booster with set network.
2747
        """
2748
2749
        if isinstance(machines, (list, set)):
            machines = ','.join(machines)
2750
2751
2752
2753
2754
        _safe_call(_LIB.LGBM_NetworkInit(c_str(machines),
                                         ctypes.c_int(local_listen_port),
                                         ctypes.c_int(listen_time_out),
                                         ctypes.c_int(num_machines)))
        self.network = True
Nikita Titov's avatar
Nikita Titov committed
2755
        return self
2756
2757

    def free_network(self):
Nikita Titov's avatar
Nikita Titov committed
2758
2759
2760
2761
2762
2763
2764
        """Free Booster's network.

        Returns
        -------
        self : Booster
            Booster with freed network.
        """
2765
2766
        _safe_call(_LIB.LGBM_NetworkFree())
        self.network = False
Nikita Titov's avatar
Nikita Titov committed
2767
        return self
2768

2769
2770
2771
    def trees_to_dataframe(self):
        """Parse the fitted model and return in an easy-to-read pandas DataFrame.

2772
2773
2774
2775
        The returned DataFrame has the following columns.

            - ``tree_index`` : int64, which tree a node belongs to. 0-based, so a value of ``6``, for example, means "this node is in the 7th tree".
            - ``node_depth`` : int64, how far a node is from the root of the tree. The root node has a value of ``1``, its direct children are ``2``, etc.
2776
2777
2778
2779
2780
            - ``node_index`` : str, unique identifier for a node.
            - ``left_child`` : str, ``node_index`` of the child node to the left of a split. ``None`` for leaf nodes.
            - ``right_child`` : str, ``node_index`` of the child node to the right of a split. ``None`` for leaf nodes.
            - ``parent_index`` : str, ``node_index`` of this node's parent. ``None`` for the root node.
            - ``split_feature`` : str, name of the feature used for splitting. ``None`` for leaf nodes.
2781
2782
            - ``split_gain`` : float64, gain from adding this split to the tree. ``NaN`` for leaf nodes.
            - ``threshold`` : float64, value of the feature used to decide which side of the split a record will go down. ``NaN`` for leaf nodes.
2783
            - ``decision_type`` : str, logical operator describing how to compare a value to ``threshold``.
2784
2785
              For example, ``split_feature = "Column_10", threshold = 15, decision_type = "<="`` means that
              records where ``Column_10 <= 15`` follow the left side of the split, otherwise follows the right side of the split. ``None`` for leaf nodes.
2786
2787
            - ``missing_direction`` : str, split direction that missing values should go to. ``None`` for leaf nodes.
            - ``missing_type`` : str, describes what types of values are treated as missing.
2788
            - ``value`` : float64, predicted value for this leaf node, multiplied by the learning rate.
2789
            - ``weight`` : float64 or int64, sum of Hessian (second-order derivative of objective), summed over observations that fall in this node.
2790
2791
            - ``count`` : int64, number of records in the training data that fall into this node.

2792
2793
2794
2795
2796
2797
        Returns
        -------
        result : pandas DataFrame
            Returns a pandas DataFrame of the parsed model.
        """
        if not PANDAS_INSTALLED:
2798
2799
            raise LightGBMError('This method cannot be run without pandas installed. '
                                'You must install pandas and restart your session to use this method.')
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810

        if self.num_trees() == 0:
            raise LightGBMError('There are no trees in this Booster and thus nothing to parse')

        def _is_split_node(tree):
            return 'split_index' in tree.keys()

        def create_node_record(tree, node_depth=1, tree_index=None,
                               feature_names=None, parent_node=None):

            def _get_node_index(tree, tree_index):
2811
                tree_num = f'{tree_index}-' if tree_index is not None else ''
2812
2813
2814
                is_split = _is_split_node(tree)
                node_type = 'S' if is_split else 'L'
                # if a single node tree it won't have `leaf_index` so return 0
2815
2816
                node_num = tree.get('split_index' if is_split else 'leaf_index', 0)
                return f"{tree_num}{node_type}{node_num}"
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828

            def _get_split_feature(tree, feature_names):
                if _is_split_node(tree):
                    if feature_names is not None:
                        feature_name = feature_names[tree['split_feature']]
                    else:
                        feature_name = tree['split_feature']
                else:
                    feature_name = None
                return feature_name

            def _is_single_node_tree(tree):
2829
                return set(tree.keys()) == {'leaf_value'}
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902

            # Create the node record, and populate universal data members
            node = OrderedDict()
            node['tree_index'] = tree_index
            node['node_depth'] = node_depth
            node['node_index'] = _get_node_index(tree, tree_index)
            node['left_child'] = None
            node['right_child'] = None
            node['parent_index'] = parent_node
            node['split_feature'] = _get_split_feature(tree, feature_names)
            node['split_gain'] = None
            node['threshold'] = None
            node['decision_type'] = None
            node['missing_direction'] = None
            node['missing_type'] = None
            node['value'] = None
            node['weight'] = None
            node['count'] = None

            # Update values to reflect node type (leaf or split)
            if _is_split_node(tree):
                node['left_child'] = _get_node_index(tree['left_child'], tree_index)
                node['right_child'] = _get_node_index(tree['right_child'], tree_index)
                node['split_gain'] = tree['split_gain']
                node['threshold'] = tree['threshold']
                node['decision_type'] = tree['decision_type']
                node['missing_direction'] = 'left' if tree['default_left'] else 'right'
                node['missing_type'] = tree['missing_type']
                node['value'] = tree['internal_value']
                node['weight'] = tree['internal_weight']
                node['count'] = tree['internal_count']
            else:
                node['value'] = tree['leaf_value']
                if not _is_single_node_tree(tree):
                    node['weight'] = tree['leaf_weight']
                    node['count'] = tree['leaf_count']

            return node

        def tree_dict_to_node_list(tree, node_depth=1, tree_index=None,
                                   feature_names=None, parent_node=None):

            node = create_node_record(tree,
                                      node_depth=node_depth,
                                      tree_index=tree_index,
                                      feature_names=feature_names,
                                      parent_node=parent_node)

            res = [node]

            if _is_split_node(tree):
                # traverse the next level of the tree
                children = ['left_child', 'right_child']
                for child in children:
                    subtree_list = tree_dict_to_node_list(
                        tree[child],
                        node_depth=node_depth + 1,
                        tree_index=tree_index,
                        feature_names=feature_names,
                        parent_node=node['node_index'])
                    # In tree format, "subtree_list" is a list of node records (dicts),
                    # and we add node to the list.
                    res.extend(subtree_list)
            return res

        model_dict = self.dump_model()
        feature_names = model_dict['feature_names']
        model_list = []
        for tree in model_dict['tree_info']:
            model_list.extend(tree_dict_to_node_list(tree['tree_structure'],
                                                     tree_index=tree['tree_index'],
                                                     feature_names=feature_names))

2903
        return pd_DataFrame(model_list, columns=model_list[0].keys())
2904

wxchan's avatar
wxchan committed
2905
    def set_train_data_name(self, name):
2906
2907
2908
2909
        """Set the name to the training Dataset.

        Parameters
        ----------
2910
        name : str
Nikita Titov's avatar
Nikita Titov committed
2911
2912
2913
2914
2915
2916
            Name for the training Dataset.

        Returns
        -------
        self : Booster
            Booster with set training Dataset name.
2917
        """
2918
        self._train_data_name = name
Nikita Titov's avatar
Nikita Titov committed
2919
        return self
wxchan's avatar
wxchan committed
2920
2921

    def add_valid(self, data, name):
2922
        """Add validation data.
wxchan's avatar
wxchan committed
2923
2924
2925
2926

        Parameters
        ----------
        data : Dataset
2927
            Validation data.
2928
        name : str
2929
            Name of validation data.
Nikita Titov's avatar
Nikita Titov committed
2930
2931
2932
2933
2934

        Returns
        -------
        self : Booster
            Booster with set validation data.
wxchan's avatar
wxchan committed
2935
        """
Guolin Ke's avatar
Guolin Ke committed
2936
        if not isinstance(data, Dataset):
2937
            raise TypeError(f'Validation data should be Dataset instance, met {type(data).__name__}')
Guolin Ke's avatar
Guolin Ke committed
2938
        if data._predictor is not self.__init_predictor:
2939
2940
            raise LightGBMError("Add validation data failed, "
                                "you should use same predictor for these data")
wxchan's avatar
wxchan committed
2941
2942
        _safe_call(_LIB.LGBM_BoosterAddValidData(
            self.handle,
wxchan's avatar
wxchan committed
2943
            data.construct().handle))
wxchan's avatar
wxchan committed
2944
2945
2946
2947
2948
        self.valid_sets.append(data)
        self.name_valid_sets.append(name)
        self.__num_dataset += 1
        self.__inner_predict_buffer.append(None)
        self.__is_predicted_cur_iter.append(False)
Nikita Titov's avatar
Nikita Titov committed
2949
        return self
wxchan's avatar
wxchan committed
2950
2951

    def reset_parameter(self, params):
2952
        """Reset parameters of Booster.
wxchan's avatar
wxchan committed
2953
2954
2955
2956

        Parameters
        ----------
        params : dict
2957
            New parameters for Booster.
Nikita Titov's avatar
Nikita Titov committed
2958
2959
2960
2961
2962

        Returns
        -------
        self : Booster
            Booster with new parameters.
wxchan's avatar
wxchan committed
2963
2964
2965
2966
2967
2968
        """
        params_str = param_dict_to_str(params)
        if params_str:
            _safe_call(_LIB.LGBM_BoosterResetParameter(
                self.handle,
                c_str(params_str)))
Guolin Ke's avatar
Guolin Ke committed
2969
        self.params.update(params)
Nikita Titov's avatar
Nikita Titov committed
2970
        return self
wxchan's avatar
wxchan committed
2971
2972

    def update(self, train_set=None, fobj=None):
Nikita Titov's avatar
Nikita Titov committed
2973
        """Update Booster for one iteration.
2974

wxchan's avatar
wxchan committed
2975
2976
        Parameters
        ----------
2977
2978
2979
2980
        train_set : Dataset or None, optional (default=None)
            Training data.
            If None, last training data is used.
        fobj : callable or None, optional (default=None)
wxchan's avatar
wxchan committed
2981
            Customized objective function.
2982
2983
2984
            Should accept two parameters: preds, train_data,
            and return (grad, hess).

2985
                preds : numpy 1-D array or numpy 2-D array (for multi-class task)
2986
                    The predicted values.
2987
2988
                    Predicted values are returned before any transformation,
                    e.g. they are raw margin instead of probability of positive class for binary task.
2989
2990
                train_data : Dataset
                    The training dataset.
2991
                grad : numpy 1-D array or numpy 2-D array (for multi-class task)
2992
2993
                    The value of the first order derivative (gradient) of the loss
                    with respect to the elements of preds for each sample point.
2994
                hess : numpy 1-D array or numpy 2-D array (for multi-class task)
2995
2996
                    The value of the second order derivative (Hessian) of the loss
                    with respect to the elements of preds for each sample point.
wxchan's avatar
wxchan committed
2997

2998
            For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes],
2999
            and grad and hess should be returned in the same format.
3000

wxchan's avatar
wxchan committed
3001
3002
        Returns
        -------
3003
3004
        is_finished : bool
            Whether the update was successfully finished.
wxchan's avatar
wxchan committed
3005
        """
3006
        # need reset training data
3007
3008
3009
3010
3011
3012
        if train_set is None and self.train_set_version != self.train_set.version:
            train_set = self.train_set
            is_the_same_train_set = False
        else:
            is_the_same_train_set = train_set is self.train_set and self.train_set_version == train_set.version
        if train_set is not None and not is_the_same_train_set:
Guolin Ke's avatar
Guolin Ke committed
3013
            if not isinstance(train_set, Dataset):
3014
                raise TypeError(f'Training data should be Dataset instance, met {type(train_set).__name__}')
Guolin Ke's avatar
Guolin Ke committed
3015
            if train_set._predictor is not self.__init_predictor:
3016
3017
                raise LightGBMError("Replace training data failed, "
                                    "you should use same predictor for these data")
wxchan's avatar
wxchan committed
3018
3019
3020
            self.train_set = train_set
            _safe_call(_LIB.LGBM_BoosterResetTrainingData(
                self.handle,
wxchan's avatar
wxchan committed
3021
                self.train_set.construct().handle))
wxchan's avatar
wxchan committed
3022
            self.__inner_predict_buffer[0] = None
3023
            self.train_set_version = self.train_set.version
wxchan's avatar
wxchan committed
3024
3025
        is_finished = ctypes.c_int(0)
        if fobj is None:
3026
            if self.__set_objective_to_none:
3027
                raise LightGBMError('Cannot update due to null objective function.')
wxchan's avatar
wxchan committed
3028
3029
3030
            _safe_call(_LIB.LGBM_BoosterUpdateOneIter(
                self.handle,
                ctypes.byref(is_finished)))
3031
            self.__is_predicted_cur_iter = [False for _ in range(self.__num_dataset)]
wxchan's avatar
wxchan committed
3032
3033
            return is_finished.value == 1
        else:
3034
            if not self.__set_objective_to_none:
Nikita Titov's avatar
Nikita Titov committed
3035
                self.reset_parameter({"objective": "none"}).__set_objective_to_none = True
wxchan's avatar
wxchan committed
3036
3037
3038
3039
            grad, hess = fobj(self.__inner_predict(0), self.train_set)
            return self.__boost(grad, hess)

    def __boost(self, grad, hess):
3040
        """Boost Booster for one iteration with customized gradient statistics.
Nikita Titov's avatar
Nikita Titov committed
3041

Nikita Titov's avatar
Nikita Titov committed
3042
3043
        .. note::

3044
3045
            Score is returned before any transformation,
            e.g. it is raw margin instead of probability of positive class for binary task.
3046
            For multi-class task, score are numpy 2-D array of shape = [n_samples, n_classes],
3047
            and grad and hess should be returned in the same format.
3048

wxchan's avatar
wxchan committed
3049
3050
        Parameters
        ----------
3051
        grad : numpy 1-D array or numpy 2-D array (for multi-class task)
3052
3053
            The value of the first order derivative (gradient) of the loss
            with respect to the elements of score for each sample point.
3054
        hess : numpy 1-D array or numpy 2-D array (for multi-class task)
3055
3056
            The value of the second order derivative (Hessian) of the loss
            with respect to the elements of score for each sample point.
wxchan's avatar
wxchan committed
3057
3058
3059

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
3060
3061
        is_finished : bool
            Whether the boost was successfully finished.
wxchan's avatar
wxchan committed
3062
        """
3063
3064
3065
        if self.__num_class > 1:
            grad = grad.ravel(order='F')
            hess = hess.ravel(order='F')
3066
3067
        grad = list_to_1d_numpy(grad, name='gradient')
        hess = list_to_1d_numpy(hess, name='hessian')
3068
3069
        assert grad.flags.c_contiguous
        assert hess.flags.c_contiguous
wxchan's avatar
wxchan committed
3070
        if len(grad) != len(hess):
3071
3072
            raise ValueError(f"Lengths of gradient ({len(grad)}) and Hessian ({len(hess)}) don't match")
        num_train_data = self.train_set.num_data()
3073
        if len(grad) != num_train_data * self.__num_class:
3074
3075
3076
            raise ValueError(
                f"Lengths of gradient ({len(grad)}) and Hessian ({len(hess)}) "
                f"don't match training data length ({num_train_data}) * "
3077
                f"number of models per one iteration ({self.__num_class})"
3078
            )
wxchan's avatar
wxchan committed
3079
3080
3081
3082
3083
3084
        is_finished = ctypes.c_int(0)
        _safe_call(_LIB.LGBM_BoosterUpdateOneIterCustom(
            self.handle,
            grad.ctypes.data_as(ctypes.POINTER(ctypes.c_float)),
            hess.ctypes.data_as(ctypes.POINTER(ctypes.c_float)),
            ctypes.byref(is_finished)))
3085
        self.__is_predicted_cur_iter = [False for _ in range(self.__num_dataset)]
wxchan's avatar
wxchan committed
3086
3087
3088
        return is_finished.value == 1

    def rollback_one_iter(self):
Nikita Titov's avatar
Nikita Titov committed
3089
3090
3091
3092
3093
3094
3095
        """Rollback one iteration.

        Returns
        -------
        self : Booster
            Booster with rolled back one iteration.
        """
wxchan's avatar
wxchan committed
3096
3097
        _safe_call(_LIB.LGBM_BoosterRollbackOneIter(
            self.handle))
3098
        self.__is_predicted_cur_iter = [False for _ in range(self.__num_dataset)]
Nikita Titov's avatar
Nikita Titov committed
3099
        return self
wxchan's avatar
wxchan committed
3100
3101

    def current_iteration(self):
3102
3103
3104
3105
3106
3107
3108
        """Get the index of the current iteration.

        Returns
        -------
        cur_iter : int
            The index of the current iteration.
        """
Guolin Ke's avatar
Guolin Ke committed
3109
        out_cur_iter = ctypes.c_int(0)
wxchan's avatar
wxchan committed
3110
3111
3112
3113
3114
        _safe_call(_LIB.LGBM_BoosterGetCurrentIteration(
            self.handle,
            ctypes.byref(out_cur_iter)))
        return out_cur_iter.value

3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
    def num_model_per_iteration(self):
        """Get number of models per iteration.

        Returns
        -------
        model_per_iter : int
            The number of models per iteration.
        """
        model_per_iter = ctypes.c_int(0)
        _safe_call(_LIB.LGBM_BoosterNumModelPerIteration(
            self.handle,
            ctypes.byref(model_per_iter)))
        return model_per_iter.value

    def num_trees(self):
        """Get number of weak sub-models.

        Returns
        -------
        num_trees : int
            The number of weak sub-models.
        """
        num_trees = ctypes.c_int(0)
        _safe_call(_LIB.LGBM_BoosterNumberOfTotalModel(
            self.handle,
            ctypes.byref(num_trees)))
        return num_trees.value

3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
    def upper_bound(self):
        """Get upper bound value of a model.

        Returns
        -------
        upper_bound : double
            Upper bound value of the model.
        """
        ret = ctypes.c_double(0)
        _safe_call(_LIB.LGBM_BoosterGetUpperBoundValue(
            self.handle,
            ctypes.byref(ret)))
        return ret.value

    def lower_bound(self):
        """Get lower bound value of a model.

        Returns
        -------
        lower_bound : double
            Lower bound value of the model.
        """
        ret = ctypes.c_double(0)
        _safe_call(_LIB.LGBM_BoosterGetLowerBoundValue(
            self.handle,
            ctypes.byref(ret)))
        return ret.value

wxchan's avatar
wxchan committed
3171
    def eval(self, data, name, feval=None):
3172
        """Evaluate for data.
wxchan's avatar
wxchan committed
3173
3174
3175

        Parameters
        ----------
3176
3177
        data : Dataset
            Data for the evaluating.
3178
        name : str
3179
            Name of the data.
3180
        feval : callable, list of callable, or None, optional (default=None)
3181
            Customized evaluation function.
3182
            Each evaluation function should accept two parameters: preds, eval_data,
3183
            and return (eval_name, eval_result, is_higher_better) or list of such tuples.
3184

3185
                preds : numpy 1-D array or numpy 2-D array (for multi-class task)
3186
                    The predicted values.
3187
                    For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes].
3188
3189
                    If ``fobj`` is specified, predicted values are returned before any transformation,
                    e.g. they are raw margin instead of probability of positive class for binary task in this case.
3190
                eval_data : Dataset
3191
                    A ``Dataset`` to evaluate.
3192
                eval_name : str
Andrew Ziem's avatar
Andrew Ziem committed
3193
                    The name of evaluation function (without whitespace).
3194
3195
3196
3197
3198
                eval_result : float
                    The eval result.
                is_higher_better : bool
                    Is eval result higher better, e.g. AUC is ``is_higher_better``.

wxchan's avatar
wxchan committed
3199
3200
        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
3201
        result : list
3202
            List with evaluation results.
wxchan's avatar
wxchan committed
3203
        """
Guolin Ke's avatar
Guolin Ke committed
3204
3205
        if not isinstance(data, Dataset):
            raise TypeError("Can only eval for Dataset instance")
wxchan's avatar
wxchan committed
3206
3207
3208
3209
        data_idx = -1
        if data is self.train_set:
            data_idx = 0
        else:
3210
            for i in range(len(self.valid_sets)):
wxchan's avatar
wxchan committed
3211
3212
3213
                if data is self.valid_sets[i]:
                    data_idx = i + 1
                    break
3214
        # need to push new valid data
wxchan's avatar
wxchan committed
3215
3216
3217
3218
3219
3220
3221
        if data_idx == -1:
            self.add_valid(data, name)
            data_idx = self.__num_dataset - 1

        return self.__inner_eval(name, data_idx, feval)

    def eval_train(self, feval=None):
3222
        """Evaluate for training data.
wxchan's avatar
wxchan committed
3223
3224
3225

        Parameters
        ----------
3226
        feval : callable, list of callable, or None, optional (default=None)
3227
            Customized evaluation function.
3228
            Each evaluation function should accept two parameters: preds, eval_data,
3229
            and return (eval_name, eval_result, is_higher_better) or list of such tuples.
3230

3231
                preds : numpy 1-D array or numpy 2-D array (for multi-class task)
3232
                    The predicted values.
3233
                    For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes].
3234
3235
                    If ``fobj`` is specified, predicted values are returned before any transformation,
                    e.g. they are raw margin instead of probability of positive class for binary task in this case.
Akshita Dixit's avatar
Akshita Dixit committed
3236
                eval_data : Dataset
3237
                    The training dataset.
3238
                eval_name : str
Andrew Ziem's avatar
Andrew Ziem committed
3239
                    The name of evaluation function (without whitespace).
3240
3241
3242
3243
3244
                eval_result : float
                    The eval result.
                is_higher_better : bool
                    Is eval result higher better, e.g. AUC is ``is_higher_better``.

wxchan's avatar
wxchan committed
3245
3246
        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
3247
        result : list
3248
            List with evaluation results.
wxchan's avatar
wxchan committed
3249
        """
3250
        return self.__inner_eval(self._train_data_name, 0, feval)
wxchan's avatar
wxchan committed
3251
3252

    def eval_valid(self, feval=None):
3253
        """Evaluate for validation data.
wxchan's avatar
wxchan committed
3254
3255
3256

        Parameters
        ----------
3257
        feval : callable, list of callable, or None, optional (default=None)
3258
            Customized evaluation function.
3259
            Each evaluation function should accept two parameters: preds, eval_data,
3260
            and return (eval_name, eval_result, is_higher_better) or list of such tuples.
3261

3262
                preds : numpy 1-D array or numpy 2-D array (for multi-class task)
3263
                    The predicted values.
3264
                    For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes].
3265
3266
                    If ``fobj`` is specified, predicted values are returned before any transformation,
                    e.g. they are raw margin instead of probability of positive class for binary task in this case.
Akshita Dixit's avatar
Akshita Dixit committed
3267
                eval_data : Dataset
3268
                    The validation dataset.
3269
                eval_name : str
Andrew Ziem's avatar
Andrew Ziem committed
3270
                    The name of evaluation function (without whitespace).
3271
3272
3273
3274
3275
                eval_result : float
                    The eval result.
                is_higher_better : bool
                    Is eval result higher better, e.g. AUC is ``is_higher_better``.

wxchan's avatar
wxchan committed
3276
3277
        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
3278
        result : list
3279
            List with evaluation results.
wxchan's avatar
wxchan committed
3280
        """
3281
        return [item for i in range(1, self.__num_dataset)
wxchan's avatar
wxchan committed
3282
                for item in self.__inner_eval(self.name_valid_sets[i - 1], i, feval)]
wxchan's avatar
wxchan committed
3283

3284
    def save_model(self, filename, num_iteration=None, start_iteration=0, importance_type='split'):
3285
        """Save Booster to file.
wxchan's avatar
wxchan committed
3286
3287
3288

        Parameters
        ----------
3289
        filename : str or pathlib.Path
3290
            Filename to save Booster.
3291
3292
3293
3294
        num_iteration : int or None, optional (default=None)
            Index of the iteration that should be saved.
            If None, if the best iteration exists, it is saved; otherwise, all iterations are saved.
            If <= 0, all iterations are saved.
Nikita Titov's avatar
Nikita Titov committed
3295
        start_iteration : int, optional (default=0)
3296
            Start index of the iteration that should be saved.
3297
        importance_type : str, optional (default="split")
3298
3299
3300
            What type of feature importance should be saved.
            If "split", result contains numbers of times the feature is used in a model.
            If "gain", result contains total gains of splits which use the feature.
Nikita Titov's avatar
Nikita Titov committed
3301
3302
3303
3304
3305

        Returns
        -------
        self : Booster
            Returns self.
wxchan's avatar
wxchan committed
3306
        """
3307
        if num_iteration is None:
3308
            num_iteration = self.best_iteration
3309
        importance_type_int = FEATURE_IMPORTANCE_TYPE_MAPPER[importance_type]
wxchan's avatar
wxchan committed
3310
3311
        _safe_call(_LIB.LGBM_BoosterSaveModel(
            self.handle,
3312
            ctypes.c_int(start_iteration),
Guolin Ke's avatar
Guolin Ke committed
3313
            ctypes.c_int(num_iteration),
3314
            ctypes.c_int(importance_type_int),
3315
            c_str(str(filename))))
3316
        _dump_pandas_categorical(self.pandas_categorical, filename)
Nikita Titov's avatar
Nikita Titov committed
3317
        return self
wxchan's avatar
wxchan committed
3318

3319
    def shuffle_models(self, start_iteration=0, end_iteration=-1):
3320
        """Shuffle models.
Nikita Titov's avatar
Nikita Titov committed
3321

3322
3323
3324
        Parameters
        ----------
        start_iteration : int, optional (default=0)
3325
            The first iteration that will be shuffled.
3326
3327
        end_iteration : int, optional (default=-1)
            The last iteration that will be shuffled.
3328
            If <= 0, means the last available iteration.
3329

Nikita Titov's avatar
Nikita Titov committed
3330
3331
3332
3333
        Returns
        -------
        self : Booster
            Booster with shuffled models.
3334
        """
3335
3336
        _safe_call(_LIB.LGBM_BoosterShuffleModels(
            self.handle,
Guolin Ke's avatar
Guolin Ke committed
3337
3338
            ctypes.c_int(start_iteration),
            ctypes.c_int(end_iteration)))
Nikita Titov's avatar
Nikita Titov committed
3339
        return self
3340

3341
    def model_from_string(self, model_str):
3342
3343
3344
3345
        """Load Booster from a string.

        Parameters
        ----------
3346
        model_str : str
3347
3348
3349
3350
            Model will be loaded from this string.

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
3351
        self : Booster
3352
3353
            Loaded Booster object.
        """
3354
3355
3356
3357
        if self.handle is not None:
            _safe_call(_LIB.LGBM_BoosterFree(self.handle))
        self._free_buffer()
        self.handle = ctypes.c_void_p()
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
        out_num_iterations = ctypes.c_int(0)
        _safe_call(_LIB.LGBM_BoosterLoadModelFromString(
            c_str(model_str),
            ctypes.byref(out_num_iterations),
            ctypes.byref(self.handle)))
        out_num_class = ctypes.c_int(0)
        _safe_call(_LIB.LGBM_BoosterGetNumClasses(
            self.handle,
            ctypes.byref(out_num_class)))
        self.__num_class = out_num_class.value
3368
        self.pandas_categorical = _load_pandas_categorical(model_str=model_str)
3369
3370
        return self

3371
    def model_to_string(self, num_iteration=None, start_iteration=0, importance_type='split'):
3372
        """Save Booster to string.
3373

3374
3375
3376
3377
3378
3379
        Parameters
        ----------
        num_iteration : int or None, optional (default=None)
            Index of the iteration that should be saved.
            If None, if the best iteration exists, it is saved; otherwise, all iterations are saved.
            If <= 0, all iterations are saved.
Nikita Titov's avatar
Nikita Titov committed
3380
        start_iteration : int, optional (default=0)
3381
            Start index of the iteration that should be saved.
3382
        importance_type : str, optional (default="split")
3383
3384
3385
            What type of feature importance should be saved.
            If "split", result contains numbers of times the feature is used in a model.
            If "gain", result contains total gains of splits which use the feature.
3386
3387
3388

        Returns
        -------
3389
        str_repr : str
3390
3391
            String representation of Booster.
        """
3392
        if num_iteration is None:
3393
            num_iteration = self.best_iteration
3394
        importance_type_int = FEATURE_IMPORTANCE_TYPE_MAPPER[importance_type]
3395
        buffer_len = 1 << 20
3396
        tmp_out_len = ctypes.c_int64(0)
3397
3398
3399
3400
        string_buffer = ctypes.create_string_buffer(buffer_len)
        ptr_string_buffer = ctypes.c_char_p(*[ctypes.addressof(string_buffer)])
        _safe_call(_LIB.LGBM_BoosterSaveModelToString(
            self.handle,
3401
            ctypes.c_int(start_iteration),
3402
            ctypes.c_int(num_iteration),
3403
            ctypes.c_int(importance_type_int),
3404
            ctypes.c_int64(buffer_len),
3405
3406
3407
            ctypes.byref(tmp_out_len),
            ptr_string_buffer))
        actual_len = tmp_out_len.value
3408
        # if buffer length is not long enough, re-allocate a buffer
3409
3410
3411
3412
3413
        if actual_len > buffer_len:
            string_buffer = ctypes.create_string_buffer(actual_len)
            ptr_string_buffer = ctypes.c_char_p(*[ctypes.addressof(string_buffer)])
            _safe_call(_LIB.LGBM_BoosterSaveModelToString(
                self.handle,
3414
                ctypes.c_int(start_iteration),
3415
                ctypes.c_int(num_iteration),
3416
                ctypes.c_int(importance_type_int),
3417
                ctypes.c_int64(actual_len),
3418
3419
                ctypes.byref(tmp_out_len),
                ptr_string_buffer))
3420
        ret = string_buffer.value.decode('utf-8')
3421
3422
        ret += _dump_pandas_categorical(self.pandas_categorical)
        return ret
3423

3424
    def dump_model(self, num_iteration=None, start_iteration=0, importance_type='split', object_hook=None):
Nikita Titov's avatar
Nikita Titov committed
3425
        """Dump Booster to JSON format.
wxchan's avatar
wxchan committed
3426

3427
3428
        Parameters
        ----------
3429
3430
3431
3432
        num_iteration : int or None, optional (default=None)
            Index of the iteration that should be dumped.
            If None, if the best iteration exists, it is dumped; otherwise, all iterations are dumped.
            If <= 0, all iterations are dumped.
Nikita Titov's avatar
Nikita Titov committed
3433
        start_iteration : int, optional (default=0)
3434
            Start index of the iteration that should be dumped.
3435
        importance_type : str, optional (default="split")
3436
3437
3438
            What type of feature importance should be dumped.
            If "split", result contains numbers of times the feature is used in a model.
            If "gain", result contains total gains of splits which use the feature.
3439
3440
3441
3442
3443
3444
3445
3446
3447
        object_hook : callable or None, optional (default=None)
            If not None, ``object_hook`` is a function called while parsing the json
            string returned by the C API. It may be used to alter the json, to store
            specific values while building the json structure. It avoids
            walking through the structure again. It saves a significant amount
            of time if the number of trees is huge.
            Signature is ``def object_hook(node: dict) -> dict``.
            None is equivalent to ``lambda node: node``.
            See documentation of ``json.loads()`` for further details.
3448

wxchan's avatar
wxchan committed
3449
3450
        Returns
        -------
3451
        json_repr : dict
Nikita Titov's avatar
Nikita Titov committed
3452
            JSON format of Booster.
wxchan's avatar
wxchan committed
3453
        """
3454
        if num_iteration is None:
3455
            num_iteration = self.best_iteration
3456
        importance_type_int = FEATURE_IMPORTANCE_TYPE_MAPPER[importance_type]
wxchan's avatar
wxchan committed
3457
        buffer_len = 1 << 20
3458
        tmp_out_len = ctypes.c_int64(0)
wxchan's avatar
wxchan committed
3459
3460
3461
3462
        string_buffer = ctypes.create_string_buffer(buffer_len)
        ptr_string_buffer = ctypes.c_char_p(*[ctypes.addressof(string_buffer)])
        _safe_call(_LIB.LGBM_BoosterDumpModel(
            self.handle,
3463
            ctypes.c_int(start_iteration),
Guolin Ke's avatar
Guolin Ke committed
3464
            ctypes.c_int(num_iteration),
3465
            ctypes.c_int(importance_type_int),
3466
            ctypes.c_int64(buffer_len),
wxchan's avatar
wxchan committed
3467
            ctypes.byref(tmp_out_len),
Guolin Ke's avatar
Guolin Ke committed
3468
            ptr_string_buffer))
wxchan's avatar
wxchan committed
3469
        actual_len = tmp_out_len.value
3470
        # if buffer length is not long enough, reallocate a buffer
wxchan's avatar
wxchan committed
3471
3472
3473
3474
3475
        if actual_len > buffer_len:
            string_buffer = ctypes.create_string_buffer(actual_len)
            ptr_string_buffer = ctypes.c_char_p(*[ctypes.addressof(string_buffer)])
            _safe_call(_LIB.LGBM_BoosterDumpModel(
                self.handle,
3476
                ctypes.c_int(start_iteration),
Guolin Ke's avatar
Guolin Ke committed
3477
                ctypes.c_int(num_iteration),
3478
                ctypes.c_int(importance_type_int),
3479
                ctypes.c_int64(actual_len),
wxchan's avatar
wxchan committed
3480
                ctypes.byref(tmp_out_len),
Guolin Ke's avatar
Guolin Ke committed
3481
                ptr_string_buffer))
3482
        ret = json.loads(string_buffer.value.decode('utf-8'), object_hook=object_hook)
3483
3484
3485
        ret['pandas_categorical'] = json.loads(json.dumps(self.pandas_categorical,
                                                          default=json_default_with_numpy))
        return ret
wxchan's avatar
wxchan committed
3486

3487
    def predict(self, data, start_iteration=0, num_iteration=None,
3488
                raw_score=False, pred_leaf=False, pred_contrib=False,
3489
                data_has_header=False, **kwargs):
3490
        """Make a prediction.
wxchan's avatar
wxchan committed
3491
3492
3493

        Parameters
        ----------
3494
        data : str, pathlib.Path, numpy array, pandas DataFrame, H2O DataTable's Frame or scipy.sparse
3495
            Data source for prediction.
3496
            If str or pathlib.Path, it represents the path to a text file (CSV, TSV, or LibSVM).
3497
        start_iteration : int, optional (default=0)
3498
            Start index of the iteration to predict.
3499
            If <= 0, starts from the first iteration.
3500
        num_iteration : int or None, optional (default=None)
3501
3502
3503
3504
            Total number of iterations used in the prediction.
            If None, if the best iteration exists and start_iteration <= 0, the best iteration is used;
            otherwise, all iterations from ``start_iteration`` are used (no limits).
            If <= 0, all iterations from ``start_iteration`` are used (no limits).
3505
3506
3507
3508
        raw_score : bool, optional (default=False)
            Whether to predict raw scores.
        pred_leaf : bool, optional (default=False)
            Whether to predict leaf index.
3509
3510
        pred_contrib : bool, optional (default=False)
            Whether to predict feature contributions.
3511

Nikita Titov's avatar
Nikita Titov committed
3512
3513
3514
3515
3516
3517
3518
            .. note::

                If you want to get more explanations for your model's predictions using SHAP values,
                like SHAP interaction values,
                you can install the shap package (https://github.com/slundberg/shap).
                Note that unlike the shap package, with ``pred_contrib`` we return a matrix with an extra
                column, where the last column is the expected value.
3519

3520
3521
        data_has_header : bool, optional (default=False)
            Whether the data has header.
3522
            Used only if data is str.
3523
3524
        **kwargs
            Other parameters for the prediction.
wxchan's avatar
wxchan committed
3525
3526
3527

        Returns
        -------
3528
        result : numpy array, scipy.sparse or list of scipy.sparse
3529
            Prediction result.
3530
            Can be sparse or a list of sparse objects (each element represents predictions for one class) for feature contributions (when ``pred_contrib=True``).
wxchan's avatar
wxchan committed
3531
        """
3532
        predictor = self._to_predictor(deepcopy(kwargs))
3533
        if num_iteration is None:
3534
            if start_iteration <= 0:
3535
3536
3537
3538
                num_iteration = self.best_iteration
            else:
                num_iteration = -1
        return predictor.predict(data, start_iteration, num_iteration,
3539
                                 raw_score, pred_leaf, pred_contrib,
3540
                                 data_has_header)
wxchan's avatar
wxchan committed
3541

3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
    def refit(
        self,
        data,
        label,
        decay_rate=0.9,
        reference=None,
        weight=None,
        group=None,
        init_score=None,
        feature_name='auto',
        categorical_feature='auto',
        dataset_params=None,
        free_raw_data=True,
        **kwargs
    ):
Guolin Ke's avatar
Guolin Ke committed
3557
3558
3559
3560
        """Refit the existing Booster by new data.

        Parameters
        ----------
3561
        data : str, pathlib.Path, numpy array, pandas DataFrame, H2O DataTable's Frame or scipy.sparse
Guolin Ke's avatar
Guolin Ke committed
3562
            Data source for refit.
3563
            If str or pathlib.Path, it represents the path to a text file (CSV, TSV, or LibSVM).
3564
        label : list, numpy 1-D array or pandas Series / one-column DataFrame
Guolin Ke's avatar
Guolin Ke committed
3565
3566
            Label for refit.
        decay_rate : float, optional (default=0.9)
3567
3568
            Decay rate of refit,
            will use ``leaf_output = decay_rate * old_leaf_output + (1.0 - decay_rate) * new_leaf_output`` to refit trees.
3569
3570
3571
        reference : Dataset or None, optional (default=None)
            Reference for ``data``.
        weight : list, numpy 1-D array, pandas Series or None, optional (default=None)
3572
            Weight for each ``data`` instance. Weights should be non-negative.
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
        group : list, numpy 1-D array, pandas Series or None, optional (default=None)
            Group/query size for ``data``.
            Only used in the learning-to-rank task.
            sum(group) = n_samples.
            For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
            where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
        init_score : list, list of lists (for multi-class task), numpy array, pandas Series, pandas DataFrame (for multi-class task), or None, optional (default=None)
            Init score for ``data``.
        feature_name : list of str, or 'auto', optional (default="auto")
            Feature names for ``data``.
            If 'auto' and data is pandas DataFrame, data columns names are used.
        categorical_feature : list of str or int, or 'auto', optional (default="auto")
            Categorical features for ``data``.
            If list of int, interpreted as indices.
            If list of str, interpreted as feature names (need to specify ``feature_name`` as well).
            If 'auto' and data is pandas DataFrame, pandas unordered categorical columns are used.
3589
            All values in categorical features will be cast to int32 and thus should be less than int32 max value (2147483647).
3590
3591
3592
            Large values could be memory consuming. Consider using consecutive integers starting from zero.
            All negative values in categorical features will be treated as missing values.
            The output cannot be monotonically constrained with respect to a categorical feature.
3593
            Floating point numbers in categorical features will be rounded towards 0.
3594
3595
3596
3597
        dataset_params : dict or None, optional (default=None)
            Other parameters for Dataset ``data``.
        free_raw_data : bool, optional (default=True)
            If True, raw data is freed after constructing inner Dataset for ``data``.
3598
3599
        **kwargs
            Other parameters for refit.
3600
            These parameters will be passed to ``predict`` method.
Guolin Ke's avatar
Guolin Ke committed
3601
3602
3603
3604
3605
3606

        Returns
        -------
        result : Booster
            Refitted Booster.
        """
3607
3608
        if self.__set_objective_to_none:
            raise LightGBMError('Cannot refit due to null objective function.')
3609
3610
        if dataset_params is None:
            dataset_params = {}
3611
        predictor = self._to_predictor(deepcopy(kwargs))
3612
        leaf_preds = predictor.predict(data, -1, pred_leaf=True)
3613
        nrow, ncol = leaf_preds.shape
3614
        out_is_linear = ctypes.c_int(0)
3615
3616
3617
        _safe_call(_LIB.LGBM_BoosterGetLinear(
            self.handle,
            ctypes.byref(out_is_linear)))
Nikita Titov's avatar
Nikita Titov committed
3618
3619
3620
3621
3622
        new_params = _choose_param_value(
            main_param_name="linear_tree",
            params=self.params,
            default_value=None
        )
3623
        new_params["linear_tree"] = bool(out_is_linear.value)
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
        new_params.update(dataset_params)
        train_set = Dataset(
            data=data,
            label=label,
            reference=reference,
            weight=weight,
            group=group,
            init_score=init_score,
            feature_name=feature_name,
            categorical_feature=categorical_feature,
            params=new_params,
            free_raw_data=free_raw_data,
        )
3637
        new_params['refit_decay_rate'] = decay_rate
3638
        new_booster = Booster(new_params, train_set)
Guolin Ke's avatar
Guolin Ke committed
3639
3640
3641
3642
3643
        # Copy models
        _safe_call(_LIB.LGBM_BoosterMerge(
            new_booster.handle,
            predictor.handle))
        leaf_preds = leaf_preds.reshape(-1)
3644
        ptr_data, _, _ = c_int_array(leaf_preds)
Guolin Ke's avatar
Guolin Ke committed
3645
3646
3647
        _safe_call(_LIB.LGBM_BoosterRefit(
            new_booster.handle,
            ptr_data,
3648
3649
            ctypes.c_int32(nrow),
            ctypes.c_int32(ncol)))
3650
3651
        new_booster.network = self.network
        new_booster.__attr = self.__attr.copy()
Guolin Ke's avatar
Guolin Ke committed
3652
3653
        return new_booster

3654
    def get_leaf_output(self, tree_id, leaf_id):
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
        """Get the output of a leaf.

        Parameters
        ----------
        tree_id : int
            The index of the tree.
        leaf_id : int
            The index of the leaf in the tree.

        Returns
        -------
        result : float
            The output of the leaf.
        """
3669
3670
3671
3672
3673
3674
3675
3676
        ret = ctypes.c_double(0)
        _safe_call(_LIB.LGBM_BoosterGetLeafValue(
            self.handle,
            ctypes.c_int(tree_id),
            ctypes.c_int(leaf_id),
            ctypes.byref(ret)))
        return ret.value

3677
    def _to_predictor(self, pred_parameter=None):
3678
        """Convert to predictor."""
3679
        predictor = _InnerPredictor(booster_handle=self.handle, pred_parameter=pred_parameter)
3680
        predictor.pandas_categorical = self.pandas_categorical
wxchan's avatar
wxchan committed
3681
3682
        return predictor

3683
    def num_feature(self):
3684
3685
3686
3687
3688
3689
3690
        """Get number of features.

        Returns
        -------
        num_feature : int
            The number of features.
        """
3691
3692
3693
3694
3695
3696
        out_num_feature = ctypes.c_int(0)
        _safe_call(_LIB.LGBM_BoosterGetNumFeature(
            self.handle,
            ctypes.byref(out_num_feature)))
        return out_num_feature.value

wxchan's avatar
wxchan committed
3697
    def feature_name(self):
3698
        """Get names of features.
wxchan's avatar
wxchan committed
3699
3700
3701

        Returns
        -------
3702
        result : list of str
3703
            List with names of features.
wxchan's avatar
wxchan committed
3704
        """
3705
        num_feature = self.num_feature()
3706
        # Get name of features
wxchan's avatar
wxchan committed
3707
        tmp_out_len = ctypes.c_int(0)
3708
3709
        reserved_string_buffer_size = 255
        required_string_buffer_size = ctypes.c_size_t(0)
3710
        string_buffers = [ctypes.create_string_buffer(reserved_string_buffer_size) for _ in range(num_feature)]
wxchan's avatar
wxchan committed
3711
3712
3713
        ptr_string_buffers = (ctypes.c_char_p * num_feature)(*map(ctypes.addressof, string_buffers))
        _safe_call(_LIB.LGBM_BoosterGetFeatureNames(
            self.handle,
3714
            ctypes.c_int(num_feature),
wxchan's avatar
wxchan committed
3715
            ctypes.byref(tmp_out_len),
3716
            ctypes.c_size_t(reserved_string_buffer_size),
3717
            ctypes.byref(required_string_buffer_size),
wxchan's avatar
wxchan committed
3718
3719
3720
            ptr_string_buffers))
        if num_feature != tmp_out_len.value:
            raise ValueError("Length of feature names doesn't equal with num_feature")
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
        actual_string_buffer_size = required_string_buffer_size.value
        # if buffer length is not long enough, reallocate buffers
        if reserved_string_buffer_size < actual_string_buffer_size:
            string_buffers = [ctypes.create_string_buffer(actual_string_buffer_size) for _ in range(num_feature)]
            ptr_string_buffers = (ctypes.c_char_p * num_feature)(*map(ctypes.addressof, string_buffers))
            _safe_call(_LIB.LGBM_BoosterGetFeatureNames(
                self.handle,
                ctypes.c_int(num_feature),
                ctypes.byref(tmp_out_len),
                ctypes.c_size_t(actual_string_buffer_size),
                ctypes.byref(required_string_buffer_size),
                ptr_string_buffers))
3733
        return [string_buffers[i].value.decode('utf-8') for i in range(num_feature)]
wxchan's avatar
wxchan committed
3734

3735
    def feature_importance(self, importance_type='split', iteration=None):
3736
        """Get feature importances.
3737

3738
3739
        Parameters
        ----------
3740
        importance_type : str, optional (default="split")
3741
3742
3743
            How the importance is calculated.
            If "split", result contains numbers of times the feature is used in a model.
            If "gain", result contains total gains of splits which use the feature.
3744
3745
3746
3747
        iteration : int or None, optional (default=None)
            Limit number of iterations in the feature importance calculation.
            If None, if the best iteration exists, it is used; otherwise, all trees are used.
            If <= 0, all trees are used (no limits).
3748

3749
3750
        Returns
        -------
3751
3752
        result : numpy array
            Array with feature importances.
3753
        """
3754
3755
        if iteration is None:
            iteration = self.best_iteration
3756
        importance_type_int = FEATURE_IMPORTANCE_TYPE_MAPPER[importance_type]
3757
        result = np.empty(self.num_feature(), dtype=np.float64)
3758
3759
3760
3761
3762
3763
        _safe_call(_LIB.LGBM_BoosterFeatureImportance(
            self.handle,
            ctypes.c_int(iteration),
            ctypes.c_int(importance_type_int),
            result.ctypes.data_as(ctypes.POINTER(ctypes.c_double))))
        if importance_type_int == 0:
3764
            return result.astype(np.int32)
3765
3766
        else:
            return result
3767

3768
3769
3770
3771
3772
    def get_split_value_histogram(self, feature, bins=None, xgboost_style=False):
        """Get split value histogram for the specified feature.

        Parameters
        ----------
3773
        feature : int or str
3774
3775
            The feature name or index the histogram is calculated for.
            If int, interpreted as index.
3776
            If str, interpreted as name.
3777

Nikita Titov's avatar
Nikita Titov committed
3778
3779
3780
            .. warning::

                Categorical features are not supported.
3781

3782
        bins : int, str or None, optional (default=None)
3783
3784
3785
            The maximum number of bins.
            If None, or int and > number of unique split values and ``xgboost_style=True``,
            the number of bins equals number of unique split values.
3786
            If str, it should be one from the list of the supported values by ``numpy.histogram()`` function.
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
        xgboost_style : bool, optional (default=False)
            Whether the returned result should be in the same form as it is in XGBoost.
            If False, the returned value is tuple of 2 numpy arrays as it is in ``numpy.histogram()`` function.
            If True, the returned value is matrix, in which the first column is the right edges of non-empty bins
            and the second one is the histogram values.

        Returns
        -------
        result_tuple : tuple of 2 numpy arrays
            If ``xgboost_style=False``, the values of the histogram of used splitting values for the specified feature
            and the bin edges.
        result_array_like : numpy array or pandas DataFrame (if pandas is installed)
            If ``xgboost_style=True``, the histogram of used splitting values for the specified feature.
        """
        def add(root):
            """Recursively add thresholds."""
            if 'split_index' in root:  # non-leaf
3804
                if feature_names is not None and isinstance(feature, str):
3805
3806
3807
3808
                    split_feature = feature_names[root['split_feature']]
                else:
                    split_feature = root['split_feature']
                if split_feature == feature:
3809
                    if isinstance(root['threshold'], str):
3810
3811
3812
                        raise LightGBMError('Cannot compute split value histogram for the categorical feature')
                    else:
                        values.append(root['threshold'])
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
                add(root['left_child'])
                add(root['right_child'])

        model = self.dump_model()
        feature_names = model.get('feature_names')
        tree_infos = model['tree_info']
        values = []
        for tree_info in tree_infos:
            add(tree_info['tree_structure'])

3823
        if bins is None or isinstance(bins, int) and xgboost_style:
3824
3825
3826
3827
3828
3829
3830
            n_unique = len(np.unique(values))
            bins = max(min(n_unique, bins) if bins is not None else n_unique, 1)
        hist, bin_edges = np.histogram(values, bins=bins)
        if xgboost_style:
            ret = np.column_stack((bin_edges[1:], hist))
            ret = ret[ret[:, 1] > 0]
            if PANDAS_INSTALLED:
3831
                return pd_DataFrame(ret, columns=['SplitValue', 'Count'])
3832
3833
3834
3835
3836
            else:
                return ret
        else:
            return hist, bin_edges

wxchan's avatar
wxchan committed
3837
    def __inner_eval(self, data_name, data_idx, feval=None):
3838
        """Evaluate training or validation data."""
wxchan's avatar
wxchan committed
3839
        if data_idx >= self.__num_dataset:
3840
            raise ValueError("Data_idx should be smaller than number of dataset")
wxchan's avatar
wxchan committed
3841
3842
3843
        self.__get_eval_info()
        ret = []
        if self.__num_inner_eval > 0:
3844
            result = np.empty(self.__num_inner_eval, dtype=np.float64)
Guolin Ke's avatar
Guolin Ke committed
3845
            tmp_out_len = ctypes.c_int(0)
wxchan's avatar
wxchan committed
3846
3847
            _safe_call(_LIB.LGBM_BoosterGetEval(
                self.handle,
Guolin Ke's avatar
Guolin Ke committed
3848
                ctypes.c_int(data_idx),
wxchan's avatar
wxchan committed
3849
                ctypes.byref(tmp_out_len),
Guolin Ke's avatar
Guolin Ke committed
3850
                result.ctypes.data_as(ctypes.POINTER(ctypes.c_double))))
wxchan's avatar
wxchan committed
3851
            if tmp_out_len.value != self.__num_inner_eval:
3852
                raise ValueError("Wrong length of eval results")
3853
            for i in range(self.__num_inner_eval):
3854
3855
                ret.append((data_name, self.__name_inner_eval[i],
                            result[i], self.__higher_better_inner_eval[i]))
3856
3857
        if callable(feval):
            feval = [feval]
wxchan's avatar
wxchan committed
3858
3859
3860
3861
3862
        if feval is not None:
            if data_idx == 0:
                cur_data = self.train_set
            else:
                cur_data = self.valid_sets[data_idx - 1]
3863
3864
3865
3866
3867
3868
3869
3870
3871
            for eval_function in feval:
                if eval_function is None:
                    continue
                feval_ret = eval_function(self.__inner_predict(data_idx), cur_data)
                if isinstance(feval_ret, list):
                    for eval_name, val, is_higher_better in feval_ret:
                        ret.append((data_name, eval_name, val, is_higher_better))
                else:
                    eval_name, val, is_higher_better = feval_ret
wxchan's avatar
wxchan committed
3872
3873
3874
3875
                    ret.append((data_name, eval_name, val, is_higher_better))
        return ret

    def __inner_predict(self, data_idx):
3876
        """Predict for training and validation dataset."""
wxchan's avatar
wxchan committed
3877
        if data_idx >= self.__num_dataset:
3878
            raise ValueError("Data_idx should be smaller than number of dataset")
wxchan's avatar
wxchan committed
3879
3880
3881
3882
3883
        if self.__inner_predict_buffer[data_idx] is None:
            if data_idx == 0:
                n_preds = self.train_set.num_data() * self.__num_class
            else:
                n_preds = self.valid_sets[data_idx - 1].num_data() * self.__num_class
3884
            self.__inner_predict_buffer[data_idx] = np.empty(n_preds, dtype=np.float64)
3885
        # avoid to predict many time in one iteration
wxchan's avatar
wxchan committed
3886
3887
        if not self.__is_predicted_cur_iter[data_idx]:
            tmp_out_len = ctypes.c_int64(0)
Guolin Ke's avatar
Guolin Ke committed
3888
            data_ptr = self.__inner_predict_buffer[data_idx].ctypes.data_as(ctypes.POINTER(ctypes.c_double))
wxchan's avatar
wxchan committed
3889
3890
            _safe_call(_LIB.LGBM_BoosterGetPredict(
                self.handle,
Guolin Ke's avatar
Guolin Ke committed
3891
                ctypes.c_int(data_idx),
wxchan's avatar
wxchan committed
3892
3893
3894
                ctypes.byref(tmp_out_len),
                data_ptr))
            if tmp_out_len.value != len(self.__inner_predict_buffer[data_idx]):
3895
                raise ValueError(f"Wrong length of predict results for data {data_idx}")
wxchan's avatar
wxchan committed
3896
            self.__is_predicted_cur_iter[data_idx] = True
3897
3898
3899
3900
3901
        result = self.__inner_predict_buffer[data_idx]
        if self.__num_class > 1:
            num_data = result.size // self.__num_class
            result = result.reshape(num_data, self.__num_class, order='F')
        return result
wxchan's avatar
wxchan committed
3902
3903

    def __get_eval_info(self):
3904
        """Get inner evaluation count and names."""
wxchan's avatar
wxchan committed
3905
3906
        if self.__need_reload_eval_info:
            self.__need_reload_eval_info = False
Guolin Ke's avatar
Guolin Ke committed
3907
            out_num_eval = ctypes.c_int(0)
3908
            # Get num of inner evals
wxchan's avatar
wxchan committed
3909
3910
3911
3912
3913
            _safe_call(_LIB.LGBM_BoosterGetEvalCounts(
                self.handle,
                ctypes.byref(out_num_eval)))
            self.__num_inner_eval = out_num_eval.value
            if self.__num_inner_eval > 0:
3914
                # Get name of eval metrics
Guolin Ke's avatar
Guolin Ke committed
3915
                tmp_out_len = ctypes.c_int(0)
3916
3917
3918
                reserved_string_buffer_size = 255
                required_string_buffer_size = ctypes.c_size_t(0)
                string_buffers = [
3919
                    ctypes.create_string_buffer(reserved_string_buffer_size) for _ in range(self.__num_inner_eval)
3920
                ]
wxchan's avatar
wxchan committed
3921
                ptr_string_buffers = (ctypes.c_char_p * self.__num_inner_eval)(*map(ctypes.addressof, string_buffers))
wxchan's avatar
wxchan committed
3922
3923
                _safe_call(_LIB.LGBM_BoosterGetEvalNames(
                    self.handle,
3924
                    ctypes.c_int(self.__num_inner_eval),
wxchan's avatar
wxchan committed
3925
                    ctypes.byref(tmp_out_len),
3926
                    ctypes.c_size_t(reserved_string_buffer_size),
3927
                    ctypes.byref(required_string_buffer_size),
wxchan's avatar
wxchan committed
3928
3929
                    ptr_string_buffers))
                if self.__num_inner_eval != tmp_out_len.value:
3930
                    raise ValueError("Length of eval names doesn't equal with num_evals")
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
                actual_string_buffer_size = required_string_buffer_size.value
                # if buffer length is not long enough, reallocate buffers
                if reserved_string_buffer_size < actual_string_buffer_size:
                    string_buffers = [
                        ctypes.create_string_buffer(actual_string_buffer_size) for _ in range(self.__num_inner_eval)
                    ]
                    ptr_string_buffers = (ctypes.c_char_p * self.__num_inner_eval)(*map(ctypes.addressof, string_buffers))
                    _safe_call(_LIB.LGBM_BoosterGetEvalNames(
                        self.handle,
                        ctypes.c_int(self.__num_inner_eval),
                        ctypes.byref(tmp_out_len),
                        ctypes.c_size_t(actual_string_buffer_size),
                        ctypes.byref(required_string_buffer_size),
                        ptr_string_buffers))
                self.__name_inner_eval = [
                    string_buffers[i].value.decode('utf-8') for i in range(self.__num_inner_eval)
                ]
                self.__higher_better_inner_eval = [
                    name.startswith(('auc', 'ndcg@', 'map@', 'average_precision')) for name in self.__name_inner_eval
                ]
3951

wxchan's avatar
wxchan committed
3952
    def attr(self, key):
3953
        """Get attribute string from the Booster.
wxchan's avatar
wxchan committed
3954
3955
3956

        Parameters
        ----------
3957
        key : str
3958
            The name of the attribute.
wxchan's avatar
wxchan committed
3959
3960
3961

        Returns
        -------
3962
        value : str or None
3963
            The attribute value.
Nikita Titov's avatar
Nikita Titov committed
3964
            Returns None if attribute does not exist.
wxchan's avatar
wxchan committed
3965
        """
3966
        return self.__attr.get(key, None)
wxchan's avatar
wxchan committed
3967
3968

    def set_attr(self, **kwargs):
3969
        """Set attributes to the Booster.
wxchan's avatar
wxchan committed
3970
3971
3972
3973

        Parameters
        ----------
        **kwargs
3974
3975
            The attributes to set.
            Setting a value to None deletes an attribute.
Nikita Titov's avatar
Nikita Titov committed
3976
3977
3978
3979

        Returns
        -------
        self : Booster
3980
            Booster with set attributes.
wxchan's avatar
wxchan committed
3981
3982
3983
        """
        for key, value in kwargs.items():
            if value is not None:
3984
                if not isinstance(value, str):
Nikita Titov's avatar
Nikita Titov committed
3985
                    raise ValueError("Only string values are accepted")
wxchan's avatar
wxchan committed
3986
3987
3988
                self.__attr[key] = value
            else:
                self.__attr.pop(key, None)
Nikita Titov's avatar
Nikita Titov committed
3989
        return self