basic.py 173 KB
Newer Older
wxchan's avatar
wxchan committed
1
# coding: utf-8
2
"""Wrapper for C API of LightGBM."""
3
import abc
wxchan's avatar
wxchan committed
4
import ctypes
5
import json
wxchan's avatar
wxchan committed
6
import warnings
7
from collections import OrderedDict
8
from copy import deepcopy
9
from enum import Enum
10
from functools import wraps
11
from os import SEEK_END, environ
12
13
from os.path import getsize
from pathlib import Path
14
from tempfile import NamedTemporaryFile
15
from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Union
wxchan's avatar
wxchan committed
16
17
18
19

import numpy as np
import scipy.sparse

20
from .compat import PANDAS_INSTALLED, concat, dt_DataTable, pd_CategoricalDtype, pd_DataFrame, pd_Series
wxchan's avatar
wxchan committed
21
22
from .libpath import find_lib_path

23
_DatasetHandle = ctypes.c_void_p
24
25
_LGBM_EvalFunctionResultType = Tuple[str, float, bool]
_LGBM_BoosterEvalMethodResultType = Tuple[str, str, float, bool]
26
27
28
29
30
31
_LGBM_LabelType = Union[
    list,
    np.ndarray,
    pd_Series,
    pd_DataFrame
]
32

33
34
35
ZERO_THRESHOLD = 1e-35


36
37
38
39
def _is_zero(x: float) -> bool:
    return -ZERO_THRESHOLD <= x <= ZERO_THRESHOLD


40
def _get_sample_count(total_nrow: int, params: str) -> int:
41
42
43
    sample_cnt = ctypes.c_int(0)
    _safe_call(_LIB.LGBM_GetSampleCount(
        ctypes.c_int32(total_nrow),
44
        _c_str(params),
45
46
47
48
        ctypes.byref(sample_cnt),
    ))
    return sample_cnt.value

wxchan's avatar
wxchan committed
49

50
51
52
53
54
55
class _MissingType(Enum):
    NONE = 'None'
    NAN = 'NaN'
    ZERO = 'Zero'


56
class _DummyLogger:
57
    def info(self, msg: str) -> None:
58
59
        print(msg)

60
    def warning(self, msg: str) -> None:
61
62
63
        warnings.warn(msg, stacklevel=3)


64
65
66
_LOGGER: Any = _DummyLogger()
_INFO_METHOD_NAME = "info"
_WARNING_METHOD_NAME = "warning"
67
68


69
70
71
def register_logger(
    logger: Any, info_method_name: str = "info", warning_method_name: str = "warning"
) -> None:
72
73
74
75
    """Register custom logger.

    Parameters
    ----------
76
    logger : Any
77
        Custom logger.
78
79
80
81
    info_method_name : str, optional (default="info")
        Method used to log info messages.
    warning_method_name : str, optional (default="warning")
        Method used to log warning messages.
82
    """
83
84
85
86
87
88
89
90
91
    def _has_method(logger: Any, method_name: str) -> bool:
        return callable(getattr(logger, method_name, None))

    if not _has_method(logger, info_method_name) or not _has_method(logger, warning_method_name):
        raise TypeError(
            f"Logger must provide '{info_method_name}' and '{warning_method_name}' method"
        )

    global _LOGGER, _INFO_METHOD_NAME, _WARNING_METHOD_NAME
92
    _LOGGER = logger
93
94
    _INFO_METHOD_NAME = info_method_name
    _WARNING_METHOD_NAME = warning_method_name
95
96


97
def _normalize_native_string(func: Callable[[str], None]) -> Callable[[str], None]:
98
    """Join log messages from native library which come by chunks."""
99
    msg_normalized: List[str] = []
100
101

    @wraps(func)
102
    def wrapper(msg: str) -> None:
103
104
105
106
107
108
109
110
111
112
113
        nonlocal msg_normalized
        if msg.strip() == '':
            msg = ''.join(msg_normalized)
            msg_normalized = []
            return func(msg)
        else:
            msg_normalized.append(msg)

    return wrapper


114
def _log_info(msg: str) -> None:
115
    getattr(_LOGGER, _INFO_METHOD_NAME)(msg)
116
117


118
def _log_warning(msg: str) -> None:
119
    getattr(_LOGGER, _WARNING_METHOD_NAME)(msg)
120
121
122


@_normalize_native_string
123
def _log_native(msg: str) -> None:
124
    getattr(_LOGGER, _INFO_METHOD_NAME)(msg)
125
126


127
def _log_callback(msg: bytes) -> None:
128
    """Redirect logs from native library into Python."""
129
    _log_native(str(msg.decode('utf-8')))
130
131


132
def _load_lib() -> ctypes.CDLL:
133
    """Load LightGBM library."""
wxchan's avatar
wxchan committed
134
135
136
    lib_path = find_lib_path()
    lib = ctypes.cdll.LoadLibrary(lib_path[0])
    lib.LGBM_GetLastError.restype = ctypes.c_char_p
137
138
139
    callback = ctypes.CFUNCTYPE(None, ctypes.c_char_p)
    lib.callback = callback(_log_callback)
    if lib.LGBM_RegisterLogCallback(lib.callback) != 0:
140
        raise LightGBMError(lib.LGBM_GetLastError().decode('utf-8'))
wxchan's avatar
wxchan committed
141
142
    return lib

wxchan's avatar
wxchan committed
143

144
145
146
147
148
149
150
# we don't need lib_lightgbm while building docs
_LIB: ctypes.CDLL
if environ.get('LIGHTGBM_BUILD_DOC', False):
    from unittest.mock import Mock  # isort: skip
    _LIB = Mock(ctypes.CDLL)  # type: ignore
else:
    _LIB = _load_lib()
wxchan's avatar
wxchan committed
151

wxchan's avatar
wxchan committed
152

153
_NUMERIC_TYPES = (int, float, bool)
154
_ArrayLike = Union[List, np.ndarray, pd_Series]
155
156


157
def _safe_call(ret: int) -> None:
158
159
    """Check the return value from C API call.

wxchan's avatar
wxchan committed
160
161
162
    Parameters
    ----------
    ret : int
163
        The return value from C API calls.
wxchan's avatar
wxchan committed
164
165
    """
    if ret != 0:
166
        raise LightGBMError(_LIB.LGBM_GetLastError().decode('utf-8'))
wxchan's avatar
wxchan committed
167

wxchan's avatar
wxchan committed
168

169
def _is_numeric(obj: Any) -> bool:
170
    """Check whether object is a number or not, include numpy number, etc."""
wxchan's avatar
wxchan committed
171
172
173
    try:
        float(obj)
        return True
wxchan's avatar
wxchan committed
174
175
176
    except (TypeError, ValueError):
        # TypeError: obj is not a string or a number
        # ValueError: invalid literal
wxchan's avatar
wxchan committed
177
178
        return False

wxchan's avatar
wxchan committed
179

180
def _is_numpy_1d_array(data: Any) -> bool:
181
    """Check whether data is a numpy 1-D array."""
182
    return isinstance(data, np.ndarray) and len(data.shape) == 1
wxchan's avatar
wxchan committed
183

wxchan's avatar
wxchan committed
184

185
def _is_numpy_column_array(data: Any) -> bool:
186
187
188
189
190
191
192
    """Check whether data is a column numpy array."""
    if not isinstance(data, np.ndarray):
        return False
    shape = data.shape
    return len(shape) == 2 and shape[1] == 1


193
def _cast_numpy_array_to_dtype(array, dtype):
194
    """Cast numpy array to given dtype."""
195
196
197
198
199
    if array.dtype == dtype:
        return array
    return array.astype(dtype=dtype, copy=False)


200
def _is_1d_list(data: Any) -> bool:
201
    """Check whether data is a 1-D list."""
202
    return isinstance(data, list) and (not data or _is_numeric(data[0]))
wxchan's avatar
wxchan committed
203

wxchan's avatar
wxchan committed
204

205
206
207
def _is_1d_collection(data: Any) -> bool:
    """Check whether data is a 1-D collection."""
    return (
208
        _is_numpy_1d_array(data)
209
        or _is_numpy_column_array(data)
210
        or _is_1d_list(data)
211
212
213
214
        or isinstance(data, pd_Series)
    )


215
def _list_to_1d_numpy(data, dtype=np.float32, name='list'):
216
    """Convert data to numpy 1-D array."""
217
    if _is_numpy_1d_array(data):
218
        return _cast_numpy_array_to_dtype(data, dtype)
219
    elif _is_numpy_column_array(data):
220
221
        _log_warning('Converting column-vector to 1d array')
        array = data.ravel()
222
        return _cast_numpy_array_to_dtype(array, dtype)
223
    elif _is_1d_list(data):
wxchan's avatar
wxchan committed
224
        return np.array(data, dtype=dtype, copy=False)
225
    elif isinstance(data, pd_Series):
226
        _check_for_bad_pandas_dtypes(data.to_frame().dtypes)
227
        return np.array(data, dtype=dtype, copy=False)  # SparseArray should be supported as well
wxchan's avatar
wxchan committed
228
    else:
229
230
        raise TypeError(f"Wrong type({type(data).__name__}) for {name}.\n"
                        "It should be list, numpy 1-D array or pandas Series")
wxchan's avatar
wxchan committed
231

wxchan's avatar
wxchan committed
232

233
234
235
236
237
238
239
def _is_numpy_2d_array(data: Any) -> bool:
    """Check whether data is a numpy 2-D array."""
    return isinstance(data, np.ndarray) and len(data.shape) == 2 and data.shape[1] > 1


def _is_2d_list(data: Any) -> bool:
    """Check whether data is a 2-D list."""
240
    return isinstance(data, list) and len(data) > 0 and _is_1d_list(data[0])
241
242
243
244
245
246
247
248
249
250
251
252
253
254


def _is_2d_collection(data: Any) -> bool:
    """Check whether data is a 2-D collection."""
    return (
        _is_numpy_2d_array(data)
        or _is_2d_list(data)
        or isinstance(data, pd_DataFrame)
    )


def _data_to_2d_numpy(data: Any, dtype: type = np.float32, name: str = 'list') -> np.ndarray:
    """Convert data to numpy 2-D array."""
    if _is_numpy_2d_array(data):
255
        return _cast_numpy_array_to_dtype(data, dtype)
256
257
258
    if _is_2d_list(data):
        return np.array(data, dtype=dtype)
    if isinstance(data, pd_DataFrame):
259
        _check_for_bad_pandas_dtypes(data.dtypes)
260
        return _cast_numpy_array_to_dtype(data.values, dtype)
261
262
263
264
    raise TypeError(f"Wrong type({type(data).__name__}) for {name}.\n"
                    "It should be list of lists, numpy 2-D array or pandas DataFrame")


265
def _cfloat32_array_to_numpy(cptr: Any, length: int) -> np.ndarray:
266
    """Convert a ctypes float pointer array to a numpy array."""
wxchan's avatar
wxchan committed
267
    if isinstance(cptr, ctypes.POINTER(ctypes.c_float)):
268
        return np.ctypeslib.as_array(cptr, shape=(length,)).copy()
wxchan's avatar
wxchan committed
269
    else:
270
        raise RuntimeError('Expected float pointer')
wxchan's avatar
wxchan committed
271

Guolin Ke's avatar
Guolin Ke committed
272

273
def _cfloat64_array_to_numpy(cptr: Any, length: int) -> np.ndarray:
274
    """Convert a ctypes double pointer array to a numpy array."""
Guolin Ke's avatar
Guolin Ke committed
275
    if isinstance(cptr, ctypes.POINTER(ctypes.c_double)):
276
        return np.ctypeslib.as_array(cptr, shape=(length,)).copy()
Guolin Ke's avatar
Guolin Ke committed
277
278
279
    else:
        raise RuntimeError('Expected double pointer')

wxchan's avatar
wxchan committed
280

281
def _cint32_array_to_numpy(cptr: Any, length: int) -> np.ndarray:
282
    """Convert a ctypes int pointer array to a numpy array."""
wxchan's avatar
wxchan committed
283
    if isinstance(cptr, ctypes.POINTER(ctypes.c_int32)):
284
        return np.ctypeslib.as_array(cptr, shape=(length,)).copy()
wxchan's avatar
wxchan committed
285
    else:
286
287
288
        raise RuntimeError('Expected int32 pointer')


289
def _cint64_array_to_numpy(cptr: Any, length: int) -> np.ndarray:
290
291
    """Convert a ctypes int pointer array to a numpy array."""
    if isinstance(cptr, ctypes.POINTER(ctypes.c_int64)):
292
        return np.ctypeslib.as_array(cptr, shape=(length,)).copy()
293
294
    else:
        raise RuntimeError('Expected int64 pointer')
wxchan's avatar
wxchan committed
295

wxchan's avatar
wxchan committed
296

297
def _c_str(string: str) -> ctypes.c_char_p:
298
    """Convert a Python string to C string."""
wxchan's avatar
wxchan committed
299
300
    return ctypes.c_char_p(string.encode('utf-8'))

wxchan's avatar
wxchan committed
301

302
def _c_array(ctype: type, values: List[Any]) -> ctypes.Array:
303
    """Convert a Python array to C array."""
wxchan's avatar
wxchan committed
304
305
    return (ctype * len(values))(*values)

wxchan's avatar
wxchan committed
306

307
def _json_default_with_numpy(obj: Any) -> Any:
308
309
310
311
312
313
314
315
316
    """Convert numpy classes to JSON serializable objects."""
    if isinstance(obj, (np.integer, np.floating, np.bool_)):
        return obj.item()
    elif isinstance(obj, np.ndarray):
        return obj.tolist()
    else:
        return obj


317
def _param_dict_to_str(data: Optional[Dict[str, Any]]) -> str:
318
    """Convert Python dictionary to string, which is passed to C API."""
319
    if data is None or not data:
wxchan's avatar
wxchan committed
320
321
322
        return ""
    pairs = []
    for key, val in data.items():
323
        if isinstance(val, (list, tuple, set)) or _is_numpy_1d_array(val):
324
325
            def to_string(x):
                if isinstance(x, list):
326
                    return f"[{','.join(map(str, x))}]"
327
328
                else:
                    return str(x)
329
            pairs.append(f"{key}={','.join(map(to_string, val))}")
330
        elif isinstance(val, (str, Path, _NUMERIC_TYPES)) or _is_numeric(val):
331
            pairs.append(f"{key}={val}")
332
        elif val is not None:
333
            raise TypeError(f'Unknown type of parameter:{key}, got:{type(val).__name__}')
wxchan's avatar
wxchan committed
334
    return ' '.join(pairs)
335

wxchan's avatar
wxchan committed
336

337
class _TempFile:
338
339
    """Proxy class to workaround errors on Windows."""

340
341
342
    def __enter__(self):
        with NamedTemporaryFile(prefix="lightgbm_tmp_", delete=True) as f:
            self.name = f.name
343
            self.path = Path(self.name)
344
        return self
wxchan's avatar
wxchan committed
345

346
    def __exit__(self, exc_type, exc_val, exc_tb):
347
348
        if self.path.is_file():
            self.path.unlink()
349

wxchan's avatar
wxchan committed
350

351
class LightGBMError(Exception):
352
353
    """Error thrown by LightGBM."""

354
355
356
    pass


357
358
359
360
361
362
363
364
# DeprecationWarning is not shown by default, so let's create our own with higher level
class LGBMDeprecationWarning(UserWarning):
    """Custom deprecation warning."""

    pass


class _ConfigAliases:
365
366
367
368
    # lazy evaluation to allow import without dynamic library, e.g., for docs generation
    aliases = None

    @staticmethod
369
    def _get_all_param_aliases() -> Dict[str, List[str]]:
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
        buffer_len = 1 << 20
        tmp_out_len = ctypes.c_int64(0)
        string_buffer = ctypes.create_string_buffer(buffer_len)
        ptr_string_buffer = ctypes.c_char_p(*[ctypes.addressof(string_buffer)])
        _safe_call(_LIB.LGBM_DumpParamAliases(
            ctypes.c_int64(buffer_len),
            ctypes.byref(tmp_out_len),
            ptr_string_buffer))
        actual_len = tmp_out_len.value
        # if buffer length is not long enough, re-allocate a buffer
        if actual_len > buffer_len:
            string_buffer = ctypes.create_string_buffer(actual_len)
            ptr_string_buffer = ctypes.c_char_p(*[ctypes.addressof(string_buffer)])
            _safe_call(_LIB.LGBM_DumpParamAliases(
                ctypes.c_int64(actual_len),
                ctypes.byref(tmp_out_len),
                ptr_string_buffer))
        aliases = json.loads(
            string_buffer.value.decode('utf-8'),
389
            object_hook=lambda obj: {k: [k] + v for k, v in obj.items()}
390
391
        )
        return aliases
392
393

    @classmethod
394
395
396
    def get(cls, *args) -> Set[str]:
        if cls.aliases is None:
            cls.aliases = cls._get_all_param_aliases()
397
398
        ret = set()
        for i in args:
399
            ret.update(cls.get_sorted(i))
400
401
        return ret

402
403
404
405
406
407
    @classmethod
    def get_sorted(cls, name: str) -> List[str]:
        if cls.aliases is None:
            cls.aliases = cls._get_all_param_aliases()
        return cls.aliases.get(name, [name])

408
    @classmethod
409
410
411
    def get_by_alias(cls, *args) -> Set[str]:
        if cls.aliases is None:
            cls.aliases = cls._get_all_param_aliases()
412
413
414
415
        ret = set(args)
        for arg in args:
            for aliases in cls.aliases.values():
                if arg in aliases:
416
                    ret.update(aliases)
417
418
419
                    break
        return ret

420

421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
def _choose_param_value(main_param_name: str, params: Dict[str, Any], default_value: Any) -> Dict[str, Any]:
    """Get a single parameter value, accounting for aliases.

    Parameters
    ----------
    main_param_name : str
        Name of the main parameter to get a value for. One of the keys of ``_ConfigAliases``.
    params : dict
        Dictionary of LightGBM parameters.
    default_value : Any
        Default value to use for the parameter, if none is found in ``params``.

    Returns
    -------
    params : dict
        A ``params`` dict with exactly one value for ``main_param_name``, and all aliases ``main_param_name`` removed.
        If both ``main_param_name`` and one or more aliases for it are found, the value of ``main_param_name`` will be preferred.
    """
    # avoid side effects on passed-in parameters
    params = deepcopy(params)

442
443
    aliases = _ConfigAliases.get_sorted(main_param_name)
    aliases = [a for a in aliases if a != main_param_name]
444
445

    # if main_param_name was provided, keep that value and remove all aliases
446
    if main_param_name in params.keys():
447
448
449
        for param in aliases:
            params.pop(param, None)
        return params
450

451
452
453
454
455
    # if main param name was not found, search for an alias
    for param in aliases:
        if param in params.keys():
            params[main_param_name] = params[param]
            break
456

457
458
459
460
461
462
463
    if main_param_name in params.keys():
        for param in aliases:
            params.pop(param, None)
        return params

    # neither of main_param_name, aliases were found
    params[main_param_name] = default_value
464
465
466
467

    return params


468
_MAX_INT32 = (1 << 31) - 1
469

470
"""Macro definition of data type in C API of LightGBM"""
471
472
473
474
_C_API_DTYPE_FLOAT32 = 0
_C_API_DTYPE_FLOAT64 = 1
_C_API_DTYPE_INT32 = 2
_C_API_DTYPE_INT64 = 3
Guolin Ke's avatar
Guolin Ke committed
475

476
"""Matrix is row major in Python"""
477
_C_API_IS_ROW_MAJOR = 1
wxchan's avatar
wxchan committed
478

479
"""Macro definition of prediction type in C API of LightGBM"""
480
481
482
483
_C_API_PREDICT_NORMAL = 0
_C_API_PREDICT_RAW_SCORE = 1
_C_API_PREDICT_LEAF_INDEX = 2
_C_API_PREDICT_CONTRIB = 3
wxchan's avatar
wxchan committed
484

485
"""Macro definition of sparse matrix type"""
486
487
_C_API_MATRIX_TYPE_CSR = 0
_C_API_MATRIX_TYPE_CSC = 1
488

489
"""Macro definition of feature importance type"""
490
491
_C_API_FEATURE_IMPORTANCE_SPLIT = 0
_C_API_FEATURE_IMPORTANCE_GAIN = 1
492

493
"""Data type of data field"""
494
495
496
497
498
499
_FIELD_TYPE_MAPPER = {
    "label": _C_API_DTYPE_FLOAT32,
    "weight": _C_API_DTYPE_FLOAT32,
    "init_score": _C_API_DTYPE_FLOAT64,
    "group": _C_API_DTYPE_INT32
}
wxchan's avatar
wxchan committed
500

501
"""String name to int feature importance type mapper"""
502
503
504
505
_FEATURE_IMPORTANCE_TYPE_MAPPER = {
    "split": _C_API_FEATURE_IMPORTANCE_SPLIT,
    "gain": _C_API_FEATURE_IMPORTANCE_GAIN
}
506

wxchan's avatar
wxchan committed
507

508
def _convert_from_sliced_object(data):
509
    """Fix the memory of multi-dimensional sliced object."""
510
    if isinstance(data, np.ndarray) and isinstance(data.base, np.ndarray):
511
        if not data.flags.c_contiguous:
512
513
            _log_warning("Usage of np.ndarray subset (sliced data) is not recommended "
                         "due to it will double the peak memory cost in LightGBM.")
514
515
516
517
            return np.copy(data)
    return data


518
def _c_float_array(data):
519
    """Get pointer of float numpy array / list."""
520
    if _is_1d_list(data):
wxchan's avatar
wxchan committed
521
        data = np.array(data, copy=False)
522
    if _is_numpy_1d_array(data):
523
        data = _convert_from_sliced_object(data)
524
        assert data.flags.c_contiguous
wxchan's avatar
wxchan committed
525
526
        if data.dtype == np.float32:
            ptr_data = data.ctypes.data_as(ctypes.POINTER(ctypes.c_float))
527
            type_data = _C_API_DTYPE_FLOAT32
wxchan's avatar
wxchan committed
528
529
        elif data.dtype == np.float64:
            ptr_data = data.ctypes.data_as(ctypes.POINTER(ctypes.c_double))
530
            type_data = _C_API_DTYPE_FLOAT64
wxchan's avatar
wxchan committed
531
        else:
532
            raise TypeError(f"Expected np.float32 or np.float64, met type({data.dtype})")
wxchan's avatar
wxchan committed
533
    else:
534
        raise TypeError(f"Unknown type({type(data).__name__})")
535
    return (ptr_data, type_data, data)  # return `data` to avoid the temporary copy is freed
wxchan's avatar
wxchan committed
536

wxchan's avatar
wxchan committed
537

538
def _c_int_array(data):
539
    """Get pointer of int numpy array / list."""
540
    if _is_1d_list(data):
wxchan's avatar
wxchan committed
541
        data = np.array(data, copy=False)
542
    if _is_numpy_1d_array(data):
543
        data = _convert_from_sliced_object(data)
544
        assert data.flags.c_contiguous
wxchan's avatar
wxchan committed
545
546
        if data.dtype == np.int32:
            ptr_data = data.ctypes.data_as(ctypes.POINTER(ctypes.c_int32))
547
            type_data = _C_API_DTYPE_INT32
wxchan's avatar
wxchan committed
548
549
        elif data.dtype == np.int64:
            ptr_data = data.ctypes.data_as(ctypes.POINTER(ctypes.c_int64))
550
            type_data = _C_API_DTYPE_INT64
wxchan's avatar
wxchan committed
551
        else:
552
            raise TypeError(f"Expected np.int32 or np.int64, met type({data.dtype})")
wxchan's avatar
wxchan committed
553
    else:
554
        raise TypeError(f"Unknown type({type(data).__name__})")
555
    return (ptr_data, type_data, data)  # return `data` to avoid the temporary copy is freed
wxchan's avatar
wxchan committed
556

wxchan's avatar
wxchan committed
557

558
def _check_for_bad_pandas_dtypes(pandas_dtypes_series):
559
560
561
562
563
564
565
566
    float128 = getattr(np, 'float128', type(None))

    def is_allowed_numpy_dtype(dtype):
        return (
            issubclass(dtype, (np.integer, np.floating, np.bool_))
            and not issubclass(dtype, (np.timedelta64, float128))
        )

567
568
    bad_pandas_dtypes = [
        f'{column_name}: {pandas_dtype}'
569
        for column_name, pandas_dtype in pandas_dtypes_series.items()
570
571
572
573
574
        if not is_allowed_numpy_dtype(pandas_dtype.type)
    ]
    if bad_pandas_dtypes:
        raise ValueError('pandas dtypes must be int, float or bool.\n'
                         f'Fields with bad pandas dtypes: {", ".join(bad_pandas_dtypes)}')
575
576


577
def _data_from_pandas(data, feature_name, categorical_feature, pandas_categorical):
578
    if isinstance(data, pd_DataFrame):
579
580
        if len(data.shape) != 2 or data.shape[0] < 1:
            raise ValueError('Input data must be 2 dimensional and non empty.')
581
        if feature_name == 'auto' or feature_name is None:
582
            data = data.rename(columns=str, copy=False)
583
        cat_cols = [col for col, dtype in zip(data.columns, data.dtypes) if isinstance(dtype, pd_CategoricalDtype)]
584
        cat_cols_not_ordered = [col for col in cat_cols if not data[col].cat.ordered]
585
586
587
588
589
        if pandas_categorical is None:  # train dataset
            pandas_categorical = [list(data[col].cat.categories) for col in cat_cols]
        else:
            if len(cat_cols) != len(pandas_categorical):
                raise ValueError('train and valid dataset categorical_feature do not match.')
590
            for col, category in zip(cat_cols, pandas_categorical):
591
592
                if list(data[col].cat.categories) != list(category):
                    data[col] = data[col].cat.set_categories(category)
593
        if len(cat_cols):  # cat_cols is list
594
            data = data.copy(deep=False)  # not alter origin DataFrame
595
            data[cat_cols] = data[cat_cols].apply(lambda x: x.cat.codes).replace({-1: np.nan})
596
597
598
        if categorical_feature is not None:
            if feature_name is None:
                feature_name = list(data.columns)
599
            if categorical_feature == 'auto':  # use cat cols from DataFrame
600
                categorical_feature = cat_cols_not_ordered
601
602
            else:  # use cat cols specified by user
                categorical_feature = list(categorical_feature)
603
604
        if feature_name == 'auto':
            feature_name = list(data.columns)
605
        _check_for_bad_pandas_dtypes(data.dtypes)
606
607
608
        df_dtypes = [dtype.type for dtype in data.dtypes]
        df_dtypes.append(np.float32)  # so that the target dtype considers floats
        target_dtype = np.find_common_type(df_dtypes, [])
609
610
611
612
613
614
615
616
617
618
        try:
            # most common case (no nullable dtypes)
            data = data.to_numpy(dtype=target_dtype, copy=False)
        except TypeError:
            # 1.0 <= pd version < 1.1 and nullable dtypes, least common case
            # raises error because array is casted to type(pd.NA) and there's no na_value argument
            data = data.astype(target_dtype, copy=False).values
        except ValueError:
            # data has nullable dtypes, but we can specify na_value argument and copy will be made
            data = data.to_numpy(dtype=target_dtype, na_value=np.nan)
619
620
621
622
623
624
    else:
        if feature_name == 'auto':
            feature_name = None
        if categorical_feature == 'auto':
            categorical_feature = None
    return data, feature_name, categorical_feature, pandas_categorical
625
626


627
def _dump_pandas_categorical(pandas_categorical, file_name=None):
628
    categorical_json = json.dumps(pandas_categorical, default=_json_default_with_numpy)
629
    pandas_str = f'\npandas_categorical:{categorical_json}\n'
630
631
632
633
634
635
    if file_name is not None:
        with open(file_name, 'a') as f:
            f.write(pandas_str)
    return pandas_str


636
637
638
639
def _load_pandas_categorical(
    file_name: Optional[Union[str, Path]] = None,
    model_str: Optional[str] = None
) -> Optional[str]:
640
641
    pandas_key = 'pandas_categorical:'
    offset = -len(pandas_key)
642
    if file_name is not None:
643
        max_offset = -getsize(file_name)
644
645
646
647
        with open(file_name, 'rb') as f:
            while True:
                if offset < max_offset:
                    offset = max_offset
648
                f.seek(offset, SEEK_END)
649
650
651
652
                lines = f.readlines()
                if len(lines) >= 2:
                    break
                offset *= 2
653
        last_line = lines[-1].decode('utf-8').strip()
654
        if not last_line.startswith(pandas_key):
655
            last_line = lines[-2].decode('utf-8').strip()
656
    elif model_str is not None:
657
658
659
660
661
662
        idx = model_str.rfind('\n', 0, offset)
        last_line = model_str[idx:].strip()
    if last_line.startswith(pandas_key):
        return json.loads(last_line[len(pandas_key):])
    else:
        return None
663
664


665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
class Sequence(abc.ABC):
    """
    Generic data access interface.

    Object should support the following operations:

    .. code-block::

        # Get total row number.
        >>> len(seq)
        # Random access by row index. Used for data sampling.
        >>> seq[10]
        # Range data access. Used to read data in batch when constructing Dataset.
        >>> seq[0:100]
        # Optionally specify batch_size to control range data read size.
        >>> seq.batch_size

    - With random access, **data sampling does not need to go through all data**.
    - With range data access, there's **no need to read all data into memory thus reduce memory usage**.

685
686
    .. versionadded:: 3.3.0

687
688
689
690
691
692
693
694
695
    Attributes
    ----------
    batch_size : int
        Default size of a batch.
    """

    batch_size = 4096  # Defaults to read 4K rows in each batch.

    @abc.abstractmethod
696
    def __getitem__(self, idx: Union[int, slice, List[int]]) -> np.ndarray:
697
698
699
700
701
702
703
        """Return data for given row index.

        A basic implementation should look like this:

        .. code-block:: python

            if isinstance(idx, numbers.Integral):
704
                return self._get_one_line(idx)
705
            elif isinstance(idx, slice):
706
707
                return np.stack([self._get_one_line(i) for i in range(idx.start, idx.stop)])
            elif isinstance(idx, list):
708
                # Only required if using ``Dataset.subset()``.
709
                return np.array([self._get_one_line(i) for i in idx])
710
            else:
711
                raise TypeError(f"Sequence index must be integer, slice or list, got {type(idx).__name__}")
712
713
714

        Parameters
        ----------
715
        idx : int, slice[int], list[int]
716
717
718
719
            Item index.

        Returns
        -------
720
        result : numpy 1-D array or numpy 2-D array
721
            1-D array if idx is int, 2-D array if idx is slice or list.
722
723
724
725
726
727
728
729
730
        """
        raise NotImplementedError("Sub-classes of lightgbm.Sequence must implement __getitem__()")

    @abc.abstractmethod
    def __len__(self) -> int:
        """Return row count of this sequence."""
        raise NotImplementedError("Sub-classes of lightgbm.Sequence must implement __len__()")


731
class _InnerPredictor:
732
733
734
735
736
    """_InnerPredictor of LightGBM.

    Not exposed to user.
    Used only for prediction, usually used for continued training.

Nikita Titov's avatar
Nikita Titov committed
737
738
739
    .. note::

        Can be converted from Booster, but cannot be converted to Booster.
Guolin Ke's avatar
Guolin Ke committed
740
    """
741

742
743
744
745
746
747
    def __init__(
        self,
        model_file: Optional[Union[str, Path]] = None,
        booster_handle: Optional[ctypes.c_void_p] = None,
        pred_parameter: Optional[Dict[str, Any]] = None
    ):
748
        """Initialize the _InnerPredictor.
wxchan's avatar
wxchan committed
749
750
751

        Parameters
        ----------
752
        model_file : str, pathlib.Path or None, optional (default=None)
wxchan's avatar
wxchan committed
753
            Path to the model file.
754
755
756
        booster_handle : object or None, optional (default=None)
            Handle of Booster.
        pred_parameter: dict or None, optional (default=None)
757
            Other parameters for the prediction.
wxchan's avatar
wxchan committed
758
759
760
761
762
        """
        self.handle = ctypes.c_void_p()
        self.__is_manage_handle = True
        if model_file is not None:
            """Prediction task"""
Guolin Ke's avatar
Guolin Ke committed
763
            out_num_iterations = ctypes.c_int(0)
wxchan's avatar
wxchan committed
764
            _safe_call(_LIB.LGBM_BoosterCreateFromModelfile(
765
                _c_str(str(model_file)),
wxchan's avatar
wxchan committed
766
767
                ctypes.byref(out_num_iterations),
                ctypes.byref(self.handle)))
Guolin Ke's avatar
Guolin Ke committed
768
            out_num_class = ctypes.c_int(0)
wxchan's avatar
wxchan committed
769
770
771
772
            _safe_call(_LIB.LGBM_BoosterGetNumClasses(
                self.handle,
                ctypes.byref(out_num_class)))
            self.num_class = out_num_class.value
773
            self.num_total_iteration = out_num_iterations.value
774
            self.pandas_categorical = _load_pandas_categorical(file_name=model_file)
wxchan's avatar
wxchan committed
775
        elif booster_handle is not None:
Guolin Ke's avatar
Guolin Ke committed
776
            self.__is_manage_handle = False
wxchan's avatar
wxchan committed
777
            self.handle = booster_handle
Guolin Ke's avatar
Guolin Ke committed
778
            out_num_class = ctypes.c_int(0)
wxchan's avatar
wxchan committed
779
780
781
782
            _safe_call(_LIB.LGBM_BoosterGetNumClasses(
                self.handle,
                ctypes.byref(out_num_class)))
            self.num_class = out_num_class.value
783
            self.num_total_iteration = self.current_iteration()
784
            self.pandas_categorical = None
wxchan's avatar
wxchan committed
785
        else:
786
            raise TypeError('Need model_file or booster_handle to create a predictor')
wxchan's avatar
wxchan committed
787

788
        pred_parameter = {} if pred_parameter is None else pred_parameter
789
        self.pred_parameter = _param_dict_to_str(pred_parameter)
cbecker's avatar
cbecker committed
790

791
    def __del__(self) -> None:
792
793
794
795
796
        try:
            if self.__is_manage_handle:
                _safe_call(_LIB.LGBM_BoosterFree(self.handle))
        except AttributeError:
            pass
wxchan's avatar
wxchan committed
797

798
    def __getstate__(self) -> Dict[str, Any]:
799
800
801
802
        this = self.__dict__.copy()
        this.pop('handle', None)
        return this

803
804
805
806
807
808
809
810
811
812
813
    def predict(
        self,
        data,
        start_iteration: int = 0,
        num_iteration: int = -1,
        raw_score: bool = False,
        pred_leaf: bool = False,
        pred_contrib: bool = False,
        data_has_header: bool = False,
        validate_features: bool = False
    ):
814
        """Predict logic.
wxchan's avatar
wxchan committed
815
816
817

        Parameters
        ----------
818
        data : str, pathlib.Path, numpy array, pandas DataFrame, H2O DataTable's Frame or scipy.sparse
819
            Data source for prediction.
820
            If str or pathlib.Path, it represents the path to a text file (CSV, TSV, or LibSVM).
821
822
        start_iteration : int, optional (default=0)
            Start index of the iteration to predict.
823
824
825
826
827
828
829
830
831
832
833
        num_iteration : int, optional (default=-1)
            Iteration used for prediction.
        raw_score : bool, optional (default=False)
            Whether to predict raw scores.
        pred_leaf : bool, optional (default=False)
            Whether to predict leaf index.
        pred_contrib : bool, optional (default=False)
            Whether to predict feature contributions.
        data_has_header : bool, optional (default=False)
            Whether data has header.
            Used only for txt data.
834
835
836
        validate_features : bool, optional (default=False)
            If True, ensure that the features used to predict match the ones used to train.
            Used only if data is pandas DataFrame.
wxchan's avatar
wxchan committed
837
838
839

        Returns
        -------
840
        result : numpy array, scipy.sparse or list of scipy.sparse
841
            Prediction result.
842
            Can be sparse or a list of sparse objects (each element represents predictions for one class) for feature contributions (when ``pred_contrib=True``).
wxchan's avatar
wxchan committed
843
        """
wxchan's avatar
wxchan committed
844
        if isinstance(data, Dataset):
845
            raise TypeError("Cannot use Dataset instance for prediction, please use raw data instead")
846
847
848
849
850
851
852
853
854
855
856
        elif isinstance(data, pd_DataFrame) and validate_features:
            data_names = [str(x) for x in data.columns]
            ptr_names = (ctypes.c_char_p * len(data_names))()
            ptr_names[:] = [x.encode('utf-8') for x in data_names]
            _safe_call(
                _LIB.LGBM_BoosterValidateFeatureNames(
                    self.handle,
                    ptr_names,
                    ctypes.c_int(len(data_names)),
                )
            )
857
        data = _data_from_pandas(data, None, None, self.pandas_categorical)[0]
858
        predict_type = _C_API_PREDICT_NORMAL
wxchan's avatar
wxchan committed
859
        if raw_score:
860
            predict_type = _C_API_PREDICT_RAW_SCORE
wxchan's avatar
wxchan committed
861
        if pred_leaf:
862
            predict_type = _C_API_PREDICT_LEAF_INDEX
863
        if pred_contrib:
864
            predict_type = _C_API_PREDICT_CONTRIB
wxchan's avatar
wxchan committed
865
        int_data_has_header = 1 if data_has_header else 0
cbecker's avatar
cbecker committed
866

867
        if isinstance(data, (str, Path)):
868
            with _TempFile() as f:
wxchan's avatar
wxchan committed
869
870
                _safe_call(_LIB.LGBM_BoosterPredictForFile(
                    self.handle,
871
                    _c_str(str(data)),
Guolin Ke's avatar
Guolin Ke committed
872
873
                    ctypes.c_int(int_data_has_header),
                    ctypes.c_int(predict_type),
874
                    ctypes.c_int(start_iteration),
Guolin Ke's avatar
Guolin Ke committed
875
                    ctypes.c_int(num_iteration),
876
877
                    _c_str(self.pred_parameter),
                    _c_str(f.name)))
878
879
                preds = np.loadtxt(f.name, dtype=np.float64)
                nrow = preds.shape[0]
wxchan's avatar
wxchan committed
880
        elif isinstance(data, scipy.sparse.csr_matrix):
881
            preds, nrow = self.__pred_for_csr(data, start_iteration, num_iteration, predict_type)
Guolin Ke's avatar
Guolin Ke committed
882
        elif isinstance(data, scipy.sparse.csc_matrix):
883
            preds, nrow = self.__pred_for_csc(data, start_iteration, num_iteration, predict_type)
wxchan's avatar
wxchan committed
884
        elif isinstance(data, np.ndarray):
885
            preds, nrow = self.__pred_for_np2d(data, start_iteration, num_iteration, predict_type)
886
887
888
        elif isinstance(data, list):
            try:
                data = np.array(data)
889
            except BaseException:
890
                raise ValueError('Cannot convert data list to numpy array.')
891
            preds, nrow = self.__pred_for_np2d(data, start_iteration, num_iteration, predict_type)
892
        elif isinstance(data, dt_DataTable):
893
            preds, nrow = self.__pred_for_np2d(data.to_numpy(), start_iteration, num_iteration, predict_type)
wxchan's avatar
wxchan committed
894
895
        else:
            try:
896
                _log_warning('Converting data to scipy sparse matrix.')
wxchan's avatar
wxchan committed
897
                csr = scipy.sparse.csr_matrix(data)
898
            except BaseException:
899
                raise TypeError(f'Cannot predict data for type {type(data).__name__}')
900
            preds, nrow = self.__pred_for_csr(csr, start_iteration, num_iteration, predict_type)
wxchan's avatar
wxchan committed
901
902
        if pred_leaf:
            preds = preds.astype(np.int32)
903
        is_sparse = scipy.sparse.issparse(preds) or isinstance(preds, list)
904
        if not is_sparse and preds.size != nrow:
wxchan's avatar
wxchan committed
905
            if preds.size % nrow == 0:
906
                preds = preds.reshape(nrow, -1)
wxchan's avatar
wxchan committed
907
            else:
908
                raise ValueError(f'Length of predict result ({preds.size}) cannot be divide nrow ({nrow})')
wxchan's avatar
wxchan committed
909
910
        return preds

911
    def __get_num_preds(self, start_iteration, num_iteration, nrow, predict_type):
912
        """Get size of prediction result."""
913
        if nrow > _MAX_INT32:
914
            raise LightGBMError('LightGBM cannot perform prediction for data '
915
                                f'with number of rows greater than MAX_INT32 ({_MAX_INT32}).\n'
916
                                'You can split your data into chunks '
917
                                'and then concatenate predictions for them')
Guolin Ke's avatar
Guolin Ke committed
918
919
920
        n_preds = ctypes.c_int64(0)
        _safe_call(_LIB.LGBM_BoosterCalcNumPredict(
            self.handle,
Guolin Ke's avatar
Guolin Ke committed
921
922
            ctypes.c_int(nrow),
            ctypes.c_int(predict_type),
923
            ctypes.c_int(start_iteration),
Guolin Ke's avatar
Guolin Ke committed
924
            ctypes.c_int(num_iteration),
Guolin Ke's avatar
Guolin Ke committed
925
926
            ctypes.byref(n_preds)))
        return n_preds.value
wxchan's avatar
wxchan committed
927

928
    def __pred_for_np2d(self, mat, start_iteration, num_iteration, predict_type):
929
        """Predict for a 2-D numpy matrix."""
wxchan's avatar
wxchan committed
930
        if len(mat.shape) != 2:
931
            raise ValueError('Input numpy.ndarray or list must be 2 dimensional')
wxchan's avatar
wxchan committed
932

933
        def inner_predict(mat, start_iteration, num_iteration, predict_type, preds=None):
934
935
            if mat.dtype == np.float32 or mat.dtype == np.float64:
                data = np.array(mat.reshape(mat.size), dtype=mat.dtype, copy=False)
936
            else:  # change non-float data to float data, need to copy
937
                data = np.array(mat.reshape(mat.size), dtype=np.float32)
938
            ptr_data, type_ptr_data, _ = _c_float_array(data)
939
            n_preds = self.__get_num_preds(start_iteration, num_iteration, mat.shape[0], predict_type)
940
            if preds is None:
941
                preds = np.empty(n_preds, dtype=np.float64)
942
943
944
945
946
947
948
            elif len(preds.shape) != 1 or len(preds) != n_preds:
                raise ValueError("Wrong length of pre-allocated predict array")
            out_num_preds = ctypes.c_int64(0)
            _safe_call(_LIB.LGBM_BoosterPredictForMat(
                self.handle,
                ptr_data,
                ctypes.c_int(type_ptr_data),
949
950
                ctypes.c_int32(mat.shape[0]),
                ctypes.c_int32(mat.shape[1]),
951
                ctypes.c_int(_C_API_IS_ROW_MAJOR),
952
                ctypes.c_int(predict_type),
953
                ctypes.c_int(start_iteration),
954
                ctypes.c_int(num_iteration),
955
                _c_str(self.pred_parameter),
956
957
958
959
960
961
962
                ctypes.byref(out_num_preds),
                preds.ctypes.data_as(ctypes.POINTER(ctypes.c_double))))
            if n_preds != out_num_preds.value:
                raise ValueError("Wrong length for predict results")
            return preds, mat.shape[0]

        nrow = mat.shape[0]
963
964
        if nrow > _MAX_INT32:
            sections = np.arange(start=_MAX_INT32, stop=nrow, step=_MAX_INT32)
965
            # __get_num_preds() cannot work with nrow > MAX_INT32, so calculate overall number of predictions piecemeal
966
            n_preds = [self.__get_num_preds(start_iteration, num_iteration, i, predict_type) for i in np.diff([0] + list(sections) + [nrow])]
967
            n_preds_sections = np.array([0] + n_preds, dtype=np.intp).cumsum()
968
            preds = np.empty(sum(n_preds), dtype=np.float64)
969
970
            for chunk, (start_idx_pred, end_idx_pred) in zip(np.array_split(mat, sections),
                                                             zip(n_preds_sections, n_preds_sections[1:])):
971
                # avoid memory consumption by arrays concatenation operations
972
                inner_predict(chunk, start_iteration, num_iteration, predict_type, preds[start_idx_pred:end_idx_pred])
973
            return preds, nrow
wxchan's avatar
wxchan committed
974
        else:
975
            return inner_predict(mat, start_iteration, num_iteration, predict_type)
wxchan's avatar
wxchan committed
976

977
978
979
980
981
    def __create_sparse_native(self, cs, out_shape, out_ptr_indptr, out_ptr_indices, out_ptr_data,
                               indptr_type, data_type, is_csr=True):
        # create numpy array from output arrays
        data_indices_len = out_shape[0]
        indptr_len = out_shape[1]
982
        if indptr_type == _C_API_DTYPE_INT32:
983
            out_indptr = _cint32_array_to_numpy(out_ptr_indptr, indptr_len)
984
        elif indptr_type == _C_API_DTYPE_INT64:
985
            out_indptr = _cint64_array_to_numpy(out_ptr_indptr, indptr_len)
986
987
        else:
            raise TypeError("Expected int32 or int64 type for indptr")
988
        if data_type == _C_API_DTYPE_FLOAT32:
989
            out_data = _cfloat32_array_to_numpy(out_ptr_data, data_indices_len)
990
        elif data_type == _C_API_DTYPE_FLOAT64:
991
            out_data = _cfloat64_array_to_numpy(out_ptr_data, data_indices_len)
992
993
        else:
            raise TypeError("Expected float32 or float64 type for data")
994
        out_indices = _cint32_array_to_numpy(out_ptr_indices, data_indices_len)
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
        # break up indptr based on number of rows (note more than one matrix in multiclass case)
        per_class_indptr_shape = cs.indptr.shape[0]
        # for CSC there is extra column added
        if not is_csr:
            per_class_indptr_shape += 1
        out_indptr_arrays = np.split(out_indptr, out_indptr.shape[0] / per_class_indptr_shape)
        # reformat output into a csr or csc matrix or list of csr or csc matrices
        cs_output_matrices = []
        offset = 0
        for cs_indptr in out_indptr_arrays:
            matrix_indptr_len = cs_indptr[cs_indptr.shape[0] - 1]
            cs_indices = out_indices[offset + cs_indptr[0]:offset + matrix_indptr_len]
            cs_data = out_data[offset + cs_indptr[0]:offset + matrix_indptr_len]
            offset += matrix_indptr_len
            # same shape as input csr or csc matrix except extra column for expected value
            cs_shape = [cs.shape[0], cs.shape[1] + 1]
            # note: make sure we copy data as it will be deallocated next
            if is_csr:
                cs_output_matrices.append(scipy.sparse.csr_matrix((cs_data, cs_indices, cs_indptr), cs_shape))
            else:
                cs_output_matrices.append(scipy.sparse.csc_matrix((cs_data, cs_indices, cs_indptr), cs_shape))
        # free the temporary native indptr, indices, and data
        _safe_call(_LIB.LGBM_BoosterFreePredictSparse(out_ptr_indptr, out_ptr_indices, out_ptr_data,
                                                      ctypes.c_int(indptr_type), ctypes.c_int(data_type)))
        if len(cs_output_matrices) == 1:
            return cs_output_matrices[0]
        return cs_output_matrices

1023
    def __pred_for_csr(self, csr, start_iteration, num_iteration, predict_type):
1024
        """Predict for a CSR data."""
1025
        def inner_predict(csr, start_iteration, num_iteration, predict_type, preds=None):
1026
            nrow = len(csr.indptr) - 1
1027
            n_preds = self.__get_num_preds(start_iteration, num_iteration, nrow, predict_type)
1028
            if preds is None:
1029
                preds = np.empty(n_preds, dtype=np.float64)
1030
1031
1032
1033
            elif len(preds.shape) != 1 or len(preds) != n_preds:
                raise ValueError("Wrong length of pre-allocated predict array")
            out_num_preds = ctypes.c_int64(0)

1034
1035
            ptr_indptr, type_ptr_indptr, __ = _c_int_array(csr.indptr)
            ptr_data, type_ptr_data, _ = _c_float_array(csr.data)
1036

1037
            assert csr.shape[1] <= _MAX_INT32
1038
            csr_indices = csr.indices.astype(np.int32, copy=False)
1039

1040
1041
1042
            _safe_call(_LIB.LGBM_BoosterPredictForCSR(
                self.handle,
                ptr_indptr,
1043
                ctypes.c_int(type_ptr_indptr),
1044
                csr_indices.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
1045
1046
1047
1048
1049
1050
                ptr_data,
                ctypes.c_int(type_ptr_data),
                ctypes.c_int64(len(csr.indptr)),
                ctypes.c_int64(len(csr.data)),
                ctypes.c_int64(csr.shape[1]),
                ctypes.c_int(predict_type),
1051
                ctypes.c_int(start_iteration),
1052
                ctypes.c_int(num_iteration),
1053
                _c_str(self.pred_parameter),
1054
1055
1056
1057
1058
                ctypes.byref(out_num_preds),
                preds.ctypes.data_as(ctypes.POINTER(ctypes.c_double))))
            if n_preds != out_num_preds.value:
                raise ValueError("Wrong length for predict results")
            return preds, nrow
wxchan's avatar
wxchan committed
1059

1060
        def inner_predict_sparse(csr, start_iteration, num_iteration, predict_type):
1061
1062
            ptr_indptr, type_ptr_indptr, __ = _c_int_array(csr.indptr)
            ptr_data, type_ptr_data, _ = _c_float_array(csr.data)
1063
            csr_indices = csr.indices.astype(np.int32, copy=False)
1064
1065
            matrix_type = _C_API_MATRIX_TYPE_CSR
            if type_ptr_indptr == _C_API_DTYPE_INT32:
1066
1067
1068
1069
                out_ptr_indptr = ctypes.POINTER(ctypes.c_int32)()
            else:
                out_ptr_indptr = ctypes.POINTER(ctypes.c_int64)()
            out_ptr_indices = ctypes.POINTER(ctypes.c_int32)()
1070
            if type_ptr_data == _C_API_DTYPE_FLOAT32:
1071
1072
1073
                out_ptr_data = ctypes.POINTER(ctypes.c_float)()
            else:
                out_ptr_data = ctypes.POINTER(ctypes.c_double)()
1074
            out_shape = np.empty(2, dtype=np.int64)
1075
1076
1077
            _safe_call(_LIB.LGBM_BoosterPredictSparseOutput(
                self.handle,
                ptr_indptr,
1078
                ctypes.c_int(type_ptr_indptr),
1079
1080
1081
1082
1083
1084
1085
                csr_indices.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
                ptr_data,
                ctypes.c_int(type_ptr_data),
                ctypes.c_int64(len(csr.indptr)),
                ctypes.c_int64(len(csr.data)),
                ctypes.c_int64(csr.shape[1]),
                ctypes.c_int(predict_type),
1086
                ctypes.c_int(start_iteration),
1087
                ctypes.c_int(num_iteration),
1088
                _c_str(self.pred_parameter),
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
                ctypes.c_int(matrix_type),
                out_shape.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)),
                ctypes.byref(out_ptr_indptr),
                ctypes.byref(out_ptr_indices),
                ctypes.byref(out_ptr_data)))
            matrices = self.__create_sparse_native(csr, out_shape, out_ptr_indptr, out_ptr_indices, out_ptr_data,
                                                   type_ptr_indptr, type_ptr_data, is_csr=True)
            nrow = len(csr.indptr) - 1
            return matrices, nrow

1099
        if predict_type == _C_API_PREDICT_CONTRIB:
1100
            return inner_predict_sparse(csr, start_iteration, num_iteration, predict_type)
1101
        nrow = len(csr.indptr) - 1
1102
1103
        if nrow > _MAX_INT32:
            sections = [0] + list(np.arange(start=_MAX_INT32, stop=nrow, step=_MAX_INT32)) + [nrow]
1104
            # __get_num_preds() cannot work with nrow > MAX_INT32, so calculate overall number of predictions piecemeal
1105
            n_preds = [self.__get_num_preds(start_iteration, num_iteration, i, predict_type) for i in np.diff(sections)]
1106
            n_preds_sections = np.array([0] + n_preds, dtype=np.intp).cumsum()
1107
            preds = np.empty(sum(n_preds), dtype=np.float64)
1108
1109
            for (start_idx, end_idx), (start_idx_pred, end_idx_pred) in zip(zip(sections, sections[1:]),
                                                                            zip(n_preds_sections, n_preds_sections[1:])):
1110
                # avoid memory consumption by arrays concatenation operations
1111
                inner_predict(csr[start_idx:end_idx], start_iteration, num_iteration, predict_type, preds[start_idx_pred:end_idx_pred])
1112
1113
            return preds, nrow
        else:
1114
            return inner_predict(csr, start_iteration, num_iteration, predict_type)
Guolin Ke's avatar
Guolin Ke committed
1115

1116
    def __pred_for_csc(self, csc, start_iteration, num_iteration, predict_type):
1117
        """Predict for a CSC data."""
1118
        def inner_predict_sparse(csc, start_iteration, num_iteration, predict_type):
1119
1120
            ptr_indptr, type_ptr_indptr, __ = _c_int_array(csc.indptr)
            ptr_data, type_ptr_data, _ = _c_float_array(csc.data)
1121
            csc_indices = csc.indices.astype(np.int32, copy=False)
1122
1123
            matrix_type = _C_API_MATRIX_TYPE_CSC
            if type_ptr_indptr == _C_API_DTYPE_INT32:
1124
1125
1126
1127
                out_ptr_indptr = ctypes.POINTER(ctypes.c_int32)()
            else:
                out_ptr_indptr = ctypes.POINTER(ctypes.c_int64)()
            out_ptr_indices = ctypes.POINTER(ctypes.c_int32)()
1128
            if type_ptr_data == _C_API_DTYPE_FLOAT32:
1129
1130
1131
                out_ptr_data = ctypes.POINTER(ctypes.c_float)()
            else:
                out_ptr_data = ctypes.POINTER(ctypes.c_double)()
1132
            out_shape = np.empty(2, dtype=np.int64)
1133
1134
1135
            _safe_call(_LIB.LGBM_BoosterPredictSparseOutput(
                self.handle,
                ptr_indptr,
1136
                ctypes.c_int(type_ptr_indptr),
1137
1138
1139
1140
1141
1142
1143
                csc_indices.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
                ptr_data,
                ctypes.c_int(type_ptr_data),
                ctypes.c_int64(len(csc.indptr)),
                ctypes.c_int64(len(csc.data)),
                ctypes.c_int64(csc.shape[0]),
                ctypes.c_int(predict_type),
1144
                ctypes.c_int(start_iteration),
1145
                ctypes.c_int(num_iteration),
1146
                _c_str(self.pred_parameter),
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
                ctypes.c_int(matrix_type),
                out_shape.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)),
                ctypes.byref(out_ptr_indptr),
                ctypes.byref(out_ptr_indices),
                ctypes.byref(out_ptr_data)))
            matrices = self.__create_sparse_native(csc, out_shape, out_ptr_indptr, out_ptr_indices, out_ptr_data,
                                                   type_ptr_indptr, type_ptr_data, is_csr=False)
            nrow = csc.shape[0]
            return matrices, nrow

Guolin Ke's avatar
Guolin Ke committed
1157
        nrow = csc.shape[0]
1158
        if nrow > _MAX_INT32:
1159
            return self.__pred_for_csr(csc.tocsr(), start_iteration, num_iteration, predict_type)
1160
        if predict_type == _C_API_PREDICT_CONTRIB:
1161
1162
            return inner_predict_sparse(csc, start_iteration, num_iteration, predict_type)
        n_preds = self.__get_num_preds(start_iteration, num_iteration, nrow, predict_type)
1163
        preds = np.empty(n_preds, dtype=np.float64)
Guolin Ke's avatar
Guolin Ke committed
1164
1165
        out_num_preds = ctypes.c_int64(0)

1166
1167
        ptr_indptr, type_ptr_indptr, __ = _c_int_array(csc.indptr)
        ptr_data, type_ptr_data, _ = _c_float_array(csc.data)
Guolin Ke's avatar
Guolin Ke committed
1168

1169
        assert csc.shape[0] <= _MAX_INT32
1170
        csc_indices = csc.indices.astype(np.int32, copy=False)
1171

Guolin Ke's avatar
Guolin Ke committed
1172
1173
1174
        _safe_call(_LIB.LGBM_BoosterPredictForCSC(
            self.handle,
            ptr_indptr,
1175
            ctypes.c_int(type_ptr_indptr),
1176
            csc_indices.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
Guolin Ke's avatar
Guolin Ke committed
1177
            ptr_data,
Guolin Ke's avatar
Guolin Ke committed
1178
1179
1180
1181
1182
            ctypes.c_int(type_ptr_data),
            ctypes.c_int64(len(csc.indptr)),
            ctypes.c_int64(len(csc.data)),
            ctypes.c_int64(csc.shape[0]),
            ctypes.c_int(predict_type),
1183
            ctypes.c_int(start_iteration),
Guolin Ke's avatar
Guolin Ke committed
1184
            ctypes.c_int(num_iteration),
1185
            _c_str(self.pred_parameter),
Guolin Ke's avatar
Guolin Ke committed
1186
            ctypes.byref(out_num_preds),
wxchan's avatar
wxchan committed
1187
            preds.ctypes.data_as(ctypes.POINTER(ctypes.c_double))))
wxchan's avatar
wxchan committed
1188
        if n_preds != out_num_preds.value:
1189
            raise ValueError("Wrong length for predict results")
wxchan's avatar
wxchan committed
1190
1191
        return preds, nrow

1192
    def current_iteration(self) -> int:
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
        """Get the index of the current iteration.

        Returns
        -------
        cur_iter : int
            The index of the current iteration.
        """
        out_cur_iter = ctypes.c_int(0)
        _safe_call(_LIB.LGBM_BoosterGetCurrentIteration(
            self.handle,
            ctypes.byref(out_cur_iter)))
        return out_cur_iter.value

wxchan's avatar
wxchan committed
1206

1207
class Dataset:
wxchan's avatar
wxchan committed
1208
    """Dataset in LightGBM."""
1209

1210
1211
1212
    def __init__(
        self,
        data,
1213
        label: Optional[_LGBM_LabelType] = None,
1214
1215
1216
1217
1218
1219
1220
1221
1222
        reference: Optional["Dataset"] = None,
        weight=None,
        group=None,
        init_score=None,
        feature_name='auto',
        categorical_feature='auto',
        params: Optional[Dict[str, Any]] = None,
        free_raw_data: bool = True
    ):
1223
        """Initialize Dataset.
1224

wxchan's avatar
wxchan committed
1225
1226
        Parameters
        ----------
1227
        data : str, pathlib.Path, numpy array, pandas DataFrame, H2O DataTable's Frame, scipy.sparse, Sequence, list of Sequence or list of numpy array
wxchan's avatar
wxchan committed
1228
            Data source of Dataset.
1229
            If str or pathlib.Path, it represents the path to a text file (CSV, TSV, or LibSVM) or a LightGBM Dataset binary file.
1230
        label : list, numpy 1-D array, pandas Series / one-column DataFrame or None, optional (default=None)
1231
1232
1233
            Label of the data.
        reference : Dataset or None, optional (default=None)
            If this is Dataset for validation, training data should be used as reference.
1234
        weight : list, numpy 1-D array, pandas Series or None, optional (default=None)
1235
            Weight for each instance. Weights should be non-negative.
1236
        group : list, numpy 1-D array, pandas Series or None, optional (default=None)
1237
1238
1239
            Group/query data.
            Only used in the learning-to-rank task.
            sum(group) = n_samples.
1240
1241
            For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
            where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
1242
        init_score : list, list of lists (for multi-class task), numpy array, pandas Series, pandas DataFrame (for multi-class task), or None, optional (default=None)
1243
            Init score for Dataset.
1244
        feature_name : list of str, or 'auto', optional (default="auto")
1245
1246
            Feature names.
            If 'auto' and data is pandas DataFrame, data columns names are used.
1247
        categorical_feature : list of str or int, or 'auto', optional (default="auto")
1248
1249
            Categorical features.
            If list of int, interpreted as indices.
1250
            If list of str, interpreted as feature names (need to specify ``feature_name`` as well).
1251
            If 'auto' and data is pandas DataFrame, pandas unordered categorical columns are used.
1252
            All values in categorical features will be cast to int32 and thus should be less than int32 max value (2147483647).
1253
            Large values could be memory consuming. Consider using consecutive integers starting from zero.
1254
            All negative values in categorical features will be treated as missing values.
1255
            The output cannot be monotonically constrained with respect to a categorical feature.
1256
            Floating point numbers in categorical features will be rounded towards 0.
Nikita Titov's avatar
Nikita Titov committed
1257
        params : dict or None, optional (default=None)
1258
            Other parameters for Dataset.
Nikita Titov's avatar
Nikita Titov committed
1259
        free_raw_data : bool, optional (default=True)
1260
            If True, raw data is freed after constructing inner Dataset.
wxchan's avatar
wxchan committed
1261
        """
wxchan's avatar
wxchan committed
1262
1263
1264
1265
1266
1267
        self.handle = None
        self.data = data
        self.label = label
        self.reference = reference
        self.weight = weight
        self.group = group
1268
        self.init_score = init_score
wxchan's avatar
wxchan committed
1269
        self.feature_name = feature_name
1270
        self.categorical_feature = categorical_feature
1271
        self.params = deepcopy(params)
wxchan's avatar
wxchan committed
1272
1273
        self.free_raw_data = free_raw_data
        self.used_indices = None
1274
        self.need_slice = True
wxchan's avatar
wxchan committed
1275
        self._predictor = None
1276
        self.pandas_categorical = None
1277
        self.params_back_up = None
1278
        self.monotone_constraints = None
1279
        self.version = 0
1280
        self._start_row = 0  # Used when pushing rows one by one.
wxchan's avatar
wxchan committed
1281

1282
    def __del__(self) -> None:
1283
1284
1285
1286
        try:
            self._free_handle()
        except AttributeError:
            pass
1287

1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
    def _create_sample_indices(self, total_nrow: int) -> np.ndarray:
        """Get an array of randomly chosen indices from this ``Dataset``.

        Indices are sampled without replacement.

        Parameters
        ----------
        total_nrow : int
            Total number of rows to sample from.
            If this value is greater than the value of parameter ``bin_construct_sample_cnt``, only ``bin_construct_sample_cnt`` indices will be used.
            If Dataset has multiple input data, this should be the sum of rows of every file.

        Returns
        -------
        indices : numpy array
            Indices for sampled data.
        """
1305
        param_str = _param_dict_to_str(self.get_params())
1306
1307
        sample_cnt = _get_sample_count(total_nrow, param_str)
        indices = np.empty(sample_cnt, dtype=np.int32)
1308
        ptr_data, _, _ = _c_int_array(indices)
1309
1310
1311
1312
        actual_sample_cnt = ctypes.c_int32(0)

        _safe_call(_LIB.LGBM_SampleIndices(
            ctypes.c_int32(total_nrow),
1313
            _c_str(param_str),
1314
1315
1316
            ptr_data,
            ctypes.byref(actual_sample_cnt),
        ))
1317
1318
        assert sample_cnt == actual_sample_cnt.value
        return indices
1319

1320
1321
1322
1323
1324
    def _init_from_ref_dataset(
        self,
        total_nrow: int,
        ref_dataset: _DatasetHandle
    ) -> 'Dataset':
1325
1326
1327
1328
1329
1330
        """Create dataset from a reference dataset.

        Parameters
        ----------
        total_nrow : int
            Number of rows expected to add to dataset.
1331
1332
        ref_dataset : object
            Handle of reference dataset to extract metadata from.
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357

        Returns
        -------
        self : Dataset
            Constructed Dataset object.
        """
        self.handle = ctypes.c_void_p()
        _safe_call(_LIB.LGBM_DatasetCreateByReference(
            ref_dataset,
            ctypes.c_int64(total_nrow),
            ctypes.byref(self.handle),
        ))
        return self

    def _init_from_sample(
        self,
        sample_data: List[np.ndarray],
        sample_indices: List[np.ndarray],
        sample_cnt: int,
        total_nrow: int,
    ) -> "Dataset":
        """Create Dataset from sampled data structures.

        Parameters
        ----------
1358
        sample_data : list of numpy array
1359
            Sample data for each column.
1360
        sample_indices : list of numpy array
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
            Sample data row index for each column.
        sample_cnt : int
            Number of samples.
        total_nrow : int
            Total number of rows for all input files.

        Returns
        -------
        self : Dataset
            Constructed Dataset object.
        """
        ncol = len(sample_indices)
        assert len(sample_data) == ncol, "#sample data column != #column indices"

        for i in range(ncol):
            if sample_data[i].dtype != np.double:
                raise ValueError(f"sample_data[{i}] type {sample_data[i].dtype} is not double")
            if sample_indices[i].dtype != np.int32:
                raise ValueError(f"sample_indices[{i}] type {sample_indices[i].dtype} is not int32")

        # c type: double**
        # each double* element points to start of each column of sample data.
        sample_col_ptr = (ctypes.POINTER(ctypes.c_double) * ncol)()
        # c type int**
        # each int* points to start of indices for each column
        indices_col_ptr = (ctypes.POINTER(ctypes.c_int32) * ncol)()
        for i in range(ncol):
1388
1389
            sample_col_ptr[i] = _c_float_array(sample_data[i])[0]
            indices_col_ptr[i] = _c_int_array(sample_indices[i])[0]
1390
1391

        num_per_col = np.array([len(d) for d in sample_indices], dtype=np.int32)
1392
        num_per_col_ptr, _, _ = _c_int_array(num_per_col)
1393
1394

        self.handle = ctypes.c_void_p()
1395
        params_str = _param_dict_to_str(self.get_params())
1396
1397
1398
1399
1400
1401
1402
        _safe_call(_LIB.LGBM_DatasetCreateFromSampledColumn(
            ctypes.cast(sample_col_ptr, ctypes.POINTER(ctypes.POINTER(ctypes.c_double))),
            ctypes.cast(indices_col_ptr, ctypes.POINTER(ctypes.POINTER(ctypes.c_int32))),
            ctypes.c_int32(ncol),
            num_per_col_ptr,
            ctypes.c_int32(sample_cnt),
            ctypes.c_int32(total_nrow),
1403
            ctypes.c_int64(total_nrow),
1404
            _c_str(params_str),
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
            ctypes.byref(self.handle),
        ))
        return self

    def _push_rows(self, data: np.ndarray) -> 'Dataset':
        """Add rows to Dataset.

        Parameters
        ----------
        data : numpy 1-D array
            New data to add to the Dataset.

        Returns
        -------
        self : Dataset
            Dataset object.
        """
        nrow, ncol = data.shape
        data = data.reshape(data.size)
1424
        data_ptr, data_type, _ = _c_float_array(data)
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436

        _safe_call(_LIB.LGBM_DatasetPushRows(
            self.handle,
            data_ptr,
            data_type,
            ctypes.c_int32(nrow),
            ctypes.c_int32(ncol),
            ctypes.c_int32(self._start_row),
        ))
        self._start_row += nrow
        return self

1437
    def get_params(self) -> Dict[str, Any]:
1438
1439
1440
1441
        """Get the used parameters in the Dataset.

        Returns
        -------
1442
        params : dict
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
            The used parameters in this Dataset object.
        """
        if self.params is not None:
            # no min_data, nthreads and verbose in this function
            dataset_params = _ConfigAliases.get("bin_construct_sample_cnt",
                                                "categorical_feature",
                                                "data_random_seed",
                                                "enable_bundle",
                                                "feature_pre_filter",
                                                "forcedbins_filename",
                                                "group_column",
                                                "header",
                                                "ignore_column",
                                                "is_enable_sparse",
                                                "label_column",
1458
                                                "linear_tree",
1459
1460
1461
1462
                                                "max_bin",
                                                "max_bin_by_feature",
                                                "min_data_in_bin",
                                                "pre_partition",
Nikita Titov's avatar
Nikita Titov committed
1463
                                                "precise_float_parser",
1464
1465
1466
1467
1468
                                                "two_round",
                                                "use_missing",
                                                "weight_column",
                                                "zero_as_missing")
            return {k: v for k, v in self.params.items() if k in dataset_params}
1469
1470
        else:
            return {}
1471

1472
    def _free_handle(self) -> "Dataset":
1473
        if self.handle is not None:
1474
            _safe_call(_LIB.LGBM_DatasetFree(self.handle))
1475
            self.handle = None
Guolin Ke's avatar
Guolin Ke committed
1476
1477
1478
        self.need_slice = True
        if self.used_indices is not None:
            self.data = None
Nikita Titov's avatar
Nikita Titov committed
1479
        return self
wxchan's avatar
wxchan committed
1480

Guolin Ke's avatar
Guolin Ke committed
1481
1482
    def _set_init_score_by_predictor(self, predictor, data, used_indices=None):
        data_has_header = False
1483
        if isinstance(data, (str, Path)):
Guolin Ke's avatar
Guolin Ke committed
1484
            # check data has header or not
1485
            data_has_header = any(self.params.get(alias, False) for alias in _ConfigAliases.get("header"))
Guolin Ke's avatar
Guolin Ke committed
1486
        num_data = self.num_data()
1487
1488
1489
        if predictor is not None:
            init_score = predictor.predict(data,
                                           raw_score=True,
1490
1491
                                           data_has_header=data_has_header)
            init_score = init_score.ravel()
1492
1493
            if used_indices is not None:
                assert not self.need_slice
1494
                if isinstance(data, (str, Path)):
1495
                    sub_init_score = np.empty(num_data * predictor.num_class, dtype=np.float64)
1496
                    assert num_data == len(used_indices)
1497
1498
                    for i in range(len(used_indices)):
                        for j in range(predictor.num_class):
1499
1500
1501
1502
                            sub_init_score[i * predictor.num_class + j] = init_score[used_indices[i] * predictor.num_class + j]
                    init_score = sub_init_score
            if predictor.num_class > 1:
                # need to regroup init_score
1503
                new_init_score = np.empty(init_score.size, dtype=np.float64)
1504
1505
                for i in range(num_data):
                    for j in range(predictor.num_class):
1506
1507
1508
                        new_init_score[j * num_data + i] = init_score[i * predictor.num_class + j]
                init_score = new_init_score
        elif self.init_score is not None:
1509
            init_score = np.zeros(self.init_score.shape, dtype=np.float64)
1510
1511
        else:
            return self
Guolin Ke's avatar
Guolin Ke committed
1512
1513
        self.set_init_score(init_score)

1514
1515
1516
    def _lazy_init(
        self,
        data,
1517
        label: Optional[_LGBM_LabelType] = None,
1518
1519
1520
1521
1522
1523
1524
1525
1526
        reference: Optional["Dataset"] = None,
        weight=None,
        group=None,
        init_score=None,
        predictor=None,
        feature_name='auto',
        categorical_feature='auto',
        params: Optional[Dict[str, Any]] = None
    ) -> "Dataset":
wxchan's avatar
wxchan committed
1527
1528
        if data is None:
            self.handle = None
Nikita Titov's avatar
Nikita Titov committed
1529
            return self
Guolin Ke's avatar
Guolin Ke committed
1530
1531
1532
        if reference is not None:
            self.pandas_categorical = reference.pandas_categorical
            categorical_feature = reference.categorical_feature
1533
1534
1535
1536
        data, feature_name, categorical_feature, self.pandas_categorical = _data_from_pandas(data,
                                                                                             feature_name,
                                                                                             categorical_feature,
                                                                                             self.pandas_categorical)
Guolin Ke's avatar
Guolin Ke committed
1537

1538
        # process for args
wxchan's avatar
wxchan committed
1539
        params = {} if params is None else params
1540
1541
1542
        args_names = (getattr(self.__class__, '_lazy_init')
                      .__code__
                      .co_varnames[:getattr(self.__class__, '_lazy_init').__code__.co_argcount])
1543
        for key in params.keys():
1544
            if key in args_names:
1545
1546
                _log_warning(f'{key} keyword has been found in `params` and will be ignored.\n'
                             f'Please use {key} argument of the Dataset constructor to pass this parameter.')
1547
        # get categorical features
1548
1549
1550
1551
1552
1553
        if categorical_feature is not None:
            categorical_indices = set()
            feature_dict = {}
            if feature_name is not None:
                feature_dict = {name: i for i, name in enumerate(feature_name)}
            for name in categorical_feature:
1554
                if isinstance(name, str) and name in feature_dict:
1555
                    categorical_indices.add(feature_dict[name])
1556
                elif isinstance(name, int):
1557
1558
                    categorical_indices.add(name)
                else:
1559
                    raise TypeError(f"Wrong type({type(name).__name__}) or unknown name({name}) in categorical_feature")
1560
            if categorical_indices:
1561
1562
                for cat_alias in _ConfigAliases.get("categorical_feature"):
                    if cat_alias in params:
1563
                        # If the params[cat_alias] is equal to categorical_indices, do not report the warning.
1564
                        if not (isinstance(params[cat_alias], list) and set(params[cat_alias]) == categorical_indices):
1565
                            _log_warning(f'{cat_alias} in param dict is overridden.')
1566
                        params.pop(cat_alias, None)
1567
                params['categorical_column'] = sorted(categorical_indices)
1568

1569
        params_str = _param_dict_to_str(params)
1570
        self.params = params
1571
        # process for reference dataset
wxchan's avatar
wxchan committed
1572
        ref_dataset = None
wxchan's avatar
wxchan committed
1573
        if isinstance(reference, Dataset):
1574
            ref_dataset = reference.construct().handle
wxchan's avatar
wxchan committed
1575
1576
        elif reference is not None:
            raise TypeError('Reference dataset should be None or dataset instance')
1577
        # start construct data
1578
        if isinstance(data, (str, Path)):
wxchan's avatar
wxchan committed
1579
1580
            self.handle = ctypes.c_void_p()
            _safe_call(_LIB.LGBM_DatasetCreateFromFile(
1581
1582
                _c_str(str(data)),
                _c_str(params_str),
wxchan's avatar
wxchan committed
1583
1584
1585
1586
                ref_dataset,
                ctypes.byref(self.handle)))
        elif isinstance(data, scipy.sparse.csr_matrix):
            self.__init_from_csr(data, params_str, ref_dataset)
Guolin Ke's avatar
Guolin Ke committed
1587
1588
        elif isinstance(data, scipy.sparse.csc_matrix):
            self.__init_from_csc(data, params_str, ref_dataset)
wxchan's avatar
wxchan committed
1589
1590
        elif isinstance(data, np.ndarray):
            self.__init_from_np2d(data, params_str, ref_dataset)
1591
1592
1593
1594
1595
1596
1597
1598
1599
        elif isinstance(data, list) and len(data) > 0:
            if all(isinstance(x, np.ndarray) for x in data):
                self.__init_from_list_np2d(data, params_str, ref_dataset)
            elif all(isinstance(x, Sequence) for x in data):
                self.__init_from_seqs(data, ref_dataset)
            else:
                raise TypeError('Data list can only be of ndarray or Sequence')
        elif isinstance(data, Sequence):
            self.__init_from_seqs([data], ref_dataset)
1600
        elif isinstance(data, dt_DataTable):
1601
            self.__init_from_np2d(data.to_numpy(), params_str, ref_dataset)
wxchan's avatar
wxchan committed
1602
1603
1604
1605
        else:
            try:
                csr = scipy.sparse.csr_matrix(data)
                self.__init_from_csr(csr, params_str, ref_dataset)
1606
            except BaseException:
1607
                raise TypeError(f'Cannot initialize Dataset from {type(data).__name__}')
wxchan's avatar
wxchan committed
1608
1609
1610
        if label is not None:
            self.set_label(label)
        if self.get_label() is None:
1611
            raise ValueError("Label should not be None")
wxchan's avatar
wxchan committed
1612
1613
1614
1615
        if weight is not None:
            self.set_weight(weight)
        if group is not None:
            self.set_group(group)
1616
1617
        if isinstance(predictor, _InnerPredictor):
            if self._predictor is None and init_score is not None:
1618
                _log_warning("The init_score will be overridden by the prediction of init_model.")
Guolin Ke's avatar
Guolin Ke committed
1619
            self._set_init_score_by_predictor(predictor, data)
1620
1621
        elif init_score is not None:
            self.set_init_score(init_score)
Guolin Ke's avatar
Guolin Ke committed
1622
        elif predictor is not None:
1623
            raise TypeError(f'Wrong predictor type {type(predictor).__name__}')
Guolin Ke's avatar
Guolin Ke committed
1624
        # set feature names
Nikita Titov's avatar
Nikita Titov committed
1625
        return self.set_feature_name(feature_name)
wxchan's avatar
wxchan committed
1626

1627
1628
    @staticmethod
    def _yield_row_from_seqlist(seqs: List[Sequence], indices: Iterable[int]):
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
        offset = 0
        seq_id = 0
        seq = seqs[seq_id]
        for row_id in indices:
            assert row_id >= offset, "sample indices are expected to be monotonic"
            while row_id >= offset + len(seq):
                offset += len(seq)
                seq_id += 1
                seq = seqs[seq_id]
            id_in_seq = row_id - offset
            row = seq[id_in_seq]
            yield row if row.flags['OWNDATA'] else row.copy()

    def __sample(self, seqs: List[Sequence], total_nrow: int) -> Tuple[List[np.ndarray], List[np.ndarray]]:
        """Sample data from seqs.

        Mimics behavior in c_api.cpp:LGBM_DatasetCreateFromMats()

        Returns
        -------
            sampled_rows, sampled_row_indices
        """
        indices = self._create_sample_indices(total_nrow)

        # Select sampled rows, transpose to column order.
1654
        sampled = np.array([row for row in self._yield_row_from_seqlist(seqs, indices)])
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
        sampled = sampled.T

        filtered = []
        filtered_idx = []
        sampled_row_range = np.arange(len(indices), dtype=np.int32)
        for col in sampled:
            col_predicate = (np.abs(col) > ZERO_THRESHOLD) | np.isnan(col)
            filtered_col = col[col_predicate]
            filtered_row_idx = sampled_row_range[col_predicate]

            filtered.append(filtered_col)
            filtered_idx.append(filtered_row_idx)

        return filtered, filtered_idx

1670
1671
1672
    def __init_from_seqs(
        self,
        seqs: List[Sequence],
1673
        ref_dataset: Optional[_DatasetHandle] = None
1674
    ) -> "Dataset":
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
        """
        Initialize data from list of Sequence objects.

        Sequence: Generic Data Access Object
            Supports random access and access by batch if properly defined by user

        Data scheme uniformity are trusted, not checked
        """
        total_nrow = sum(len(seq) for seq in seqs)

        # create validation dataset from ref_dataset
        if ref_dataset is not None:
            self._init_from_ref_dataset(total_nrow, ref_dataset)
        else:
1689
            param_str = _param_dict_to_str(self.get_params())
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
            sample_cnt = _get_sample_count(total_nrow, param_str)

            sample_data, col_indices = self.__sample(seqs, total_nrow)
            self._init_from_sample(sample_data, col_indices, sample_cnt, total_nrow)

        for seq in seqs:
            nrow = len(seq)
            batch_size = getattr(seq, 'batch_size', None) or Sequence.batch_size
            for start in range(0, nrow, batch_size):
                end = min(start + batch_size, nrow)
                self._push_rows(seq[start:end])
        return self

1703
1704
1705
1706
1707
1708
    def __init_from_np2d(
        self,
        mat: np.ndarray,
        params_str: str,
        ref_dataset: Optional[_DatasetHandle]
    ) -> "Dataset":
1709
        """Initialize data from a 2-D numpy matrix."""
wxchan's avatar
wxchan committed
1710
1711
1712
1713
1714
1715
        if len(mat.shape) != 2:
            raise ValueError('Input numpy.ndarray must be 2 dimensional')

        self.handle = ctypes.c_void_p()
        if mat.dtype == np.float32 or mat.dtype == np.float64:
            data = np.array(mat.reshape(mat.size), dtype=mat.dtype, copy=False)
1716
        else:  # change non-float data to float data, need to copy
wxchan's avatar
wxchan committed
1717
1718
            data = np.array(mat.reshape(mat.size), dtype=np.float32)

1719
        ptr_data, type_ptr_data, _ = _c_float_array(data)
wxchan's avatar
wxchan committed
1720
1721
        _safe_call(_LIB.LGBM_DatasetCreateFromMat(
            ptr_data,
Guolin Ke's avatar
Guolin Ke committed
1722
            ctypes.c_int(type_ptr_data),
1723
1724
            ctypes.c_int32(mat.shape[0]),
            ctypes.c_int32(mat.shape[1]),
1725
            ctypes.c_int(_C_API_IS_ROW_MAJOR),
1726
            _c_str(params_str),
wxchan's avatar
wxchan committed
1727
1728
            ref_dataset,
            ctypes.byref(self.handle)))
Nikita Titov's avatar
Nikita Titov committed
1729
        return self
wxchan's avatar
wxchan committed
1730

1731
1732
1733
1734
1735
1736
    def __init_from_list_np2d(
        self,
        mats: List[np.ndarray],
        params_str: str,
        ref_dataset: Optional[_DatasetHandle]
    ) -> "Dataset":
1737
        """Initialize data from a list of 2-D numpy matrices."""
1738
        ncol = mats[0].shape[1]
1739
        nrow = np.empty((len(mats),), np.int32)
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
        if mats[0].dtype == np.float64:
            ptr_data = (ctypes.POINTER(ctypes.c_double) * len(mats))()
        else:
            ptr_data = (ctypes.POINTER(ctypes.c_float) * len(mats))()

        holders = []
        type_ptr_data = None

        for i, mat in enumerate(mats):
            if len(mat.shape) != 2:
                raise ValueError('Input numpy.ndarray must be 2 dimensional')

            if mat.shape[1] != ncol:
                raise ValueError('Input arrays must have same number of columns')

            nrow[i] = mat.shape[0]

            if mat.dtype == np.float32 or mat.dtype == np.float64:
                mats[i] = np.array(mat.reshape(mat.size), dtype=mat.dtype, copy=False)
1759
            else:  # change non-float data to float data, need to copy
1760
1761
                mats[i] = np.array(mat.reshape(mat.size), dtype=np.float32)

1762
            chunk_ptr_data, chunk_type_ptr_data, holder = _c_float_array(mats[i])
1763
1764
1765
1766
1767
1768
1769
1770
            if type_ptr_data is not None and chunk_type_ptr_data != type_ptr_data:
                raise ValueError('Input chunks must have same type')
            ptr_data[i] = chunk_ptr_data
            type_ptr_data = chunk_type_ptr_data
            holders.append(holder)

        self.handle = ctypes.c_void_p()
        _safe_call(_LIB.LGBM_DatasetCreateFromMats(
1771
            ctypes.c_int32(len(mats)),
1772
1773
1774
            ctypes.cast(ptr_data, ctypes.POINTER(ctypes.POINTER(ctypes.c_double))),
            ctypes.c_int(type_ptr_data),
            nrow.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
1775
            ctypes.c_int32(ncol),
1776
            ctypes.c_int(_C_API_IS_ROW_MAJOR),
1777
            _c_str(params_str),
1778
1779
            ref_dataset,
            ctypes.byref(self.handle)))
Nikita Titov's avatar
Nikita Titov committed
1780
        return self
1781

1782
1783
1784
1785
1786
1787
    def __init_from_csr(
        self,
        csr: scipy.sparse.csr_matrix,
        params_str: str,
        ref_dataset: Optional[_DatasetHandle]
    ) -> "Dataset":
1788
        """Initialize data from a CSR matrix."""
wxchan's avatar
wxchan committed
1789
        if len(csr.indices) != len(csr.data):
1790
            raise ValueError(f'Length mismatch: {len(csr.indices)} vs {len(csr.data)}')
wxchan's avatar
wxchan committed
1791
1792
        self.handle = ctypes.c_void_p()

1793
1794
        ptr_indptr, type_ptr_indptr, __ = _c_int_array(csr.indptr)
        ptr_data, type_ptr_data, _ = _c_float_array(csr.data)
wxchan's avatar
wxchan committed
1795

1796
        assert csr.shape[1] <= _MAX_INT32
1797
        csr_indices = csr.indices.astype(np.int32, copy=False)
1798

wxchan's avatar
wxchan committed
1799
1800
        _safe_call(_LIB.LGBM_DatasetCreateFromCSR(
            ptr_indptr,
Guolin Ke's avatar
Guolin Ke committed
1801
            ctypes.c_int(type_ptr_indptr),
1802
            csr_indices.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
wxchan's avatar
wxchan committed
1803
            ptr_data,
Guolin Ke's avatar
Guolin Ke committed
1804
1805
1806
1807
            ctypes.c_int(type_ptr_data),
            ctypes.c_int64(len(csr.indptr)),
            ctypes.c_int64(len(csr.data)),
            ctypes.c_int64(csr.shape[1]),
1808
            _c_str(params_str),
wxchan's avatar
wxchan committed
1809
1810
            ref_dataset,
            ctypes.byref(self.handle)))
Nikita Titov's avatar
Nikita Titov committed
1811
        return self
wxchan's avatar
wxchan committed
1812

1813
1814
1815
1816
1817
1818
    def __init_from_csc(
        self,
        csc: scipy.sparse.csc_matrix,
        params_str: str,
        ref_dataset: Optional[_DatasetHandle]
    ) -> "Dataset":
1819
        """Initialize data from a CSC matrix."""
Guolin Ke's avatar
Guolin Ke committed
1820
        if len(csc.indices) != len(csc.data):
1821
            raise ValueError(f'Length mismatch: {len(csc.indices)} vs {len(csc.data)}')
Guolin Ke's avatar
Guolin Ke committed
1822
1823
        self.handle = ctypes.c_void_p()

1824
1825
        ptr_indptr, type_ptr_indptr, __ = _c_int_array(csc.indptr)
        ptr_data, type_ptr_data, _ = _c_float_array(csc.data)
Guolin Ke's avatar
Guolin Ke committed
1826

1827
        assert csc.shape[0] <= _MAX_INT32
1828
        csc_indices = csc.indices.astype(np.int32, copy=False)
1829

Guolin Ke's avatar
Guolin Ke committed
1830
1831
        _safe_call(_LIB.LGBM_DatasetCreateFromCSC(
            ptr_indptr,
Guolin Ke's avatar
Guolin Ke committed
1832
            ctypes.c_int(type_ptr_indptr),
1833
            csc_indices.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
Guolin Ke's avatar
Guolin Ke committed
1834
            ptr_data,
Guolin Ke's avatar
Guolin Ke committed
1835
1836
1837
1838
            ctypes.c_int(type_ptr_data),
            ctypes.c_int64(len(csc.indptr)),
            ctypes.c_int64(len(csc.data)),
            ctypes.c_int64(csc.shape[0]),
1839
            _c_str(params_str),
Guolin Ke's avatar
Guolin Ke committed
1840
1841
            ref_dataset,
            ctypes.byref(self.handle)))
Nikita Titov's avatar
Nikita Titov committed
1842
        return self
Guolin Ke's avatar
Guolin Ke committed
1843

1844
    @staticmethod
1845
1846
1847
1848
1849
1850
    def _compare_params_for_warning(
        params: Optional[Dict[str, Any]],
        other_params: Optional[Dict[str, Any]],
        ignore_keys: Set[str]
    ) -> bool:
        """Compare two dictionaries with params ignoring some keys.
1851

1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
        It is only for the warning purpose.

        Parameters
        ----------
        params : dict or None
            One dictionary with parameters to compare.
        other_params : dict or None
            Another dictionary with parameters to compare.
        ignore_keys : set
            Keys that should be ignored during comparing two dictionaries.
1862
1863
1864

        Returns
        -------
1865
1866
        compare_result : bool
          Returns whether two dictionaries with params are equal.
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
        """
        if params is None:
            params = {}
        if other_params is None:
            other_params = {}
        for k in other_params:
            if k not in ignore_keys:
                if k not in params or params[k] != other_params[k]:
                    return False
        for k in params:
            if k not in ignore_keys:
                if k not in other_params or params[k] != other_params[k]:
                    return False
        return True

1882
    def construct(self) -> "Dataset":
1883
1884
1885
1886
1887
        """Lazy init.

        Returns
        -------
        self : Dataset
Nikita Titov's avatar
Nikita Titov committed
1888
            Constructed Dataset object.
1889
        """
1890
        if self.handle is None:
wxchan's avatar
wxchan committed
1891
            if self.reference is not None:
1892
                reference_params = self.reference.get_params()
1893
1894
                params = self.get_params()
                if params != reference_params:
1895
1896
1897
1898
1899
                    if not self._compare_params_for_warning(
                        params=params,
                        other_params=reference_params,
                        ignore_keys=_ConfigAliases.get("categorical_feature")
                    ):
1900
                        _log_warning('Overriding the parameters from Reference Dataset.')
1901
                    self._update_params(reference_params)
wxchan's avatar
wxchan committed
1902
                if self.used_indices is None:
1903
                    # create valid
1904
                    self._lazy_init(self.data, label=self.label, reference=self.reference,
1905
1906
                                    weight=self.weight, group=self.group,
                                    init_score=self.init_score, predictor=self._predictor,
1907
                                    feature_name=self.feature_name, params=self.params)
wxchan's avatar
wxchan committed
1908
                else:
1909
                    # construct subset
1910
                    used_indices = _list_to_1d_numpy(self.used_indices, np.int32, name='used_indices')
1911
                    assert used_indices.flags.c_contiguous
Guolin Ke's avatar
Guolin Ke committed
1912
                    if self.reference.group is not None:
1913
                        group_info = np.array(self.reference.group).astype(np.int32, copy=False)
1914
                        _, self.group = np.unique(np.repeat(range(len(group_info)), repeats=group_info)[self.used_indices],
1915
                                                  return_counts=True)
1916
                    self.handle = ctypes.c_void_p()
1917
                    params_str = _param_dict_to_str(self.params)
wxchan's avatar
wxchan committed
1918
                    _safe_call(_LIB.LGBM_DatasetGetSubset(
1919
                        self.reference.construct().handle,
wxchan's avatar
wxchan committed
1920
                        used_indices.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
1921
                        ctypes.c_int32(used_indices.shape[0]),
1922
                        _c_str(params_str),
wxchan's avatar
wxchan committed
1923
                        ctypes.byref(self.handle)))
Guolin Ke's avatar
Guolin Ke committed
1924
1925
                    if not self.free_raw_data:
                        self.get_data()
Guolin Ke's avatar
Guolin Ke committed
1926
1927
                    if self.group is not None:
                        self.set_group(self.group)
wxchan's avatar
wxchan committed
1928
1929
                    if self.get_label() is None:
                        raise ValueError("Label should not be None.")
Guolin Ke's avatar
Guolin Ke committed
1930
1931
1932
                    if isinstance(self._predictor, _InnerPredictor) and self._predictor is not self.reference._predictor:
                        self.get_data()
                        self._set_init_score_by_predictor(self._predictor, self.data, used_indices)
wxchan's avatar
wxchan committed
1933
            else:
1934
                # create train
1935
                self._lazy_init(self.data, label=self.label,
1936
1937
                                weight=self.weight, group=self.group,
                                init_score=self.init_score, predictor=self._predictor,
1938
                                feature_name=self.feature_name, categorical_feature=self.categorical_feature, params=self.params)
wxchan's avatar
wxchan committed
1939
1940
            if self.free_raw_data:
                self.data = None
1941
            self.feature_name = self.get_feature_name()
wxchan's avatar
wxchan committed
1942
        return self
wxchan's avatar
wxchan committed
1943

1944
1945
1946
    def create_valid(
        self,
        data,
1947
        label: Optional[_LGBM_LabelType] = None,
1948
1949
1950
1951
1952
        weight=None,
        group=None,
        init_score=None,
        params: Optional[Dict[str, Any]] = None
    ) -> "Dataset":
1953
        """Create validation data align with current Dataset.
wxchan's avatar
wxchan committed
1954
1955
1956

        Parameters
        ----------
1957
        data : str, pathlib.Path, numpy array, pandas DataFrame, H2O DataTable's Frame, scipy.sparse, Sequence, list of Sequence or list of numpy array
wxchan's avatar
wxchan committed
1958
            Data source of Dataset.
1959
            If str or pathlib.Path, it represents the path to a text file (CSV, TSV, or LibSVM) or a LightGBM Dataset binary file.
1960
        label : list, numpy 1-D array, pandas Series / one-column DataFrame or None, optional (default=None)
1961
1962
            Label of the data.
        weight : list, numpy 1-D array, pandas Series or None, optional (default=None)
1963
            Weight for each instance. Weights should be non-negative.
1964
        group : list, numpy 1-D array, pandas Series or None, optional (default=None)
1965
1966
1967
            Group/query data.
            Only used in the learning-to-rank task.
            sum(group) = n_samples.
1968
1969
            For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
            where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
1970
        init_score : list, list of lists (for multi-class task), numpy array, pandas Series, pandas DataFrame (for multi-class task), or None, optional (default=None)
1971
            Init score for Dataset.
Nikita Titov's avatar
Nikita Titov committed
1972
        params : dict or None, optional (default=None)
1973
            Other parameters for validation Dataset.
1974
1975
1976

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
1977
1978
        valid : Dataset
            Validation Dataset with reference to self.
wxchan's avatar
wxchan committed
1979
        """
1980
        ret = Dataset(data, label=label, reference=self,
1981
                      weight=weight, group=group, init_score=init_score,
1982
                      params=params, free_raw_data=self.free_raw_data)
wxchan's avatar
wxchan committed
1983
        ret._predictor = self._predictor
1984
        ret.pandas_categorical = self.pandas_categorical
wxchan's avatar
wxchan committed
1985
        return ret
wxchan's avatar
wxchan committed
1986

1987
1988
1989
1990
1991
    def subset(
        self,
        used_indices: List[int],
        params: Optional[Dict[str, Any]] = None
    ) -> "Dataset":
1992
        """Get subset of current Dataset.
wxchan's avatar
wxchan committed
1993
1994
1995
1996

        Parameters
        ----------
        used_indices : list of int
1997
            Indices used to create the subset.
Nikita Titov's avatar
Nikita Titov committed
1998
        params : dict or None, optional (default=None)
1999
            These parameters will be passed to Dataset constructor.
2000
2001
2002
2003
2004

        Returns
        -------
        subset : Dataset
            Subset of the current Dataset.
wxchan's avatar
wxchan committed
2005
        """
wxchan's avatar
wxchan committed
2006
2007
        if params is None:
            params = self.params
wxchan's avatar
wxchan committed
2008
        ret = Dataset(None, reference=self, feature_name=self.feature_name,
2009
2010
                      categorical_feature=self.categorical_feature, params=params,
                      free_raw_data=self.free_raw_data)
wxchan's avatar
wxchan committed
2011
        ret._predictor = self._predictor
2012
        ret.pandas_categorical = self.pandas_categorical
2013
        ret.used_indices = sorted(used_indices)
wxchan's avatar
wxchan committed
2014
2015
        return ret

2016
    def save_binary(self, filename: Union[str, Path]) -> "Dataset":
2017
        """Save Dataset to a binary file.
wxchan's avatar
wxchan committed
2018

2019
2020
2021
2022
2023
        .. note::

            Please note that `init_score` is not saved in binary file.
            If you need it, please set it again after loading Dataset.

wxchan's avatar
wxchan committed
2024
2025
        Parameters
        ----------
2026
        filename : str or pathlib.Path
wxchan's avatar
wxchan committed
2027
            Name of the output file.
Nikita Titov's avatar
Nikita Titov committed
2028
2029
2030
2031
2032

        Returns
        -------
        self : Dataset
            Returns self.
wxchan's avatar
wxchan committed
2033
2034
2035
        """
        _safe_call(_LIB.LGBM_DatasetSaveBinary(
            self.construct().handle,
2036
            _c_str(str(filename))))
Nikita Titov's avatar
Nikita Titov committed
2037
        return self
wxchan's avatar
wxchan committed
2038

2039
    def _update_params(self, params: Optional[Dict[str, Any]]) -> "Dataset":
2040
2041
        if not params:
            return self
2042
        params = deepcopy(params)
2043
2044
2045
2046
2047

        def update():
            if not self.params:
                self.params = params
            else:
2048
                self.params_back_up = deepcopy(self.params)
2049
2050
2051
2052
2053
2054
                self.params.update(params)

        if self.handle is None:
            update()
        elif params is not None:
            ret = _LIB.LGBM_DatasetUpdateParamChecking(
2055
2056
                _c_str(_param_dict_to_str(self.params)),
                _c_str(_param_dict_to_str(params)))
2057
2058
2059
2060
2061
2062
            if ret != 0:
                # could be updated if data is not freed
                if self.data is not None:
                    update()
                    self._free_handle()
                else:
2063
                    raise LightGBMError(_LIB.LGBM_GetLastError().decode('utf-8'))
Nikita Titov's avatar
Nikita Titov committed
2064
        return self
wxchan's avatar
wxchan committed
2065

2066
    def _reverse_update_params(self) -> "Dataset":
2067
        if self.handle is None:
2068
            self.params = deepcopy(self.params_back_up)
2069
            self.params_back_up = None
Nikita Titov's avatar
Nikita Titov committed
2070
        return self
2071

2072
2073
2074
2075
2076
    def set_field(
        self,
        field_name: str,
        data
    ) -> "Dataset":
wxchan's avatar
wxchan committed
2077
        """Set property into the Dataset.
wxchan's avatar
wxchan committed
2078
2079
2080

        Parameters
        ----------
2081
        field_name : str
2082
            The field name of the information.
2083
2084
        data : list, list of lists (for multi-class task), numpy array, pandas Series, pandas DataFrame (for multi-class task), or None
            The data to be set.
Nikita Titov's avatar
Nikita Titov committed
2085
2086
2087
2088
2089

        Returns
        -------
        self : Dataset
            Dataset with set property.
wxchan's avatar
wxchan committed
2090
        """
2091
        if self.handle is None:
2092
            raise Exception(f"Cannot set {field_name} before construct dataset")
wxchan's avatar
wxchan committed
2093
        if data is None:
2094
            # set to None
wxchan's avatar
wxchan committed
2095
2096
            _safe_call(_LIB.LGBM_DatasetSetField(
                self.handle,
2097
                _c_str(field_name),
wxchan's avatar
wxchan committed
2098
                None,
Guolin Ke's avatar
Guolin Ke committed
2099
                ctypes.c_int(0),
2100
                ctypes.c_int(_FIELD_TYPE_MAPPER[field_name])))
Nikita Titov's avatar
Nikita Titov committed
2101
            return self
2102
        if field_name == 'init_score':
Guolin Ke's avatar
Guolin Ke committed
2103
            dtype = np.float64
2104
            if _is_1d_collection(data):
2105
                data = _list_to_1d_numpy(data, dtype, name=field_name)
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
            elif _is_2d_collection(data):
                data = _data_to_2d_numpy(data, dtype, name=field_name)
                data = data.ravel(order='F')
            else:
                raise TypeError(
                    'init_score must be list, numpy 1-D array or pandas Series.\n'
                    'In multiclass classification init_score can also be a list of lists, numpy 2-D array or pandas DataFrame.'
                )
        else:
            dtype = np.int32 if field_name == 'group' else np.float32
2116
            data = _list_to_1d_numpy(data, dtype, name=field_name)
2117

2118
        if data.dtype == np.float32 or data.dtype == np.float64:
2119
            ptr_data, type_data, _ = _c_float_array(data)
wxchan's avatar
wxchan committed
2120
        elif data.dtype == np.int32:
2121
            ptr_data, type_data, _ = _c_int_array(data)
wxchan's avatar
wxchan committed
2122
        else:
2123
            raise TypeError(f"Expected np.float32/64 or np.int32, met type({data.dtype})")
2124
        if type_data != _FIELD_TYPE_MAPPER[field_name]:
2125
            raise TypeError("Input type error for set_field")
wxchan's avatar
wxchan committed
2126
2127
        _safe_call(_LIB.LGBM_DatasetSetField(
            self.handle,
2128
            _c_str(field_name),
wxchan's avatar
wxchan committed
2129
            ptr_data,
Guolin Ke's avatar
Guolin Ke committed
2130
2131
            ctypes.c_int(len(data)),
            ctypes.c_int(type_data)))
2132
        self.version += 1
Nikita Titov's avatar
Nikita Titov committed
2133
        return self
wxchan's avatar
wxchan committed
2134

2135
    def get_field(self, field_name: str) -> Optional[np.ndarray]:
wxchan's avatar
wxchan committed
2136
        """Get property from the Dataset.
wxchan's avatar
wxchan committed
2137
2138
2139

        Parameters
        ----------
2140
        field_name : str
2141
            The field name of the information.
wxchan's avatar
wxchan committed
2142
2143
2144

        Returns
        -------
2145
        info : numpy array or None
2146
            A numpy array with information from the Dataset.
Guolin Ke's avatar
Guolin Ke committed
2147
        """
2148
        if self.handle is None:
2149
            raise Exception(f"Cannot get {field_name} before construct Dataset")
2150
2151
        tmp_out_len = ctypes.c_int(0)
        out_type = ctypes.c_int(0)
wxchan's avatar
wxchan committed
2152
2153
2154
        ret = ctypes.POINTER(ctypes.c_void_p)()
        _safe_call(_LIB.LGBM_DatasetGetField(
            self.handle,
2155
            _c_str(field_name),
wxchan's avatar
wxchan committed
2156
2157
2158
            ctypes.byref(tmp_out_len),
            ctypes.byref(ret),
            ctypes.byref(out_type)))
2159
        if out_type.value != _FIELD_TYPE_MAPPER[field_name]:
wxchan's avatar
wxchan committed
2160
2161
2162
            raise TypeError("Return type error for get_field")
        if tmp_out_len.value == 0:
            return None
2163
        if out_type.value == _C_API_DTYPE_INT32:
2164
            arr = _cint32_array_to_numpy(ctypes.cast(ret, ctypes.POINTER(ctypes.c_int32)), tmp_out_len.value)
2165
        elif out_type.value == _C_API_DTYPE_FLOAT32:
2166
            arr = _cfloat32_array_to_numpy(ctypes.cast(ret, ctypes.POINTER(ctypes.c_float)), tmp_out_len.value)
2167
        elif out_type.value == _C_API_DTYPE_FLOAT64:
2168
            arr = _cfloat64_array_to_numpy(ctypes.cast(ret, ctypes.POINTER(ctypes.c_double)), tmp_out_len.value)
2169
        else:
wxchan's avatar
wxchan committed
2170
            raise TypeError("Unknown type")
2171
2172
2173
2174
2175
2176
        if field_name == 'init_score':
            num_data = self.num_data()
            num_classes = arr.size // num_data
            if num_classes > 1:
                arr = arr.reshape((num_data, num_classes), order='F')
        return arr
Guolin Ke's avatar
Guolin Ke committed
2177

2178
2179
2180
2181
    def set_categorical_feature(
        self,
        categorical_feature: Union[List[int], List[str]]
    ) -> "Dataset":
2182
        """Set categorical features.
2183
2184
2185

        Parameters
        ----------
2186
        categorical_feature : list of int or str
2187
            Names or indices of categorical features.
Nikita Titov's avatar
Nikita Titov committed
2188
2189
2190
2191
2192

        Returns
        -------
        self : Dataset
            Dataset with set categorical features.
2193
2194
        """
        if self.categorical_feature == categorical_feature:
Nikita Titov's avatar
Nikita Titov committed
2195
            return self
2196
        if self.data is not None:
2197
2198
            if self.categorical_feature is None:
                self.categorical_feature = categorical_feature
Nikita Titov's avatar
Nikita Titov committed
2199
                return self._free_handle()
2200
            elif categorical_feature == 'auto':
Nikita Titov's avatar
Nikita Titov committed
2201
                return self
2202
            else:
2203
2204
2205
                if self.categorical_feature != 'auto':
                    _log_warning('categorical_feature in Dataset is overridden.\n'
                                 f'New categorical_feature is {sorted(list(categorical_feature))}')
2206
                self.categorical_feature = categorical_feature
Nikita Titov's avatar
Nikita Titov committed
2207
                return self._free_handle()
2208
        else:
2209
2210
            raise LightGBMError("Cannot set categorical feature after freed raw data, "
                                "set free_raw_data=False when construct Dataset to avoid this.")
2211

2212
2213
2214
2215
    def _set_predictor(
        self,
        predictor: Optional[_InnerPredictor]
    ) -> "Dataset":
2216
2217
2218
2219
        """Set predictor for continued training.

        It is not recommended for user to call this function.
        Please use init_model argument in engine.train() or engine.cv() instead.
Guolin Ke's avatar
Guolin Ke committed
2220
        """
2221
        if predictor is self._predictor and (predictor is None or predictor.current_iteration() == self._predictor.current_iteration()):
Nikita Titov's avatar
Nikita Titov committed
2222
            return self
2223
        if self.handle is None:
Guolin Ke's avatar
Guolin Ke committed
2224
            self._predictor = predictor
2225
2226
2227
2228
2229
2230
        elif self.data is not None:
            self._predictor = predictor
            self._set_init_score_by_predictor(self._predictor, self.data)
        elif self.used_indices is not None and self.reference is not None and self.reference.data is not None:
            self._predictor = predictor
            self._set_init_score_by_predictor(self._predictor, self.reference.data, self.used_indices)
Guolin Ke's avatar
Guolin Ke committed
2231
        else:
2232
2233
            raise LightGBMError("Cannot set predictor after freed raw data, "
                                "set free_raw_data=False when construct Dataset to avoid this.")
2234
        return self
Guolin Ke's avatar
Guolin Ke committed
2235

2236
    def set_reference(self, reference: "Dataset") -> "Dataset":
2237
        """Set reference Dataset.
Guolin Ke's avatar
Guolin Ke committed
2238
2239
2240
2241

        Parameters
        ----------
        reference : Dataset
2242
            Reference that is used as a template to construct the current Dataset.
Nikita Titov's avatar
Nikita Titov committed
2243
2244
2245
2246
2247

        Returns
        -------
        self : Dataset
            Dataset with set reference.
Guolin Ke's avatar
Guolin Ke committed
2248
        """
2249
2250
2251
        self.set_categorical_feature(reference.categorical_feature) \
            .set_feature_name(reference.feature_name) \
            ._set_predictor(reference._predictor)
2252
        # we're done if self and reference share a common upstream reference
2253
        if self.get_ref_chain().intersection(reference.get_ref_chain()):
Nikita Titov's avatar
Nikita Titov committed
2254
            return self
Guolin Ke's avatar
Guolin Ke committed
2255
2256
        if self.data is not None:
            self.reference = reference
Nikita Titov's avatar
Nikita Titov committed
2257
            return self._free_handle()
Guolin Ke's avatar
Guolin Ke committed
2258
        else:
2259
2260
            raise LightGBMError("Cannot set reference after freed raw data, "
                                "set free_raw_data=False when construct Dataset to avoid this.")
Guolin Ke's avatar
Guolin Ke committed
2261

2262
    def set_feature_name(self, feature_name: List[str]) -> "Dataset":
2263
        """Set feature name.
Guolin Ke's avatar
Guolin Ke committed
2264
2265
2266

        Parameters
        ----------
2267
        feature_name : list of str
2268
            Feature names.
Nikita Titov's avatar
Nikita Titov committed
2269
2270
2271
2272
2273

        Returns
        -------
        self : Dataset
            Dataset with set feature name.
Guolin Ke's avatar
Guolin Ke committed
2274
        """
2275
2276
        if feature_name != 'auto':
            self.feature_name = feature_name
2277
        if self.handle is not None and feature_name is not None and feature_name != 'auto':
wxchan's avatar
wxchan committed
2278
            if len(feature_name) != self.num_feature():
2279
                raise ValueError(f"Length of feature_name({len(feature_name)}) and num_feature({self.num_feature()}) don't match")
2280
            c_feature_name = [_c_str(name) for name in feature_name]
wxchan's avatar
wxchan committed
2281
2282
            _safe_call(_LIB.LGBM_DatasetSetFeatureNames(
                self.handle,
2283
                _c_array(ctypes.c_char_p, c_feature_name),
Guolin Ke's avatar
Guolin Ke committed
2284
                ctypes.c_int(len(feature_name))))
Nikita Titov's avatar
Nikita Titov committed
2285
        return self
Guolin Ke's avatar
Guolin Ke committed
2286

2287
    def set_label(self, label: Optional[_LGBM_LabelType]) -> "Dataset":
2288
        """Set label of Dataset.
Guolin Ke's avatar
Guolin Ke committed
2289
2290
2291

        Parameters
        ----------
2292
        label : list, numpy 1-D array, pandas Series / one-column DataFrame or None
2293
            The label information to be set into Dataset.
Nikita Titov's avatar
Nikita Titov committed
2294
2295
2296
2297
2298

        Returns
        -------
        self : Dataset
            Dataset with set label.
Guolin Ke's avatar
Guolin Ke committed
2299
2300
        """
        self.label = label
2301
        if self.handle is not None:
2302
2303
2304
2305
            if isinstance(label, pd_DataFrame):
                if len(label.columns) > 1:
                    raise ValueError('DataFrame for label cannot have multiple columns')
                _check_for_bad_pandas_dtypes(label.dtypes)
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
                try:
                    # most common case (no nullable dtypes)
                    label = label.to_numpy(dtype=np.float32, copy=False)
                except TypeError:
                    # 1.0 <= pd version < 1.1 and nullable dtypes, least common case
                    # raises error because array is casted to type(pd.NA) and there's no na_value argument
                    label = label.astype(np.float32, copy=False).values
                except ValueError:
                    # data has nullable dtypes, but we can specify na_value argument and copy will be made
                    label = label.to_numpy(dtype=np.float32, na_value=np.nan)
                label_array = np.ravel(label)
2317
            else:
2318
                label_array = _list_to_1d_numpy(label, name='label')
2319
            self.set_field('label', label_array)
2320
            self.label = self.get_field('label')  # original values can be modified at cpp side
Nikita Titov's avatar
Nikita Titov committed
2321
        return self
Guolin Ke's avatar
Guolin Ke committed
2322

2323
    def set_weight(self, weight) -> "Dataset":
2324
        """Set weight of each instance.
Guolin Ke's avatar
Guolin Ke committed
2325
2326
2327

        Parameters
        ----------
2328
        weight : list, numpy 1-D array, pandas Series or None
2329
            Weight to be set for each data point. Weights should be non-negative.
Nikita Titov's avatar
Nikita Titov committed
2330
2331
2332
2333
2334

        Returns
        -------
        self : Dataset
            Dataset with set weight.
Guolin Ke's avatar
Guolin Ke committed
2335
        """
2336
2337
        if weight is not None and np.all(weight == 1):
            weight = None
Guolin Ke's avatar
Guolin Ke committed
2338
        self.weight = weight
2339
        if self.handle is not None and weight is not None:
2340
            weight = _list_to_1d_numpy(weight, name='weight')
wxchan's avatar
wxchan committed
2341
            self.set_field('weight', weight)
2342
            self.weight = self.get_field('weight')  # original values can be modified at cpp side
Nikita Titov's avatar
Nikita Titov committed
2343
        return self
Guolin Ke's avatar
Guolin Ke committed
2344

2345
    def set_init_score(self, init_score) -> "Dataset":
2346
        """Set init score of Booster to start from.
Guolin Ke's avatar
Guolin Ke committed
2347
2348
2349

        Parameters
        ----------
2350
        init_score : list, list of lists (for multi-class task), numpy array, pandas Series, pandas DataFrame (for multi-class task), or None
2351
            Init score for Booster.
Nikita Titov's avatar
Nikita Titov committed
2352
2353
2354
2355
2356

        Returns
        -------
        self : Dataset
            Dataset with set init score.
Guolin Ke's avatar
Guolin Ke committed
2357
2358
        """
        self.init_score = init_score
2359
        if self.handle is not None and init_score is not None:
wxchan's avatar
wxchan committed
2360
            self.set_field('init_score', init_score)
2361
            self.init_score = self.get_field('init_score')  # original values can be modified at cpp side
Nikita Titov's avatar
Nikita Titov committed
2362
        return self
Guolin Ke's avatar
Guolin Ke committed
2363

2364
    def set_group(self, group) -> "Dataset":
2365
        """Set group size of Dataset (used for ranking).
Guolin Ke's avatar
Guolin Ke committed
2366
2367
2368

        Parameters
        ----------
2369
        group : list, numpy 1-D array, pandas Series or None
2370
2371
2372
            Group/query data.
            Only used in the learning-to-rank task.
            sum(group) = n_samples.
2373
2374
            For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
            where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
Nikita Titov's avatar
Nikita Titov committed
2375
2376
2377
2378
2379

        Returns
        -------
        self : Dataset
            Dataset with set group.
Guolin Ke's avatar
Guolin Ke committed
2380
2381
        """
        self.group = group
2382
        if self.handle is not None and group is not None:
2383
            group = _list_to_1d_numpy(group, np.int32, name='group')
wxchan's avatar
wxchan committed
2384
            self.set_field('group', group)
Nikita Titov's avatar
Nikita Titov committed
2385
        return self
Guolin Ke's avatar
Guolin Ke committed
2386

2387
    def get_feature_name(self) -> List[str]:
2388
2389
2390
2391
        """Get the names of columns (features) in the Dataset.

        Returns
        -------
2392
        feature_names : list of str
2393
2394
2395
2396
2397
2398
2399
2400
            The names of columns (features) in the Dataset.
        """
        if self.handle is None:
            raise LightGBMError("Cannot get feature_name before construct dataset")
        num_feature = self.num_feature()
        tmp_out_len = ctypes.c_int(0)
        reserved_string_buffer_size = 255
        required_string_buffer_size = ctypes.c_size_t(0)
2401
        string_buffers = [ctypes.create_string_buffer(reserved_string_buffer_size) for _ in range(num_feature)]
2402
2403
2404
        ptr_string_buffers = (ctypes.c_char_p * num_feature)(*map(ctypes.addressof, string_buffers))
        _safe_call(_LIB.LGBM_DatasetGetFeatureNames(
            self.handle,
2405
            ctypes.c_int(num_feature),
2406
            ctypes.byref(tmp_out_len),
2407
            ctypes.c_size_t(reserved_string_buffer_size),
2408
2409
2410
2411
            ctypes.byref(required_string_buffer_size),
            ptr_string_buffers))
        if num_feature != tmp_out_len.value:
            raise ValueError("Length of feature names doesn't equal with num_feature")
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
        actual_string_buffer_size = required_string_buffer_size.value
        # if buffer length is not long enough, reallocate buffers
        if reserved_string_buffer_size < actual_string_buffer_size:
            string_buffers = [ctypes.create_string_buffer(actual_string_buffer_size) for _ in range(num_feature)]
            ptr_string_buffers = (ctypes.c_char_p * num_feature)(*map(ctypes.addressof, string_buffers))
            _safe_call(_LIB.LGBM_DatasetGetFeatureNames(
                self.handle,
                ctypes.c_int(num_feature),
                ctypes.byref(tmp_out_len),
                ctypes.c_size_t(actual_string_buffer_size),
                ctypes.byref(required_string_buffer_size),
                ptr_string_buffers))
2424
        return [string_buffers[i].value.decode('utf-8') for i in range(num_feature)]
2425

2426
    def get_label(self) -> Optional[np.ndarray]:
2427
        """Get the label of the Dataset.
Guolin Ke's avatar
Guolin Ke committed
2428
2429
2430

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
2431
        label : numpy array or None
2432
            The label information from the Dataset.
Guolin Ke's avatar
Guolin Ke committed
2433
        """
2434
        if self.label is None:
wxchan's avatar
wxchan committed
2435
            self.label = self.get_field('label')
Guolin Ke's avatar
Guolin Ke committed
2436
2437
        return self.label

2438
    def get_weight(self) -> Optional[np.ndarray]:
2439
        """Get the weight of the Dataset.
Guolin Ke's avatar
Guolin Ke committed
2440
2441
2442

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
2443
        weight : numpy array or None
2444
            Weight for each data point from the Dataset. Weights should be non-negative.
Guolin Ke's avatar
Guolin Ke committed
2445
        """
2446
        if self.weight is None:
wxchan's avatar
wxchan committed
2447
            self.weight = self.get_field('weight')
Guolin Ke's avatar
Guolin Ke committed
2448
2449
        return self.weight

2450
    def get_init_score(self) -> Optional[np.ndarray]:
2451
        """Get the initial score of the Dataset.
Guolin Ke's avatar
Guolin Ke committed
2452
2453
2454

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
2455
        init_score : numpy array or None
2456
            Init score of Booster.
Guolin Ke's avatar
Guolin Ke committed
2457
        """
2458
        if self.init_score is None:
wxchan's avatar
wxchan committed
2459
            self.init_score = self.get_field('init_score')
Guolin Ke's avatar
Guolin Ke committed
2460
2461
        return self.init_score

2462
2463
2464
2465
2466
    def get_data(self):
        """Get the raw data of the Dataset.

        Returns
        -------
2467
        data : str, pathlib.Path, numpy array, pandas DataFrame, H2O DataTable's Frame, scipy.sparse, Sequence, list of Sequence or list of numpy array or None
2468
2469
2470
2471
            Raw data used in the Dataset construction.
        """
        if self.handle is None:
            raise Exception("Cannot get data before construct Dataset")
Guolin Ke's avatar
Guolin Ke committed
2472
2473
2474
2475
2476
        if self.need_slice and self.used_indices is not None and self.reference is not None:
            self.data = self.reference.data
            if self.data is not None:
                if isinstance(self.data, np.ndarray) or scipy.sparse.issparse(self.data):
                    self.data = self.data[self.used_indices, :]
2477
                elif isinstance(self.data, pd_DataFrame):
Guolin Ke's avatar
Guolin Ke committed
2478
                    self.data = self.data.iloc[self.used_indices].copy()
2479
                elif isinstance(self.data, dt_DataTable):
Guolin Ke's avatar
Guolin Ke committed
2480
                    self.data = self.data[self.used_indices, :]
2481
2482
2483
2484
                elif isinstance(self.data, Sequence):
                    self.data = self.data[self.used_indices]
                elif isinstance(self.data, list) and len(self.data) > 0 and all(isinstance(x, Sequence) for x in self.data):
                    self.data = np.array([row for row in self._yield_row_from_seqlist(self.data, self.used_indices)])
Guolin Ke's avatar
Guolin Ke committed
2485
                else:
2486
2487
                    _log_warning(f"Cannot subset {type(self.data).__name__} type of raw data.\n"
                                 "Returning original raw data")
2488
            self.need_slice = False
Guolin Ke's avatar
Guolin Ke committed
2489
2490
2491
        if self.data is None:
            raise LightGBMError("Cannot call `get_data` after freed raw data, "
                                "set free_raw_data=False when construct Dataset to avoid this.")
2492
2493
        return self.data

Guolin Ke's avatar
Guolin Ke committed
2494
    def get_group(self):
2495
        """Get the group of the Dataset.
Guolin Ke's avatar
Guolin Ke committed
2496
2497
2498

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
2499
        group : numpy array or None
2500
2501
2502
            Group/query data.
            Only used in the learning-to-rank task.
            sum(group) = n_samples.
2503
2504
            For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
            where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
Guolin Ke's avatar
Guolin Ke committed
2505
        """
2506
        if self.group is None:
wxchan's avatar
wxchan committed
2507
            self.group = self.get_field('group')
Guolin Ke's avatar
Guolin Ke committed
2508
2509
            if self.group is not None:
                # group data from LightGBM is boundaries data, need to convert to group size
Nikita Titov's avatar
Nikita Titov committed
2510
                self.group = np.diff(self.group)
Guolin Ke's avatar
Guolin Ke committed
2511
2512
        return self.group

2513
    def num_data(self) -> int:
2514
        """Get the number of rows in the Dataset.
Guolin Ke's avatar
Guolin Ke committed
2515
2516
2517

        Returns
        -------
2518
2519
        number_of_rows : int
            The number of rows in the Dataset.
Guolin Ke's avatar
Guolin Ke committed
2520
        """
2521
        if self.handle is not None:
2522
            ret = ctypes.c_int(0)
wxchan's avatar
wxchan committed
2523
2524
2525
            _safe_call(_LIB.LGBM_DatasetGetNumData(self.handle,
                                                   ctypes.byref(ret)))
            return ret.value
Guolin Ke's avatar
Guolin Ke committed
2526
        else:
2527
            raise LightGBMError("Cannot get num_data before construct dataset")
Guolin Ke's avatar
Guolin Ke committed
2528

2529
    def num_feature(self) -> int:
2530
        """Get the number of columns (features) in the Dataset.
Guolin Ke's avatar
Guolin Ke committed
2531
2532
2533

        Returns
        -------
2534
2535
        number_of_columns : int
            The number of columns (features) in the Dataset.
Guolin Ke's avatar
Guolin Ke committed
2536
        """
2537
        if self.handle is not None:
2538
            ret = ctypes.c_int(0)
wxchan's avatar
wxchan committed
2539
2540
2541
            _safe_call(_LIB.LGBM_DatasetGetNumFeature(self.handle,
                                                      ctypes.byref(ret)))
            return ret.value
Guolin Ke's avatar
Guolin Ke committed
2542
        else:
2543
            raise LightGBMError("Cannot get num_feature before construct dataset")
Guolin Ke's avatar
Guolin Ke committed
2544

2545
    def feature_num_bin(self, feature: Union[int, str]) -> int:
2546
2547
2548
2549
        """Get the number of bins for a feature.

        Parameters
        ----------
2550
2551
        feature : int or str
            Index or name of the feature.
2552
2553
2554
2555
2556
2557
2558

        Returns
        -------
        number_of_bins : int
            The number of constructed bins for the feature in the Dataset.
        """
        if self.handle is not None:
2559
            if isinstance(feature, str):
2560
2561
2562
                feature_index = self.feature_name.index(feature)
            else:
                feature_index = feature
2563
2564
            ret = ctypes.c_int(0)
            _safe_call(_LIB.LGBM_DatasetGetFeatureNumBin(self.handle,
2565
                                                         ctypes.c_int(feature_index),
2566
2567
2568
2569
2570
                                                         ctypes.byref(ret)))
            return ret.value
        else:
            raise LightGBMError("Cannot get feature_num_bin before construct dataset")

2571
    def get_ref_chain(self, ref_limit: int = 100) -> Set["Dataset"]:
2572
2573
2574
2575
2576
        """Get a chain of Dataset objects.

        Starts with r, then goes to r.reference (if exists),
        then to r.reference.reference, etc.
        until we hit ``ref_limit`` or a reference loop.
2577
2578
2579
2580
2581

        Parameters
        ----------
        ref_limit : int, optional (default=100)
            The limit number of references.
2582
2583
2584

        Returns
        -------
2585
2586
2587
        ref_chain : set of Dataset
            Chain of references of the Datasets.
        """
2588
        head = self
2589
        ref_chain: Set[Dataset] = set()
2590
2591
        while len(ref_chain) < ref_limit:
            if isinstance(head, Dataset):
2592
                ref_chain.add(head)
2593
2594
2595
2596
2597
2598
                if (head.reference is not None) and (head.reference not in ref_chain):
                    head = head.reference
                else:
                    break
            else:
                break
Nikita Titov's avatar
Nikita Titov committed
2599
        return ref_chain
2600

2601
    def add_features_from(self, other: "Dataset") -> "Dataset":
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
        """Add features from other Dataset to the current Dataset.

        Both Datasets must be constructed before calling this method.

        Parameters
        ----------
        other : Dataset
            The Dataset to take features from.

        Returns
        -------
        self : Dataset
            Dataset with the new features added.
        """
        if self.handle is None or other.handle is None:
            raise ValueError('Both source and target Datasets must be constructed before adding features')
        _safe_call(_LIB.LGBM_DatasetAddFeaturesFrom(self.handle, other.handle))
Guolin Ke's avatar
Guolin Ke committed
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
        was_none = self.data is None
        old_self_data_type = type(self.data).__name__
        if other.data is None:
            self.data = None
        elif self.data is not None:
            if isinstance(self.data, np.ndarray):
                if isinstance(other.data, np.ndarray):
                    self.data = np.hstack((self.data, other.data))
                elif scipy.sparse.issparse(other.data):
                    self.data = np.hstack((self.data, other.data.toarray()))
2629
                elif isinstance(other.data, pd_DataFrame):
Guolin Ke's avatar
Guolin Ke committed
2630
                    self.data = np.hstack((self.data, other.data.values))
2631
                elif isinstance(other.data, dt_DataTable):
Guolin Ke's avatar
Guolin Ke committed
2632
2633
2634
2635
2636
2637
2638
                    self.data = np.hstack((self.data, other.data.to_numpy()))
                else:
                    self.data = None
            elif scipy.sparse.issparse(self.data):
                sparse_format = self.data.getformat()
                if isinstance(other.data, np.ndarray) or scipy.sparse.issparse(other.data):
                    self.data = scipy.sparse.hstack((self.data, other.data), format=sparse_format)
2639
                elif isinstance(other.data, pd_DataFrame):
Guolin Ke's avatar
Guolin Ke committed
2640
                    self.data = scipy.sparse.hstack((self.data, other.data.values), format=sparse_format)
2641
                elif isinstance(other.data, dt_DataTable):
Guolin Ke's avatar
Guolin Ke committed
2642
2643
2644
                    self.data = scipy.sparse.hstack((self.data, other.data.to_numpy()), format=sparse_format)
                else:
                    self.data = None
2645
            elif isinstance(self.data, pd_DataFrame):
Guolin Ke's avatar
Guolin Ke committed
2646
2647
                if not PANDAS_INSTALLED:
                    raise LightGBMError("Cannot add features to DataFrame type of raw data "
2648
2649
                                        "without pandas installed. "
                                        "Install pandas and restart your session.")
Guolin Ke's avatar
Guolin Ke committed
2650
                if isinstance(other.data, np.ndarray):
2651
                    self.data = concat((self.data, pd_DataFrame(other.data)),
Guolin Ke's avatar
Guolin Ke committed
2652
2653
                                       axis=1, ignore_index=True)
                elif scipy.sparse.issparse(other.data):
2654
                    self.data = concat((self.data, pd_DataFrame(other.data.toarray())),
Guolin Ke's avatar
Guolin Ke committed
2655
                                       axis=1, ignore_index=True)
2656
                elif isinstance(other.data, pd_DataFrame):
Guolin Ke's avatar
Guolin Ke committed
2657
2658
                    self.data = concat((self.data, other.data),
                                       axis=1, ignore_index=True)
2659
2660
                elif isinstance(other.data, dt_DataTable):
                    self.data = concat((self.data, pd_DataFrame(other.data.to_numpy())),
Guolin Ke's avatar
Guolin Ke committed
2661
2662
2663
                                       axis=1, ignore_index=True)
                else:
                    self.data = None
2664
            elif isinstance(self.data, dt_DataTable):
Guolin Ke's avatar
Guolin Ke committed
2665
                if isinstance(other.data, np.ndarray):
2666
                    self.data = dt_DataTable(np.hstack((self.data.to_numpy(), other.data)))
Guolin Ke's avatar
Guolin Ke committed
2667
                elif scipy.sparse.issparse(other.data):
2668
2669
2670
2671
2672
                    self.data = dt_DataTable(np.hstack((self.data.to_numpy(), other.data.toarray())))
                elif isinstance(other.data, pd_DataFrame):
                    self.data = dt_DataTable(np.hstack((self.data.to_numpy(), other.data.values)))
                elif isinstance(other.data, dt_DataTable):
                    self.data = dt_DataTable(np.hstack((self.data.to_numpy(), other.data.to_numpy())))
Guolin Ke's avatar
Guolin Ke committed
2673
2674
2675
2676
2677
                else:
                    self.data = None
            else:
                self.data = None
        if self.data is None:
2678
2679
            err_msg = (f"Cannot add features from {type(other.data).__name__} type of raw data to "
                       f"{old_self_data_type} type of raw data.\n")
Guolin Ke's avatar
Guolin Ke committed
2680
2681
            err_msg += ("Set free_raw_data=False when construct Dataset to avoid this"
                        if was_none else "Freeing raw data")
2682
            _log_warning(err_msg)
Guolin Ke's avatar
Guolin Ke committed
2683
        self.feature_name = self.get_feature_name()
2684
2685
        _log_warning("Reseting categorical features.\n"
                     "You can set new categorical features via ``set_categorical_feature`` method")
Guolin Ke's avatar
Guolin Ke committed
2686
2687
        self.categorical_feature = "auto"
        self.pandas_categorical = None
2688
2689
        return self

2690
    def _dump_text(self, filename: Union[str, Path]) -> "Dataset":
2691
2692
2693
2694
2695
2696
        """Save Dataset to a text file.

        This format cannot be loaded back in by LightGBM, but is useful for debugging purposes.

        Parameters
        ----------
2697
        filename : str or pathlib.Path
2698
2699
2700
2701
2702
2703
2704
2705
2706
            Name of the output file.

        Returns
        -------
        self : Dataset
            Returns self.
        """
        _safe_call(_LIB.LGBM_DatasetDumpText(
            self.construct().handle,
2707
            _c_str(str(filename))))
2708
2709
        return self

wxchan's avatar
wxchan committed
2710

2711
2712
2713
2714
_LGBM_CustomObjectiveFunction = Callable[
    [np.ndarray, Dataset],
    Tuple[np.ndarray, np.ndarray]
]
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
_LGBM_CustomEvalFunction = Union[
    Callable[
        [np.ndarray, Dataset],
        _LGBM_EvalFunctionResultType
    ],
    Callable[
        [np.ndarray, Dataset],
        List[_LGBM_EvalFunctionResultType]
    ]
]
2725
2726


2727
class Booster:
2728
    """Booster in LightGBM."""
2729

2730
2731
2732
2733
2734
2735
2736
    def __init__(
        self,
        params: Optional[Dict[str, Any]] = None,
        train_set: Optional[Dataset] = None,
        model_file: Optional[Union[str, Path]] = None,
        model_str: Optional[str] = None
    ):
2737
        """Initialize the Booster.
wxchan's avatar
wxchan committed
2738
2739
2740

        Parameters
        ----------
Nikita Titov's avatar
Nikita Titov committed
2741
        params : dict or None, optional (default=None)
2742
2743
2744
            Parameters for Booster.
        train_set : Dataset or None, optional (default=None)
            Training dataset.
2745
        model_file : str, pathlib.Path or None, optional (default=None)
wxchan's avatar
wxchan committed
2746
            Path to the model file.
2747
        model_str : str or None, optional (default=None)
2748
            Model will be loaded from this string.
wxchan's avatar
wxchan committed
2749
        """
2750
        self.handle = None
2751
        self.network = False
wxchan's avatar
wxchan committed
2752
        self.__need_reload_eval_info = True
2753
        self._train_data_name = "training"
2754
        self.__set_objective_to_none = False
wxchan's avatar
wxchan committed
2755
        self.best_iteration = -1
wxchan's avatar
wxchan committed
2756
        self.best_score = {}
2757
        params = {} if params is None else deepcopy(params)
wxchan's avatar
wxchan committed
2758
        if train_set is not None:
2759
            # Training task
wxchan's avatar
wxchan committed
2760
            if not isinstance(train_set, Dataset):
2761
                raise TypeError(f'Training data should be Dataset instance, met {type(train_set).__name__}')
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
            params = _choose_param_value(
                main_param_name="machines",
                params=params,
                default_value=None
            )
            # if "machines" is given, assume user wants to do distributed learning, and set up network
            if params["machines"] is None:
                params.pop("machines", None)
            else:
                machines = params["machines"]
                if isinstance(machines, str):
                    num_machines_from_machine_list = len(machines.split(','))
                elif isinstance(machines, (list, set)):
                    num_machines_from_machine_list = len(machines)
                    machines = ','.join(machines)
                else:
                    raise ValueError("Invalid machines in params.")

                params = _choose_param_value(
                    main_param_name="num_machines",
                    params=params,
                    default_value=num_machines_from_machine_list
                )
                params = _choose_param_value(
                    main_param_name="local_listen_port",
                    params=params,
                    default_value=12400
                )
                self.set_network(
                    machines=machines,
                    local_listen_port=params["local_listen_port"],
                    listen_time_out=params.get("time_out", 120),
                    num_machines=params["num_machines"]
                )
2796
            # construct booster object
2797
2798
2799
            train_set.construct()
            # copy the parameters from train_set
            params.update(train_set.get_params())
2800
            params_str = _param_dict_to_str(params)
2801
            self.handle = ctypes.c_void_p()
wxchan's avatar
wxchan committed
2802
            _safe_call(_LIB.LGBM_BoosterCreate(
2803
                train_set.handle,
2804
                _c_str(params_str),
wxchan's avatar
wxchan committed
2805
                ctypes.byref(self.handle)))
2806
            # save reference to data
wxchan's avatar
wxchan committed
2807
            self.train_set = train_set
2808
2809
            self.valid_sets: List[Dataset] = []
            self.name_valid_sets: List[str] = []
wxchan's avatar
wxchan committed
2810
            self.__num_dataset = 1
Guolin Ke's avatar
Guolin Ke committed
2811
2812
            self.__init_predictor = train_set._predictor
            if self.__init_predictor is not None:
wxchan's avatar
wxchan committed
2813
2814
                _safe_call(_LIB.LGBM_BoosterMerge(
                    self.handle,
Guolin Ke's avatar
Guolin Ke committed
2815
                    self.__init_predictor.handle))
Guolin Ke's avatar
Guolin Ke committed
2816
            out_num_class = ctypes.c_int(0)
wxchan's avatar
wxchan committed
2817
2818
2819
2820
            _safe_call(_LIB.LGBM_BoosterGetNumClasses(
                self.handle,
                ctypes.byref(out_num_class)))
            self.__num_class = out_num_class.value
2821
            # buffer for inner predict
wxchan's avatar
wxchan committed
2822
2823
2824
            self.__inner_predict_buffer = [None]
            self.__is_predicted_cur_iter = [False]
            self.__get_eval_info()
2825
            self.pandas_categorical = train_set.pandas_categorical
2826
            self.train_set_version = train_set.version
wxchan's avatar
wxchan committed
2827
        elif model_file is not None:
2828
            # Prediction task
Guolin Ke's avatar
Guolin Ke committed
2829
            out_num_iterations = ctypes.c_int(0)
2830
            self.handle = ctypes.c_void_p()
wxchan's avatar
wxchan committed
2831
            _safe_call(_LIB.LGBM_BoosterCreateFromModelfile(
2832
                _c_str(str(model_file)),
wxchan's avatar
wxchan committed
2833
2834
                ctypes.byref(out_num_iterations),
                ctypes.byref(self.handle)))
Guolin Ke's avatar
Guolin Ke committed
2835
            out_num_class = ctypes.c_int(0)
wxchan's avatar
wxchan committed
2836
2837
2838
2839
            _safe_call(_LIB.LGBM_BoosterGetNumClasses(
                self.handle,
                ctypes.byref(out_num_class)))
            self.__num_class = out_num_class.value
2840
            self.pandas_categorical = _load_pandas_categorical(file_name=model_file)
2841
2842
2843
            if params:
                _log_warning('Ignoring params argument, using parameters from model file.')
            params = self._get_loaded_param()
2844
        elif model_str is not None:
2845
            self.model_from_string(model_str)
wxchan's avatar
wxchan committed
2846
        else:
2847
2848
            raise TypeError('Need at least one training dataset or model file or model string '
                            'to create Booster instance')
2849
        self.params = params
wxchan's avatar
wxchan committed
2850

2851
    def __del__(self) -> None:
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
        try:
            if self.network:
                self.free_network()
        except AttributeError:
            pass
        try:
            if self.handle is not None:
                _safe_call(_LIB.LGBM_BoosterFree(self.handle))
        except AttributeError:
            pass
wxchan's avatar
wxchan committed
2862

2863
    def __copy__(self) -> "Booster":
wxchan's avatar
wxchan committed
2864
2865
        return self.__deepcopy__(None)

2866
    def __deepcopy__(self, _) -> "Booster":
2867
        model_str = self.model_to_string(num_iteration=-1)
2868
        booster = Booster(model_str=model_str)
2869
        return booster
wxchan's avatar
wxchan committed
2870

2871
    def __getstate__(self) -> Dict[str, Any]:
wxchan's avatar
wxchan committed
2872
2873
2874
2875
2876
        this = self.__dict__.copy()
        handle = this['handle']
        this.pop('train_set', None)
        this.pop('valid_sets', None)
        if handle is not None:
2877
            this["handle"] = self.model_to_string(num_iteration=-1)
wxchan's avatar
wxchan committed
2878
2879
        return this

2880
    def __setstate__(self, state: Dict[str, Any]) -> None:
2881
2882
        model_str = state.get('handle', None)
        if model_str is not None:
wxchan's avatar
wxchan committed
2883
            handle = ctypes.c_void_p()
Guolin Ke's avatar
Guolin Ke committed
2884
            out_num_iterations = ctypes.c_int(0)
2885
            _safe_call(_LIB.LGBM_BoosterLoadModelFromString(
2886
                _c_str(model_str),
2887
2888
                ctypes.byref(out_num_iterations),
                ctypes.byref(handle)))
wxchan's avatar
wxchan committed
2889
2890
2891
            state['handle'] = handle
        self.__dict__.update(state)

2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
    def _get_loaded_param(self) -> Dict[str, Any]:
        buffer_len = 1 << 20
        tmp_out_len = ctypes.c_int64(0)
        string_buffer = ctypes.create_string_buffer(buffer_len)
        ptr_string_buffer = ctypes.c_char_p(*[ctypes.addressof(string_buffer)])
        _safe_call(_LIB.LGBM_BoosterGetLoadedParam(
            self.handle,
            ctypes.c_int64(buffer_len),
            ctypes.byref(tmp_out_len),
            ptr_string_buffer))
        actual_len = tmp_out_len.value
        # if buffer length is not long enough, re-allocate a buffer
        if actual_len > buffer_len:
            string_buffer = ctypes.create_string_buffer(actual_len)
            ptr_string_buffer = ctypes.c_char_p(*[ctypes.addressof(string_buffer)])
            _safe_call(_LIB.LGBM_BoosterGetLoadedParam(
                self.handle,
                ctypes.c_int64(actual_len),
                ctypes.byref(tmp_out_len),
                ptr_string_buffer))
        return json.loads(string_buffer.value.decode('utf-8'))

2914
    def free_dataset(self) -> "Booster":
Nikita Titov's avatar
Nikita Titov committed
2915
2916
2917
2918
2919
2920
2921
        """Free Booster's Datasets.

        Returns
        -------
        self : Booster
            Booster without Datasets.
        """
wxchan's avatar
wxchan committed
2922
2923
        self.__dict__.pop('train_set', None)
        self.__dict__.pop('valid_sets', None)
2924
        self.__num_dataset = 0
Nikita Titov's avatar
Nikita Titov committed
2925
        return self
wxchan's avatar
wxchan committed
2926

2927
    def _free_buffer(self) -> "Booster":
2928
2929
        self.__inner_predict_buffer = []
        self.__is_predicted_cur_iter = []
Nikita Titov's avatar
Nikita Titov committed
2930
        return self
2931

2932
2933
2934
2935
2936
2937
2938
    def set_network(
        self,
        machines: Union[List[str], Set[str], str],
        local_listen_port: int = 12400,
        listen_time_out: int = 120,
        num_machines: int = 1
    ) -> "Booster":
2939
2940
2941
2942
        """Set the network configuration.

        Parameters
        ----------
2943
        machines : list, set or str
2944
            Names of machines.
Nikita Titov's avatar
Nikita Titov committed
2945
        local_listen_port : int, optional (default=12400)
2946
            TCP listen port for local machines.
Nikita Titov's avatar
Nikita Titov committed
2947
        listen_time_out : int, optional (default=120)
2948
            Socket time-out in minutes.
Nikita Titov's avatar
Nikita Titov committed
2949
        num_machines : int, optional (default=1)
2950
            The number of machines for distributed learning application.
Nikita Titov's avatar
Nikita Titov committed
2951
2952
2953
2954
2955

        Returns
        -------
        self : Booster
            Booster with set network.
2956
        """
2957
2958
        if isinstance(machines, (list, set)):
            machines = ','.join(machines)
2959
        _safe_call(_LIB.LGBM_NetworkInit(_c_str(machines),
2960
2961
2962
2963
                                         ctypes.c_int(local_listen_port),
                                         ctypes.c_int(listen_time_out),
                                         ctypes.c_int(num_machines)))
        self.network = True
Nikita Titov's avatar
Nikita Titov committed
2964
        return self
2965

2966
    def free_network(self) -> "Booster":
Nikita Titov's avatar
Nikita Titov committed
2967
2968
2969
2970
2971
2972
2973
        """Free Booster's network.

        Returns
        -------
        self : Booster
            Booster with freed network.
        """
2974
2975
        _safe_call(_LIB.LGBM_NetworkFree())
        self.network = False
Nikita Titov's avatar
Nikita Titov committed
2976
        return self
2977

2978
    def trees_to_dataframe(self) -> pd_DataFrame:
2979
2980
        """Parse the fitted model and return in an easy-to-read pandas DataFrame.

2981
2982
2983
2984
        The returned DataFrame has the following columns.

            - ``tree_index`` : int64, which tree a node belongs to. 0-based, so a value of ``6``, for example, means "this node is in the 7th tree".
            - ``node_depth`` : int64, how far a node is from the root of the tree. The root node has a value of ``1``, its direct children are ``2``, etc.
2985
2986
2987
2988
2989
            - ``node_index`` : str, unique identifier for a node.
            - ``left_child`` : str, ``node_index`` of the child node to the left of a split. ``None`` for leaf nodes.
            - ``right_child`` : str, ``node_index`` of the child node to the right of a split. ``None`` for leaf nodes.
            - ``parent_index`` : str, ``node_index`` of this node's parent. ``None`` for the root node.
            - ``split_feature`` : str, name of the feature used for splitting. ``None`` for leaf nodes.
2990
2991
            - ``split_gain`` : float64, gain from adding this split to the tree. ``NaN`` for leaf nodes.
            - ``threshold`` : float64, value of the feature used to decide which side of the split a record will go down. ``NaN`` for leaf nodes.
2992
            - ``decision_type`` : str, logical operator describing how to compare a value to ``threshold``.
2993
2994
              For example, ``split_feature = "Column_10", threshold = 15, decision_type = "<="`` means that
              records where ``Column_10 <= 15`` follow the left side of the split, otherwise follows the right side of the split. ``None`` for leaf nodes.
2995
2996
            - ``missing_direction`` : str, split direction that missing values should go to. ``None`` for leaf nodes.
            - ``missing_type`` : str, describes what types of values are treated as missing.
2997
            - ``value`` : float64, predicted value for this leaf node, multiplied by the learning rate.
2998
            - ``weight`` : float64 or int64, sum of Hessian (second-order derivative of objective), summed over observations that fall in this node.
2999
3000
            - ``count`` : int64, number of records in the training data that fall into this node.

3001
3002
3003
3004
3005
3006
        Returns
        -------
        result : pandas DataFrame
            Returns a pandas DataFrame of the parsed model.
        """
        if not PANDAS_INSTALLED:
3007
3008
            raise LightGBMError('This method cannot be run without pandas installed. '
                                'You must install pandas and restart your session to use this method.')
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019

        if self.num_trees() == 0:
            raise LightGBMError('There are no trees in this Booster and thus nothing to parse')

        def _is_split_node(tree):
            return 'split_index' in tree.keys()

        def create_node_record(tree, node_depth=1, tree_index=None,
                               feature_names=None, parent_node=None):

            def _get_node_index(tree, tree_index):
3020
                tree_num = f'{tree_index}-' if tree_index is not None else ''
3021
3022
3023
                is_split = _is_split_node(tree)
                node_type = 'S' if is_split else 'L'
                # if a single node tree it won't have `leaf_index` so return 0
3024
3025
                node_num = tree.get('split_index' if is_split else 'leaf_index', 0)
                return f"{tree_num}{node_type}{node_num}"
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037

            def _get_split_feature(tree, feature_names):
                if _is_split_node(tree):
                    if feature_names is not None:
                        feature_name = feature_names[tree['split_feature']]
                    else:
                        feature_name = tree['split_feature']
                else:
                    feature_name = None
                return feature_name

            def _is_single_node_tree(tree):
3038
                return set(tree.keys()) == {'leaf_value'}
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111

            # Create the node record, and populate universal data members
            node = OrderedDict()
            node['tree_index'] = tree_index
            node['node_depth'] = node_depth
            node['node_index'] = _get_node_index(tree, tree_index)
            node['left_child'] = None
            node['right_child'] = None
            node['parent_index'] = parent_node
            node['split_feature'] = _get_split_feature(tree, feature_names)
            node['split_gain'] = None
            node['threshold'] = None
            node['decision_type'] = None
            node['missing_direction'] = None
            node['missing_type'] = None
            node['value'] = None
            node['weight'] = None
            node['count'] = None

            # Update values to reflect node type (leaf or split)
            if _is_split_node(tree):
                node['left_child'] = _get_node_index(tree['left_child'], tree_index)
                node['right_child'] = _get_node_index(tree['right_child'], tree_index)
                node['split_gain'] = tree['split_gain']
                node['threshold'] = tree['threshold']
                node['decision_type'] = tree['decision_type']
                node['missing_direction'] = 'left' if tree['default_left'] else 'right'
                node['missing_type'] = tree['missing_type']
                node['value'] = tree['internal_value']
                node['weight'] = tree['internal_weight']
                node['count'] = tree['internal_count']
            else:
                node['value'] = tree['leaf_value']
                if not _is_single_node_tree(tree):
                    node['weight'] = tree['leaf_weight']
                    node['count'] = tree['leaf_count']

            return node

        def tree_dict_to_node_list(tree, node_depth=1, tree_index=None,
                                   feature_names=None, parent_node=None):

            node = create_node_record(tree,
                                      node_depth=node_depth,
                                      tree_index=tree_index,
                                      feature_names=feature_names,
                                      parent_node=parent_node)

            res = [node]

            if _is_split_node(tree):
                # traverse the next level of the tree
                children = ['left_child', 'right_child']
                for child in children:
                    subtree_list = tree_dict_to_node_list(
                        tree[child],
                        node_depth=node_depth + 1,
                        tree_index=tree_index,
                        feature_names=feature_names,
                        parent_node=node['node_index'])
                    # In tree format, "subtree_list" is a list of node records (dicts),
                    # and we add node to the list.
                    res.extend(subtree_list)
            return res

        model_dict = self.dump_model()
        feature_names = model_dict['feature_names']
        model_list = []
        for tree in model_dict['tree_info']:
            model_list.extend(tree_dict_to_node_list(tree['tree_structure'],
                                                     tree_index=tree['tree_index'],
                                                     feature_names=feature_names))

3112
        return pd_DataFrame(model_list, columns=model_list[0].keys())
3113

3114
    def set_train_data_name(self, name: str) -> "Booster":
3115
3116
3117
3118
        """Set the name to the training Dataset.

        Parameters
        ----------
3119
        name : str
Nikita Titov's avatar
Nikita Titov committed
3120
3121
3122
3123
3124
3125
            Name for the training Dataset.

        Returns
        -------
        self : Booster
            Booster with set training Dataset name.
3126
        """
3127
        self._train_data_name = name
Nikita Titov's avatar
Nikita Titov committed
3128
        return self
wxchan's avatar
wxchan committed
3129

3130
    def add_valid(self, data: Dataset, name: str) -> "Booster":
3131
        """Add validation data.
wxchan's avatar
wxchan committed
3132
3133
3134
3135

        Parameters
        ----------
        data : Dataset
3136
            Validation data.
3137
        name : str
3138
            Name of validation data.
Nikita Titov's avatar
Nikita Titov committed
3139
3140
3141
3142
3143

        Returns
        -------
        self : Booster
            Booster with set validation data.
wxchan's avatar
wxchan committed
3144
        """
Guolin Ke's avatar
Guolin Ke committed
3145
        if not isinstance(data, Dataset):
3146
            raise TypeError(f'Validation data should be Dataset instance, met {type(data).__name__}')
Guolin Ke's avatar
Guolin Ke committed
3147
        if data._predictor is not self.__init_predictor:
3148
3149
            raise LightGBMError("Add validation data failed, "
                                "you should use same predictor for these data")
wxchan's avatar
wxchan committed
3150
3151
        _safe_call(_LIB.LGBM_BoosterAddValidData(
            self.handle,
wxchan's avatar
wxchan committed
3152
            data.construct().handle))
wxchan's avatar
wxchan committed
3153
3154
3155
3156
3157
        self.valid_sets.append(data)
        self.name_valid_sets.append(name)
        self.__num_dataset += 1
        self.__inner_predict_buffer.append(None)
        self.__is_predicted_cur_iter.append(False)
Nikita Titov's avatar
Nikita Titov committed
3158
        return self
wxchan's avatar
wxchan committed
3159

3160
    def reset_parameter(self, params: Dict[str, Any]) -> "Booster":
3161
        """Reset parameters of Booster.
wxchan's avatar
wxchan committed
3162
3163
3164
3165

        Parameters
        ----------
        params : dict
3166
            New parameters for Booster.
Nikita Titov's avatar
Nikita Titov committed
3167
3168
3169
3170
3171

        Returns
        -------
        self : Booster
            Booster with new parameters.
wxchan's avatar
wxchan committed
3172
        """
3173
        params_str = _param_dict_to_str(params)
wxchan's avatar
wxchan committed
3174
3175
3176
        if params_str:
            _safe_call(_LIB.LGBM_BoosterResetParameter(
                self.handle,
3177
                _c_str(params_str)))
Guolin Ke's avatar
Guolin Ke committed
3178
        self.params.update(params)
Nikita Titov's avatar
Nikita Titov committed
3179
        return self
wxchan's avatar
wxchan committed
3180

3181
3182
3183
3184
3185
    def update(
        self,
        train_set: Optional[Dataset] = None,
        fobj: Optional[_LGBM_CustomObjectiveFunction] = None
    ) -> bool:
Nikita Titov's avatar
Nikita Titov committed
3186
        """Update Booster for one iteration.
3187

wxchan's avatar
wxchan committed
3188
3189
        Parameters
        ----------
3190
3191
3192
3193
        train_set : Dataset or None, optional (default=None)
            Training data.
            If None, last training data is used.
        fobj : callable or None, optional (default=None)
wxchan's avatar
wxchan committed
3194
            Customized objective function.
3195
3196
3197
            Should accept two parameters: preds, train_data,
            and return (grad, hess).

3198
                preds : numpy 1-D array or numpy 2-D array (for multi-class task)
3199
                    The predicted values.
3200
3201
                    Predicted values are returned before any transformation,
                    e.g. they are raw margin instead of probability of positive class for binary task.
3202
3203
                train_data : Dataset
                    The training dataset.
3204
                grad : numpy 1-D array or numpy 2-D array (for multi-class task)
3205
3206
                    The value of the first order derivative (gradient) of the loss
                    with respect to the elements of preds for each sample point.
3207
                hess : numpy 1-D array or numpy 2-D array (for multi-class task)
3208
3209
                    The value of the second order derivative (Hessian) of the loss
                    with respect to the elements of preds for each sample point.
wxchan's avatar
wxchan committed
3210

3211
            For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes],
3212
            and grad and hess should be returned in the same format.
3213

wxchan's avatar
wxchan committed
3214
3215
        Returns
        -------
3216
3217
        is_finished : bool
            Whether the update was successfully finished.
wxchan's avatar
wxchan committed
3218
        """
3219
        # need reset training data
3220
3221
3222
3223
3224
3225
        if train_set is None and self.train_set_version != self.train_set.version:
            train_set = self.train_set
            is_the_same_train_set = False
        else:
            is_the_same_train_set = train_set is self.train_set and self.train_set_version == train_set.version
        if train_set is not None and not is_the_same_train_set:
Guolin Ke's avatar
Guolin Ke committed
3226
            if not isinstance(train_set, Dataset):
3227
                raise TypeError(f'Training data should be Dataset instance, met {type(train_set).__name__}')
Guolin Ke's avatar
Guolin Ke committed
3228
            if train_set._predictor is not self.__init_predictor:
3229
3230
                raise LightGBMError("Replace training data failed, "
                                    "you should use same predictor for these data")
wxchan's avatar
wxchan committed
3231
3232
3233
            self.train_set = train_set
            _safe_call(_LIB.LGBM_BoosterResetTrainingData(
                self.handle,
wxchan's avatar
wxchan committed
3234
                self.train_set.construct().handle))
wxchan's avatar
wxchan committed
3235
            self.__inner_predict_buffer[0] = None
3236
            self.train_set_version = self.train_set.version
wxchan's avatar
wxchan committed
3237
3238
        is_finished = ctypes.c_int(0)
        if fobj is None:
3239
            if self.__set_objective_to_none:
3240
                raise LightGBMError('Cannot update due to null objective function.')
wxchan's avatar
wxchan committed
3241
3242
3243
            _safe_call(_LIB.LGBM_BoosterUpdateOneIter(
                self.handle,
                ctypes.byref(is_finished)))
3244
            self.__is_predicted_cur_iter = [False for _ in range(self.__num_dataset)]
wxchan's avatar
wxchan committed
3245
3246
            return is_finished.value == 1
        else:
3247
            if not self.__set_objective_to_none:
Nikita Titov's avatar
Nikita Titov committed
3248
                self.reset_parameter({"objective": "none"}).__set_objective_to_none = True
wxchan's avatar
wxchan committed
3249
3250
3251
            grad, hess = fobj(self.__inner_predict(0), self.train_set)
            return self.__boost(grad, hess)

3252
3253
3254
3255
3256
    def __boost(
        self,
        grad: np.ndarray,
        hess: np.ndarray
    ) -> bool:
3257
        """Boost Booster for one iteration with customized gradient statistics.
Nikita Titov's avatar
Nikita Titov committed
3258

Nikita Titov's avatar
Nikita Titov committed
3259
3260
        .. note::

3261
3262
            Score is returned before any transformation,
            e.g. it is raw margin instead of probability of positive class for binary task.
3263
            For multi-class task, score are numpy 2-D array of shape = [n_samples, n_classes],
3264
            and grad and hess should be returned in the same format.
3265

wxchan's avatar
wxchan committed
3266
3267
        Parameters
        ----------
3268
        grad : numpy 1-D array or numpy 2-D array (for multi-class task)
3269
3270
            The value of the first order derivative (gradient) of the loss
            with respect to the elements of score for each sample point.
3271
        hess : numpy 1-D array or numpy 2-D array (for multi-class task)
3272
3273
            The value of the second order derivative (Hessian) of the loss
            with respect to the elements of score for each sample point.
wxchan's avatar
wxchan committed
3274
3275
3276

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
3277
3278
        is_finished : bool
            Whether the boost was successfully finished.
wxchan's avatar
wxchan committed
3279
        """
3280
3281
3282
        if self.__num_class > 1:
            grad = grad.ravel(order='F')
            hess = hess.ravel(order='F')
3283
3284
        grad = _list_to_1d_numpy(grad, name='gradient')
        hess = _list_to_1d_numpy(hess, name='hessian')
3285
3286
        assert grad.flags.c_contiguous
        assert hess.flags.c_contiguous
wxchan's avatar
wxchan committed
3287
        if len(grad) != len(hess):
3288
3289
            raise ValueError(f"Lengths of gradient ({len(grad)}) and Hessian ({len(hess)}) don't match")
        num_train_data = self.train_set.num_data()
3290
        if len(grad) != num_train_data * self.__num_class:
3291
3292
3293
            raise ValueError(
                f"Lengths of gradient ({len(grad)}) and Hessian ({len(hess)}) "
                f"don't match training data length ({num_train_data}) * "
3294
                f"number of models per one iteration ({self.__num_class})"
3295
            )
wxchan's avatar
wxchan committed
3296
3297
3298
3299
3300
3301
        is_finished = ctypes.c_int(0)
        _safe_call(_LIB.LGBM_BoosterUpdateOneIterCustom(
            self.handle,
            grad.ctypes.data_as(ctypes.POINTER(ctypes.c_float)),
            hess.ctypes.data_as(ctypes.POINTER(ctypes.c_float)),
            ctypes.byref(is_finished)))
3302
        self.__is_predicted_cur_iter = [False for _ in range(self.__num_dataset)]
wxchan's avatar
wxchan committed
3303
3304
        return is_finished.value == 1

3305
    def rollback_one_iter(self) -> "Booster":
Nikita Titov's avatar
Nikita Titov committed
3306
3307
3308
3309
3310
3311
3312
        """Rollback one iteration.

        Returns
        -------
        self : Booster
            Booster with rolled back one iteration.
        """
wxchan's avatar
wxchan committed
3313
3314
        _safe_call(_LIB.LGBM_BoosterRollbackOneIter(
            self.handle))
3315
        self.__is_predicted_cur_iter = [False for _ in range(self.__num_dataset)]
Nikita Titov's avatar
Nikita Titov committed
3316
        return self
wxchan's avatar
wxchan committed
3317

3318
    def current_iteration(self) -> int:
3319
3320
3321
3322
3323
3324
3325
        """Get the index of the current iteration.

        Returns
        -------
        cur_iter : int
            The index of the current iteration.
        """
Guolin Ke's avatar
Guolin Ke committed
3326
        out_cur_iter = ctypes.c_int(0)
wxchan's avatar
wxchan committed
3327
3328
3329
3330
3331
        _safe_call(_LIB.LGBM_BoosterGetCurrentIteration(
            self.handle,
            ctypes.byref(out_cur_iter)))
        return out_cur_iter.value

3332
    def num_model_per_iteration(self) -> int:
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
        """Get number of models per iteration.

        Returns
        -------
        model_per_iter : int
            The number of models per iteration.
        """
        model_per_iter = ctypes.c_int(0)
        _safe_call(_LIB.LGBM_BoosterNumModelPerIteration(
            self.handle,
            ctypes.byref(model_per_iter)))
        return model_per_iter.value

3346
    def num_trees(self) -> int:
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
        """Get number of weak sub-models.

        Returns
        -------
        num_trees : int
            The number of weak sub-models.
        """
        num_trees = ctypes.c_int(0)
        _safe_call(_LIB.LGBM_BoosterNumberOfTotalModel(
            self.handle,
            ctypes.byref(num_trees)))
        return num_trees.value

3360
    def upper_bound(self) -> float:
3361
3362
3363
3364
        """Get upper bound value of a model.

        Returns
        -------
3365
        upper_bound : float
3366
3367
3368
3369
3370
3371
3372
3373
            Upper bound value of the model.
        """
        ret = ctypes.c_double(0)
        _safe_call(_LIB.LGBM_BoosterGetUpperBoundValue(
            self.handle,
            ctypes.byref(ret)))
        return ret.value

3374
    def lower_bound(self) -> float:
3375
3376
3377
3378
        """Get lower bound value of a model.

        Returns
        -------
3379
        lower_bound : float
3380
3381
3382
3383
3384
3385
3386
3387
            Lower bound value of the model.
        """
        ret = ctypes.c_double(0)
        _safe_call(_LIB.LGBM_BoosterGetLowerBoundValue(
            self.handle,
            ctypes.byref(ret)))
        return ret.value

3388
3389
3390
3391
3392
3393
    def eval(
        self,
        data: Dataset,
        name: str,
        feval: Optional[Union[_LGBM_CustomEvalFunction, List[_LGBM_CustomEvalFunction]]] = None
    ) -> List[_LGBM_BoosterEvalMethodResultType]:
3394
        """Evaluate for data.
wxchan's avatar
wxchan committed
3395
3396
3397

        Parameters
        ----------
3398
3399
        data : Dataset
            Data for the evaluating.
3400
        name : str
3401
            Name of the data.
3402
        feval : callable, list of callable, or None, optional (default=None)
3403
            Customized evaluation function.
3404
            Each evaluation function should accept two parameters: preds, eval_data,
3405
            and return (eval_name, eval_result, is_higher_better) or list of such tuples.
3406

3407
                preds : numpy 1-D array or numpy 2-D array (for multi-class task)
3408
                    The predicted values.
3409
                    For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes].
3410
                    If custom objective function is used, predicted values are returned before any transformation,
3411
                    e.g. they are raw margin instead of probability of positive class for binary task in this case.
3412
                eval_data : Dataset
3413
                    A ``Dataset`` to evaluate.
3414
                eval_name : str
Andrew Ziem's avatar
Andrew Ziem committed
3415
                    The name of evaluation function (without whitespace).
3416
3417
3418
3419
3420
                eval_result : float
                    The eval result.
                is_higher_better : bool
                    Is eval result higher better, e.g. AUC is ``is_higher_better``.

wxchan's avatar
wxchan committed
3421
3422
        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
3423
        result : list
3424
            List with (dataset_name, eval_name, eval_result, is_higher_better) tuples.
wxchan's avatar
wxchan committed
3425
        """
Guolin Ke's avatar
Guolin Ke committed
3426
3427
        if not isinstance(data, Dataset):
            raise TypeError("Can only eval for Dataset instance")
wxchan's avatar
wxchan committed
3428
3429
3430
3431
        data_idx = -1
        if data is self.train_set:
            data_idx = 0
        else:
3432
            for i in range(len(self.valid_sets)):
wxchan's avatar
wxchan committed
3433
3434
3435
                if data is self.valid_sets[i]:
                    data_idx = i + 1
                    break
3436
        # need to push new valid data
wxchan's avatar
wxchan committed
3437
3438
3439
3440
3441
3442
        if data_idx == -1:
            self.add_valid(data, name)
            data_idx = self.__num_dataset - 1

        return self.__inner_eval(name, data_idx, feval)

3443
3444
3445
3446
    def eval_train(
        self,
        feval: Optional[Union[_LGBM_CustomEvalFunction, List[_LGBM_CustomEvalFunction]]] = None
    ) -> List[_LGBM_BoosterEvalMethodResultType]:
3447
        """Evaluate for training data.
wxchan's avatar
wxchan committed
3448
3449
3450

        Parameters
        ----------
3451
        feval : callable, list of callable, or None, optional (default=None)
3452
            Customized evaluation function.
3453
            Each evaluation function should accept two parameters: preds, eval_data,
3454
            and return (eval_name, eval_result, is_higher_better) or list of such tuples.
3455

3456
                preds : numpy 1-D array or numpy 2-D array (for multi-class task)
3457
                    The predicted values.
3458
                    For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes].
3459
                    If custom objective function is used, predicted values are returned before any transformation,
3460
                    e.g. they are raw margin instead of probability of positive class for binary task in this case.
Akshita Dixit's avatar
Akshita Dixit committed
3461
                eval_data : Dataset
3462
                    The training dataset.
3463
                eval_name : str
Andrew Ziem's avatar
Andrew Ziem committed
3464
                    The name of evaluation function (without whitespace).
3465
3466
3467
3468
3469
                eval_result : float
                    The eval result.
                is_higher_better : bool
                    Is eval result higher better, e.g. AUC is ``is_higher_better``.

wxchan's avatar
wxchan committed
3470
3471
        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
3472
        result : list
3473
            List with (train_dataset_name, eval_name, eval_result, is_higher_better) tuples.
wxchan's avatar
wxchan committed
3474
        """
3475
        return self.__inner_eval(self._train_data_name, 0, feval)
wxchan's avatar
wxchan committed
3476

3477
3478
3479
3480
    def eval_valid(
        self,
        feval: Optional[Union[_LGBM_CustomEvalFunction, List[_LGBM_CustomEvalFunction]]] = None
    ) -> List[_LGBM_BoosterEvalMethodResultType]:
3481
        """Evaluate for validation data.
wxchan's avatar
wxchan committed
3482
3483
3484

        Parameters
        ----------
3485
        feval : callable, list of callable, or None, optional (default=None)
3486
            Customized evaluation function.
3487
            Each evaluation function should accept two parameters: preds, eval_data,
3488
            and return (eval_name, eval_result, is_higher_better) or list of such tuples.
3489

3490
                preds : numpy 1-D array or numpy 2-D array (for multi-class task)
3491
                    The predicted values.
3492
                    For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes].
3493
                    If custom objective function is used, predicted values are returned before any transformation,
3494
                    e.g. they are raw margin instead of probability of positive class for binary task in this case.
Akshita Dixit's avatar
Akshita Dixit committed
3495
                eval_data : Dataset
3496
                    The validation dataset.
3497
                eval_name : str
Andrew Ziem's avatar
Andrew Ziem committed
3498
                    The name of evaluation function (without whitespace).
3499
3500
3501
3502
3503
                eval_result : float
                    The eval result.
                is_higher_better : bool
                    Is eval result higher better, e.g. AUC is ``is_higher_better``.

wxchan's avatar
wxchan committed
3504
3505
        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
3506
        result : list
3507
            List with (validation_dataset_name, eval_name, eval_result, is_higher_better) tuples.
wxchan's avatar
wxchan committed
3508
        """
3509
        return [item for i in range(1, self.__num_dataset)
wxchan's avatar
wxchan committed
3510
                for item in self.__inner_eval(self.name_valid_sets[i - 1], i, feval)]
wxchan's avatar
wxchan committed
3511

3512
3513
3514
3515
3516
3517
3518
    def save_model(
        self,
        filename: Union[str, Path],
        num_iteration: Optional[int] = None,
        start_iteration: int = 0,
        importance_type: str = 'split'
    ) -> "Booster":
3519
        """Save Booster to file.
wxchan's avatar
wxchan committed
3520
3521
3522

        Parameters
        ----------
3523
        filename : str or pathlib.Path
3524
            Filename to save Booster.
3525
3526
3527
3528
        num_iteration : int or None, optional (default=None)
            Index of the iteration that should be saved.
            If None, if the best iteration exists, it is saved; otherwise, all iterations are saved.
            If <= 0, all iterations are saved.
Nikita Titov's avatar
Nikita Titov committed
3529
        start_iteration : int, optional (default=0)
3530
            Start index of the iteration that should be saved.
3531
        importance_type : str, optional (default="split")
3532
3533
3534
            What type of feature importance should be saved.
            If "split", result contains numbers of times the feature is used in a model.
            If "gain", result contains total gains of splits which use the feature.
Nikita Titov's avatar
Nikita Titov committed
3535
3536
3537
3538
3539

        Returns
        -------
        self : Booster
            Returns self.
wxchan's avatar
wxchan committed
3540
        """
3541
        if num_iteration is None:
3542
            num_iteration = self.best_iteration
3543
        importance_type_int = _FEATURE_IMPORTANCE_TYPE_MAPPER[importance_type]
wxchan's avatar
wxchan committed
3544
3545
        _safe_call(_LIB.LGBM_BoosterSaveModel(
            self.handle,
3546
            ctypes.c_int(start_iteration),
Guolin Ke's avatar
Guolin Ke committed
3547
            ctypes.c_int(num_iteration),
3548
            ctypes.c_int(importance_type_int),
3549
            _c_str(str(filename))))
3550
        _dump_pandas_categorical(self.pandas_categorical, filename)
Nikita Titov's avatar
Nikita Titov committed
3551
        return self
wxchan's avatar
wxchan committed
3552

3553
3554
3555
3556
3557
    def shuffle_models(
        self,
        start_iteration: int = 0,
        end_iteration: int = -1
    ) -> "Booster":
3558
        """Shuffle models.
Nikita Titov's avatar
Nikita Titov committed
3559

3560
3561
3562
        Parameters
        ----------
        start_iteration : int, optional (default=0)
3563
            The first iteration that will be shuffled.
3564
3565
        end_iteration : int, optional (default=-1)
            The last iteration that will be shuffled.
3566
            If <= 0, means the last available iteration.
3567

Nikita Titov's avatar
Nikita Titov committed
3568
3569
3570
3571
        Returns
        -------
        self : Booster
            Booster with shuffled models.
3572
        """
3573
3574
        _safe_call(_LIB.LGBM_BoosterShuffleModels(
            self.handle,
Guolin Ke's avatar
Guolin Ke committed
3575
3576
            ctypes.c_int(start_iteration),
            ctypes.c_int(end_iteration)))
Nikita Titov's avatar
Nikita Titov committed
3577
        return self
3578

3579
    def model_from_string(self, model_str: str) -> "Booster":
3580
3581
3582
3583
        """Load Booster from a string.

        Parameters
        ----------
3584
        model_str : str
3585
3586
3587
3588
            Model will be loaded from this string.

        Returns
        -------
Nikita Titov's avatar
Nikita Titov committed
3589
        self : Booster
3590
3591
            Loaded Booster object.
        """
3592
3593
3594
3595
        if self.handle is not None:
            _safe_call(_LIB.LGBM_BoosterFree(self.handle))
        self._free_buffer()
        self.handle = ctypes.c_void_p()
3596
3597
        out_num_iterations = ctypes.c_int(0)
        _safe_call(_LIB.LGBM_BoosterLoadModelFromString(
3598
            _c_str(model_str),
3599
3600
3601
3602
3603
3604
3605
            ctypes.byref(out_num_iterations),
            ctypes.byref(self.handle)))
        out_num_class = ctypes.c_int(0)
        _safe_call(_LIB.LGBM_BoosterGetNumClasses(
            self.handle,
            ctypes.byref(out_num_class)))
        self.__num_class = out_num_class.value
3606
        self.pandas_categorical = _load_pandas_categorical(model_str=model_str)
3607
3608
        return self

3609
3610
3611
3612
3613
3614
    def model_to_string(
        self,
        num_iteration: Optional[int] = None,
        start_iteration: int = 0,
        importance_type: str = 'split'
    ) -> str:
3615
        """Save Booster to string.
3616

3617
3618
3619
3620
3621
3622
        Parameters
        ----------
        num_iteration : int or None, optional (default=None)
            Index of the iteration that should be saved.
            If None, if the best iteration exists, it is saved; otherwise, all iterations are saved.
            If <= 0, all iterations are saved.
Nikita Titov's avatar
Nikita Titov committed
3623
        start_iteration : int, optional (default=0)
3624
            Start index of the iteration that should be saved.
3625
        importance_type : str, optional (default="split")
3626
3627
3628
            What type of feature importance should be saved.
            If "split", result contains numbers of times the feature is used in a model.
            If "gain", result contains total gains of splits which use the feature.
3629
3630
3631

        Returns
        -------
3632
        str_repr : str
3633
3634
            String representation of Booster.
        """
3635
        if num_iteration is None:
3636
            num_iteration = self.best_iteration
3637
        importance_type_int = _FEATURE_IMPORTANCE_TYPE_MAPPER[importance_type]
3638
        buffer_len = 1 << 20
3639
        tmp_out_len = ctypes.c_int64(0)
3640
3641
3642
3643
        string_buffer = ctypes.create_string_buffer(buffer_len)
        ptr_string_buffer = ctypes.c_char_p(*[ctypes.addressof(string_buffer)])
        _safe_call(_LIB.LGBM_BoosterSaveModelToString(
            self.handle,
3644
            ctypes.c_int(start_iteration),
3645
            ctypes.c_int(num_iteration),
3646
            ctypes.c_int(importance_type_int),
3647
            ctypes.c_int64(buffer_len),
3648
3649
3650
            ctypes.byref(tmp_out_len),
            ptr_string_buffer))
        actual_len = tmp_out_len.value
3651
        # if buffer length is not long enough, re-allocate a buffer
3652
3653
3654
3655
3656
        if actual_len > buffer_len:
            string_buffer = ctypes.create_string_buffer(actual_len)
            ptr_string_buffer = ctypes.c_char_p(*[ctypes.addressof(string_buffer)])
            _safe_call(_LIB.LGBM_BoosterSaveModelToString(
                self.handle,
3657
                ctypes.c_int(start_iteration),
3658
                ctypes.c_int(num_iteration),
3659
                ctypes.c_int(importance_type_int),
3660
                ctypes.c_int64(actual_len),
3661
3662
                ctypes.byref(tmp_out_len),
                ptr_string_buffer))
3663
        ret = string_buffer.value.decode('utf-8')
3664
3665
        ret += _dump_pandas_categorical(self.pandas_categorical)
        return ret
3666

3667
3668
3669
3670
3671
3672
3673
    def dump_model(
        self,
        num_iteration: Optional[int] = None,
        start_iteration: int = 0,
        importance_type: str = 'split',
        object_hook: Optional[Callable[[Dict[str, Any]], Dict[str, Any]]] = None
    ) -> Dict[str, Any]:
Nikita Titov's avatar
Nikita Titov committed
3674
        """Dump Booster to JSON format.
wxchan's avatar
wxchan committed
3675

3676
3677
        Parameters
        ----------
3678
3679
3680
3681
        num_iteration : int or None, optional (default=None)
            Index of the iteration that should be dumped.
            If None, if the best iteration exists, it is dumped; otherwise, all iterations are dumped.
            If <= 0, all iterations are dumped.
Nikita Titov's avatar
Nikita Titov committed
3682
        start_iteration : int, optional (default=0)
3683
            Start index of the iteration that should be dumped.
3684
        importance_type : str, optional (default="split")
3685
3686
3687
            What type of feature importance should be dumped.
            If "split", result contains numbers of times the feature is used in a model.
            If "gain", result contains total gains of splits which use the feature.
3688
3689
3690
3691
3692
3693
3694
3695
3696
        object_hook : callable or None, optional (default=None)
            If not None, ``object_hook`` is a function called while parsing the json
            string returned by the C API. It may be used to alter the json, to store
            specific values while building the json structure. It avoids
            walking through the structure again. It saves a significant amount
            of time if the number of trees is huge.
            Signature is ``def object_hook(node: dict) -> dict``.
            None is equivalent to ``lambda node: node``.
            See documentation of ``json.loads()`` for further details.
3697

wxchan's avatar
wxchan committed
3698
3699
        Returns
        -------
3700
        json_repr : dict
Nikita Titov's avatar
Nikita Titov committed
3701
            JSON format of Booster.
wxchan's avatar
wxchan committed
3702
        """
3703
        if num_iteration is None:
3704
            num_iteration = self.best_iteration
3705
        importance_type_int = _FEATURE_IMPORTANCE_TYPE_MAPPER[importance_type]
wxchan's avatar
wxchan committed
3706
        buffer_len = 1 << 20
3707
        tmp_out_len = ctypes.c_int64(0)
wxchan's avatar
wxchan committed
3708
3709
3710
3711
        string_buffer = ctypes.create_string_buffer(buffer_len)
        ptr_string_buffer = ctypes.c_char_p(*[ctypes.addressof(string_buffer)])
        _safe_call(_LIB.LGBM_BoosterDumpModel(
            self.handle,
3712
            ctypes.c_int(start_iteration),
Guolin Ke's avatar
Guolin Ke committed
3713
            ctypes.c_int(num_iteration),
3714
            ctypes.c_int(importance_type_int),
3715
            ctypes.c_int64(buffer_len),
wxchan's avatar
wxchan committed
3716
            ctypes.byref(tmp_out_len),
Guolin Ke's avatar
Guolin Ke committed
3717
            ptr_string_buffer))
wxchan's avatar
wxchan committed
3718
        actual_len = tmp_out_len.value
3719
        # if buffer length is not long enough, reallocate a buffer
wxchan's avatar
wxchan committed
3720
3721
3722
3723
3724
        if actual_len > buffer_len:
            string_buffer = ctypes.create_string_buffer(actual_len)
            ptr_string_buffer = ctypes.c_char_p(*[ctypes.addressof(string_buffer)])
            _safe_call(_LIB.LGBM_BoosterDumpModel(
                self.handle,
3725
                ctypes.c_int(start_iteration),
Guolin Ke's avatar
Guolin Ke committed
3726
                ctypes.c_int(num_iteration),
3727
                ctypes.c_int(importance_type_int),
3728
                ctypes.c_int64(actual_len),
wxchan's avatar
wxchan committed
3729
                ctypes.byref(tmp_out_len),
Guolin Ke's avatar
Guolin Ke committed
3730
                ptr_string_buffer))
3731
        ret = json.loads(string_buffer.value.decode('utf-8'), object_hook=object_hook)
3732
        ret['pandas_categorical'] = json.loads(json.dumps(self.pandas_categorical,
3733
                                                          default=_json_default_with_numpy))
3734
        return ret
wxchan's avatar
wxchan committed
3735

3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
    def predict(
        self,
        data,
        start_iteration: int = 0,
        num_iteration: Optional[int] = None,
        raw_score: bool = False,
        pred_leaf: bool = False,
        pred_contrib: bool = False,
        data_has_header: bool = False,
        validate_features: bool = False,
        **kwargs: Any
    ):
3748
        """Make a prediction.
wxchan's avatar
wxchan committed
3749
3750
3751

        Parameters
        ----------
3752
        data : str, pathlib.Path, numpy array, pandas DataFrame, H2O DataTable's Frame or scipy.sparse
3753
            Data source for prediction.
3754
            If str or pathlib.Path, it represents the path to a text file (CSV, TSV, or LibSVM).
3755
        start_iteration : int, optional (default=0)
3756
            Start index of the iteration to predict.
3757
            If <= 0, starts from the first iteration.
3758
        num_iteration : int or None, optional (default=None)
3759
3760
3761
3762
            Total number of iterations used in the prediction.
            If None, if the best iteration exists and start_iteration <= 0, the best iteration is used;
            otherwise, all iterations from ``start_iteration`` are used (no limits).
            If <= 0, all iterations from ``start_iteration`` are used (no limits).
3763
3764
3765
3766
        raw_score : bool, optional (default=False)
            Whether to predict raw scores.
        pred_leaf : bool, optional (default=False)
            Whether to predict leaf index.
3767
3768
        pred_contrib : bool, optional (default=False)
            Whether to predict feature contributions.
3769

Nikita Titov's avatar
Nikita Titov committed
3770
3771
3772
3773
3774
3775
3776
            .. note::

                If you want to get more explanations for your model's predictions using SHAP values,
                like SHAP interaction values,
                you can install the shap package (https://github.com/slundberg/shap).
                Note that unlike the shap package, with ``pred_contrib`` we return a matrix with an extra
                column, where the last column is the expected value.
3777

3778
3779
        data_has_header : bool, optional (default=False)
            Whether the data has header.
3780
            Used only if data is str.
3781
3782
3783
        validate_features : bool, optional (default=False)
            If True, ensure that the features used to predict match the ones used to train.
            Used only if data is pandas DataFrame.
3784
3785
        **kwargs
            Other parameters for the prediction.
wxchan's avatar
wxchan committed
3786
3787
3788

        Returns
        -------
3789
        result : numpy array, scipy.sparse or list of scipy.sparse
3790
            Prediction result.
3791
            Can be sparse or a list of sparse objects (each element represents predictions for one class) for feature contributions (when ``pred_contrib=True``).
wxchan's avatar
wxchan committed
3792
        """
3793
        predictor = self._to_predictor(deepcopy(kwargs))
3794
        if num_iteration is None:
3795
            if start_iteration <= 0:
3796
3797
3798
3799
                num_iteration = self.best_iteration
            else:
                num_iteration = -1
        return predictor.predict(data, start_iteration, num_iteration,
3800
                                 raw_score, pred_leaf, pred_contrib,
3801
                                 data_has_header, validate_features)
wxchan's avatar
wxchan committed
3802

3803
3804
3805
3806
    def refit(
        self,
        data,
        label,
3807
3808
        decay_rate: float = 0.9,
        reference: Optional[Dataset] = None,
3809
3810
3811
        weight=None,
        group=None,
        init_score=None,
3812
3813
3814
3815
3816
        feature_name: Union[str, List[str]] = 'auto',
        categorical_feature: Union[str, List[str], List[int]] = 'auto',
        dataset_params: Optional[Dict[str, Any]] = None,
        free_raw_data: bool = True,
        validate_features: bool = False,
3817
3818
        **kwargs
    ):
Guolin Ke's avatar
Guolin Ke committed
3819
3820
3821
3822
        """Refit the existing Booster by new data.

        Parameters
        ----------
3823
        data : str, pathlib.Path, numpy array, pandas DataFrame, H2O DataTable's Frame or scipy.sparse
Guolin Ke's avatar
Guolin Ke committed
3824
            Data source for refit.
3825
            If str or pathlib.Path, it represents the path to a text file (CSV, TSV, or LibSVM).
3826
        label : list, numpy 1-D array or pandas Series / one-column DataFrame
Guolin Ke's avatar
Guolin Ke committed
3827
3828
            Label for refit.
        decay_rate : float, optional (default=0.9)
3829
3830
            Decay rate of refit,
            will use ``leaf_output = decay_rate * old_leaf_output + (1.0 - decay_rate) * new_leaf_output`` to refit trees.
3831
3832
3833
        reference : Dataset or None, optional (default=None)
            Reference for ``data``.
        weight : list, numpy 1-D array, pandas Series or None, optional (default=None)
3834
            Weight for each ``data`` instance. Weights should be non-negative.
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
        group : list, numpy 1-D array, pandas Series or None, optional (default=None)
            Group/query size for ``data``.
            Only used in the learning-to-rank task.
            sum(group) = n_samples.
            For example, if you have a 100-document dataset with ``group = [10, 20, 40, 10, 10, 10]``, that means that you have 6 groups,
            where the first 10 records are in the first group, records 11-30 are in the second group, records 31-70 are in the third group, etc.
        init_score : list, list of lists (for multi-class task), numpy array, pandas Series, pandas DataFrame (for multi-class task), or None, optional (default=None)
            Init score for ``data``.
        feature_name : list of str, or 'auto', optional (default="auto")
            Feature names for ``data``.
            If 'auto' and data is pandas DataFrame, data columns names are used.
        categorical_feature : list of str or int, or 'auto', optional (default="auto")
            Categorical features for ``data``.
            If list of int, interpreted as indices.
            If list of str, interpreted as feature names (need to specify ``feature_name`` as well).
            If 'auto' and data is pandas DataFrame, pandas unordered categorical columns are used.
3851
            All values in categorical features will be cast to int32 and thus should be less than int32 max value (2147483647).
3852
3853
3854
            Large values could be memory consuming. Consider using consecutive integers starting from zero.
            All negative values in categorical features will be treated as missing values.
            The output cannot be monotonically constrained with respect to a categorical feature.
3855
            Floating point numbers in categorical features will be rounded towards 0.
3856
3857
3858
3859
        dataset_params : dict or None, optional (default=None)
            Other parameters for Dataset ``data``.
        free_raw_data : bool, optional (default=True)
            If True, raw data is freed after constructing inner Dataset for ``data``.
3860
3861
3862
        validate_features : bool, optional (default=False)
            If True, ensure that the features used to refit the model match the original ones.
            Used only if data is pandas DataFrame.
3863
3864
        **kwargs
            Other parameters for refit.
3865
            These parameters will be passed to ``predict`` method.
Guolin Ke's avatar
Guolin Ke committed
3866
3867
3868
3869
3870
3871

        Returns
        -------
        result : Booster
            Refitted Booster.
        """
3872
3873
        if self.__set_objective_to_none:
            raise LightGBMError('Cannot refit due to null objective function.')
3874
3875
        if dataset_params is None:
            dataset_params = {}
3876
        predictor = self._to_predictor(deepcopy(kwargs))
3877
        leaf_preds = predictor.predict(data, -1, pred_leaf=True, validate_features=validate_features)
3878
        nrow, ncol = leaf_preds.shape
3879
        out_is_linear = ctypes.c_int(0)
3880
3881
3882
        _safe_call(_LIB.LGBM_BoosterGetLinear(
            self.handle,
            ctypes.byref(out_is_linear)))
Nikita Titov's avatar
Nikita Titov committed
3883
3884
3885
3886
3887
        new_params = _choose_param_value(
            main_param_name="linear_tree",
            params=self.params,
            default_value=None
        )
3888
        new_params["linear_tree"] = bool(out_is_linear.value)
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
        new_params.update(dataset_params)
        train_set = Dataset(
            data=data,
            label=label,
            reference=reference,
            weight=weight,
            group=group,
            init_score=init_score,
            feature_name=feature_name,
            categorical_feature=categorical_feature,
            params=new_params,
            free_raw_data=free_raw_data,
        )
3902
        new_params['refit_decay_rate'] = decay_rate
3903
        new_booster = Booster(new_params, train_set)
Guolin Ke's avatar
Guolin Ke committed
3904
3905
3906
3907
3908
        # Copy models
        _safe_call(_LIB.LGBM_BoosterMerge(
            new_booster.handle,
            predictor.handle))
        leaf_preds = leaf_preds.reshape(-1)
3909
        ptr_data, _, _ = _c_int_array(leaf_preds)
Guolin Ke's avatar
Guolin Ke committed
3910
3911
3912
        _safe_call(_LIB.LGBM_BoosterRefit(
            new_booster.handle,
            ptr_data,
3913
3914
            ctypes.c_int32(nrow),
            ctypes.c_int32(ncol)))
3915
        new_booster.network = self.network
Guolin Ke's avatar
Guolin Ke committed
3916
3917
        return new_booster

3918
    def get_leaf_output(self, tree_id: int, leaf_id: int) -> float:
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
        """Get the output of a leaf.

        Parameters
        ----------
        tree_id : int
            The index of the tree.
        leaf_id : int
            The index of the leaf in the tree.

        Returns
        -------
        result : float
            The output of the leaf.
        """
3933
3934
3935
3936
3937
3938
3939
3940
        ret = ctypes.c_double(0)
        _safe_call(_LIB.LGBM_BoosterGetLeafValue(
            self.handle,
            ctypes.c_int(tree_id),
            ctypes.c_int(leaf_id),
            ctypes.byref(ret)))
        return ret.value

3941
3942
3943
3944
    def _to_predictor(
        self,
        pred_parameter: Optional[Dict[str, Any]] = None
    ) -> _InnerPredictor:
3945
        """Convert to predictor."""
3946
        predictor = _InnerPredictor(booster_handle=self.handle, pred_parameter=pred_parameter)
3947
        predictor.pandas_categorical = self.pandas_categorical
wxchan's avatar
wxchan committed
3948
3949
        return predictor

3950
    def num_feature(self) -> int:
3951
3952
3953
3954
3955
3956
3957
        """Get number of features.

        Returns
        -------
        num_feature : int
            The number of features.
        """
3958
3959
3960
3961
3962
3963
        out_num_feature = ctypes.c_int(0)
        _safe_call(_LIB.LGBM_BoosterGetNumFeature(
            self.handle,
            ctypes.byref(out_num_feature)))
        return out_num_feature.value

3964
    def feature_name(self) -> List[str]:
3965
        """Get names of features.
wxchan's avatar
wxchan committed
3966
3967
3968

        Returns
        -------
3969
        result : list of str
3970
            List with names of features.
wxchan's avatar
wxchan committed
3971
        """
3972
        num_feature = self.num_feature()
3973
        # Get name of features
wxchan's avatar
wxchan committed
3974
        tmp_out_len = ctypes.c_int(0)
3975
3976
        reserved_string_buffer_size = 255
        required_string_buffer_size = ctypes.c_size_t(0)
3977
        string_buffers = [ctypes.create_string_buffer(reserved_string_buffer_size) for _ in range(num_feature)]
wxchan's avatar
wxchan committed
3978
3979
3980
        ptr_string_buffers = (ctypes.c_char_p * num_feature)(*map(ctypes.addressof, string_buffers))
        _safe_call(_LIB.LGBM_BoosterGetFeatureNames(
            self.handle,
3981
            ctypes.c_int(num_feature),
wxchan's avatar
wxchan committed
3982
            ctypes.byref(tmp_out_len),
3983
            ctypes.c_size_t(reserved_string_buffer_size),
3984
            ctypes.byref(required_string_buffer_size),
wxchan's avatar
wxchan committed
3985
3986
3987
            ptr_string_buffers))
        if num_feature != tmp_out_len.value:
            raise ValueError("Length of feature names doesn't equal with num_feature")
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
        actual_string_buffer_size = required_string_buffer_size.value
        # if buffer length is not long enough, reallocate buffers
        if reserved_string_buffer_size < actual_string_buffer_size:
            string_buffers = [ctypes.create_string_buffer(actual_string_buffer_size) for _ in range(num_feature)]
            ptr_string_buffers = (ctypes.c_char_p * num_feature)(*map(ctypes.addressof, string_buffers))
            _safe_call(_LIB.LGBM_BoosterGetFeatureNames(
                self.handle,
                ctypes.c_int(num_feature),
                ctypes.byref(tmp_out_len),
                ctypes.c_size_t(actual_string_buffer_size),
                ctypes.byref(required_string_buffer_size),
                ptr_string_buffers))
4000
        return [string_buffers[i].value.decode('utf-8') for i in range(num_feature)]
wxchan's avatar
wxchan committed
4001

4002
4003
4004
4005
4006
    def feature_importance(
        self,
        importance_type: str = 'split',
        iteration: Optional[int] = None
    ) -> np.ndarray:
4007
        """Get feature importances.
4008

4009
4010
        Parameters
        ----------
4011
        importance_type : str, optional (default="split")
4012
4013
4014
            How the importance is calculated.
            If "split", result contains numbers of times the feature is used in a model.
            If "gain", result contains total gains of splits which use the feature.
4015
4016
4017
4018
        iteration : int or None, optional (default=None)
            Limit number of iterations in the feature importance calculation.
            If None, if the best iteration exists, it is used; otherwise, all trees are used.
            If <= 0, all trees are used (no limits).
4019

4020
4021
        Returns
        -------
4022
4023
        result : numpy array
            Array with feature importances.
4024
        """
4025
4026
        if iteration is None:
            iteration = self.best_iteration
4027
        importance_type_int = _FEATURE_IMPORTANCE_TYPE_MAPPER[importance_type]
4028
        result = np.empty(self.num_feature(), dtype=np.float64)
4029
4030
4031
4032
4033
        _safe_call(_LIB.LGBM_BoosterFeatureImportance(
            self.handle,
            ctypes.c_int(iteration),
            ctypes.c_int(importance_type_int),
            result.ctypes.data_as(ctypes.POINTER(ctypes.c_double))))
4034
        if importance_type_int == _C_API_FEATURE_IMPORTANCE_SPLIT:
4035
            return result.astype(np.int32)
4036
4037
        else:
            return result
4038

4039
4040
4041
4042
4043
4044
    def get_split_value_histogram(
        self,
        feature: Union[int, str],
        bins: Optional[Union[int, str]] = None,
        xgboost_style: bool = False
    ) -> Union[Tuple[np.ndarray, np.ndarray], np.ndarray, pd_DataFrame]:
4045
4046
4047
4048
        """Get split value histogram for the specified feature.

        Parameters
        ----------
4049
        feature : int or str
4050
4051
            The feature name or index the histogram is calculated for.
            If int, interpreted as index.
4052
            If str, interpreted as name.
4053

Nikita Titov's avatar
Nikita Titov committed
4054
4055
4056
            .. warning::

                Categorical features are not supported.
4057

4058
        bins : int, str or None, optional (default=None)
4059
4060
4061
            The maximum number of bins.
            If None, or int and > number of unique split values and ``xgboost_style=True``,
            the number of bins equals number of unique split values.
4062
            If str, it should be one from the list of the supported values by ``numpy.histogram()`` function.
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
        xgboost_style : bool, optional (default=False)
            Whether the returned result should be in the same form as it is in XGBoost.
            If False, the returned value is tuple of 2 numpy arrays as it is in ``numpy.histogram()`` function.
            If True, the returned value is matrix, in which the first column is the right edges of non-empty bins
            and the second one is the histogram values.

        Returns
        -------
        result_tuple : tuple of 2 numpy arrays
            If ``xgboost_style=False``, the values of the histogram of used splitting values for the specified feature
            and the bin edges.
        result_array_like : numpy array or pandas DataFrame (if pandas is installed)
            If ``xgboost_style=True``, the histogram of used splitting values for the specified feature.
        """
        def add(root):
            """Recursively add thresholds."""
            if 'split_index' in root:  # non-leaf
4080
                if feature_names is not None and isinstance(feature, str):
4081
4082
4083
4084
                    split_feature = feature_names[root['split_feature']]
                else:
                    split_feature = root['split_feature']
                if split_feature == feature:
4085
                    if isinstance(root['threshold'], str):
4086
4087
4088
                        raise LightGBMError('Cannot compute split value histogram for the categorical feature')
                    else:
                        values.append(root['threshold'])
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
                add(root['left_child'])
                add(root['right_child'])

        model = self.dump_model()
        feature_names = model.get('feature_names')
        tree_infos = model['tree_info']
        values = []
        for tree_info in tree_infos:
            add(tree_info['tree_structure'])

4099
        if bins is None or isinstance(bins, int) and xgboost_style:
4100
4101
4102
4103
4104
4105
4106
            n_unique = len(np.unique(values))
            bins = max(min(n_unique, bins) if bins is not None else n_unique, 1)
        hist, bin_edges = np.histogram(values, bins=bins)
        if xgboost_style:
            ret = np.column_stack((bin_edges[1:], hist))
            ret = ret[ret[:, 1] > 0]
            if PANDAS_INSTALLED:
4107
                return pd_DataFrame(ret, columns=['SplitValue', 'Count'])
4108
4109
4110
4111
4112
            else:
                return ret
        else:
            return hist, bin_edges

4113
4114
4115
4116
4117
4118
    def __inner_eval(
        self,
        data_name: str,
        data_idx: int,
        feval: Optional[Union[_LGBM_CustomEvalFunction, List[_LGBM_CustomEvalFunction]]] = None
    ) -> List[_LGBM_BoosterEvalMethodResultType]:
4119
        """Evaluate training or validation data."""
wxchan's avatar
wxchan committed
4120
        if data_idx >= self.__num_dataset:
4121
            raise ValueError("Data_idx should be smaller than number of dataset")
wxchan's avatar
wxchan committed
4122
4123
4124
        self.__get_eval_info()
        ret = []
        if self.__num_inner_eval > 0:
4125
            result = np.empty(self.__num_inner_eval, dtype=np.float64)
Guolin Ke's avatar
Guolin Ke committed
4126
            tmp_out_len = ctypes.c_int(0)
wxchan's avatar
wxchan committed
4127
4128
            _safe_call(_LIB.LGBM_BoosterGetEval(
                self.handle,
Guolin Ke's avatar
Guolin Ke committed
4129
                ctypes.c_int(data_idx),
wxchan's avatar
wxchan committed
4130
                ctypes.byref(tmp_out_len),
Guolin Ke's avatar
Guolin Ke committed
4131
                result.ctypes.data_as(ctypes.POINTER(ctypes.c_double))))
wxchan's avatar
wxchan committed
4132
            if tmp_out_len.value != self.__num_inner_eval:
4133
                raise ValueError("Wrong length of eval results")
4134
            for i in range(self.__num_inner_eval):
4135
4136
                ret.append((data_name, self.__name_inner_eval[i],
                            result[i], self.__higher_better_inner_eval[i]))
4137
4138
        if callable(feval):
            feval = [feval]
wxchan's avatar
wxchan committed
4139
4140
4141
4142
4143
        if feval is not None:
            if data_idx == 0:
                cur_data = self.train_set
            else:
                cur_data = self.valid_sets[data_idx - 1]
4144
4145
4146
4147
4148
4149
4150
4151
4152
            for eval_function in feval:
                if eval_function is None:
                    continue
                feval_ret = eval_function(self.__inner_predict(data_idx), cur_data)
                if isinstance(feval_ret, list):
                    for eval_name, val, is_higher_better in feval_ret:
                        ret.append((data_name, eval_name, val, is_higher_better))
                else:
                    eval_name, val, is_higher_better = feval_ret
wxchan's avatar
wxchan committed
4153
4154
4155
                    ret.append((data_name, eval_name, val, is_higher_better))
        return ret

4156
    def __inner_predict(self, data_idx: int):
4157
        """Predict for training and validation dataset."""
wxchan's avatar
wxchan committed
4158
        if data_idx >= self.__num_dataset:
4159
            raise ValueError("Data_idx should be smaller than number of dataset")
wxchan's avatar
wxchan committed
4160
4161
4162
4163
4164
        if self.__inner_predict_buffer[data_idx] is None:
            if data_idx == 0:
                n_preds = self.train_set.num_data() * self.__num_class
            else:
                n_preds = self.valid_sets[data_idx - 1].num_data() * self.__num_class
4165
            self.__inner_predict_buffer[data_idx] = np.empty(n_preds, dtype=np.float64)
4166
        # avoid to predict many time in one iteration
wxchan's avatar
wxchan committed
4167
4168
        if not self.__is_predicted_cur_iter[data_idx]:
            tmp_out_len = ctypes.c_int64(0)
Guolin Ke's avatar
Guolin Ke committed
4169
            data_ptr = self.__inner_predict_buffer[data_idx].ctypes.data_as(ctypes.POINTER(ctypes.c_double))
wxchan's avatar
wxchan committed
4170
4171
            _safe_call(_LIB.LGBM_BoosterGetPredict(
                self.handle,
Guolin Ke's avatar
Guolin Ke committed
4172
                ctypes.c_int(data_idx),
wxchan's avatar
wxchan committed
4173
4174
4175
                ctypes.byref(tmp_out_len),
                data_ptr))
            if tmp_out_len.value != len(self.__inner_predict_buffer[data_idx]):
4176
                raise ValueError(f"Wrong length of predict results for data {data_idx}")
wxchan's avatar
wxchan committed
4177
            self.__is_predicted_cur_iter[data_idx] = True
4178
4179
4180
4181
4182
        result = self.__inner_predict_buffer[data_idx]
        if self.__num_class > 1:
            num_data = result.size // self.__num_class
            result = result.reshape(num_data, self.__num_class, order='F')
        return result
wxchan's avatar
wxchan committed
4183

4184
    def __get_eval_info(self) -> None:
4185
        """Get inner evaluation count and names."""
wxchan's avatar
wxchan committed
4186
4187
        if self.__need_reload_eval_info:
            self.__need_reload_eval_info = False
Guolin Ke's avatar
Guolin Ke committed
4188
            out_num_eval = ctypes.c_int(0)
4189
            # Get num of inner evals
wxchan's avatar
wxchan committed
4190
4191
4192
4193
4194
            _safe_call(_LIB.LGBM_BoosterGetEvalCounts(
                self.handle,
                ctypes.byref(out_num_eval)))
            self.__num_inner_eval = out_num_eval.value
            if self.__num_inner_eval > 0:
4195
                # Get name of eval metrics
Guolin Ke's avatar
Guolin Ke committed
4196
                tmp_out_len = ctypes.c_int(0)
4197
4198
4199
                reserved_string_buffer_size = 255
                required_string_buffer_size = ctypes.c_size_t(0)
                string_buffers = [
4200
                    ctypes.create_string_buffer(reserved_string_buffer_size) for _ in range(self.__num_inner_eval)
4201
                ]
wxchan's avatar
wxchan committed
4202
                ptr_string_buffers = (ctypes.c_char_p * self.__num_inner_eval)(*map(ctypes.addressof, string_buffers))
wxchan's avatar
wxchan committed
4203
4204
                _safe_call(_LIB.LGBM_BoosterGetEvalNames(
                    self.handle,
4205
                    ctypes.c_int(self.__num_inner_eval),
wxchan's avatar
wxchan committed
4206
                    ctypes.byref(tmp_out_len),
4207
                    ctypes.c_size_t(reserved_string_buffer_size),
4208
                    ctypes.byref(required_string_buffer_size),
wxchan's avatar
wxchan committed
4209
4210
                    ptr_string_buffers))
                if self.__num_inner_eval != tmp_out_len.value:
4211
                    raise ValueError("Length of eval names doesn't equal with num_evals")
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
                actual_string_buffer_size = required_string_buffer_size.value
                # if buffer length is not long enough, reallocate buffers
                if reserved_string_buffer_size < actual_string_buffer_size:
                    string_buffers = [
                        ctypes.create_string_buffer(actual_string_buffer_size) for _ in range(self.__num_inner_eval)
                    ]
                    ptr_string_buffers = (ctypes.c_char_p * self.__num_inner_eval)(*map(ctypes.addressof, string_buffers))
                    _safe_call(_LIB.LGBM_BoosterGetEvalNames(
                        self.handle,
                        ctypes.c_int(self.__num_inner_eval),
                        ctypes.byref(tmp_out_len),
                        ctypes.c_size_t(actual_string_buffer_size),
                        ctypes.byref(required_string_buffer_size),
                        ptr_string_buffers))
                self.__name_inner_eval = [
                    string_buffers[i].value.decode('utf-8') for i in range(self.__num_inner_eval)
                ]
                self.__higher_better_inner_eval = [
                    name.startswith(('auc', 'ndcg@', 'map@', 'average_precision')) for name in self.__name_inner_eval
                ]