Add draft for dataset

python-package/lightgbm/basic.py

+"""Wrapper c_api of LightGBM"""
+from __future__ import absolute_import
+
+import sys
+import os
+import ctypes
+import collections
+import re
+
+import numpy as np
+import scipy.sparse
+
+
+IS_PY3 = (sys.version_info[0] == 3)
+
+
+def find_lib_path():
+    """Find the path to LightGBM library files.
+    Returns
+    -------
+    lib_path: list(string)
+       List of all found library path to LightGBM
+    """
+    curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
+    dll_path = [curr_path, os.path.join(curr_path, '../../lib/'),
+                os.path.join(curr_path, './lib/'),
+                os.path.join(sys.prefix, 'lightgbm')]
+    if os.name == 'nt':
+        dll_path.append(os.path.join(curr_path, '../../windows/x64/Dll/'))
+        dll_path.append(os.path.join(curr_path, './windows/x64/Dll/'))
+        dll_path = [os.path.join(p, 'lib_lightgbm.dll') for p in dll_path]
+    else:
+        dll_path = [os.path.join(p, 'lib_lightgbm.so') for p in dll_path]
+    lib_path = [p for p in dll_path if os.path.exists(p) and os.path.isfile(p)]
+    if not lib_path:
+        raise Exception('Cannot find lightgbm Library')
+    return lib_path
+
+def _load_lib():
+    """Load LightGBM Library."""
+    lib_path = find_lib_path()
+    if len(lib_path) == 0:
+        return None
+    lib = ctypes.cdll.LoadLibrary(lib_path[0])
+    lib.LGBM_GetLastError.restype = ctypes.c_char_p
+    return lib
+
+_LIB = _load_lib()
+
+class LightGBMError(Exception):
+    """Error throwed by LightGBM"""
+    pass
+
+def _safe_call(ret):
+    """Check the return value of C API call
+    Parameters
+    ----------
+    ret : int
+        return value from API calls
+    """
+    if ret != 0:
+        raise LightGBMError(_LIB.LGBM_GetLastError())
+
+def is_str(s):
+    if IS_PY3:
+        return isinstance(s, str)
+    else:
+        return isinstance(s, basestring)
+
+def is_numpy_object(data):
+    return type(data).__module__ == np.__name__
+
+def is_numpy_1d_array(data):
+    if isinstance(data, np.ndarray) and len(data.shape) == 1:
+        return True
+    else:
+        return False
+
+def list_to_1d_numpy(data, dtype):
+    if is_numpy_1d_array(data):
+        return data
+    elif isinstance(data, list):
+        return np.array(data, dtype=dtype, copy=False)
+    else:
+        raise TypeError("Unknow type({})".format(type(data).__name__))
+
+def cfloat32_array_to_numpy(cptr, length):
+    """Convert a ctypes float pointer array to a numpy array.
+    """
+    if isinstance(cptr, ctypes.POINTER(ctypes.c_float)):
+        res = np.fromiter(cptr, dtype=np.float32, count=length)
+        return res
+    else:
+        raise RuntimeError('expected float pointer')
+
+def cint32_array_to_numpy(cptr, length):
+    """Convert a ctypes float pointer array to a numpy array.
+    """
+    if isinstance(cptr, ctypes.POINTER(ctypes.c_int32)):
+        res = np.fromiter(cptr, dtype=np.int32, count=length)
+        return res
+    else:
+        raise RuntimeError('expected int pointer')
+
+def c_str(string):
+    """Convert a python string to cstring."""
+    return ctypes.c_char_p(string.encode('utf-8'))
+
+def c_array(ctype, values):
+    """Convert a python array to c array."""
+    return (ctype * len(values))(*values)
+
+"""marco definition of data type in c_api of LightGBM"""
+C_API_DTYPE_FLOAT32 =0
+C_API_DTYPE_FLOAT64 =1
+C_API_DTYPE_INT32   =2
+C_API_DTYPE_INT64   =3
+"""Matric is row major in python"""
+C_API_IS_ROW_MAJOR  =1
+
+def c_float_array(data):
+    """Convert numpy array / list to c float array."""
+    if isinstance(data, list):
+        data = np.array(data, copy=False)
+    if is_numpy_1d_array(data):
+        if data.dtype == np.float32:
+            ptr_data = c_array(ctypes.c_float, data)
+            type_data = C_API_DTYPE_FLOAT32
+        elif data.dtype == np.float64:
+            ptr_data = c_array(ctypes.c_double, data)
+            type_data = C_API_DTYPE_FLOAT64
+        else:
+            raise TypeError("expected np.float32 or np.float64, met type({})".format(data.dtype))
+    else:
+        raise TypeError("Unknow type({})".format(type(data).__name__))
+    return (ptr_data, type_data)
+
+def c_int_array(data):
+    """Convert numpy array to c int array."""
+    if isinstance(data, list):
+        data = np.array(data, copy=False)
+    if is_numpy_1d_array(data):
+        if data.dtype == np.int32:
+            ptr_data = c_array(ctypes.c_int32, data)
+            type_data = C_API_DTYPE_INT32
+        elif data.dtype == np.int64:
+            ptr_data = c_array(ctypes.c_int64, data)
+            type_data = C_API_DTYPE_INT64
+        else:
+            raise TypeError("expected np.int32 or np.int64, met type({})".format(data.dtype))
+    else:
+        raise TypeError("Unknow type({})".format(type(data).__name__))
+    return (ptr_data, type_data)
+
+class Dataset(object):
+    """Dataset used in LightGBM.
+
+    Dataset is a internal data structure that used by LightGBM
+    You can construct Dataset from numpy.arrays
+    """
+
+    _feature_names = None
+
+    def __init__(self, data, max_bin=255, reference=None,
+        label=None, weight=None, group_id=None, 
+        silent=False, feature_names=None, 
+        other_args=None):
+        """
+        Dataset used in LightGBM.
+
+        Parameters
+        ----------
+        data : string/numpy array/scipy.sparse
+            Data source of Dataset.
+            When data is string type, it represents the path of txt file,
+        max_bin : int, required
+            max number of discrete bin for features 
+        reference : Other Dataset, optional
+            If this dataset validation, need to use training data as reference
+        label : list or numpy 1-D array, optional
+            Label of the training data.
+        weight : list or numpy 1-D array , optional
+            Weight for each instance.
+        group_id : list or numpy 1-D array , optional
+            group/query id for each instance. Note: if having group/query id, data should group by this id
+        silent : boolean, optional
+            Whether print messages during construction
+        feature_names : list, optional
+            Set names for features.
+        other_args: list, optional
+            other parameters, format: ['key1=val1','key2=val2']
+        """
+
+        if data is None:
+            self.handle = None
+            return
+        """process for args"""
+        pass_args = ["max_bin={}".format(max_bin)]
+        if silent:
+            pass_args.append("verbose=0")
+        if other_args:
+            pass_args += other_args
+        pass_args_str = ' '.join(pass_args)
+        """process for reference dataset"""
+        ref_dataset = None
+        if isinstance(reference, Dataset):
+            ref_dataset = ctypes.byref(reference.handle)
+        elif reference is not None:
+            raise TypeError('Reference dataset should be None or dataset instance')
+        """start construct data"""
+        if is_str(data):
+            self.handle = ctypes.c_void_p()
+            _safe_call(_LIB.LGBM_CreateDatasetFromFile(
+                c_str(data), 
+                c_str(pass_args_str), 
+                ref_dataset,
+                ctypes.byref(self.handle)))
+        elif isinstance(data, scipy.sparse.csr_matrix):
+            self._init_from_csr(data, pass_args_str, ref_dataset)
+        elif isinstance(data, scipy.sparse.csc_matrix):
+            self._init_from_csc(data, pass_args_str, ref_dataset)
+        elif isinstance(data, np.ndarray):
+            self._init_from_npy2d(data, pass_args_str, ref_dataset)
+        else:
+            try:
+                csr = scipy.sparse.csr_matrix(data)
+                self._init_from_csr(csr)
+            except:
+                raise TypeError('can not initialize Dataset from {}'.format(type(data).__name__))
+        if label is not None:
+            self.set_label(label)
+        if weight is not None:
+            self.set_weight(weight)
+        if group_id is not None:
+            self.set_group_id(group_id)
+        self.feature_names = feature_names
+
+    def _init_from_csr(self, csr, pass_args_str, ref_dataset):
+        """
+        Initialize data from a CSR matrix.
+        """
+        if len(csr.indices) != len(csr.data):
+            raise ValueError('length mismatch: {} vs {}'.format(len(csr.indices), len(csr.data)))
+        self.handle = ctypes.c_void_p()
+
+        ptr_indptr, type_ptr_indptr = c_int_array(csr.indptr)
+        ptr_data, type_ptr_data = c_float_array(csr.data)
+
+        _safe_call(_LIB.LGBM_CreateDatasetFromCSR(
+            ptr_indptr, 
+            type_ptr_indptr, 
+            c_array(ctypes.c_int32, csr.indices), 
+            ptr_data,
+            type_ptr_data, 
+            len(csr.indptr), 
+            len(csr.data),
+            csr.shape[1], 
+            c_str(pass_args_str), 
+            ref_dataset, 
+            ctypes.byref(self.handle)))
+
+    def _init_from_csc(self, csr, pass_args_str, ref_dataset):
+        """
+        Initialize data from a CSC matrix.
+        """
+        if len(csc.indices) != len(csc.data):
+            raise ValueError('length mismatch: {} vs {}'.format(len(csc.indices), len(csc.data)))
+        self.handle = ctypes.c_void_p()
+
+        ptr_indptr, type_ptr_indptr = c_int_array(csc.indptr)
+        ptr_data, type_ptr_data = c_float_array(csc.data)
+
+        _safe_call(_LIB.LGBM_CreateDatasetFromCSC(
+            ptr_indptr, 
+            type_ptr_indptr, 
+            c_array(ctypes.c_int32, csc.indices), 
+            ptr_data,
+            type_ptr_data, 
+            len(csc.indptr), 
+            len(csc.data),
+            csc.shape[0], 
+            c_str(pass_args_str), 
+            ref_dataset, 
+            ctypes.byref(self.handle)))
+
+    def _init_from_npy2d(self, mat, pass_args_str, ref_dataset):
+        """
+        Initialize data from a 2-D numpy matrix.
+        """
+        if len(mat.shape) != 2:
+            raise ValueError('Input numpy.ndarray must be 2 dimensional')
+
+        self.handle = ctypes.c_void_p()
+        if mat.dtype == np.float32 or mat.dtype == np.float64:
+            data = np.array(mat.reshape(mat.size), dtype=mat.dtype, copy=False)
+        else:
+            """change non-float data to float data, need to copy"""
+            data = np.array(mat.reshape(mat.size), dtype=np.float32)
+
+        ptr_data, type_ptr_data = c_float_array(data)
+        _safe_call(LIB.LGBM_CreateDatasetFromMat(
+            ptr_data, 
+            type_ptr_data,
+            mat.shape[0],
+            mat.shape[1],
+            C_API_IS_ROW_MAJOR,
+            c_str(pass_args_str), 
+            ref_dataset, 
+            ctypes.byref(self.handle)))
+
+    def __del__(self):
+        _safe_call(_LIB.LGBM_DatasetFree(self.handle))
+
+    def get_field(self, field_name):
+        """Get property from the Dataset.
+
+        Parameters
+        ----------
+        field_name: str
+            The field name of the information
+
+        Returns
+        -------
+        info : array
+            a numpy array of information of the data
+        """
+        out_len = ctypes.c_int32()
+        out_type = ctypes.c_int32()
+        ret = ctypes.POINTER(ctypes.c_void_p)()
+        _safe_call(_LIB.LGBM_DatasetGetField(
+            self.handle,
+            c_str(field_name),
+            ctypes.byref(out_len),
+            ctypes.byref(ret),
+            ctypes.byref(out_type)))
+        if out_type.value == C_API_DTYPE_INT32:
+            return cint32_array_to_numpy(ctypes.cast(ret, ctypes.POINTER(c_int32), out_len.value))
+        elif out_type.value == C_API_DTYPE_FLOAT32:
+            return cfloat32_array_to_numpy(ctypes.cast(ret, ctypes.POINTER(c_float), out_len.value))
+        else:
+            raise TypeError("unknow type")
+
+    def set_field(self, field_name, data):
+        """Set property into the Dataset.
+
+        Parameters
+        ----------
+        field_name: str
+            The field name of the information
+
+        data: numpy array or list
+            The array ofdata to be set
+        """
+        if not is_numpy_1d_array(data):
+            raise TypeError("Unknow type({})".format(type(data).__name__))
+        if data.dtype == np.float32:
+            ptr_data = c_array(ctypes.c_float, data)
+            type_data = C_API_DTYPE_FLOAT32
+        elif data.dtype == np.int32:
+            ptr_data = c_array(ctypes.c_int32, data)
+            type_data = C_API_DTYPE_INT32
+        else:
+            raise TypeError("excepted np.float32 or np.int32, met type({})".format(data.dtype))
+        _safe_call(_LIB.LGBM_DatasetSetField(
+            self.handle,
+            c_str(field_name),
+            ptr_data,
+            len(data),
+            type_data))
+
+
+    def save_binary(self, filename):
+        """Save Dataset to binary file
+
+        Parameters
+        ----------
+        filename : string
+            Name of the output file.
+        """
+        _safe_call(_LIB.LGBM_DatasetSaveBinary(
+            self.handle,
+            c_str(filename)))
+
+    def set_label(self, label):
+        """Set label of Dataset
+
+        Parameters
+        ----------
+        label: array like
+            The label information to be set into Dataset
+        """
+        label = list_to_1d_numpy(label, np.float32)
+        if label.dtype != np.float32:
+            label = label.astype(np.float32, copy=False)
+        self.set_field('label', label)
+
+    def set_weight(self, weight):
+        """ Set weight of each instance.
+
+        Parameters
+        ----------
+        weight : array like
+            Weight for each data point
+        """
+        weight = list_to_1d_numpy(weight, np.float32)
+        if weight.dtype != np.float32:
+            weight = weight.astype(np.float32, copy=False)
+        self.set_field('weight', weight)
+
+    def set_init_score(self, score):
+        """ Set init score of booster to start from.
+        Parameters
+        ----------
+        score: array like
+
+        """
+        score = list_to_1d_numpy(score, np.float32)
+        if score.dtype != np.float32:
+            score = score.astype(np.float32, copy=False)
+        self.set_field('init_score', score)
+
+    def set_group(self, group):
+        """Set group size of Dataset (used for ranking).
+
+        Parameters
+        ----------
+        group : array like
+            Group size of each group
+        """
+        group = list_to_1d_numpy(group, np.int32)
+        if group.dtype != np.int32:
+            group = group.astype(np.int32, copy=False)
+        self.set_field('group', group)
+
+    def set_group_id(self, group_id):
+
+        """Set group_id of Dataset (used for ranking).
+
+        Parameters
+        ----------
+        group : array like
+            group_id of Dataset (used for ranking).
+        """
+        group_id = list_to_1d_numpy(group_id, np.int32)
+        if group_id.dtype != np.int32:
+            group_id = group_id.astype(np.int32, copy=False)
+        self.set_field('group_id', group_id)
+
+    def get_label(self):
+        """Get the label of the Dataset.
+
+        Returns
+        -------
+        label : array
+        """
+        return self.get_field('label')
+
+    def get_weight(self):
+        """Get the weight of the Dataset.
+
+        Returns
+        -------
+        weight : array
+        """
+        return self.get_field('weight')
+
+    def get_init_score(self):
+        """Get the initial score of the Dataset.
+
+        Returns
+        -------
+        init_score : array
+        """
+        return self.get_field('init_score')
+
+    def num_data(self):
+        """Get the number of rows in the Dataset.
+
+        Returns
+        -------
+        number of rows : int
+        """
+        ret = ctypes.c_int64()
+        _safe_call(_LIB.LGBM_DatasetGetNumData(self.handle,
+                                         ctypes.byref(ret)))
+        return ret.value
+
+    def num_feature(self):
+        """Get the number of columns (features) in the Dataset.
+
+        Returns
+        -------
+        number of columns : int
+        """
+        ret = ctypes.c_int64()
+        _safe_call(_LIB.LGBM_DatasetGetNumFeature(self.handle,
+                                         ctypes.byref(ret)))
+        return ret.value
+
+    @property
+    def feature_names(self):
+        """Get feature names (column labels).
+
+        Returns
+        -------
+        feature_names : list
+        """
+        if self._feature_names is None:
+            self._feature_names = ['Column_{0}'.format(i) for i in range(self.num_col())]
+        return self._feature_names
+
+    @feature_names.setter
+    def feature_names(self, feature_names):
+        """Set feature names (column labels).
+
+        Parameters
+        ----------
+        feature_names : list
+            Labels for features
+        """
+        if feature_names is not None:
+            # validate feature name
+            if not isinstance(feature_names, list):
+                feature_names = list(feature_names)
+            if len(feature_names) != len(set(feature_names)):
+                raise ValueError('feature_names must be unique')
+            if len(feature_names) != self.num_col():
+                msg = 'feature_names must have the same length as data'
+                raise ValueError(msg)
+            # prohibit to use symbols may affect to parse. e.g. []<
+            if not all(isinstance(f, STRING_TYPES) and
+                       not any(x in f for x in set(('[', ']', '<')))
+                       for f in feature_names):
+                raise ValueError('feature_names may not contain [, ] or <')
+            self._feature_names = feature_names
+        else:
+            self._feature_names = None
+