test_.py 9.54 KB
Newer Older
wxchan's avatar
wxchan committed
1
2
# coding: utf-8
# pylint: skip-file
Guolin Ke's avatar
Guolin Ke committed
3
import ctypes
wxchan's avatar
wxchan committed
4
import os
Guolin Ke's avatar
Guolin Ke committed
5
import sys
Guolin Ke's avatar
Guolin Ke committed
6

7
8
from platform import system

Guolin Ke's avatar
Guolin Ke committed
9
10
11
import numpy as np
from scipy import sparse

wxchan's avatar
wxchan committed
12

Guolin Ke's avatar
Guolin Ke committed
13
14
15
16
17
18
def find_lib_path():
    if os.environ.get('LIGHTGBM_BUILD_DOC', False):
        # we don't need lib_lightgbm while building docs
        return []

    curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
19
20
    dll_path = [curr_path,
                os.path.join(curr_path, '../../'),
21
22
23
                os.path.join(curr_path, '../../python-package/lightgbm/compile'),
                os.path.join(curr_path, '../../python-package/compile'),
                os.path.join(curr_path, '../../lib/')]
24
    if system() in ('Windows', 'Microsoft'):
25
26
        dll_path.append(os.path.join(curr_path, '../../python-package/compile/Release/'))
        dll_path.append(os.path.join(curr_path, '../../python-package/compile/windows/x64/DLL/'))
Guolin Ke's avatar
Guolin Ke committed
27
28
29
        dll_path.append(os.path.join(curr_path, '../../Release/'))
        dll_path.append(os.path.join(curr_path, '../../windows/x64/DLL/'))
        dll_path = [os.path.join(p, 'lib_lightgbm.dll') for p in dll_path]
Guolin Ke's avatar
Guolin Ke committed
30
    else:
Guolin Ke's avatar
Guolin Ke committed
31
32
33
34
        dll_path = [os.path.join(p, 'lib_lightgbm.so') for p in dll_path]
    lib_path = [p for p in dll_path if os.path.exists(p) and os.path.isfile(p)]
    if not lib_path:
        dll_path = [os.path.realpath(p) for p in dll_path]
35
        raise Exception('Cannot find lightgbm library file in following paths:\n' + '\n'.join(dll_path))
Guolin Ke's avatar
Guolin Ke committed
36
37
38
39
40
41
42
43
    return lib_path


def LoadDll():
    lib_path = find_lib_path()
    if len(lib_path) == 0:
        return None
    lib = ctypes.cdll.LoadLibrary(lib_path[0])
Guolin Ke's avatar
Guolin Ke committed
44
45
    return lib

wxchan's avatar
wxchan committed
46

Guolin Ke's avatar
Guolin Ke committed
47
48
LIB = LoadDll()

Guolin Ke's avatar
Guolin Ke committed
49
50
LIB.LGBM_GetLastError.restype = ctypes.c_char_p

51
52
53
54
55
56
dtype_float32 = 0
dtype_float64 = 1
dtype_int32 = 2
dtype_int64 = 3


Guolin Ke's avatar
Guolin Ke committed
57
58
59
def c_array(ctype, values):
    return (ctype * len(values))(*values)

wxchan's avatar
wxchan committed
60

Guolin Ke's avatar
Guolin Ke committed
61
def c_str(string):
Guolin Ke's avatar
Guolin Ke committed
62
    return ctypes.c_char_p(string.encode('ascii'))
Guolin Ke's avatar
Guolin Ke committed
63

wxchan's avatar
wxchan committed
64

65
def load_from_file(filename, reference):
66
    ref = None
wxchan's avatar
wxchan committed
67
    if reference is not None:
Guolin Ke's avatar
Guolin Ke committed
68
        ref = reference
69
    handle = ctypes.c_void_p()
wxchan's avatar
wxchan committed
70
71
72
73
    LIB.LGBM_DatasetCreateFromFile(
        c_str(filename),
        c_str('max_bin=15'),
        ref, ctypes.byref(handle))
Guolin Ke's avatar
Guolin Ke committed
74
    print(LIB.LGBM_GetLastError())
75
    num_data = ctypes.c_long()
wxchan's avatar
wxchan committed
76
    LIB.LGBM_DatasetGetNumData(handle, ctypes.byref(num_data))
77
    num_feature = ctypes.c_long()
wxchan's avatar
wxchan committed
78
    LIB.LGBM_DatasetGetNumFeature(handle, ctypes.byref(num_feature))
79
    print('#data: %d #feature: %d' % (num_data.value, num_feature.value))
80
81
    return handle

wxchan's avatar
wxchan committed
82

83
def save_to_binary(handle, filename):
84
85
86
    LIB.LGBM_DatasetSaveBinary(handle, c_str(filename))


87
def load_from_csr(filename, reference):
Guolin Ke's avatar
Guolin Ke committed
88
89
    data = []
    label = []
90
91
92
93
    with open(filename, 'r') as inp:
        for line in inp.readlines():
            data.append([float(x) for x in line.split('\t')[1:]])
            label.append(float(line.split('\t')[0]))
Guolin Ke's avatar
Guolin Ke committed
94
95
96
97
98
    mat = np.array(data)
    label = np.array(label, dtype=np.float32)
    csr = sparse.csr_matrix(mat)
    handle = ctypes.c_void_p()
    ref = None
wxchan's avatar
wxchan committed
99
    if reference is not None:
Guolin Ke's avatar
Guolin Ke committed
100
        ref = reference
Guolin Ke's avatar
Guolin Ke committed
101

wxchan's avatar
wxchan committed
102
103
104
105
    LIB.LGBM_DatasetCreateFromCSR(
        c_array(ctypes.c_int, csr.indptr),
        dtype_int32,
        c_array(ctypes.c_int, csr.indices),
Guolin Ke's avatar
Guolin Ke committed
106
        csr.data.ctypes.data_as(ctypes.POINTER(ctypes.c_void_p)),
wxchan's avatar
wxchan committed
107
108
        dtype_float64,
        len(csr.indptr),
109
        len(csr.data),
wxchan's avatar
wxchan committed
110
111
112
113
        csr.shape[1],
        c_str('max_bin=15'),
        ref,
        ctypes.byref(handle))
114
    num_data = ctypes.c_long()
wxchan's avatar
wxchan committed
115
    LIB.LGBM_DatasetGetNumData(handle, ctypes.byref(num_data))
116
    num_feature = ctypes.c_long()
wxchan's avatar
wxchan committed
117
    LIB.LGBM_DatasetGetNumFeature(handle, ctypes.byref(num_feature))
118
    LIB.LGBM_DatasetSetField(handle, c_str('label'), c_array(ctypes.c_float, label), len(label), 0)
119
    print('#data: %d #feature: %d' % (num_data.value, num_feature.value))
120
121
    return handle

wxchan's avatar
wxchan committed
122

123
def load_from_csc(filename, reference):
124
125
    data = []
    label = []
126
127
128
129
    with open(filename, 'r') as inp:
        for line in inp.readlines():
            data.append([float(x) for x in line.split('\t')[1:]])
            label.append(float(line.split('\t')[0]))
130
131
132
133
134
    mat = np.array(data)
    label = np.array(label, dtype=np.float32)
    csr = sparse.csc_matrix(mat)
    handle = ctypes.c_void_p()
    ref = None
wxchan's avatar
wxchan committed
135
    if reference is not None:
Guolin Ke's avatar
Guolin Ke committed
136
        ref = reference
Guolin Ke's avatar
Guolin Ke committed
137

wxchan's avatar
wxchan committed
138
139
140
141
    LIB.LGBM_DatasetCreateFromCSC(
        c_array(ctypes.c_int, csr.indptr),
        dtype_int32,
        c_array(ctypes.c_int, csr.indices),
142
        csr.data.ctypes.data_as(ctypes.POINTER(ctypes.c_void_p)),
wxchan's avatar
wxchan committed
143
144
        dtype_float64,
        len(csr.indptr),
145
        len(csr.data),
wxchan's avatar
wxchan committed
146
147
148
149
        csr.shape[0],
        c_str('max_bin=15'),
        ref,
        ctypes.byref(handle))
150
    num_data = ctypes.c_long()
wxchan's avatar
wxchan committed
151
    LIB.LGBM_DatasetGetNumData(handle, ctypes.byref(num_data))
152
    num_feature = ctypes.c_long()
wxchan's avatar
wxchan committed
153
    LIB.LGBM_DatasetGetNumFeature(handle, ctypes.byref(num_feature))
154
    LIB.LGBM_DatasetSetField(handle, c_str('label'), c_array(ctypes.c_float, label), len(label), 0)
155
    print('#data: %d #feature: %d' % (num_data.value, num_feature.value))
156
157
    return handle

wxchan's avatar
wxchan committed
158

159
def load_from_mat(filename, reference):
160
161
    data = []
    label = []
162
163
164
165
    with open(filename, 'r') as inp:
        for line in inp.readlines():
            data.append([float(x) for x in line.split('\t')[1:]])
            label.append(float(line.split('\t')[0]))
166
167
168
169
170
    mat = np.array(data)
    data = np.array(mat.reshape(mat.size), copy=False)
    label = np.array(label, dtype=np.float32)
    handle = ctypes.c_void_p()
    ref = None
wxchan's avatar
wxchan committed
171
    if reference is not None:
Guolin Ke's avatar
Guolin Ke committed
172
        ref = reference
Guolin Ke's avatar
Guolin Ke committed
173

174
175
    LIB.LGBM_DatasetCreateFromMat(
        data.ctypes.data_as(ctypes.POINTER(ctypes.c_void_p)),
176
177
178
179
        dtype_float64,
        mat.shape[0],
        mat.shape[1],
        1,
wxchan's avatar
wxchan committed
180
181
182
        c_str('max_bin=15'),
        ref,
        ctypes.byref(handle))
183
    num_data = ctypes.c_long()
wxchan's avatar
wxchan committed
184
    LIB.LGBM_DatasetGetNumData(handle, ctypes.byref(num_data))
185
    num_feature = ctypes.c_long()
wxchan's avatar
wxchan committed
186
    LIB.LGBM_DatasetGetNumFeature(handle, ctypes.byref(num_feature))
Guolin Ke's avatar
Guolin Ke committed
187
    LIB.LGBM_DatasetSetField(handle, c_str('label'), c_array(ctypes.c_float, label), len(label), 0)
188
    print('#data: %d #feature: %d' % (num_data.value, num_feature.value))
Guolin Ke's avatar
Guolin Ke committed
189
    return handle
wxchan's avatar
wxchan committed
190
191


192
def free_dataset(handle):
193
194
    LIB.LGBM_DatasetFree(handle)

wxchan's avatar
wxchan committed
195

196
def test_dataset():
197
198
199
200
    train = load_from_file(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                        '../../examples/binary_classification/binary.train'), None)
    test = load_from_mat(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                      '../../examples/binary_classification/binary.test'), train)
201
    free_dataset(test)
202
203
    test = load_from_csr(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                      '../../examples/binary_classification/binary.test'), train)
204
    free_dataset(test)
205
206
    test = load_from_csc(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                      '../../examples/binary_classification/binary.test'), train)
207
208
209
210
211
    free_dataset(test)
    save_to_binary(train, 'train.binary.bin')
    free_dataset(train)
    train = load_from_file('train.binary.bin', None)
    free_dataset(train)
wxchan's avatar
wxchan committed
212
213


214
def test_booster():
215
216
217
218
    train = load_from_mat(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                       '../../examples/binary_classification/binary.train'), None)
    test = load_from_mat(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                      '../../examples/binary_classification/binary.test'), train)
219
    booster = ctypes.c_void_p()
220
221
222
223
    LIB.LGBM_BoosterCreate(
        train,
        c_str("app=binary metric=auc num_leaves=31 verbose=0"),
        ctypes.byref(booster))
224
    LIB.LGBM_BoosterAddValidData(booster, test)
225
    is_finished = ctypes.c_int(0)
wxchan's avatar
wxchan committed
226
    for i in range(1, 101):
wxchan's avatar
wxchan committed
227
        LIB.LGBM_BoosterUpdateOneIter(booster, ctypes.byref(is_finished))
Guolin Ke's avatar
Guolin Ke committed
228
        result = np.array([0.0], dtype=np.float64)
229
        out_len = ctypes.c_ulong(0)
230
231
232
233
234
        LIB.LGBM_BoosterGetEval(
            booster,
            0,
            ctypes.byref(out_len),
            result.ctypes.data_as(ctypes.POINTER(ctypes.c_double)))
wxchan's avatar
wxchan committed
235
        if i % 10 == 0:
236
            print('%d iteration test AUC %f' % (i, result[0]))
237
    LIB.LGBM_BoosterSaveModel(booster, 0, -1, c_str('model.txt'))
238
    LIB.LGBM_BoosterFree(booster)
239
240
    free_dataset(train)
    free_dataset(test)
241
    booster2 = ctypes.c_void_p()
Guolin Ke's avatar
Guolin Ke committed
242
    num_total_model = ctypes.c_long()
243
244
245
246
    LIB.LGBM_BoosterCreateFromModelfile(
        c_str('model.txt'),
        ctypes.byref(num_total_model),
        ctypes.byref(booster2))
247
    data = []
248
249
250
251
    with open(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                           '../../examples/binary_classification/binary.test'), 'r') as inp:
        for line in inp.readlines():
            data.append([float(x) for x in line.split('\t')[1:]])
252
    mat = np.array(data)
Guolin Ke's avatar
Guolin Ke committed
253
    preb = np.zeros(mat.shape[0], dtype=np.float64)
Guolin Ke's avatar
Guolin Ke committed
254
    num_preb = ctypes.c_long()
255
    data = np.array(mat.reshape(mat.size), copy=False)
wxchan's avatar
wxchan committed
256
257
258
    LIB.LGBM_BoosterPredictForMat(
        booster2,
        data.ctypes.data_as(ctypes.POINTER(ctypes.c_void_p)),
259
260
261
262
263
264
        dtype_float64,
        mat.shape[0],
        mat.shape[1],
        1,
        1,
        50,
265
        c_str(''),
Guolin Ke's avatar
Guolin Ke committed
266
        ctypes.byref(num_preb),
267
        preb.ctypes.data_as(ctypes.POINTER(ctypes.c_double)))
268
269
    LIB.LGBM_BoosterPredictForFile(
        booster2,
270
271
        c_str(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                           '../../examples/binary_classification/binary.test')),
272
273
274
275
276
        0,
        0,
        50,
        c_str(''),
        c_str('preb.txt'))
277
    LIB.LGBM_BoosterFree(booster2)