test.py 7.44 KB
Newer Older
wxchan's avatar
wxchan committed
1
2
# coding: utf-8
# pylint: skip-file
Guolin Ke's avatar
Guolin Ke committed
3
import ctypes
wxchan's avatar
wxchan committed
4
import os
Guolin Ke's avatar
Guolin Ke committed
5
6
7
8

import numpy as np
from scipy import sparse

wxchan's avatar
wxchan committed
9

Guolin Ke's avatar
Guolin Ke committed
10
def LoadDll():
Guolin Ke's avatar
Guolin Ke committed
11
12
13
14
    if os.name == 'nt':
        lib_path = '../../windows/x64/DLL/lib_lightgbm.dll'
    else:
        lib_path = '../../lib_lightgbm.so'
Guolin Ke's avatar
Guolin Ke committed
15
16
17
    lib = ctypes.cdll.LoadLibrary(lib_path)
    return lib

wxchan's avatar
wxchan committed
18

Guolin Ke's avatar
Guolin Ke committed
19
20
LIB = LoadDll()

Guolin Ke's avatar
Guolin Ke committed
21
22
LIB.LGBM_GetLastError.restype = ctypes.c_char_p

23
24
25
26
27
28
dtype_float32 = 0
dtype_float64 = 1
dtype_int32 = 2
dtype_int64 = 3


Guolin Ke's avatar
Guolin Ke committed
29
30
31
def c_array(ctype, values):
    return (ctype * len(values))(*values)

wxchan's avatar
wxchan committed
32

Guolin Ke's avatar
Guolin Ke committed
33
def c_str(string):
Guolin Ke's avatar
Guolin Ke committed
34
    return ctypes.c_char_p(string.encode('ascii'))
Guolin Ke's avatar
Guolin Ke committed
35

wxchan's avatar
wxchan committed
36

37
38
def test_load_from_file(filename, reference):
    ref = None
wxchan's avatar
wxchan committed
39
    if reference is not None:
Guolin Ke's avatar
Guolin Ke committed
40
        ref = reference
41
    handle = ctypes.c_void_p()
wxchan's avatar
wxchan committed
42
43
44
45
    LIB.LGBM_DatasetCreateFromFile(
        c_str(filename),
        c_str('max_bin=15'),
        ref, ctypes.byref(handle))
Guolin Ke's avatar
Guolin Ke committed
46
    print(LIB.LGBM_GetLastError())
47
    num_data = ctypes.c_long()
wxchan's avatar
wxchan committed
48
    LIB.LGBM_DatasetGetNumData(handle, ctypes.byref(num_data))
49
    num_feature = ctypes.c_long()
wxchan's avatar
wxchan committed
50
51
    LIB.LGBM_DatasetGetNumFeature(handle, ctypes.byref(num_feature))
    print('#data:%d #feature:%d' % (num_data.value, num_feature.value))
52
53
    return handle

wxchan's avatar
wxchan committed
54

55
56
57
58
def test_save_to_binary(handle, filename):
    LIB.LGBM_DatasetSaveBinary(handle, c_str(filename))


Guolin Ke's avatar
Guolin Ke committed
59
60
61
62
63
def test_load_from_csr(filename, reference):
    data = []
    label = []
    inp = open(filename, 'r')
    for line in inp.readlines():
wxchan's avatar
wxchan committed
64
65
        data.append([float(x) for x in line.split('\t')[1:]])
        label.append(float(line.split('\t')[0]))
Guolin Ke's avatar
Guolin Ke committed
66
67
68
69
70
71
    inp.close()
    mat = np.array(data)
    label = np.array(label, dtype=np.float32)
    csr = sparse.csr_matrix(mat)
    handle = ctypes.c_void_p()
    ref = None
wxchan's avatar
wxchan committed
72
    if reference is not None:
Guolin Ke's avatar
Guolin Ke committed
73
        ref = reference
Guolin Ke's avatar
Guolin Ke committed
74

wxchan's avatar
wxchan committed
75
76
77
78
    LIB.LGBM_DatasetCreateFromCSR(
        c_array(ctypes.c_int, csr.indptr),
        dtype_int32,
        c_array(ctypes.c_int, csr.indices),
Guolin Ke's avatar
Guolin Ke committed
79
        csr.data.ctypes.data_as(ctypes.POINTER(ctypes.c_void_p)),
wxchan's avatar
wxchan committed
80
81
        dtype_float64,
        len(csr.indptr),
82
        len(csr.data),
wxchan's avatar
wxchan committed
83
84
85
86
        csr.shape[1],
        c_str('max_bin=15'),
        ref,
        ctypes.byref(handle))
87
    num_data = ctypes.c_long()
wxchan's avatar
wxchan committed
88
    LIB.LGBM_DatasetGetNumData(handle, ctypes.byref(num_data))
89
    num_feature = ctypes.c_long()
wxchan's avatar
wxchan committed
90
    LIB.LGBM_DatasetGetNumFeature(handle, ctypes.byref(num_feature))
91
    LIB.LGBM_DatasetSetField(handle, c_str('label'), c_array(ctypes.c_float, label), len(label), 0)
wxchan's avatar
wxchan committed
92
    print('#data:%d #feature:%d' % (num_data.value, num_feature.value))
93
94
    return handle

wxchan's avatar
wxchan committed
95

96
97
98
99
100
def test_load_from_csc(filename, reference):
    data = []
    label = []
    inp = open(filename, 'r')
    for line in inp.readlines():
wxchan's avatar
wxchan committed
101
102
        data.append([float(x) for x in line.split('\t')[1:]])
        label.append(float(line.split('\t')[0]))
103
104
105
106
107
108
    inp.close()
    mat = np.array(data)
    label = np.array(label, dtype=np.float32)
    csr = sparse.csc_matrix(mat)
    handle = ctypes.c_void_p()
    ref = None
wxchan's avatar
wxchan committed
109
    if reference is not None:
Guolin Ke's avatar
Guolin Ke committed
110
        ref = reference
Guolin Ke's avatar
Guolin Ke committed
111

wxchan's avatar
wxchan committed
112
113
114
115
    LIB.LGBM_DatasetCreateFromCSC(
        c_array(ctypes.c_int, csr.indptr),
        dtype_int32,
        c_array(ctypes.c_int, csr.indices),
116
        csr.data.ctypes.data_as(ctypes.POINTER(ctypes.c_void_p)),
wxchan's avatar
wxchan committed
117
118
        dtype_float64,
        len(csr.indptr),
119
        len(csr.data),
wxchan's avatar
wxchan committed
120
121
122
123
        csr.shape[0],
        c_str('max_bin=15'),
        ref,
        ctypes.byref(handle))
124
    num_data = ctypes.c_long()
wxchan's avatar
wxchan committed
125
    LIB.LGBM_DatasetGetNumData(handle, ctypes.byref(num_data))
126
    num_feature = ctypes.c_long()
wxchan's avatar
wxchan committed
127
    LIB.LGBM_DatasetGetNumFeature(handle, ctypes.byref(num_feature))
128
    LIB.LGBM_DatasetSetField(handle, c_str('label'), c_array(ctypes.c_float, label), len(label), 0)
wxchan's avatar
wxchan committed
129
    print('#data:%d #feature:%d' % (num_data.value, num_feature.value))
130
131
    return handle

wxchan's avatar
wxchan committed
132

133
134
135
136
137
def test_load_from_mat(filename, reference):
    data = []
    label = []
    inp = open(filename, 'r')
    for line in inp.readlines():
wxchan's avatar
wxchan committed
138
139
        data.append([float(x) for x in line.split('\t')[1:]])
        label.append(float(line.split('\t')[0]))
140
141
142
143
144
145
    inp.close()
    mat = np.array(data)
    data = np.array(mat.reshape(mat.size), copy=False)
    label = np.array(label, dtype=np.float32)
    handle = ctypes.c_void_p()
    ref = None
wxchan's avatar
wxchan committed
146
    if reference is not None:
Guolin Ke's avatar
Guolin Ke committed
147
        ref = reference
Guolin Ke's avatar
Guolin Ke committed
148

wxchan's avatar
wxchan committed
149
150
    LIB.LGBM_DatasetCreateFromMat(data.ctypes.data_as(
        ctypes.POINTER(ctypes.c_void_p)),
151
152
153
154
        dtype_float64,
        mat.shape[0],
        mat.shape[1],
        1,
wxchan's avatar
wxchan committed
155
156
157
        c_str('max_bin=15'),
        ref,
        ctypes.byref(handle))
158
    num_data = ctypes.c_long()
wxchan's avatar
wxchan committed
159
    LIB.LGBM_DatasetGetNumData(handle, ctypes.byref(num_data))
160
    num_feature = ctypes.c_long()
wxchan's avatar
wxchan committed
161
    LIB.LGBM_DatasetGetNumFeature(handle, ctypes.byref(num_feature))
Guolin Ke's avatar
Guolin Ke committed
162
    LIB.LGBM_DatasetSetField(handle, c_str('label'), c_array(ctypes.c_float, label), len(label), 0)
wxchan's avatar
wxchan committed
163
    print('#data:%d #feature:%d' % (num_data.value, num_feature.value))
Guolin Ke's avatar
Guolin Ke committed
164
    return handle
wxchan's avatar
wxchan committed
165
166


167
168
169
def test_free_dataset(handle):
    LIB.LGBM_DatasetFree(handle)

wxchan's avatar
wxchan committed
170

171
172
173
174
175
176
177
178
179
180
def test_dataset():
    train = test_load_from_file('../../examples/binary_classification/binary.train', None)
    test = test_load_from_mat('../../examples/binary_classification/binary.test', train)
    test_free_dataset(test)
    test = test_load_from_csr('../../examples/binary_classification/binary.test', train)
    test_free_dataset(test)
    test = test_load_from_csc('../../examples/binary_classification/binary.test', train)
    test_free_dataset(test)
    test_save_to_binary(train, 'train.binary.bin')
    test_free_dataset(train)
wxchan's avatar
wxchan committed
181
    train = test_load_from_file('train.binary.bin', None)
182
    test_free_dataset(train)
wxchan's avatar
wxchan committed
183
184


185
186
def test_booster():
    train = test_load_from_mat('../../examples/binary_classification/binary.train', None)
187
    test = test_load_from_mat('../../examples/binary_classification/binary.test', train)
188
    booster = ctypes.c_void_p()
189
190
    LIB.LGBM_BoosterCreate(train, c_str("app=binary metric=auc num_leaves=31 verbose=0"), ctypes.byref(booster))
    LIB.LGBM_BoosterAddValidData(booster, test)
191
    is_finished = ctypes.c_int(0)
wxchan's avatar
wxchan committed
192
    for i in range(1, 101):
wxchan's avatar
wxchan committed
193
        LIB.LGBM_BoosterUpdateOneIter(booster, ctypes.byref(is_finished))
Guolin Ke's avatar
Guolin Ke committed
194
        result = np.array([0.0], dtype=np.float64)
195
        out_len = ctypes.c_ulong(0)
Guolin Ke's avatar
Guolin Ke committed
196
        LIB.LGBM_BoosterGetEval(booster, 0, ctypes.byref(out_len), result.ctypes.data_as(ctypes.POINTER(ctypes.c_double)))
wxchan's avatar
wxchan committed
197
198
        if i % 10 == 0:
            print('%d Iteration test AUC %f' % (i, result[0]))
199
200
201
    LIB.LGBM_BoosterSaveModel(booster, -1, c_str('model.txt'))
    LIB.LGBM_BoosterFree(booster)
    test_free_dataset(train)
202
    test_free_dataset(test)
203
    booster2 = ctypes.c_void_p()
Guolin Ke's avatar
Guolin Ke committed
204
205
    num_total_model = ctypes.c_long()
    LIB.LGBM_BoosterCreateFromModelfile(c_str('model.txt'), ctypes.byref(num_total_model), ctypes.byref(booster2))
206
207
208
    data = []
    inp = open('../../examples/binary_classification/binary.test', 'r')
    for line in inp.readlines():
wxchan's avatar
wxchan committed
209
        data.append([float(x) for x in line.split('\t')[1:]])
210
211
    inp.close()
    mat = np.array(data)
Guolin Ke's avatar
Guolin Ke committed
212
    preb = np.zeros(mat.shape[0], dtype=np.float64)
Guolin Ke's avatar
Guolin Ke committed
213
    num_preb = ctypes.c_long()
214
    data = np.array(mat.reshape(mat.size), copy=False)
wxchan's avatar
wxchan committed
215
216
217
    LIB.LGBM_BoosterPredictForMat(
        booster2,
        data.ctypes.data_as(ctypes.POINTER(ctypes.c_void_p)),
218
219
220
221
222
223
        dtype_float64,
        mat.shape[0],
        mat.shape[1],
        1,
        1,
        50,
Guolin Ke's avatar
Guolin Ke committed
224
        ctypes.byref(num_preb),
225
        preb.ctypes.data_as(ctypes.POINTER(ctypes.c_double)))
wxchan's avatar
wxchan committed
226
    LIB.LGBM_BoosterPredictForFile(booster2, c_str('../../examples/binary_classification/binary.test'), 0, 0, 50, c_str('preb.txt'))
227
    LIB.LGBM_BoosterFree(booster2)
Guolin Ke's avatar
Guolin Ke committed
228

wxchan's avatar
wxchan committed
229

230
231
test_dataset()
test_booster()