dataset.cpp 15.6 KB
Newer Older
Guolin Ke's avatar
Guolin Ke committed
1
#include <LightGBM/dataset.h>
Guolin Ke's avatar
Guolin Ke committed
2
#include <LightGBM/feature_group.h>
3
#include <LightGBM/utils/openmp_wrapper.h>
Guolin Ke's avatar
Guolin Ke committed
4
5
#include <LightGBM/utils/threading.h>
#include <LightGBM/utils/array_args.h>
Guolin Ke's avatar
Guolin Ke committed
6
7
8
9
10
11
12

#include <cstdio>
#include <unordered_map>
#include <limits>
#include <vector>
#include <utility>
#include <string>
Guolin Ke's avatar
Guolin Ke committed
13
#include <sstream>
Guolin Ke's avatar
Guolin Ke committed
14
15
16

namespace LightGBM {

17
const char* Dataset::binary_file_token = "______LightGBM_Binary_File_Token______\n";
Guolin Ke's avatar
Guolin Ke committed
18

Guolin Ke's avatar
Guolin Ke committed
19
Dataset::Dataset() {
20
  data_filename_ = "noname";
Guolin Ke's avatar
Guolin Ke committed
21
  num_data_ = 0;
Guolin Ke's avatar
Guolin Ke committed
22
23
}

24
Dataset::Dataset(data_size_t num_data) {
Guolin Ke's avatar
Guolin Ke committed
25
  data_filename_ = "noname";
Guolin Ke's avatar
Guolin Ke committed
26
  num_data_ = num_data;
Guolin Ke's avatar
Guolin Ke committed
27
  metadata_.Init(num_data_, NO_SPECIFIC, NO_SPECIFIC);
Guolin Ke's avatar
Guolin Ke committed
28
29
}

Guolin Ke's avatar
Guolin Ke committed
30
Dataset::~Dataset() {
Guolin Ke's avatar
Guolin Ke committed
31
}
Guolin Ke's avatar
Guolin Ke committed
32

Guolin Ke's avatar
Guolin Ke committed
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
std::vector<std::vector<int>> NoGroup(
  const std::vector<int>& used_features) {
  std::vector<std::vector<int>> features_in_group;
  features_in_group.resize(used_features.size());
  for (size_t i = 0; i < used_features.size(); ++i) {
    features_in_group[i].emplace_back(used_features[i]);
  }
  return features_in_group;
}

void Dataset::Construct(
  std::vector<std::unique_ptr<BinMapper>>& bin_mappers,
  const std::vector<std::vector<int>>& sample_indices,
  size_t total_sample_cnt,
  const IOConfig& io_config) {
  num_total_features_ = static_cast<int>(bin_mappers.size());
  // get num_features
  std::vector<int> used_features;
  for (int i = 0; i < static_cast<int>(bin_mappers.size()); ++i) {
    if (bin_mappers[i] != nullptr && !bin_mappers[i]->is_trival()) {
      used_features.emplace_back(i);
    } 
  }

  auto features_in_group = NoGroup(used_features);

  num_features_ = 0;
  for (const auto& fs : features_in_group) {
    num_features_ += static_cast<int>(fs.size());
  }
  int cur_fidx = 0;
  used_feature_map_ = std::vector<int>(num_total_features_, -1);
  num_groups_ = static_cast<int>(features_in_group.size());
  real_feature_idx_.resize(num_features_);
  feature2group_.resize(num_features_);
  feature2subfeature_.resize(num_features_);
  for (int i = 0; i < num_groups_; ++i) {
    auto cur_features = features_in_group[i];
    int cur_cnt_features = static_cast<int>(cur_features.size());
    // get bin_mappers
    std::vector<std::unique_ptr<BinMapper>> cur_bin_mappers;
    for (int j = 0; j < cur_cnt_features; ++j) {
      int real_fidx = cur_features[j];
      used_feature_map_[real_fidx] = cur_fidx;
      real_feature_idx_[cur_fidx] = real_fidx;
      feature2group_[cur_fidx] = i;
      feature2subfeature_[cur_fidx] = j;
      cur_bin_mappers.emplace_back(bin_mappers[real_fidx].release());
      ++cur_fidx;
    }
    feature_groups_.emplace_back(std::unique_ptr<FeatureGroup>(
      new FeatureGroup(cur_cnt_features, cur_bin_mappers, num_data_, io_config.is_enable_sparse)));
  }
  feature_groups_.shrink_to_fit();
  group_bin_boundaries_.clear();
  uint64_t num_total_bin = 0;
  group_bin_boundaries_.push_back(num_total_bin);
  for (int i = 0; i < num_groups_; ++i) {
    num_total_bin += feature_groups_[i]->num_total_bin_;
    group_bin_boundaries_.push_back(num_total_bin);
  }
  int last_group = 0;
  group_feature_start_.reserve(num_groups_);
  group_feature_cnt_.reserve(num_groups_);
  group_feature_start_.push_back(0);
  group_feature_cnt_.push_back(1);
  for (int i = 1; i < num_features_; ++i) {
    const int group = feature2group_[i];
    if (group == last_group) {
      group_feature_cnt_.back() = group_feature_cnt_.back() + 1;
    } else {
      group_feature_start_.push_back(i);
      group_feature_cnt_.push_back(1);
      last_group = group;
    }
  }
Guolin Ke's avatar
Guolin Ke committed
109
110
}

Guolin Ke's avatar
Guolin Ke committed
111
112
void Dataset::FinishLoad() {
#pragma omp parallel for schedule(guided)
Guolin Ke's avatar
Guolin Ke committed
113
114
  for (int i = 0; i < num_groups_; ++i) {
    feature_groups_[i]->bin_data_->FinishLoad();
Guolin Ke's avatar
Guolin Ke committed
115
116
  }
}
Guolin Ke's avatar
Guolin Ke committed
117

118
void Dataset::CopyFeatureMapperFrom(const Dataset* dataset) {
Guolin Ke's avatar
Guolin Ke committed
119
  feature_groups_.clear();
Guolin Ke's avatar
Guolin Ke committed
120
  num_features_ = dataset->num_features_;
Guolin Ke's avatar
Guolin Ke committed
121
  num_groups_ = dataset->num_groups_;
122
  bool is_enable_sparse = false;
Guolin Ke's avatar
Guolin Ke committed
123
124
  for (int i = 0; i < num_groups_; ++i) {
    if (dataset->feature_groups_[i]->is_sparse_) {
125
126
127
128
      is_enable_sparse = true;
      break;
    }
  }
Guolin Ke's avatar
Guolin Ke committed
129
  // copy feature bin mapper data
Guolin Ke's avatar
Guolin Ke committed
130
131
132
133
134
135
136
137
  for (int i = 0; i < num_groups_; ++i) {
    std::vector<std::unique_ptr<BinMapper>> bin_mappers;
    for (int j = 0; j < dataset->feature_groups_[i]->num_feature_; ++j) {
      bin_mappers.emplace_back(new BinMapper(*(dataset->feature_groups_[i]->bin_mappers_[j])));
    }
    feature_groups_.emplace_back(new FeatureGroup(
      dataset->feature_groups_[i]->num_feature_,
      bin_mappers,
Guolin Ke's avatar
Guolin Ke committed
138
139
      num_data_,
      is_enable_sparse));
Guolin Ke's avatar
Guolin Ke committed
140
  }
Guolin Ke's avatar
Guolin Ke committed
141
  feature_groups_.shrink_to_fit();
Guolin Ke's avatar
Guolin Ke committed
142
143
144
  used_feature_map_ = dataset->used_feature_map_;
  num_total_features_ = dataset->num_total_features_;
  feature_names_ = dataset->feature_names_;
Guolin Ke's avatar
Guolin Ke committed
145
  label_idx_ = dataset->label_idx_;
Guolin Ke's avatar
Guolin Ke committed
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
  real_feature_idx_ = dataset->real_feature_idx_;
  feature2group_ = dataset->feature2group_;
  feature2subfeature_ = dataset->feature2subfeature_;
  group_bin_boundaries_ = dataset->group_bin_boundaries_;
  group_feature_start_ = dataset->group_feature_start_;
  group_feature_cnt_ = dataset->group_feature_cnt_;
}

void Dataset::CreateValid(const Dataset* dataset) {
  feature_groups_.clear();
  num_features_ = dataset->num_features_;
  num_groups_ = num_features_;
  bool is_enable_sparse = true;
  feature2group_.clear();
  feature2subfeature_.clear();
  // copy feature bin mapper data
  for (int i = 0; i < num_features_; ++i) {
    std::vector<std::unique_ptr<BinMapper>> bin_mappers;
    bin_mappers.emplace_back(new BinMapper(*(dataset->FeatureBinMapper(i))));
    feature_groups_.emplace_back(new FeatureGroup(
      1,
      bin_mappers,
      num_data_,
      is_enable_sparse));
    feature2group_.push_back(i);
    feature2subfeature_.push_back(0);
  }

  feature_groups_.shrink_to_fit();
  used_feature_map_ = dataset->used_feature_map_;
  num_total_features_ = dataset->num_total_features_;
  feature_names_ = dataset->feature_names_;
  label_idx_ = dataset->label_idx_;
  real_feature_idx_ = dataset->real_feature_idx_;
  group_bin_boundaries_.clear();
  uint64_t num_total_bin = 0;
  group_bin_boundaries_.push_back(num_total_bin);
  for (int i = 0; i < num_groups_; ++i) {
    num_total_bin += feature_groups_[i]->num_total_bin_;
    group_bin_boundaries_.push_back(num_total_bin);
  }
  int last_group = 0;
  group_feature_start_.reserve(num_groups_);
  group_feature_cnt_.reserve(num_groups_);
  group_feature_start_.push_back(0);
  group_feature_cnt_.push_back(1);
  for (int i = 1; i < num_features_; ++i) {
    const int group = feature2group_[i];
    if (group == last_group) {
      group_feature_cnt_.back() = group_feature_cnt_.back() + 1;
    } else {
      group_feature_start_.push_back(i);
      group_feature_cnt_.push_back(1);
      last_group = group;
    }
  }
Guolin Ke's avatar
Guolin Ke committed
202
203
}

Guolin Ke's avatar
Guolin Ke committed
204
205
206
void Dataset::ReSize(data_size_t num_data) {
  if (num_data_ != num_data) {
    num_data_ = num_data;
Guolin Ke's avatar
Guolin Ke committed
207
#pragma omp parallel for schedule(static)
Guolin Ke's avatar
Guolin Ke committed
208
209
    for (int group = 0; group < num_groups_; ++group) {
      feature_groups_[group]->bin_data_->ReSize(num_data_);
Guolin Ke's avatar
Guolin Ke committed
210
211
212
213
214
215
    }
  }
}

void Dataset::CopySubset(const Dataset* fullset, const data_size_t* used_indices, data_size_t num_used_indices, bool need_meta_data) {
  CHECK(num_used_indices == num_data_);
Guolin Ke's avatar
Guolin Ke committed
216
#pragma omp parallel for schedule(static)
Guolin Ke's avatar
Guolin Ke committed
217
218
  for (int group = 0; group < num_groups_; ++group) {
    feature_groups_[group]->CopySubset(fullset->feature_groups_[group].get(), used_indices, num_used_indices);
Guolin Ke's avatar
Guolin Ke committed
219
  }
Guolin Ke's avatar
Guolin Ke committed
220
  if (need_meta_data) {
Guolin Ke's avatar
Guolin Ke committed
221
    metadata_.Init(fullset->metadata_, used_indices, num_used_indices);
Guolin Ke's avatar
Guolin Ke committed
222
  }
Guolin Ke's avatar
Guolin Ke committed
223
224
}

225
bool Dataset::SetFloatField(const char* field_name, const float* field_data, data_size_t num_element) {
Guolin Ke's avatar
Guolin Ke committed
226
227
228
  std::string name(field_name);
  name = Common::Trim(name);
  if (name == std::string("label") || name == std::string("target")) {
229
    metadata_.SetLabel(field_data, num_element);
Guolin Ke's avatar
Guolin Ke committed
230
  } else if (name == std::string("weight") || name == std::string("weights")) {
231
    metadata_.SetWeights(field_data, num_element);
Guolin Ke's avatar
Guolin Ke committed
232
233
234
235
236
237
238
239
240
241
  } else {
    return false;
  }
  return true;
}

bool Dataset::SetDoubleField(const char* field_name, const double* field_data, data_size_t num_element) {
  std::string name(field_name);
  name = Common::Trim(name);
  if (name == std::string("init_score")) {
242
    metadata_.SetInitScore(field_data, num_element);
Guolin Ke's avatar
Guolin Ke committed
243
  } else {
244
    return false;
Guolin Ke's avatar
Guolin Ke committed
245
  }
246
  return true;
Guolin Ke's avatar
Guolin Ke committed
247
248
}

249
250
251
252
bool Dataset::SetIntField(const char* field_name, const int* field_data, data_size_t num_element) {
  std::string name(field_name);
  name = Common::Trim(name);
  if (name == std::string("query") || name == std::string("group")) {
Guolin Ke's avatar
Guolin Ke committed
253
    metadata_.SetQuery(field_data, num_element);
254
255
256
257
258
259
  } else {
    return false;
  }
  return true;
}

Guolin Ke's avatar
Guolin Ke committed
260
bool Dataset::GetFloatField(const char* field_name, data_size_t* out_len, const float** out_ptr) {
261
262
263
264
265
266
267
268
  std::string name(field_name);
  name = Common::Trim(name);
  if (name == std::string("label") || name == std::string("target")) {
    *out_ptr = metadata_.label();
    *out_len = num_data_;
  } else if (name == std::string("weight") || name == std::string("weights")) {
    *out_ptr = metadata_.weights();
    *out_len = num_data_;
Guolin Ke's avatar
Guolin Ke committed
269
270
271
272
273
274
275
276
277
278
  } else {
    return false;
  }
  return true;
}

bool Dataset::GetDoubleField(const char* field_name, data_size_t* out_len, const double** out_ptr) {
  std::string name(field_name);
  name = Common::Trim(name);
  if (name == std::string("init_score")) {
279
    *out_ptr = metadata_.init_score();
Guolin Ke's avatar
Guolin Ke committed
280
    *out_len = static_cast<data_size_t>(metadata_.num_init_score());
281
282
283
  } else {
    return false;
  }
284
  return true;
285
286
}

Guolin Ke's avatar
Guolin Ke committed
287
bool Dataset::GetIntField(const char* field_name, data_size_t* out_len, const int** out_ptr) {
288
289
290
  std::string name(field_name);
  name = Common::Trim(name);
  if (name == std::string("query") || name == std::string("group")) {
291
    *out_ptr = metadata_.query_boundaries();
Guolin Ke's avatar
Guolin Ke committed
292
    *out_len = metadata_.num_queries() + 1;
Guolin Ke's avatar
Guolin Ke committed
293
294
295
  } else {
    return false;
  }
296
  return true;
297
298
}

Guolin Ke's avatar
Guolin Ke committed
299
void Dataset::SaveBinaryFile(const char* bin_filename) {
Guolin Ke's avatar
Guolin Ke committed
300
301
  if (bin_filename != nullptr
    && std::string(bin_filename) == std::string(data_filename_)) {
Guolin Ke's avatar
Guolin Ke committed
302
303
304
    Log::Warning("Bianry file %s already existed", bin_filename);
    return;
  }
Guolin Ke's avatar
Guolin Ke committed
305
  // if not pass a filename, just append ".bin" of original file
Guolin Ke's avatar
Guolin Ke committed
306
  std::string bin_filename_str(data_filename_);
Guolin Ke's avatar
Guolin Ke committed
307
308
309
310
  if (bin_filename == nullptr || bin_filename[0] == '\0') {
    bin_filename_str.append(".bin");
    bin_filename = bin_filename_str.c_str();
  }
Guolin Ke's avatar
Guolin Ke committed
311
312
313
314
315
316
317
318
319
320
321
322
323
  bool is_file_existed = false;
  FILE* file;
#ifdef _MSC_VER
  fopen_s(&file, bin_filename, "rb");
#else
  file = fopen(bin_filename, "rb");
#endif

  if (file != NULL) {
    is_file_existed = true;
    Log::Warning("File %s existed, cannot save binary to it", bin_filename);
    fclose(file);
  }
Guolin Ke's avatar
Guolin Ke committed
324

Guolin Ke's avatar
Guolin Ke committed
325
  if (!is_file_existed) {
Guolin Ke's avatar
Guolin Ke committed
326
#ifdef _MSC_VER
Guolin Ke's avatar
Guolin Ke committed
327
    fopen_s(&file, bin_filename, "wb");
Guolin Ke's avatar
Guolin Ke committed
328
#else
Guolin Ke's avatar
Guolin Ke committed
329
    file = fopen(bin_filename, "wb");
Guolin Ke's avatar
Guolin Ke committed
330
#endif
Guolin Ke's avatar
Guolin Ke committed
331
    if (file == NULL) {
Guolin Ke's avatar
Guolin Ke committed
332
      Log::Fatal("Cannot write binary data to %s ", bin_filename);
Guolin Ke's avatar
Guolin Ke committed
333
    }
334
    Log::Info("Saving data to binary file %s", bin_filename);
335
336
    size_t size_of_token = std::strlen(binary_file_token);
    fwrite(binary_file_token, sizeof(char), size_of_token, file);
Guolin Ke's avatar
Guolin Ke committed
337
    // get size of header
Guolin Ke's avatar
Guolin Ke committed
338
339
340
    size_t size_of_header = sizeof(num_data_) + sizeof(num_features_) + sizeof(num_total_features_)
      + sizeof(int) * num_total_features_ + sizeof(num_groups_)
      + 3 * sizeof(int) * num_features_ + sizeof(uint64_t) * (num_groups_ + 1) + 2 * sizeof(int) * num_groups_;
341
342
343
344
    // size of feature names
    for (int i = 0; i < num_total_features_; ++i) {
      size_of_header += feature_names_[i].size() + sizeof(int);
    }
Guolin Ke's avatar
Guolin Ke committed
345
346
347
348
    fwrite(&size_of_header, sizeof(size_of_header), 1, file);
    // write header
    fwrite(&num_data_, sizeof(num_data_), 1, file);
    fwrite(&num_features_, sizeof(num_features_), 1, file);
Guolin Ke's avatar
Guolin Ke committed
349
350
351
352
353
354
355
356
357
    fwrite(&num_total_features_, sizeof(num_total_features_), 1, file);
    fwrite(used_feature_map_.data(), sizeof(int), num_total_features_, file);
    fwrite(&num_groups_, sizeof(num_groups_), 1, file);
    fwrite(real_feature_idx_.data(), sizeof(int), num_features_, file);
    fwrite(feature2group_.data(), sizeof(int), num_features_, file);
    fwrite(feature2subfeature_.data(), sizeof(int), num_features_, file);
    fwrite(group_bin_boundaries_.data(), sizeof(uint64_t), num_groups_ + 1, file);
    fwrite(group_feature_start_.data(), sizeof(int), num_groups_, file);
    fwrite(group_feature_cnt_.data(), sizeof(int), num_groups_, file);
Guolin Ke's avatar
Guolin Ke committed
358

359
360
361
362
363
364
365
366
    // write feature names
    for (int i = 0; i < num_total_features_; ++i) {
      int str_len = static_cast<int>(feature_names_[i].size());
      fwrite(&str_len, sizeof(int), 1, file);
      const char* c_str = feature_names_[i].c_str();
      fwrite(c_str, sizeof(char), str_len, file);
    }

Guolin Ke's avatar
Guolin Ke committed
367
368
369
370
371
372
373
    // get size of meta data
    size_t size_of_metadata = metadata_.SizesInByte();
    fwrite(&size_of_metadata, sizeof(size_of_metadata), 1, file);
    // write meta data
    metadata_.SaveBinaryToFile(file);

    // write feature data
Guolin Ke's avatar
Guolin Ke committed
374
    for (int i = 0; i < num_groups_; ++i) {
Guolin Ke's avatar
Guolin Ke committed
375
      // get size of feature
Guolin Ke's avatar
Guolin Ke committed
376
      size_t size_of_feature = feature_groups_[i]->SizesInByte();
Guolin Ke's avatar
Guolin Ke committed
377
378
      fwrite(&size_of_feature, sizeof(size_of_feature), 1, file);
      // write feature
Guolin Ke's avatar
Guolin Ke committed
379
      feature_groups_[i]->SaveBinaryToFile(file);
Guolin Ke's avatar
Guolin Ke committed
380
381
382
383
384
    }
    fclose(file);
  }
}

Guolin Ke's avatar
Guolin Ke committed
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
void Dataset::ConstructHistograms(
  const std::vector<int8_t>& is_feature_used,
  const data_size_t* data_indices, data_size_t num_data,
  int leaf_idx,
  std::vector<std::unique_ptr<OrderedBin>>& ordered_bins,
  const score_t* gradients, const score_t* hessians,
  score_t* ordered_gradients, score_t* ordered_hessians,
  HistogramBinEntry* hist_data) const {

  if (leaf_idx < 0 || num_data <= 0 || hist_data == nullptr) {
    return;
  }
  auto ptr_ordered_grad = gradients;
  auto ptr_ordered_hess = hessians;
  if (data_indices != nullptr && num_data < num_data_) {
#pragma omp parallel for schedule(static)
    for (data_size_t i = 0; i < num_data; ++i) {
      ordered_gradients[i] = gradients[data_indices[i]];
      ordered_hessians[i] = hessians[data_indices[i]];
    }
    ptr_ordered_grad = ordered_gradients;
    ptr_ordered_hess = ordered_hessians;
  }
Guolin Ke's avatar
Guolin Ke committed
408
409

#pragma omp parallel for schedule(static)
Guolin Ke's avatar
Guolin Ke committed
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
  for (int group = 0; group < num_groups_; ++group) {
    bool is_groud_used = false;
    const int f_cnt = group_feature_cnt_[group];
    for (int j = 0; j < f_cnt; ++j) {
      const int fidx = group_feature_start_[group] + j;
      if (is_feature_used[fidx]) {
        is_groud_used = true;
        break;
      }
    }
    if (!is_groud_used) { continue; }
    // feature is not used
    auto data_ptr = hist_data + group_bin_boundaries_[group];
    const int num_bin = feature_groups_[group]->num_total_bin_;
    std::memset(data_ptr + 1, 0, (num_bin - 1) * sizeof(HistogramBinEntry));
    // construct histograms for smaller leaf
    if (ordered_bins[group] == nullptr) {
      // if not use ordered bin
      feature_groups_[group]->bin_data_->ConstructHistogram(
        data_indices,
        num_data,
        ptr_ordered_grad,
        ptr_ordered_hess,
        data_ptr);
    } else {
      // used ordered bin
      ordered_bins[group]->ConstructHistogram(leaf_idx,
        gradients,
        hessians,
        data_ptr);
    }
  }
}

void Dataset::FixHistogram(int feature_idx, double sum_gradient, double sum_hessian, data_size_t num_data,
  HistogramBinEntry* data) const {
  const int group = feature2group_[feature_idx];
  const int sub_feature = feature2subfeature_[feature_idx];
  const BinMapper* bin_mapper = feature_groups_[group]->bin_mappers_[sub_feature].get();
  const int default_bin = bin_mapper->GetDefaultBin();
  if (default_bin > 0) {
    const int num_bin = bin_mapper->num_bin();
    data[default_bin].sum_gradients = sum_gradient;
    data[default_bin].sum_hessians = sum_hessian;
    data[default_bin].cnt = num_data;
    for (int i = 0; i < num_bin; ++i) {
      if (i != default_bin) {
        data[default_bin].sum_gradients -= data[i].sum_gradients;
        data[default_bin].sum_hessians -= data[i].sum_hessians;
        data[default_bin].cnt -= data[i].cnt;
      }
    }
  }
}

Guolin Ke's avatar
Guolin Ke committed
465
}  // namespace LightGBM