dataset.h 19.4 KB
Newer Older
Guolin Ke's avatar
Guolin Ke committed
1
2
#ifndef LIGHTGBM_DATASET_H_
#define LIGHTGBM_DATASET_H_
Guolin Ke's avatar
Guolin Ke committed
3
4
5

#include <LightGBM/utils/random.h>
#include <LightGBM/utils/text_reader.h>
6
#include <LightGBM/utils/openmp_wrapper.h>
Guolin Ke's avatar
Guolin Ke committed
7
8

#include <LightGBM/meta.h>
Guolin Ke's avatar
Guolin Ke committed
9
#include <LightGBM/config.h>
Guolin Ke's avatar
Guolin Ke committed
10
#include <LightGBM/feature_group.h>
Guolin Ke's avatar
Guolin Ke committed
11
12
13
14
15

#include <vector>
#include <utility>
#include <functional>
#include <string>
Guolin Ke's avatar
Guolin Ke committed
16
#include <unordered_set>
17
#include <mutex>
Guolin Ke's avatar
Guolin Ke committed
18
19
20
21

namespace LightGBM {

/*! \brief forward declaration */
Guolin Ke's avatar
Guolin Ke committed
22
class DatasetLoader;
Guolin Ke's avatar
Guolin Ke committed
23
/*!
Hui Xue's avatar
Hui Xue committed
24
* \brief This class is used to store some meta(non-feature) data for training data,
Guolin Ke's avatar
Guolin Ke committed
25
26
*        e.g. labels, weights, initial scores, qurey level informations.
*
Qiwei Ye's avatar
Qiwei Ye committed
27
28
29
30
31
32
33
34
*        Some details:
*        1. Label, used for traning.
*        2. Weights, weighs of records, optional
*        3. Query Boundaries, necessary for lambdarank.
*           The documents of i-th query is in [ query_boundarise[i], query_boundarise[i+1] )
*        4. Query Weights, auto calculate by weights and query_boundarise(if both of them are existed)
*           the weight for i-th query is sum(query_boundarise[i] , .., query_boundarise[i+1]) / (query_boundarise[i + 1] -  query_boundarise[i+1])
*        5. Initial score. optional. if exsitng, the model will boost from this score, otherwise will start from 0.
Guolin Ke's avatar
Guolin Ke committed
35
36
37
*/
class Metadata {
public:
38
  /*!
Guolin Ke's avatar
Guolin Ke committed
39
40
41
42
  * \brief Null costructor
  */
  Metadata();
  /*!
Qiwei Ye's avatar
Qiwei Ye committed
43
  * \brief Initialization will load qurey level informations, since it is need for sampling data
Guolin Ke's avatar
Guolin Ke committed
44
45
46
  * \param data_filename Filename of data
  * \param init_score_filename Filename of initial score
  */
47
  void Init(const char* data_filename, const char* initscore_file);
Guolin Ke's avatar
Guolin Ke committed
48
  /*!
Guolin Ke's avatar
Guolin Ke committed
49
50
  * \brief init as subset
  * \param metadata Filename of data
51
  * \param used_indices
Guolin Ke's avatar
Guolin Ke committed
52
53
54
55
  * \param num_used_indices
  */
  void Init(const Metadata& metadata, const data_size_t* used_indices, data_size_t num_used_indices);
  /*!
Guolin Ke's avatar
Guolin Ke committed
56
57
58
59
60
61
62
63
  * \brief Initial with binary memory
  * \param memory Pointer to memory
  */
  void LoadFromMemory(const void* memory);
  /*! \brief Destructor */
  ~Metadata();

  /*!
Guolin Ke's avatar
Guolin Ke committed
64
  * \brief Initial work, will allocate space for label, weight(if exists) and query(if exists)
Guolin Ke's avatar
Guolin Ke committed
65
  * \param num_data Number of training data
Guolin Ke's avatar
Guolin Ke committed
66
67
  * \param weight_idx Index of weight column, < 0 means doesn't exists
  * \param query_idx Index of query id column, < 0 means doesn't exists
Guolin Ke's avatar
Guolin Ke committed
68
  */
69
  void Init(data_size_t num_data, int weight_idx, int query_idx);
Guolin Ke's avatar
Guolin Ke committed
70
71
72
73
74
75
76
77
78
79
80
81
82

  /*!
  * \brief Partition label by used indices
  * \param used_indices Indice of local used
  */
  void PartitionLabel(const std::vector<data_size_t>& used_indices);

  /*!
  * \brief Partition meta data according to local used indices if need
  * \param num_all_data Number of total training data, including other machines' data on parallel learning
  * \param used_data_indices Indices of local used training data
  */
  void CheckOrPartition(data_size_t num_all_data,
83
                        const std::vector<data_size_t>& used_data_indices);
Guolin Ke's avatar
Guolin Ke committed
84

85
  void SetLabel(const label_t* label, data_size_t len);
Guolin Ke's avatar
Guolin Ke committed
86

87
  void SetWeights(const label_t* weights, data_size_t len);
Guolin Ke's avatar
Guolin Ke committed
88

Guolin Ke's avatar
Guolin Ke committed
89
  void SetQuery(const data_size_t* query, data_size_t len);
Guolin Ke's avatar
Guolin Ke committed
90

Guolin Ke's avatar
Guolin Ke committed
91
92
93
94
  /*!
  * \brief Set initial scores
  * \param init_score Initial scores, this class will manage memory for init_score.
  */
95
  void SetInitScore(const double* init_score, data_size_t len);
Guolin Ke's avatar
Guolin Ke committed
96
97
98
99
100
101


  /*!
  * \brief Save binary data to file
  * \param file File want to write
  */
102
  void SaveBinaryToFile(const VirtualFileWriter* writer) const;
Guolin Ke's avatar
Guolin Ke committed
103
104
105
106
107
108
109
110
111
112

  /*!
  * \brief Get sizes in byte of this object
  */
  size_t SizesInByte() const;

  /*!
  * \brief Get pointer of label
  * \return Pointer of label
  */
113
  inline const label_t* label() const { return label_.data(); }
Guolin Ke's avatar
Guolin Ke committed
114
115
116
117
118
119

  /*!
  * \brief Set label for one record
  * \param idx Index of this record
  * \param value Label value of this record
  */
120
  inline void SetLabelAt(data_size_t idx, label_t value)
Guolin Ke's avatar
Guolin Ke committed
121
  {
122
    label_[idx] = value;
Guolin Ke's avatar
Guolin Ke committed
123
124
  }

Guolin Ke's avatar
Guolin Ke committed
125
126
127
128
129
  /*!
  * \brief Set Weight for one record
  * \param idx Index of this record
  * \param value Weight value of this record
  */
130
  inline void SetWeightAt(data_size_t idx, label_t value)
Guolin Ke's avatar
Guolin Ke committed
131
  {
132
    weights_[idx] = value;
Guolin Ke's avatar
Guolin Ke committed
133
134
135
136
137
138
139
  }

  /*!
  * \brief Set Query Id for one record
  * \param idx Index of this record
  * \param value Query Id value of this record
  */
Guolin Ke's avatar
Guolin Ke committed
140
  inline void SetQueryAt(data_size_t idx, data_size_t value)
Guolin Ke's avatar
Guolin Ke committed
141
142
143
144
  {
    queries_[idx] = static_cast<data_size_t>(value);
  }

Guolin Ke's avatar
Guolin Ke committed
145
  /*!
Hui Xue's avatar
Hui Xue committed
146
  * \brief Get weights, if not exists, will return nullptr
Guolin Ke's avatar
Guolin Ke committed
147
148
  * \return Pointer of weights
  */
149
  inline const label_t* weights() const {
Guolin Ke's avatar
Guolin Ke committed
150
    if (!weights_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
151
152
153
154
155
      return weights_.data();
    } else {
      return nullptr;
    }
  }
Guolin Ke's avatar
Guolin Ke committed
156
157

  /*!
Hui Xue's avatar
Hui Xue committed
158
  * \brief Get data boundaries on queries, if not exists, will return nullptr
159
  *        we assume data will order by query,
Guolin Ke's avatar
Guolin Ke committed
160
161
162
163
  *        the interval of [query_boundaris[i], query_boundaris[i+1])
  *        is the data indices for query i.
  * \return Pointer of data boundaries on queries
  */
164
  inline const data_size_t* query_boundaries() const {
Guolin Ke's avatar
Guolin Ke committed
165
    if (!query_boundaries_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
166
167
168
169
170
      return query_boundaries_.data();
    } else {
      return nullptr;
    }
  }
Guolin Ke's avatar
Guolin Ke committed
171
172
173
174
175

  /*!
  * \brief Get Number of queries
  * \return Number of queries
  */
176
  inline data_size_t num_queries() const { return num_queries_; }
Guolin Ke's avatar
Guolin Ke committed
177
178

  /*!
Hui Xue's avatar
Hui Xue committed
179
  * \brief Get weights for queries, if not exists, will return nullptr
Guolin Ke's avatar
Guolin Ke committed
180
181
  * \return Pointer of weights for queries
  */
182
  inline const label_t* query_weights() const {
Guolin Ke's avatar
Guolin Ke committed
183
    if (!query_weights_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
184
185
186
187
188
      return query_weights_.data();
    } else {
      return nullptr;
    }
  }
Guolin Ke's avatar
Guolin Ke committed
189
190

  /*!
Hui Xue's avatar
Hui Xue committed
191
  * \brief Get initial scores, if not exists, will return nullptr
Guolin Ke's avatar
Guolin Ke committed
192
193
  * \return Pointer of initial scores
  */
194
  inline const double* init_score() const {
Guolin Ke's avatar
Guolin Ke committed
195
    if (!init_score_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
196
197
198
199
200
      return init_score_.data();
    } else {
      return nullptr;
    }
  }
Guolin Ke's avatar
Guolin Ke committed
201

202
203
204
  /*!
  * \brief Get size of initial scores
  */
Guolin Ke's avatar
Guolin Ke committed
205
  inline int64_t num_init_score() const { return num_init_score_; }
206

Guolin Ke's avatar
Guolin Ke committed
207
208
209
210
  /*! \brief Disable copy */
  Metadata& operator=(const Metadata&) = delete;
  /*! \brief Disable copy */
  Metadata(const Metadata&) = delete;
Guolin Ke's avatar
Guolin Ke committed
211
212

private:
Guolin Ke's avatar
Guolin Ke committed
213
  /*! \brief Load initial scores from file */
214
  void LoadInitialScore(const char* initscore_file);
Guolin Ke's avatar
Guolin Ke committed
215
216
217
218
219
220
221
  /*! \brief Load wights from file */
  void LoadWeights();
  /*! \brief Load query boundaries from file */
  void LoadQueryBoundaries();
  /*! \brief Load query wights */
  void LoadQueryWeights();
  /*! \brief Filename of current data */
Guolin Ke's avatar
Guolin Ke committed
222
  std::string data_filename_;
Guolin Ke's avatar
Guolin Ke committed
223
224
225
226
227
  /*! \brief Number of data */
  data_size_t num_data_;
  /*! \brief Number of weights, used to check correct weight file */
  data_size_t num_weights_;
  /*! \brief Label data */
228
  std::vector<label_t> label_;
Guolin Ke's avatar
Guolin Ke committed
229
  /*! \brief Weights data */
230
  std::vector<label_t> weights_;
Guolin Ke's avatar
Guolin Ke committed
231
  /*! \brief Query boundaries */
Guolin Ke's avatar
Guolin Ke committed
232
  std::vector<data_size_t> query_boundaries_;
Guolin Ke's avatar
Guolin Ke committed
233
  /*! \brief Query weights */
234
  std::vector<label_t> query_weights_;
Guolin Ke's avatar
Guolin Ke committed
235
236
237
  /*! \brief Number of querys */
  data_size_t num_queries_;
  /*! \brief Number of Initial score, used to check correct weight file */
Guolin Ke's avatar
Guolin Ke committed
238
  int64_t num_init_score_;
Guolin Ke's avatar
Guolin Ke committed
239
  /*! \brief Initial score */
Guolin Ke's avatar
Guolin Ke committed
240
  std::vector<double> init_score_;
Guolin Ke's avatar
Guolin Ke committed
241
  /*! \brief Queries data */
Guolin Ke's avatar
Guolin Ke committed
242
  std::vector<data_size_t> queries_;
243
244
  /*! \brief mutex for threading safe call */
  std::mutex mutex_;
245
246
247
  bool weight_load_from_file_;
  bool query_load_from_file_;
  bool init_score_load_from_file_;
Guolin Ke's avatar
Guolin Ke committed
248
249
250
251
252
253
};


/*! \brief Interface for Parser */
class Parser {
public:
Guolin Ke's avatar
Guolin Ke committed
254

Guolin Ke's avatar
Guolin Ke committed
255
256
257
258
259
260
  /*! \brief virtual destructor */
  virtual ~Parser() {}

  /*!
  * \brief Parse one line with label
  * \param str One line record, string format, should end with '\0'
Guolin Ke's avatar
Guolin Ke committed
261
262
  * \param out_features Output columns, store in (column_idx, values)
  * \param out_label Label will store to this if exists
Guolin Ke's avatar
Guolin Ke committed
263
264
  */
  virtual void ParseOneLine(const char* str,
265
                            std::vector<std::pair<int, double>>* out_features, double* out_label) const = 0;
Guolin Ke's avatar
Guolin Ke committed
266

Guolin Ke's avatar
Guolin Ke committed
267
268
  virtual int TotalColumns() const = 0;

Guolin Ke's avatar
Guolin Ke committed
269
270
271
  /*!
  * \brief Create a object of parser, will auto choose the format depend on file
  * \param filename One Filename of data
272
  * \param num_features Pass num_features of this data file if you know, <=0 means don't know
Guolin Ke's avatar
Guolin Ke committed
273
  * \param label_idx index of label column
Guolin Ke's avatar
Guolin Ke committed
274
275
  * \return Object of parser
  */
Guolin Ke's avatar
Guolin Ke committed
276
  static Parser* CreateParser(const char* filename, bool has_header, int num_features, int label_idx);
Guolin Ke's avatar
Guolin Ke committed
277
278
279
280
281
282
283
};

/*! \brief The main class of data set,
*          which are used to traning or validation
*/
class Dataset {
public:
Guolin Ke's avatar
Guolin Ke committed
284
  friend DatasetLoader;
Guolin Ke's avatar
Guolin Ke committed
285

286
  LIGHTGBM_EXPORT Dataset();
Guolin Ke's avatar
Guolin Ke committed
287

288
  LIGHTGBM_EXPORT Dataset(data_size_t num_data);
Guolin Ke's avatar
Guolin Ke committed
289

Guolin Ke's avatar
Guolin Ke committed
290
291
  void Construct(
    std::vector<std::unique_ptr<BinMapper>>& bin_mappers,
292
293
    int** sample_non_zero_indices,
    const int* num_per_col,
Guolin Ke's avatar
Guolin Ke committed
294
295
296
    size_t total_sample_cnt,
    const IOConfig& io_config);

Guolin Ke's avatar
Guolin Ke committed
297
  /*! \brief Destructor */
298
  LIGHTGBM_EXPORT ~Dataset();
Guolin Ke's avatar
Guolin Ke committed
299

300
  LIGHTGBM_EXPORT bool CheckAlign(const Dataset& other) const {
301
302
303
304
305
306
307
308
309
310
    if (num_features_ != other.num_features_) {
      return false;
    }
    if (num_total_features_ != other.num_total_features_) {
      return false;
    }
    if (label_idx_ != other.label_idx_) {
      return false;
    }
    for (int i = 0; i < num_features_; ++i) {
Guolin Ke's avatar
Guolin Ke committed
311
      if (!FeatureBinMapper(i)->CheckAlign(*(other.FeatureBinMapper(i)))) {
312
313
314
315
316
317
        return false;
      }
    }
    return true;
  }

Guolin Ke's avatar
Guolin Ke committed
318
  inline void PushOneRow(int tid, data_size_t row_idx, const std::vector<double>& feature_values) {
Guolin Ke's avatar
Guolin Ke committed
319
    if (is_finish_load_) { return; }
Guolin Ke's avatar
Guolin Ke committed
320
    for (size_t i = 0; i < feature_values.size() && i < static_cast<size_t>(num_total_features_); ++i) {
Guolin Ke's avatar
Guolin Ke committed
321
322
      int feature_idx = used_feature_map_[i];
      if (feature_idx >= 0) {
Guolin Ke's avatar
Guolin Ke committed
323
324
325
        const int group = feature2group_[feature_idx];
        const int sub_feature = feature2subfeature_[feature_idx];
        feature_groups_[group]->PushData(tid, sub_feature, row_idx, feature_values[i]);
Guolin Ke's avatar
Guolin Ke committed
326
327
328
329
      }
    }
  }

330
  inline void PushOneRow(int tid, data_size_t row_idx, const std::vector<std::pair<int, double>>& feature_values) {
Guolin Ke's avatar
Guolin Ke committed
331
    if (is_finish_load_) { return; }
332
    for (auto& inner_data : feature_values) {
333
      if (inner_data.first >= num_total_features_) { continue; }
334
335
      int feature_idx = used_feature_map_[inner_data.first];
      if (feature_idx >= 0) {
Guolin Ke's avatar
Guolin Ke committed
336
337
338
        const int group = feature2group_[feature_idx];
        const int sub_feature = feature2subfeature_[feature_idx];
        feature_groups_[group]->PushData(tid, sub_feature, row_idx, inner_data.second);
339
340
341
342
      }
    }
  }

Guolin Ke's avatar
Guolin Ke committed
343
344
345
346
347
348
349
350
351
  inline void PushOneData(int tid, data_size_t row_idx, int group, int sub_feature, double value) {
    feature_groups_[group]->PushData(tid, sub_feature, row_idx, value);
  }

  inline int RealFeatureIndex(int fidx) const {
    return real_feature_idx_[fidx];
  }

  inline int InnerFeatureIndex(int col_idx) const {
Guolin Ke's avatar
Guolin Ke committed
352
    return used_feature_map_[col_idx];
Guolin Ke's avatar
Guolin Ke committed
353
  }
Guolin Ke's avatar
Guolin Ke committed
354
355
356
357
358
359
  inline int Feature2Group(int feature_idx) const {
    return feature2group_[feature_idx];
  }
  inline int Feture2SubFeature(int feature_idx) const {
    return feature2subfeature_[feature_idx];
  }
360
361
362
  inline uint64_t GroupBinBoundary(int group_idx) const {
    return group_bin_boundaries_[group_idx];
  }
Guolin Ke's avatar
Guolin Ke committed
363
364
365
  inline uint64_t NumTotalBin() const {
    return group_bin_boundaries_.back();
  }
366
367
368
369
370
371
372
373
374
  inline std::vector<int> ValidFeatureIndices() const {
    std::vector<int> ret;
    for (int i = 0; i < num_total_features_; ++i) {
      if (used_feature_map_[i] >= 0) {
        ret.push_back(i);
      }
    }
    return ret;
  }
Guolin Ke's avatar
Guolin Ke committed
375
376
377
  void ReSize(data_size_t num_data);

  void CopySubset(const Dataset* fullset, const data_size_t* used_indices, data_size_t num_used_indices, bool need_meta_data);
Guolin Ke's avatar
Guolin Ke committed
378

379
  LIGHTGBM_EXPORT void FinishLoad();
Guolin Ke's avatar
Guolin Ke committed
380

381
  LIGHTGBM_EXPORT bool SetFloatField(const char* field_name, const float* field_data, data_size_t num_element);
Guolin Ke's avatar
Guolin Ke committed
382

383
  LIGHTGBM_EXPORT bool SetDoubleField(const char* field_name, const double* field_data, data_size_t num_element);
Guolin Ke's avatar
Guolin Ke committed
384

385
  LIGHTGBM_EXPORT bool SetIntField(const char* field_name, const int* field_data, data_size_t num_element);
386

387
  LIGHTGBM_EXPORT bool GetFloatField(const char* field_name, data_size_t* out_len, const float** out_ptr);
388

389
  LIGHTGBM_EXPORT bool GetDoubleField(const char* field_name, data_size_t* out_len, const double** out_ptr);
Guolin Ke's avatar
Guolin Ke committed
390

391
  LIGHTGBM_EXPORT bool GetIntField(const char* field_name, data_size_t* out_len, const int** out_ptr);
392

Guolin Ke's avatar
Guolin Ke committed
393
394
395
  /*!
  * \brief Save current dataset into binary file, will save to "filename.bin"
  */
396
  LIGHTGBM_EXPORT void SaveBinaryFile(const char* bin_filename);
Guolin Ke's avatar
Guolin Ke committed
397

398
  LIGHTGBM_EXPORT void CopyFeatureMapperFrom(const Dataset* dataset);
Guolin Ke's avatar
Guolin Ke committed
399

Guolin Ke's avatar
Guolin Ke committed
400
401
  LIGHTGBM_EXPORT void CreateValid(const Dataset* dataset);

402
403
404
405
  void ConstructHistograms(const std::vector<int8_t>& is_feature_used,
                           const data_size_t* data_indices, data_size_t num_data,
                           int leaf_idx,
                           std::vector<std::unique_ptr<OrderedBin>>& ordered_bins,
406
407
                           const score_t* gradients, const score_t* hessians,
                           score_t* ordered_gradients, score_t* ordered_hessians,
408
409
                           bool is_constant_hessian,
                           HistogramBinEntry* histogram_data) const;
Guolin Ke's avatar
Guolin Ke committed
410
411

  void FixHistogram(int feature_idx, double sum_gradient, double sum_hessian, data_size_t num_data,
412
                    HistogramBinEntry* data) const;
Guolin Ke's avatar
Guolin Ke committed
413

414
  inline data_size_t Split(int feature,
415
                           const uint32_t* threshold, int num_threshold,  bool default_left,
416
417
                           data_size_t* data_indices, data_size_t num_data,
                           data_size_t* lte_indices, data_size_t* gt_indices) const {
Guolin Ke's avatar
Guolin Ke committed
418
419
    const int group = feature2group_[feature];
    const int sub_feature = feature2subfeature_[feature];
420
    return feature_groups_[group]->Split(sub_feature, threshold, num_threshold, default_left, data_indices, num_data, lte_indices, gt_indices);
Guolin Ke's avatar
Guolin Ke committed
421
422
423
424
425
426
427
428
429
430
431
432
433
434
  }

  inline int SubFeatureBinOffset(int i) const {
    const int sub_feature = feature2subfeature_[i];
    if (sub_feature == 0) {
      return 1;
    } else {
      return 0;
    }
  }

  inline int FeatureNumBin(int i) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
435
    return feature_groups_[group]->bin_mappers_[sub_feature]->num_bin();
Guolin Ke's avatar
Guolin Ke committed
436
  }
Guolin Ke's avatar
Guolin Ke committed
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457

  inline int8_t FeatureMonotone(int i) const {
    if (monotone_types_.empty()) {
      return 0;
    } else {
      return monotone_types_[i];
    }
  }

  bool HasMonotone() const {
    if (monotone_types_.empty()) {
      return false;
    } else {
      for (size_t i = 0; i < monotone_types_.size(); ++i) {
        if (monotone_types_[i] != 0) {
          return true;
        }
      }
      return false;
    }
  }
458
459
460
461
  
  inline int FeatureGroupNumBin(int group) const {
    return feature_groups_[group]->num_total_bin_;
  }
462

Guolin Ke's avatar
Guolin Ke committed
463
464
465
466
467
468
  inline const BinMapper* FeatureBinMapper(int i) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
    return feature_groups_[group]->bin_mappers_[sub_feature].get();
  }

469
470
471
472
473
474
475
476
477
  inline const Bin* FeatureBin(int i) const {
    const int group = feature2group_[i];
    return feature_groups_[group]->bin_data_.get();
  }
  
  inline const Bin* FeatureGroupBin(int group) const {
    return feature_groups_[group]->bin_data_.get();
  }

478
479
480
481
  inline bool FeatureGroupIsSparse(int group) const {
    return feature_groups_[group]->is_sparse_;
  }

Guolin Ke's avatar
Guolin Ke committed
482
483
484
  inline BinIterator* FeatureIterator(int i) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
zhangyafeikimi's avatar
zhangyafeikimi committed
485
    return feature_groups_[group]->SubFeatureIterator(sub_feature);
Guolin Ke's avatar
Guolin Ke committed
486
487
  }

488
489
490
491
  inline BinIterator* FeatureGroupIterator(int group) const {
    return feature_groups_[group]->FeatureGroupIterator();
  }
  
Guolin Ke's avatar
Guolin Ke committed
492
493
494
495
496
497
  inline double RealThreshold(int i, uint32_t threshold) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
    return feature_groups_[group]->bin_mappers_[sub_feature]->BinToValue(threshold);
  }

498
499
500
501
502
503
504
  // given a real threshold, find the closest threshold bin
  inline uint32_t BinThreshold(int i, double threshold_double) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
    return feature_groups_[group]->bin_mappers_[sub_feature]->ValueToBin(threshold_double);
  }

Guolin Ke's avatar
Guolin Ke committed
505
506
  inline void CreateOrderedBins(std::vector<std::unique_ptr<OrderedBin>>* ordered_bins) const {
    ordered_bins->resize(num_groups_);
507
508
    OMP_INIT_EX();
    #pragma omp parallel for schedule(guided)
Guolin Ke's avatar
Guolin Ke committed
509
    for (int i = 0; i < num_groups_; ++i) {
510
511
512
      OMP_LOOP_EX_BEGIN();
      ordered_bins->at(i).reset(feature_groups_[i]->bin_data_->CreateOrderedBin());
      OMP_LOOP_EX_END();
Guolin Ke's avatar
Guolin Ke committed
513
    }
514
    OMP_THROW_EX();
Guolin Ke's avatar
Guolin Ke committed
515
  }
Guolin Ke's avatar
Guolin Ke committed
516
517
518
519
520
521
522
523
524
525

  /*!
  * \brief Get meta data pointer
  * \return Pointer of meta data
  */
  inline const Metadata& metadata() const { return metadata_; }

  /*! \brief Get Number of used features */
  inline int num_features() const { return num_features_; }

526
527
528
  /*! \brief Get Number of feature groups */
  inline int num_feature_groups() const { return num_groups_;}

529
530
531
  /*! \brief Get Number of total features */
  inline int num_total_features() const { return num_total_features_; }

Guolin Ke's avatar
Guolin Ke committed
532
533
534
535
  /*! \brief Get the index of label column */
  inline int label_idx() const { return label_idx_; }

  /*! \brief Get names of current data set */
Guolin Ke's avatar
Guolin Ke committed
536
537
538
539
  inline const std::vector<std::string>& feature_names() const { return feature_names_; }

  inline void set_feature_names(const std::vector<std::string>& feature_names) {
    if (feature_names.size() != static_cast<size_t>(num_total_features_)) {
540
      Log::Fatal("Size of feature_names error, should equal with total number of features");
Guolin Ke's avatar
Guolin Ke committed
541
542
    }
    feature_names_ = std::vector<std::string>(feature_names);
543
544
545
546
547
548
549
550
551
552
553
    // replace ' ' in feature_names with '_'
    bool spaceInFeatureName = false;
    for (auto& feature_name: feature_names_){
      if (feature_name.find(' ') != std::string::npos){
        spaceInFeatureName = true;
        std::replace(feature_name.begin(), feature_name.end(), ' ', '_');
      }
    }
    if (spaceInFeatureName){
      Log::Warning("Find whitespaces in feature_names, replace with underlines");
    }
Guolin Ke's avatar
Guolin Ke committed
554
  }
Guolin Ke's avatar
Guolin Ke committed
555

Guolin Ke's avatar
Guolin Ke committed
556
557
558
559
560
561
562
563
564
565
566
567
568
569
  inline std::vector<std::string> feature_infos() const {
    std::vector<std::string> bufs;
    for (int i = 0; i < num_total_features_; i++) {
      int fidx = used_feature_map_[i];
      if (fidx == -1) {
        bufs.push_back("none");
      } else {
        const auto bin_mapper = FeatureBinMapper(fidx);
        bufs.push_back(bin_mapper->bin_info());
      }
    }
    return bufs;
  }

Guolin Ke's avatar
Guolin Ke committed
570
571
572
573
574
575
576
577
578
  /*! \brief Get Number of data */
  inline data_size_t num_data() const { return num_data_; }

  /*! \brief Disable copy */
  Dataset& operator=(const Dataset&) = delete;
  /*! \brief Disable copy */
  Dataset(const Dataset&) = delete;

private:
Guolin Ke's avatar
Guolin Ke committed
579
  std::string data_filename_;
Guolin Ke's avatar
Guolin Ke committed
580
  /*! \brief Store used features */
Guolin Ke's avatar
Guolin Ke committed
581
  std::vector<std::unique_ptr<FeatureGroup>> feature_groups_;
Guolin Ke's avatar
Guolin Ke committed
582
583
584
585
  /*! \brief Mapper from real feature index to used index*/
  std::vector<int> used_feature_map_;
  /*! \brief Number of used features*/
  int num_features_;
586
587
  /*! \brief Number of total features*/
  int num_total_features_;
Guolin Ke's avatar
Guolin Ke committed
588
589
590
591
  /*! \brief Number of total data*/
  data_size_t num_data_;
  /*! \brief Store some label level data*/
  Metadata metadata_;
Guolin Ke's avatar
Guolin Ke committed
592
593
  /*! \brief index of label column */
  int label_idx_ = 0;
594
595
  /*! \brief Threshold for treating a feature as a sparse feature */
  double sparse_threshold_;
Guolin Ke's avatar
Guolin Ke committed
596
597
  /*! \brief store feature names */
  std::vector<std::string> feature_names_;
598
599
  /*! \brief store feature names */
  static const char* binary_file_token;
Guolin Ke's avatar
Guolin Ke committed
600
601
602
603
604
605
606
  int num_groups_;
  std::vector<int> real_feature_idx_;
  std::vector<int> feature2group_;
  std::vector<int> feature2subfeature_;
  std::vector<uint64_t> group_bin_boundaries_;
  std::vector<int> group_feature_start_;
  std::vector<int> group_feature_cnt_;
Guolin Ke's avatar
Guolin Ke committed
607
  std::vector<int8_t> monotone_types_;
Guolin Ke's avatar
Guolin Ke committed
608
  bool is_finish_load_;
Guolin Ke's avatar
Guolin Ke committed
609
610
611
612
};

}  // namespace LightGBM

Guolin Ke's avatar
Guolin Ke committed
613
#endif   // LightGBM_DATA_H_