dataset.h 16.5 KB
Newer Older
Guolin Ke's avatar
Guolin Ke committed
1
2
#ifndef LIGHTGBM_DATASET_H_
#define LIGHTGBM_DATASET_H_
Guolin Ke's avatar
Guolin Ke committed
3
4
5
6
7

#include <LightGBM/utils/random.h>
#include <LightGBM/utils/text_reader.h>

#include <LightGBM/meta.h>
Guolin Ke's avatar
Guolin Ke committed
8
#include <LightGBM/config.h>
Guolin Ke's avatar
Guolin Ke committed
9
#include <LightGBM/feature_group.h>
Guolin Ke's avatar
Guolin Ke committed
10
11
12
13
14

#include <vector>
#include <utility>
#include <functional>
#include <string>
Guolin Ke's avatar
Guolin Ke committed
15
#include <unordered_set>
16
#include <mutex>
Guolin Ke's avatar
Guolin Ke committed
17
18
19
20

namespace LightGBM {

/*! \brief forward declaration */
Guolin Ke's avatar
Guolin Ke committed
21
class DatasetLoader;
Guolin Ke's avatar
Guolin Ke committed
22
/*!
Hui Xue's avatar
Hui Xue committed
23
* \brief This class is used to store some meta(non-feature) data for training data,
Guolin Ke's avatar
Guolin Ke committed
24
25
*        e.g. labels, weights, initial scores, qurey level informations.
*
Qiwei Ye's avatar
Qiwei Ye committed
26
27
28
29
30
31
32
33
*        Some details:
*        1. Label, used for traning.
*        2. Weights, weighs of records, optional
*        3. Query Boundaries, necessary for lambdarank.
*           The documents of i-th query is in [ query_boundarise[i], query_boundarise[i+1] )
*        4. Query Weights, auto calculate by weights and query_boundarise(if both of them are existed)
*           the weight for i-th query is sum(query_boundarise[i] , .., query_boundarise[i+1]) / (query_boundarise[i + 1] -  query_boundarise[i+1])
*        5. Initial score. optional. if exsitng, the model will boost from this score, otherwise will start from 0.
Guolin Ke's avatar
Guolin Ke committed
34
35
36
37
38
39
40
41
*/
class Metadata {
public:
 /*!
  * \brief Null costructor
  */
  Metadata();
  /*!
Qiwei Ye's avatar
Qiwei Ye committed
42
  * \brief Initialization will load qurey level informations, since it is need for sampling data
Guolin Ke's avatar
Guolin Ke committed
43
44
45
  * \param data_filename Filename of data
  * \param init_score_filename Filename of initial score
  */
46
  void Init(const char* data_filename);
Guolin Ke's avatar
Guolin Ke committed
47
  /*!
Guolin Ke's avatar
Guolin Ke committed
48
49
50
51
52
53
54
  * \brief init as subset
  * \param metadata Filename of data
  * \param used_indices 
  * \param num_used_indices
  */
  void Init(const Metadata& metadata, const data_size_t* used_indices, data_size_t num_used_indices);
  /*!
Guolin Ke's avatar
Guolin Ke committed
55
56
57
58
59
60
61
62
  * \brief Initial with binary memory
  * \param memory Pointer to memory
  */
  void LoadFromMemory(const void* memory);
  /*! \brief Destructor */
  ~Metadata();

  /*!
Guolin Ke's avatar
Guolin Ke committed
63
  * \brief Initial work, will allocate space for label, weight(if exists) and query(if exists)
Guolin Ke's avatar
Guolin Ke committed
64
  * \param num_data Number of training data
Guolin Ke's avatar
Guolin Ke committed
65
66
  * \param weight_idx Index of weight column, < 0 means doesn't exists
  * \param query_idx Index of query id column, < 0 means doesn't exists
Guolin Ke's avatar
Guolin Ke committed
67
  */
68
  void Init(data_size_t num_data, int weight_idx, int query_idx);
Guolin Ke's avatar
Guolin Ke committed
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

  /*!
  * \brief Partition label by used indices
  * \param used_indices Indice of local used
  */
  void PartitionLabel(const std::vector<data_size_t>& used_indices);

  /*!
  * \brief Partition meta data according to local used indices if need
  * \param num_all_data Number of total training data, including other machines' data on parallel learning
  * \param used_data_indices Indices of local used training data
  */
  void CheckOrPartition(data_size_t num_all_data,
    const std::vector<data_size_t>& used_data_indices);

Guolin Ke's avatar
Guolin Ke committed
84
85
86
87
  void SetLabel(const float* label, data_size_t len);

  void SetWeights(const float* weights, data_size_t len);

Guolin Ke's avatar
Guolin Ke committed
88
  void SetQuery(const data_size_t* query, data_size_t len);
Guolin Ke's avatar
Guolin Ke committed
89

Guolin Ke's avatar
Guolin Ke committed
90
91
92
93
  /*!
  * \brief Set initial scores
  * \param init_score Initial scores, this class will manage memory for init_score.
  */
94
  void SetInitScore(const double* init_score, data_size_t len);
Guolin Ke's avatar
Guolin Ke committed
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111


  /*!
  * \brief Save binary data to file
  * \param file File want to write
  */
  void SaveBinaryToFile(FILE* file) const;

  /*!
  * \brief Get sizes in byte of this object
  */
  size_t SizesInByte() const;

  /*!
  * \brief Get pointer of label
  * \return Pointer of label
  */
Guolin Ke's avatar
Guolin Ke committed
112
  inline const float* label() const { return label_.data(); }
Guolin Ke's avatar
Guolin Ke committed
113
114
115
116
117
118

  /*!
  * \brief Set label for one record
  * \param idx Index of this record
  * \param value Label value of this record
  */
119
  inline void SetLabelAt(data_size_t idx, float value)
Guolin Ke's avatar
Guolin Ke committed
120
  {
121
    label_[idx] = value;
Guolin Ke's avatar
Guolin Ke committed
122
123
  }

Guolin Ke's avatar
Guolin Ke committed
124
125
126
127
128
  /*!
  * \brief Set Weight for one record
  * \param idx Index of this record
  * \param value Weight value of this record
  */
129
  inline void SetWeightAt(data_size_t idx, float value)
Guolin Ke's avatar
Guolin Ke committed
130
  {
131
    weights_[idx] = value;
Guolin Ke's avatar
Guolin Ke committed
132
133
134
135
136
137
138
  }

  /*!
  * \brief Set Query Id for one record
  * \param idx Index of this record
  * \param value Query Id value of this record
  */
Guolin Ke's avatar
Guolin Ke committed
139
  inline void SetQueryAt(data_size_t idx, data_size_t value)
Guolin Ke's avatar
Guolin Ke committed
140
141
142
143
  {
    queries_[idx] = static_cast<data_size_t>(value);
  }

Guolin Ke's avatar
Guolin Ke committed
144
  /*!
Hui Xue's avatar
Hui Xue committed
145
  * \brief Get weights, if not exists, will return nullptr
Guolin Ke's avatar
Guolin Ke committed
146
147
  * \return Pointer of weights
  */
Guolin Ke's avatar
Guolin Ke committed
148
  inline const float* weights() const {
Guolin Ke's avatar
Guolin Ke committed
149
    if (!weights_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
150
151
152
153
154
      return weights_.data();
    } else {
      return nullptr;
    }
  }
Guolin Ke's avatar
Guolin Ke committed
155
156

  /*!
Hui Xue's avatar
Hui Xue committed
157
  * \brief Get data boundaries on queries, if not exists, will return nullptr
Guolin Ke's avatar
Guolin Ke committed
158
159
160
161
162
  *        we assume data will order by query, 
  *        the interval of [query_boundaris[i], query_boundaris[i+1])
  *        is the data indices for query i.
  * \return Pointer of data boundaries on queries
  */
Guolin Ke's avatar
Guolin Ke committed
163
  inline const data_size_t* query_boundaries() const { 
Guolin Ke's avatar
Guolin Ke committed
164
    if (!query_boundaries_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
165
166
167
168
169
      return query_boundaries_.data();
    } else {
      return nullptr;
    }
  }
Guolin Ke's avatar
Guolin Ke committed
170
171
172
173
174

  /*!
  * \brief Get Number of queries
  * \return Number of queries
  */
175
  inline data_size_t num_queries() const { return num_queries_; }
Guolin Ke's avatar
Guolin Ke committed
176
177

  /*!
Hui Xue's avatar
Hui Xue committed
178
  * \brief Get weights for queries, if not exists, will return nullptr
Guolin Ke's avatar
Guolin Ke committed
179
180
  * \return Pointer of weights for queries
  */
Guolin Ke's avatar
Guolin Ke committed
181
  inline const float* query_weights() const { 
Guolin Ke's avatar
Guolin Ke committed
182
    if (!query_weights_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
183
184
185
186
187
      return query_weights_.data();
    } else {
      return nullptr;
    }
  }
Guolin Ke's avatar
Guolin Ke committed
188
189

  /*!
Hui Xue's avatar
Hui Xue committed
190
  * \brief Get initial scores, if not exists, will return nullptr
Guolin Ke's avatar
Guolin Ke committed
191
192
  * \return Pointer of initial scores
  */
Guolin Ke's avatar
Guolin Ke committed
193
  inline const double* init_score() const { 
Guolin Ke's avatar
Guolin Ke committed
194
    if (!init_score_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
195
196
197
198
199
      return init_score_.data();
    } else {
      return nullptr;
    }
  }
Guolin Ke's avatar
Guolin Ke committed
200

201
202
203
  /*!
  * \brief Get size of initial scores
  */
Guolin Ke's avatar
Guolin Ke committed
204
  inline int64_t num_init_score() const { return num_init_score_; }
205

Guolin Ke's avatar
Guolin Ke committed
206
207
208
209
  /*! \brief Disable copy */
  Metadata& operator=(const Metadata&) = delete;
  /*! \brief Disable copy */
  Metadata(const Metadata&) = delete;
Guolin Ke's avatar
Guolin Ke committed
210
211

private:
Guolin Ke's avatar
Guolin Ke committed
212
213
  /*! \brief Load initial scores from file */
  void LoadInitialScore();
Guolin Ke's avatar
Guolin Ke committed
214
215
216
217
218
219
220
221
222
223
224
225
226
  /*! \brief Load wights from file */
  void LoadWeights();
  /*! \brief Load query boundaries from file */
  void LoadQueryBoundaries();
  /*! \brief Load query wights */
  void LoadQueryWeights();
  /*! \brief Filename of current data */
  const char* data_filename_;
  /*! \brief Number of data */
  data_size_t num_data_;
  /*! \brief Number of weights, used to check correct weight file */
  data_size_t num_weights_;
  /*! \brief Label data */
Guolin Ke's avatar
Guolin Ke committed
227
  std::vector<float> label_;
Guolin Ke's avatar
Guolin Ke committed
228
  /*! \brief Weights data */
Guolin Ke's avatar
Guolin Ke committed
229
  std::vector<float> weights_;
Guolin Ke's avatar
Guolin Ke committed
230
  /*! \brief Query boundaries */
Guolin Ke's avatar
Guolin Ke committed
231
  std::vector<data_size_t> query_boundaries_;
Guolin Ke's avatar
Guolin Ke committed
232
  /*! \brief Query weights */
Guolin Ke's avatar
Guolin Ke committed
233
  std::vector<float> query_weights_;
Guolin Ke's avatar
Guolin Ke committed
234
235
236
  /*! \brief Number of querys */
  data_size_t num_queries_;
  /*! \brief Number of Initial score, used to check correct weight file */
Guolin Ke's avatar
Guolin Ke committed
237
  int64_t num_init_score_;
Guolin Ke's avatar
Guolin Ke committed
238
  /*! \brief Initial score */
Guolin Ke's avatar
Guolin Ke committed
239
  std::vector<double> init_score_;
Guolin Ke's avatar
Guolin Ke committed
240
  /*! \brief Queries data */
Guolin Ke's avatar
Guolin Ke committed
241
  std::vector<data_size_t> queries_;
242
243
  /*! \brief mutex for threading safe call */
  std::mutex mutex_;
244
245
246
  bool weight_load_from_file_;
  bool query_load_from_file_;
  bool init_score_load_from_file_;
Guolin Ke's avatar
Guolin Ke committed
247
248
249
250
251
252
};


/*! \brief Interface for Parser */
class Parser {
public:
Guolin Ke's avatar
Guolin Ke committed
253

Guolin Ke's avatar
Guolin Ke committed
254
255
256
257
258
259
  /*! \brief virtual destructor */
  virtual ~Parser() {}

  /*!
  * \brief Parse one line with label
  * \param str One line record, string format, should end with '\0'
Guolin Ke's avatar
Guolin Ke committed
260
261
  * \param out_features Output columns, store in (column_idx, values)
  * \param out_label Label will store to this if exists
Guolin Ke's avatar
Guolin Ke committed
262
263
  */
  virtual void ParseOneLine(const char* str,
264
    std::vector<std::pair<int, double>>* out_features, double* out_label) const = 0;
Guolin Ke's avatar
Guolin Ke committed
265
266
267
268

  /*!
  * \brief Create a object of parser, will auto choose the format depend on file
  * \param filename One Filename of data
269
  * \param num_features Pass num_features of this data file if you know, <=0 means don't know
Guolin Ke's avatar
Guolin Ke committed
270
  * \param label_idx index of label column
Guolin Ke's avatar
Guolin Ke committed
271
272
  * \return Object of parser
  */
Guolin Ke's avatar
Guolin Ke committed
273
  static Parser* CreateParser(const char* filename, bool has_header, int num_features, int label_idx);
Guolin Ke's avatar
Guolin Ke committed
274
275
276
277
278
279
280
};

/*! \brief The main class of data set,
*          which are used to traning or validation
*/
class Dataset {
public:
Guolin Ke's avatar
Guolin Ke committed
281
  friend DatasetLoader;
Guolin Ke's avatar
Guolin Ke committed
282

283
  LIGHTGBM_EXPORT Dataset();
Guolin Ke's avatar
Guolin Ke committed
284

285
  LIGHTGBM_EXPORT Dataset(data_size_t num_data);
Guolin Ke's avatar
Guolin Ke committed
286

Guolin Ke's avatar
Guolin Ke committed
287
288
289
290
291
292
  void Construct(
    std::vector<std::unique_ptr<BinMapper>>& bin_mappers,
    const std::vector<std::vector<int>>& sample_non_zero_indices,
    size_t total_sample_cnt,
    const IOConfig& io_config);

Guolin Ke's avatar
Guolin Ke committed
293
  /*! \brief Destructor */
294
  LIGHTGBM_EXPORT ~Dataset();
Guolin Ke's avatar
Guolin Ke committed
295

296
  LIGHTGBM_EXPORT bool CheckAlign(const Dataset& other) const {
297
298
299
300
301
302
303
304
305
306
    if (num_features_ != other.num_features_) {
      return false;
    }
    if (num_total_features_ != other.num_total_features_) {
      return false;
    }
    if (label_idx_ != other.label_idx_) {
      return false;
    }
    for (int i = 0; i < num_features_; ++i) {
Guolin Ke's avatar
Guolin Ke committed
307
      if (!FeatureBinMapper(i)->CheckAlign(*(other.FeatureBinMapper(i)))) {
308
309
310
311
312
313
        return false;
      }
    }
    return true;
  }

Guolin Ke's avatar
Guolin Ke committed
314
  inline void PushOneRow(int tid, data_size_t row_idx, const std::vector<double>& feature_values) {
Guolin Ke's avatar
Guolin Ke committed
315
    if (is_finish_load_) { return; }
Guolin Ke's avatar
Guolin Ke committed
316
    for (size_t i = 0; i < feature_values.size() && i < static_cast<size_t>(num_total_features_); ++i) {
Guolin Ke's avatar
Guolin Ke committed
317
318
      int feature_idx = used_feature_map_[i];
      if (feature_idx >= 0) {
Guolin Ke's avatar
Guolin Ke committed
319
320
321
        const int group = feature2group_[feature_idx];
        const int sub_feature = feature2subfeature_[feature_idx];
        feature_groups_[group]->PushData(tid, sub_feature, row_idx, feature_values[i]);
Guolin Ke's avatar
Guolin Ke committed
322
323
324
325
      }
    }
  }

326
  inline void PushOneRow(int tid, data_size_t row_idx, const std::vector<std::pair<int, double>>& feature_values) {
Guolin Ke's avatar
Guolin Ke committed
327
    if (is_finish_load_) { return; }
328
    for (auto& inner_data : feature_values) {
329
      if (inner_data.first >= num_total_features_) { continue; }
330
331
      int feature_idx = used_feature_map_[inner_data.first];
      if (feature_idx >= 0) {
Guolin Ke's avatar
Guolin Ke committed
332
333
334
        const int group = feature2group_[feature_idx];
        const int sub_feature = feature2subfeature_[feature_idx];
        feature_groups_[group]->PushData(tid, sub_feature, row_idx, inner_data.second);
335
336
337
338
      }
    }
  }

Guolin Ke's avatar
Guolin Ke committed
339
340
341
342
343
344
345
346
347
  inline void PushOneData(int tid, data_size_t row_idx, int group, int sub_feature, double value) {
    feature_groups_[group]->PushData(tid, sub_feature, row_idx, value);
  }

  inline int RealFeatureIndex(int fidx) const {
    return real_feature_idx_[fidx];
  }

  inline int InnerFeatureIndex(int col_idx) const {
Guolin Ke's avatar
Guolin Ke committed
348
    return used_feature_map_[col_idx];
Guolin Ke's avatar
Guolin Ke committed
349
  }
Guolin Ke's avatar
Guolin Ke committed
350
351
352
353
354
355
356
357
358
  inline int Feature2Group(int feature_idx) const {
    return feature2group_[feature_idx];
  }
  inline int Feture2SubFeature(int feature_idx) const {
    return feature2subfeature_[feature_idx];
  }
  inline uint64_t NumTotalBin() const {
    return group_bin_boundaries_.back();
  }
Guolin Ke's avatar
Guolin Ke committed
359

Guolin Ke's avatar
Guolin Ke committed
360
361
362
  void ReSize(data_size_t num_data);

  void CopySubset(const Dataset* fullset, const data_size_t* used_indices, data_size_t num_used_indices, bool need_meta_data);
Guolin Ke's avatar
Guolin Ke committed
363

364
  LIGHTGBM_EXPORT void FinishLoad();
Guolin Ke's avatar
Guolin Ke committed
365

366
  LIGHTGBM_EXPORT bool SetFloatField(const char* field_name, const float* field_data, data_size_t num_element);
Guolin Ke's avatar
Guolin Ke committed
367

368
  LIGHTGBM_EXPORT bool SetDoubleField(const char* field_name, const double* field_data, data_size_t num_element);
Guolin Ke's avatar
Guolin Ke committed
369

370
  LIGHTGBM_EXPORT bool SetIntField(const char* field_name, const int* field_data, data_size_t num_element);
371

372
  LIGHTGBM_EXPORT bool GetFloatField(const char* field_name, data_size_t* out_len, const float** out_ptr);
373

374
  LIGHTGBM_EXPORT bool GetDoubleField(const char* field_name, data_size_t* out_len, const double** out_ptr);
Guolin Ke's avatar
Guolin Ke committed
375

376
  LIGHTGBM_EXPORT bool GetIntField(const char* field_name, data_size_t* out_len, const int** out_ptr);
377

Guolin Ke's avatar
Guolin Ke committed
378
379
380
  /*!
  * \brief Save current dataset into binary file, will save to "filename.bin"
  */
381
  LIGHTGBM_EXPORT void SaveBinaryFile(const char* bin_filename);
Guolin Ke's avatar
Guolin Ke committed
382

383
  LIGHTGBM_EXPORT void CopyFeatureMapperFrom(const Dataset* dataset);
Guolin Ke's avatar
Guolin Ke committed
384

Guolin Ke's avatar
Guolin Ke committed
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
  LIGHTGBM_EXPORT void CreateValid(const Dataset* dataset);

  void ConstructHistograms(
    const std::vector<int8_t>& is_feature_used,
    const data_size_t* data_indices, data_size_t num_data,
    int leaf_idx,
    std::vector<std::unique_ptr<OrderedBin>>& ordered_bins,
    const score_t* gradients, const score_t* hessians,
    score_t* ordered_gradients, score_t* ordered_hessians,
    HistogramBinEntry* histogram_data) const;

  void FixHistogram(int feature_idx, double sum_gradient, double sum_hessian, data_size_t num_data,
    HistogramBinEntry* data) const;

  inline data_size_t Split(
    int feature,
    uint32_t threshold,
    data_size_t* data_indices, data_size_t num_data,
    data_size_t* lte_indices, data_size_t* gt_indices) const {
    const int group = feature2group_[feature];
    const int sub_feature = feature2subfeature_[feature];
    return feature_groups_[group]->Split(sub_feature, threshold, data_indices, num_data, lte_indices, gt_indices);
  }

  inline int SubFeatureBinOffset(int i) const {
    const int sub_feature = feature2subfeature_[i];
    if (sub_feature == 0) {
      return 1;
    } else {
      return 0;
    }
  }

  inline int FeatureNumBin(int i) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
	  return feature_groups_[group]->bin_mappers_[sub_feature]->num_bin();
  }
  
  inline const BinMapper* FeatureBinMapper(int i) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
    return feature_groups_[group]->bin_mappers_[sub_feature].get();
  }

  inline BinIterator* FeatureIterator(int i) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
zhangyafeikimi's avatar
zhangyafeikimi committed
433
    return feature_groups_[group]->SubFeatureIterator(sub_feature);
Guolin Ke's avatar
Guolin Ke committed
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
  }

  inline double RealThreshold(int i, uint32_t threshold) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
    return feature_groups_[group]->bin_mappers_[sub_feature]->BinToValue(threshold);
  }

  inline void CreateOrderedBins(std::vector<std::unique_ptr<OrderedBin>>* ordered_bins) const {
    ordered_bins->resize(num_groups_);
#pragma omp parallel for schedule(guided)
    for (int i = 0; i < num_groups_; ++i) {
       ordered_bins->at(i).reset(feature_groups_[i]->bin_data_->CreateOrderedBin());
    }
  }
Guolin Ke's avatar
Guolin Ke committed
449
450
451
452
453
454
455
456
457
458

  /*!
  * \brief Get meta data pointer
  * \return Pointer of meta data
  */
  inline const Metadata& metadata() const { return metadata_; }

  /*! \brief Get Number of used features */
  inline int num_features() const { return num_features_; }

459
460
461
  /*! \brief Get Number of total features */
  inline int num_total_features() const { return num_total_features_; }

Guolin Ke's avatar
Guolin Ke committed
462
463
464
465
  /*! \brief Get the index of label column */
  inline int label_idx() const { return label_idx_; }

  /*! \brief Get names of current data set */
Guolin Ke's avatar
Guolin Ke committed
466
467
468
469
470
471
472
473
474
  inline const std::vector<std::string>& feature_names() const { return feature_names_; }

  inline void set_feature_names(const std::vector<std::string>& feature_names) {
    if (feature_names.size() != static_cast<size_t>(num_total_features_)) {
      Log::Warning("size of feature_names error, should equal with total number of features");
      return;
    }
    feature_names_ = std::vector<std::string>(feature_names);
  }
Guolin Ke's avatar
Guolin Ke committed
475

Guolin Ke's avatar
Guolin Ke committed
476
477
478
479
480
481
482
483
484
485
486
487
488
489
  inline std::vector<std::string> feature_infos() const {
    std::vector<std::string> bufs;
    for (int i = 0; i < num_total_features_; i++) {
      int fidx = used_feature_map_[i];
      if (fidx == -1) {
        bufs.push_back("none");
      } else {
        const auto bin_mapper = FeatureBinMapper(fidx);
        bufs.push_back(bin_mapper->bin_info());
      }
    }
    return bufs;
  }

Guolin Ke's avatar
Guolin Ke committed
490
491
492
493
494
495
496
497
498
499
500
  /*! \brief Get Number of data */
  inline data_size_t num_data() const { return num_data_; }

  /*! \brief Disable copy */
  Dataset& operator=(const Dataset&) = delete;
  /*! \brief Disable copy */
  Dataset(const Dataset&) = delete;

private:
  const char* data_filename_;
  /*! \brief Store used features */
Guolin Ke's avatar
Guolin Ke committed
501
  std::vector<std::unique_ptr<FeatureGroup>> feature_groups_;
Guolin Ke's avatar
Guolin Ke committed
502
503
504
505
  /*! \brief Mapper from real feature index to used index*/
  std::vector<int> used_feature_map_;
  /*! \brief Number of used features*/
  int num_features_;
506
507
  /*! \brief Number of total features*/
  int num_total_features_;
Guolin Ke's avatar
Guolin Ke committed
508
509
510
511
  /*! \brief Number of total data*/
  data_size_t num_data_;
  /*! \brief Store some label level data*/
  Metadata metadata_;
Guolin Ke's avatar
Guolin Ke committed
512
513
514
515
  /*! \brief index of label column */
  int label_idx_ = 0;
  /*! \brief store feature names */
  std::vector<std::string> feature_names_;
516
517
  /*! \brief store feature names */
  static const char* binary_file_token;
Guolin Ke's avatar
Guolin Ke committed
518
519
520
521
522
523
524
  int num_groups_;
  std::vector<int> real_feature_idx_;
  std::vector<int> feature2group_;
  std::vector<int> feature2subfeature_;
  std::vector<uint64_t> group_bin_boundaries_;
  std::vector<int> group_feature_start_;
  std::vector<int> group_feature_cnt_;
Guolin Ke's avatar
Guolin Ke committed
525
  bool is_finish_load_;
Guolin Ke's avatar
Guolin Ke committed
526
527
528
529
};

}  // namespace LightGBM

Guolin Ke's avatar
Guolin Ke committed
530
#endif   // LightGBM_DATA_H_