dataset.h 24.3 KB
Newer Older
1
2
3
4
/*!
 * Copyright (c) 2016 Microsoft Corporation. All rights reserved.
 * Licensed under the MIT License. See LICENSE file in the project root for license information.
 */
Guolin Ke's avatar
Guolin Ke committed
5
6
#ifndef LIGHTGBM_DATASET_H_
#define LIGHTGBM_DATASET_H_
Guolin Ke's avatar
Guolin Ke committed
7

8
9
10
#include <LightGBM/config.h>
#include <LightGBM/feature_group.h>
#include <LightGBM/meta.h>
11
#include <LightGBM/train_share_states.h>
12
13
14
15
#include <LightGBM/utils/openmp_wrapper.h>
#include <LightGBM/utils/random.h>
#include <LightGBM/utils/text_reader.h>

Guolin Ke's avatar
Guolin Ke committed
16
#include <string>
17
18
#include <functional>
#include <memory>
19
#include <mutex>
20
21
22
#include <unordered_set>
#include <utility>
#include <vector>
Guolin Ke's avatar
Guolin Ke committed
23
24
25
26

namespace LightGBM {

/*! \brief forward declaration */
Guolin Ke's avatar
Guolin Ke committed
27
class DatasetLoader;
Guolin Ke's avatar
Guolin Ke committed
28
/*!
Hui Xue's avatar
Hui Xue committed
29
* \brief This class is used to store some meta(non-feature) data for training data,
Andrew Ziem's avatar
Andrew Ziem committed
30
*        e.g. labels, weights, initial scores, query level information.
Guolin Ke's avatar
Guolin Ke committed
31
*
Qiwei Ye's avatar
Qiwei Ye committed
32
*        Some details:
33
*        1. Label, used for training.
Qiwei Ye's avatar
Qiwei Ye committed
34
*        2. Weights, weighs of records, optional
Andrew Ziem's avatar
Andrew Ziem committed
35
*        3. Query Boundaries, necessary for LambdaRank.
36
37
38
39
*           The documents of i-th query is in [ query_boundaries[i], query_boundaries[i+1] )
*        4. Query Weights, auto calculate by weights and query_boundaries(if both of them are existed)
*           the weight for i-th query is sum(query_boundaries[i] , .., query_boundaries[i+1]) / (query_boundaries[i + 1] -  query_boundaries[i+1])
*        5. Initial score. optional. if existing, the model will boost from this score, otherwise will start from 0.
Guolin Ke's avatar
Guolin Ke committed
40
41
*/
class Metadata {
Nikita Titov's avatar
Nikita Titov committed
42
 public:
43
  /*!
44
  * \brief Null constructor
Guolin Ke's avatar
Guolin Ke committed
45
46
47
  */
  Metadata();
  /*!
Andrew Ziem's avatar
Andrew Ziem committed
48
  * \brief Initialization will load query level information, since it is need for sampling data
Guolin Ke's avatar
Guolin Ke committed
49
50
  * \param data_filename Filename of data
  */
51
  void Init(const char* data_filename);
Guolin Ke's avatar
Guolin Ke committed
52
  /*!
Guolin Ke's avatar
Guolin Ke committed
53
54
  * \brief init as subset
  * \param metadata Filename of data
55
  * \param used_indices
Guolin Ke's avatar
Guolin Ke committed
56
57
58
59
  * \param num_used_indices
  */
  void Init(const Metadata& metadata, const data_size_t* used_indices, data_size_t num_used_indices);
  /*!
Guolin Ke's avatar
Guolin Ke committed
60
61
62
63
64
65
66
67
  * \brief Initial with binary memory
  * \param memory Pointer to memory
  */
  void LoadFromMemory(const void* memory);
  /*! \brief Destructor */
  ~Metadata();

  /*!
Guolin Ke's avatar
Guolin Ke committed
68
  * \brief Initial work, will allocate space for label, weight(if exists) and query(if exists)
Guolin Ke's avatar
Guolin Ke committed
69
  * \param num_data Number of training data
Guolin Ke's avatar
Guolin Ke committed
70
71
  * \param weight_idx Index of weight column, < 0 means doesn't exists
  * \param query_idx Index of query id column, < 0 means doesn't exists
Guolin Ke's avatar
Guolin Ke committed
72
  */
73
  void Init(data_size_t num_data, int weight_idx, int query_idx);
Guolin Ke's avatar
Guolin Ke committed
74
75
76

  /*!
  * \brief Partition label by used indices
77
  * \param used_indices Indices of local used
Guolin Ke's avatar
Guolin Ke committed
78
79
80
81
82
  */
  void PartitionLabel(const std::vector<data_size_t>& used_indices);

  /*!
  * \brief Partition meta data according to local used indices if need
83
  * \param num_all_data Number of total training data, including other machines' data on distributed learning
Guolin Ke's avatar
Guolin Ke committed
84
85
86
  * \param used_data_indices Indices of local used training data
  */
  void CheckOrPartition(data_size_t num_all_data,
87
                        const std::vector<data_size_t>& used_data_indices);
Guolin Ke's avatar
Guolin Ke committed
88

89
  void SetLabel(const label_t* label, data_size_t len);
Guolin Ke's avatar
Guolin Ke committed
90

91
  void SetWeights(const label_t* weights, data_size_t len);
Guolin Ke's avatar
Guolin Ke committed
92

Guolin Ke's avatar
Guolin Ke committed
93
  void SetQuery(const data_size_t* query, data_size_t len);
Guolin Ke's avatar
Guolin Ke committed
94

Guolin Ke's avatar
Guolin Ke committed
95
96
97
98
  /*!
  * \brief Set initial scores
  * \param init_score Initial scores, this class will manage memory for init_score.
  */
99
  void SetInitScore(const double* init_score, data_size_t len);
Guolin Ke's avatar
Guolin Ke committed
100
101
102
103
104
105


  /*!
  * \brief Save binary data to file
  * \param file File want to write
  */
106
  void SaveBinaryToFile(const VirtualFileWriter* writer) const;
Guolin Ke's avatar
Guolin Ke committed
107
108
109
110
111
112
113
114
115
116

  /*!
  * \brief Get sizes in byte of this object
  */
  size_t SizesInByte() const;

  /*!
  * \brief Get pointer of label
  * \return Pointer of label
  */
117
  inline const label_t* label() const { return label_.data(); }
Guolin Ke's avatar
Guolin Ke committed
118
119
120
121
122
123

  /*!
  * \brief Set label for one record
  * \param idx Index of this record
  * \param value Label value of this record
  */
124
  inline void SetLabelAt(data_size_t idx, label_t value) {
125
    label_[idx] = value;
Guolin Ke's avatar
Guolin Ke committed
126
127
  }

Guolin Ke's avatar
Guolin Ke committed
128
129
130
131
132
  /*!
  * \brief Set Weight for one record
  * \param idx Index of this record
  * \param value Weight value of this record
  */
133
  inline void SetWeightAt(data_size_t idx, label_t value) {
134
    weights_[idx] = value;
Guolin Ke's avatar
Guolin Ke committed
135
136
137
138
139
140
141
  }

  /*!
  * \brief Set Query Id for one record
  * \param idx Index of this record
  * \param value Query Id value of this record
  */
142
  inline void SetQueryAt(data_size_t idx, data_size_t value) {
Guolin Ke's avatar
Guolin Ke committed
143
144
145
    queries_[idx] = static_cast<data_size_t>(value);
  }

Guolin Ke's avatar
Guolin Ke committed
146
  /*!
Hui Xue's avatar
Hui Xue committed
147
  * \brief Get weights, if not exists, will return nullptr
Guolin Ke's avatar
Guolin Ke committed
148
149
  * \return Pointer of weights
  */
150
  inline const label_t* weights() const {
Guolin Ke's avatar
Guolin Ke committed
151
    if (!weights_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
152
153
154
155
156
      return weights_.data();
    } else {
      return nullptr;
    }
  }
Guolin Ke's avatar
Guolin Ke committed
157
158

  /*!
Hui Xue's avatar
Hui Xue committed
159
  * \brief Get data boundaries on queries, if not exists, will return nullptr
160
  *        we assume data will order by query,
Guolin Ke's avatar
Guolin Ke committed
161
162
163
164
  *        the interval of [query_boundaris[i], query_boundaris[i+1])
  *        is the data indices for query i.
  * \return Pointer of data boundaries on queries
  */
165
  inline const data_size_t* query_boundaries() const {
Guolin Ke's avatar
Guolin Ke committed
166
    if (!query_boundaries_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
167
168
169
170
171
      return query_boundaries_.data();
    } else {
      return nullptr;
    }
  }
Guolin Ke's avatar
Guolin Ke committed
172
173
174
175
176

  /*!
  * \brief Get Number of queries
  * \return Number of queries
  */
177
  inline data_size_t num_queries() const { return num_queries_; }
Guolin Ke's avatar
Guolin Ke committed
178
179

  /*!
Hui Xue's avatar
Hui Xue committed
180
  * \brief Get weights for queries, if not exists, will return nullptr
Guolin Ke's avatar
Guolin Ke committed
181
182
  * \return Pointer of weights for queries
  */
183
  inline const label_t* query_weights() const {
Guolin Ke's avatar
Guolin Ke committed
184
    if (!query_weights_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
185
186
187
188
189
      return query_weights_.data();
    } else {
      return nullptr;
    }
  }
Guolin Ke's avatar
Guolin Ke committed
190
191

  /*!
Hui Xue's avatar
Hui Xue committed
192
  * \brief Get initial scores, if not exists, will return nullptr
Guolin Ke's avatar
Guolin Ke committed
193
194
  * \return Pointer of initial scores
  */
195
  inline const double* init_score() const {
Guolin Ke's avatar
Guolin Ke committed
196
    if (!init_score_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
197
198
199
200
201
      return init_score_.data();
    } else {
      return nullptr;
    }
  }
Guolin Ke's avatar
Guolin Ke committed
202

203
204
205
  /*!
  * \brief Get size of initial scores
  */
Guolin Ke's avatar
Guolin Ke committed
206
  inline int64_t num_init_score() const { return num_init_score_; }
207

Guolin Ke's avatar
Guolin Ke committed
208
209
210
211
  /*! \brief Disable copy */
  Metadata& operator=(const Metadata&) = delete;
  /*! \brief Disable copy */
  Metadata(const Metadata&) = delete;
Guolin Ke's avatar
Guolin Ke committed
212

Nikita Titov's avatar
Nikita Titov committed
213
 private:
Guolin Ke's avatar
Guolin Ke committed
214
  /*! \brief Load initial scores from file */
215
  void LoadInitialScore();
Guolin Ke's avatar
Guolin Ke committed
216
217
218
219
220
221
222
  /*! \brief Load wights from file */
  void LoadWeights();
  /*! \brief Load query boundaries from file */
  void LoadQueryBoundaries();
  /*! \brief Load query wights */
  void LoadQueryWeights();
  /*! \brief Filename of current data */
Guolin Ke's avatar
Guolin Ke committed
223
  std::string data_filename_;
Guolin Ke's avatar
Guolin Ke committed
224
225
226
227
228
  /*! \brief Number of data */
  data_size_t num_data_;
  /*! \brief Number of weights, used to check correct weight file */
  data_size_t num_weights_;
  /*! \brief Label data */
229
  std::vector<label_t> label_;
Guolin Ke's avatar
Guolin Ke committed
230
  /*! \brief Weights data */
231
  std::vector<label_t> weights_;
Guolin Ke's avatar
Guolin Ke committed
232
  /*! \brief Query boundaries */
Guolin Ke's avatar
Guolin Ke committed
233
  std::vector<data_size_t> query_boundaries_;
Guolin Ke's avatar
Guolin Ke committed
234
  /*! \brief Query weights */
235
  std::vector<label_t> query_weights_;
Guolin Ke's avatar
Guolin Ke committed
236
237
238
  /*! \brief Number of querys */
  data_size_t num_queries_;
  /*! \brief Number of Initial score, used to check correct weight file */
Guolin Ke's avatar
Guolin Ke committed
239
  int64_t num_init_score_;
Guolin Ke's avatar
Guolin Ke committed
240
  /*! \brief Initial score */
Guolin Ke's avatar
Guolin Ke committed
241
  std::vector<double> init_score_;
Guolin Ke's avatar
Guolin Ke committed
242
  /*! \brief Queries data */
Guolin Ke's avatar
Guolin Ke committed
243
  std::vector<data_size_t> queries_;
244
245
  /*! \brief mutex for threading safe call */
  std::mutex mutex_;
246
247
248
  bool weight_load_from_file_;
  bool query_load_from_file_;
  bool init_score_load_from_file_;
Guolin Ke's avatar
Guolin Ke committed
249
250
251
252
253
};


/*! \brief Interface for Parser */
class Parser {
Nikita Titov's avatar
Nikita Titov committed
254
 public:
Chen Yufei's avatar
Chen Yufei committed
255
256
  typedef const char* (*AtofFunc)(const char* p, double* out);

Guolin Ke's avatar
Guolin Ke committed
257
258
259
260
261
262
  /*! \brief virtual destructor */
  virtual ~Parser() {}

  /*!
  * \brief Parse one line with label
  * \param str One line record, string format, should end with '\0'
Guolin Ke's avatar
Guolin Ke committed
263
264
  * \param out_features Output columns, store in (column_idx, values)
  * \param out_label Label will store to this if exists
Guolin Ke's avatar
Guolin Ke committed
265
266
  */
  virtual void ParseOneLine(const char* str,
267
                            std::vector<std::pair<int, double>>* out_features, double* out_label) const = 0;
Guolin Ke's avatar
Guolin Ke committed
268

269
  virtual int NumFeatures() const = 0;
Guolin Ke's avatar
Guolin Ke committed
270

Guolin Ke's avatar
Guolin Ke committed
271
  /*!
272
  * \brief Create an object of parser, will auto choose the format depend on file
Guolin Ke's avatar
Guolin Ke committed
273
  * \param filename One Filename of data
274
  * \param num_features Pass num_features of this data file if you know, <=0 means don't know
Guolin Ke's avatar
Guolin Ke committed
275
  * \param label_idx index of label column
Chen Yufei's avatar
Chen Yufei committed
276
  * \param precise_float_parser using precise floating point number parsing if true
Guolin Ke's avatar
Guolin Ke committed
277
278
  * \return Object of parser
  */
Chen Yufei's avatar
Chen Yufei committed
279
  static Parser* CreateParser(const char* filename, bool header, int num_features, int label_idx, bool precise_float_parser);
Guolin Ke's avatar
Guolin Ke committed
280
281
282
};

/*! \brief The main class of data set,
283
*          which are used to training or validation
Guolin Ke's avatar
Guolin Ke committed
284
285
*/
class Dataset {
Nikita Titov's avatar
Nikita Titov committed
286
 public:
Guolin Ke's avatar
Guolin Ke committed
287
  friend DatasetLoader;
Guolin Ke's avatar
Guolin Ke committed
288

289
  LIGHTGBM_EXPORT Dataset();
Guolin Ke's avatar
Guolin Ke committed
290

291
  LIGHTGBM_EXPORT Dataset(data_size_t num_data);
Guolin Ke's avatar
Guolin Ke committed
292

Guolin Ke's avatar
Guolin Ke committed
293
  void Construct(
Guolin Ke's avatar
Guolin Ke committed
294
    std::vector<std::unique_ptr<BinMapper>>* bin_mappers,
295
    int num_total_features,
296
    const std::vector<std::vector<double>>& forced_bins,
297
    int** sample_non_zero_indices,
Guolin Ke's avatar
Guolin Ke committed
298
    double** sample_values,
299
    const int* num_per_col,
300
    int num_sample_col,
Guolin Ke's avatar
Guolin Ke committed
301
    size_t total_sample_cnt,
Guolin Ke's avatar
Guolin Ke committed
302
    const Config& io_config);
Guolin Ke's avatar
Guolin Ke committed
303

Guolin Ke's avatar
Guolin Ke committed
304
  /*! \brief Destructor */
305
  LIGHTGBM_EXPORT ~Dataset();
Guolin Ke's avatar
Guolin Ke committed
306

307
  LIGHTGBM_EXPORT bool CheckAlign(const Dataset& other) const {
308
309
310
311
312
313
314
315
316
317
    if (num_features_ != other.num_features_) {
      return false;
    }
    if (num_total_features_ != other.num_total_features_) {
      return false;
    }
    if (label_idx_ != other.label_idx_) {
      return false;
    }
    for (int i = 0; i < num_features_; ++i) {
Guolin Ke's avatar
Guolin Ke committed
318
      if (!FeatureBinMapper(i)->CheckAlign(*(other.FeatureBinMapper(i)))) {
319
320
321
322
323
324
        return false;
      }
    }
    return true;
  }

Guolin Ke's avatar
Guolin Ke committed
325
326
327
328
329
330
331
332
333
334
  inline void FinishOneRow(int tid, data_size_t row_idx, const std::vector<bool>& is_feature_added) {
    if (is_finish_load_) { return; }
    for (auto fidx : feature_need_push_zeros_) {
      if (is_feature_added[fidx]) { continue; }
      const int group = feature2group_[fidx];
      const int sub_feature = feature2subfeature_[fidx];
      feature_groups_[group]->PushData(tid, sub_feature, row_idx, 0.0f);
    }
  }

Guolin Ke's avatar
Guolin Ke committed
335
  inline void PushOneRow(int tid, data_size_t row_idx, const std::vector<double>& feature_values) {
Guolin Ke's avatar
Guolin Ke committed
336
    if (is_finish_load_) { return; }
Guolin Ke's avatar
Guolin Ke committed
337
    for (size_t i = 0; i < feature_values.size() && i < static_cast<size_t>(num_total_features_); ++i) {
Guolin Ke's avatar
Guolin Ke committed
338
339
      int feature_idx = used_feature_map_[i];
      if (feature_idx >= 0) {
Guolin Ke's avatar
Guolin Ke committed
340
341
342
        const int group = feature2group_[feature_idx];
        const int sub_feature = feature2subfeature_[feature_idx];
        feature_groups_[group]->PushData(tid, sub_feature, row_idx, feature_values[i]);
343
344
345
        if (has_raw_) {
          int feat_ind = numeric_feature_map_[feature_idx];
          if (feat_ind >= 0) {
sisco0's avatar
sisco0 committed
346
            raw_data_[feat_ind][row_idx] = static_cast<float>(feature_values[i]);
347
348
          }
        }
Guolin Ke's avatar
Guolin Ke committed
349
350
351
352
      }
    }
  }

353
  inline void PushOneRow(int tid, data_size_t row_idx, const std::vector<std::pair<int, double>>& feature_values) {
Guolin Ke's avatar
Guolin Ke committed
354
    if (is_finish_load_) { return; }
Guolin Ke's avatar
Guolin Ke committed
355
    std::vector<bool> is_feature_added(num_features_, false);
356
    for (auto& inner_data : feature_values) {
357
      if (inner_data.first >= num_total_features_) { continue; }
358
359
      int feature_idx = used_feature_map_[inner_data.first];
      if (feature_idx >= 0) {
Guolin Ke's avatar
Guolin Ke committed
360
        is_feature_added[feature_idx] = true;
Guolin Ke's avatar
Guolin Ke committed
361
362
363
        const int group = feature2group_[feature_idx];
        const int sub_feature = feature2subfeature_[feature_idx];
        feature_groups_[group]->PushData(tid, sub_feature, row_idx, inner_data.second);
364
365
366
        if (has_raw_) {
          int feat_ind = numeric_feature_map_[feature_idx];
          if (feat_ind >= 0) {
367
            raw_data_[feat_ind][row_idx] = static_cast<float>(inner_data.second);
368
369
          }
        }
370
371
      }
    }
Guolin Ke's avatar
Guolin Ke committed
372
    FinishOneRow(tid, row_idx, is_feature_added);
373
374
  }

375
  inline void PushOneData(int tid, data_size_t row_idx, int group, int feature_idx, int sub_feature, double value) {
Guolin Ke's avatar
Guolin Ke committed
376
    feature_groups_[group]->PushData(tid, sub_feature, row_idx, value);
377
378
379
    if (has_raw_) {
      int feat_ind = numeric_feature_map_[feature_idx];
      if (feat_ind >= 0) {
380
        raw_data_[feat_ind][row_idx] = static_cast<float>(value);
381
382
      }
    }
Guolin Ke's avatar
Guolin Ke committed
383
384
385
386
387
388
389
  }

  inline int RealFeatureIndex(int fidx) const {
    return real_feature_idx_[fidx];
  }

  inline int InnerFeatureIndex(int col_idx) const {
Guolin Ke's avatar
Guolin Ke committed
390
    return used_feature_map_[col_idx];
Guolin Ke's avatar
Guolin Ke committed
391
  }
Guolin Ke's avatar
Guolin Ke committed
392
393
394
395
396
397
  inline int Feature2Group(int feature_idx) const {
    return feature2group_[feature_idx];
  }
  inline int Feture2SubFeature(int feature_idx) const {
    return feature2subfeature_[feature_idx];
  }
398
399
400
  inline uint64_t GroupBinBoundary(int group_idx) const {
    return group_bin_boundaries_[group_idx];
  }
Guolin Ke's avatar
Guolin Ke committed
401
402
403
  inline uint64_t NumTotalBin() const {
    return group_bin_boundaries_.back();
  }
404

405
406
407
408
409
410
411
412
413
  inline std::vector<int> ValidFeatureIndices() const {
    std::vector<int> ret;
    for (int i = 0; i < num_total_features_; ++i) {
      if (used_feature_map_[i] >= 0) {
        ret.push_back(i);
      }
    }
    return ret;
  }
Guolin Ke's avatar
Guolin Ke committed
414
415
  void ReSize(data_size_t num_data);

416
  void CopySubrow(const Dataset* fullset, const data_size_t* used_indices, data_size_t num_used_indices, bool need_meta_data);
Guolin Ke's avatar
Guolin Ke committed
417

418
  MultiValBin* GetMultiBinFromSparseFeatures(const std::vector<uint32_t>& offsets) const;
419

420
  MultiValBin* GetMultiBinFromAllFeatures(const std::vector<uint32_t>& offsets) const;
421

422
423
424
  TrainingShareStates* GetShareStates(
      score_t* gradients, score_t* hessians,
      const std::vector<int8_t>& is_feature_used, bool is_constant_hessian,
425
      bool force_col_wise, bool force_row_wise) const;
426

427
  LIGHTGBM_EXPORT void FinishLoad();
Guolin Ke's avatar
Guolin Ke committed
428

429
  LIGHTGBM_EXPORT bool SetFloatField(const char* field_name, const float* field_data, data_size_t num_element);
Guolin Ke's avatar
Guolin Ke committed
430

431
  LIGHTGBM_EXPORT bool SetDoubleField(const char* field_name, const double* field_data, data_size_t num_element);
Guolin Ke's avatar
Guolin Ke committed
432

433
  LIGHTGBM_EXPORT bool SetIntField(const char* field_name, const int* field_data, data_size_t num_element);
434

435
  LIGHTGBM_EXPORT bool GetFloatField(const char* field_name, data_size_t* out_len, const float** out_ptr);
436

437
  LIGHTGBM_EXPORT bool GetDoubleField(const char* field_name, data_size_t* out_len, const double** out_ptr);
Guolin Ke's avatar
Guolin Ke committed
438

439
  LIGHTGBM_EXPORT bool GetIntField(const char* field_name, data_size_t* out_len, const int** out_ptr);
440

Guolin Ke's avatar
Guolin Ke committed
441
442
443
  /*!
  * \brief Save current dataset into binary file, will save to "filename.bin"
  */
444
  LIGHTGBM_EXPORT void SaveBinaryFile(const char* bin_filename);
Guolin Ke's avatar
Guolin Ke committed
445

446
447
  LIGHTGBM_EXPORT void DumpTextFile(const char* text_filename);

448
  LIGHTGBM_EXPORT void CopyFeatureMapperFrom(const Dataset* dataset);
Guolin Ke's avatar
Guolin Ke committed
449

Guolin Ke's avatar
Guolin Ke committed
450
451
  LIGHTGBM_EXPORT void CreateValid(const Dataset* dataset);

452
  void InitTrain(const std::vector<int8_t>& is_feature_used,
453
                 TrainingShareStates* share_state) const;
454

Guolin Ke's avatar
Guolin Ke committed
455
456
457
458
459
460
461
462
463
464
465
  template <bool USE_INDICES, bool USE_HESSIAN>
  void ConstructHistogramsInner(const std::vector<int8_t>& is_feature_used,
                                const data_size_t* data_indices,
                                data_size_t num_data, const score_t* gradients,
                                const score_t* hessians,
                                score_t* ordered_gradients,
                                score_t* ordered_hessians,
                                TrainingShareStates* share_state,
                                hist_t* hist_data) const;

  template <bool USE_INDICES, bool ORDERED>
466
467
468
469
  void ConstructHistogramsMultiVal(const data_size_t* data_indices,
                                   data_size_t num_data,
                                   const score_t* gradients,
                                   const score_t* hessians,
470
                                   TrainingShareStates* share_state,
Guolin Ke's avatar
Guolin Ke committed
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
                                   hist_t* hist_data) const;

  inline void ConstructHistograms(
      const std::vector<int8_t>& is_feature_used,
      const data_size_t* data_indices, data_size_t num_data,
      const score_t* gradients, const score_t* hessians,
      score_t* ordered_gradients, score_t* ordered_hessians,
      TrainingShareStates* share_state, hist_t* hist_data) const {
    if (num_data <= 0) {
      return;
    }
    bool use_indices = data_indices != nullptr && (num_data < num_data_);
    if (share_state->is_constant_hessian) {
      if (use_indices) {
        ConstructHistogramsInner<true, false>(
            is_feature_used, data_indices, num_data, gradients, hessians,
            ordered_gradients, ordered_hessians, share_state, hist_data);
      } else {
        ConstructHistogramsInner<false, false>(
            is_feature_used, data_indices, num_data, gradients, hessians,
            ordered_gradients, ordered_hessians, share_state, hist_data);
      }
    } else {
      if (use_indices) {
        ConstructHistogramsInner<true, true>(
            is_feature_used, data_indices, num_data, gradients, hessians,
            ordered_gradients, ordered_hessians, share_state, hist_data);
      } else {
        ConstructHistogramsInner<false, true>(
            is_feature_used, data_indices, num_data, gradients, hessians,
            ordered_gradients, ordered_hessians, share_state, hist_data);
      }
    }
  }
Guolin Ke's avatar
Guolin Ke committed
505

506
  void FixHistogram(int feature_idx, double sum_gradient, double sum_hessian, hist_t* data) const;
Guolin Ke's avatar
Guolin Ke committed
507

508
509
  inline data_size_t Split(int feature, const uint32_t* threshold,
                           int num_threshold, bool default_left,
510
                           const data_size_t* data_indices,
511
512
                           data_size_t cnt, data_size_t* lte_indices,
                           data_size_t* gt_indices) const {
Guolin Ke's avatar
Guolin Ke committed
513
514
    const int group = feature2group_[feature];
    const int sub_feature = feature2subfeature_[feature];
515
516
517
    return feature_groups_[group]->Split(
        sub_feature, threshold, num_threshold, default_left, data_indices,
        cnt, lte_indices, gt_indices);
Guolin Ke's avatar
Guolin Ke committed
518
519
520
521
522
523
524
525
526
527
528
529
530
531
  }

  inline int SubFeatureBinOffset(int i) const {
    const int sub_feature = feature2subfeature_[i];
    if (sub_feature == 0) {
      return 1;
    } else {
      return 0;
    }
  }

  inline int FeatureNumBin(int i) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
532
    return feature_groups_[group]->bin_mappers_[sub_feature]->num_bin();
Guolin Ke's avatar
Guolin Ke committed
533
  }
Guolin Ke's avatar
Guolin Ke committed
534

535
536
537
  inline int FeatureGroupNumBin(int group) const {
    return feature_groups_[group]->num_total_bin_;
  }
538

Guolin Ke's avatar
Guolin Ke committed
539
540
541
542
543
544
  inline const BinMapper* FeatureBinMapper(int i) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
    return feature_groups_[group]->bin_mappers_[sub_feature].get();
  }

545
546
547
548
  inline const Bin* FeatureGroupBin(int group) const {
    return feature_groups_[group]->bin_data_.get();
  }

Guolin Ke's avatar
Guolin Ke committed
549
550
551
  inline BinIterator* FeatureIterator(int i) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
zhangyafeikimi's avatar
zhangyafeikimi committed
552
    return feature_groups_[group]->SubFeatureIterator(sub_feature);
Guolin Ke's avatar
Guolin Ke committed
553
554
  }

555
556
557
  inline BinIterator* FeatureGroupIterator(int group) const {
    return feature_groups_[group]->FeatureGroupIterator();
  }
558

559
560
561
562
  inline bool IsMultiGroup(int i) const {
    return feature_groups_[i]->is_multi_val_;
  }

563
564
565
566
567
568
569
570
  inline size_t FeatureGroupSizesInByte(int group) const {
    return feature_groups_[group]->FeatureGroupSizesInByte();
  }

  inline void* FeatureGroupData(int group) const {
    return feature_groups_[group]->FeatureGroupData();
  }

Guolin Ke's avatar
Guolin Ke committed
571
572
573
574
575
576
  inline double RealThreshold(int i, uint32_t threshold) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
    return feature_groups_[group]->bin_mappers_[sub_feature]->BinToValue(threshold);
  }

577
578
579
580
581
582
583
  // given a real threshold, find the closest threshold bin
  inline uint32_t BinThreshold(int i, double threshold_double) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
    return feature_groups_[group]->bin_mappers_[sub_feature]->ValueToBin(threshold_double);
  }

Guolin Ke's avatar
Guolin Ke committed
584
585
586
587
588
589
590
591
592
  /*!
  * \brief Get meta data pointer
  * \return Pointer of meta data
  */
  inline const Metadata& metadata() const { return metadata_; }

  /*! \brief Get Number of used features */
  inline int num_features() const { return num_features_; }

593
594
595
  /*! \brief Get number of numeric features */
  inline int num_numeric_features() const { return num_numeric_features_; }

596
597
598
  /*! \brief Get Number of feature groups */
  inline int num_feature_groups() const { return num_groups_;}

599
600
601
  /*! \brief Get Number of total features */
  inline int num_total_features() const { return num_total_features_; }

Guolin Ke's avatar
Guolin Ke committed
602
603
604
605
  /*! \brief Get the index of label column */
  inline int label_idx() const { return label_idx_; }

  /*! \brief Get names of current data set */
Guolin Ke's avatar
Guolin Ke committed
606
607
608
609
  inline const std::vector<std::string>& feature_names() const { return feature_names_; }

  inline void set_feature_names(const std::vector<std::string>& feature_names) {
    if (feature_names.size() != static_cast<size_t>(num_total_features_)) {
610
      Log::Fatal("Size of feature_names error, should equal with total number of features");
Guolin Ke's avatar
Guolin Ke committed
611
612
    }
    feature_names_ = std::vector<std::string>(feature_names);
Guolin Ke's avatar
Guolin Ke committed
613
    std::unordered_set<std::string> feature_name_set;
614
615
    // replace ' ' in feature_names with '_'
    bool spaceInFeatureName = false;
616
    for (auto& feature_name : feature_names_) {
Andrew Ziem's avatar
Andrew Ziem committed
617
      // check JSON
618
619
      if (!Common::CheckAllowedJSON(feature_name)) {
        Log::Fatal("Do not support special JSON characters in feature name.");
620
      }
621
      if (feature_name.find(' ') != std::string::npos) {
622
623
624
        spaceInFeatureName = true;
        std::replace(feature_name.begin(), feature_name.end(), ' ', '_');
      }
Guolin Ke's avatar
Guolin Ke committed
625
626
627
628
      if (feature_name_set.count(feature_name) > 0) {
        Log::Fatal("Feature (%s) appears more than one time.", feature_name.c_str());
      }
      feature_name_set.insert(feature_name);
629
    }
630
    if (spaceInFeatureName) {
Andrew Ziem's avatar
Andrew Ziem committed
631
      Log::Warning("Found whitespace in feature_names, replace with underlines");
632
    }
Guolin Ke's avatar
Guolin Ke committed
633
  }
Guolin Ke's avatar
Guolin Ke committed
634

Guolin Ke's avatar
Guolin Ke committed
635
636
  inline std::vector<std::string> feature_infos() const {
    std::vector<std::string> bufs;
637
    for (int i = 0; i < num_total_features_; ++i) {
Guolin Ke's avatar
Guolin Ke committed
638
      int fidx = used_feature_map_[i];
639
      if (fidx < 0) {
Guolin Ke's avatar
Guolin Ke committed
640
641
642
        bufs.push_back("none");
      } else {
        const auto bin_mapper = FeatureBinMapper(fidx);
643
        bufs.push_back(bin_mapper->bin_info_string());
Guolin Ke's avatar
Guolin Ke committed
644
645
646
647
648
      }
    }
    return bufs;
  }

Guolin Ke's avatar
Guolin Ke committed
649
650
651
652
653
654
655
656
  /*! \brief Get Number of data */
  inline data_size_t num_data() const { return num_data_; }

  /*! \brief Disable copy */
  Dataset& operator=(const Dataset&) = delete;
  /*! \brief Disable copy */
  Dataset(const Dataset&) = delete;

657
  void AddFeaturesFrom(Dataset* other);
658

659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
  /*! \brief Get has_raw_ */
  inline bool has_raw() const { return has_raw_; }

  /*! \brief Set has_raw_ */
  inline void SetHasRaw(bool has_raw) { has_raw_ = has_raw; }

  /*! \brief Resize raw_data_ */
  inline void ResizeRaw(int num_rows) {
    if (static_cast<int>(raw_data_.size()) > num_numeric_features_) {
      raw_data_.resize(num_numeric_features_);
    }
    for (size_t i = 0; i < raw_data_.size(); ++i) {
      raw_data_[i].resize(num_rows);
    }
    int curr_size = static_cast<int>(raw_data_.size());
    for (int i = curr_size; i < num_numeric_features_; ++i) {
      raw_data_.push_back(std::vector<float>(num_rows, 0));
    }
  }

  /*! \brief Get pointer to raw_data_ feature */
  inline const float* raw_index(int feat_ind) const {
    return raw_data_[numeric_feature_map_[feat_ind]].data();
  }

Nikita Titov's avatar
Nikita Titov committed
684
 private:
Guolin Ke's avatar
Guolin Ke committed
685
  std::string data_filename_;
Guolin Ke's avatar
Guolin Ke committed
686
  /*! \brief Store used features */
Guolin Ke's avatar
Guolin Ke committed
687
  std::vector<std::unique_ptr<FeatureGroup>> feature_groups_;
Guolin Ke's avatar
Guolin Ke committed
688
689
690
691
  /*! \brief Mapper from real feature index to used index*/
  std::vector<int> used_feature_map_;
  /*! \brief Number of used features*/
  int num_features_;
692
693
  /*! \brief Number of total features*/
  int num_total_features_;
Guolin Ke's avatar
Guolin Ke committed
694
695
696
697
  /*! \brief Number of total data*/
  data_size_t num_data_;
  /*! \brief Store some label level data*/
  Metadata metadata_;
Guolin Ke's avatar
Guolin Ke committed
698
699
700
701
  /*! \brief index of label column */
  int label_idx_ = 0;
  /*! \brief store feature names */
  std::vector<std::string> feature_names_;
702
703
  /*! \brief store feature names */
  static const char* binary_file_token;
Guolin Ke's avatar
Guolin Ke committed
704
705
706
707
708
709
710
  int num_groups_;
  std::vector<int> real_feature_idx_;
  std::vector<int> feature2group_;
  std::vector<int> feature2subfeature_;
  std::vector<uint64_t> group_bin_boundaries_;
  std::vector<int> group_feature_start_;
  std::vector<int> group_feature_cnt_;
Guolin Ke's avatar
Guolin Ke committed
711
  bool is_finish_load_;
712
  int max_bin_;
Belinda Trotta's avatar
Belinda Trotta committed
713
  std::vector<int32_t> max_bin_by_feature_;
714
  std::vector<std::vector<double>> forced_bin_bounds_;
715
716
717
718
  int bin_construct_sample_cnt_;
  int min_data_in_bin_;
  bool use_missing_;
  bool zero_as_missing_;
Guolin Ke's avatar
Guolin Ke committed
719
  std::vector<int> feature_need_push_zeros_;
720
721
722
723
724
  std::vector<std::vector<float>> raw_data_;
  bool has_raw_;
  /*! map feature (inner index) to its index in the list of numeric (non-categorical) features */
  std::vector<int> numeric_feature_map_;
  int num_numeric_features_;
Guolin Ke's avatar
Guolin Ke committed
725
726
727
728
};

}  // namespace LightGBM

Guolin Ke's avatar
Guolin Ke committed
729
#endif   // LightGBM_DATA_H_