dataset.h 26.7 KB
Newer Older
1
2
3
4
/*!
 * Copyright (c) 2016 Microsoft Corporation. All rights reserved.
 * Licensed under the MIT License. See LICENSE file in the project root for license information.
 */
Guolin Ke's avatar
Guolin Ke committed
5
6
#ifndef LIGHTGBM_DATASET_H_
#define LIGHTGBM_DATASET_H_
Guolin Ke's avatar
Guolin Ke committed
7

8
9
10
#include <LightGBM/config.h>
#include <LightGBM/feature_group.h>
#include <LightGBM/meta.h>
11
#include <LightGBM/train_share_states.h>
12
13
14
15
#include <LightGBM/utils/openmp_wrapper.h>
#include <LightGBM/utils/random.h>
#include <LightGBM/utils/text_reader.h>

Guolin Ke's avatar
Guolin Ke committed
16
#include <string>
17
#include <functional>
18
#include <map>
19
#include <memory>
20
#include <mutex>
21
22
23
#include <unordered_set>
#include <utility>
#include <vector>
Guolin Ke's avatar
Guolin Ke committed
24
25
26
27

namespace LightGBM {

/*! \brief forward declaration */
Guolin Ke's avatar
Guolin Ke committed
28
class DatasetLoader;
Guolin Ke's avatar
Guolin Ke committed
29
/*!
Hui Xue's avatar
Hui Xue committed
30
* \brief This class is used to store some meta(non-feature) data for training data,
Andrew Ziem's avatar
Andrew Ziem committed
31
*        e.g. labels, weights, initial scores, query level information.
Guolin Ke's avatar
Guolin Ke committed
32
*
Qiwei Ye's avatar
Qiwei Ye committed
33
*        Some details:
34
*        1. Label, used for training.
Qiwei Ye's avatar
Qiwei Ye committed
35
*        2. Weights, weighs of records, optional
Andrew Ziem's avatar
Andrew Ziem committed
36
*        3. Query Boundaries, necessary for LambdaRank.
37
38
39
40
*           The documents of i-th query is in [ query_boundaries[i], query_boundaries[i+1] )
*        4. Query Weights, auto calculate by weights and query_boundaries(if both of them are existed)
*           the weight for i-th query is sum(query_boundaries[i] , .., query_boundaries[i+1]) / (query_boundaries[i + 1] -  query_boundaries[i+1])
*        5. Initial score. optional. if existing, the model will boost from this score, otherwise will start from 0.
Guolin Ke's avatar
Guolin Ke committed
41
42
*/
class Metadata {
Nikita Titov's avatar
Nikita Titov committed
43
 public:
44
  /*!
45
  * \brief Null constructor
Guolin Ke's avatar
Guolin Ke committed
46
47
48
  */
  Metadata();
  /*!
Andrew Ziem's avatar
Andrew Ziem committed
49
  * \brief Initialization will load query level information, since it is need for sampling data
Guolin Ke's avatar
Guolin Ke committed
50
51
  * \param data_filename Filename of data
  */
52
  void Init(const char* data_filename);
Guolin Ke's avatar
Guolin Ke committed
53
  /*!
Guolin Ke's avatar
Guolin Ke committed
54
55
  * \brief init as subset
  * \param metadata Filename of data
56
  * \param used_indices
Guolin Ke's avatar
Guolin Ke committed
57
58
59
60
  * \param num_used_indices
  */
  void Init(const Metadata& metadata, const data_size_t* used_indices, data_size_t num_used_indices);
  /*!
Guolin Ke's avatar
Guolin Ke committed
61
62
63
64
65
66
67
68
  * \brief Initial with binary memory
  * \param memory Pointer to memory
  */
  void LoadFromMemory(const void* memory);
  /*! \brief Destructor */
  ~Metadata();

  /*!
Guolin Ke's avatar
Guolin Ke committed
69
  * \brief Initial work, will allocate space for label, weight(if exists) and query(if exists)
Guolin Ke's avatar
Guolin Ke committed
70
  * \param num_data Number of training data
Guolin Ke's avatar
Guolin Ke committed
71
72
  * \param weight_idx Index of weight column, < 0 means doesn't exists
  * \param query_idx Index of query id column, < 0 means doesn't exists
Guolin Ke's avatar
Guolin Ke committed
73
  */
74
  void Init(data_size_t num_data, int weight_idx, int query_idx);
Guolin Ke's avatar
Guolin Ke committed
75
76
77

  /*!
  * \brief Partition label by used indices
78
  * \param used_indices Indices of local used
Guolin Ke's avatar
Guolin Ke committed
79
80
81
82
83
  */
  void PartitionLabel(const std::vector<data_size_t>& used_indices);

  /*!
  * \brief Partition meta data according to local used indices if need
84
  * \param num_all_data Number of total training data, including other machines' data on distributed learning
Guolin Ke's avatar
Guolin Ke committed
85
86
87
  * \param used_data_indices Indices of local used training data
  */
  void CheckOrPartition(data_size_t num_all_data,
88
                        const std::vector<data_size_t>& used_data_indices);
Guolin Ke's avatar
Guolin Ke committed
89

90
  void SetLabel(const label_t* label, data_size_t len);
Guolin Ke's avatar
Guolin Ke committed
91

92
  void SetWeights(const label_t* weights, data_size_t len);
Guolin Ke's avatar
Guolin Ke committed
93

Guolin Ke's avatar
Guolin Ke committed
94
  void SetQuery(const data_size_t* query, data_size_t len);
Guolin Ke's avatar
Guolin Ke committed
95

Guolin Ke's avatar
Guolin Ke committed
96
97
98
99
  /*!
  * \brief Set initial scores
  * \param init_score Initial scores, this class will manage memory for init_score.
  */
100
  void SetInitScore(const double* init_score, data_size_t len);
Guolin Ke's avatar
Guolin Ke committed
101
102
103
104
105
106


  /*!
  * \brief Save binary data to file
  * \param file File want to write
  */
107
  void SaveBinaryToFile(const VirtualFileWriter* writer) const;
Guolin Ke's avatar
Guolin Ke committed
108
109
110
111
112
113
114
115
116
117

  /*!
  * \brief Get sizes in byte of this object
  */
  size_t SizesInByte() const;

  /*!
  * \brief Get pointer of label
  * \return Pointer of label
  */
118
  inline const label_t* label() const { return label_.data(); }
Guolin Ke's avatar
Guolin Ke committed
119
120
121
122
123
124

  /*!
  * \brief Set label for one record
  * \param idx Index of this record
  * \param value Label value of this record
  */
125
  inline void SetLabelAt(data_size_t idx, label_t value) {
126
    label_[idx] = value;
Guolin Ke's avatar
Guolin Ke committed
127
128
  }

Guolin Ke's avatar
Guolin Ke committed
129
130
131
132
133
  /*!
  * \brief Set Weight for one record
  * \param idx Index of this record
  * \param value Weight value of this record
  */
134
  inline void SetWeightAt(data_size_t idx, label_t value) {
135
    weights_[idx] = value;
Guolin Ke's avatar
Guolin Ke committed
136
137
138
139
140
141
142
  }

  /*!
  * \brief Set Query Id for one record
  * \param idx Index of this record
  * \param value Query Id value of this record
  */
143
  inline void SetQueryAt(data_size_t idx, data_size_t value) {
Guolin Ke's avatar
Guolin Ke committed
144
145
146
    queries_[idx] = static_cast<data_size_t>(value);
  }

Guolin Ke's avatar
Guolin Ke committed
147
  /*!
Hui Xue's avatar
Hui Xue committed
148
  * \brief Get weights, if not exists, will return nullptr
Guolin Ke's avatar
Guolin Ke committed
149
150
  * \return Pointer of weights
  */
151
  inline const label_t* weights() const {
Guolin Ke's avatar
Guolin Ke committed
152
    if (!weights_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
153
154
155
156
157
      return weights_.data();
    } else {
      return nullptr;
    }
  }
Guolin Ke's avatar
Guolin Ke committed
158
159

  /*!
Hui Xue's avatar
Hui Xue committed
160
  * \brief Get data boundaries on queries, if not exists, will return nullptr
161
  *        we assume data will order by query,
Guolin Ke's avatar
Guolin Ke committed
162
163
164
165
  *        the interval of [query_boundaris[i], query_boundaris[i+1])
  *        is the data indices for query i.
  * \return Pointer of data boundaries on queries
  */
166
  inline const data_size_t* query_boundaries() const {
Guolin Ke's avatar
Guolin Ke committed
167
    if (!query_boundaries_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
168
169
170
171
172
      return query_boundaries_.data();
    } else {
      return nullptr;
    }
  }
Guolin Ke's avatar
Guolin Ke committed
173
174
175
176
177

  /*!
  * \brief Get Number of queries
  * \return Number of queries
  */
178
  inline data_size_t num_queries() const { return num_queries_; }
Guolin Ke's avatar
Guolin Ke committed
179
180

  /*!
Hui Xue's avatar
Hui Xue committed
181
  * \brief Get weights for queries, if not exists, will return nullptr
Guolin Ke's avatar
Guolin Ke committed
182
183
  * \return Pointer of weights for queries
  */
184
  inline const label_t* query_weights() const {
Guolin Ke's avatar
Guolin Ke committed
185
    if (!query_weights_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
186
187
188
189
190
      return query_weights_.data();
    } else {
      return nullptr;
    }
  }
Guolin Ke's avatar
Guolin Ke committed
191
192

  /*!
Hui Xue's avatar
Hui Xue committed
193
  * \brief Get initial scores, if not exists, will return nullptr
Guolin Ke's avatar
Guolin Ke committed
194
195
  * \return Pointer of initial scores
  */
196
  inline const double* init_score() const {
Guolin Ke's avatar
Guolin Ke committed
197
    if (!init_score_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
198
199
200
201
202
      return init_score_.data();
    } else {
      return nullptr;
    }
  }
Guolin Ke's avatar
Guolin Ke committed
203

204
205
206
  /*!
  * \brief Get size of initial scores
  */
Guolin Ke's avatar
Guolin Ke committed
207
  inline int64_t num_init_score() const { return num_init_score_; }
208

Guolin Ke's avatar
Guolin Ke committed
209
210
211
212
  /*! \brief Disable copy */
  Metadata& operator=(const Metadata&) = delete;
  /*! \brief Disable copy */
  Metadata(const Metadata&) = delete;
Guolin Ke's avatar
Guolin Ke committed
213

Nikita Titov's avatar
Nikita Titov committed
214
 private:
Guolin Ke's avatar
Guolin Ke committed
215
  /*! \brief Load initial scores from file */
216
  void LoadInitialScore();
Guolin Ke's avatar
Guolin Ke committed
217
218
219
220
221
222
223
  /*! \brief Load wights from file */
  void LoadWeights();
  /*! \brief Load query boundaries from file */
  void LoadQueryBoundaries();
  /*! \brief Load query wights */
  void LoadQueryWeights();
  /*! \brief Filename of current data */
Guolin Ke's avatar
Guolin Ke committed
224
  std::string data_filename_;
Guolin Ke's avatar
Guolin Ke committed
225
226
227
228
229
  /*! \brief Number of data */
  data_size_t num_data_;
  /*! \brief Number of weights, used to check correct weight file */
  data_size_t num_weights_;
  /*! \brief Label data */
230
  std::vector<label_t> label_;
Guolin Ke's avatar
Guolin Ke committed
231
  /*! \brief Weights data */
232
  std::vector<label_t> weights_;
Guolin Ke's avatar
Guolin Ke committed
233
  /*! \brief Query boundaries */
Guolin Ke's avatar
Guolin Ke committed
234
  std::vector<data_size_t> query_boundaries_;
Guolin Ke's avatar
Guolin Ke committed
235
  /*! \brief Query weights */
236
  std::vector<label_t> query_weights_;
Guolin Ke's avatar
Guolin Ke committed
237
238
239
  /*! \brief Number of querys */
  data_size_t num_queries_;
  /*! \brief Number of Initial score, used to check correct weight file */
Guolin Ke's avatar
Guolin Ke committed
240
  int64_t num_init_score_;
Guolin Ke's avatar
Guolin Ke committed
241
  /*! \brief Initial score */
Guolin Ke's avatar
Guolin Ke committed
242
  std::vector<double> init_score_;
Guolin Ke's avatar
Guolin Ke committed
243
  /*! \brief Queries data */
Guolin Ke's avatar
Guolin Ke committed
244
  std::vector<data_size_t> queries_;
245
246
  /*! \brief mutex for threading safe call */
  std::mutex mutex_;
247
248
249
  bool weight_load_from_file_;
  bool query_load_from_file_;
  bool init_score_load_from_file_;
Guolin Ke's avatar
Guolin Ke committed
250
251
252
253
254
};


/*! \brief Interface for Parser */
class Parser {
Nikita Titov's avatar
Nikita Titov committed
255
 public:
Chen Yufei's avatar
Chen Yufei committed
256
257
  typedef const char* (*AtofFunc)(const char* p, double* out);

258
259
260
261
262
263
264
265
  /*! \brief Default constructor */
  Parser() {}

  /*!
  * \brief Constructor for customized parser. The constructor accepts content not path because need to save/load the config along with model string
  */
  explicit Parser(std::string) {}

Guolin Ke's avatar
Guolin Ke committed
266
267
268
269
270
271
  /*! \brief virtual destructor */
  virtual ~Parser() {}

  /*!
  * \brief Parse one line with label
  * \param str One line record, string format, should end with '\0'
Guolin Ke's avatar
Guolin Ke committed
272
273
  * \param out_features Output columns, store in (column_idx, values)
  * \param out_label Label will store to this if exists
Guolin Ke's avatar
Guolin Ke committed
274
275
  */
  virtual void ParseOneLine(const char* str,
276
                            std::vector<std::pair<int, double>>* out_features, double* out_label) const = 0;
Guolin Ke's avatar
Guolin Ke committed
277

278
  virtual int NumFeatures() const = 0;
Guolin Ke's avatar
Guolin Ke committed
279

Guolin Ke's avatar
Guolin Ke committed
280
  /*!
281
  * \brief Create an object of parser, will auto choose the format depend on file
Guolin Ke's avatar
Guolin Ke committed
282
  * \param filename One Filename of data
283
  * \param header whether input file contains header
284
  * \param num_features Pass num_features of this data file if you know, <=0 means don't know
Guolin Ke's avatar
Guolin Ke committed
285
  * \param label_idx index of label column
Chen Yufei's avatar
Chen Yufei committed
286
  * \param precise_float_parser using precise floating point number parsing if true
Guolin Ke's avatar
Guolin Ke committed
287
288
  * \return Object of parser
  */
Chen Yufei's avatar
Chen Yufei committed
289
  static Parser* CreateParser(const char* filename, bool header, int num_features, int label_idx, bool precise_float_parser);
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334

  /*!
  * \brief Create an object of parser, could use customized parser, or auto choose the format depend on file
  * \param filename One Filename of data
  * \param header whether input file contains header
  * \param num_features Pass num_features of this data file if you know, <=0 means don't know
  * \param label_idx index of label column
  * \param precise_float_parser using precise floating point number parsing if true
  * \param parser_config_str Customized parser config content
  * \return Object of parser
  */
  static Parser* CreateParser(const char* filename, bool header, int num_features, int label_idx, bool precise_float_parser,
                              std::string parser_config_str);

  /*!
  * \brief Generate parser config str used for custom parser initialization, may save values of label id and header
  * \param filename One Filename of data
  * \param parser_config_filename One Filename of parser config
  * \param header whether input file contains header
  * \param label_idx index of label column
  * \return Parser config str
  */
  static std::string GenerateParserConfigStr(const char* filename, const char* parser_config_filename, bool header, int label_idx);
};

/*! \brief Interface for parser factory, used by customized parser */
class ParserFactory {
 private:
  ParserFactory() {}
  std::map<std::string, std::function<Parser*(std::string)>> object_map_;

 public:
  ~ParserFactory() {}
  static ParserFactory& getInstance();
  void Register(std::string class_name, std::function<Parser*(std::string)> objc);
  Parser* getObject(std::string class_name, std::string config_str);
};

/*! \brief Interface for parser reflector, used by customized parser */
class ParserReflector {
 public:
  ParserReflector(std::string class_name, std::function<Parser*(std::string)> objc) {
    ParserFactory::getInstance().Register(class_name, objc);
  }
  virtual ~ParserReflector() {}
Guolin Ke's avatar
Guolin Ke committed
335
336
337
};

/*! \brief The main class of data set,
338
*          which are used to training or validation
Guolin Ke's avatar
Guolin Ke committed
339
340
*/
class Dataset {
Nikita Titov's avatar
Nikita Titov committed
341
 public:
Guolin Ke's avatar
Guolin Ke committed
342
  friend DatasetLoader;
Guolin Ke's avatar
Guolin Ke committed
343

344
  LIGHTGBM_EXPORT Dataset();
Guolin Ke's avatar
Guolin Ke committed
345

346
  LIGHTGBM_EXPORT Dataset(data_size_t num_data);
Guolin Ke's avatar
Guolin Ke committed
347

Guolin Ke's avatar
Guolin Ke committed
348
  void Construct(
Guolin Ke's avatar
Guolin Ke committed
349
    std::vector<std::unique_ptr<BinMapper>>* bin_mappers,
350
    int num_total_features,
351
    const std::vector<std::vector<double>>& forced_bins,
352
    int** sample_non_zero_indices,
Guolin Ke's avatar
Guolin Ke committed
353
    double** sample_values,
354
    const int* num_per_col,
355
    int num_sample_col,
Guolin Ke's avatar
Guolin Ke committed
356
    size_t total_sample_cnt,
Guolin Ke's avatar
Guolin Ke committed
357
    const Config& io_config);
Guolin Ke's avatar
Guolin Ke committed
358

Guolin Ke's avatar
Guolin Ke committed
359
  /*! \brief Destructor */
360
  LIGHTGBM_EXPORT ~Dataset();
Guolin Ke's avatar
Guolin Ke committed
361

362
  LIGHTGBM_EXPORT bool CheckAlign(const Dataset& other) const {
363
364
365
366
367
368
369
370
371
372
    if (num_features_ != other.num_features_) {
      return false;
    }
    if (num_total_features_ != other.num_total_features_) {
      return false;
    }
    if (label_idx_ != other.label_idx_) {
      return false;
    }
    for (int i = 0; i < num_features_; ++i) {
Guolin Ke's avatar
Guolin Ke committed
373
      if (!FeatureBinMapper(i)->CheckAlign(*(other.FeatureBinMapper(i)))) {
374
375
376
377
378
379
        return false;
      }
    }
    return true;
  }

Guolin Ke's avatar
Guolin Ke committed
380
381
382
383
384
385
386
387
388
389
  inline void FinishOneRow(int tid, data_size_t row_idx, const std::vector<bool>& is_feature_added) {
    if (is_finish_load_) { return; }
    for (auto fidx : feature_need_push_zeros_) {
      if (is_feature_added[fidx]) { continue; }
      const int group = feature2group_[fidx];
      const int sub_feature = feature2subfeature_[fidx];
      feature_groups_[group]->PushData(tid, sub_feature, row_idx, 0.0f);
    }
  }

Guolin Ke's avatar
Guolin Ke committed
390
  inline void PushOneRow(int tid, data_size_t row_idx, const std::vector<double>& feature_values) {
Guolin Ke's avatar
Guolin Ke committed
391
    if (is_finish_load_) { return; }
Guolin Ke's avatar
Guolin Ke committed
392
    for (size_t i = 0; i < feature_values.size() && i < static_cast<size_t>(num_total_features_); ++i) {
Guolin Ke's avatar
Guolin Ke committed
393
394
      int feature_idx = used_feature_map_[i];
      if (feature_idx >= 0) {
Guolin Ke's avatar
Guolin Ke committed
395
396
397
        const int group = feature2group_[feature_idx];
        const int sub_feature = feature2subfeature_[feature_idx];
        feature_groups_[group]->PushData(tid, sub_feature, row_idx, feature_values[i]);
398
399
400
        if (has_raw_) {
          int feat_ind = numeric_feature_map_[feature_idx];
          if (feat_ind >= 0) {
sisco0's avatar
sisco0 committed
401
            raw_data_[feat_ind][row_idx] = static_cast<float>(feature_values[i]);
402
403
          }
        }
Guolin Ke's avatar
Guolin Ke committed
404
405
406
407
      }
    }
  }

408
  inline void PushOneRow(int tid, data_size_t row_idx, const std::vector<std::pair<int, double>>& feature_values) {
Guolin Ke's avatar
Guolin Ke committed
409
    if (is_finish_load_) { return; }
Guolin Ke's avatar
Guolin Ke committed
410
    std::vector<bool> is_feature_added(num_features_, false);
411
    for (auto& inner_data : feature_values) {
412
      if (inner_data.first >= num_total_features_) { continue; }
413
414
      int feature_idx = used_feature_map_[inner_data.first];
      if (feature_idx >= 0) {
Guolin Ke's avatar
Guolin Ke committed
415
        is_feature_added[feature_idx] = true;
Guolin Ke's avatar
Guolin Ke committed
416
417
418
        const int group = feature2group_[feature_idx];
        const int sub_feature = feature2subfeature_[feature_idx];
        feature_groups_[group]->PushData(tid, sub_feature, row_idx, inner_data.second);
419
420
421
        if (has_raw_) {
          int feat_ind = numeric_feature_map_[feature_idx];
          if (feat_ind >= 0) {
422
            raw_data_[feat_ind][row_idx] = static_cast<float>(inner_data.second);
423
424
          }
        }
425
426
      }
    }
Guolin Ke's avatar
Guolin Ke committed
427
    FinishOneRow(tid, row_idx, is_feature_added);
428
429
  }

430
  inline void PushOneData(int tid, data_size_t row_idx, int group, int feature_idx, int sub_feature, double value) {
Guolin Ke's avatar
Guolin Ke committed
431
    feature_groups_[group]->PushData(tid, sub_feature, row_idx, value);
432
433
434
    if (has_raw_) {
      int feat_ind = numeric_feature_map_[feature_idx];
      if (feat_ind >= 0) {
435
        raw_data_[feat_ind][row_idx] = static_cast<float>(value);
436
437
      }
    }
Guolin Ke's avatar
Guolin Ke committed
438
439
440
441
442
443
444
  }

  inline int RealFeatureIndex(int fidx) const {
    return real_feature_idx_[fidx];
  }

  inline int InnerFeatureIndex(int col_idx) const {
Guolin Ke's avatar
Guolin Ke committed
445
    return used_feature_map_[col_idx];
Guolin Ke's avatar
Guolin Ke committed
446
  }
Guolin Ke's avatar
Guolin Ke committed
447
448
449
450
451
452
  inline int Feature2Group(int feature_idx) const {
    return feature2group_[feature_idx];
  }
  inline int Feture2SubFeature(int feature_idx) const {
    return feature2subfeature_[feature_idx];
  }
453
454
455
  inline uint64_t GroupBinBoundary(int group_idx) const {
    return group_bin_boundaries_[group_idx];
  }
Guolin Ke's avatar
Guolin Ke committed
456
457
458
  inline uint64_t NumTotalBin() const {
    return group_bin_boundaries_.back();
  }
459

460
461
462
463
464
465
466
467
468
  inline std::vector<int> ValidFeatureIndices() const {
    std::vector<int> ret;
    for (int i = 0; i < num_total_features_; ++i) {
      if (used_feature_map_[i] >= 0) {
        ret.push_back(i);
      }
    }
    return ret;
  }
Guolin Ke's avatar
Guolin Ke committed
469
470
  void ReSize(data_size_t num_data);

471
  void CopySubrow(const Dataset* fullset, const data_size_t* used_indices, data_size_t num_used_indices, bool need_meta_data);
Guolin Ke's avatar
Guolin Ke committed
472

473
  MultiValBin* GetMultiBinFromSparseFeatures(const std::vector<uint32_t>& offsets) const;
474

475
  MultiValBin* GetMultiBinFromAllFeatures(const std::vector<uint32_t>& offsets) const;
476

477
478
479
  TrainingShareStates* GetShareStates(
      score_t* gradients, score_t* hessians,
      const std::vector<int8_t>& is_feature_used, bool is_constant_hessian,
480
      bool force_col_wise, bool force_row_wise) const;
481

482
  LIGHTGBM_EXPORT void FinishLoad();
Guolin Ke's avatar
Guolin Ke committed
483

484
  LIGHTGBM_EXPORT bool SetFloatField(const char* field_name, const float* field_data, data_size_t num_element);
Guolin Ke's avatar
Guolin Ke committed
485

486
  LIGHTGBM_EXPORT bool SetDoubleField(const char* field_name, const double* field_data, data_size_t num_element);
Guolin Ke's avatar
Guolin Ke committed
487

488
  LIGHTGBM_EXPORT bool SetIntField(const char* field_name, const int* field_data, data_size_t num_element);
489

490
  LIGHTGBM_EXPORT bool GetFloatField(const char* field_name, data_size_t* out_len, const float** out_ptr);
491

492
  LIGHTGBM_EXPORT bool GetDoubleField(const char* field_name, data_size_t* out_len, const double** out_ptr);
Guolin Ke's avatar
Guolin Ke committed
493

494
  LIGHTGBM_EXPORT bool GetIntField(const char* field_name, data_size_t* out_len, const int** out_ptr);
495

Guolin Ke's avatar
Guolin Ke committed
496
497
498
  /*!
  * \brief Save current dataset into binary file, will save to "filename.bin"
  */
499
  LIGHTGBM_EXPORT void SaveBinaryFile(const char* bin_filename);
Guolin Ke's avatar
Guolin Ke committed
500

501
502
  LIGHTGBM_EXPORT void DumpTextFile(const char* text_filename);

503
  LIGHTGBM_EXPORT void CopyFeatureMapperFrom(const Dataset* dataset);
Guolin Ke's avatar
Guolin Ke committed
504

Guolin Ke's avatar
Guolin Ke committed
505
506
  LIGHTGBM_EXPORT void CreateValid(const Dataset* dataset);

507
  void InitTrain(const std::vector<int8_t>& is_feature_used,
508
                 TrainingShareStates* share_state) const;
509

Guolin Ke's avatar
Guolin Ke committed
510
511
512
513
514
515
516
517
518
519
520
  template <bool USE_INDICES, bool USE_HESSIAN>
  void ConstructHistogramsInner(const std::vector<int8_t>& is_feature_used,
                                const data_size_t* data_indices,
                                data_size_t num_data, const score_t* gradients,
                                const score_t* hessians,
                                score_t* ordered_gradients,
                                score_t* ordered_hessians,
                                TrainingShareStates* share_state,
                                hist_t* hist_data) const;

  template <bool USE_INDICES, bool ORDERED>
521
522
523
524
  void ConstructHistogramsMultiVal(const data_size_t* data_indices,
                                   data_size_t num_data,
                                   const score_t* gradients,
                                   const score_t* hessians,
525
                                   TrainingShareStates* share_state,
Guolin Ke's avatar
Guolin Ke committed
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
                                   hist_t* hist_data) const;

  inline void ConstructHistograms(
      const std::vector<int8_t>& is_feature_used,
      const data_size_t* data_indices, data_size_t num_data,
      const score_t* gradients, const score_t* hessians,
      score_t* ordered_gradients, score_t* ordered_hessians,
      TrainingShareStates* share_state, hist_t* hist_data) const {
    if (num_data <= 0) {
      return;
    }
    bool use_indices = data_indices != nullptr && (num_data < num_data_);
    if (share_state->is_constant_hessian) {
      if (use_indices) {
        ConstructHistogramsInner<true, false>(
            is_feature_used, data_indices, num_data, gradients, hessians,
            ordered_gradients, ordered_hessians, share_state, hist_data);
      } else {
        ConstructHistogramsInner<false, false>(
            is_feature_used, data_indices, num_data, gradients, hessians,
            ordered_gradients, ordered_hessians, share_state, hist_data);
      }
    } else {
      if (use_indices) {
        ConstructHistogramsInner<true, true>(
            is_feature_used, data_indices, num_data, gradients, hessians,
            ordered_gradients, ordered_hessians, share_state, hist_data);
      } else {
        ConstructHistogramsInner<false, true>(
            is_feature_used, data_indices, num_data, gradients, hessians,
            ordered_gradients, ordered_hessians, share_state, hist_data);
      }
    }
  }
Guolin Ke's avatar
Guolin Ke committed
560

561
  void FixHistogram(int feature_idx, double sum_gradient, double sum_hessian, hist_t* data) const;
Guolin Ke's avatar
Guolin Ke committed
562

563
564
  inline data_size_t Split(int feature, const uint32_t* threshold,
                           int num_threshold, bool default_left,
565
                           const data_size_t* data_indices,
566
567
                           data_size_t cnt, data_size_t* lte_indices,
                           data_size_t* gt_indices) const {
Guolin Ke's avatar
Guolin Ke committed
568
569
    const int group = feature2group_[feature];
    const int sub_feature = feature2subfeature_[feature];
570
571
572
    return feature_groups_[group]->Split(
        sub_feature, threshold, num_threshold, default_left, data_indices,
        cnt, lte_indices, gt_indices);
Guolin Ke's avatar
Guolin Ke committed
573
574
575
576
577
578
579
580
581
582
583
584
585
586
  }

  inline int SubFeatureBinOffset(int i) const {
    const int sub_feature = feature2subfeature_[i];
    if (sub_feature == 0) {
      return 1;
    } else {
      return 0;
    }
  }

  inline int FeatureNumBin(int i) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
587
    return feature_groups_[group]->bin_mappers_[sub_feature]->num_bin();
Guolin Ke's avatar
Guolin Ke committed
588
  }
Guolin Ke's avatar
Guolin Ke committed
589

590
591
592
  inline int FeatureGroupNumBin(int group) const {
    return feature_groups_[group]->num_total_bin_;
  }
593

Guolin Ke's avatar
Guolin Ke committed
594
595
596
597
598
599
  inline const BinMapper* FeatureBinMapper(int i) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
    return feature_groups_[group]->bin_mappers_[sub_feature].get();
  }

600
601
602
603
  inline const Bin* FeatureGroupBin(int group) const {
    return feature_groups_[group]->bin_data_.get();
  }

Guolin Ke's avatar
Guolin Ke committed
604
605
606
  inline BinIterator* FeatureIterator(int i) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
zhangyafeikimi's avatar
zhangyafeikimi committed
607
    return feature_groups_[group]->SubFeatureIterator(sub_feature);
Guolin Ke's avatar
Guolin Ke committed
608
609
  }

610
611
612
  inline BinIterator* FeatureGroupIterator(int group) const {
    return feature_groups_[group]->FeatureGroupIterator();
  }
613

614
615
616
617
  inline bool IsMultiGroup(int i) const {
    return feature_groups_[i]->is_multi_val_;
  }

618
619
620
621
622
623
624
625
  inline size_t FeatureGroupSizesInByte(int group) const {
    return feature_groups_[group]->FeatureGroupSizesInByte();
  }

  inline void* FeatureGroupData(int group) const {
    return feature_groups_[group]->FeatureGroupData();
  }

Guolin Ke's avatar
Guolin Ke committed
626
627
628
629
630
631
  inline double RealThreshold(int i, uint32_t threshold) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
    return feature_groups_[group]->bin_mappers_[sub_feature]->BinToValue(threshold);
  }

632
633
634
635
636
637
638
  // given a real threshold, find the closest threshold bin
  inline uint32_t BinThreshold(int i, double threshold_double) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
    return feature_groups_[group]->bin_mappers_[sub_feature]->ValueToBin(threshold_double);
  }

Guolin Ke's avatar
Guolin Ke committed
639
640
641
642
643
644
645
646
647
  /*!
  * \brief Get meta data pointer
  * \return Pointer of meta data
  */
  inline const Metadata& metadata() const { return metadata_; }

  /*! \brief Get Number of used features */
  inline int num_features() const { return num_features_; }

648
649
650
  /*! \brief Get number of numeric features */
  inline int num_numeric_features() const { return num_numeric_features_; }

651
652
653
  /*! \brief Get Number of feature groups */
  inline int num_feature_groups() const { return num_groups_;}

654
655
656
  /*! \brief Get Number of total features */
  inline int num_total_features() const { return num_total_features_; }

Guolin Ke's avatar
Guolin Ke committed
657
658
659
660
  /*! \brief Get the index of label column */
  inline int label_idx() const { return label_idx_; }

  /*! \brief Get names of current data set */
Guolin Ke's avatar
Guolin Ke committed
661
662
  inline const std::vector<std::string>& feature_names() const { return feature_names_; }

663
664
665
  /*! \brief Get content of parser config file */
  inline const std::string parser_config_str() const { return parser_config_str_; }

Guolin Ke's avatar
Guolin Ke committed
666
667
  inline void set_feature_names(const std::vector<std::string>& feature_names) {
    if (feature_names.size() != static_cast<size_t>(num_total_features_)) {
668
      Log::Fatal("Size of feature_names error, should equal with total number of features");
Guolin Ke's avatar
Guolin Ke committed
669
670
    }
    feature_names_ = std::vector<std::string>(feature_names);
Guolin Ke's avatar
Guolin Ke committed
671
    std::unordered_set<std::string> feature_name_set;
672
673
    // replace ' ' in feature_names with '_'
    bool spaceInFeatureName = false;
674
    for (auto& feature_name : feature_names_) {
Andrew Ziem's avatar
Andrew Ziem committed
675
      // check JSON
676
677
      if (!Common::CheckAllowedJSON(feature_name)) {
        Log::Fatal("Do not support special JSON characters in feature name.");
678
      }
679
      if (feature_name.find(' ') != std::string::npos) {
680
681
682
        spaceInFeatureName = true;
        std::replace(feature_name.begin(), feature_name.end(), ' ', '_');
      }
Guolin Ke's avatar
Guolin Ke committed
683
684
685
686
      if (feature_name_set.count(feature_name) > 0) {
        Log::Fatal("Feature (%s) appears more than one time.", feature_name.c_str());
      }
      feature_name_set.insert(feature_name);
687
    }
688
    if (spaceInFeatureName) {
Andrew Ziem's avatar
Andrew Ziem committed
689
      Log::Warning("Found whitespace in feature_names, replace with underlines");
690
    }
Guolin Ke's avatar
Guolin Ke committed
691
  }
Guolin Ke's avatar
Guolin Ke committed
692

Guolin Ke's avatar
Guolin Ke committed
693
694
  inline std::vector<std::string> feature_infos() const {
    std::vector<std::string> bufs;
695
    for (int i = 0; i < num_total_features_; ++i) {
Guolin Ke's avatar
Guolin Ke committed
696
      int fidx = used_feature_map_[i];
697
      if (fidx < 0) {
Guolin Ke's avatar
Guolin Ke committed
698
699
700
        bufs.push_back("none");
      } else {
        const auto bin_mapper = FeatureBinMapper(fidx);
701
        bufs.push_back(bin_mapper->bin_info_string());
Guolin Ke's avatar
Guolin Ke committed
702
703
704
705
706
      }
    }
    return bufs;
  }

Guolin Ke's avatar
Guolin Ke committed
707
708
709
710
711
712
713
714
  /*! \brief Get Number of data */
  inline data_size_t num_data() const { return num_data_; }

  /*! \brief Disable copy */
  Dataset& operator=(const Dataset&) = delete;
  /*! \brief Disable copy */
  Dataset(const Dataset&) = delete;

715
  void AddFeaturesFrom(Dataset* other);
716

717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
  /*! \brief Get has_raw_ */
  inline bool has_raw() const { return has_raw_; }

  /*! \brief Set has_raw_ */
  inline void SetHasRaw(bool has_raw) { has_raw_ = has_raw; }

  /*! \brief Resize raw_data_ */
  inline void ResizeRaw(int num_rows) {
    if (static_cast<int>(raw_data_.size()) > num_numeric_features_) {
      raw_data_.resize(num_numeric_features_);
    }
    for (size_t i = 0; i < raw_data_.size(); ++i) {
      raw_data_[i].resize(num_rows);
    }
    int curr_size = static_cast<int>(raw_data_.size());
    for (int i = curr_size; i < num_numeric_features_; ++i) {
      raw_data_.push_back(std::vector<float>(num_rows, 0));
    }
  }

  /*! \brief Get pointer to raw_data_ feature */
  inline const float* raw_index(int feat_ind) const {
    return raw_data_[numeric_feature_map_[feat_ind]].data();
  }

Nikita Titov's avatar
Nikita Titov committed
742
 private:
Guolin Ke's avatar
Guolin Ke committed
743
  std::string data_filename_;
Guolin Ke's avatar
Guolin Ke committed
744
  /*! \brief Store used features */
Guolin Ke's avatar
Guolin Ke committed
745
  std::vector<std::unique_ptr<FeatureGroup>> feature_groups_;
Guolin Ke's avatar
Guolin Ke committed
746
747
748
749
  /*! \brief Mapper from real feature index to used index*/
  std::vector<int> used_feature_map_;
  /*! \brief Number of used features*/
  int num_features_;
750
751
  /*! \brief Number of total features*/
  int num_total_features_;
Guolin Ke's avatar
Guolin Ke committed
752
753
754
755
  /*! \brief Number of total data*/
  data_size_t num_data_;
  /*! \brief Store some label level data*/
  Metadata metadata_;
Guolin Ke's avatar
Guolin Ke committed
756
757
758
759
  /*! \brief index of label column */
  int label_idx_ = 0;
  /*! \brief store feature names */
  std::vector<std::string> feature_names_;
760
761
  /*! \brief store feature names */
  static const char* binary_file_token;
Guolin Ke's avatar
Guolin Ke committed
762
763
764
765
766
767
768
  int num_groups_;
  std::vector<int> real_feature_idx_;
  std::vector<int> feature2group_;
  std::vector<int> feature2subfeature_;
  std::vector<uint64_t> group_bin_boundaries_;
  std::vector<int> group_feature_start_;
  std::vector<int> group_feature_cnt_;
Guolin Ke's avatar
Guolin Ke committed
769
  bool is_finish_load_;
770
  int max_bin_;
Belinda Trotta's avatar
Belinda Trotta committed
771
  std::vector<int32_t> max_bin_by_feature_;
772
  std::vector<std::vector<double>> forced_bin_bounds_;
773
774
775
776
  int bin_construct_sample_cnt_;
  int min_data_in_bin_;
  bool use_missing_;
  bool zero_as_missing_;
Guolin Ke's avatar
Guolin Ke committed
777
  std::vector<int> feature_need_push_zeros_;
778
779
780
781
782
  std::vector<std::vector<float>> raw_data_;
  bool has_raw_;
  /*! map feature (inner index) to its index in the list of numeric (non-categorical) features */
  std::vector<int> numeric_feature_map_;
  int num_numeric_features_;
783
  std::string parser_config_str_;
Guolin Ke's avatar
Guolin Ke committed
784
785
786
787
};

}  // namespace LightGBM

Guolin Ke's avatar
Guolin Ke committed
788
#endif   // LightGBM_DATA_H_