metadata.cpp 17.9 KB
Newer Older
Guolin Ke's avatar
Guolin Ke committed
1
2
3
4
5
6
7
8
9
#include <LightGBM/dataset.h>

#include <LightGBM/utils/common.h>

#include <vector>
#include <string>

namespace LightGBM {

Guolin Ke's avatar
Guolin Ke committed
10
Metadata::Metadata() {
Guolin Ke's avatar
Guolin Ke committed
11
12
13
14
  num_weights_ = 0;
  num_init_score_ = 0;
  num_data_ = 0;
  num_queries_ = 0;
Guolin Ke's avatar
Guolin Ke committed
15
16
}

17
void Metadata::Init(const char * data_filename) {
Guolin Ke's avatar
Guolin Ke committed
18
19
20
21
22
23
24
25
26
27
28
  data_filename_ = data_filename;
  // for lambdarank, it needs query data for partition data in parallel learning
  LoadQueryBoundaries();
  LoadWeights();
  LoadQueryWeights();
  LoadInitialScore();
}

Metadata::~Metadata() {
}

29
void Metadata::Init(data_size_t num_data, int weight_idx, int query_idx) {
Guolin Ke's avatar
Guolin Ke committed
30
  num_data_ = num_data;
Guolin Ke's avatar
Guolin Ke committed
31
  label_ = std::vector<float>(num_data_);
Guolin Ke's avatar
Guolin Ke committed
32
  if (weight_idx >= 0) {
Guolin Ke's avatar
Guolin Ke committed
33
    if (!weights_.empty()) {
34
      Log::Info("Using weights in data file, ignoring the additional weights file");
Guolin Ke's avatar
Guolin Ke committed
35
      weights_.clear();
Guolin Ke's avatar
Guolin Ke committed
36
    }
Guolin Ke's avatar
Guolin Ke committed
37
    weights_ = std::vector<float>(num_data_);
Guolin Ke's avatar
Guolin Ke committed
38
    num_weights_ = num_data_;
Guolin Ke's avatar
Guolin Ke committed
39
    std::fill(weights_.begin(), weights_.end(), 0.0f);
Guolin Ke's avatar
Guolin Ke committed
40
41
  }
  if (query_idx >= 0) {
Guolin Ke's avatar
Guolin Ke committed
42
    if (!query_boundaries_.empty()) {
43
      Log::Info("Using query id in data file, ignoring the additional query file");
Guolin Ke's avatar
Guolin Ke committed
44
      query_boundaries_.clear();
Guolin Ke's avatar
Guolin Ke committed
45
    }
Guolin Ke's avatar
Guolin Ke committed
46
    if (!query_weights_.empty()) { query_weights_.clear(); }
Guolin Ke's avatar
Guolin Ke committed
47
48
    queries_ = std::vector<data_size_t>(num_data_);
    std::fill(queries_.begin(), queries_.end(), 0);
Guolin Ke's avatar
Guolin Ke committed
49
  }
Guolin Ke's avatar
Guolin Ke committed
50
51
}

Guolin Ke's avatar
Guolin Ke committed
52
53
54
55
56
57
58
59
void Metadata::Init(const Metadata& fullset, const data_size_t* used_indices, data_size_t num_used_indices) {
  num_data_ = num_used_indices;

  label_ = std::vector<float>(num_used_indices);
  for (data_size_t i = 0; i < num_used_indices; i++) {
    label_[i] = fullset.label_[used_indices[i]];
  }

Guolin Ke's avatar
Guolin Ke committed
60
  if (!fullset.weights_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
61
62
63
64
65
66
67
68
69
    weights_ = std::vector<float>(num_used_indices);
    num_weights_ = num_used_indices;
    for (data_size_t i = 0; i < num_used_indices; i++) {
      weights_[i] = fullset.weights_[used_indices[i]];
    }
  } else {
    num_weights_ = 0;
  }

Guolin Ke's avatar
Guolin Ke committed
70
  if (!fullset.init_score_.empty()) {
71
72
73
74
75
76
77
    int num_class = static_cast<int>(fullset.num_init_score_) / fullset.num_data_;
    init_score_ = std::vector<float>(num_used_indices*num_class);
    num_init_score_ = num_used_indices*num_class;
    for (int k = 0; k < num_class; ++k) {
      for (data_size_t i = 0; i < num_used_indices; i++) {
        init_score_[k*num_data_ + i] = fullset.init_score_[k* fullset.num_data_ + used_indices[i]];
      }
Guolin Ke's avatar
Guolin Ke committed
78
79
80
81
82
    }
  } else {
    num_init_score_ = 0;
  }

Guolin Ke's avatar
Guolin Ke committed
83
  if (!fullset.query_boundaries_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
    std::vector<data_size_t> used_query;
    data_size_t data_idx = 0;
    for (data_size_t qid = 0; qid < num_queries_ && data_idx < num_used_indices; ++qid) {
      data_size_t start = fullset.query_boundaries_[qid];
      data_size_t end = fullset.query_boundaries_[qid + 1];
      data_size_t len = end - start;
      if (used_indices[data_idx] > start) {
        continue;
      } else if (used_indices[data_idx] == start) {
        if (num_used_indices >= data_idx + len && used_indices[data_idx + len - 1] == end - 1) {
          used_query.push_back(qid);
          data_idx += len;
        } else {
          Log::Fatal("Data partition error, data didn't match queries");
        }
      } else {
        Log::Fatal("Data partition error, data didn't match queries");
      }
    }
    query_boundaries_ = std::vector<data_size_t>(used_query.size() + 1);
    num_queries_ = static_cast<data_size_t>(used_query.size());
    query_boundaries_[0] = 0;
    for (data_size_t i = 0; i < num_queries_; ++i) {
      data_size_t qid = used_query[i];
      data_size_t len = fullset.query_boundaries_[qid + 1] - fullset.query_boundaries_[qid];
      query_boundaries_[i + 1] = query_boundaries_[i] + len;
    }
  } else {
    num_queries_ = 0;
  }

}

Guolin Ke's avatar
Guolin Ke committed
117
void Metadata::PartitionLabel(const std::vector<data_size_t>& used_indices) {
Guolin Ke's avatar
Guolin Ke committed
118
  if (used_indices.empty()) {
Guolin Ke's avatar
Guolin Ke committed
119
120
    return;
  }
Guolin Ke's avatar
Guolin Ke committed
121
  auto old_label = label_;
Guolin Ke's avatar
Guolin Ke committed
122
  num_data_ = static_cast<data_size_t>(used_indices.size());
Guolin Ke's avatar
Guolin Ke committed
123
  label_ = std::vector<float>(num_data_);
Guolin Ke's avatar
Guolin Ke committed
124
125
126
  for (data_size_t i = 0; i < num_data_; ++i) {
    label_[i] = old_label[used_indices[i]];
  }
Guolin Ke's avatar
Guolin Ke committed
127
  old_label.clear();
Guolin Ke's avatar
Guolin Ke committed
128
129
130
}

void Metadata::CheckOrPartition(data_size_t num_all_data, const std::vector<data_size_t>& used_data_indices) {
Guolin Ke's avatar
Guolin Ke committed
131
132
  if (used_data_indices.empty()) {
    if (!queries_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
      // need convert query_id to boundaries
      std::vector<data_size_t> tmp_buffer;
      data_size_t last_qid = -1;
      data_size_t cur_cnt = 0;
      for (data_size_t i = 0; i < num_data_; ++i) {
        if (last_qid != queries_[i]) {
          if (cur_cnt > 0) {
            tmp_buffer.push_back(cur_cnt);
          }
          cur_cnt = 0;
          last_qid = queries_[i];
        }
        ++cur_cnt;
      }
      tmp_buffer.push_back(cur_cnt);
Guolin Ke's avatar
Guolin Ke committed
148
      query_boundaries_ = std::vector<data_size_t>(tmp_buffer.size() + 1);
Guolin Ke's avatar
Guolin Ke committed
149
150
151
152
153
154
      num_queries_ = static_cast<data_size_t>(tmp_buffer.size());
      query_boundaries_[0] = 0;
      for (size_t i = 0; i < tmp_buffer.size(); ++i) {
        query_boundaries_[i + 1] = query_boundaries_[i] + tmp_buffer[i];
      }
      LoadQueryWeights();
Guolin Ke's avatar
Guolin Ke committed
155
      queries_.clear();
Guolin Ke's avatar
Guolin Ke committed
156
    }
Guolin Ke's avatar
Guolin Ke committed
157
    // check weights
Guolin Ke's avatar
Guolin Ke committed
158
    if (!weights_.empty() && num_weights_ != num_data_) {
Guolin Ke's avatar
Guolin Ke committed
159
      weights_.clear();
Guolin Ke's avatar
Guolin Ke committed
160
      num_weights_ = 0;
Guolin Ke's avatar
Guolin Ke committed
161
      Log::Fatal("Weights size doesn't match data size");
Guolin Ke's avatar
Guolin Ke committed
162
163
164
    }

    // check query boundries
Guolin Ke's avatar
Guolin Ke committed
165
    if (!query_boundaries_.empty() && query_boundaries_[num_queries_] != num_data_) {
Guolin Ke's avatar
Guolin Ke committed
166
      query_boundaries_.clear();
Guolin Ke's avatar
Guolin Ke committed
167
      num_queries_ = 0;
Guolin Ke's avatar
Guolin Ke committed
168
      Log::Fatal("Query size doesn't match data size");
Guolin Ke's avatar
Guolin Ke committed
169
170
171
    }

    // contain initial score file
172
    if (!init_score_.empty() && (num_init_score_ % num_data_) != 0) {
Guolin Ke's avatar
Guolin Ke committed
173
      init_score_.clear();
Guolin Ke's avatar
Guolin Ke committed
174
      num_init_score_ = 0;
Guolin Ke's avatar
Guolin Ke committed
175
      Log::Fatal("Initial score size doesn't match data size");
Guolin Ke's avatar
Guolin Ke committed
176
177
178
179
    }
  } else {
    data_size_t num_used_data = static_cast<data_size_t>(used_data_indices.size());
    // check weights
Guolin Ke's avatar
Guolin Ke committed
180
181
    if (weights_.size() > 0 && num_weights_ != num_all_data) {
      weights_.clear();
Guolin Ke's avatar
Guolin Ke committed
182
      num_weights_ = 0;
Guolin Ke's avatar
Guolin Ke committed
183
      Log::Fatal("Weights size doesn't match data size");
Guolin Ke's avatar
Guolin Ke committed
184
185
    }
    // check query boundries
Guolin Ke's avatar
Guolin Ke committed
186
    if (!query_boundaries_.empty() && query_boundaries_[num_queries_] != num_all_data) {
Guolin Ke's avatar
Guolin Ke committed
187
      query_boundaries_.clear();
Guolin Ke's avatar
Guolin Ke committed
188
      num_queries_ = 0;
Guolin Ke's avatar
Guolin Ke committed
189
      Log::Fatal("Query size doesn't match data size");
Guolin Ke's avatar
Guolin Ke committed
190
191
192
    }

    // contain initial score file
193
    if (!init_score_.empty() && (num_init_score_ % num_all_data) != 0) {
Guolin Ke's avatar
Guolin Ke committed
194
      init_score_.clear();
Guolin Ke's avatar
Guolin Ke committed
195
      num_init_score_ = 0;
Guolin Ke's avatar
Guolin Ke committed
196
      Log::Fatal("Initial score size doesn't match data size");
Guolin Ke's avatar
Guolin Ke committed
197
198
199
    }

    // get local weights
Guolin Ke's avatar
Guolin Ke committed
200
    if (!weights_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
201
      auto old_weights = weights_;
Guolin Ke's avatar
Guolin Ke committed
202
      num_weights_ = num_data_;
Guolin Ke's avatar
Guolin Ke committed
203
      weights_ = std::vector<float>(num_data_);
Guolin Ke's avatar
Guolin Ke committed
204
205
206
      for (size_t i = 0; i < used_data_indices.size(); ++i) {
        weights_[i] = old_weights[used_data_indices[i]];
      }
Guolin Ke's avatar
Guolin Ke committed
207
      old_weights.clear();
Guolin Ke's avatar
Guolin Ke committed
208
209
210
    }

    // get local query boundaries
Guolin Ke's avatar
Guolin Ke committed
211
    if (!query_boundaries_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
212
213
214
215
216
217
218
219
220
221
222
223
224
      std::vector<data_size_t> used_query;
      data_size_t data_idx = 0;
      for (data_size_t qid = 0; qid < num_queries_ && data_idx < num_used_data; ++qid) {
        data_size_t start = query_boundaries_[qid];
        data_size_t end = query_boundaries_[qid + 1];
        data_size_t len = end - start;
        if (used_data_indices[data_idx] > start) {
          continue;
        } else if (used_data_indices[data_idx] == start) {
          if (num_used_data >= data_idx + len && used_data_indices[data_idx + len - 1] == end - 1) {
            used_query.push_back(qid);
            data_idx += len;
          } else {
Guolin Ke's avatar
Guolin Ke committed
225
            Log::Fatal("Data partition error, data didn't match queries");
Guolin Ke's avatar
Guolin Ke committed
226
227
          }
        } else {
Guolin Ke's avatar
Guolin Ke committed
228
          Log::Fatal("Data partition error, data didn't match queries");
Guolin Ke's avatar
Guolin Ke committed
229
230
        }
      }
Guolin Ke's avatar
Guolin Ke committed
231
232
      auto old_query_boundaries = query_boundaries_;
      query_boundaries_ = std::vector<data_size_t>(used_query.size() + 1);
Guolin Ke's avatar
Guolin Ke committed
233
234
235
236
237
238
239
      num_queries_ = static_cast<data_size_t>(used_query.size());
      query_boundaries_[0] = 0;
      for (data_size_t i = 0; i < num_queries_; ++i) {
        data_size_t qid = used_query[i];
        data_size_t len = old_query_boundaries[qid + 1] - old_query_boundaries[qid];
        query_boundaries_[i + 1] = query_boundaries_[i] + len;
      }
Guolin Ke's avatar
Guolin Ke committed
240
      old_query_boundaries.clear();
Guolin Ke's avatar
Guolin Ke committed
241
242
243
    }

    // get local initial scores
Guolin Ke's avatar
Guolin Ke committed
244
    if (!init_score_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
245
      auto old_scores = init_score_;
246
247
248
249
      int num_class = num_init_score_ / num_all_data;
      num_init_score_ = num_data_ * num_class;
      init_score_ = std::vector<float>(num_init_score_);
      for (int k = 0; k < num_class; ++k){
250
251
        for (size_t i = 0; i < used_data_indices.size(); ++i) {
          init_score_[k * num_data_ + i] = old_scores[k * num_all_data + used_data_indices[i]];
252
        }
Guolin Ke's avatar
Guolin Ke committed
253
      }
Guolin Ke's avatar
Guolin Ke committed
254
      old_scores.clear();
Guolin Ke's avatar
Guolin Ke committed
255
256
257
258
259
260
261
262
    }

    // re-load query weight
    LoadQueryWeights();
  }
}


Guolin Ke's avatar
Guolin Ke committed
263
void Metadata::SetInitScore(const float* init_score, data_size_t len) {
264
  std::lock_guard<std::mutex> lock(mutex_);
265
266
267
268
269
270
  // save to nullptr
  if (init_score == nullptr || len == 0) {
    init_score_.clear();
    num_init_score_ = 0;
    return;
  }
271
  if ((len % num_data_) != 0) {
272
    Log::Fatal("Initial score size doesn't match data size");
Guolin Ke's avatar
Guolin Ke committed
273
  }
Guolin Ke's avatar
Guolin Ke committed
274
  if (!init_score_.empty()) { init_score_.clear(); }
275
  num_init_score_ = len;
Guolin Ke's avatar
Guolin Ke committed
276
  init_score_ = std::vector<float>(len);
277
  for (data_size_t i = 0; i < len; ++i) {
Guolin Ke's avatar
Guolin Ke committed
278
279
    init_score_[i] = init_score[i];
  }
Guolin Ke's avatar
Guolin Ke committed
280
281
}

282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
void Metadata::SetInitScore(const double* init_score, data_size_t len) {
  std::lock_guard<std::mutex> lock(mutex_);
  // save to nullptr
  if (init_score == nullptr || len == 0) {
    init_score_.clear();
    num_init_score_ = 0;
    return;
  }
  if ((len % num_data_) != 0) {
    Log::Fatal("Initial score size doesn't match data size");
  }
  if (!init_score_.empty()) { init_score_.clear(); }
  num_init_score_ = len;
  init_score_ = std::vector<float>(len);
  for (data_size_t i = 0; i < len; ++i) {
    init_score_[i] = static_cast<float>(init_score[i]);
  }
}

Guolin Ke's avatar
Guolin Ke committed
301
void Metadata::SetLabel(const float* label, data_size_t len) {
302
  std::lock_guard<std::mutex> lock(mutex_);
303
304
305
  if (label == nullptr) {
    Log::Fatal("label cannot be nullptr");
  }
Guolin Ke's avatar
Guolin Ke committed
306
307
308
  if (num_data_ != len) {
    Log::Fatal("len of label is not same with #data");
  }
Guolin Ke's avatar
Guolin Ke committed
309
  if (!label_.empty()) { label_.clear(); }
Guolin Ke's avatar
Guolin Ke committed
310
  label_ = std::vector<float>(num_data_);
Guolin Ke's avatar
Guolin Ke committed
311
312
313
314
315
316
  for (data_size_t i = 0; i < num_data_; ++i) {
    label_[i] = label[i];
  }
}

void Metadata::SetWeights(const float* weights, data_size_t len) {
317
  std::lock_guard<std::mutex> lock(mutex_);
318
319
320
321
322
323
  // save to nullptr
  if (weights == nullptr || len == 0) {
    weights_.clear();
    num_weights_ = 0;
    return;
  }
Guolin Ke's avatar
Guolin Ke committed
324
325
326
  if (num_data_ != len) {
    Log::Fatal("len of weights is not same with #data");
  }
Guolin Ke's avatar
Guolin Ke committed
327
  if (!weights_.empty()) { weights_.clear(); }
Guolin Ke's avatar
Guolin Ke committed
328
  num_weights_ = num_data_;
Guolin Ke's avatar
Guolin Ke committed
329
  weights_ = std::vector<float>(num_weights_);
Guolin Ke's avatar
Guolin Ke committed
330
331
332
333
334
335
  for (data_size_t i = 0; i < num_weights_; ++i) {
    weights_[i] = weights[i];
  }
  LoadQueryWeights();
}

Guolin Ke's avatar
Guolin Ke committed
336
void Metadata::SetQuery(const data_size_t* query, data_size_t len) {
337
  std::lock_guard<std::mutex> lock(mutex_);
338
  // save to nullptr
Guolin Ke's avatar
Guolin Ke committed
339
  if (query == nullptr || len == 0) {
340
341
342
343
    query_boundaries_.clear();
    num_queries_ = 0;
    return;
  }
Guolin Ke's avatar
Guolin Ke committed
344
345
  data_size_t sum = 0;
  for (data_size_t i = 0; i < len; ++i) {
Guolin Ke's avatar
Guolin Ke committed
346
    sum += query[i];
Guolin Ke's avatar
Guolin Ke committed
347
348
349
350
  }
  if (num_data_ != sum) {
    Log::Fatal("sum of query counts is not same with #data");
  }
Guolin Ke's avatar
Guolin Ke committed
351
  if (!query_boundaries_.empty()) { query_boundaries_.clear(); }
Guolin Ke's avatar
Guolin Ke committed
352
  num_queries_ = len;
Guolin Ke's avatar
Guolin Ke committed
353
354
  query_boundaries_ = std::vector<data_size_t>(num_queries_ + 1);
  query_boundaries_[0] = 0;
Guolin Ke's avatar
Guolin Ke committed
355
  for (data_size_t i = 0; i < num_queries_; ++i) {
Guolin Ke's avatar
Guolin Ke committed
356
    query_boundaries_[i + 1] = query_boundaries_[i] + query[i];
Guolin Ke's avatar
Guolin Ke committed
357
358
359
360
  }
  LoadQueryWeights();
}

361
void Metadata::SetQueryId(const data_size_t* query_id, data_size_t len) {
362
  std::lock_guard<std::mutex> lock(mutex_);
363
364
365
366
367
368
369
  // save to nullptr
  if (query_id == nullptr || len == 0) {
    query_boundaries_.clear();
    queries_.clear();
    num_queries_ = 0;
    return;
  }
370
371
372
  if (num_data_ != len) {
    Log::Fatal("len of query id is not same with #data");
  }
Guolin Ke's avatar
Guolin Ke committed
373
  if (!queries_.empty()) { queries_.clear(); }
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
  queries_ = std::vector<data_size_t>(num_data_);
  for (data_size_t i = 0; i < num_weights_; ++i) {
    queries_[i] = query_id[i];
  }
  // need convert query_id to boundaries
  std::vector<data_size_t> tmp_buffer;
  data_size_t last_qid = -1;
  data_size_t cur_cnt = 0;
  for (data_size_t i = 0; i < num_data_; ++i) {
    if (last_qid != queries_[i]) {
      if (cur_cnt > 0) {
        tmp_buffer.push_back(cur_cnt);
      }
      cur_cnt = 0;
      last_qid = queries_[i];
    }
    ++cur_cnt;
  }
  tmp_buffer.push_back(cur_cnt);
  query_boundaries_ = std::vector<data_size_t>(tmp_buffer.size() + 1);
  num_queries_ = static_cast<data_size_t>(tmp_buffer.size());
  query_boundaries_[0] = 0;
  for (size_t i = 0; i < tmp_buffer.size(); ++i) {
    query_boundaries_[i + 1] = query_boundaries_[i] + tmp_buffer[i];
  }
  queries_.clear();
  LoadQueryWeights();
}
Guolin Ke's avatar
Guolin Ke committed
402

Guolin Ke's avatar
Guolin Ke committed
403
404
405
406
407
void Metadata::LoadWeights() {
  num_weights_ = 0;
  std::string weight_filename(data_filename_);
  // default weight file name
  weight_filename.append(".weight");
Guolin Ke's avatar
Guolin Ke committed
408
  TextReader<size_t> reader(weight_filename.c_str(), false);
Guolin Ke's avatar
Guolin Ke committed
409
  reader.ReadAllLines();
Guolin Ke's avatar
Guolin Ke committed
410
  if (reader.Lines().empty()) {
Guolin Ke's avatar
Guolin Ke committed
411
412
    return;
  }
413
  Log::Info("Loading weights...");
Guolin Ke's avatar
Guolin Ke committed
414
  num_weights_ = static_cast<data_size_t>(reader.Lines().size());
Guolin Ke's avatar
Guolin Ke committed
415
  weights_ = std::vector<float>(num_weights_);
Guolin Ke's avatar
Guolin Ke committed
416
  for (data_size_t i = 0; i < num_weights_; ++i) {
417
    double tmp_weight = 0.0f;
Guolin Ke's avatar
Guolin Ke committed
418
    Common::Atof(reader.Lines()[i].c_str(), &tmp_weight);
419
    weights_[i] = static_cast<float>(tmp_weight);
Guolin Ke's avatar
Guolin Ke committed
420
421
422
423
424
  }
}

void Metadata::LoadInitialScore() {
  num_init_score_ = 0;
Guolin Ke's avatar
Guolin Ke committed
425
426
427
428
  std::string init_score_filename(data_filename_);
  // default weight file name
  init_score_filename.append(".init");
  TextReader<size_t> reader(init_score_filename.c_str(), false);
Guolin Ke's avatar
Guolin Ke committed
429
  reader.ReadAllLines();
Guolin Ke's avatar
Guolin Ke committed
430
  if (reader.Lines().empty()) {
Guolin Ke's avatar
Guolin Ke committed
431
432
    return;
  }
433
434
  Log::Info("Loading initial scores...");

435
436
437
438
439
440
  // use first line to count number class
  int num_class = static_cast<int>(Common::Split(reader.Lines()[0].c_str(), '\t').size());
  data_size_t num_line = static_cast<data_size_t>(reader.Lines().size());
  num_init_score_ = static_cast<data_size_t>(num_line * num_class);

  init_score_ = std::vector<float>(num_init_score_);
441
  double tmp = 0.0f;
442

443
444
445
446
447
  if (num_class == 1) {
    for (data_size_t i = 0; i < num_line; ++i) {
      Common::Atof(reader.Lines()[i].c_str(), &tmp);
      init_score_[i] = static_cast<float>(tmp);
    }
448
  } else {
449
450
451
452
453
454
455
456
457
    std::vector<std::string> oneline_init_score;
    for (data_size_t i = 0; i < num_line; ++i) {
      oneline_init_score = Common::Split(reader.Lines()[i].c_str(), '\t');
      if (static_cast<int>(oneline_init_score.size()) != num_class) {
        Log::Fatal("Invalid initial score file. Redundant or insufficient columns.");
      }
      for (int k = 0; k < num_class; ++k) {
        Common::Atof(oneline_init_score[k].c_str(), &tmp);
        init_score_[k * num_line + i] = static_cast<float>(tmp);
458
      }
459
    }
Guolin Ke's avatar
Guolin Ke committed
460
461
462
463
464
465
466
467
  }
}

void Metadata::LoadQueryBoundaries() {
  num_queries_ = 0;
  std::string query_filename(data_filename_);
  // default query file name
  query_filename.append(".query");
Guolin Ke's avatar
Guolin Ke committed
468
  TextReader<size_t> reader(query_filename.c_str(), false);
Guolin Ke's avatar
Guolin Ke committed
469
  reader.ReadAllLines();
Guolin Ke's avatar
Guolin Ke committed
470
  if (reader.Lines().empty()) {
Guolin Ke's avatar
Guolin Ke committed
471
472
    return;
  }
473
  Log::Info("Loading query boundaries...");
Guolin Ke's avatar
Guolin Ke committed
474
  query_boundaries_ = std::vector<data_size_t>(reader.Lines().size() + 1);
Guolin Ke's avatar
Guolin Ke committed
475
476
477
478
479
480
481
482
483
484
  num_queries_ = static_cast<data_size_t>(reader.Lines().size());
  query_boundaries_[0] = 0;
  for (size_t i = 0; i < reader.Lines().size(); ++i) {
    int tmp_cnt;
    Common::Atoi(reader.Lines()[i].c_str(), &tmp_cnt);
    query_boundaries_[i + 1] = query_boundaries_[i] + static_cast<data_size_t>(tmp_cnt);
  }
}

void Metadata::LoadQueryWeights() {
Guolin Ke's avatar
Guolin Ke committed
485
  if (weights_.size() == 0 || query_boundaries_.size() == 0) {
Guolin Ke's avatar
Guolin Ke committed
486
487
    return;
  }
Guolin Ke's avatar
Guolin Ke committed
488
  query_weights_.clear();
489
  Log::Info("Loading query weights...");
Guolin Ke's avatar
Guolin Ke committed
490
  query_weights_ = std::vector<float>(num_queries_);
Guolin Ke's avatar
Guolin Ke committed
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
  for (data_size_t i = 0; i < num_queries_; ++i) {
    query_weights_[i] = 0.0f;
    for (data_size_t j = query_boundaries_[i]; j < query_boundaries_[i + 1]; ++j) {
      query_weights_[i] += weights_[j];
    }
    query_weights_[i] /= (query_boundaries_[i + 1] - query_boundaries_[i]);
  }
}

void Metadata::LoadFromMemory(const void* memory) {
  const char* mem_ptr = reinterpret_cast<const char*>(memory);

  num_data_ = *(reinterpret_cast<const data_size_t*>(mem_ptr));
  mem_ptr += sizeof(num_data_);
  num_weights_ = *(reinterpret_cast<const data_size_t*>(mem_ptr));
  mem_ptr += sizeof(num_weights_);
  num_queries_ = *(reinterpret_cast<const data_size_t*>(mem_ptr));
  mem_ptr += sizeof(num_queries_);

Guolin Ke's avatar
Guolin Ke committed
510
  if (!label_.empty()) { label_.clear(); }
Guolin Ke's avatar
Guolin Ke committed
511
512
  label_ = std::vector<float>(num_data_);
  std::memcpy(label_.data(), mem_ptr, sizeof(float)*num_data_);
Guolin Ke's avatar
Guolin Ke committed
513
  mem_ptr += sizeof(float)*num_data_;
Guolin Ke's avatar
Guolin Ke committed
514
515

  if (num_weights_ > 0) {
Guolin Ke's avatar
Guolin Ke committed
516
    if (!weights_.empty()) { weights_.clear(); }
Guolin Ke's avatar
Guolin Ke committed
517
518
    weights_ = std::vector<float>(num_weights_);
    std::memcpy(weights_.data(), mem_ptr, sizeof(float)*num_weights_);
Guolin Ke's avatar
Guolin Ke committed
519
520
521
    mem_ptr += sizeof(float)*num_weights_;
  }
  if (num_queries_ > 0) {
Guolin Ke's avatar
Guolin Ke committed
522
    if (!query_boundaries_.empty()) { query_boundaries_.clear(); }
Guolin Ke's avatar
Guolin Ke committed
523
524
    query_boundaries_ = std::vector<data_size_t>(num_queries_ + 1);
    std::memcpy(query_boundaries_.data(), mem_ptr, sizeof(data_size_t)*(num_queries_ + 1));
Guolin Ke's avatar
Guolin Ke committed
525
526
    mem_ptr += sizeof(data_size_t)*(num_queries_ + 1);
  }
Guolin Ke's avatar
Guolin Ke committed
527
  LoadQueryWeights();
Guolin Ke's avatar
Guolin Ke committed
528
529
530
531
532
533
}

void Metadata::SaveBinaryToFile(FILE* file) const {
  fwrite(&num_data_, sizeof(num_data_), 1, file);
  fwrite(&num_weights_, sizeof(num_weights_), 1, file);
  fwrite(&num_queries_, sizeof(num_queries_), 1, file);
Guolin Ke's avatar
Guolin Ke committed
534
  fwrite(label_.data(), sizeof(float), num_data_, file);
Guolin Ke's avatar
Guolin Ke committed
535
  if (!weights_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
536
    fwrite(weights_.data(), sizeof(float), num_weights_, file);
Guolin Ke's avatar
Guolin Ke committed
537
  }
Guolin Ke's avatar
Guolin Ke committed
538
  if (!query_boundaries_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
539
    fwrite(query_boundaries_.data(), sizeof(data_size_t), num_queries_ + 1, file);
Guolin Ke's avatar
Guolin Ke committed
540
541
542
543
544
545
546
547
  }

}

size_t Metadata::SizesInByte() const  {
  size_t size = sizeof(num_data_) + sizeof(num_weights_)
    + sizeof(num_queries_);
  size += sizeof(float) * num_data_;
Guolin Ke's avatar
Guolin Ke committed
548
  if (!weights_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
549
550
    size += sizeof(float) * num_weights_;
  }
Guolin Ke's avatar
Guolin Ke committed
551
  if (!query_boundaries_.empty()) {
Guolin Ke's avatar
Guolin Ke committed
552
553
554
555
556
557
558
    size += sizeof(data_size_t) * (num_queries_ + 1);
  }
  return size;
}


}  // namespace LightGBM