parser.cpp 10.2 KB
Newer Older
1
2
3
4
/*!
 * Copyright (c) 2016 Microsoft Corporation. All rights reserved.
 * Licensed under the MIT License. See LICENSE file in the project root for license information.
 */
5
#include "parser.hpp"
Guolin Ke's avatar
Guolin Ke committed
6

7
#include <algorithm>
8
#include <functional>
9
#include <map>
Guolin Ke's avatar
Guolin Ke committed
10
#include <memory>
11
12
#include <string>
#include <vector>
Guolin Ke's avatar
Guolin Ke committed
13
14
15

namespace LightGBM {

Guolin Ke's avatar
Guolin Ke committed
16
void GetStatistic(const char* str, int* comma_cnt, int* tab_cnt, int* colon_cnt) {
Guolin Ke's avatar
Guolin Ke committed
17
18
19
20
21
22
23
24
25
26
27
28
29
30
  *comma_cnt = 0;
  *tab_cnt = 0;
  *colon_cnt = 0;
  for (int i = 0; str[i] != '\0'; ++i) {
    if (str[i] == ',') {
      ++(*comma_cnt);
    } else if (str[i] == '\t') {
      ++(*tab_cnt);
    } else if (str[i] == ':') {
      ++(*colon_cnt);
    }
  }
}

Guolin Ke's avatar
Guolin Ke committed
31
int GetLabelIdxForLibsvm(const std::string& str, int num_features, int label_idx) {
Guolin Ke's avatar
Guolin Ke committed
32
33
34
  if (num_features <= 0) {
    return label_idx;
  }
Guolin Ke's avatar
Guolin Ke committed
35
36
37
  auto str2 = Common::Trim(str);
  auto pos_space = str2.find_first_of(" \f\n\r\t\v");
  auto pos_colon = str2.find_first_of(":");
38
  if (pos_space == std::string::npos || pos_space < pos_colon) {
Guolin Ke's avatar
Guolin Ke committed
39
    return label_idx;
40
41
  } else {
    return -1;
42
43
44
  }
}

Guolin Ke's avatar
Guolin Ke committed
45
int GetLabelIdxForTSV(const std::string& str, int num_features, int label_idx) {
Guolin Ke's avatar
Guolin Ke committed
46
47
48
  if (num_features <= 0) {
    return label_idx;
  }
Guolin Ke's avatar
Guolin Ke committed
49
50
  auto str2 = Common::Trim(str);
  auto tokens = Common::Split(str2.c_str(), '\t');
Guolin Ke's avatar
Guolin Ke committed
51
  if (static_cast<int>(tokens.size()) == num_features) {
Guolin Ke's avatar
Guolin Ke committed
52
    return -1;
53
  } else {
Guolin Ke's avatar
Guolin Ke committed
54
    return label_idx;
55
56
57
  }
}

Guolin Ke's avatar
Guolin Ke committed
58
int GetLabelIdxForCSV(const std::string& str, int num_features, int label_idx) {
Guolin Ke's avatar
Guolin Ke committed
59
60
61
  if (num_features <= 0) {
    return label_idx;
  }
Guolin Ke's avatar
Guolin Ke committed
62
63
  auto str2 = Common::Trim(str);
  auto tokens = Common::Split(str2.c_str(), ',');
Guolin Ke's avatar
Guolin Ke committed
64
  if (static_cast<int>(tokens.size()) == num_features) {
Guolin Ke's avatar
Guolin Ke committed
65
    return -1;
66
  } else {
Guolin Ke's avatar
Guolin Ke committed
67
    return label_idx;
68
69
70
  }
}

Guolin Ke's avatar
Guolin Ke committed
71
72
73
74
75
76
77
enum DataType {
  INVALID,
  CSV,
  TSV,
  LIBSVM
};

Guolin Ke's avatar
Guolin Ke committed
78
79
80
81
void GetLine(std::stringstream* ss, std::string* line, const VirtualFileReader* reader, std::vector<char>* buffer, size_t buffer_size) {
  std::getline(*ss, *line);
  while (ss->eof()) {
    size_t read_len = reader->Read(buffer->data(), buffer_size);
82
83
84
    if (read_len <= 0) {
      break;
    }
Guolin Ke's avatar
Guolin Ke committed
85
86
    ss->clear();
    ss->str(std::string(buffer->data(), read_len));
87
    std::string tmp;
Guolin Ke's avatar
Guolin Ke committed
88
89
    std::getline(*ss, tmp);
    *line += tmp;
90
91
92
  }
}

93
std::vector<std::string> ReadKLineFromFile(const char* filename, bool header, int k) {
94
95
  auto reader = VirtualFileReader::Make(filename);
  if (!reader->Init()) {
96
    Log::Fatal("Data file %s doesn't exist.", filename);
Guolin Ke's avatar
Guolin Ke committed
97
  }
98
99
100
  std::vector<std::string> ret;
  std::string cur_line;
  const size_t buffer_size = 1024 * 1024;
101
102
103
  auto buffer = std::vector<char>(buffer_size);
  size_t read_len = reader->Read(buffer.data(), buffer_size);
  if (read_len <= 0) {
104
    Log::Fatal("Data file %s couldn't be read.", filename);
105
  }
106
107
  std::string read_str = std::string(buffer.data(), read_len);
  std::stringstream tmp_file(read_str);
Guolin Ke's avatar
Guolin Ke committed
108
  if (header) {
Guolin Ke's avatar
Guolin Ke committed
109
    if (!tmp_file.eof()) {
110
      GetLine(&tmp_file, &cur_line, reader.get(), &buffer, buffer_size);
Guolin Ke's avatar
Guolin Ke committed
111
112
    }
  }
113
114
115
  for (int i = 0; i < k; ++i) {
    if (!tmp_file.eof()) {
      GetLine(&tmp_file, &cur_line, reader.get(), &buffer, buffer_size);
Guolin Ke's avatar
Guolin Ke committed
116
117
118
119
      cur_line = Common::Trim(cur_line);
      if (!cur_line.empty()) {
        ret.push_back(cur_line);
      }
120
121
122
    } else {
      break;
    }
Guolin Ke's avatar
Guolin Ke committed
123
  }
124
125
126
127
128
129
130
  if (ret.empty()) {
    Log::Fatal("Data file %s should have at least one line.", filename);
  } else if (ret.size() == 1) {
    Log::Warning("Data file %s only has one line.", filename);
  }
  return ret;
}
Guolin Ke's avatar
Guolin Ke committed
131

132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
int GetNumColFromLIBSVMFile(const char* filename, bool header) {
  auto reader = VirtualFileReader::Make(filename);
  if (!reader->Init()) {
    Log::Fatal("Data file %s doesn't exist.", filename);
  }
  std::vector<std::string> ret;
  std::string cur_line;
  const size_t buffer_size = 1024 * 1024;
  auto buffer = std::vector<char>(buffer_size);
  size_t read_len = reader->Read(buffer.data(), buffer_size);
  if (read_len <= 0) {
    Log::Fatal("Data file %s couldn't be read.", filename);
  }
  std::string read_str = std::string(buffer.data(), read_len);
  std::stringstream tmp_file(read_str);
  if (header) {
    if (!tmp_file.eof()) {
      GetLine(&tmp_file, &cur_line, reader.get(), &buffer, buffer_size);
    }
  }
  int max_col_idx = 0;
  int max_line_idx = 0;
  const int stop_round = 1 << 7;
  const int max_line = 1 << 13;
  for (int i = 0; i < max_line; ++i) {
    if (!tmp_file.eof()) {
      GetLine(&tmp_file, &cur_line, reader.get(), &buffer, buffer_size);
      cur_line = Common::Trim(cur_line);
      auto colon_pos = cur_line.find_last_of(":");
      auto space_pos = cur_line.find_last_of(" \f\t\v");
      auto sub_str = cur_line.substr(space_pos + 1, space_pos - colon_pos - 1);
      int cur_idx = 0;
      Common::Atoi(sub_str.c_str(), &cur_idx);
      if (cur_idx > max_col_idx) {
        max_col_idx = cur_idx;
        max_line_idx = i;
      }
      if (i - max_line_idx >= stop_round) {
        break;
      }
    } else {
      break;
    }
  }
  CHECK_GT(max_col_idx, 0);
  return max_col_idx;
}

DataType GetDataType(const char* filename, bool header,
                     const std::vector<std::string>& lines, int* num_col) {
Guolin Ke's avatar
Guolin Ke committed
182
  DataType type = DataType::INVALID;
183
184
185
186
187
188
189
  if (lines.empty()) {
    return type;
  }
  int comma_cnt = 0;
  int tab_cnt = 0;
  int colon_cnt = 0;
  GetStatistic(lines[0].c_str(), &comma_cnt, &tab_cnt, &colon_cnt);
190
191
  size_t num_lines = lines.size();
  if (num_lines == 1) {
Guolin Ke's avatar
Guolin Ke committed
192
    if (colon_cnt > 0) {
Guolin Ke's avatar
Guolin Ke committed
193
      type = DataType::LIBSVM;
Guolin Ke's avatar
Guolin Ke committed
194
    } else if (tab_cnt > 0) {
Guolin Ke's avatar
Guolin Ke committed
195
      type = DataType::TSV;
Guolin Ke's avatar
Guolin Ke committed
196
    } else if (comma_cnt > 0) {
Guolin Ke's avatar
Guolin Ke committed
197
198
      type = DataType::CSV;
    }
199
  } else {
Guolin Ke's avatar
Guolin Ke committed
200
201
202
203
204
205
206
207
208
209
210
211
212
    int comma_cnt2 = 0;
    int tab_cnt2 = 0;
    int colon_cnt2 = 0;
    GetStatistic(lines[1].c_str(), &comma_cnt2, &tab_cnt2, &colon_cnt2);
    if (colon_cnt > 0 || colon_cnt2 > 0) {
      type = DataType::LIBSVM;
    } else if (tab_cnt == tab_cnt2 && tab_cnt > 0) {
      type = DataType::TSV;
    } else if (comma_cnt == comma_cnt2 && comma_cnt > 0) {
      type = DataType::CSV;
    }
    if (type == DataType::TSV || type == DataType::CSV) {
      // valid the type
213
      for (size_t i = 2; i < num_lines; ++i) {
Guolin Ke's avatar
Guolin Ke committed
214
215
216
217
218
219
220
221
        GetStatistic(lines[i].c_str(), &comma_cnt2, &tab_cnt2, &colon_cnt2);
        if (type == DataType::TSV && tab_cnt2 != tab_cnt) {
          type = DataType::INVALID;
          break;
        } else if (type == DataType::CSV && comma_cnt != comma_cnt2) {
          type = DataType::INVALID;
          break;
        }
222
223
224
225
      }
    }
  }
  if (type == DataType::LIBSVM) {
226
    int max_col_idx = GetNumColFromLIBSVMFile(filename, header);
227
228
229
230
231
    *num_col = max_col_idx + 1;
  } else if (type == DataType::CSV) {
    *num_col = comma_cnt + 1;
  } else if (type == DataType::TSV) {
    *num_col = tab_cnt + 1;
Guolin Ke's avatar
Guolin Ke committed
232
  }
233
234
235
  return type;
}

236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
// parser factory implementation.
ParserFactory& ParserFactory::getInstance() {
  static ParserFactory factory;
  return factory;
}

void ParserFactory::Register(std::string class_name, std::function<Parser*(std::string)> m_objc) {
  if (m_objc) {
    object_map_.insert(
        std::map<std::string, std::function<Parser*(std::string)>>::value_type(class_name, m_objc));
  }
}

Parser* ParserFactory::getObject(std::string class_name, std::string config_str) {
  std::map<std::string, std::function<Parser*(std::string)>>::const_iterator iter =
      object_map_.find(class_name);
  if (iter != object_map_.end()) {
    return iter->second(config_str);
  } else {
    Log::Fatal("Cannot find parser class '%s', please register first or check config format.", class_name.c_str());
    return nullptr;
  }
}

Chen Yufei's avatar
Chen Yufei committed
260
Parser* Parser::CreateParser(const char* filename, bool header, int num_features, int label_idx, bool precise_float_parser) {
261
  const int n_read_line = 32;
262
263
  auto lines = ReadKLineFromFile(filename, header, n_read_line);
  int num_col = 0;
264
  DataType type = GetDataType(filename, header, lines, &num_col);
Guolin Ke's avatar
Guolin Ke committed
265
  if (type == DataType::INVALID) {
266
    Log::Fatal("Unknown format of training data. Only CSV, TSV, and LibSVM (zero-based) formatted text files are supported.");
Guolin Ke's avatar
Guolin Ke committed
267
  }
Guolin Ke's avatar
Guolin Ke committed
268
  std::unique_ptr<Parser> ret;
Guolin Ke's avatar
Guolin Ke committed
269
  int output_label_index = -1;
Chen Yufei's avatar
Chen Yufei committed
270
  AtofFunc atof = precise_float_parser ? Common::AtofPrecise : Common::Atof;
Guolin Ke's avatar
Guolin Ke committed
271
  if (type == DataType::LIBSVM) {
Guolin Ke's avatar
Guolin Ke committed
272
    output_label_index = GetLabelIdxForLibsvm(lines[0], num_features, label_idx);
Chen Yufei's avatar
Chen Yufei committed
273
    ret.reset(new LibSVMParser(output_label_index, num_col, atof));
274
  } else if (type == DataType::TSV) {
Guolin Ke's avatar
Guolin Ke committed
275
    output_label_index = GetLabelIdxForTSV(lines[0], num_features, label_idx);
Chen Yufei's avatar
Chen Yufei committed
276
    ret.reset(new TSVParser(output_label_index, num_col, atof));
277
  } else if (type == DataType::CSV) {
Guolin Ke's avatar
Guolin Ke committed
278
    output_label_index = GetLabelIdxForCSV(lines[0], num_features, label_idx);
Chen Yufei's avatar
Chen Yufei committed
279
    ret.reset(new CSVParser(output_label_index, num_col, atof));
Guolin Ke's avatar
Guolin Ke committed
280
281
  }

Guolin Ke's avatar
Guolin Ke committed
282
  if (output_label_index < 0 && label_idx >= 0) {
283
    Log::Info("Data file %s doesn't contain a label column.", filename);
Guolin Ke's avatar
Guolin Ke committed
284
  }
Guolin Ke's avatar
Guolin Ke committed
285
  return ret.release();
Guolin Ke's avatar
Guolin Ke committed
286
287
}

288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
Parser* Parser::CreateParser(const char* filename, bool header, int num_features, int label_idx, bool precise_float_parser, std::string parser_config_str) {
  // customized parser add-on.
  if (!parser_config_str.empty()) {
    std::unique_ptr<Parser> ret;
    std::string class_name = Common::GetFromParserConfig(parser_config_str, "className");
    Log::Info("Custom parser class name: %s", class_name.c_str());
    Parser* p = ParserFactory::getInstance().getObject(class_name, parser_config_str);
    ret.reset(p);
    return ret.release();
  }
  return CreateParser(filename, header, num_features, label_idx, precise_float_parser);
}

std::string Parser::GenerateParserConfigStr(const char* filename, const char* parser_config_filename, bool header, int label_idx) {
  TextReader<data_size_t> parser_config_reader(parser_config_filename, false);
  parser_config_reader.ReadAllLines();
  std::string parser_config_str = parser_config_reader.JoinedLines();
  if (!parser_config_str.empty()) {
    // save header to parser config in case needed.
    if (header && Common::GetFromParserConfig(parser_config_str, "header").empty()) {
      TextReader<data_size_t> text_reader(filename, header);
      parser_config_str = Common::SaveToParserConfig(parser_config_str, "header", text_reader.first_line());
    }
    // save label id to parser config in case needed.
    if (Common::GetFromParserConfig(parser_config_str, "labelId").empty()) {
      parser_config_str = Common::SaveToParserConfig(parser_config_str, "labelId", std::to_string(label_idx));
    }
  }
  return parser_config_str;
}
Guolin Ke's avatar
Guolin Ke committed
318
}  // namespace LightGBM