application.cpp 10.1 KB
Newer Older
1
2
3
4
/*!
 * Copyright (c) 2016 Microsoft Corporation. All rights reserved.
 * Licensed under the MIT License. See LICENSE file in the project root for license information.
 */
Guolin Ke's avatar
Guolin Ke committed
5
6
#include <LightGBM/application.h>

7
#include <LightGBM/boosting.h>
Guolin Ke's avatar
Guolin Ke committed
8
#include <LightGBM/dataset.h>
Guolin Ke's avatar
Guolin Ke committed
9
#include <LightGBM/dataset_loader.h>
10
11
#include <LightGBM/metric.h>
#include <LightGBM/network.h>
Guolin Ke's avatar
Guolin Ke committed
12
#include <LightGBM/objective_function.h>
13
#include <LightGBM/prediction_early_stop.h>
14
#include <LightGBM/cuda/vector_cudahost.h>
15
#include <LightGBM/utils/common.h>
16
#include <LightGBM/utils/openmp_wrapper.h>
17
#include <LightGBM/utils/text_reader.h>
Guolin Ke's avatar
Guolin Ke committed
18

19
20
21
22
23
24
25
26
#include <string>
#include <chrono>
#include <cstdio>
#include <ctime>
#include <fstream>
#include <sstream>
#include <utility>

27
#include "predictor.hpp"
Guolin Ke's avatar
Guolin Ke committed
28
29
30

namespace LightGBM {

31
32
Common::Timer global_timer;

Guolin Ke's avatar
Guolin Ke committed
33
Application::Application(int argc, char** argv) {
Guolin Ke's avatar
Guolin Ke committed
34
35
36
37
38
  LoadParameters(argc, argv);
  // set number of threads for openmp
  if (config_.num_threads > 0) {
    omp_set_num_threads(config_.num_threads);
  }
Guolin Ke's avatar
Guolin Ke committed
39
  if (config_.data.size() == 0 && config_.task != TaskType::kConvertModel) {
40
    Log::Fatal("No training/prediction data, application quit");
41
  }
42
43
44
45

  if (config_.device_type == std::string("cuda")) {
      LGBM_config_::current_device = lgbm_device_cuda;
  }
Guolin Ke's avatar
Guolin Ke committed
46
47
48
49
50
51
52
53
54
55
}

Application::~Application() {
  if (config_.is_parallel) {
    Network::Dispose();
  }
}

void Application::LoadParameters(int argc, char** argv) {
  std::unordered_map<std::string, std::string> params;
Guolin Ke's avatar
Guolin Ke committed
56
  for (int i = 1; i < argc; ++i) {
Guolin Ke's avatar
Guolin Ke committed
57
    Config::KV2Map(&params, argv[i]);
Guolin Ke's avatar
Guolin Ke committed
58
59
60
61
  }
  // check for alias
  ParameterAlias::KeyAliasTransform(&params);
  // read parameters from config file
Guolin Ke's avatar
Guolin Ke committed
62
63
  if (params.count("config") > 0) {
    TextReader<size_t> config_reader(params["config"].c_str(), false);
Guolin Ke's avatar
Guolin Ke committed
64
    config_reader.ReadAllLines();
Guolin Ke's avatar
Guolin Ke committed
65
    if (!config_reader.Lines().empty()) {
Guolin Ke's avatar
Guolin Ke committed
66
      for (auto& line : config_reader.Lines()) {
67
        // remove str after "#"
Guolin Ke's avatar
Guolin Ke committed
68
69
70
        if (line.size() > 0 && std::string::npos != line.find_first_of("#")) {
          line.erase(line.find_first_of("#"));
        }
Guolin Ke's avatar
Guolin Ke committed
71
        line = Common::Trim(line);
Guolin Ke's avatar
Guolin Ke committed
72
        if (line.size() == 0) {
Guolin Ke's avatar
Guolin Ke committed
73
74
          continue;
        }
Guolin Ke's avatar
Guolin Ke committed
75
        Config::KV2Map(&params, line.c_str());
Guolin Ke's avatar
Guolin Ke committed
76
77
      }
    } else {
78
      Log::Warning("Config file %s doesn't exist, will ignore",
Guolin Ke's avatar
Guolin Ke committed
79
                   params["config"].c_str());
Guolin Ke's avatar
Guolin Ke committed
80
81
82
83
84
85
    }
  }
  // check for alias again
  ParameterAlias::KeyAliasTransform(&params);
  // load configs
  config_.Set(params);
86
  Log::Info("Finished loading parameters");
Guolin Ke's avatar
Guolin Ke committed
87
88
89
90
}

void Application::LoadData() {
  auto start_time = std::chrono::high_resolution_clock::now();
Guolin Ke's avatar
Guolin Ke committed
91
  std::unique_ptr<Predictor> predictor;
92
  // prediction is needed if using input initial model(continued train)
Guolin Ke's avatar
Guolin Ke committed
93
  PredictFunction predict_fun = nullptr;
94
  // need to continue training
Guolin Ke's avatar
Guolin Ke committed
95
  if (boosting_->NumberOfTotalModel() > 0 && config_.task != TaskType::KRefitTree) {
96
    predictor.reset(new Predictor(boosting_.get(), 0, -1, true, false, false, false, -1, -1));
Guolin Ke's avatar
Guolin Ke committed
97
    predict_fun = predictor->GetPredictFunction();
Guolin Ke's avatar
Guolin Ke committed
98
  }
99

Guolin Ke's avatar
Guolin Ke committed
100
  // sync up random seed for data partition
101
  if (config_.is_data_based_parallel) {
Guolin Ke's avatar
Guolin Ke committed
102
    config_.data_random_seed = Network::GlobalSyncUpByMin(config_.data_random_seed);
Guolin Ke's avatar
Guolin Ke committed
103
  }
Guolin Ke's avatar
Guolin Ke committed
104

105
  Log::Debug("Loading train file...");
Guolin Ke's avatar
Guolin Ke committed
106
107
  DatasetLoader dataset_loader(config_, predict_fun,
                               config_.num_class, config_.data.c_str());
Guolin Ke's avatar
Guolin Ke committed
108
  // load Training data
109
  if (config_.is_data_based_parallel) {
Guolin Ke's avatar
Guolin Ke committed
110
    // load data for parallel training
Guolin Ke's avatar
Guolin Ke committed
111
    train_data_.reset(dataset_loader.LoadFromFile(config_.data.c_str(),
Guolin Ke's avatar
Guolin Ke committed
112
                                                  Network::rank(), Network::num_machines()));
Guolin Ke's avatar
Guolin Ke committed
113
114
  } else {
    // load data for single machine
115
    train_data_.reset(dataset_loader.LoadFromFile(config_.data.c_str(), 0, 1));
Guolin Ke's avatar
Guolin Ke committed
116
117
  }
  // need save binary file
Guolin Ke's avatar
Guolin Ke committed
118
  if (config_.save_binary) {
Guolin Ke's avatar
Guolin Ke committed
119
    train_data_->SaveBinaryFile(nullptr);
Guolin Ke's avatar
Guolin Ke committed
120
121
  }
  // create training metric
Guolin Ke's avatar
Guolin Ke committed
122
123
124
  if (config_.is_provide_training_metric) {
    for (auto metric_type : config_.metric) {
      auto metric = std::unique_ptr<Metric>(Metric::CreateMetric(metric_type, config_));
Guolin Ke's avatar
Guolin Ke committed
125
      if (metric == nullptr) { continue; }
Guolin Ke's avatar
Guolin Ke committed
126
      metric->Init(train_data_->metadata(), train_data_->num_data());
Guolin Ke's avatar
Guolin Ke committed
127
      train_metric_.push_back(std::move(metric));
Guolin Ke's avatar
Guolin Ke committed
128
129
    }
  }
Guolin Ke's avatar
Guolin Ke committed
130
  train_metric_.shrink_to_fit();
Guolin Ke's avatar
Guolin Ke committed
131

Guolin Ke's avatar
Guolin Ke committed
132
  if (!config_.metric.empty()) {
133
134
135
    // only when have metrics then need to construct validation data

    // Add validation data, if it exists
Guolin Ke's avatar
Guolin Ke committed
136
    for (size_t i = 0; i < config_.valid.size(); ++i) {
137
      Log::Debug("Loading validation file #%zu...", (i + 1));
138
139
140
      // add
      auto new_dataset = std::unique_ptr<Dataset>(
        dataset_loader.LoadFromFileAlignWithOtherDataset(
Guolin Ke's avatar
Guolin Ke committed
141
          config_.valid[i].c_str(),
142
          train_data_.get()));
143
144
      valid_datas_.push_back(std::move(new_dataset));
      // need save binary file
Guolin Ke's avatar
Guolin Ke committed
145
      if (config_.save_binary) {
146
147
148
149
150
        valid_datas_.back()->SaveBinaryFile(nullptr);
      }

      // add metric for validation data
      valid_metrics_.emplace_back();
Guolin Ke's avatar
Guolin Ke committed
151
152
      for (auto metric_type : config_.metric) {
        auto metric = std::unique_ptr<Metric>(Metric::CreateMetric(metric_type, config_));
153
154
        if (metric == nullptr) { continue; }
        metric->Init(valid_datas_.back()->metadata(),
Guolin Ke's avatar
Guolin Ke committed
155
                     valid_datas_.back()->num_data());
156
157
158
        valid_metrics_.back().push_back(std::move(metric));
      }
      valid_metrics_.back().shrink_to_fit();
Guolin Ke's avatar
Guolin Ke committed
159
    }
160
161
    valid_datas_.shrink_to_fit();
    valid_metrics_.shrink_to_fit();
Guolin Ke's avatar
Guolin Ke committed
162
163
164
  }
  auto end_time = std::chrono::high_resolution_clock::now();
  // output used time on each iteration
165
  Log::Info("Finished loading data in %f seconds",
Guolin Ke's avatar
Guolin Ke committed
166
            std::chrono::duration<double, std::milli>(end_time - start_time) * 1e-3);
Guolin Ke's avatar
Guolin Ke committed
167
168
169
170
171
}

void Application::InitTrain() {
  if (config_.is_parallel) {
    // need init network
Guolin Ke's avatar
Guolin Ke committed
172
    Network::Init(config_);
173
    Log::Info("Finished initializing network");
Guolin Ke's avatar
Guolin Ke committed
174
175
176
177
178
179
    config_.feature_fraction_seed =
      Network::GlobalSyncUpByMin(config_.feature_fraction_seed);
    config_.feature_fraction =
      Network::GlobalSyncUpByMin(config_.feature_fraction);
    config_.drop_seed =
      Network::GlobalSyncUpByMin(config_.drop_seed);
Guolin Ke's avatar
Guolin Ke committed
180
  }
Guolin Ke's avatar
Guolin Ke committed
181

Guolin Ke's avatar
Guolin Ke committed
182
  // create boosting
Guolin Ke's avatar
Guolin Ke committed
183
  boosting_.reset(
Guolin Ke's avatar
Guolin Ke committed
184
185
    Boosting::CreateBoosting(config_.boosting,
                             config_.input_model.c_str()));
Guolin Ke's avatar
Guolin Ke committed
186
  // create objective function
Guolin Ke's avatar
Guolin Ke committed
187
  objective_fun_.reset(
Guolin Ke's avatar
Guolin Ke committed
188
189
    ObjectiveFunction::CreateObjectiveFunction(config_.objective,
                                               config_));
Guolin Ke's avatar
Guolin Ke committed
190
191
192
193
194
  // load training data
  LoadData();
  // initialize the objective function
  objective_fun_->Init(train_data_->metadata(), train_data_->num_data());
  // initialize the boosting
Guolin Ke's avatar
Guolin Ke committed
195
  boosting_->Init(&config_, train_data_.get(), objective_fun_.get(),
Guolin Ke's avatar
Guolin Ke committed
196
                  Common::ConstPtrInVectorWrapper<Metric>(train_metric_));
Guolin Ke's avatar
Guolin Ke committed
197
198
  // add validation data into boosting
  for (size_t i = 0; i < valid_datas_.size(); ++i) {
199
    boosting_->AddValidDataset(valid_datas_[i].get(),
Guolin Ke's avatar
Guolin Ke committed
200
                               Common::ConstPtrInVectorWrapper<Metric>(valid_metrics_[i]));
201
    Log::Debug("Number of data points in validation set #%zu: %zu", i + 1, valid_datas_[i]->num_data());
Guolin Ke's avatar
Guolin Ke committed
202
  }
203
  Log::Info("Finished initializing training");
Guolin Ke's avatar
Guolin Ke committed
204
205
206
}

void Application::Train() {
207
  Log::Info("Started training...");
Guolin Ke's avatar
Guolin Ke committed
208
  boosting_->Train(config_.snapshot_freq, config_.output_model);
209
210
  boosting_->SaveModelToFile(0, -1, config_.saved_feature_importance_type,
                             config_.output_model.c_str());
211
212
  // convert model to if-else statement code
  if (config_.convert_model_language == std::string("cpp")) {
Guolin Ke's avatar
Guolin Ke committed
213
    boosting_->SaveModelToIfElse(-1, config_.convert_model.c_str());
214
  }
215
  Log::Info("Finished training");
Guolin Ke's avatar
Guolin Ke committed
216
217
218
}

void Application::Predict() {
Guolin Ke's avatar
Guolin Ke committed
219
  if (config_.task == TaskType::KRefitTree) {
220
    // create predictor
221
    Predictor predictor(boosting_.get(), 0, -1, false, true, false, false, 1, 1);
222
    predictor.Predict(config_.data.c_str(), config_.output_result.c_str(), config_.header, config_.predict_disable_shape_check);
Guolin Ke's avatar
Guolin Ke committed
223
    TextReader<int> result_reader(config_.output_result.c_str(), false);
224
225
226
227
228
229
230
231
    result_reader.ReadAllLines();
    std::vector<std::vector<int>> pred_leaf(result_reader.Lines().size());
    #pragma omp parallel for schedule(static)
    for (int i = 0; i < static_cast<int>(result_reader.Lines().size()); ++i) {
      pred_leaf[i] = Common::StringToArray<int>(result_reader.Lines()[i], '\t');
      // Free memory
      result_reader.Lines()[i].clear();
    }
Guolin Ke's avatar
Guolin Ke committed
232
233
    DatasetLoader dataset_loader(config_, nullptr,
                                 config_.num_class, config_.data.c_str());
234
    train_data_.reset(dataset_loader.LoadFromFile(config_.data.c_str(), 0, 1));
235
    train_metric_.clear();
Guolin Ke's avatar
Guolin Ke committed
236
237
    objective_fun_.reset(ObjectiveFunction::CreateObjectiveFunction(config_.objective,
                                                                    config_));
238
    objective_fun_->Init(train_data_->metadata(), train_data_->num_data());
Guolin Ke's avatar
Guolin Ke committed
239
    boosting_->Init(&config_, train_data_.get(), objective_fun_.get(),
240
241
                    Common::ConstPtrInVectorWrapper<Metric>(train_metric_));
    boosting_->RefitTree(pred_leaf);
242
243
    boosting_->SaveModelToFile(0, -1, config_.saved_feature_importance_type,
                               config_.output_model.c_str());
244
245
246
    Log::Info("Finished RefitTree");
  } else {
    // create predictor
247
    Predictor predictor(boosting_.get(), config_.start_iteration_predict, config_.num_iteration_predict, config_.predict_raw_score,
Guolin Ke's avatar
Guolin Ke committed
248
249
250
251
                        config_.predict_leaf_index, config_.predict_contrib,
                        config_.pred_early_stop, config_.pred_early_stop_freq,
                        config_.pred_early_stop_margin);
    predictor.Predict(config_.data.c_str(),
252
                      config_.output_result.c_str(), config_.header, config_.predict_disable_shape_check);
253
254
    Log::Info("Finished prediction");
  }
Guolin Ke's avatar
Guolin Ke committed
255
256
257
}

void Application::InitPredict() {
Guolin Ke's avatar
Guolin Ke committed
258
  boosting_.reset(
Guolin Ke's avatar
Guolin Ke committed
259
    Boosting::CreateBoosting("gbdt", config_.input_model.c_str()));
260
  Log::Info("Finished initializing prediction, total used %d iterations", boosting_->GetCurrentIteration());
Guolin Ke's avatar
Guolin Ke committed
261
262
}

263
264
void Application::ConvertModel() {
  boosting_.reset(
Guolin Ke's avatar
Guolin Ke committed
265
266
    Boosting::CreateBoosting(config_.boosting, config_.input_model.c_str()));
  boosting_->SaveModelToIfElse(-1, config_.convert_model.c_str());
267
268
}

Guolin Ke's avatar
Guolin Ke committed
269
270

}  // namespace LightGBM