Skip to content

GitLab

Commit 2a8d38c5 authored by Qiwei Ye's avatar Qiwei Ye
Browse files

Merge branches 'master' and 'master' of https://github.com/Microsoft/LightGBM

parents 351b3d7e ed958eb2
Expand all Show whitespace changes
Inline Side-by-side
Showing with 1375 additions and 669 deletions
src/application/application.cpp
View file @ 2a8d38c5
......@@ -76,7 +76,7 @@ void Application::LoadParameters(int argc, char** argv) {
ParameterAlias::KeyAliasTransform(&params);
// read parameters from config file
if (params.count("config_file") > 0) {
TextReader<size_t> config_reader(params["config_file"].c_str());
TextReader<size_t> config_reader(params["config_file"].c_str(), false);
config_reader.ReadAllLines();
if (config_reader.Lines().size() > 0) {
for (auto& line : config_reader.Lines()) {
......@@ -121,17 +121,14 @@ void Application::LoadData() {
// predition is needed if using input initial model(continued train)
PredictFunction predict_fun = nullptr;
Predictor* predictor = nullptr;
// load init model
if (config_.io_config.input_model.size() > 0) {
LoadModel();
// need to continue train
if (boosting_->NumberOfSubModels() > 0) {
predictor = new Predictor(boosting_, config_.io_config.is_sigmoid, config_.predict_leaf_index);
predictor = new Predictor(boosting_, config_.io_config.is_sigmoid, config_.predict_leaf_index, -1);
predict_fun =
[&predictor](const std::vector<std::pair<int, double>>& features) {
[&predictor](const std::vector<std::pair<int, float>>& features) {
return predictor->PredictRawOneLine(features);
};
}
}
// sync up random seed for data partition
if (config_.is_parallel_find_bin) {
config_.io_config.data_random_seed =
......@@ -139,9 +136,7 @@ void Application::LoadData() {
}
train_data_ = new Dataset(config_.io_config.data_filename.c_str(),
config_.io_config.input_init_score.c_str(),
config_.io_config.max_bin,
config_.io_config.data_random_seed,
config_.io_config.is_enable_sparse,
config_.io_config,
predict_fun);
// load Training data
if (config_.is_parallel_find_bin) {
......@@ -158,7 +153,7 @@ void Application::LoadData() {
train_data_->SaveBinaryFile();
}
// create training metric
if (config_.metric_config.is_provide_training_metric) {
if (config_.boosting_config->is_provide_training_metric) {
for (auto metric_type : config_.metric_types) {
Metric* metric =
Metric::CreateMetric(metric_type, config_.metric_config);
......@@ -173,9 +168,7 @@ void Application::LoadData() {
// add
valid_datas_.push_back(
new Dataset(config_.io_config.valid_data_filenames[i].c_str(),
config_.io_config.max_bin,
config_.io_config.data_random_seed,
config_.io_config.is_enable_sparse,
config_.io_config,
predict_fun));
// load validation data like train data
valid_datas_.back()->LoadValidationData(train_data_,
......@@ -217,12 +210,13 @@ void Application::InitTrain() {
gbdt_config->tree_config.feature_fraction_seed =
GlobalSyncUpByMin<int>(gbdt_config->tree_config.feature_fraction_seed);
gbdt_config->tree_config.feature_fraction =
GlobalSyncUpByMin<double>(gbdt_config->tree_config.feature_fraction);
GlobalSyncUpByMin<float>(gbdt_config->tree_config.feature_fraction);
}
}
// create boosting
boosting_ =
Boosting::CreateBoosting(config_.boosting_type, config_.boosting_config);
Boosting::CreateBoosting(config_.boosting_type,
config_.io_config.input_model.c_str());
// create objective function
objective_fun_ =
ObjectiveFunction::CreateObjectiveFunction(config_.objective_type,
......@@ -232,9 +226,8 @@ void Application::InitTrain() {
// initialize the objective function
objective_fun_->Init(train_data_->metadata(), train_data_->num_data());
// initialize the boosting
boosting_->Init(train_data_, objective_fun_,
ConstPtrInVectorWarpper<Metric>(train_metric_),
config_.io_config.output_model.c_str());
boosting_->Init(config_.boosting_config, train_data_, objective_fun_,
ConstPtrInVectorWarpper<Metric>(train_metric_));
// add validation data into boosting
for (size_t i = 0; i < valid_datas_.size(); ++i) {
boosting_->AddDataset(valid_datas_[i],
......@@ -244,36 +237,41 @@ void Application::InitTrain() {
}
void Application::Train() {
Log::Info("Start train");
boosting_->Train();
Log::Info("Finish train");
Log::Info("Start train ...");
int total_iter = config_.boosting_config->num_iterations;
bool is_finished = false;
bool need_eval = true;
auto start_time = std::chrono::high_resolution_clock::now();
for (int iter = 0; iter < total_iter && !is_finished; ++iter) {
is_finished = boosting_->TrainOneIter(nullptr, nullptr, need_eval);
auto end_time = std::chrono::high_resolution_clock::now();
// output used time per iteration
Log::Info("%f seconds elapsed, finished %d iteration", std::chrono::duration<double,
std::milli>(end_time - start_time) * 1e-3, iter + 1);
boosting_->SaveModelToFile(is_finished, config_.io_config.output_model.c_str());
}
is_finished = true;
// save model to file
boosting_->SaveModelToFile(is_finished, config_.io_config.output_model.c_str());
Log::Info("Finished train");
}
void Application::Predict() {
// create predictor
Predictor predictor(boosting_, config_.io_config.is_sigmoid, config_.predict_leaf_index);
predictor.Predict(config_.io_config.data_filename.c_str(), config_.io_config.output_result.c_str());
Predictor predictor(boosting_, config_.io_config.is_sigmoid,
config_.predict_leaf_index, config_.io_config.num_model_predict);
predictor.Predict(config_.io_config.data_filename.c_str(),
config_.io_config.output_result.c_str(), config_.io_config.has_header);
Log::Info("Finish predict.");
}
void Application::InitPredict() {
boosting_ =
Boosting::CreateBoosting(config_.boosting_type, config_.boosting_config);
LoadModel();
Boosting::CreateBoosting(config_.io_config.input_model.c_str());
Log::Info("Finish predict initilization.");
}
void Application::LoadModel() {
TextReader<size_t> model_reader(config_.io_config.input_model.c_str());
model_reader.ReadAllLines();
std::stringstream ss;
for (auto& line : model_reader.Lines()) {
ss << line << '\n';
}
boosting_->ModelsFromString(ss.str(), config_.io_config.num_model_predict);
}
template<typename T>
T Application::GlobalSyncUpByMin(T& local) {
T global = local;
......
src/application/predictor.hpp
View file @ 2a8d38c5
......@@ -28,18 +28,20 @@ public:
* \param is_sigmoid True if need to predict result with sigmoid transform(if needed, like binary classification)
* \param predict_leaf_index True if output leaf index instead of prediction score
*/
Predictor(const Boosting* boosting, bool is_simgoid, bool predict_leaf_index)
: is_simgoid_(is_simgoid), predict_leaf_index(predict_leaf_index) {
Predictor(const Boosting* boosting, bool is_simgoid, bool is_predict_leaf_index, int num_used_model)
: is_simgoid_(is_simgoid), is_predict_leaf_index_(is_predict_leaf_index),
num_used_model_(num_used_model) {
boosting_ = boosting;
num_features_ = boosting_->MaxFeatureIdx() + 1;
num_class_ = boosting_->NumberOfClass();
#pragma omp parallel
#pragma omp master
{
num_threads_ = omp_get_num_threads();
}
features_ = new double*[num_threads_];
features_ = new float*[num_threads_];
for (int i = 0; i < num_threads_; ++i) {
features_[i] = new double[num_features_];
features_[i] = new float[num_features_];
}
}
/*!
......@@ -59,10 +61,10 @@ public:
* \param features Feature for this record
* \return Prediction result
*/
double PredictRawOneLine(const std::vector<std::pair<int, double>>& features) {
float PredictRawOneLine(const std::vector<std::pair<int, float>>& features) {
const int tid = PutFeatureValuesToBuffer(features);
// get result without sigmoid transformation
return boosting_->PredictRaw(features_[tid]);
return boosting_->PredictRaw(features_[tid], num_used_model_);
}
/*!
......@@ -70,10 +72,10 @@ public:
* \param features Feature for this record
* \return Predictied leaf index
*/
std::vector<int> PredictLeafIndexOneLine(const std::vector<std::pair<int, double>>& features) {
std::vector<int> PredictLeafIndexOneLine(const std::vector<std::pair<int, float>>& features) {
const int tid = PutFeatureValuesToBuffer(features);
// get result for leaf index
return boosting_->PredictLeafIndex(features_[tid]);
return boosting_->PredictLeafIndex(features_[tid], num_used_model_);
}
/*!
......@@ -81,18 +83,30 @@ public:
* \param features Feature of this record
* \return Prediction result
*/
double PredictOneLine(const std::vector<std::pair<int, double>>& features) {
float PredictOneLine(const std::vector<std::pair<int, float>>& features) {
const int tid = PutFeatureValuesToBuffer(features);
// get result with sigmoid transform if needed
return boosting_->Predict(features_[tid]);
return boosting_->Predict(features_[tid], num_used_model_);
}
/*!
* \brief prediction for multiclass classification
* \param features Feature of this record
* \return Prediction result
*/
std::vector<float> PredictMulticlassOneLine(const std::vector<std::pair<int, float>>& features) {
const int tid = PutFeatureValuesToBuffer(features);
// get result with sigmoid transform if needed
return boosting_->PredictMulticlass(features_[tid], num_used_model_);
}
/*!
* \brief predicting on data, then saving result to disk
* \param data_filename Filename of data
* \param has_label True if this data contains label
* \param result_filename Filename of output result
*/
void Predict(const char* data_filename, const char* result_filename) {
void Predict(const char* data_filename, const char* result_filename, bool has_header) {
FILE* result_file;
#ifdef _MSC_VER
......@@ -104,53 +118,55 @@ public:
if (result_file == NULL) {
Log::Fatal("Predition result file %s doesn't exists", data_filename);
}
bool has_label = false;
Parser* parser = Parser::CreateParser(data_filename, num_features_, &has_label);
Parser* parser = Parser::CreateParser(data_filename, has_header, num_features_, boosting_->LabelIdx());
if (parser == nullptr) {
Log::Fatal("Recongnizing input data format failed, filename %s", data_filename);
}
// function for parse data
std::function<void(const char*, std::vector<std::pair<int, double>>*)> parser_fun;
double tmp_label;
if (has_label) {
// parse function with label
std::function<void(const char*, std::vector<std::pair<int, float>>*)> parser_fun;
float tmp_label;
parser_fun = [this, &parser, &tmp_label]
(const char* buffer, std::vector<std::pair<int, double>>* feature) {
(const char* buffer, std::vector<std::pair<int, float>>* feature) {
parser->ParseOneLine(buffer, feature, &tmp_label);
};
Log::Info("Start prediction for data %s with labels", data_filename);
} else {
// parse function without label
parser_fun = [this, &parser]
(const char* buffer, std::vector<std::pair<int, double>>* feature) {
parser->ParseOneLine(buffer, feature);
std::function<std::string(const std::vector<std::pair<int, float>>&)> predict_fun;
if (num_class_ > 1) {
predict_fun = [this](const std::vector<std::pair<int, float>>& features){
std::vector<float> prediction = PredictMulticlassOneLine(features);
std::stringstream result_stream_buf;
for (size_t i = 0; i < prediction.size(); ++i){
if (i > 0) {
result_stream_buf << '\t';
}
result_stream_buf << prediction[i];
}
return result_stream_buf.str();
};
Log::Info("Start prediction for data %s without label", data_filename);
}
std::function<std::string(const std::vector<std::pair<int, double>>&)> predict_fun;
if (predict_leaf_index) {
predict_fun = [this](const std::vector<std::pair<int, double>>& features){
else if (is_predict_leaf_index_) {
predict_fun = [this](const std::vector<std::pair<int, float>>& features){
std::vector<int> predicted_leaf_index = PredictLeafIndexOneLine(features);
std::stringstream result_ss;
std::stringstream result_stream_buf;
for (size_t i = 0; i < predicted_leaf_index.size(); ++i){
if (i > 0) {
result_ss << '\t';
result_stream_buf << '\t';
}
result_ss << predicted_leaf_index[i];
result_stream_buf << predicted_leaf_index[i];
}
return result_ss.str();
return result_stream_buf.str();
};
}
else {
if (is_simgoid_) {
predict_fun = [this](const std::vector<std::pair<int, double>>& features){
predict_fun = [this](const std::vector<std::pair<int, float>>& features){
return std::to_string(PredictOneLine(features));
};
}
else {
predict_fun = [this](const std::vector<std::pair<int, double>>& features){
predict_fun = [this](const std::vector<std::pair<int, float>>& features){
return std::to_string(PredictRawOneLine(features));
};
}
......@@ -158,10 +174,10 @@ public:
std::function<void(data_size_t, const std::vector<std::string>&)> process_fun =
[this, &parser_fun, &predict_fun, &result_file]
(data_size_t, const std::vector<std::string>& lines) {
std::vector<std::pair<int, double>> oneline_features;
std::vector<std::pair<int, float>> oneline_features;
std::vector<std::string> pred_result(lines.size(), "");
#pragma omp parallel for schedule(static) private(oneline_features)
for (data_size_t i = 0; i < static_cast<data_size_t>(lines.size()); i++) {
for (data_size_t i = 0; i < static_cast<data_size_t>(lines.size()); ++i) {
oneline_features.clear();
// parser
parser_fun(lines[i].c_str(), &oneline_features);
......@@ -173,7 +189,7 @@ public:
fprintf(result_file, "%s\n", pred_result[i].c_str());
}
};
TextReader<data_size_t> predict_data_reader(data_filename);
TextReader<data_size_t> predict_data_reader(data_filename, has_header);
predict_data_reader.ReadAllAndProcessParallel(process_fun);
fclose(result_file);
......@@ -181,10 +197,10 @@ public:
}
private:
int PutFeatureValuesToBuffer(const std::vector<std::pair<int, double>>& features) {
int PutFeatureValuesToBuffer(const std::vector<std::pair<int, float>>& features) {
int tid = omp_get_thread_num();
// init feature value
std::memset(features_[tid], 0, sizeof(double)*num_features_);
std::memset(features_[tid], 0, sizeof(float)*num_features_);
// put feature value
for (const auto& p : features) {
if (p.first < num_features_) {
......@@ -196,15 +212,19 @@ private:
/*! \brief Boosting model */
const Boosting* boosting_;
/*! \brief Buffer for feature values */
double** features_;
float** features_;
/*! \brief Number of features */
int num_features_;
/*! \brief Number of classes */
int num_class_;
/*! \brief True if need to predict result with sigmoid transform */
bool is_simgoid_;
/*! \brief Number of threads */
int num_threads_;
/*! \brief True if output leaf index instead of prediction score */
bool predict_leaf_index;
bool is_predict_leaf_index_;
/*! \brief Number of used model */
int num_used_model_;
};
} // namespace LightGBM
......
src/boosting/boosting.cpp
View file @ 2a8d38c5
......@@ -3,13 +3,57 @@
namespace LightGBM {
Boosting* Boosting::CreateBoosting(BoostingType type,
const BoostingConfig* config) {
BoostingType GetBoostingTypeFromModelFile(const char* filename) {
TextReader<size_t> model_reader(filename, true);
std::string type = model_reader.first_line();
if (type == std::string("gbdt")) {
return BoostingType::kGBDT;
}
return BoostingType::kUnknow;
}
void LoadFileToBoosting(Boosting* boosting, const char* filename) {
if (boosting != nullptr) {
TextReader<size_t> model_reader(filename, true);
model_reader.ReadAllLines();
std::stringstream str_buf;
for (auto& line : model_reader.Lines()) {
str_buf << line << '\n';
}
boosting->ModelsFromString(str_buf.str());
}
}
Boosting* Boosting::CreateBoosting(BoostingType type, const char* filename) {
if (filename[0] == '\0') {
if (type == BoostingType::kGBDT) {
return new GBDT(config);
return new GBDT();
} else {
return nullptr;
}
} else {
Boosting* ret = nullptr;
auto type_in_file = GetBoostingTypeFromModelFile(filename);
if (type_in_file == type) {
if (type == BoostingType::kGBDT) {
ret = new GBDT();
}
LoadFileToBoosting(ret, filename);
} else {
Log::Fatal("Boosting type in parameter is not same with the type in model file");
}
return ret;
}
}
Boosting* Boosting::CreateBoosting(const char* filename) {
auto type = GetBoostingTypeFromModelFile(filename);
Boosting* ret = nullptr;
if (type == BoostingType::kGBDT) {
ret = new GBDT();
}
LoadFileToBoosting(ret, filename);
return ret;
}
} // namespace LightGBM
src/boosting/gbdt.cpp
View file @ 2a8d38c5
......@@ -12,20 +12,20 @@
#include <chrono>
#include <string>
#include <vector>
#include <utility>
namespace LightGBM {
GBDT::GBDT(const BoostingConfig* config)
: tree_learner_(nullptr), train_score_updater_(nullptr),
GBDT::GBDT()
: train_score_updater_(nullptr),
gradients_(nullptr), hessians_(nullptr),
out_of_bag_data_indices_(nullptr), bag_data_indices_(nullptr) {
max_feature_idx_ = 0;
gbdt_config_ = dynamic_cast<const GBDTConfig*>(config);
early_stopping_round_ = gbdt_config_->early_stopping_round;
}
GBDT::~GBDT() {
if (tree_learner_ != nullptr) { delete tree_learner_; }
for (auto& tree_learner: tree_learner_){
if (tree_learner != nullptr) { delete tree_learner; }
}
if (gradients_ != nullptr) { delete[] gradients_; }
if (hessians_ != nullptr) { delete[] hessians_; }
if (out_of_bag_data_indices_ != nullptr) { delete[] out_of_bag_data_indices_; }
......@@ -39,29 +39,40 @@ GBDT::~GBDT() {
}
}
void GBDT::Init(const Dataset* train_data, const ObjectiveFunction* object_function,
const std::vector<const Metric*>& training_metrics, const char* output_model_filename) {
void GBDT::Init(const BoostingConfig* config, const Dataset* train_data, const ObjectiveFunction* object_function,
const std::vector<const Metric*>& training_metrics) {
gbdt_config_ = dynamic_cast<const GBDTConfig*>(config);
iter_ = 0;
max_feature_idx_ = 0;
early_stopping_round_ = gbdt_config_->early_stopping_round;
train_data_ = train_data;
num_class_ = config->num_class;
tree_learner_ = std::vector<TreeLearner*>(num_class_, nullptr);
// create tree learner
tree_learner_ =
for (int i = 0; i < num_class_; ++i){
tree_learner_[i] =
TreeLearner::CreateTreeLearner(gbdt_config_->tree_learner_type, gbdt_config_->tree_config);
// init tree learner
tree_learner_->Init(train_data_);
tree_learner_[i]->Init(train_data_);
}
object_function_ = object_function;
// push training metrics
for (const auto& metric : training_metrics) {
training_metrics_.push_back(metric);
}
// create score tracker
train_score_updater_ = new ScoreUpdater(train_data_);
train_score_updater_ = new ScoreUpdater(train_data_, num_class_);
num_data_ = train_data_->num_data();
// create buffer for gradients and hessians
gradients_ = new score_t[num_data_];
hessians_ = new score_t[num_data_];
if (object_function_ != nullptr) {
gradients_ = new score_t[num_data_ * num_class_];
hessians_ = new score_t[num_data_ * num_class_];
}
// get max feature index
max_feature_idx_ = train_data_->num_total_features() - 1;
// get label index
label_idx_ = train_data_->label_idx();
// if need bagging, create buffer
if (gbdt_config_->bagging_fraction < 1.0 && gbdt_config_->bagging_freq > 0) {
out_of_bag_data_indices_ = new data_size_t[num_data_];
......@@ -75,22 +86,12 @@ void GBDT::Init(const Dataset* train_data, const ObjectiveFunction* object_funct
// initialize random generator
random_ = Random(gbdt_config_->bagging_seed);
// open model output file
#ifdef _MSC_VER
fopen_s(&output_model_file, output_model_filename, "w");
#else
output_model_file = fopen(output_model_filename, "w");
#endif
// output models
fprintf(output_model_file, "%s", this->ModelsToString().c_str());
}
void GBDT::AddDataset(const Dataset* valid_data,
const std::vector<const Metric*>& valid_metrics) {
// for a validation dataset, we need its score and metric
valid_score_updater_.push_back(new ScoreUpdater(valid_data));
valid_score_updater_.push_back(new ScoreUpdater(valid_data, num_class_));
valid_metrics_.emplace_back();
best_iter_.emplace_back();
best_score_.emplace_back();
......@@ -102,7 +103,7 @@ void GBDT::AddDataset(const Dataset* valid_data,
}
void GBDT::Bagging(int iter) {
void GBDT::Bagging(int iter, const int curr_class) {
// if need bagging
if (out_of_bag_data_indices_ != nullptr && iter % gbdt_config_->bagging_freq == 0) {
// if doesn't have query data
......@@ -151,150 +152,244 @@ void GBDT::Bagging(int iter) {
}
Log::Info("re-bagging, using %d data to train", bag_data_cnt_);
// set bagging data to tree learner
tree_learner_->SetBaggingData(bag_data_indices_, bag_data_cnt_);
tree_learner_[curr_class]->SetBaggingData(bag_data_indices_, bag_data_cnt_);
}
}
void GBDT::UpdateScoreOutOfBag(const Tree* tree) {
void GBDT::UpdateScoreOutOfBag(const Tree* tree, const int curr_class) {
// we need to predict out-of-bag socres of data for boosting
if (out_of_bag_data_indices_ != nullptr) {
train_score_updater_->
AddScore(tree, out_of_bag_data_indices_, out_of_bag_data_cnt_);
AddScore(tree, out_of_bag_data_indices_, out_of_bag_data_cnt_, curr_class);
}
}
void GBDT::Train() {
// training start time
auto start_time = std::chrono::high_resolution_clock::now();
for (int iter = 0; iter < gbdt_config_->num_iterations; ++iter) {
bool GBDT::TrainOneIter(const score_t* gradient, const score_t* hessian, bool is_eval) {
// boosting first
if (gradient == nullptr || hessian == nullptr) {
Boosting();
gradient = gradients_;
hessian = hessians_;
}
for (int curr_class = 0; curr_class < num_class_; ++curr_class){
// bagging logic
Bagging(iter);
Bagging(iter_, curr_class);
// train a new tree
Tree * new_tree = TrainOneTree();
Tree * new_tree = tree_learner_[curr_class]->Train(gradient + curr_class * num_data_, hessian+ curr_class * num_data_);
// if cannot learn a new tree, then stop
if (new_tree->num_leaves() <= 1) {
Log::Info("Can't training anymore, there isn't any leaf meets split requirements.");
break;
return true;
}
// shrinkage by learning rate
new_tree->Shrinkage(gbdt_config_->learning_rate);
// update score
UpdateScore(new_tree);
UpdateScoreOutOfBag(new_tree);
// print message for metric
bool is_early_stopping = OutputMetric(iter + 1);
UpdateScore(new_tree, curr_class);
UpdateScoreOutOfBag(new_tree, curr_class);
// add model
models_.push_back(new_tree);
// save model to file per iteration
if (early_stopping_round_ > 0){
// if use early stopping, save previous model at (iter - early_stopping_round_) iteration
if (iter >= early_stopping_round_){
fprintf(output_model_file, "Tree=%d\n", iter - early_stopping_round_);
Tree * printing_tree = models_.at(iter - early_stopping_round_);
fprintf(output_model_file, "%s\n", printing_tree->ToString().c_str());
fflush(output_model_file);
}
}
else{
fprintf(output_model_file, "Tree=%d\n", iter);
fprintf(output_model_file, "%s\n", new_tree->ToString().c_str());
fflush(output_model_file);
}
auto end_time = std::chrono::high_resolution_clock::now();
// output used time per iteration
Log::Info("%f seconds elapsed, finished %d iteration", std::chrono::duration<double,
std::milli>(end_time - start_time) * 1e-3, iter + 1);
if (is_early_stopping) {
// close file with an early-stopping message
Log::Info("Early stopping at iteration %d, the best iteration round is %d", iter + 1, iter + 1 - early_stopping_round_);
fclose(output_model_file);
return;
}
bool is_met_early_stopping = false;
// print message for metric
if (is_eval) {
is_met_early_stopping = OutputMetric(iter_ + 1);
}
// close file
if (early_stopping_round_ > 0) {
// save remaining models
for (int iter = gbdt_config_->num_iterations - early_stopping_round_; iter < static_cast<int>(models_.size()); ++iter){
fprintf(output_model_file, "Tree=%d\n", iter);
fprintf(output_model_file, "%s\n", models_.at(iter)->ToString().c_str());
++iter_;
if (is_met_early_stopping) {
Log::Info("Early stopping at iteration %d, the best iteration round is %d",
iter_, iter_ - early_stopping_round_);
// pop last early_stopping_round_ models
for (int i = 0; i < early_stopping_round_ * num_class_; ++i) {
delete models_.back();
models_.pop_back();
}
fflush(output_model_file);
}
fclose(output_model_file);
}
return is_met_early_stopping;
Tree* GBDT::TrainOneTree() {
return tree_learner_->Train(gradients_, hessians_);
}
void GBDT::UpdateScore(const Tree* tree) {
void GBDT::UpdateScore(const Tree* tree, const int curr_class) {
// update training score
train_score_updater_->AddScore(tree_learner_);
train_score_updater_->AddScore(tree_learner_[curr_class], curr_class);
// update validation score
for (auto& score_tracker : valid_score_updater_) {
score_tracker->AddScore(tree);
for (auto& score_updater : valid_score_updater_) {
score_updater->AddScore(tree, curr_class);
}
}
bool GBDT::OutputMetric(int iter) {
bool ret = false;
// print training metric
if ((iter % gbdt_config_->output_freq) == 0) {
for (auto& sub_metric : training_metrics_) {
sub_metric->PrintAndGetLoss(iter, train_score_updater_->score());
auto name = sub_metric->GetName();
auto scores = sub_metric->Eval(train_score_updater_->score());
Log::Info("Iteration:%d, %s : %s", iter, name, Common::ArrayToString<float>(scores, ' ').c_str());
}
}
// print validation metric
if ((iter % gbdt_config_->output_freq) == 0 || early_stopping_round_ > 0) {
for (size_t i = 0; i < valid_metrics_.size(); ++i) {
for (size_t j = 0; j < valid_metrics_[i].size(); ++j) {
score_t test_score_ = valid_metrics_[i][j]->PrintAndGetLoss(iter, valid_score_updater_[i]->score());
if (!ret && early_stopping_round_ > 0){
bool the_bigger_the_better_ = valid_metrics_[i][j]->the_bigger_the_better;
auto test_scores = valid_metrics_[i][j]->Eval(valid_score_updater_[i]->score());
if ((iter % gbdt_config_->output_freq) == 0) {
auto name = valid_metrics_[i][j]->GetName();
Log::Info("Iteration:%d, %s : %s", iter, name, Common::ArrayToString<float>(test_scores, ' ').c_str());
}
if (!ret && early_stopping_round_ > 0) {
bool the_bigger_the_better = valid_metrics_[i][j]->is_bigger_better();
if (best_score_[i][j] < 0
|| (!the_bigger_the_better_ && test_score_ < best_score_[i][j])
|| ( the_bigger_the_better_ && test_score_ > best_score_[i][j])){
best_score_[i][j] = test_score_;
|| (!the_bigger_the_better && test_scores.back() < best_score_[i][j])
|| (the_bigger_the_better && test_scores.back() > best_score_[i][j])) {
best_score_[i][j] = test_scores.back();
best_iter_[i][j] = iter;
}
else {
} else {
if (iter - best_iter_[i][j] >= early_stopping_round_) ret = true;
}
}
}
}
}
return ret;
}
/*! \brief Get eval result */
std::vector<std::string> GBDT::EvalCurrent(bool is_eval_train) const {
std::vector<std::string> ret;
if (is_eval_train) {
for (auto& sub_metric : training_metrics_) {
auto name = sub_metric->GetName();
auto scores = sub_metric->Eval(train_score_updater_->score());
std::stringstream str_buf;
str_buf << name << " : " << Common::ArrayToString<float>(scores, ' ');
ret.emplace_back(str_buf.str());
}
}
for (size_t i = 0; i < valid_metrics_.size(); ++i) {
for (size_t j = 0; j < valid_metrics_[i].size(); ++j) {
auto name = valid_metrics_[i][j]->GetName();
auto test_scores = valid_metrics_[i][j]->Eval(valid_score_updater_[i]->score());
std::stringstream str_buf;
str_buf << name << " : " << Common::ArrayToString<float>(test_scores, ' ');
ret.emplace_back(str_buf.str());
}
}
return ret;
}
/*! \brief Get prediction result */
const std::vector<const score_t*> GBDT::PredictCurrent(bool is_predict_train) const {
std::vector<const score_t*> ret;
if (is_predict_train) {
ret.push_back(train_score_updater_->score());
}
for (size_t i = 0; i < valid_metrics_.size(); ++i) {
ret.push_back(valid_score_updater_[i]->score());
}
return ret;
}
void GBDT::Boosting() {
if (object_function_ == nullptr) {
Log::Fatal("No object function provided");
}
// objective function will calculate gradients and hessians
object_function_->
GetGradients(train_score_updater_->score(), gradients_, hessians_);
}
void GBDT::SaveModelToFile(bool is_finish, const char* filename) {
std::string GBDT::ModelsToString() const {
// serialize this object to string
std::stringstream ss;
// first time to this function, open file
if (saved_model_size_ == -1) {
model_output_file_.open(filename);
// output model type
model_output_file_ << "gbdt" << std::endl;
// output number of class
model_output_file_ << "num_class=" << num_class_ << std::endl;
// output label index
model_output_file_ << "label_index=" << label_idx_ << std::endl;
// output max_feature_idx
ss << "max_feature_idx=" << max_feature_idx_ << std::endl;
model_output_file_ << "max_feature_idx=" << max_feature_idx_ << std::endl;
// output sigmoid parameter
ss << "sigmoid=" << object_function_->GetSigmoid() << std::endl;
ss << std::endl;
model_output_file_ << "sigmoid=" << object_function_->GetSigmoid() << std::endl;
model_output_file_ << std::endl;
saved_model_size_ = 0;
}
// already saved
if (!model_output_file_.is_open()) {
return;
}
int rest = static_cast<int>(models_.size()) - early_stopping_round_ * num_class_;
// output tree models
for (size_t i = 0; i < models_.size(); ++i) {
ss << "Tree=" << i << std::endl;
ss << models_[i]->ToString() << std::endl;
for (int i = saved_model_size_; i < rest; ++i) {
model_output_file_ << "Tree=" << i << std::endl;
model_output_file_ << models_[i]->ToString() << std::endl;
}
saved_model_size_ = Common::Max(saved_model_size_, rest);
model_output_file_.flush();
// training finished, can close file
if (is_finish) {
for (int i = saved_model_size_; i < static_cast<int>(models_.size()); ++i) {
model_output_file_ << "Tree=" << i << std::endl;
model_output_file_ << models_[i]->ToString() << std::endl;
}
model_output_file_ << std::endl << FeatureImportance() << std::endl;
model_output_file_.close();
}
return ss.str();
}
void GBDT::ModelsFromString(const std::string& model_str, int num_used_model) {
void GBDT::ModelsFromString(const std::string& model_str) {
// use serialized string to restore this object
models_.clear();
std::vector<std::string> lines = Common::Split(model_str.c_str(), '\n');
size_t i = 0;
// get number of class
while (i < lines.size()) {
size_t find_pos = lines[i].find("num_class=");
if (find_pos != std::string::npos) {
std::vector<std::string> strs = Common::Split(lines[i].c_str(), '=');
Common::Atoi(strs[1].c_str(), &num_class_);
++i;
break;
} else {
++i;
}
}
if (i == lines.size()) {
Log::Fatal("Model file doesn't contain number of class");
return;
}
// get index of label
i = 0;
while (i < lines.size()) {
size_t find_pos = lines[i].find("label_index=");
if (find_pos != std::string::npos) {
std::vector<std::string> strs = Common::Split(lines[i].c_str(), '=');
Common::Atoi(strs[1].c_str(), &label_idx_);
++i;
break;
} else {
++i;
}
}
if (i == lines.size()) {
Log::Fatal("Model file doesn't contain label index");
return;
}
// get max_feature_idx first
i = 0;
while (i < lines.size()) {
size_t find_pos = lines[i].find("max_feature_idx=");
if (find_pos != std::string::npos) {
......@@ -338,40 +433,86 @@ void GBDT::ModelsFromString(const std::string& model_str, int num_used_model) {
int end = static_cast<int>(i);
std::string tree_str = Common::Join(lines, start, end, '\n');
models_.push_back(new Tree(tree_str));
if (num_used_model > 0 && models_.size() >= static_cast<size_t>(num_used_model)) {
break;
}
} else {
++i;
}
}
Log::Info("%d models has been loaded\n", models_.size());
}
double GBDT::PredictRaw(const double* value) const {
double ret = 0.0;
for (size_t i = 0; i < models_.size(); ++i) {
std::string GBDT::FeatureImportance() const {
std::vector<size_t> feature_importances(max_feature_idx_ + 1, 0);
for (size_t iter = 0; iter < models_.size(); ++iter) {
for (int split_idx = 0; split_idx < models_[iter]->num_leaves() - 1; ++split_idx) {
++feature_importances[models_[iter]->split_feature_real(split_idx)];
}
}
// store the importance first
std::vector<std::pair<size_t, std::string>> pairs;
for (size_t i = 0; i < feature_importances.size(); ++i) {
pairs.emplace_back(feature_importances[i], train_data_->feature_names()[i]);
}
// sort the importance
std::sort(pairs.begin(), pairs.end(),
[](const std::pair<size_t, std::string>& lhs,
const std::pair<size_t, std::string>& rhs) {
return lhs.first > rhs.first;
});
std::stringstream str_buf;
// write to model file
str_buf << std::endl << "feature importances:" << std::endl;
for (size_t i = 0; i < pairs.size(); ++i) {
str_buf << pairs[i].second << "=" << std::to_string(pairs[i].first) << std::endl;
}
return str_buf.str();
}
float GBDT::PredictRaw(const float* value, int num_used_model) const {
if (num_used_model < 0) {
num_used_model = static_cast<int>(models_.size());
}
float ret = 0.0f;
for (int i = 0; i < num_used_model; ++i) {
ret += models_[i]->Predict(value);
}
return ret;
}
double GBDT::Predict(const double* value) const {
double ret = 0.0;
for (size_t i = 0; i < models_.size(); ++i) {
float GBDT::Predict(const float* value, int num_used_model) const {
if (num_used_model < 0) {
num_used_model = static_cast<int>(models_.size());
}
float ret = 0.0f;
for (int i = 0; i < num_used_model; ++i) {
ret += models_[i]->Predict(value);
}
// if need sigmoid transform
if (sigmoid_ > 0) {
ret = 1.0 / (1.0 + std::exp(- 2.0f * sigmoid_ * ret));
ret = 1.0f / (1.0f + std::exp(- 2.0f * sigmoid_ * ret));
}
return ret;
}
std::vector<float> GBDT::PredictMulticlass(const float* value, int num_used_model) const {
if (num_used_model < 0) {
num_used_model = static_cast<int>(models_.size()) / num_class_;
}
std::vector<float> ret(num_class_, 0.0f);
for (int i = 0; i < num_used_model; ++i) {
for (int j = 0; j < num_class_; ++j){
ret[j] += models_[i * num_class_ + j] -> Predict(value);
}
}
Common::Softmax(&ret);
return ret;
}
std::vector<int> GBDT::PredictLeafIndex(const double* value) const {
std::vector<int> GBDT::PredictLeafIndex(const float* value, int num_used_model) const {
if (num_used_model < 0) {
num_used_model = static_cast<int>(models_.size());
}
std::vector<int> ret;
for (size_t i = 0; i < models_.size(); ++i) {
for (int i = 0; i < num_used_model; ++i) {
ret.push_back(models_[i]->PredictLeafIndex(value));
}
return ret;
......
src/boosting/gbdt.h
View file @ 2a8d38c5
......@@ -7,6 +7,7 @@
#include <cstdio>
#include <vector>
#include <string>
#include <fstream>
namespace LightGBM {
/*!
......@@ -16,9 +17,8 @@ class GBDT: public Boosting {
public:
/*!
* \brief Constructor
* \param config Config of GBDT
*/
explicit GBDT(const BoostingConfig* config);
GBDT();
/*!
* \brief Destructor
*/
......@@ -31,9 +31,8 @@ public:
* \param training_metrics Training metrics
* \param output_model_filename Filename of output model
*/
void Init(const Dataset* train_data, const ObjectiveFunction* object_function,
const std::vector<const Metric*>& training_metrics,
const char* output_model_filename)
void Init(const BoostingConfig* gbdt_config, const Dataset* train_data, const ObjectiveFunction* object_function,
const std::vector<const Metric*>& training_metrics)
override;
/*!
* \brief Adding a validation dataset
......@@ -45,92 +44,128 @@ public:
/*!
* \brief one training iteration
*/
void Train() override;
bool TrainOneIter(const score_t* gradient, const score_t* hessian, bool is_eval) override;
/*! \brief Get eval result */
std::vector<std::string> EvalCurrent(bool is_eval_train) const override;
/*! \brief Get prediction result */
const std::vector<const score_t*> PredictCurrent(bool is_predict_train) const override;
/*!
* \brief Predtion for one record without sigmoid transformation
* \param feature_values Feature value on this record
* \param num_used_model Number of used model
* \return Prediction result for this record
*/
double PredictRaw(const double * feature_values) const override;
float PredictRaw(const float* feature_values, int num_used_model) const override;
/*!
* \brief Predtion for one record with sigmoid transformation if enabled
* \param feature_values Feature value on this record
* \param num_used_model Number of used model
* \return Prediction result for this record
*/
double Predict(const double * feature_values) const override;
float Predict(const float* feature_values, int num_used_model) const override;
/*!
* \brief Predtion for multiclass classification
* \param feature_values Feature value on this record
* \return Prediction result, num_class numbers per line
*/
std::vector<float> PredictMulticlass(const float* value, int num_used_model) const override;
/*!
* \brief Predtion for one record with leaf index
* \param feature_values Feature value on this record
* \param num_used_model Number of used model
* \return Predicted leaf index for this record
*/
std::vector<int> PredictLeafIndex(const double* value) const override;
std::vector<int> PredictLeafIndex(const float* value, int num_used_model) const override;
/*!
* \brief Serialize models by string
* \return String output of tranined model
*/
std::string ModelsToString() const override;
void SaveModelToFile(bool is_finish, const char* filename) override;
/*!
* \brief Restore from a serialized string
* \param model_str The string of model
*/
void ModelsFromString(const std::string& model_str, int num_used_model) override;
void ModelsFromString(const std::string& model_str) override;
/*!
* \brief Get max feature index of this model
* \return Max feature index of this model
*/
inline int MaxFeatureIdx() const override { return max_feature_idx_; }
/*!
* \brief Get index of label column
* \return index of label column
*/
inline int LabelIdx() const override { return label_idx_; }
/*!
* \brief Get number of weak sub-models
* \return Number of weak sub-models
*/
inline int NumberOfSubModels() const override { return static_cast<int>(models_.size()); }
/*!
* \brief Get number of classes
* \return Number of classes
*/
inline int NumberOfClass() const override { return num_class_; }
/*!
* \brief Get Type name of this boosting object
*/
const char* Name() const override { return "gbdt"; }
private:
/*!
* \brief Implement bagging logic
* \param iter Current interation
* \param curr_class Current class for multiclass training
*/
void Bagging(int iter);
void Bagging(int iter, const int curr_class);
/*!
* \brief updating score for out-of-bag data.
* Data should be update since we may re-bagging data on training
* \param tree Trained tree of this iteration
* \param curr_class Current class for multiclass training
*/
void UpdateScoreOutOfBag(const Tree* tree);
void UpdateScoreOutOfBag(const Tree* tree, const int curr_class);
/*!
* \brief calculate the object function
*/
void Boosting();
/*!
* \brief training one tree
* \return Trained tree of this iteration
*/
Tree* TrainOneTree();
/*!
* \brief updating score after tree was trained
* \param tree Trained tree of this iteration
* \param curr_class Current class for multiclass training
*/
void UpdateScore(const Tree* tree);
void UpdateScore(const Tree* tree, const int curr_class);
/*!
* \brief Print Metric result of current iteration
* \brief Print metric result of current iteration
* \param iter Current interation
*/
bool OutputMetric(int iter);
int early_stopping_round_;
/*!
* \brief Calculate feature importances
* \param last_iter Last tree use to calculate
*/
std::string FeatureImportance() const;
/*! \brief current iteration */
int iter_;
/*! \brief Pointer to training data */
const Dataset* train_data_;
/*! \brief Config of gbdt */
const GBDTConfig* gbdt_config_;
/*! \brief Tree learner, will use tihs class to learn trees */
TreeLearner* tree_learner_;
/*! \brief Tree learner, will use this class to learn trees */
std::vector<TreeLearner*> tree_learner_;
/*! \brief Objective function */
const ObjectiveFunction* object_function_;
/*! \brief Store and update traning data's score */
/*! \brief Store and update training data's score */
ScoreUpdater* train_score_updater_;
/*! \brief Metrics for training data */
std::vector<const Metric*> training_metrics_;
......@@ -138,6 +173,8 @@ private:
std::vector<ScoreUpdater*> valid_score_updater_;
/*! \brief Metric for validation data */
std::vector<std::vector<const Metric*>> valid_metrics_;
/*! \brief Number of rounds for early stopping */
int early_stopping_round_;
/*! \brief Best score(s) for early stopping */
std::vector<std::vector<int>> best_iter_;
std::vector<std::vector<score_t>> best_score_;
......@@ -159,15 +196,21 @@ private:
data_size_t bag_data_cnt_;
/*! \brief Number of traning data */
data_size_t num_data_;
/*! \brief Number of classes */
int num_class_;
/*! \brief Random generator, used for bagging */
Random random_;
/*! \brief The filename that the models will save to */
FILE * output_model_file;
/*!
* \brief Sigmoid parameter, used for prediction.
* if > 0 meas output score will transform by sigmoid function
*/
double sigmoid_;
float sigmoid_;
/*! \brief Index of label column */
data_size_t label_idx_;
/*! \brief Saved number of models */
int saved_model_size_ = -1;
/*! \brief File to write models */
std::ofstream model_output_file_;
};
} // namespace LightGBM
......
src/boosting/score_updater.hpp
View file @ 2a8d38c5
......@@ -18,13 +18,13 @@ public:
* \brief Constructor, will pass a const pointer of dataset
* \param data This class will bind with this data set
*/
explicit ScoreUpdater(const Dataset* data)
explicit ScoreUpdater(const Dataset* data, int num_class)
:data_(data) {
num_data_ = data->num_data();
score_ = new score_t[num_data_];
score_ = new score_t[num_data_ * num_class];
// default start score is zero
std::memset(score_, 0, sizeof(score_t)*num_data_);
const score_t* init_score = data->metadata().init_score();
std::memset(score_, 0, sizeof(score_t) * num_data_ * num_class);
const float* init_score = data->metadata().init_score();
// if exists initial score, will start from it
if (init_score != nullptr) {
for (data_size_t i = 0; i < num_data_; ++i) {
......@@ -41,8 +41,8 @@ public:
* Note: this function generally will be used on validation data too.
* \param tree Trained tree model
*/
inline void AddScore(const Tree* tree) {
tree->AddPredictionToScore(data_, num_data_, score_);
inline void AddScore(const Tree* tree, int curr_class) {
tree->AddPredictionToScore(data_, num_data_, score_ + curr_class * num_data_);
}
/*!
* \brief Adding prediction score, only used for training data.
......@@ -50,19 +50,19 @@ public:
* Based on which We can get prediction quckily.
* \param tree_learner
*/
inline void AddScore(const TreeLearner* tree_learner) {
tree_learner->AddPredictionToScore(score_);
inline void AddScore(const TreeLearner* tree_learner, int curr_class) {
tree_learner->AddPredictionToScore(score_ + curr_class * num_data_);
}
/*!
* \brief Using tree model to get prediction number, then adding to scores for parts of data
* Used for prediction of training out-of-bag data
* \param tree Trained tree model
* \param data_indices Indices of data that want proccess to
* \param data_cnt Number of data that want proccess to
* \param data_indices Indices of data that will be proccessed
* \param data_cnt Number of data that will be proccessed
*/
inline void AddScore(const Tree* tree, const data_size_t* data_indices,
data_size_t data_cnt) {
tree->AddPredictionToScore(data_, data_indices, data_cnt, score_);
data_size_t data_cnt, int curr_class) {
tree->AddPredictionToScore(data_, data_indices, data_cnt, score_ + curr_class * num_data_);
}
/*! \brief Pointer of score */
inline const score_t * score() { return score_; }
......@@ -72,7 +72,7 @@ private:
data_size_t num_data_;
/*! \brief Pointer of data set */
const Dataset* data_;
/*! \brief scores for data set */
/*! \brief Scores for data set */
score_t* score_;
};
......
src/io/bin.cpp
View file @ 2a8d38c5
......@@ -23,7 +23,7 @@ BinMapper::BinMapper(const BinMapper& other)
num_bin_ = other.num_bin_;
is_trival_ = other.is_trival_;
sparse_rate_ = other.sparse_rate_;
bin_upper_bound_ = new double[num_bin_];
bin_upper_bound_ = new float[num_bin_];
for (int i = 0; i < num_bin_; ++i) {
bin_upper_bound_[i] = other.bin_upper_bound_[i];
}
......@@ -38,10 +38,10 @@ BinMapper::~BinMapper() {
delete[] bin_upper_bound_;
}
void BinMapper::FindBin(std::vector<double>* values, int max_bin) {
void BinMapper::FindBin(std::vector<float>* values, int max_bin) {
size_t sample_size = values->size();
// find distinct_values first
double* distinct_values = new double[sample_size];
float* distinct_values = new float[sample_size];
int *counts = new int[sample_size];
int num_values = 1;
std::sort(values->begin(), values->end());
......@@ -61,19 +61,19 @@ void BinMapper::FindBin(std::vector<double>* values, int max_bin) {
if (num_values <= max_bin) {
// use distinct value is enough
num_bin_ = num_values;
bin_upper_bound_ = new double[num_values];
bin_upper_bound_ = new float[num_values];
for (int i = 0; i < num_values - 1; ++i) {
bin_upper_bound_[i] = (distinct_values[i] + distinct_values[i + 1]) / 2;
}
cnt_in_bin0 = counts[0];
bin_upper_bound_[num_values - 1] = std::numeric_limits<double>::infinity();
bin_upper_bound_[num_values - 1] = std::numeric_limits<float>::infinity();
} else {
// need find bins
num_bin_ = max_bin;
bin_upper_bound_ = new double[max_bin];
double * bin_lower_bound = new double[max_bin];
bin_upper_bound_ = new float[max_bin];
float * bin_lower_bound = new float[max_bin];
// mean size for one bin
double mean_bin_size = sample_size / static_cast<double>(max_bin);
float mean_bin_size = sample_size / static_cast<float>(max_bin);
int rest_sample_cnt = static_cast<int>(sample_size);
int cur_cnt_inbin = 0;
int bin_cnt = 0;
......@@ -88,24 +88,24 @@ void BinMapper::FindBin(std::vector<double>* values, int max_bin) {
++bin_cnt;
bin_lower_bound[bin_cnt] = distinct_values[i + 1];
cur_cnt_inbin = 0;
mean_bin_size = rest_sample_cnt / static_cast<double>(max_bin - bin_cnt);
mean_bin_size = rest_sample_cnt / static_cast<float>(max_bin - bin_cnt);
}
}
cur_cnt_inbin += counts[num_values - 1];
// update bin upper bound
for (int i = 0; i < bin_cnt; ++i) {
bin_upper_bound_[i] = (bin_upper_bound_[i] + bin_lower_bound[i + 1]) / 2.0;
bin_upper_bound_[i] = (bin_upper_bound_[i] + bin_lower_bound[i + 1]) / 2.0f;
}
// last bin upper bound
bin_upper_bound_[bin_cnt] = std::numeric_limits<double>::infinity();
bin_upper_bound_[bin_cnt] = std::numeric_limits<float>::infinity();
++bin_cnt;
delete[] bin_lower_bound;
// if no so much bin
if (bin_cnt < max_bin) {
// old bin data
double * tmp_bin_upper_bound = bin_upper_bound_;
float* tmp_bin_upper_bound = bin_upper_bound_;
num_bin_ = bin_cnt;
bin_upper_bound_ = new double[num_bin_];
bin_upper_bound_ = new float[num_bin_];
// copy back
for (int i = 0; i < num_bin_; ++i) {
bin_upper_bound_[i] = tmp_bin_upper_bound[i];
......@@ -123,7 +123,7 @@ void BinMapper::FindBin(std::vector<double>* values, int max_bin) {
is_trival_ = false;
}
// calculate sparse rate
sparse_rate_ = static_cast<double>(cnt_in_bin0) / static_cast<double>(sample_size);
sparse_rate_ = static_cast<float>(cnt_in_bin0) / static_cast<float>(sample_size);
}
......@@ -131,8 +131,8 @@ int BinMapper::SizeForSpecificBin(int bin) {
int size = 0;
size += sizeof(int);
size += sizeof(bool);
size += sizeof(double);
size += bin * sizeof(double);
size += sizeof(float);
size += bin * sizeof(float);
return size;
}
......@@ -143,7 +143,7 @@ void BinMapper::CopyTo(char * buffer) {
buffer += sizeof(is_trival_);
std::memcpy(buffer, &sparse_rate_, sizeof(sparse_rate_));
buffer += sizeof(sparse_rate_);
std::memcpy(buffer, bin_upper_bound_, num_bin_ * sizeof(double));
std::memcpy(buffer, bin_upper_bound_, num_bin_ * sizeof(float));
}
void BinMapper::CopyFrom(const char * buffer) {
......@@ -154,19 +154,19 @@ void BinMapper::CopyFrom(const char * buffer) {
std::memcpy(&sparse_rate_, buffer, sizeof(sparse_rate_));
buffer += sizeof(sparse_rate_);
if (bin_upper_bound_ != nullptr) { delete[] bin_upper_bound_; }
bin_upper_bound_ = new double[num_bin_];
std::memcpy(bin_upper_bound_, buffer, num_bin_ * sizeof(double));
bin_upper_bound_ = new float[num_bin_];
std::memcpy(bin_upper_bound_, buffer, num_bin_ * sizeof(float));
}
void BinMapper::SaveBinaryToFile(FILE* file) const {
fwrite(&num_bin_, sizeof(num_bin_), 1, file);
fwrite(&is_trival_, sizeof(is_trival_), 1, file);
fwrite(&sparse_rate_, sizeof(sparse_rate_), 1, file);
fwrite(bin_upper_bound_, sizeof(double), num_bin_, file);
fwrite(bin_upper_bound_, sizeof(float), num_bin_, file);
}
size_t BinMapper::SizesInByte() const {
return sizeof(num_bin_) + sizeof(is_trival_) + sizeof(sparse_rate_) + sizeof(double) * num_bin_;
return sizeof(num_bin_) + sizeof(is_trival_) + sizeof(sparse_rate_) + sizeof(float) * num_bin_;
}
template class DenseBin<uint8_t>;
......@@ -182,9 +182,9 @@ template class OrderedSparseBin<uint16_t>;
template class OrderedSparseBin<uint32_t>;
Bin* Bin::CreateBin(data_size_t num_data, int num_bin, double sparse_rate, bool is_enable_sparse, bool* is_sparse, int default_bin) {
Bin* Bin::CreateBin(data_size_t num_data, int num_bin, float sparse_rate, bool is_enable_sparse, bool* is_sparse, int default_bin) {
// sparse threshold
const double kSparseThreshold = 0.8;
const float kSparseThreshold = 0.8f;
if (sparse_rate >= kSparseThreshold && is_enable_sparse) {
*is_sparse = true;
return CreateSparseBin(num_data, num_bin, default_bin);
......
src/io/config.cpp
View file @ 2a8d38c5
......@@ -10,6 +10,26 @@
namespace LightGBM {
void OverallConfig::LoadFromString(const char* str) {
std::unordered_map<std::string, std::string> params;
auto args = Common::Split(str, " \t\n\r");
for (auto arg : args) {
std::vector<std::string> tmp_strs = Common::Split(arg.c_str(), '=');
if (tmp_strs.size() == 2) {
std::string key = Common::RemoveQuotationSymbol(Common::Trim(tmp_strs[0]));
std::string value = Common::RemoveQuotationSymbol(Common::Trim(tmp_strs[1]));
if (key.size() <= 0) {
continue;
}
params[key] = value;
} else {
Log::Error("Unknown parameter %s", arg.c_str());
}
}
ParameterAlias::KeyAliasTransform(&params);
Set(params);
}
void OverallConfig::Set(const std::unordered_map<std::string, std::string>& params) {
// load main config types
GetInt(params, "num_threads", &num_threads);
......@@ -26,7 +46,6 @@ void OverallConfig::Set(const std::unordered_map<std::string, std::string>& para
boosting_config = new GBDTConfig();
}
// sub-config setup
network_config.Set(params);
io_config.Set(params);
......@@ -113,6 +132,28 @@ void OverallConfig::GetTaskType(const std::unordered_map<std::string, std::strin
void OverallConfig::CheckParamConflict() {
GBDTConfig* gbdt_config = dynamic_cast<GBDTConfig*>(boosting_config);
// check if objective_type, metric_type, and num_class match
bool objective_type_multiclass = (objective_type == std::string("multiclass"));
int num_class_check = gbdt_config->num_class;
if (objective_type_multiclass){
if (num_class_check <= 1){
Log::Fatal("You should specify number of class(>=2) for multiclass training.");
}
}
else {
if (task_type == TaskType::kTrain && num_class_check != 1){
Log::Fatal("Number of class must be 1 for non-multiclass training.");
}
}
for (std::string metric_type : metric_types){
bool metric_type_multiclass = ( metric_type == std::string("multi_logloss") || metric_type == std::string("multi_error"));
if ((objective_type_multiclass && !metric_type_multiclass)
|| (!objective_type_multiclass && metric_type_multiclass)){
Log::Fatal("Objective and metrics don't match.");
}
}
if (network_config.num_machines > 1) {
is_parallel = true;
} else {
......@@ -159,48 +200,52 @@ void IOConfig::Set(const std::unordered_map<std::string, std::string>& params) {
GetString(params, "input_model", &input_model);
GetString(params, "output_result", &output_result);
GetString(params, "input_init_score", &input_init_score);
GetString(params, "log_file", &log_file);
std::string tmp_str = "";
if (GetString(params, "valid_data", &tmp_str)) {
valid_data_filenames = Common::Split(tmp_str.c_str(), ',');
}
GetBool(params, "has_header", &has_header);
GetString(params, "label_column", &label_column);
GetString(params, "weight_column", &weight_column);
GetString(params, "group_column", &group_column);
GetString(params, "ignore_column", &ignore_column);
}
void ObjectiveConfig::Set(const std::unordered_map<std::string, std::string>& params) {
GetBool(params, "is_unbalance", &is_unbalance);
GetDouble(params, "sigmoid", &sigmoid);
GetFloat(params, "sigmoid", &sigmoid);
GetInt(params, "max_position", &max_position);
CHECK(max_position > 0);
GetInt(params, "num_class", &num_class);
CHECK(num_class >= 1);
std::string tmp_str = "";
if (GetString(params, "label_gain", &tmp_str)) {
label_gain = Common::StringToDoubleArray(tmp_str, ',');
label_gain = Common::StringToFloatArray(tmp_str, ',');
} else {
// label_gain = 2^i - 1, may overflow, so we use 31 here
const int max_label = 31;
label_gain.push_back(0.0);
label_gain.push_back(0.0f);
for (int i = 1; i < max_label; ++i) {
label_gain.push_back((1 << i) - 1);
label_gain.push_back(static_cast<float>((1 << i) - 1));
}
}
}
void MetricConfig::Set(const std::unordered_map<std::string, std::string>& params) {
GetInt(params, "early_stopping_round", &early_stopping_round);
GetInt(params, "metric_freq", &output_freq);
CHECK(output_freq >= 0);
GetDouble(params, "sigmoid", &sigmoid);
GetBool(params, "is_training_metric", &is_provide_training_metric);
GetFloat(params, "sigmoid", &sigmoid);
GetInt(params, "num_class", &num_class);
CHECK(num_class >= 1);
std::string tmp_str = "";
if (GetString(params, "label_gain", &tmp_str)) {
label_gain = Common::StringToDoubleArray(tmp_str, ',');
label_gain = Common::StringToFloatArray(tmp_str, ',');
} else {
// label_gain = 2^i - 1, may overflow, so we use 31 here
const int max_label = 31;
label_gain.push_back(0.0);
label_gain.push_back(0.0f);
for (int i = 1; i < max_label; ++i) {
label_gain.push_back((1 << i) - 1);
label_gain.push_back(static_cast<float>((1 << i) - 1));
}
}
if (GetString(params, "ndcg_eval_at", &tmp_str)) {
......@@ -220,14 +265,16 @@ void MetricConfig::Set(const std::unordered_map<std::string, std::string>& param
void TreeConfig::Set(const std::unordered_map<std::string, std::string>& params) {
GetInt(params, "min_data_in_leaf", &min_data_in_leaf);
GetDouble(params, "min_sum_hessian_in_leaf", &min_sum_hessian_in_leaf);
GetFloat(params, "min_sum_hessian_in_leaf", &min_sum_hessian_in_leaf);
CHECK(min_sum_hessian_in_leaf > 1.0f || min_data_in_leaf > 0);
GetInt(params, "num_leaves", &num_leaves);
CHECK(num_leaves > 1);
GetInt(params, "feature_fraction_seed", &feature_fraction_seed);
GetDouble(params, "feature_fraction", &feature_fraction);
CHECK(feature_fraction > 0.0 && feature_fraction <= 1.0);
GetDouble(params, "histogram_pool_size", &histogram_pool_size);
GetFloat(params, "feature_fraction", &feature_fraction);
CHECK(feature_fraction > 0.0f && feature_fraction <= 1.0f);
GetFloat(params, "histogram_pool_size", &histogram_pool_size);
GetInt(params, "max_depth", &max_depth);
CHECK(max_depth > 1 || max_depth < 0);
}
......@@ -237,12 +284,17 @@ void BoostingConfig::Set(const std::unordered_map<std::string, std::string>& par
GetInt(params, "bagging_seed", &bagging_seed);
GetInt(params, "bagging_freq", &bagging_freq);
CHECK(bagging_freq >= 0);
GetDouble(params, "bagging_fraction", &bagging_fraction);
CHECK(bagging_fraction > 0.0 && bagging_fraction <= 1.0);
GetDouble(params, "learning_rate", &learning_rate);
CHECK(learning_rate > 0.0);
GetFloat(params, "bagging_fraction", &bagging_fraction);
CHECK(bagging_fraction > 0.0f && bagging_fraction <= 1.0f);
GetFloat(params, "learning_rate", &learning_rate);
CHECK(learning_rate > 0.0f);
GetInt(params, "early_stopping_round", &early_stopping_round);
CHECK(early_stopping_round >= 0);
GetInt(params, "metric_freq", &output_freq);
CHECK(output_freq >= 0);
GetBool(params, "is_training_metric", &is_provide_training_metric);
GetInt(params, "num_class", &num_class);
CHECK(num_class >= 1);
}
void GBDTConfig::GetTreeLearnerType(const std::unordered_map<std::string, std::string>& params) {
......
src/io/dataset.cpp
View file @ 2a8d38c5
......@@ -11,13 +11,14 @@
#include <vector>
#include <utility>
#include <string>
#include <sstream>
namespace LightGBM {
Dataset::Dataset(const char* data_filename, const char* init_score_filename,
int max_bin, int random_seed, bool is_enable_sparse, const PredictFunction& predict_fun)
:data_filename_(data_filename), random_(random_seed),
max_bin_(max_bin), is_enable_sparse_(is_enable_sparse), predict_fun_(predict_fun) {
const IOConfig& io_config, const PredictFunction& predict_fun)
:data_filename_(data_filename), random_(io_config.data_random_seed),
max_bin_(io_config.max_bin), is_enable_sparse_(io_config.is_enable_sparse), predict_fun_(predict_fun) {
CheckCanLoadFromBin();
if (is_loading_from_binfile_ && predict_fun != nullptr) {
......@@ -28,15 +29,129 @@ Dataset::Dataset(const char* data_filename, const char* init_score_filename,
if (!is_loading_from_binfile_) {
// load weight, query information and initilize score
metadata_.Init(data_filename, init_score_filename);
// create text reader
text_reader_ = new TextReader<data_size_t>(data_filename, io_config.has_header);
std::unordered_map<std::string, int> name2idx;
// get column names
if (io_config.has_header) {
std::string first_line = text_reader_->first_line();
feature_names_ = Common::Split(first_line.c_str(), "\t ,");
for (size_t i = 0; i < feature_names_.size(); ++i) {
name2idx[feature_names_[i]] = static_cast<int>(i);
}
}
std::string name_prefix("name:");
// load label idx
if (io_config.label_column.size() > 0) {
if (Common::StartsWith(io_config.label_column, name_prefix)) {
std::string name = io_config.label_column.substr(name_prefix.size());
if (name2idx.count(name) > 0) {
label_idx_ = name2idx[name];
Log::Info("use %s column as label", name.c_str());
} else {
Log::Fatal("cannot find label column: %s in data file", name.c_str());
}
} else {
if (!Common::AtoiAndCheck(io_config.label_column.c_str(), &label_idx_)) {
Log::Fatal("label_column is not a number, \
if you want to use column name, \
please add prefix \"name:\" before column name");
}
Log::Info("use %d-th column as label", label_idx_);
}
}
if (feature_names_.size() > 0) {
// erase label column name
feature_names_.erase(feature_names_.begin() + label_idx_);
}
// load ignore columns
if (io_config.ignore_column.size() > 0) {
if (Common::StartsWith(io_config.ignore_column, name_prefix)) {
std::string names = io_config.ignore_column.substr(name_prefix.size());
for (auto name : Common::Split(names.c_str(), ',')) {
if (name2idx.count(name) > 0) {
int tmp = name2idx[name];
// skip for label column
if (tmp > label_idx_) { tmp -= 1; }
ignore_features_.emplace(tmp);
} else {
Log::Fatal("cannot find column: %s in data file", name.c_str());
}
}
} else {
for (auto token : Common::Split(io_config.ignore_column.c_str(), ',')) {
int tmp = 0;
if (!Common::AtoiAndCheck(token.c_str(), &tmp)) {
Log::Fatal("ignore_column is not a number, \
if you want to use column name, \
please add prefix \"name:\" before column name");
}
// skip for label column
if (tmp > label_idx_) { tmp -= 1; }
ignore_features_.emplace(tmp);
}
}
}
// load weight idx
if (io_config.weight_column.size() > 0) {
if (Common::StartsWith(io_config.weight_column, name_prefix)) {
std::string name = io_config.weight_column.substr(name_prefix.size());
if (name2idx.count(name) > 0) {
weight_idx_ = name2idx[name];
Log::Info("use %s column as weight", name.c_str());
} else {
Log::Fatal("cannot find weight column: %s in data file", name.c_str());
}
} else {
if (!Common::AtoiAndCheck(io_config.weight_column.c_str(), &weight_idx_)) {
Log::Fatal("weight_column is not a number, \
if you want to use column name, \
please add prefix \"name:\" before column name");
}
Log::Info("use %d-th column as weight", weight_idx_);
}
// skip for label column
if (weight_idx_ > label_idx_) {
weight_idx_ -= 1;
}
ignore_features_.emplace(weight_idx_);
}
if (io_config.group_column.size() > 0) {
if (Common::StartsWith(io_config.group_column, name_prefix)) {
std::string name = io_config.group_column.substr(name_prefix.size());
if (name2idx.count(name) > 0) {
group_idx_ = name2idx[name];
Log::Info("use %s column as group/query id", name.c_str());
} else {
Log::Fatal("cannot find group/query column: %s in data file", name.c_str());
}
} else {
if (!Common::AtoiAndCheck(io_config.group_column.c_str(), &group_idx_)) {
Log::Fatal("group_column is not a number, \
if you want to use column name, \
please add prefix \"name:\" before column name");
}
Log::Info("use %d-th column as group/query id", group_idx_);
}
// skip for label column
if (group_idx_ > label_idx_) {
group_idx_ -= 1;
}
ignore_features_.emplace(group_idx_);
}
// create text parser
parser_ = Parser::CreateParser(data_filename_, 0, nullptr);
parser_ = Parser::CreateParser(data_filename_, io_config.has_header, 0, label_idx_);
if (parser_ == nullptr) {
Log::Fatal("Cannot recognising input data format, filename: %s", data_filename_);
}
// create text reader
text_reader_ = new TextReader<data_size_t>(data_filename);
} else {
// only need to load initilize score, other meta data will load from bin flie
// only need to load initilize score, other meta data will be loaded from bin flie
metadata_.Init(init_score_filename);
Log::Info("Loading data set from binary file");
parser_ = nullptr;
......@@ -159,10 +274,10 @@ void Dataset::SampleDataFromFile(int rank, int num_machines, bool is_pre_partiti
void Dataset::ConstructBinMappers(int rank, int num_machines, const std::vector<std::string>& sample_data) {
// sample_values[i][j], means the value of j-th sample on i-th feature
std::vector<std::vector<double>> sample_values;
std::vector<std::vector<float>> sample_values;
// temp buffer for one line features and label
std::vector<std::pair<int, double>> oneline_features;
double label;
std::vector<std::pair<int, float>> oneline_features;
float label;
for (size_t i = 0; i < sample_data.size(); ++i) {
oneline_features.clear();
// parse features
......@@ -171,13 +286,13 @@ void Dataset::ConstructBinMappers(int rank, int num_machines, const std::vector<
for (auto& feature_values : sample_values) {
feature_values.push_back(0.0);
}
for (std::pair<int, double>& inner_data : oneline_features) {
for (std::pair<int, float>& inner_data : oneline_features) {
if (static_cast<size_t>(inner_data.first) >= sample_values.size()) {
// if need expand feature set
size_t need_size = inner_data.first - sample_values.size() + 1;
for (size_t j = 0; j < need_size; ++j) {
// push i+1 0
sample_values.emplace_back(i + 1, 0.0);
sample_values.emplace_back(i + 1, 0.0f);
}
}
// edit the feature value
......@@ -190,18 +305,40 @@ void Dataset::ConstructBinMappers(int rank, int num_machines, const std::vector<
// -1 means doesn't use this feature
used_feature_map_ = std::vector<int>(sample_values.size(), -1);
num_total_features_ = static_cast<int>(sample_values.size());
// check the range of label_idx, weight_idx and group_idx
CHECK(label_idx_ >= 0 && label_idx_ <= num_total_features_);
CHECK(weight_idx_ < 0 || weight_idx_ < num_total_features_);
CHECK(group_idx_ < 0 || group_idx_ < num_total_features_);
// fill feature_names_ if not header
if (feature_names_.size() <= 0) {
for (int i = 0; i < num_total_features_; ++i) {
std::stringstream str_buf;
str_buf << "Column_" << i;
feature_names_.push_back(str_buf.str());
}
}
// start find bins
if (num_machines == 1) {
std::vector<BinMapper*> bin_mappers(sample_values.size());
// if only 1 machines, find bin locally
#pragma omp parallel for schedule(guided)
for (int i = 0; i < static_cast<int>(sample_values.size()); ++i) {
if (ignore_features_.count(i) > 0) {
bin_mappers[i] = nullptr;
continue;
}
bin_mappers[i] = new BinMapper();
bin_mappers[i]->FindBin(&sample_values[i], max_bin_);
}
for (size_t i = 0; i < sample_values.size(); ++i) {
if (!bin_mappers[i]->is_trival()) {
if (bin_mappers[i] == nullptr) {
Log::Error("Ignore Feature %s ", feature_names_[i].c_str());
}
else if (!bin_mappers[i]->is_trival()) {
// map real feature index to used feature index
used_feature_map_[i] = static_cast<int>(features_.size());
// push new feature
......@@ -209,7 +346,7 @@ void Dataset::ConstructBinMappers(int rank, int num_machines, const std::vector<
num_data_, is_enable_sparse_));
} else {
// if feature is trival(only 1 bin), free spaces
Log::Error("Feature %d only contains one value, will be ignored", i);
Log::Error("Feature %s only contains one value, will be ignored", feature_names_[i].c_str());
delete bin_mappers[i];
}
}
......@@ -256,12 +393,17 @@ void Dataset::ConstructBinMappers(int rank, int num_machines, const std::vector<
Network::Allgather(input_buffer, buffer_size, start, len, output_buffer);
// restore features bins from buffer
for (int i = 0; i < total_num_feature; ++i) {
if (ignore_features_.count(i) > 0) {
Log::Error("Ignore Feature %s ", feature_names_[i].c_str());
continue;
}
BinMapper* bin_mapper = new BinMapper();
bin_mapper->CopyFrom(output_buffer + i * type_size);
if (!bin_mapper->is_trival()) {
used_feature_map_[i] = static_cast<int>(features_.size());
features_.push_back(new Feature(static_cast<int>(i), bin_mapper, num_data_, is_enable_sparse_));
} else {
Log::Error("Feature %s only contains one value, will be ignored", feature_names_[i].c_str());
delete bin_mapper;
}
}
......@@ -276,6 +418,13 @@ void Dataset::ConstructBinMappers(int rank, int num_machines, const std::vector<
void Dataset::LoadTrainData(int rank, int num_machines, bool is_pre_partition, bool use_two_round_loading) {
// don't support query id in data file when training in parallel
if (num_machines > 1 && !is_pre_partition) {
if (group_idx_ > 0) {
Log::Fatal("Don't support query id in data file when training parallel without pre-partition. \
Please use an additional query file or pre-partition your data");
}
}
used_data_indices_.clear();
if (!is_loading_from_binfile_ ) {
if (!use_two_round_loading) {
......@@ -287,7 +436,7 @@ void Dataset::LoadTrainData(int rank, int num_machines, bool is_pre_partition, b
// construct feature bin mappers
ConstructBinMappers(rank, num_machines, sample_data);
// initialize label
metadata_.InitLabel(num_data_);
metadata_.Init(num_data_, weight_idx_, group_idx_);
// extract features
ExtractFeaturesFromMemory();
} else {
......@@ -297,7 +446,7 @@ void Dataset::LoadTrainData(int rank, int num_machines, bool is_pre_partition, b
// construct feature bin mappers
ConstructBinMappers(rank, num_machines, sample_data);
// initialize label
metadata_.InitLabel(num_data_);
metadata_.Init(num_data_, weight_idx_, group_idx_);
// extract features
ExtractFeaturesFromFile();
......@@ -322,7 +471,7 @@ void Dataset::LoadValidationData(const Dataset* train_set, bool use_two_round_lo
// read data in memory
LoadDataToMemory(0, 1, false);
// initialize label
metadata_.InitLabel(num_data_);
metadata_.Init(num_data_, weight_idx_, group_idx_);
features_.clear();
// copy feature bin mapper data
for (Feature* feature : train_set->features_) {
......@@ -336,7 +485,7 @@ void Dataset::LoadValidationData(const Dataset* train_set, bool use_two_round_lo
// Get number of lines of data file
num_data_ = static_cast<data_size_t>(text_reader_->CountLine());
// initialize label
metadata_.InitLabel(num_data_);
metadata_.Init(num_data_, weight_idx_, group_idx_);
features_.clear();
// copy feature bin mapper data
for (Feature* feature : train_set->features_) {
......@@ -358,8 +507,8 @@ void Dataset::LoadValidationData(const Dataset* train_set, bool use_two_round_lo
}
void Dataset::ExtractFeaturesFromMemory() {
std::vector<std::pair<int, double>> oneline_features;
double tmp_label = 0.0;
std::vector<std::pair<int, float>> oneline_features;
float tmp_label = 0.0f;
if (predict_fun_ == nullptr) {
// if doesn't need to prediction with initial model
#pragma omp parallel for schedule(guided) private(oneline_features) firstprivate(tmp_label)
......@@ -381,11 +530,18 @@ void Dataset::ExtractFeaturesFromMemory() {
// if is used feature
features_[feature_idx]->PushData(tid, i, inner_data.second);
}
else {
if (inner_data.first == weight_idx_) {
metadata_.SetWeightAt(i, inner_data.second);
} else if (inner_data.first == group_idx_) {
metadata_.SetQueryAt(i, inner_data.second);
}
}
}
}
} else {
// if need to prediction with initial model
score_t* init_score = new score_t[num_data_];
float* init_score = new float[num_data_];
#pragma omp parallel for schedule(guided) private(oneline_features) firstprivate(tmp_label)
for (data_size_t i = 0; i < num_data_; ++i) {
const int tid = omp_get_thread_num();
......@@ -393,7 +549,7 @@ void Dataset::ExtractFeaturesFromMemory() {
// parser
parser_->ParseOneLine(text_reader_->Lines()[i].c_str(), &oneline_features, &tmp_label);
// set initial score
init_score[i] = static_cast<score_t>(predict_fun_(oneline_features));
init_score[i] = static_cast<float>(predict_fun_(oneline_features));
// set label
metadata_.SetLabelAt(i, tmp_label);
// free processed line:
......@@ -407,14 +563,22 @@ void Dataset::ExtractFeaturesFromMemory() {
// if is used feature
features_[feature_idx]->PushData(tid, i, inner_data.second);
}
else {
if (inner_data.first == weight_idx_) {
metadata_.SetWeightAt(i, inner_data.second);
} else if (inner_data.first == group_idx_) {
metadata_.SetQueryAt(i, inner_data.second);
}
}
}
}
// metadata_ will manage space of init_score
metadata_.SetInitScore(init_score);
metadata_.SetInitScore(init_score, num_data_);
delete[] init_score;
}
#pragma omp parallel for schedule(guided)
for (int i = 0; i < num_features_; i++) {
for (int i = 0; i < num_features_; ++i) {
features_[i]->FinishLoad();
}
// text data can be free after loaded feature values
......@@ -423,24 +587,24 @@ void Dataset::ExtractFeaturesFromMemory() {
void Dataset::ExtractFeaturesFromFile() {
score_t* init_score = nullptr;
float* init_score = nullptr;
if (predict_fun_ != nullptr) {
init_score = new score_t[num_data_];
init_score = new float[num_data_];
}
std::function<void(data_size_t, const std::vector<std::string>&)> process_fun =
[this, &init_score]
(data_size_t start_idx, const std::vector<std::string>& lines) {
std::vector<std::pair<int, double>> oneline_features;
double tmp_label = 0.0;
std::vector<std::pair<int, float>> oneline_features;
float tmp_label = 0.0f;
#pragma omp parallel for schedule(static) private(oneline_features) firstprivate(tmp_label)
for (data_size_t i = 0; i < static_cast<data_size_t>(lines.size()); i++) {
for (data_size_t i = 0; i < static_cast<data_size_t>(lines.size()); ++i) {
const int tid = omp_get_thread_num();
oneline_features.clear();
// parser
parser_->ParseOneLine(lines[i].c_str(), &oneline_features, &tmp_label);
// set initial score
if (init_score != nullptr) {
init_score[start_idx + i] = static_cast<score_t>(predict_fun_(oneline_features));
init_score[start_idx + i] = static_cast<float>(predict_fun_(oneline_features));
}
// set label
metadata_.SetLabelAt(start_idx + i, tmp_label);
......@@ -451,6 +615,13 @@ void Dataset::ExtractFeaturesFromFile() {
// if is used feature
features_[feature_idx]->PushData(tid, start_idx + i, inner_data.second);
}
else {
if (inner_data.first == weight_idx_) {
metadata_.SetWeightAt(start_idx + i, inner_data.second);
} else if (inner_data.first == group_idx_) {
metadata_.SetQueryAt(start_idx + i, inner_data.second);
}
}
}
}
};
......@@ -465,11 +636,12 @@ void Dataset::ExtractFeaturesFromFile() {
// metadata_ will manage space of init_score
if (init_score != nullptr) {
metadata_.SetInitScore(init_score);
metadata_.SetInitScore(init_score, num_data_);
delete[] init_score;
}
#pragma omp parallel for schedule(guided)
for (int i = 0; i < num_features_; i++) {
for (int i = 0; i < num_features_; ++i) {
features_[i]->FinishLoad();
}
}
......@@ -613,7 +785,7 @@ void Dataset::LoadDataFromBinFile(int rank, int num_machines, bool is_pre_partit
size_t size_of_metadata = *(reinterpret_cast<size_t*>(buffer));
// re-allocmate space if not enough
// re-allocate space if not enough
if (size_of_metadata > buffer_size) {
delete[] buffer;
buffer_size = size_of_metadata;
......@@ -635,7 +807,7 @@ void Dataset::LoadDataFromBinFile(int rank, int num_machines, bool is_pre_partit
const data_size_t* query_boundaries = metadata_.query_boundaries();
if (query_boundaries == nullptr) {
// if not contain query file, minimal sample unit is one record
for (data_size_t i = 0; i < num_data_; i++) {
for (data_size_t i = 0; i < num_data_; ++i) {
if (random_.NextInt(0, num_machines) == rank) {
used_data_indices_.push_back(i);
}
......@@ -645,7 +817,7 @@ void Dataset::LoadDataFromBinFile(int rank, int num_machines, bool is_pre_partit
data_size_t num_queries = metadata_.num_queries();
data_size_t qid = -1;
bool is_query_used = false;
for (data_size_t i = 0; i < num_data_; i++) {
for (data_size_t i = 0; i < num_data_; ++i) {
if (qid >= num_queries) {
Log::Fatal("current query is exceed the range of query file, please ensure your query file is correct");
}
......@@ -673,7 +845,7 @@ void Dataset::LoadDataFromBinFile(int rank, int num_machines, bool is_pre_partit
Log::Fatal("Binary file format error at feature %d's size", i);
}
size_t size_of_feature = *(reinterpret_cast<size_t*>(buffer));
// re-allocmate space if not enough
// re-allocate space if not enough
if (size_of_feature > buffer_size) {
delete[] buffer;
buffer_size = size_of_feature;
......
src/io/dense_bin.hpp
View file @ 2a8d38c5
......@@ -10,7 +10,7 @@
namespace LightGBM {
/*!
* \brief Used to Store bins for dense feature
* \brief Used to store bins for dense feature
* Use template to reduce memory cost
*/
template <typename VAL_T>
......
src/io/metadata.cpp
View file @ 2a8d38c5
......@@ -10,7 +10,7 @@ namespace LightGBM {
Metadata::Metadata()
:label_(nullptr), label_int_(nullptr), weights_(nullptr),
query_boundaries_(nullptr),
query_weights_(nullptr), init_score_(nullptr) {
query_weights_(nullptr), init_score_(nullptr), queries_(nullptr){
}
......@@ -36,12 +36,31 @@ Metadata::~Metadata() {
if (query_boundaries_ != nullptr) { delete[] query_boundaries_; }
if (query_weights_ != nullptr) { delete[] query_weights_; }
if (init_score_ != nullptr) { delete[] init_score_; }
if (queries_ != nullptr) { delete[] queries_; }
}
void Metadata::InitLabel(data_size_t num_data) {
void Metadata::Init(data_size_t num_data, int weight_idx, int query_idx) {
num_data_ = num_data;
label_ = new float[num_data_];
if (weight_idx >= 0) {
if (weights_ != nullptr) {
Log::Info("using weight in data file, and ignore additional weight file");
delete[] weights_;
}
weights_ = new float[num_data_];
num_weights_ = num_data_;
memset(weights_, 0, sizeof(float) * num_data_);
}
if (query_idx >= 0) {
if (query_boundaries_ != nullptr) {
Log::Info("using query id in data file, and ignore additional query file");
delete[] query_boundaries_;
}
if (query_weights_ != nullptr) { delete[] query_weights_; }
queries_ = new data_size_t[num_data_];
memset(queries_, 0, sizeof(data_size_t) * num_data_);
}
}
void Metadata::PartitionLabel(const std::vector<data_size_t>& used_indices) {
......@@ -59,9 +78,35 @@ void Metadata::PartitionLabel(const std::vector<data_size_t>& used_indices) {
void Metadata::CheckOrPartition(data_size_t num_all_data, const std::vector<data_size_t>& used_data_indices) {
if (used_data_indices.size() == 0) {
if (queries_ != nullptr) {
// need convert query_id to boundaries
std::vector<data_size_t> tmp_buffer;
data_size_t last_qid = -1;
data_size_t cur_cnt = 0;
for (data_size_t i = 0; i < num_data_; ++i) {
if (last_qid != queries_[i]) {
if (cur_cnt > 0) {
tmp_buffer.push_back(cur_cnt);
}
cur_cnt = 0;
last_qid = queries_[i];
}
++cur_cnt;
}
tmp_buffer.push_back(cur_cnt);
query_boundaries_ = new data_size_t[tmp_buffer.size() + 1];
num_queries_ = static_cast<data_size_t>(tmp_buffer.size());
query_boundaries_[0] = 0;
for (size_t i = 0; i < tmp_buffer.size(); ++i) {
query_boundaries_[i + 1] = query_boundaries_[i] + tmp_buffer[i];
}
LoadQueryWeights();
delete[] queries_;
queries_ = nullptr;
}
// check weights
if (weights_ != nullptr && num_weights_ != num_data_) {
Log::Error("Initial weight size doesn't equal to data, weights will be ignored");
Log::Fatal("Initial weight size doesn't equal to data");
delete[] weights_;
num_weights_ = 0;
weights_ = nullptr;
......@@ -69,7 +114,7 @@ void Metadata::CheckOrPartition(data_size_t num_all_data, const std::vector<data
// check query boundries
if (query_boundaries_ != nullptr && query_boundaries_[num_queries_] != num_data_) {
Log::Error("Initial query size doesn't equal to data, queies will be ignored");
Log::Fatal("Initial query size doesn't equal to data");
delete[] query_boundaries_;
num_queries_ = 0;
query_boundaries_ = nullptr;
......@@ -78,21 +123,22 @@ void Metadata::CheckOrPartition(data_size_t num_all_data, const std::vector<data
// contain initial score file
if (init_score_ != nullptr && num_init_score_ != num_data_) {
delete[] init_score_;
Log::Error("Initial score size doesn't equal to data, score file will be ignored");
Log::Fatal("Initial score size doesn't equal to data");
init_score_ = nullptr;
num_init_score_ = 0;
}
} else {
data_size_t num_used_data = static_cast<data_size_t>(used_data_indices.size());
// check weights
if (weights_ != nullptr && num_weights_ != num_all_data) {
Log::Error("Initial weights size doesn't equal to data, weights will be ignored");
Log::Fatal("Initial weights size doesn't equal to data");
delete[] weights_;
num_weights_ = 0;
weights_ = nullptr;
}
// check query boundries
if (query_boundaries_ != nullptr && query_boundaries_[num_queries_] != num_all_data) {
Log::Error("Initial query size doesn't equal to data , queries will be ignored");
Log::Fatal("Initial query size doesn't equal to data");
delete[] query_boundaries_;
num_queries_ = 0;
query_boundaries_ = nullptr;
......@@ -100,9 +146,10 @@ void Metadata::CheckOrPartition(data_size_t num_all_data, const std::vector<data
// contain initial score file
if (init_score_ != nullptr && num_init_score_ != num_all_data) {
Log::Error("Initial score size doesn't equal to data , initial scores will be ignored");
Log::Fatal("Initial score size doesn't equal to data");
delete[] init_score_;
num_init_score_ = 0;
init_score_ = nullptr;
}
// get local weights
......@@ -131,10 +178,10 @@ void Metadata::CheckOrPartition(data_size_t num_all_data, const std::vector<data
used_query.push_back(qid);
data_idx += len;
} else {
Log::Fatal("Data partition error, data didn't match queies");
Log::Fatal("Data partition error, data didn't match queries");
}
} else {
Log::Fatal("Data partition error, data didn't match queies");
Log::Fatal("Data partition error, data didn't match queries");
}
}
data_size_t * old_query_boundaries = query_boundaries_;
......@@ -151,9 +198,9 @@ void Metadata::CheckOrPartition(data_size_t num_all_data, const std::vector<data
// get local initial scores
if (init_score_ != nullptr) {
score_t* old_scores = init_score_;
float* old_scores = init_score_;
num_init_score_ = num_data_;
init_score_ = new score_t[num_init_score_];
init_score_ = new float[num_init_score_];
for (size_t i = 0; i < used_data_indices.size(); ++i) {
init_score_[i] = old_scores[used_data_indices[i]];
}
......@@ -166,10 +213,16 @@ void Metadata::CheckOrPartition(data_size_t num_all_data, const std::vector<data
}
void Metadata::SetInitScore(score_t* init_score) {
void Metadata::SetInitScore(const float* init_score, data_size_t len) {
if (num_data_ != len) {
Log::Fatal("len of initial score is not same with #data");
}
if (init_score_ != nullptr) { delete[] init_score_; }
num_init_score_ = num_data_;
init_score_ = init_score;
init_score_ = new float[num_init_score_];
for (data_size_t i = 0; i < num_init_score_; ++i) {
init_score_[i] = init_score[i];
}
}
void Metadata::LoadWeights() {
......@@ -177,7 +230,7 @@ void Metadata::LoadWeights() {
std::string weight_filename(data_filename_);
// default weight file name
weight_filename.append(".weight");
TextReader<size_t> reader(weight_filename.c_str());
TextReader<size_t> reader(weight_filename.c_str(), false);
reader.ReadAllLines();
if (reader.Lines().size() <= 0) {
return;
......@@ -186,25 +239,25 @@ void Metadata::LoadWeights() {
num_weights_ = static_cast<data_size_t>(reader.Lines().size());
weights_ = new float[num_weights_];
for (data_size_t i = 0; i < num_weights_; ++i) {
double tmp_weight = 0.0f;
float tmp_weight = 0.0f;
Common::Atof(reader.Lines()[i].c_str(), &tmp_weight);
weights_[i] = static_cast<float>(tmp_weight);
weights_[i] = tmp_weight;
}
}
void Metadata::LoadInitialScore() {
num_init_score_ = 0;
if (init_score_filename_[0] == '\0') { return; }
TextReader<size_t> reader(init_score_filename_);
TextReader<size_t> reader(init_score_filename_, false);
reader.ReadAllLines();
Log::Info("Start loading initial scores");
num_init_score_ = static_cast<data_size_t>(reader.Lines().size());
init_score_ = new score_t[num_init_score_];
double tmp = 0.0f;
init_score_ = new float[num_init_score_];
float tmp = 0.0f;
for (data_size_t i = 0; i < num_init_score_; ++i) {
Common::Atof(reader.Lines()[i].c_str(), &tmp);
init_score_[i] = static_cast<score_t>(tmp);
init_score_[i] = tmp;
}
}
......@@ -213,7 +266,7 @@ void Metadata::LoadQueryBoundaries() {
std::string query_filename(data_filename_);
// default query file name
query_filename.append(".query");
TextReader<size_t> reader(query_filename.c_str());
TextReader<size_t> reader(query_filename.c_str(), false);
reader.ReadAllLines();
if (reader.Lines().size() <= 0) {
return;
......
src/io/parser.cpp
View file @ 2a8d38c5
......@@ -2,6 +2,7 @@
#include <iostream>
#include <fstream>
#include <functional>
namespace LightGBM {
......@@ -20,44 +21,65 @@ void GetStatistic(const char* str, int* comma_cnt, int* tab_cnt, int* colon_cnt)
}
}
bool CheckHasLabelForLibsvm(std::string& str) {
int GetLabelIdxForLibsvm(std::string& str, int num_features, int label_idx) {
if (num_features <= 0) {
return label_idx;
}
str = Common::Trim(str);
auto pos_space = str.find_first_of(" \f\n\r\t\v");
auto pos_colon = str.find_first_of(":");
if (pos_colon == std::string::npos || pos_colon > pos_space) {
return true;
if (pos_space == std::string::npos || pos_space < pos_colon) {
return label_idx;
} else {
return false;
return -1;
}
}
bool CheckHasLabelForTSV(std::string& str, int num_features) {
int GetLabelIdxForTSV(std::string& str, int num_features, int label_idx) {
if (num_features <= 0) {
return label_idx;
}
str = Common::Trim(str);
auto tokens = Common::Split(str.c_str(), '\t');
if (static_cast<int>(tokens.size()) == num_features) {
return false;
return -1;
} else {
return true;
return label_idx;
}
}
bool CheckHasLabelForCSV(std::string& str, int num_features) {
int GetLabelIdxForCSV(std::string& str, int num_features, int label_idx) {
if (num_features <= 0) {
return label_idx;
}
str = Common::Trim(str);
auto tokens = Common::Split(str.c_str(), ',');
if (static_cast<int>(tokens.size()) == num_features) {
return false;
return -1;
} else {
return true;
return label_idx;
}
}
Parser* Parser::CreateParser(const char* filename, int num_features, bool* has_label) {
enum DataType {
INVALID,
CSV,
TSV,
LIBSVM
};
Parser* Parser::CreateParser(const char* filename, bool has_header, int num_features, int label_idx) {
std::ifstream tmp_file;
tmp_file.open(filename);
if (!tmp_file.is_open()) {
Log::Fatal("Data file: %s doesn't exist", filename);
}
std::string line1, line2;
if (has_header) {
if (!tmp_file.eof()) {
std::getline(tmp_file, line1);
}
}
if (!tmp_file.eof()) {
std::getline(tmp_file, line1);
} else {
......@@ -75,43 +97,47 @@ Parser* Parser::CreateParser(const char* filename, int num_features, bool* has_l
// Get some statistic from 2 line
GetStatistic(line1.c_str(), &comma_cnt, &tab_cnt, &colon_cnt);
GetStatistic(line2.c_str(), &comma_cnt2, &tab_cnt2, &colon_cnt2);
Parser* ret = nullptr;
DataType type = DataType::INVALID;
if (line2.size() == 0) {
// if only have one line on file
if (colon_cnt > 0) {
ret = new LibSVMParser();
if (num_features > 0 && has_label != nullptr) {
*has_label = CheckHasLabelForLibsvm(line1);
}
type = DataType::LIBSVM;
} else if (tab_cnt > 0) {
ret = new TSVParser();
if (num_features > 0 && has_label != nullptr) {
*has_label = CheckHasLabelForTSV(line1, num_features);
}
type = DataType::TSV;
} else if (comma_cnt > 0) {
ret = new CSVParser();
if (num_features > 0 && has_label != nullptr) {
*has_label = CheckHasLabelForCSV(line1, num_features);
}
type = DataType::CSV;
}
} else {
if (colon_cnt > 0 || colon_cnt2 > 0) {
ret = new LibSVMParser();
if (num_features > 0 && has_label != nullptr) {
*has_label = CheckHasLabelForLibsvm(line1);
type = DataType::LIBSVM;
} else if (tab_cnt == tab_cnt2 && tab_cnt > 0) {
type = DataType::TSV;
} else if (comma_cnt == comma_cnt2 && comma_cnt > 0) {
type = DataType::CSV;
}
}
else if (tab_cnt == tab_cnt2 && tab_cnt > 0) {
ret = new TSVParser();
if (num_features > 0 && has_label != nullptr) {
*has_label = CheckHasLabelForTSV(line1, num_features);
if (type == DataType::INVALID) {
Log::Fatal("Unkown format of training data");
}
} else if (comma_cnt == comma_cnt2 && comma_cnt > 0) {
ret = new CSVParser();
if (num_features > 0 && has_label != nullptr) {
*has_label = CheckHasLabelForCSV(line1, num_features);
Parser* ret = nullptr;
if (type == DataType::LIBSVM) {
label_idx = GetLabelIdxForLibsvm(line1, num_features, label_idx);
ret = new LibSVMParser(label_idx);
}
else if (type == DataType::TSV) {
label_idx = GetLabelIdxForTSV(line1, num_features, label_idx);
ret = new TSVParser(label_idx);
}
else if (type == DataType::CSV) {
label_idx = GetLabelIdxForCSV(line1, num_features, label_idx);
ret = new CSVParser(label_idx);
}
if (label_idx < 0) {
Log::Info("Data file: %s doesn't contain label column", filename);
}
return ret;
}
......
src/io/parser.hpp
View file @ 2a8d38c5
......@@ -14,14 +14,23 @@ namespace LightGBM {
class CSVParser: public Parser {
public:
explicit CSVParser(int label_idx)
:label_idx_(label_idx) {
}
inline void ParseOneLine(const char* str,
std::vector<std::pair<int, double>>* out_features) const override {
std::vector<std::pair<int, float>>* out_features, float* out_label) const override {
int idx = 0;
double val = 0.0;
float val = 0.0f;
int bias = 0;
*out_label = 0.0f;
while (*str != '\0') {
str = Common::Atof(str, &val);
if (fabs(val) > 1e-10) {
out_features->emplace_back(idx, val);
if (idx == label_idx_) {
*out_label = val;
bias = -1;
}
else if (fabs(val) > 1e-10) {
out_features->emplace_back(idx + bias, val);
}
++idx;
if (*str == ',') {
......@@ -31,28 +40,27 @@ public:
}
}
}
inline void ParseOneLine(const char* str, std::vector<std::pair<int, double>>* out_features,
double* out_label) const override {
// first column is label
str = Common::Atof(str, out_label);
if (*str == ',') {
++str;
} else if (*str != '\0') {
Log::Fatal("input format error, should be CSV");
}
return ParseOneLine(str, out_features);
}
private:
int label_idx_ = 0;
};
class TSVParser: public Parser {
public:
inline void ParseOneLine(const char* str, std::vector<std::pair<int, double>>* out_features) const override {
explicit TSVParser(int label_idx)
:label_idx_(label_idx) {
}
inline void ParseOneLine(const char* str,
std::vector<std::pair<int, float>>* out_features, float* out_label) const override {
int idx = 0;
double val = 0.0;
float val = 0.0f;
int bias = 0;
while (*str != '\0') {
str = Common::Atof(str, &val);
if (fabs(val) > 1e-10) {
out_features->emplace_back(idx, val);
if (idx == label_idx_) {
*out_label = val;
bias = -1;
} else if (fabs(val) > 1e-10) {
out_features->emplace_back(idx + bias, val);
}
++idx;
if (*str == '\t') {
......@@ -62,24 +70,27 @@ public:
}
}
}
inline void ParseOneLine(const char* str, std::vector<std::pair<int, double>>* out_features,
double* out_label) const override {
// first column is label
str = Common::Atof(str, out_label);
if (*str == '\t') {
++str;
} else if (*str != '\0') {
Log::Fatal("input format error, should be TSV");
}
return ParseOneLine(str, out_features);
}
private:
int label_idx_ = 0;
};
class LibSVMParser: public Parser {
public:
inline void ParseOneLine(const char* str, std::vector<std::pair<int, double>>* out_features) const override {
explicit LibSVMParser(int label_idx)
:label_idx_(label_idx) {
if (label_idx > 0) {
Log::Fatal("label should be the first column in Libsvm file");
}
}
inline void ParseOneLine(const char* str,
std::vector<std::pair<int, float>>* out_features, float* out_label) const override {
int idx = 0;
double val = 0.0;
float val = 0.0f;
if (label_idx_ == 0) {
str = Common::Atof(str, &val);
*out_label = val;
str = Common::SkipSpaceAndTab(str);
}
while (*str != '\0') {
str = Common::Atoi(str, &idx);
str = Common::SkipSpaceAndTab(str);
......@@ -93,13 +104,9 @@ public:
str = Common::SkipSpaceAndTab(str);
}
}
inline void ParseOneLine(const char* str, std::vector<std::pair<int, double>>* out_features,
double* out_label) const override {
// first column is label
str = Common::Atof(str, out_label);
str = Common::SkipSpaceAndTab(str);
return ParseOneLine(str, out_features);
}
private:
int label_idx_ = 0;
};
} // namespace LightGBM
#endif // LightGBM_IO_PARSER_HPP_
src/io/sparse_bin.hpp
View file @ 2a8d38c5
......@@ -64,7 +64,7 @@ public:
data_size_t cur_pos = fast_pair.second;
data_size_t lte_count = 0;
data_size_t gt_count = 0;
for (data_size_t i = 0; i < num_data; i++) {
for (data_size_t i = 0; i < num_data; ++i) {
const data_size_t idx = data_indices[i];
while (cur_pos < idx && j < num_vals_) {
++j;
......@@ -92,12 +92,12 @@ public:
void FinishLoad() override {
// get total non zero size
size_t non_zero_size = 0;
for (size_t i = 0; i < push_buffers_.size(); i++) {
for (size_t i = 0; i < push_buffers_.size(); ++i) {
non_zero_size += push_buffers_[i].size();
}
// merge
non_zero_pair_.reserve(non_zero_size);
for (size_t i = 0; i < push_buffers_.size(); i++) {
for (size_t i = 0; i < push_buffers_.size(); ++i) {
non_zero_pair_.insert(non_zero_pair_.end(), push_buffers_[i].begin(), push_buffers_[i].end());
push_buffers_[i].clear();
push_buffers_[i].shrink_to_fit();
......@@ -122,7 +122,7 @@ public:
// transform to delta array
const uint8_t kMaxDelta = 255;
data_size_t last_idx = 0;
for (size_t i = 0; i < non_zero_pair.size(); i++) {
for (size_t i = 0; i < non_zero_pair.size(); ++i) {
const data_size_t cur_idx = non_zero_pair[i].first;
const VAL_T bin = non_zero_pair[i].second;
data_size_t cur_delta = cur_idx - last_idx;
......@@ -198,7 +198,7 @@ public:
delta_.clear();
vals_.clear();
num_vals_ = tmp_num_vals;
for (data_size_t i = 0; i < num_vals_; i++) {
for (data_size_t i = 0; i < num_vals_; ++i) {
delta_.push_back(tmp_delta[i]);
vals_.push_back(tmp_vals[i]);
}
......
src/io/tree.cpp
View file @ 2a8d38c5
......@@ -23,11 +23,14 @@ Tree::Tree(int max_leaves)
split_feature_ = new int[max_leaves_ - 1];
split_feature_real_ = new int[max_leaves_ - 1];
threshold_in_bin_ = new unsigned int[max_leaves_ - 1];
threshold_ = new double[max_leaves_ - 1];
split_gain_ = new double[max_leaves_ - 1];
threshold_ = new float[max_leaves_ - 1];
split_gain_ = new float[max_leaves_ - 1];
leaf_parent_ = new int[max_leaves_];
leaf_value_ = new score_t[max_leaves_];
leaf_value_ = new float[max_leaves_];
leaf_depth_ = new int[max_leaves_];
// root is in the depth 1
leaf_depth_[0] = 1;
num_leaves_ = 1;
leaf_parent_[0] = -1;
}
......@@ -41,10 +44,11 @@ Tree::~Tree() {
if (threshold_ != nullptr) { delete[] threshold_; }
if (split_gain_ != nullptr) { delete[] split_gain_; }
if (leaf_value_ != nullptr) { delete[] leaf_value_; }
if (leaf_depth_ != nullptr) { delete[] leaf_depth_; }
}
int Tree::Split(int leaf, int feature, unsigned int threshold_bin, int real_feature,
double threshold, score_t left_value, score_t right_value, double gain) {
float threshold, float left_value, float right_value, float gain) {
int new_node_idx = num_leaves_ - 1;
// update parent info
int parent = leaf_parent_[leaf];
......@@ -70,19 +74,21 @@ int Tree::Split(int leaf, int feature, unsigned int threshold_bin, int real_feat
leaf_parent_[num_leaves_] = new_node_idx;
leaf_value_[leaf] = left_value;
leaf_value_[num_leaves_] = right_value;
// update leaf depth
leaf_depth_[num_leaves_] = leaf_depth_[leaf] + 1;
leaf_depth_[leaf]++;
++num_leaves_;
return num_leaves_ - 1;
}
void Tree::AddPredictionToScore(const Dataset* data, data_size_t num_data, score_t* score) const {
Threading::For<data_size_t>(0, num_data, [this, data, score](int, data_size_t start, data_size_t end) {
std::vector<BinIterator*> iterators;
for (int i = 0; i < data->num_features(); i++) {
for (int i = 0; i < data->num_features(); ++i) {
iterators.push_back(data->FeatureAt(i)->bin_data()->GetIterator(start));
}
for (data_size_t i = start; i < end; i++) {
for (data_size_t i = start; i < end; ++i) {
score[i] += leaf_value_[GetLeaf(iterators, i)];
}
});
......@@ -93,10 +99,10 @@ void Tree::AddPredictionToScore(const Dataset* data, const data_size_t* used_dat
Threading::For<data_size_t>(0, num_data,
[this, data, used_data_indices, score](int, data_size_t start, data_size_t end) {
std::vector<BinIterator*> iterators;
for (int i = 0; i < data->num_features(); i++) {
for (int i = 0; i < data->num_features(); ++i) {
iterators.push_back(data->FeatureAt(i)->bin_data()->GetIterator(used_data_indices[start]));
}
for (data_size_t i = start; i < end; i++) {
for (data_size_t i = start; i < end; ++i) {
score[used_data_indices[i]] += leaf_value_[GetLeaf(iterators, used_data_indices[i])];
}
});
......@@ -108,9 +114,9 @@ std::string Tree::ToString() {
ss << "split_feature="
<< Common::ArrayToString<int>(split_feature_real_, num_leaves_ - 1, ' ') << std::endl;
ss << "split_gain="
<< Common::ArrayToString<double>(split_gain_, num_leaves_ - 1, ' ') << std::endl;
<< Common::ArrayToString<float>(split_gain_, num_leaves_ - 1, ' ') << std::endl;
ss << "threshold="
<< Common::ArrayToString<double>(threshold_, num_leaves_ - 1, ' ') << std::endl;
<< Common::ArrayToString<float>(threshold_, num_leaves_ - 1, ' ') << std::endl;
ss << "left_child="
<< Common::ArrayToString<int>(left_child_, num_leaves_ - 1, ' ') << std::endl;
ss << "right_child="
......@@ -118,7 +124,7 @@ std::string Tree::ToString() {
ss << "leaf_parent="
<< Common::ArrayToString<int>(leaf_parent_, num_leaves_, ' ') << std::endl;
ss << "leaf_value="
<< Common::ArrayToString<score_t>(leaf_value_, num_leaves_, ' ') << std::endl;
<< Common::ArrayToString<float>(leaf_value_, num_leaves_, ' ') << std::endl;
ss << std::endl;
return ss.str();
}
......@@ -148,19 +154,20 @@ Tree::Tree(const std::string& str) {
left_child_ = new int[num_leaves_ - 1];
right_child_ = new int[num_leaves_ - 1];
split_feature_real_ = new int[num_leaves_ - 1];
threshold_ = new double[num_leaves_ - 1];
split_gain_ = new double[num_leaves_ - 1];
threshold_ = new float[num_leaves_ - 1];
split_gain_ = new float[num_leaves_ - 1];
leaf_parent_ = new int[num_leaves_];
leaf_value_ = new score_t[num_leaves_];
leaf_value_ = new float[num_leaves_];
split_feature_ = nullptr;
threshold_in_bin_ = nullptr;
leaf_depth_ = nullptr;
Common::StringToIntArray(key_vals["split_feature"], ' ',
num_leaves_ - 1, split_feature_real_);
Common::StringToDoubleArray(key_vals["split_gain"], ' ',
Common::StringToFloatArray(key_vals["split_gain"], ' ',
num_leaves_ - 1, split_gain_);
Common::StringToDoubleArray(key_vals["threshold"], ' ',
Common::StringToFloatArray(key_vals["threshold"], ' ',
num_leaves_ - 1, threshold_);
Common::StringToIntArray(key_vals["left_child"], ' ',
num_leaves_ - 1, left_child_);
......@@ -168,7 +175,7 @@ Tree::Tree(const std::string& str) {
num_leaves_ - 1, right_child_);
Common::StringToIntArray(key_vals["leaf_parent"], ' ',
num_leaves_ , leaf_parent_);
Common::StringToDoubleArray(key_vals["leaf_value"], ' ',
Common::StringToFloatArray(key_vals["leaf_value"], ' ',
num_leaves_ , leaf_value_);
}
......
src/metric/binary_metric.hpp
View file @ 2a8d38c5
......@@ -7,6 +7,7 @@
#include <algorithm>
#include <vector>
#include <sstream>
namespace LightGBM {
......@@ -18,9 +19,6 @@ template<typename PointWiseLossCalculator>
class BinaryMetric: public Metric {
public:
explicit BinaryMetric(const MetricConfig& config) {
early_stopping_round_ = config.early_stopping_round;
output_freq_ = config.output_freq;
the_bigger_the_better = false;
sigmoid_ = static_cast<score_t>(config.sigmoid);
if (sigmoid_ <= 0.0f) {
Log::Fatal("Sigmoid param %f should greater than zero", sigmoid_);
......@@ -32,7 +30,9 @@ public:
}
void Init(const char* test_name, const Metadata& metadata, data_size_t num_data) override {
name = test_name;
std::stringstream str_buf;
str_buf << test_name << "'s " << PointWiseLossCalculator::Name();
name_ = str_buf.str();
num_data_ = num_data;
// get label
label_ = metadata.label();
......@@ -41,7 +41,7 @@ public:
weights_ = metadata.weights();
if (weights_ == nullptr) {
sum_weights_ = static_cast<double>(num_data_);
sum_weights_ = static_cast<float>(num_data_);
} else {
sum_weights_ = 0.0f;
for (data_size_t i = 0; i < num_data; ++i) {
......@@ -50,11 +50,18 @@ public:
}
}
score_t PrintAndGetLoss(int iter, const score_t* score) const override {
const char* GetName() const override {
return name_.c_str();
}
bool is_bigger_better() const override {
return false;
}
std::vector<float> Eval(const score_t* score) const override {
score_t sum_loss = 0.0f;
if (early_stopping_round_ > 0 || (output_freq_ > 0 && iter % output_freq_ == 0)) {
if (weights_ == nullptr) {
#pragma omp parallel for schedule(static) reduction(+:sum_loss)
#pragma omp parallel for schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
// sigmoid transform
score_t prob = 1.0f / (1.0f + std::exp(-2.0f * sigmoid_ * score[i]));
......@@ -62,7 +69,7 @@ public:
sum_loss += PointWiseLossCalculator::LossOnPoint(label_[i], prob);
}
} else {
#pragma omp parallel for schedule(static) reduction(+:sum_loss)
#pragma omp parallel for schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
// sigmoid transform
score_t prob = 1.0f / (1.0f + std::exp(-2.0f * sigmoid_ * score[i]));
......@@ -71,17 +78,10 @@ public:
}
}
score_t loss = sum_loss / sum_weights_;
if (output_freq_ > 0 && iter % output_freq_ == 0){
Log::Info("Iteration:%d, %s's %s: %f", iter, name, PointWiseLossCalculator::Name(), loss);
}
return loss;
}
return 0.0f;
return std::vector<float>(1, static_cast<float>(loss));
}
private:
/*! \brief Output frequently */
int output_freq_;
/*! \brief Number of data */
data_size_t num_data_;
/*! \brief Pointer of label */
......@@ -89,9 +89,9 @@ private:
/*! \brief Pointer of weighs */
const float* weights_;
/*! \brief Sum weights */
double sum_weights_;
float sum_weights_;
/*! \brief Name of test set */
const char* name;
std::string name_;
/*! \brief Sigmoid parameter */
score_t sigmoid_;
};
......@@ -145,17 +145,26 @@ public:
*/
class AUCMetric: public Metric {
public:
explicit AUCMetric(const MetricConfig& config) {
early_stopping_round_ = config.early_stopping_round;
output_freq_ = config.output_freq;
the_bigger_the_better = true;
explicit AUCMetric(const MetricConfig&) {
}
virtual ~AUCMetric() {
}
const char* GetName() const override {
return name_.c_str();
}
bool is_bigger_better() const override {
return true;
}
void Init(const char* test_name, const Metadata& metadata, data_size_t num_data) override {
name = test_name;
std::stringstream str_buf;
str_buf << test_name << "'s AUC";
name_ = str_buf.str();
num_data_ = num_data;
// get label
label_ = metadata.label();
......@@ -163,7 +172,7 @@ public:
weights_ = metadata.weights();
if (weights_ == nullptr) {
sum_weights_ = static_cast<double>(num_data_);
sum_weights_ = static_cast<float>(num_data_);
} else {
sum_weights_ = 0.0f;
for (data_size_t i = 0; i < num_data; ++i) {
......@@ -172,8 +181,7 @@ public:
}
}
score_t PrintAndGetLoss(int iter, const score_t* score) const override {
if (early_stopping_round_ > 0 || (output_freq_ > 0 && iter % output_freq_ == 0)) {
std::vector<float> Eval(const score_t* score) const override {
// get indices sorted by score, descent order
std::vector<data_size_t> sorted_idx;
for (data_size_t i = 0; i < num_data_; ++i) {
......@@ -181,15 +189,15 @@ public:
}
std::sort(sorted_idx.begin(), sorted_idx.end(), [score](data_size_t a, data_size_t b) {return score[a] > score[b]; });
// temp sum of postive label
double cur_pos = 0.0;
score_t cur_pos = 0.0f;
// total sum of postive label
double sum_pos = 0.0;
score_t sum_pos = 0.0f;
// accumlate of auc
double accum = 0.0;
score_t accum = 0.0f;
// temp sum of negative label
double cur_neg = 0.0;
score_t cur_neg = 0.0f;
score_t threshold = score[sorted_idx[0]];
if (weights_ == nullptr) { // not weights
if (weights_ == nullptr) { // no weights
for (data_size_t i = 0; i < num_data_; ++i) {
const float cur_label = label_[sorted_idx[i]];
const score_t cur_score = score[sorted_idx[i]];
......@@ -197,12 +205,12 @@ public:
if (cur_score != threshold) {
threshold = cur_score;
// accmulate
accum += cur_neg*(cur_pos * 0.5 + sum_pos);
accum += cur_neg*(cur_pos * 0.5f + sum_pos);
sum_pos += cur_pos;
// reset
cur_neg = cur_pos = 0.0;
cur_neg = cur_pos = 0.0f;
}
cur_neg += 1.0 - cur_label;
cur_neg += 1.0f - cur_label;
cur_pos += cur_label;
}
} else { // has weights
......@@ -214,32 +222,25 @@ public:
if (cur_score != threshold) {
threshold = cur_score;
// accmulate
accum += cur_neg*(cur_pos * 0.5 + sum_pos);
accum += cur_neg*(cur_pos * 0.5f + sum_pos);
sum_pos += cur_pos;
// reset
cur_neg = cur_pos = 0.0;
cur_neg = cur_pos = 0.0f;
}
cur_neg += (1.0 - cur_label)*cur_weight;
cur_neg += (1.0f - cur_label)*cur_weight;
cur_pos += cur_label*cur_weight;
}
}
accum += cur_neg*(cur_pos * 0.5 + sum_pos);
accum += cur_neg*(cur_pos * 0.5f + sum_pos);
sum_pos += cur_pos;
double auc = 1.0;
score_t auc = 1.0f;
if (sum_pos > 0.0f && sum_pos != sum_weights_) {
auc = accum / (sum_pos *(sum_weights_ - sum_pos));
}
if (output_freq_ > 0 && iter % output_freq_ == 0){
Log::Info("Iteration:%d, %s's %s: %f", iter, name, "auc", auc);
}
return auc;
}
return 0.0f;
return std::vector<float>(1, static_cast<float>(auc));
}
private:
/*! \brief Output frequently */
int output_freq_;
/*! \brief Number of data */
data_size_t num_data_;
/*! \brief Pointer of label */
......@@ -247,9 +248,9 @@ private:
/*! \brief Pointer of weighs */
const float* weights_;
/*! \brief Sum weights */
double sum_weights_;
float sum_weights_;
/*! \brief Name of test set */
const char* name;
std::string name_;
};
} // namespace LightGBM
......
src/metric/dcg_calculator.cpp
View file @ 2a8d38c5
......@@ -11,23 +11,23 @@ namespace LightGBM {
/*! \brief Declaration for some static members */
bool DCGCalculator::is_inited_ = false;
std::vector<double> DCGCalculator::label_gain_;
std::vector<double> DCGCalculator::discount_;
std::vector<float> DCGCalculator::label_gain_;
std::vector<float> DCGCalculator::discount_;
const data_size_t DCGCalculator::kMaxPosition = 10000;
void DCGCalculator::Init(std::vector<double> input_label_gain) {
void DCGCalculator::Init(std::vector<float> input_label_gain) {
// only inited one time
if (is_inited_) { return; }
label_gain_ = input_label_gain;
discount_.clear();
for (data_size_t i = 0; i < kMaxPosition; ++i) {
discount_.emplace_back(1.0 / std::log2(2.0 + i));
discount_.emplace_back(1.0f / std::log2(2.0f + i));
}
is_inited_ = true;
}
double DCGCalculator::CalMaxDCGAtK(data_size_t k, const float* label, data_size_t num_data) {
double ret = 0.0;
float DCGCalculator::CalMaxDCGAtK(data_size_t k, const float* label, data_size_t num_data) {
float ret = 0.0f;
// counts for all labels
std::vector<data_size_t> label_cnt(label_gain_.size(), 0);
for (data_size_t i = 0; i < num_data; ++i) {
......@@ -53,14 +53,14 @@ double DCGCalculator::CalMaxDCGAtK(data_size_t k, const float* label, data_size_
void DCGCalculator::CalMaxDCG(const std::vector<data_size_t>& ks,
const float* label,
data_size_t num_data,
std::vector<double>* out) {
std::vector<float>* out) {
std::vector<data_size_t> label_cnt(label_gain_.size(), 0);
// counts for all labels
for (data_size_t i = 0; i < num_data; ++i) {
if (static_cast<size_t>(label[i]) >= label_cnt.size()) { Log::Fatal("label excel %d", label[i]); }
++label_cnt[static_cast<int>(label[i])];
}
double cur_result = 0.0;
float cur_result = 0.0f;
data_size_t cur_left = 0;
size_t top_label = label_gain_.size() - 1;
// calculate k Max DCG by one pass
......@@ -83,7 +83,7 @@ void DCGCalculator::CalMaxDCG(const std::vector<data_size_t>& ks,
}
double DCGCalculator::CalDCGAtK(data_size_t k, const float* label,
float DCGCalculator::CalDCGAtK(data_size_t k, const float* label,
const score_t* score, data_size_t num_data) {
// get sorted indices by score
std::vector<data_size_t> sorted_idx;
......@@ -94,7 +94,7 @@ double DCGCalculator::CalDCGAtK(data_size_t k, const float* label,
[score](data_size_t a, data_size_t b) {return score[a] > score[b]; });
if (k > num_data) { k = num_data; }
double dcg = 0.0;
float dcg = 0.0f;
// calculate dcg
for (data_size_t i = 0; i < k; ++i) {
data_size_t idx = sorted_idx[i];
......@@ -104,7 +104,7 @@ double DCGCalculator::CalDCGAtK(data_size_t k, const float* label,
}
void DCGCalculator::CalDCG(const std::vector<data_size_t>& ks, const float* label,
const score_t * score, data_size_t num_data, std::vector<double>* out) {
const score_t * score, data_size_t num_data, std::vector<float>* out) {
// get sorted indices by score
std::vector<data_size_t> sorted_idx;
for (data_size_t i = 0; i < num_data; ++i) {
......@@ -113,7 +113,7 @@ void DCGCalculator::CalDCG(const std::vector<data_size_t>& ks, const float* labe
std::sort(sorted_idx.begin(), sorted_idx.end(),
[score](data_size_t a, data_size_t b) {return score[a] > score[b]; });
double cur_result = 0.0;
float cur_result = 0.0f;
data_size_t cur_left = 0;
// calculate multi dcg by one pass
for (size_t i = 0; i < ks.size(); ++i) {
......
src/metric/metric.cpp
View file @ 2a8d38c5
......@@ -2,22 +2,27 @@
#include "regression_metric.hpp"
#include "binary_metric.hpp"
#include "rank_metric.hpp"
#include "multiclass_metric.hpp"
namespace LightGBM {
Metric* Metric::CreateMetric(const std::string& type, const MetricConfig& config) {
if (type == "l2") {
if (type == std::string("l2")) {
return new L2Metric(config);
} else if (type == "l1") {
} else if (type == std::string("l1")) {
return new L1Metric(config);
} else if (type == "binary_logloss") {
} else if (type == std::string("binary_logloss")) {
return new BinaryLoglossMetric(config);
} else if (type == "binary_error") {
} else if (type == std::string("binary_error")) {
return new BinaryErrorMetric(config);
} else if (type == "auc") {
} else if (type == std::string("auc")) {
return new AUCMetric(config);
} else if (type == "ndcg") {
} else if (type == std::string("ndcg")) {
return new NDCGMetric(config);
} else if (type == std::string("multi_logloss")) {
return new MultiLoglossMetric(config);
} else if (type == std::string("multi_error")) {
return new MultiErrorMetric(config);
}
return nullptr;
}
......
src/metric/multiclass_metric.hpp 0 → 100644
View file @ 2a8d38c5
#ifndef LIGHTGBM_METRIC_MULTICLASS_METRIC_HPP_
#define LIGHTGBM_METRIC_MULTICLASS_METRIC_HPP_
#include <LightGBM/utils/log.h>
#include <LightGBM/metric.h>
#include <cmath>
namespace LightGBM {
/*!
* \brief Metric for multiclass task.
* Use static class "PointWiseLossCalculator" to calculate loss point-wise
*/
template<typename PointWiseLossCalculator>
class MulticlassMetric: public Metric {
public:
explicit MulticlassMetric(const MetricConfig& config) {
num_class_ = config.num_class;
}
virtual ~MulticlassMetric() {
}
void Init(const char* test_name, const Metadata& metadata, data_size_t num_data) override {
std::stringstream str_buf;
str_buf << test_name << "'s " << PointWiseLossCalculator::Name();
name_ = str_buf.str();
num_data_ = num_data;
// get label
label_ = metadata.label();
// get weights
weights_ = metadata.weights();
if (weights_ == nullptr) {
sum_weights_ = static_cast<float>(num_data_);
} else {
sum_weights_ = 0.0f;
for (data_size_t i = 0; i < num_data_; ++i) {
sum_weights_ += weights_[i];
}
}
}
const char* GetName() const override {
return name_.c_str();
}
bool is_bigger_better() const override {
return false;
}
std::vector<float> Eval(const score_t* score) const override {
score_t sum_loss = 0.0;
if (weights_ == nullptr) {
#pragma omp parallel for schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
std::vector<float> rec(num_class_);
for (int k = 0; k < num_class_; ++k) {
rec[k] = static_cast<float>(score[k * num_data_ + i]);
}
// add loss
sum_loss += PointWiseLossCalculator::LossOnPoint(label_[i], rec);
}
} else {
#pragma omp parallel for schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
std::vector<float> rec(num_class_);
for (int k = 0; k < num_class_; ++k) {
rec[k] = static_cast<float>(score[k * num_data_ + i]);
}
// add loss
sum_loss += PointWiseLossCalculator::LossOnPoint(label_[i], rec) * weights_[i];
}
}
score_t loss = sum_loss / sum_weights_;
return std::vector<float>(1, static_cast<float>(loss));
}
private:
/*! \brief Output frequency */
int output_freq_;
/*! \brief Number of data */
data_size_t num_data_;
/*! \brief Number of classes */
int num_class_;
/*! \brief Pointer of label */
const float* label_;
/*! \brief Pointer of weighs */
const float* weights_;
/*! \brief Sum weights */
float sum_weights_;
/*! \brief Name of this test set */
std::string name_;
};
/*! \brief L2 loss for multiclass task */
class MultiErrorMetric: public MulticlassMetric<MultiErrorMetric> {
public:
explicit MultiErrorMetric(const MetricConfig& config) :MulticlassMetric<MultiErrorMetric>(config) {}
inline static score_t LossOnPoint(float label, std::vector<float> score) {
size_t k = static_cast<size_t>(label);
for (size_t i = 0; i < score.size(); ++i){
if (i != k && score[i] > score[k]) {
return 0.0f;
}
}
return 1.0f;
}
inline static const char* Name() {
return "multi error";
}
};
/*! \brief Logloss for multiclass task */
class MultiLoglossMetric: public MulticlassMetric<MultiLoglossMetric> {
public:
explicit MultiLoglossMetric(const MetricConfig& config) :MulticlassMetric<MultiLoglossMetric>(config) {}
inline static score_t LossOnPoint(float label, std::vector<float> score) {
size_t k = static_cast<size_t>(label);
Common::Softmax(&score);
if (score[k] > kEpsilon) {
return -std::log(score[k]);
} else {
return -std::log(kEpsilon);
}
}
inline static const char* Name() {
return "multi logloss";
}
};
} // namespace LightGBM
#endif // LightGBM_METRIC_MULTICLASS_METRIC_HPP_
src/metric/rank_metric.hpp
View file @ 2a8d38c5
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment