Commit 4e402370 authored by Guolin Ke's avatar Guolin Ke
Browse files

fix bug in save binary

parent 54963bb7
...@@ -20,7 +20,7 @@ public: ...@@ -20,7 +20,7 @@ public:
Dataset* LoadFromFileAlignWithOtherDataset(const char* filename, const Dataset* train_data); Dataset* LoadFromFileAlignWithOtherDataset(const char* filename, const Dataset* train_data);
Dataset* LoadFromBinFile(const char* bin_filename, int rank, int num_machines); Dataset* LoadFromBinFile(const char* data_filename, const char* bin_filename, int rank, int num_machines);
Dataset* CostructFromSampleData(std::vector<std::vector<double>>& sample_values, size_t total_sample_size, data_size_t num_data); Dataset* CostructFromSampleData(std::vector<std::vector<double>>& sample_values, size_t total_sample_size, data_size_t num_data);
......
...@@ -131,13 +131,17 @@ bool Dataset::GetIntField(const char* field_name, int64_t* out_len, const int** ...@@ -131,13 +131,17 @@ bool Dataset::GetIntField(const char* field_name, int64_t* out_len, const int**
} }
void Dataset::SaveBinaryFile(const char* bin_filename) { void Dataset::SaveBinaryFile(const char* bin_filename) {
if (bin_filename != nullptr
&& std::string(bin_filename) == std::string(data_filename_)) {
Log::Warning("Bianry file %s already existed", bin_filename);
return;
}
// if not pass a filename, just append ".bin" of original file // if not pass a filename, just append ".bin" of original file
std::string bin_filename_str(data_filename_);
if (bin_filename == nullptr || bin_filename[0] == '\0') { if (bin_filename == nullptr || bin_filename[0] == '\0') {
std::string bin_filename_str(data_filename_);
bin_filename_str.append(".bin"); bin_filename_str.append(".bin");
bin_filename = bin_filename_str.c_str(); bin_filename = bin_filename_str.c_str();
} }
bool is_file_existed = false; bool is_file_existed = false;
FILE* file; FILE* file;
#ifdef _MSC_VER #ifdef _MSC_VER
......
...@@ -210,7 +210,7 @@ Dataset* DatasetLoader::LoadFromFile(const char* filename, int rank, int num_mac ...@@ -210,7 +210,7 @@ Dataset* DatasetLoader::LoadFromFile(const char* filename, int rank, int num_mac
} }
} else { } else {
// load data from binary file // load data from binary file
dataset.reset(LoadFromBinFile(bin_filename.c_str(), rank, num_machines)); dataset.reset(LoadFromBinFile(filename, bin_filename.c_str(), rank, num_machines));
} }
// check meta data // check meta data
dataset->metadata_.CheckOrPartition(num_global_data, used_data_indices); dataset->metadata_.CheckOrPartition(num_global_data, used_data_indices);
...@@ -257,7 +257,7 @@ Dataset* DatasetLoader::LoadFromFileAlignWithOtherDataset(const char* filename, ...@@ -257,7 +257,7 @@ Dataset* DatasetLoader::LoadFromFileAlignWithOtherDataset(const char* filename,
} }
} else { } else {
// load data from binary file // load data from binary file
dataset.reset(LoadFromBinFile(bin_filename.c_str(), 0, 1)); dataset.reset(LoadFromBinFile(filename, bin_filename.c_str(), 0, 1));
} }
// not need to check validation data // not need to check validation data
// check meta data // check meta data
...@@ -265,7 +265,7 @@ Dataset* DatasetLoader::LoadFromFileAlignWithOtherDataset(const char* filename, ...@@ -265,7 +265,7 @@ Dataset* DatasetLoader::LoadFromFileAlignWithOtherDataset(const char* filename,
return dataset.release(); return dataset.release();
} }
Dataset* DatasetLoader::LoadFromBinFile(const char* bin_filename, int rank, int num_machines) { Dataset* DatasetLoader::LoadFromBinFile(const char* data_filename, const char* bin_filename, int rank, int num_machines) {
auto dataset = std::unique_ptr<Dataset>(new Dataset()); auto dataset = std::unique_ptr<Dataset>(new Dataset());
FILE* file; FILE* file;
#ifdef _MSC_VER #ifdef _MSC_VER
...@@ -273,7 +273,7 @@ Dataset* DatasetLoader::LoadFromBinFile(const char* bin_filename, int rank, int ...@@ -273,7 +273,7 @@ Dataset* DatasetLoader::LoadFromBinFile(const char* bin_filename, int rank, int
#else #else
file = fopen(bin_filename, "rb"); file = fopen(bin_filename, "rb");
#endif #endif
dataset->data_filename_ = data_filename;
if (file == NULL) { if (file == NULL) {
Log::Fatal("Could not read binary data from %s", bin_filename); Log::Fatal("Could not read binary data from %s", bin_filename);
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment