Unverified Commit 2ef3cb81 authored by xuchuanyin's avatar xuchuanyin Committed by GitHub
Browse files

clear memory of sample data right after BinMapper is constructed to save memory (#4890)

Sample data is useless after BinMapper is constructed, but the corresponding memory is still there before feature extraction is finished.
parent e4c0ca5f
...@@ -234,8 +234,9 @@ Dataset* DatasetLoader::LoadFromFile(const char* filename, int rank, int num_mac ...@@ -234,8 +234,9 @@ Dataset* DatasetLoader::LoadFromFile(const char* filename, int rank, int num_mac
auto sample_data = SampleTextDataFromMemory(text_data); auto sample_data = SampleTextDataFromMemory(text_data);
CheckSampleSize(sample_data.size(), CheckSampleSize(sample_data.size(),
static_cast<size_t>(dataset->num_data_)); static_cast<size_t>(dataset->num_data_));
// construct feature bin mappers // construct feature bin mappers & clear sample data
ConstructBinMappersFromTextData(rank, num_machines, sample_data, parser.get(), dataset.get()); ConstructBinMappersFromTextData(rank, num_machines, sample_data, parser.get(), dataset.get());
std::vector<std::string>().swap(sample_data);
if (dataset->has_raw()) { if (dataset->has_raw()) {
dataset->ResizeRaw(dataset->num_data_); dataset->ResizeRaw(dataset->num_data_);
} }
...@@ -254,8 +255,9 @@ Dataset* DatasetLoader::LoadFromFile(const char* filename, int rank, int num_mac ...@@ -254,8 +255,9 @@ Dataset* DatasetLoader::LoadFromFile(const char* filename, int rank, int num_mac
} }
CheckSampleSize(sample_data.size(), CheckSampleSize(sample_data.size(),
static_cast<size_t>(dataset->num_data_)); static_cast<size_t>(dataset->num_data_));
// construct feature bin mappers // construct feature bin mappers & clear sample data
ConstructBinMappersFromTextData(rank, num_machines, sample_data, parser.get(), dataset.get()); ConstructBinMappersFromTextData(rank, num_machines, sample_data, parser.get(), dataset.get());
std::vector<std::string>().swap(sample_data);
if (dataset->has_raw()) { if (dataset->has_raw()) {
dataset->ResizeRaw(dataset->num_data_); dataset->ResizeRaw(dataset->num_data_);
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment