Unverified Commit 2ef3cb81 authored by xuchuanyin's avatar xuchuanyin Committed by GitHub
Browse files

clear memory of sample data right after BinMapper is constructed to save memory (#4890)

Sample data is useless after BinMapper is constructed, but the corresponding memory is still there before feature extraction is finished.
parent e4c0ca5f
......@@ -234,8 +234,9 @@ Dataset* DatasetLoader::LoadFromFile(const char* filename, int rank, int num_mac
auto sample_data = SampleTextDataFromMemory(text_data);
CheckSampleSize(sample_data.size(),
static_cast<size_t>(dataset->num_data_));
// construct feature bin mappers
// construct feature bin mappers & clear sample data
ConstructBinMappersFromTextData(rank, num_machines, sample_data, parser.get(), dataset.get());
std::vector<std::string>().swap(sample_data);
if (dataset->has_raw()) {
dataset->ResizeRaw(dataset->num_data_);
}
......@@ -254,8 +255,9 @@ Dataset* DatasetLoader::LoadFromFile(const char* filename, int rank, int num_mac
}
CheckSampleSize(sample_data.size(),
static_cast<size_t>(dataset->num_data_));
// construct feature bin mappers
// construct feature bin mappers & clear sample data
ConstructBinMappersFromTextData(rank, num_machines, sample_data, parser.get(), dataset.get());
std::vector<std::string>().swap(sample_data);
if (dataset->has_raw()) {
dataset->ResizeRaw(dataset->num_data_);
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment