"vscode:/vscode.git/clone" did not exist on "e7aba2012d86a959b91b16fbd569c7b3f233d40a"
Unverified Commit 2ef3cb81 authored by xuchuanyin's avatar xuchuanyin Committed by GitHub
Browse files

clear memory of sample data right after BinMapper is constructed to save memory (#4890)

Sample data is useless after BinMapper is constructed, but the corresponding memory is still there before feature extraction is finished.
parent e4c0ca5f
......@@ -234,8 +234,9 @@ Dataset* DatasetLoader::LoadFromFile(const char* filename, int rank, int num_mac
auto sample_data = SampleTextDataFromMemory(text_data);
CheckSampleSize(sample_data.size(),
static_cast<size_t>(dataset->num_data_));
// construct feature bin mappers
// construct feature bin mappers & clear sample data
ConstructBinMappersFromTextData(rank, num_machines, sample_data, parser.get(), dataset.get());
std::vector<std::string>().swap(sample_data);
if (dataset->has_raw()) {
dataset->ResizeRaw(dataset->num_data_);
}
......@@ -254,8 +255,9 @@ Dataset* DatasetLoader::LoadFromFile(const char* filename, int rank, int num_mac
}
CheckSampleSize(sample_data.size(),
static_cast<size_t>(dataset->num_data_));
// construct feature bin mappers
// construct feature bin mappers & clear sample data
ConstructBinMappersFromTextData(rank, num_machines, sample_data, parser.get(), dataset.get());
std::vector<std::string>().swap(sample_data);
if (dataset->has_raw()) {
dataset->ResizeRaw(dataset->num_data_);
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment