Skip to content

GitLab

"vscode:/vscode.git/clone" did not exist on "f903fda39e4c3f712fa6e161be89e60fd9470952"
Commit 2b47c5e7 authored by Raul Puri's avatar Raul Puri
Browse files

mmap sample map

parent cfe35fd3
Hide whitespace changes
Inline Side-by-side
Showing with 1 addition and 1 deletion
megatron/data/bert_dataset.py
View file @ 2b47c5e7
...@@ -279,7 +279,7 @@ def get_samples_mapping_(indexed_dataset, ...@@ -279,7 +279,7 @@ def get_samples_mapping_(indexed_dataset,
print_rank_0(' > loading indexed mapping from {}'.format( print_rank_0(' > loading indexed mapping from {}'.format(
indexmap_filename)) indexmap_filename))
start_time = time.time() start_time = time.time()
samples_mapping = np.load(indexmap_filename, allow_pickle=True) samples_mapping = np.load(indexmap_filename, allow_pickle=True, mmap_mode='r')
print_rank_0(' loaded indexed file in {:3.3f} seconds'.format( print_rank_0(' loaded indexed file in {:3.3f} seconds'.format(
time.time() - start_time)) time.time() - start_time))
print_rank_0(' total number of samples: {}'.format( print_rank_0(' total number of samples: {}'.format(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment