Skip to content

GitLab

"...vision/git@developer.sourcefind.cn:gaoqiong/migraphx.git" did not exist on "9b3b10ed948d147022d311acf8ac2ad02cd5ef5e"
Commit b8e2a9c5 authored by Matthew Carrigan's avatar Matthew Carrigan
Browse files

Made --reduce_memory actually do something in finetune_on_pregenerated

parent af8a0384
Show whitespace changes
Inline Side-by-side
Showing with 2 additions and 2 deletions
examples/lm_finetuning/finetune_on_pregenerated.py
View file @ b8e2a9c5
......@@ -74,7 +74,7 @@ class PregeneratedDataset(Dataset):
mode='w+', dtype=np.int32, shape=(num_samples, seq_len))
input_masks = np.memmap(filename=self.working_dir/'input_masks.memmap',
shape=(num_samples, seq_len), mode='w+', dtype=np.bool)
segment_ids = np.memmap(filename=self.working_dir/'input_masks.memmap',
segment_ids = np.memmap(filename=self.working_dir/'segment_ids.memmap',
shape=(num_samples, seq_len), mode='w+', dtype=np.bool)
lm_label_ids = np.memmap(filename=self.working_dir/'lm_label_ids.memmap',
shape=(num_samples, seq_len), mode='w+', dtype=np.int32)
......@@ -283,7 +283,7 @@ def main():
model.train()
for epoch in range(args.epochs):
epoch_dataset = PregeneratedDataset(epoch=epoch, training_path=args.pregenerated_data, tokenizer=tokenizer,
num_data_epochs=num_data_epochs)
num_data_epochs=num_data_epochs, reduce_memory=args.reduce_memory)
if args.local_rank == -1:
train_sampler = RandomSampler(epoch_dataset)
else:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment