".github/vscode:/vscode.git/clone" did not exist on "836974229bdf0e2d329bdfdb0f9c4920eae224b6"
Commit 9dcee4c7 authored by Myle Ott's avatar Myle Ott
Browse files

Store full checkpoints instead of symlinking

parent d9a13180
......@@ -287,16 +287,8 @@ def save_checkpoint(args, trainer, epoch_itr, val_loss):
checkpoints = [os.path.join(args.save_dir, fn) for fn, cond in checkpoint_conds.items() if cond]
if len(checkpoints) > 0:
for fn in checkpoints:
if os.path.exists(fn):
os.remove(fn)
if not end_of_epoch and args.keep_interval_updates > 0:
for cp in checkpoints:
trainer.save_checkpoint(cp, extra_state)
else:
trainer.save_checkpoint(checkpoints[0], extra_state)
for fn in checkpoints[1:]:
os.symlink(os.path.basename(checkpoints[0]), fn)
if not end_of_epoch and args.keep_interval_updates > 0:
# remove old checkpoints; checkpoints are sorted in descending order
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment