Skip to content

GitLab

Commit 8d1d1ffd authored by Matthew Carrigan's avatar Matthew Carrigan
Browse files

Corrected the displayed loss when gradient_accumulation_steps > 1

parent abb7d1ff
Hide whitespace changes
Inline Side-by-side
Showing with 1 addition and 1 deletion
examples/lm_finetuning/finetune_on_pregenerated.py
View file @ 8d1d1ffd
...@@ -309,7 +309,7 @@ def main(): ...@@ -309,7 +309,7 @@ def main():
nb_tr_examples += input_ids.size(0) nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1 nb_tr_steps += 1
pbar.update(1) pbar.update(1)
mean_loss = tr_loss / nb_tr_steps mean_loss = tr_loss * args.gradient_accumulation_steps / nb_tr_steps
pbar.set_postfix_str(f"Loss: {mean_loss:.5f}") pbar.set_postfix_str(f"Loss: {mean_loss:.5f}")
if (step + 1) % args.gradient_accumulation_steps == 0: if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16: if args.fp16:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment