Skip to content

GitLab

"vscode:/vscode.git/clone" did not exist on "26dad0a9fab0694ce1128d91838b9ca78e1554f6"
Unverified Commit b6f47b53 authored by Sourab Mangrulkar's avatar Sourab Mangrulkar Committed by GitHub
Browse files

fsdp bf16 enable autocast (#21847)

parent fb76994c
Show whitespace changes
Inline Side-by-side
Showing with 0 additions and 3 deletions
src/transformers/trainer.py
View file @ b6f47b53
...@@ -618,9 +618,6 @@ class Trainer: ...@@ -618,9 +618,6 @@ class Trainer:
self.scaler = GradScaler() self.scaler = GradScaler()
else: else:
self.scaler = torch.cuda.amp.GradScaler() self.scaler = torch.cuda.amp.GradScaler()
elif self.fsdp is not None:
self.use_cuda_amp = False
self.amp_dtype = None
elif args.half_precision_backend == "cpu_amp": elif args.half_precision_backend == "cpu_amp":
self.use_cpu_amp = True self.use_cpu_amp = True
self.amp_dtype = torch.bfloat16 self.amp_dtype = torch.bfloat16
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment