Fix nan-loss condition (#13911)

5d390e9e · Anton Lozhkov · GitHub · 8f2c07d3 · 5d390e9e
Unverified Commit 5d390e9e authored Oct 06, 2021 by Anton Lozhkov Committed by GitHub Oct 06, 2021
Hide whitespace changes
Inline Side-by-side

Showing with 7 additions and 4 deletions

src/transformers/trainer.py src/transformers/trainer.py +7 -4

No files found.
--- a/src/transformers/trainer.py
+++ b/src/transformers/trainer.py
@@ -1315,10 +1315,13 @@ class Trainer:
                else:
                    tr_loss_step = self.training_step(model, inputs)
-                if args.logging_nan_inf_filter and not is_torch_tpu_available():
+                if (
-                    if torch.isnan(tr_loss_step) or torch.isinf(tr_loss_step):
+                    args.logging_nan_inf_filter
-                        # if loss is nan or inf simply add the average of previous logged losses
+                    and not is_torch_tpu_available()
-                        tr_loss += tr_loss / (1 + self.state.global_step - self._globalstep_last_logged)
+                    and (torch.isnan(tr_loss_step) or torch.isinf(tr_loss_step))
+                ):
+                    # if loss is nan or inf simply add the average of previous logged losses
+                    tr_loss += tr_loss / (1 + self.state.global_step - self._globalstep_last_logged)
                else:
                    tr_loss += tr_loss_step