Better OOM recovery

Summary: Pull Request resolved: https://github.com/pytorch/fairseq/pull/685

Differential Revision: D15154647

Pulled By: myleott

fbshipit-source-id: 36c72359755192a4a53367e19f8dd006791d483c

fairseq/trainer.py

@@ -331,7 +331,7 @@ class Trainer(object):
                     print('| WARNING: ran out of memory, retrying batch')
                     for p in self.model.parameters():
                         if p.grad is not None:
-                            del p.grad  # free some memory
+                            p.grad = None  # free some memory
                     if self.cuda:
                         torch.cuda.empty_cache()
                     return self.valid_step(sample, raise_oom=True)