Fix gradient clipping when --clip-norm=0

fairseq/multiprocessing_trainer.py

@@ -265,7 +265,10 @@ class MultiprocessingTrainer(MultiprocessingEventLoop):
         self._all_reduce_and_rescale_grads(grad_denom)
 
         # clip grads
-        grad_norm = torch.nn.utils.clip_grad_norm(self.model.parameters(), self.args.clip_norm)
+        if self.args.clip_norm > 0:
+            grad_norm = torch.nn.utils.clip_grad_norm(self.model.parameters(), self.args.clip_norm)
+        else:
+            grad_norm = math.sqrt(sum([p.grad.data.norm()**2 for p in self.model.parameters()]))
 
         # take an optimization step
         self.optimizer.step()