Switch to torch.nn.functional.gelu when available

Summary: Pull Request resolved: https://github.com/fairinternal/fairseq-py/pull/735

Differential Revision: D16377046

Pulled By: myleott

fbshipit-source-id: 9725d4a3ce6b2fc8cee0b1d1cb8921f9d59c551a

fairseq/modules/gelu.py

@@ -21,4 +21,7 @@ def gelu_accurate(x):
 
 
 def gelu(x: torch.Tensor) -> torch.Tensor:
-    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
+    if hasattr(torch.nn.functional, 'gelu'):
+        return torch.nn.functional.gelu(x.float()).type_as(x)
+    else:
+        return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))