Add linear activation

Summary: Pull Request resolved: https://github.com/fairinternal/fairseq-py/pull/698

Differential Revision: D16068477

Pulled By: myleott

fbshipit-source-id: a68f6f519dc5481f857d8e10cc443249eccb2545

fairseq/utils.py

@@ -320,6 +320,8 @@ def get_activation_fn(activation: str) -> Callable:
         return gelu_accurate
     elif activation == 'tanh':
         return torch.tanh
+    elif activation == 'linear':
+        return lambda x: x
     else:
         raise RuntimeError("--activation-fn {} not supported".format(activation))
 
@@ -331,4 +333,5 @@ def get_available_activation_fns() -> List:
         'gelu_fast',  # deprecated
         'gelu_accurate',
         'tanh',
+        'linear',
     ]