Refactor Fairseq models for BERT and XLM to use TransformerSentenceEncoder (#622)

Summary:
Pull Request resolved: https://github.com/pytorch/fairseq/pull/622

Updating some defaults to more meaningful values

Reviewed By: rutyrinott

Differential Revision: D14761263

fbshipit-source-id: 7ac670aa370f315ddfb511c63273583a6062c569

fairseq/modules/transformer_sentence_encoder_layer.py

@@ -37,8 +37,8 @@ class TransformerSentenceEncoderLayer(nn.Module):
         dropout: float = 0.1,
         attention_dropout: float = 0.1,
         activation_dropout: float = 0.1,
-        encoder_normalize_before: bool = True,
-        use_bert_layer_norm: bool = True,
+        encoder_normalize_before: bool = False,
+        use_bert_layer_norm: bool = False,
         use_gelu: bool = True,
     ) -> None:
 
@@ -108,6 +108,7 @@ class TransformerSentenceEncoderLayer(nn.Module):
         x = F.dropout(x, p=self.dropout, training=self.training)
         x = residual + x
         x = self._maybe_layer_norm(self.self_attn_layer_norm, x, after=True)
+
         residual = x
         x = self._maybe_layer_norm(self.final_layer_norm, x, before=True)
         x = self.activation_fn(self.fc1(x))