Fix non-negligible difference between GPT2 and TFGP2 (#13679)

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>

src/transformers/models/gpt2/modeling_tf_gpt2.py

@@ -174,7 +174,7 @@ class TFMLP(tf.keras.layers.Layer):
         nx = config.n_embd
         self.c_fc = TFConv1D(n_state, nx, initializer_range=config.initializer_range, name="c_fc")
         self.c_proj = TFConv1D(nx, n_state, initializer_range=config.initializer_range, name="c_proj")
-        self.act = get_tf_activation("gelu")
+        self.act = get_tf_activation(config.activation_function)
         self.dropout = tf.keras.layers.Dropout(config.resid_pdrop)
 
     def call(self, x, training=False):