fix transformer kernel preln config (#257)

fc1de4ff · eltonzheng · GitHub · 2312f04b · fc1de4ff
Unverified Commit fc1de4ff authored Jun 08, 2020 by eltonzheng Committed by GitHub Jun 08, 2020
Show whitespace changes
Inline Side-by-side

Showing with 1 addition and 1 deletion

deepspeed/pt/deepspeed_cuda.py deepspeed/pt/deepspeed_cuda.py +1 -1

No files found.
--- a/deepspeed/pt/deepspeed_cuda.py
+++ b/deepspeed/pt/deepspeed_cuda.py
@@ -106,7 +106,7 @@ class DeepSpeedTransformerConfig(TransformerConfig):
                             num_hidden_layers,
                             initializer_range)
        self.fp16 = fp16
-        self.pre_layer_norm = pre_layer_norm = True
+        self.pre_layer_norm = pre_layer_norm
        self.local_rank = local_rank
        self.seed = seed
        self.normalize_invertible = normalize_invertible