Merge pull request #171 from donglixp/patch-1

LayerNorm initialization

Merge pull request #171 from donglixp/patch-1
LayerNorm initialization
e048c7f1 · Thomas Wolf · GitHub · d3d56f9a · d0d9b384 · e048c7f1
Unverified Commit e048c7f1 authored Jan 07, 2019 by Thomas Wolf Committed by GitHub Jan 07, 2019
Show whitespace changes
Inline Side-by-side

Showing with 2 additions and 2 deletions

pytorch_pretrained_bert/modeling.py pytorch_pretrained_bert/modeling.py +2 -2

No files found.
--- a/pytorch_pretrained_bert/modeling.py
+++ b/pytorch_pretrained_bert/modeling.py
@@ -439,8 +439,8 @@ class PreTrainedBertModel(nn.Module):
            # cf https://github.com/pytorch/pytorch/pull/5617
            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
        elif isinstance(module, BertLayerNorm):
-            module.bias.data.normal_(mean=0.0, std=self.config.initializer_range)
-            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
+            module.bias.data.zero_()
+            module.weight.data.fill_(1.0)
        if isinstance(module, nn.Linear) and module.bias is not None:
            module.bias.data.zero_()