Skip to content

GitLab

Unverified Commit e048c7f1 authored by Thomas Wolf's avatar Thomas Wolf Committed by GitHub
Browse files

Merge pull request #171 from donglixp/patch-1 

LayerNorm initialization
parents d3d56f9a d0d9b384
Show whitespace changes
Inline Side-by-side
Showing with 2 additions and 2 deletions
pytorch_pretrained_bert/modeling.py
View file @ e048c7f1
...@@ -439,8 +439,8 @@ class PreTrainedBertModel(nn.Module): ...@@ -439,8 +439,8 @@ class PreTrainedBertModel(nn.Module):
# cf https://github.com/pytorch/pytorch/pull/5617 # cf https://github.com/pytorch/pytorch/pull/5617
module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
elif isinstance(module, BertLayerNorm): elif isinstance(module, BertLayerNorm):
module.bias.data.normal_(mean=0.0, std=self.config.initializer_range) module.bias.data.zero_()
module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) module.weight.data.fill_(1.0)
if isinstance(module, nn.Linear) and module.bias is not None: if isinstance(module, nn.Linear) and module.bias is not None:
module.bias.data.zero_() module.bias.data.zero_()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment