Skip to content

GitLab

"...git@developer.sourcefind.cn:chenpangpang/transformers.git" did not exist on "ef0ac063c9b9da3e4da759866736e266dbb44cfe"
Commit e6d106a0 authored by thomwolf's avatar thomwolf
Browse files

comment on gelu function

parent 01b1a053
Show whitespace changes
Inline Side-by-side
Showing with 2 additions and 2 deletions
modeling_pytorch.py
View file @ e6d106a0
...@@ -28,8 +28,8 @@ from torch.nn import CrossEntropyLoss ...@@ -28,8 +28,8 @@ from torch.nn import CrossEntropyLoss
def gelu(x): def gelu(x):
return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0))) return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
# OpenAI GPT gelu version : # For information: OpenAI GPT gelu version is a bit different:
# return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) # 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
class BertConfig(object): class BertConfig(object):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment