Skip to content

GitLab

You need to sign in or sign up before continuing.
Commit e6d106a0 authored by thomwolf's avatar thomwolf
Browse files

comment on gelu function

parent 01b1a053
Show whitespace changes
Inline Side-by-side
Showing with 2 additions and 2 deletions
modeling_pytorch.py
View file @ e6d106a0
...@@ -28,8 +28,8 @@ from torch.nn import CrossEntropyLoss ...@@ -28,8 +28,8 @@ from torch.nn import CrossEntropyLoss
def gelu(x): def gelu(x):
return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0))) return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
# OpenAI GPT gelu version : # For information: OpenAI GPT gelu version is a bit different:
# return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) # 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
class BertConfig(object): class BertConfig(object):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment