Unverified Commit dcbb3f18 authored by bogdanm's avatar bogdanm Committed by GitHub
Browse files

[Bugfix] Correct LayerNorm epsilon parameter in modernbert.py (#27008)


Signed-off-by: default avatarbogdanm <152898065+bogdan01m@users.noreply.github.com>
Co-authored-by: default avatargemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent 00417f4e
......@@ -39,9 +39,12 @@ class ModernBertEmbeddings(nn.Module):
self.tok_embeddings = VocabParallelEmbedding(
config.vocab_size, config.hidden_size
)
self.norm = nn.LayerNorm(
config.hidden_size, eps=config.layer_norm_eps, bias=config.norm_bias
eps = (
getattr(config, "norm_eps", None)
or getattr(config, "layer_norm_eps", None)
or 1e-5
)
self.norm = nn.LayerNorm(config.hidden_size, eps=eps, bias=config.norm_bias)
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.tok_embeddings(input_ids)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment