"examples/pytorch/vscode:/vscode.git/clone" did not exist on "27520bc5154be34340f7574d6034b92822883c2b"
Unverified Commit 604efe07 authored by Xinyuan Tong's avatar Xinyuan Tong Committed by GitHub
Browse files

Updates Gemma3n MLP layer to adapt latest transformers version (#7573)


Signed-off-by: default avatarXinyuan Tong <justinning0323@outlook.com>
parent 1b8cf77b
......@@ -62,7 +62,7 @@ class Gemma3nTextScaledWordEmbedding(Gemma3TextScaledWordEmbedding):
pass
class Gemma3nMLP(nn.Module):
class Gemma3nTextMLP(nn.Module):
def __init__(
self,
hidden_size: int,
......@@ -514,10 +514,11 @@ class Gemma3nDecoderLayer(nn.Module):
prefix=add_prefix("self_attn", prefix),
)
intermediate_size = config.intermediate_size[layer_id]
activation_sparsity = config.activation_sparsity_pattern[layer_id]
self.mlp = Gemma3nMLP(
self.mlp = Gemma3nTextMLP(
hidden_size=self.hidden_size,
intermediate_size=config.intermediate_size,
intermediate_size=intermediate_size,
hidden_activation=config.hidden_activation,
activation_sparsity=activation_sparsity,
quant_config=quant_config,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment