Unverified Commit 604efe07 authored by Xinyuan Tong's avatar Xinyuan Tong Committed by GitHub
Browse files

Updates Gemma3n MLP layer to adapt latest transformers version (#7573)


Signed-off-by: default avatarXinyuan Tong <justinning0323@outlook.com>
parent 1b8cf77b
...@@ -62,7 +62,7 @@ class Gemma3nTextScaledWordEmbedding(Gemma3TextScaledWordEmbedding): ...@@ -62,7 +62,7 @@ class Gemma3nTextScaledWordEmbedding(Gemma3TextScaledWordEmbedding):
pass pass
class Gemma3nMLP(nn.Module): class Gemma3nTextMLP(nn.Module):
def __init__( def __init__(
self, self,
hidden_size: int, hidden_size: int,
...@@ -514,10 +514,11 @@ class Gemma3nDecoderLayer(nn.Module): ...@@ -514,10 +514,11 @@ class Gemma3nDecoderLayer(nn.Module):
prefix=add_prefix("self_attn", prefix), prefix=add_prefix("self_attn", prefix),
) )
intermediate_size = config.intermediate_size[layer_id]
activation_sparsity = config.activation_sparsity_pattern[layer_id] activation_sparsity = config.activation_sparsity_pattern[layer_id]
self.mlp = Gemma3nMLP( self.mlp = Gemma3nTextMLP(
hidden_size=self.hidden_size, hidden_size=self.hidden_size,
intermediate_size=config.intermediate_size, intermediate_size=intermediate_size,
hidden_activation=config.hidden_activation, hidden_activation=config.hidden_activation,
activation_sparsity=activation_sparsity, activation_sparsity=activation_sparsity,
quant_config=quant_config, quant_config=quant_config,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment