Unverified Commit 426cc862 authored by Yong Hoon Shin's avatar Yong Hoon Shin Committed by GitHub
Browse files

[BugFix] Fix routed_scaling_factor double mul for dots1 and glm4 MoE models (#24132)


Signed-off-by: default avatarYong Hoon Shin <yhshin@meta.com>
parent e81d4e69
......@@ -137,7 +137,8 @@ class Dots1MoE(nn.Module):
topk_group=config.topk_group,
prefix=f"{prefix}.experts",
scoring_func=config.scoring_func,
routed_scaling_factor=self.routed_scaling_factor,
# we do scaling outside, set factor to 1.0 to avoid double mul
routed_scaling_factor=1.0,
e_score_correction_bias=self.gate.e_score_correction_bias)
if config.n_shared_experts is not None:
......
......@@ -159,7 +159,8 @@ class Glm4MoE(nn.Module):
topk_group=config.topk_group,
prefix=f"{prefix}.experts",
scoring_func="sigmoid",
routed_scaling_factor=self.routed_scaling_factor,
# we do scaling outside, set factor to 1.0 to avoid double mul
routed_scaling_factor=1.0,
e_score_correction_bias=self.gate.e_score_correction_bias,
enable_eplb=self.enable_eplb,
num_redundant_experts=self.n_redundant_experts)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment