Unverified Commit 760286e3 authored by Yuxuan Zhang's avatar Yuxuan Zhang Committed by GitHub
Browse files

use fp32 for e_score_correction_bias in GLM-4.5 (#8729)

parent 3435a24e
......@@ -343,7 +343,7 @@ class Glm4MoeGate(nn.Module):
torch.empty((config.n_routed_experts, config.hidden_size))
)
self.e_score_correction_bias = nn.Parameter(
torch.empty((config.n_routed_experts))
torch.empty((config.n_routed_experts), dtype=torch.float32)
)
if _is_cpu and _is_cpu_amx_available:
self.quant_method = PackWeightMethod(weight_names=["weight"])
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment