Skip to content

GitLab

"examples/sampling/vscode:/vscode.git/clone" did not exist on "c0ac2f60b7e6622bae3a5b8a79686f55bc7b4ae3"
Unverified Commit 1b1b47a9 authored by lambert0312's avatar lambert0312 Committed by GitHub
Browse files

Fix w8a8_int8 model shared experts fusion load weights error (#5120)

parent 3c9740d2
Hide whitespace changes
Inline Side-by-side
Showing with 18 additions and 8 deletions
python/sglang/srt/models/deepseek_v2.py
View file @ 1b1b47a9
...@@ -1480,14 +1480,24 @@ class DeepseekV2ForCausalLM(nn.Module): ...@@ -1480,14 +1480,24 @@ class DeepseekV2ForCausalLM(nn.Module):
if self.n_share_experts_fusion is not None and self.n_share_experts_fusion > 0: if self.n_share_experts_fusion is not None and self.n_share_experts_fusion > 0:
weights_list = list(weights) weights_list = list(weights)
weights_dict = dict(weights_list) weights_dict = dict(weights_list)
suffix_list = [ if self.quant_config.get_name() == "w8a8_int8":
"down_proj.weight", suffix_list = [
"down_proj.weight_scale_inv", "down_proj.weight",
"gate_proj.weight", "down_proj.weight_scale",
"gate_proj.weight_scale_inv", "gate_proj.weight",
"up_proj.weight", "gate_proj.weight_scale",
"up_proj.weight_scale_inv", "up_proj.weight",
] "up_proj.weight_scale",
]
else:
suffix_list = [
"down_proj.weight",
"down_proj.weight_scale_inv",
"gate_proj.weight",
"gate_proj.weight_scale_inv",
"up_proj.weight",
"up_proj.weight_scale_inv",
]
names_to_remove = [] names_to_remove = []
for moe_layer in tqdm( for moe_layer in tqdm(
range( range(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment