Unverified Commit ad6c2e1a authored by Seiji Eicher's avatar Seiji Eicher Committed by GitHub
Browse files

Correct PPMissingLayer handling in Deepseek-V2-Lite PP deployment (#20665)


Signed-off-by: default avatarSeiji Eicher <seiji@anyscale.com>
parent 49e8c7ea
...@@ -739,14 +739,20 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP, MixtureOfExperts): ...@@ -739,14 +739,20 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP, MixtureOfExperts):
self.num_expert_groups = config.n_group self.num_expert_groups = config.n_group
self.moe_layers: list[FusedMoE] = [] self.moe_layers: list[FusedMoE] = []
example_moe = None
for layer in self.model.layers: for layer in self.model.layers:
if isinstance(layer, PPMissingLayer):
continue
assert isinstance(layer, DeepseekV2DecoderLayer) assert isinstance(layer, DeepseekV2DecoderLayer)
if isinstance(layer.mlp, DeepseekV2MoE): if isinstance(layer.mlp, DeepseekV2MoE):
# Pick last one layer since the first ones may be dense layers.
example_moe = layer.mlp
self.moe_layers.append(layer.mlp.experts) self.moe_layers.append(layer.mlp.experts)
# Pick last one layer since the first ones may be dense layers. if example_moe is None:
example_moe = typing.cast( raise RuntimeError("No DeepseekV2MoE layer found in model.layers.")
DeepseekV2MoE, self.model.layers[config.num_hidden_layers - 1].mlp)
self.num_logical_experts = example_moe.n_logical_experts self.num_logical_experts = example_moe.n_logical_experts
self.num_physical_experts = example_moe.n_physical_experts self.num_physical_experts = example_moe.n_physical_experts
self.num_local_physical_experts = example_moe.n_local_physical_experts self.num_local_physical_experts = example_moe.n_local_physical_experts
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment