Skip to content

GitLab

Unverified Commit 5b2a9422 authored by danisereb's avatar danisereb Committed by GitHub
Browse files

[BugFix] Fix LoRA Fp8 (#33879) 


Signed-off-by: default avatarDaniel Serebrenik <daserebrenik@nvidia.com>
parent c1858b7e
Hide whitespace changes
Inline Side-by-side
Showing with 14 additions and 8 deletions
vllm/lora/layers/fused_moe.py
View file @ 5b2a9422
...@@ -130,14 +130,20 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA): ...@@ -130,14 +130,20 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
self.base_layer.ensure_moe_quant_config_init() self.base_layer.ensure_moe_quant_config_init()
quant_config = self.base_layer.quant_method.moe_quant_config quant_config = self.base_layer.quant_method.moe_quant_config
prepare_finalize = MoEPrepareAndFinalizeNoEP() if getattr(self.base_layer.quant_method, "supports_internal_mk", False):
m_fused_moe_fn = FusedMoEModularKernel( # Use the existing modular kernel from the quant method
prepare_finalize, m_fused_moe_fn = self.base_layer.quant_method.moe_mk
self.base_layer.quant_method.select_gemm_impl( else:
prepare_finalize, self.base_layer # Create a new modular kernel via select_gemm_impl
), prepare_finalize = MoEPrepareAndFinalizeNoEP()
self.base_layer.shared_experts, m_fused_moe_fn = FusedMoEModularKernel(
) prepare_finalize,
self.base_layer.quant_method.select_gemm_impl(
prepare_finalize, self.base_layer
),
self.base_layer.shared_experts,
)
if quant_config.use_mxfp4_w4a16: if quant_config.use_mxfp4_w4a16:
assert isinstance( assert isinstance(
m_fused_moe_fn.fused_experts, (MarlinExperts, UnfusedOAITritonExperts) m_fused_moe_fn.fused_experts, (MarlinExperts, UnfusedOAITritonExperts)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment