Skip to content

GitLab

Unverified Commit 33d3914b authored by Philipp Moritz's avatar Philipp Moritz Committed by GitHub
Browse files

[Bugfix] Fix dynamic FP8 quantization for Mixtral (#4793)

parent 1356df53
Hide whitespace changes
Inline Side-by-side
Showing with 1 addition and 1 deletion
vllm/model_executor/models/mixtral.py
View file @ 33d3914b
...@@ -95,7 +95,7 @@ class MixtralMoE(nn.Module): ...@@ -95,7 +95,7 @@ class MixtralMoE(nn.Module):
params_dtype=self.params_dtype, params_dtype=self.params_dtype,
quant_config=None) quant_config=None)
if self.use_fp8: if self.use_fp8 and self.quant_config.is_checkpoint_fp8_serialized:
params_dtype = torch.float8_e4m3fn params_dtype = torch.float8_e4m3fn
self.w13_weight = nn.Parameter( self.w13_weight = nn.Parameter(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment