[Misc] Add assertion and helpful message for marlin24 compressed models (#11388)

b866cdbd · Dipika Sikka · GitHub · 2e726680 · b866cdbd
Unverified Commit b866cdbd authored Dec 23, 2024 by Dipika Sikka Committed by GitHub Dec 24, 2024
Show whitespace changes
Inline Side-by-side

Showing with 4 additions and 0 deletions

vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py ...compressed_tensors/schemes/compressed_tensors_w4a16_24.py +4 -0

No files found.
--- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py
+++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py
@@ -61,6 +61,10 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
                       params_dtype: torch.dtype, weight_loader: Callable,
                       **kwargs):

+        assert params_dtype == torch.float16, (
+            "float16 is required for marlin24 compressd models. Set dtype=torch.float16"  # noqa: E501
+        )
+
        pack_factor = 32 // self.quant_type.size_bits
        output_size_per_partition = sum(output_partition_sizes)