[Bugfix] Fix lora loading for Compressed Tensors in #9120 (#9179)

21906a6f · Ahmad Fahadh Ilyas · GitHub · dc4aea67 · 21906a6f
Unverified Commit 21906a6f authored Oct 09, 2024 by Ahmad Fahadh Ilyas Committed by GitHub Oct 09, 2024
Show whitespace changes
Inline Side-by-side

Showing with 3 additions and 0 deletions

vllm/lora/layers.py vllm/lora/layers.py +3 -0

No files found.
--- a/vllm/lora/layers.py
+++ b/vllm/lora/layers.py
@@ -39,6 +39,9 @@ def _get_lora_device(base_layer: nn.Module) -> torch.device:
    # unquantizedLinear
    if hasattr(base_layer, "weight"):
        return base_layer.weight.device
+    # Compressed Tensor
+    elif hasattr(base_layer, "weight_packed"):
+        return base_layer.weight_packed.device
    # GPTQ/AWQ
    elif hasattr(base_layer, "qweight"):
        return base_layer.qweight.device