[Bugfix] Fix ColumnParallelLinearWithLoRA slice (#11708)

Signed-off-by: ZincCat <zincchloride@outlook.com>

[Bugfix] Fix ColumnParallelLinearWithLoRA slice (#11708)
Signed-off-by: ZincCat <zincchloride@outlook.com>
61fed92c · ZincCat · GitHub · 80c751e7 · 61fed92c
Unverified Commit 61fed92c authored Jan 03, 2025 by ZincCat Committed by GitHub Jan 03, 2025
Show whitespace changes
Inline Side-by-side

Showing with 2 additions and 2 deletions

vllm/lora/layers.py vllm/lora/layers.py +2 -2

No files found.
--- a/vllm/lora/layers.py
+++ b/vllm/lora/layers.py
@@ -479,7 +479,7 @@ class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
        # ColumnParallelLinear.
        else:
            tensor_model_parallel_rank = get_tensor_model_parallel_rank()
-            shard_size = self.output_dim
+            shard_size = self.output_size
            start_idx = tensor_model_parallel_rank * shard_size
            end_idx = (tensor_model_parallel_rank + 1) * shard_size
            lora_b = lora_b[:, start_idx:end_idx]
@@ -490,7 +490,7 @@ class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
        if bias is None:
            return bias
        tensor_model_parallel_rank = get_tensor_model_parallel_rank()
-        shard_size = self.output_dim
+        shard_size = self.output_size
        start_idx = tensor_model_parallel_rank * shard_size
        end_idx = (tensor_model_parallel_rank + 1) * shard_size
        bias = bias[start_idx:end_idx]