[Qwen2.5] Support bnb quant for Qwen2.5 (#9467)

154a8ae8 · Haoyu Wang · GitHub · de4008e2 · 154a8ae8
Unverified Commit 154a8ae8 authored Oct 18, 2024 by Haoyu Wang Committed by GitHub Oct 18, 2024
Hide whitespace changes
Inline Side-by-side

Showing with 8 additions and 0 deletions

vllm/model_executor/models/qwen2.py vllm/model_executor/models/qwen2.py +8 -0

No files found.
--- a/vllm/model_executor/models/qwen2.py
+++ b/vllm/model_executor/models/qwen2.py
@@ -364,6 +364,14 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
    ]
    embedding_modules = {}
    embedding_padding_modules = []
+    bitsandbytes_stacked_params_mapping = {
+        # shard_name, weight_name, index
+        "q_proj": ("qkv_proj", 0),
+        "k_proj": ("qkv_proj", 1),
+        "v_proj": ("qkv_proj", 2),
+        "gate_proj": ("gate_up_proj", 0),
+        "up_proj": ("gate_up_proj", 1),
+    }
    def __init__(
        self,