Skip to content

GitLab

Unverified Commit 948c8595 authored by zixuanzhang226's avatar zixuanzhang226 Committed by GitHub
Browse files

support bitsandbytes quantization with qwen model (#10549) 


Signed-off-by: default avatarUbuntu <zixuanzhang@bytedance.com>
parent 97814fbf
Hide whitespace changes
Inline Side-by-side
Showing with 12 additions and 0 deletions
vllm/model_executor/models/qwen.py
View file @ 948c8595
...@@ -1028,6 +1028,18 @@ class QWenLLM(QWenBaseModel): ...@@ -1028,6 +1028,18 @@ class QWenLLM(QWenBaseModel):
embedding_modules = {} embedding_modules = {}
embedding_padding_modules = [] embedding_padding_modules = []
default_bitsandbytes_target_modules = [
".c_attn.",
".c_proj.",
".w1.",
".w2.",
]
bitsandbytes_stacked_params_mapping = {
# shard_name, weight_name, index
"w2": ("gate_up_proj", 0),
"w1": ("gate_up_proj", 1),
}
class QWenVL(QWenBaseModel, SupportsMultiModal): class QWenVL(QWenBaseModel, SupportsMultiModal):
packed_modules_mapping = { packed_modules_mapping = {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment