Skip to content

GitLab

Unverified Commit cfea9c04 authored by Chen Wu's avatar Chen Wu Committed by GitHub
Browse files

[Model] Fix Baichuan BNB online quantization (#10572) 


Signed-off-by: default avatarChen Wu <cntryroa@gmail.com>
parent 7d8ffb34
Show whitespace changes
Inline Side-by-side
Showing with 15 additions and 0 deletions
vllm/model_executor/models/baichuan.py
View file @ cfea9c04
......@@ -350,6 +350,21 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
embedding_modules = {}
embedding_padding_modules = []
# BitandBytes specific attributes
default_bitsandbytes_target_modules = [
".W_pack.",
".o_proj.",
".down_proj.",
".up_proj.",
".gate_proj.",
".up_proj.",
]
bitsandbytes_stacked_params_mapping = {
# shard_name, weight_name, index
"gate_proj": ("gate_up_proj", 0),
"up_proj": ("gate_up_proj", 1),
}
def __init__(
self,
*,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment