Skip to content

GitLab

"ssh:/git@developer.sourcefind.cn:2222/OpenDAS/vllm_cscc.git" did not exist on "ec13e549d3e1de13d05af759cc8bef3f7cf5e318"
Unverified Commit 8334c39f authored by Isotr0py's avatar Isotr0py Committed by GitHub
Browse files

[Bugfix] Fix new Llama3.1 GGUF model loading (#7269)

parent e9045767
Hide whitespace changes
Inline Side-by-side
Showing with 16 additions and 14 deletions
vllm/model_executor/model_loader/weight_utils.py
View file @ 8334c39f
...@@ -435,23 +435,25 @@ def gguf_quant_weights_iterator( ...@@ -435,23 +435,25 @@ def gguf_quant_weights_iterator(
reader = gguf.GGUFReader(gguf_file) reader = gguf.GGUFReader(gguf_file)
for tensor in reader.tensors: for tensor in reader.tensors:
weight_type = tensor.tensor_type if tensor.name in gguf_to_hf_name_map:
name = gguf_to_hf_name_map[tensor.name] weight_type = tensor.tensor_type
name = gguf_to_hf_name_map[tensor.name]
if weight_type.name != "F32": if weight_type.name != "F32":
weight_type_name = name.replace("weight", "qweight_type") weight_type_name = name.replace("weight", "qweight_type")
weight_type = torch.tensor(weight_type) weight_type = torch.tensor(weight_type)
yield weight_type_name, weight_type yield weight_type_name, weight_type
for tensor in reader.tensors: for tensor in reader.tensors:
weight = tensor.data if tensor.name in gguf_to_hf_name_map:
weight_type = tensor.tensor_type weight = tensor.data
name = gguf_to_hf_name_map[tensor.name] weight_type = tensor.tensor_type
name = gguf_to_hf_name_map[tensor.name]
if weight_type.name != "F32":
name = name.replace("weight", "qweight") if weight_type.name != "F32":
param = torch.tensor(weight) name = name.replace("weight", "qweight")
yield name, param param = torch.tensor(weight)
yield name, param
def kv_cache_scales_loader( def kv_cache_scales_loader(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment