Skip to content

GitLab

"git@developer.sourcefind.cn:modelzoo/paddleocr.git" did not exist on "da2d01b3774ebadd3cc4d16200f6a2e2bbf43dd1"
Unverified Commit 8a7749b8 authored by drbh's avatar drbh Committed by GitHub
Browse files

fix: include create_exllama_buffers and set_device for exllama (#2407)

parent 9a7830bd
Hide whitespace changes
Inline Side-by-side
Showing with 4 additions and 0 deletions
server/text_generation_server/layers/gptq/__init__.py
View file @ 8a7749b8
...@@ -422,12 +422,16 @@ elif CAN_EXLLAMA: ...@@ -422,12 +422,16 @@ elif CAN_EXLLAMA:
if V2: if V2:
from text_generation_server.layers.gptq.exllamav2 import ( from text_generation_server.layers.gptq.exllamav2 import (
QuantLinear as ExllamaQuantLinear, # noqa: F401 QuantLinear as ExllamaQuantLinear, # noqa: F401
create_exllama_buffers, # noqa: F401
set_device, # noqa: F401
) )
HAS_EXLLAMA = "2" HAS_EXLLAMA = "2"
else: else:
from text_generation_server.layers.gptq.exllama import ( from text_generation_server.layers.gptq.exllama import (
Ex4bitLinear as ExllamaQuantLinear, # noqa: F401 Ex4bitLinear as ExllamaQuantLinear, # noqa: F401
create_exllama_buffers, # noqa: F401
set_device, # noqa: F401
) )
HAS_EXLLAMA = "1" HAS_EXLLAMA = "1"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment