Commit af884028 authored by baberabb's avatar baberabb
Browse files

fix tensor parallel

parent 5ba54539
...@@ -37,7 +37,7 @@ class VLLM(LM): ...@@ -37,7 +37,7 @@ class VLLM(LM):
dtype=dtype, dtype=dtype,
tokenizer_mode=tokenizer_mode, tokenizer_mode=tokenizer_mode,
trust_remote_code=trust_remote_code, trust_remote_code=trust_remote_code,
tensor_parallel_size=tensor_parallel_size, tensor_parallel_size=int(tensor_parallel_size),
swap_space=swap_space, swap_space=swap_space,
quantization=quantization, quantization=quantization,
) )
...@@ -87,7 +87,7 @@ class VLLM(LM): ...@@ -87,7 +87,7 @@ class VLLM(LM):
generate: bool = False, generate: bool = False,
max_tokens: int = None, max_tokens: int = None,
stop: Optional[List[str]] = None, stop: Optional[List[str]] = None,
use_tqdm=False, use_tqdm=True,
**kwargs, **kwargs,
): ):
if "do_sample" in kwargs.keys(): if "do_sample" in kwargs.keys():
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment