Unverified Commit 120bf8b3 authored by Mo Li's avatar Mo Li Committed by GitHub
Browse files

add vllm model configs (#938)

parent 3e9844ed
from opencompass.models import VLLM
_meta_template = dict(
begin='<s>',
round=[
dict(role="HUMAN", begin='Human: ', end='\n'),
dict(role="BOT", begin="Assistant: ", end='</s>', generate=True),
],
eos_token_id=2
)
models = [
dict(
abbr='orionstar-14b-longchat-vllm',
type=VLLM,
path='OrionStarAI/Orion-14B-LongChat',
model_kwargs=dict(tensor_parallel_size=4),
generation_kwargs=dict(temperature=0),
meta_template=_meta_template,
max_out_len=100,
max_seq_len=4096,
batch_size=32,
run_cfg=dict(num_gpus=4, num_procs=1),
end_str='<|endoftext|>',
)
]
from opencompass.models import VLLM
_meta_template = dict(
round=[
dict(role="HUMAN", begin='<|im_start|>user\n', end='<|im_end|>\n'),
dict(role="BOT", begin="<|im_start|>assistant\n", end='<|im_end|>\n',
generate=True),
],
eos_token_id=151645,
)
models = [
dict(
type=VLLM,
abbr='qwen1.5-14b-chat-vllm',
path="Qwen/Qwen1.5-14B-Chat",
model_kwargs=dict(tensor_parallel_size=2),
meta_template=_meta_template,
max_out_len=100,
max_seq_len=2048,
batch_size=32,
generation_kwargs=dict(temperature=0),
end_str='<|im_end|>',
run_cfg=dict(num_gpus=2, num_procs=1),
)
]
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment