Skip to content

GitLab

Unverified Commit 7505b3ca authored by Fengzhe Zhou's avatar Fengzhe Zhou Committed by GitHub
Browse files

[Feature] Add huggingface apply_chat_template (#1098) 

* add TheoremQA with 5-shot

* add huggingface_above_v4_33 classes

* use num_worker partitioner in cli

* update theoremqa

* update TheoremQA

* add TheoremQA

* rename theoremqa -> TheoremQA

* update TheoremQA output path

* rewrite many model configs

* update huggingface

* further update

* refine configs

* update configs

* update configs

* add configs/eval_llama3_instruct.py

* add summarizer multi faceted

* update bbh datasets

* update configs/models/hf_llama/lmdeploy_llama3_8b_instruct.py

* rename class

* update readme

* update hf above v4.33
parent 6c711cb2
Hide whitespace changes
Inline Side-by-side
Showing with 99 additions and 378 deletions
configs/models/qwen/hf_qwen1_5_7b.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM from opencompass.models import HuggingFaceBaseModel
models = [ models = [
dict( dict(
type=HuggingFaceCausalLM, type=HuggingFaceBaseModel,
abbr='qwen1.5-7b-hf', abbr='qwen1.5-7b-hf',
path="Qwen/Qwen1.5-7B", path='Qwen/Qwen1.5-7B',
tokenizer_path='Qwen/Qwen1.5-7B', max_out_len=1024,
model_kwargs=dict(
device_map='auto',
trust_remote_code=True
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
use_fast=False,
),
pad_token_id=151645,
max_out_len=100,
max_seq_len=2048,
batch_size=8, batch_size=8,
run_cfg=dict(num_gpus=1, num_procs=1), run_cfg=dict(num_gpus=1),
) )
] ]
configs/models/qwen/hf_qwen1_5_7b_chat.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM from opencompass.models import HuggingFacewithChatTemplate
_meta_template = dict(
round=[
dict(role="HUMAN", begin='<|im_start|>user\n', end='<|im_end|>\n'),
dict(role="BOT", begin="<|im_start|>assistant\n", end='<|im_end|>\n', generate=True),
],
)
models = [ models = [
dict( dict(
type=HuggingFaceCausalLM, type=HuggingFacewithChatTemplate,
abbr='qwen1.5-7b-chat-hf', abbr='qwen1.5-7b-chat-hf',
path="Qwen/Qwen1.5-7B-Chat", path='Qwen/Qwen1.5-7B-Chat',
model_kwargs=dict( max_out_len=1024,
device_map='auto',
trust_remote_code=True
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
use_fast=False,
),
meta_template=_meta_template,
max_out_len=100,
max_seq_len=2048,
batch_size=8, batch_size=8,
run_cfg=dict(num_gpus=4, num_procs=1), run_cfg=dict(num_gpus=1),
end_str='<|im_end|>',
batch_padding=True,
) )
] ]
configs/models/qwen/hf_qwen_14b.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM from opencompass.models import HuggingFaceBaseModel
models = [ models = [
dict( dict(
type=HuggingFaceCausalLM, type=HuggingFaceBaseModel,
abbr='qwen-14b-hf', abbr='qwen-14b-hf',
path="Qwen/Qwen-14B", path='Qwen/Qwen-14B',
tokenizer_path='Qwen/Qwen-14B', max_out_len=1024,
model_kwargs=dict(
device_map='auto',
trust_remote_code=True,
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
use_fast=False,
),
pad_token_id=151643,
min_out_len=1,
max_out_len=100,
max_seq_len=2048,
batch_size=8, batch_size=8,
run_cfg=dict(num_gpus=1, num_procs=1), run_cfg=dict(num_gpus=1),
) )
] ]
configs/models/qwen/hf_qwen_14b_chat.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM from opencompass.models import HuggingFacewithChatTemplate
_meta_template = dict(
round=[
dict(role="HUMAN", begin='\n<|im_start|>user\n', end='<|im_end|>'),
dict(role="BOT", begin="\n<|im_start|>assistant\n", end='<|im_end|>', generate=True),
],
)
models = [ models = [
dict( dict(
type=HuggingFaceCausalLM, type=HuggingFacewithChatTemplate,
abbr='qwen-14b-chat-hf', abbr='qwen-14b-chat-hf',
path="Qwen/Qwen-14B-Chat", path='Qwen/Qwen-14B-Chat',
tokenizer_path='Qwen/Qwen-14B-Chat', max_out_len=1024,
model_kwargs=dict(
device_map='auto',
trust_remote_code=True
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
use_fast=False,
),
pad_token_id=151643,
max_out_len=100,
max_seq_len=2048,
batch_size=8, batch_size=8,
meta_template=_meta_template, run_cfg=dict(num_gpus=1),
run_cfg=dict(num_gpus=1, num_procs=1),
end_str='<|im_end|>',
) )
] ]
\ No newline at end of file
configs/models/qwen/hf_qwen_1_8b.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM from opencompass.models import HuggingFaceBaseModel
models = [ models = [
dict( dict(
type=HuggingFaceCausalLM, type=HuggingFaceBaseModel,
abbr='qwen-1.8b-hf', abbr='qwen-1.8b-hf',
path="Qwen/Qwen-1_8B", path='Qwen/Qwen-1_8B',
tokenizer_path='Qwen/Qwen-1_8B', max_out_len=1024,
model_kwargs=dict(
device_map='auto',
trust_remote_code=True,
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
use_fast=False,
),
pad_token_id=151643,
min_out_len=1,
max_out_len=100,
max_seq_len=2048,
batch_size=8, batch_size=8,
run_cfg=dict(num_gpus=1, num_procs=1), run_cfg=dict(num_gpus=1),
) )
] ]
configs/models/qwen/hf_qwen_1_8b_chat.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM from opencompass.models import HuggingFacewithChatTemplate
_meta_template = dict(
round=[
dict(role="HUMAN", begin='\n<|im_start|>user\n', end='<|im_end|>'),
dict(role="BOT", begin="\n<|im_start|>assistant\n", end='<|im_end|>', generate=True),
],
eos_token_id=151645,
)
models = [ models = [
dict( dict(
type=HuggingFaceCausalLM, type=HuggingFacewithChatTemplate,
abbr='qwen-1.8b-chat-hf', abbr='qwen-1.8b-chat-hf',
path="Qwen/Qwen-1_8B-Chat", path='Qwen/Qwen-1_8B-Chat',
tokenizer_path='Qwen/Qwen-1_8B-Chat', max_out_len=1024,
model_kwargs=dict(
device_map='auto',
trust_remote_code=True
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
use_fast=False,
),
pad_token_id=151643,
max_out_len=100,
max_seq_len=2048,
batch_size=8, batch_size=8,
meta_template=_meta_template, run_cfg=dict(num_gpus=1),
run_cfg=dict(num_gpus=1, num_procs=1),
end_str='<|im_end|>',
) )
] ]
configs/models/qwen/hf_qwen_72b.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM from opencompass.models import HuggingFaceBaseModel
models = [ models = [
dict( dict(
type=HuggingFaceCausalLM, type=HuggingFaceBaseModel,
abbr='qwen-72b-hf', abbr='qwen-72b-hf',
path="Qwen/Qwen-72B", path='Qwen/Qwen-72B',
tokenizer_path='Qwen/Qwen-72B', max_out_len=1024,
model_kwargs=dict(
device_map='auto',
trust_remote_code=True,
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
use_fast=False,
),
pad_token_id=151643,
min_out_len=1,
max_out_len=100,
max_seq_len=2048,
batch_size=8, batch_size=8,
run_cfg=dict(num_gpus=4, num_procs=1), run_cfg=dict(num_gpus=4),
) )
] ]
configs/models/qwen/hf_qwen_72b_chat.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM from opencompass.models import HuggingFacewithChatTemplate
_meta_template = dict(
round=[
dict(role="HUMAN", begin='\n<|im_start|>user\n', end='<|im_end|>'),
dict(role="BOT", begin="\n<|im_start|>assistant\n", end='<|im_end|>', generate=True),
],
)
models = [ models = [
dict( dict(
type=HuggingFaceCausalLM, type=HuggingFacewithChatTemplate,
abbr='qwen-72b-chat-hf', abbr='qwen-72b-chat-hf',
path="Qwen/Qwen-72B-Chat", path='Qwen/Qwen-72B-Chat',
tokenizer_path='Qwen/Qwen-72B-Chat', max_out_len=1024,
model_kwargs=dict(
device_map='auto',
trust_remote_code=True
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
use_fast=False,),
pad_token_id=151643,
max_out_len=100,
max_seq_len=2048,
batch_size=8, batch_size=8,
meta_template=_meta_template, run_cfg=dict(num_gpus=4),
run_cfg=dict(num_gpus=4, num_procs=1),
end_str='<|im_end|>',
) )
] ]
configs/models/qwen/hf_qwen_7b.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM from opencompass.models import HuggingFaceBaseModel
models = [ models = [
dict( dict(
type=HuggingFaceCausalLM, type=HuggingFaceBaseModel,
abbr='qwen-7b-hf', abbr='qwen-7b-hf',
path="Qwen/Qwen-7B", path='Qwen/Qwen-7B',
tokenizer_path='Qwen/Qwen-7B', max_out_len=1024,
model_kwargs=dict(
device_map='auto',
trust_remote_code=True,
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
use_fast=False,
),
pad_token_id=151643,
min_out_len=1,
max_out_len=100,
max_seq_len=2048,
batch_size=8, batch_size=8,
run_cfg=dict(num_gpus=1, num_procs=1), run_cfg=dict(num_gpus=1),
) )
] ]
configs/models/qwen/hf_qwen_7b_chat.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM from opencompass.models import HuggingFacewithChatTemplate
_meta_template = dict(
round=[
dict(role="HUMAN", begin='\n<|im_start|>user\n', end='<|im_end|>'),
dict(role="BOT", begin="\n<|im_start|>assistant\n", end='<|im_end|>', generate=True),
],
)
models = [ models = [
dict( dict(
type=HuggingFaceCausalLM, type=HuggingFacewithChatTemplate,
abbr='qwen-7b-chat-hf', abbr='qwen-7b-chat-hf',
path="Qwen/Qwen-7B-Chat", path='Qwen/Qwen-7B-Chat',
tokenizer_path='Qwen/Qwen-7B-Chat', max_out_len=1024,
model_kwargs=dict(
device_map='auto',
trust_remote_code=True
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
use_fast=False,
),
pad_token_id=151643,
max_out_len=100,
max_seq_len=2048,
batch_size=8, batch_size=8,
meta_template=_meta_template, run_cfg=dict(num_gpus=1),
run_cfg=dict(num_gpus=1, num_procs=1),
end_str='<|im_end|>',
) )
] ]
\ No newline at end of file
configs/models/qwen/vllm_qwen1_5_14b_chat.py
View file @ 7505b3ca
...@@ -6,7 +6,6 @@ _meta_template = dict( ...@@ -6,7 +6,6 @@ _meta_template = dict(
dict(role="HUMAN", begin='<|im_start|>user\n', end='<|im_end|>\n'), dict(role="HUMAN", begin='<|im_start|>user\n', end='<|im_end|>\n'),
dict(role="BOT", begin="<|im_start|>assistant\n", end='<|im_end|>\n', generate=True), dict(role="BOT", begin="<|im_start|>assistant\n", end='<|im_end|>\n', generate=True),
], ],
eos_token_id=151645,
) )
models = [ models = [
......
configs/models/qwen/vllm_qwen1_5_72b_chat.py
View file @ 7505b3ca
...@@ -6,7 +6,6 @@ _meta_template = dict( ...@@ -6,7 +6,6 @@ _meta_template = dict(
dict(role="HUMAN", begin='<|im_start|>user\n', end='<|im_end|>\n'), dict(role="HUMAN", begin='<|im_start|>user\n', end='<|im_end|>\n'),
dict(role="BOT", begin="<|im_start|>assistant\n", end='<|im_end|>\n', generate=True), dict(role="BOT", begin="<|im_start|>assistant\n", end='<|im_end|>\n', generate=True),
], ],
eos_token_id=151645,
) )
models = [ models = [
......
configs/models/skywork/hf_skywork_13b.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM from opencompass.models import HuggingFaceBaseModel
models = [ models = [
dict( dict(
type=HuggingFaceCausalLM, type=HuggingFaceBaseModel,
abbr='skywork-13b-hf', abbr='skywork-13b-hf',
path="Skywork/Skywork-13B-base", path='Skywork/Skywork-13B-base',
tokenizer_path='Skywork/Skywork-13B-base', max_out_len=1024,
model_kwargs=dict(
device_map='auto',
trust_remote_code=True,
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
use_fast=False,
),
max_out_len=100,
max_seq_len=2048,
batch_size=8, batch_size=8,
run_cfg=dict(num_gpus=1, num_procs=1), run_cfg=dict(num_gpus=1),
) )
] ]
configs/models/vicuna/hf_vicuna_13b_v13.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM from opencompass.models import HuggingFacewithChatTemplate
models = [ models = [
dict( dict(
type=HuggingFaceCausalLM, type=HuggingFacewithChatTemplate,
abbr='vicuna-13b-v1.3-hf', abbr='vicuna-13b-v1.3-hf',
path="lmsys/vicuna-13b-v1.3", path='lmsys/vicuna-13b-v1.3',
tokenizer_path='lmsys/vicuna-13b-v1.3', max_out_len=1024,
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
use_fast=False,
),
max_out_len=100,
max_seq_len=2048,
batch_size=8, batch_size=8,
model_kwargs=dict(device_map='auto'), run_cfg=dict(num_gpus=1),
batch_padding=False, # if false, inference with for-loop without batch padding fastchat_template='vicuna',
use_fastchat_template=True,
run_cfg=dict(num_gpus=2, num_procs=1)
) )
] ]
configs/models/vicuna/hf_vicuna_13b_v15.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM from opencompass.models import HuggingFacewithChatTemplate
models = [ models = [
dict( dict(
type=HuggingFaceCausalLM, type=HuggingFacewithChatTemplate,
abbr='vicuna-13b-v1.5-hf', abbr='vicuna-13b-v1.5-hf',
path="lmsys/vicuna-13b-v1.5", path='lmsys/vicuna-13b-v1.5',
tokenizer_path='lmsys/vicuna-13b-v1.5', max_out_len=1024,
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
use_fast=False,
),
max_out_len=100,
max_seq_len=2048,
batch_size=8, batch_size=8,
model_kwargs=dict(device_map='auto'), run_cfg=dict(num_gpus=1),
batch_padding=False, # if false, inference with for-loop without batch padding fastchat_template='vicuna',
use_fastchat_template=True,
run_cfg=dict(num_gpus=1, num_procs=1)
) )
] ]
configs/models/vicuna/hf_vicuna_13b_v15_16k.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM from opencompass.models import HuggingFacewithChatTemplate
_meta_template = dict(
round=[
dict(role="HUMAN", begin='USER: '),
dict(role="BOT", begin=" ASSISTANT:", end='</s>', generate=True),
],
)
models = [ models = [
dict( dict(
type=HuggingFaceCausalLM, type=HuggingFacewithChatTemplate,
abbr='vicuna-13b-v1.5-16k-hf', abbr='vicuna-13b-v1.5-16k-hf',
path="lmsys/vicuna-13b-v1.5-16k", path='lmsys/vicuna-13b-v1.5-16k',
tokenizer_path='lmsys/vicuna-13b-v1.5-16k', max_out_len=1024,
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
use_fast=False,
),
meta_template=_meta_template,
max_out_len=100,
max_seq_len=8192,
batch_size=8, batch_size=8,
model_kwargs=dict(device_map='auto'), run_cfg=dict(num_gpus=1),
batch_padding=False, # if false, inference with for-loop without batch padding fastchat_template='vicuna',
run_cfg=dict(num_gpus=2, num_procs=1),
end_str='</s>',
) )
] ]
configs/models/vicuna/hf_vicuna_33b_v13.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM from opencompass.models import HuggingFacewithChatTemplate
models = [ models = [
dict( dict(
type=HuggingFaceCausalLM, type=HuggingFacewithChatTemplate,
abbr='vicuna-33b-v1.3-hf', abbr='vicuna-33b-v1.3-hf',
path="lmsys/vicuna-33b-v1.3", path='lmsys/vicuna-33b-v1.3',
tokenizer_path='lmsys/vicuna-33b-v1.3', max_out_len=1024,
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
use_fast=False,
),
max_out_len=100,
max_seq_len=2048,
batch_size=8, batch_size=8,
model_kwargs=dict(device_map='auto'), run_cfg=dict(num_gpus=2),
batch_padding=False, # if false, inference with for-loop without batch padding fastchat_template='vicuna',
use_fastchat_template=True,
run_cfg=dict(num_gpus=4, num_procs=1)
) )
] ]
configs/models/vicuna/hf_vicuna_7b_v13.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM from opencompass.models import HuggingFacewithChatTemplate
models = [ models = [
dict( dict(
type=HuggingFaceCausalLM, type=HuggingFacewithChatTemplate,
abbr='vicuna-7b-v1.3-hf', abbr='vicuna-7b-v1.3-hf',
path="lmsys/vicuna-7b-v1.3", path='lmsys/vicuna-7b-v1.3',
tokenizer_path='lmsys/vicuna-7b-v1.3', max_out_len=1024,
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
use_fast=False,
),
max_out_len=100,
max_seq_len=2048,
batch_size=8, batch_size=8,
model_kwargs=dict(device_map='auto'), run_cfg=dict(num_gpus=1),
batch_padding=False, # if false, inference with for-loop without batch padding fastchat_template='vicuna',
use_fastchat_template=True,
run_cfg=dict(num_gpus=1, num_procs=1)
) )
] ]
configs/models/vicuna/hf_vicuna_7b_v15.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM from opencompass.models import HuggingFacewithChatTemplate
models = [ models = [
dict( dict(
type=HuggingFaceCausalLM, type=HuggingFacewithChatTemplate,
abbr='vicuna-7b-v1.5-hf', abbr='vicuna-7b-v1.5-hf',
path="lmsys/vicuna-7b-v1.5", path='lmsys/vicuna-7b-v1.5',
tokenizer_path='lmsys/vicuna-7b-v1.5', max_out_len=1024,
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
use_fast=False,
),
max_out_len=100,
max_seq_len=2048,
batch_size=8, batch_size=8,
model_kwargs=dict(device_map='auto'), run_cfg=dict(num_gpus=1),
batch_padding=False, # if false, inference with for-loop without batch padding fastchat_template='vicuna',
use_fastchat_template=True,
run_cfg=dict(num_gpus=1, num_procs=1)
) )
] ]
configs/models/vicuna/hf_vicuna_7b_v15_16k.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM from opencompass.models import HuggingFacewithChatTemplate
_meta_template = dict(
round=[
dict(role="HUMAN", begin='USER: '),
dict(role="BOT", begin=" ASSISTANT:", end='</s>', generate=True),
],
)
models = [ models = [
dict( dict(
type=HuggingFaceCausalLM, type=HuggingFacewithChatTemplate,
abbr='vicuna-7b-v1.5-16k-hf', abbr='vicuna-7b-v1.5-16k-hf',
path="lmsys/vicuna-7b-v1.5-16k", path='lmsys/vicuna-7b-v1.5-16k',
tokenizer_path='lmsys/vicuna-7b-v1.5-16k', max_out_len=1024,
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
use_fast=False,
),
meta_template=_meta_template,
max_out_len=100,
max_seq_len=8192,
batch_size=8, batch_size=8,
model_kwargs=dict(device_map='auto'), run_cfg=dict(num_gpus=1),
batch_padding=False, # if false, inference with for-loop without batch padding fastchat_template='vicuna',
run_cfg=dict(num_gpus=1, num_procs=1),
end_str='</s>',
) )
] ]
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment