Skip to content

GitLab

Unverified Commit 7505b3ca authored by Fengzhe Zhou's avatar Fengzhe Zhou Committed by GitHub
Browse files

[Feature] Add huggingface apply_chat_template (#1098) 

* add TheoremQA with 5-shot

* add huggingface_above_v4_33 classes

* use num_worker partitioner in cli

* update theoremqa

* update TheoremQA

* add TheoremQA

* rename theoremqa -> TheoremQA

* update TheoremQA output path

* rewrite many model configs

* update huggingface

* further update

* refine configs

* update configs

* update configs

* add configs/eval_llama3_instruct.py

* add summarizer multi faceted

* update bbh datasets

* update configs/models/hf_llama/lmdeploy_llama3_8b_instruct.py

* rename class

* update readme

* update hf above v4.33
parent 6c711cb2
Hide whitespace changes
Inline Side-by-side
Showing with 99 additions and 378 deletions
configs/models/qwen/hf_qwen1_5_7b.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM
from opencompass.models import HuggingFaceBaseModel
models = [
dict(
type=HuggingFaceCausalLM,
type=HuggingFaceBaseModel,
abbr='qwen1.5-7b-hf',
path="Qwen/Qwen1.5-7B",
tokenizer_path='Qwen/Qwen1.5-7B',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
use_fast=False,
),
pad_token_id=151645,
max_out_len=100,
max_seq_len=2048,
path='Qwen/Qwen1.5-7B',
max_out_len=1024,
batch_size=8,
run_cfg=dict(num_gpus=1, num_procs=1),
run_cfg=dict(num_gpus=1),
)
]
configs/models/qwen/hf_qwen1_5_7b_chat.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM
_meta_template = dict(
round=[
dict(role="HUMAN", begin='<|im_start|>user\n', end='<|im_end|>\n'),
dict(role="BOT", begin="<|im_start|>assistant\n", end='<|im_end|>\n', generate=True),
],
)
from opencompass.models import HuggingFacewithChatTemplate
models = [
dict(
type=HuggingFaceCausalLM,
type=HuggingFacewithChatTemplate,
abbr='qwen1.5-7b-chat-hf',
path="Qwen/Qwen1.5-7B-Chat",
model_kwargs=dict(
device_map='auto',
trust_remote_code=True
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
use_fast=False,
),
meta_template=_meta_template,
max_out_len=100,
max_seq_len=2048,
path='Qwen/Qwen1.5-7B-Chat',
max_out_len=1024,
batch_size=8,
run_cfg=dict(num_gpus=4, num_procs=1),
end_str='<|im_end|>',
batch_padding=True,
run_cfg=dict(num_gpus=1),
)
]
configs/models/qwen/hf_qwen_14b.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM
from opencompass.models import HuggingFaceBaseModel
models = [
dict(
type=HuggingFaceCausalLM,
type=HuggingFaceBaseModel,
abbr='qwen-14b-hf',
path="Qwen/Qwen-14B",
tokenizer_path='Qwen/Qwen-14B',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True,
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
use_fast=False,
),
pad_token_id=151643,
min_out_len=1,
max_out_len=100,
max_seq_len=2048,
path='Qwen/Qwen-14B',
max_out_len=1024,
batch_size=8,
run_cfg=dict(num_gpus=1, num_procs=1),
run_cfg=dict(num_gpus=1),
)
]
configs/models/qwen/hf_qwen_14b_chat.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM
_meta_template = dict(
round=[
dict(role="HUMAN", begin='\n<|im_start|>user\n', end='<|im_end|>'),
dict(role="BOT", begin="\n<|im_start|>assistant\n", end='<|im_end|>', generate=True),
],
)
from opencompass.models import HuggingFacewithChatTemplate
models = [
dict(
type=HuggingFaceCausalLM,
type=HuggingFacewithChatTemplate,
abbr='qwen-14b-chat-hf',
path="Qwen/Qwen-14B-Chat",
tokenizer_path='Qwen/Qwen-14B-Chat',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
use_fast=False,
),
pad_token_id=151643,
max_out_len=100,
max_seq_len=2048,
path='Qwen/Qwen-14B-Chat',
max_out_len=1024,
batch_size=8,
meta_template=_meta_template,
run_cfg=dict(num_gpus=1, num_procs=1),
end_str='<|im_end|>',
run_cfg=dict(num_gpus=1),
)
]
\ No newline at end of file
]
configs/models/qwen/hf_qwen_1_8b.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM
from opencompass.models import HuggingFaceBaseModel
models = [
dict(
type=HuggingFaceCausalLM,
type=HuggingFaceBaseModel,
abbr='qwen-1.8b-hf',
path="Qwen/Qwen-1_8B",
tokenizer_path='Qwen/Qwen-1_8B',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True,
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
use_fast=False,
),
pad_token_id=151643,
min_out_len=1,
max_out_len=100,
max_seq_len=2048,
path='Qwen/Qwen-1_8B',
max_out_len=1024,
batch_size=8,
run_cfg=dict(num_gpus=1, num_procs=1),
run_cfg=dict(num_gpus=1),
)
]
configs/models/qwen/hf_qwen_1_8b_chat.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM
_meta_template = dict(
round=[
dict(role="HUMAN", begin='\n<|im_start|>user\n', end='<|im_end|>'),
dict(role="BOT", begin="\n<|im_start|>assistant\n", end='<|im_end|>', generate=True),
],
eos_token_id=151645,
)
from opencompass.models import HuggingFacewithChatTemplate
models = [
dict(
type=HuggingFaceCausalLM,
type=HuggingFacewithChatTemplate,
abbr='qwen-1.8b-chat-hf',
path="Qwen/Qwen-1_8B-Chat",
tokenizer_path='Qwen/Qwen-1_8B-Chat',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
use_fast=False,
),
pad_token_id=151643,
max_out_len=100,
max_seq_len=2048,
path='Qwen/Qwen-1_8B-Chat',
max_out_len=1024,
batch_size=8,
meta_template=_meta_template,
run_cfg=dict(num_gpus=1, num_procs=1),
end_str='<|im_end|>',
run_cfg=dict(num_gpus=1),
)
]
configs/models/qwen/hf_qwen_72b.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM
from opencompass.models import HuggingFaceBaseModel
models = [
dict(
type=HuggingFaceCausalLM,
type=HuggingFaceBaseModel,
abbr='qwen-72b-hf',
path="Qwen/Qwen-72B",
tokenizer_path='Qwen/Qwen-72B',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True,
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
use_fast=False,
),
pad_token_id=151643,
min_out_len=1,
max_out_len=100,
max_seq_len=2048,
path='Qwen/Qwen-72B',
max_out_len=1024,
batch_size=8,
run_cfg=dict(num_gpus=4, num_procs=1),
run_cfg=dict(num_gpus=4),
)
]
configs/models/qwen/hf_qwen_72b_chat.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM
_meta_template = dict(
round=[
dict(role="HUMAN", begin='\n<|im_start|>user\n', end='<|im_end|>'),
dict(role="BOT", begin="\n<|im_start|>assistant\n", end='<|im_end|>', generate=True),
],
)
from opencompass.models import HuggingFacewithChatTemplate
models = [
dict(
type=HuggingFaceCausalLM,
type=HuggingFacewithChatTemplate,
abbr='qwen-72b-chat-hf',
path="Qwen/Qwen-72B-Chat",
tokenizer_path='Qwen/Qwen-72B-Chat',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
use_fast=False,),
pad_token_id=151643,
max_out_len=100,
max_seq_len=2048,
path='Qwen/Qwen-72B-Chat',
max_out_len=1024,
batch_size=8,
meta_template=_meta_template,
run_cfg=dict(num_gpus=4, num_procs=1),
end_str='<|im_end|>',
run_cfg=dict(num_gpus=4),
)
]
configs/models/qwen/hf_qwen_7b.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM
from opencompass.models import HuggingFaceBaseModel
models = [
dict(
type=HuggingFaceCausalLM,
type=HuggingFaceBaseModel,
abbr='qwen-7b-hf',
path="Qwen/Qwen-7B",
tokenizer_path='Qwen/Qwen-7B',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True,
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
use_fast=False,
),
pad_token_id=151643,
min_out_len=1,
max_out_len=100,
max_seq_len=2048,
path='Qwen/Qwen-7B',
max_out_len=1024,
batch_size=8,
run_cfg=dict(num_gpus=1, num_procs=1),
run_cfg=dict(num_gpus=1),
)
]
configs/models/qwen/hf_qwen_7b_chat.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM
_meta_template = dict(
round=[
dict(role="HUMAN", begin='\n<|im_start|>user\n', end='<|im_end|>'),
dict(role="BOT", begin="\n<|im_start|>assistant\n", end='<|im_end|>', generate=True),
],
)
from opencompass.models import HuggingFacewithChatTemplate
models = [
dict(
type=HuggingFaceCausalLM,
type=HuggingFacewithChatTemplate,
abbr='qwen-7b-chat-hf',
path="Qwen/Qwen-7B-Chat",
tokenizer_path='Qwen/Qwen-7B-Chat',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
use_fast=False,
),
pad_token_id=151643,
max_out_len=100,
max_seq_len=2048,
path='Qwen/Qwen-7B-Chat',
max_out_len=1024,
batch_size=8,
meta_template=_meta_template,
run_cfg=dict(num_gpus=1, num_procs=1),
end_str='<|im_end|>',
run_cfg=dict(num_gpus=1),
)
]
\ No newline at end of file
]
configs/models/qwen/vllm_qwen1_5_14b_chat.py
View file @ 7505b3ca
......@@ -6,7 +6,6 @@ _meta_template = dict(
dict(role="HUMAN", begin='<|im_start|>user\n', end='<|im_end|>\n'),
dict(role="BOT", begin="<|im_start|>assistant\n", end='<|im_end|>\n', generate=True),
],
eos_token_id=151645,
)
models = [
......
configs/models/qwen/vllm_qwen1_5_72b_chat.py
View file @ 7505b3ca
......@@ -6,7 +6,6 @@ _meta_template = dict(
dict(role="HUMAN", begin='<|im_start|>user\n', end='<|im_end|>\n'),
dict(role="BOT", begin="<|im_start|>assistant\n", end='<|im_end|>\n', generate=True),
],
eos_token_id=151645,
)
models = [
......
configs/models/skywork/hf_skywork_13b.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM
from opencompass.models import HuggingFaceBaseModel
models = [
dict(
type=HuggingFaceCausalLM,
type=HuggingFaceBaseModel,
abbr='skywork-13b-hf',
path="Skywork/Skywork-13B-base",
tokenizer_path='Skywork/Skywork-13B-base',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True,
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
use_fast=False,
),
max_out_len=100,
max_seq_len=2048,
path='Skywork/Skywork-13B-base',
max_out_len=1024,
batch_size=8,
run_cfg=dict(num_gpus=1, num_procs=1),
run_cfg=dict(num_gpus=1),
)
]
configs/models/vicuna/hf_vicuna_13b_v13.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM
from opencompass.models import HuggingFacewithChatTemplate
models = [
dict(
type=HuggingFaceCausalLM,
type=HuggingFacewithChatTemplate,
abbr='vicuna-13b-v1.3-hf',
path="lmsys/vicuna-13b-v1.3",
tokenizer_path='lmsys/vicuna-13b-v1.3',
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
use_fast=False,
),
max_out_len=100,
max_seq_len=2048,
path='lmsys/vicuna-13b-v1.3',
max_out_len=1024,
batch_size=8,
model_kwargs=dict(device_map='auto'),
batch_padding=False, # if false, inference with for-loop without batch padding
use_fastchat_template=True,
run_cfg=dict(num_gpus=2, num_procs=1)
run_cfg=dict(num_gpus=1),
fastchat_template='vicuna',
)
]
configs/models/vicuna/hf_vicuna_13b_v15.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM
from opencompass.models import HuggingFacewithChatTemplate
models = [
dict(
type=HuggingFaceCausalLM,
type=HuggingFacewithChatTemplate,
abbr='vicuna-13b-v1.5-hf',
path="lmsys/vicuna-13b-v1.5",
tokenizer_path='lmsys/vicuna-13b-v1.5',
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
use_fast=False,
),
max_out_len=100,
max_seq_len=2048,
path='lmsys/vicuna-13b-v1.5',
max_out_len=1024,
batch_size=8,
model_kwargs=dict(device_map='auto'),
batch_padding=False, # if false, inference with for-loop without batch padding
use_fastchat_template=True,
run_cfg=dict(num_gpus=1, num_procs=1)
run_cfg=dict(num_gpus=1),
fastchat_template='vicuna',
)
]
configs/models/vicuna/hf_vicuna_13b_v15_16k.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM
_meta_template = dict(
round=[
dict(role="HUMAN", begin='USER: '),
dict(role="BOT", begin=" ASSISTANT:", end='</s>', generate=True),
],
)
from opencompass.models import HuggingFacewithChatTemplate
models = [
dict(
type=HuggingFaceCausalLM,
type=HuggingFacewithChatTemplate,
abbr='vicuna-13b-v1.5-16k-hf',
path="lmsys/vicuna-13b-v1.5-16k",
tokenizer_path='lmsys/vicuna-13b-v1.5-16k',
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
use_fast=False,
),
meta_template=_meta_template,
max_out_len=100,
max_seq_len=8192,
path='lmsys/vicuna-13b-v1.5-16k',
max_out_len=1024,
batch_size=8,
model_kwargs=dict(device_map='auto'),
batch_padding=False, # if false, inference with for-loop without batch padding
run_cfg=dict(num_gpus=2, num_procs=1),
end_str='</s>',
run_cfg=dict(num_gpus=1),
fastchat_template='vicuna',
)
]
configs/models/vicuna/hf_vicuna_33b_v13.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM
from opencompass.models import HuggingFacewithChatTemplate
models = [
dict(
type=HuggingFaceCausalLM,
type=HuggingFacewithChatTemplate,
abbr='vicuna-33b-v1.3-hf',
path="lmsys/vicuna-33b-v1.3",
tokenizer_path='lmsys/vicuna-33b-v1.3',
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
use_fast=False,
),
max_out_len=100,
max_seq_len=2048,
path='lmsys/vicuna-33b-v1.3',
max_out_len=1024,
batch_size=8,
model_kwargs=dict(device_map='auto'),
batch_padding=False, # if false, inference with for-loop without batch padding
use_fastchat_template=True,
run_cfg=dict(num_gpus=4, num_procs=1)
run_cfg=dict(num_gpus=2),
fastchat_template='vicuna',
)
]
configs/models/vicuna/hf_vicuna_7b_v13.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM
from opencompass.models import HuggingFacewithChatTemplate
models = [
dict(
type=HuggingFaceCausalLM,
type=HuggingFacewithChatTemplate,
abbr='vicuna-7b-v1.3-hf',
path="lmsys/vicuna-7b-v1.3",
tokenizer_path='lmsys/vicuna-7b-v1.3',
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
use_fast=False,
),
max_out_len=100,
max_seq_len=2048,
path='lmsys/vicuna-7b-v1.3',
max_out_len=1024,
batch_size=8,
model_kwargs=dict(device_map='auto'),
batch_padding=False, # if false, inference with for-loop without batch padding
use_fastchat_template=True,
run_cfg=dict(num_gpus=1, num_procs=1)
run_cfg=dict(num_gpus=1),
fastchat_template='vicuna',
)
]
configs/models/vicuna/hf_vicuna_7b_v15.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM
from opencompass.models import HuggingFacewithChatTemplate
models = [
dict(
type=HuggingFaceCausalLM,
type=HuggingFacewithChatTemplate,
abbr='vicuna-7b-v1.5-hf',
path="lmsys/vicuna-7b-v1.5",
tokenizer_path='lmsys/vicuna-7b-v1.5',
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
use_fast=False,
),
max_out_len=100,
max_seq_len=2048,
path='lmsys/vicuna-7b-v1.5',
max_out_len=1024,
batch_size=8,
model_kwargs=dict(device_map='auto'),
batch_padding=False, # if false, inference with for-loop without batch padding
use_fastchat_template=True,
run_cfg=dict(num_gpus=1, num_procs=1)
run_cfg=dict(num_gpus=1),
fastchat_template='vicuna',
)
]
configs/models/vicuna/hf_vicuna_7b_v15_16k.py
View file @ 7505b3ca
from opencompass.models import HuggingFaceCausalLM
_meta_template = dict(
round=[
dict(role="HUMAN", begin='USER: '),
dict(role="BOT", begin=" ASSISTANT:", end='</s>', generate=True),
],
)
from opencompass.models import HuggingFacewithChatTemplate
models = [
dict(
type=HuggingFaceCausalLM,
type=HuggingFacewithChatTemplate,
abbr='vicuna-7b-v1.5-16k-hf',
path="lmsys/vicuna-7b-v1.5-16k",
tokenizer_path='lmsys/vicuna-7b-v1.5-16k',
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
use_fast=False,
),
meta_template=_meta_template,
max_out_len=100,
max_seq_len=8192,
path='lmsys/vicuna-7b-v1.5-16k',
max_out_len=1024,
batch_size=8,
model_kwargs=dict(device_map='auto'),
batch_padding=False, # if false, inference with for-loop without batch padding
run_cfg=dict(num_gpus=1, num_procs=1),
end_str='</s>',
run_cfg=dict(num_gpus=1),
fastchat_template='vicuna',
)
]
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment