Skip to content

GitLab

"vscode:/vscode.git/clone" did not exist on "b8972b3675edbd27a76b342c0fc2add256cd29fc"
Unverified Commit 32b5948f authored by bittersweet1999's avatar bittersweet1999 Committed by GitHub
Browse files

[Fix] add do sample demo for subjective dataset (#873) 



* add do sample demo for subjective dataset

* fix strings

* format

---------
Co-authored-by: default avatarLeymore <zfz-960727@163.com>
parent 7ad11680
Show whitespace changes
Inline Side-by-side
Showing with 431 additions and 345 deletions
configs/eval_subjective_alignbench.py
View file @ 32b5948f
from mmengine.config import read_base from mmengine.config import read_base
with read_base(): with read_base():
from .models.qwen.hf_qwen_7b_chat import models as hf_qwen_7b_chat
from .models.qwen.hf_qwen_14b_chat import models as hf_qwen_14b_chat
from .models.chatglm.hf_chatglm3_6b import models as hf_chatglm3_6b
from .models.baichuan.hf_baichuan2_7b_chat import models as hf_baichuan2_7b
from .models.hf_internlm.hf_internlm_chat_20b import models as hf_internlm_chat_20b
from .models.judge_llm.auto_j.hf_autoj_eng_13b import models as hf_autoj
from .models.judge_llm.judgelm.hf_judgelm_33b_v1 import models as hf_judgelm
from .models.judge_llm.pandalm.hf_pandalm_7b_v1 import models as hf_pandalm
from .datasets.subjective.alignbench.alignbench_judgeby_critiquellm import subjective_datasets from .datasets.subjective.alignbench.alignbench_judgeby_critiquellm import subjective_datasets
datasets = [*subjective_datasets]
from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3 from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3
from opencompass.models.openai_api import OpenAIAllesAPIN from opencompass.models.openai_api import OpenAIAllesAPIN
from opencompass.partitioners import NaivePartitioner, SizePartitioner from opencompass.partitioners import NaivePartitioner, SizePartitioner
...@@ -23,10 +14,42 @@ from opencompass.tasks import OpenICLInferTask ...@@ -23,10 +14,42 @@ from opencompass.tasks import OpenICLInferTask
from opencompass.tasks.subjective_eval import SubjectiveEvalTask from opencompass.tasks.subjective_eval import SubjectiveEvalTask
from opencompass.summarizers import AlignmentBenchSummarizer from opencompass.summarizers import AlignmentBenchSummarizer
api_meta_template = dict(
round=[
dict(role='HUMAN', api_role='HUMAN'),
dict(role='BOT', api_role='BOT', generate=True),
]
)
# -------------Inferen Stage ---------------------------------------- # -------------Inference Stage ----------------------------------------
# For subjective evaluation, we often set do sample for models
models = [
dict(
type=HuggingFaceChatGLM3,
abbr='chatglm3-6b-hf',
path='THUDM/chatglm3-6b',
tokenizer_path='THUDM/chatglm3-6b',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True,
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
),
generation_kwargs=dict(
do_sample=True,
),
meta_template=api_meta_template,
max_out_len=2048,
max_seq_len=4096,
batch_size=1,
run_cfg=dict(num_gpus=1, num_procs=1),
)
]
models = [*hf_chatglm3_6b]#, *hf_chatglm3_6b, *hf_internlm_chat_20b, *hf_qwen_7b_chat, *hf_qwen_14b_chat] datasets = [*subjective_datasets]
infer = dict( infer = dict(
partitioner=dict(type=NaivePartitioner), partitioner=dict(type=NaivePartitioner),
...@@ -35,23 +58,17 @@ infer = dict( ...@@ -35,23 +58,17 @@ infer = dict(
partition='llmeval', partition='llmeval',
quotatype='auto', quotatype='auto',
max_num_workers=256, max_num_workers=256,
task=dict(type=OpenICLInferTask)), task=dict(type=OpenICLInferTask),
),
) )
# -------------Evalation Stage ---------------------------------------- # -------------Evalation Stage ----------------------------------------
## ------------- JudgeLLM Configuration ## ------------- JudgeLLM Configuration
api_meta_template = dict(
round=[
dict(role='HUMAN', api_role='HUMAN'),
dict(role='BOT', api_role='BOT', generate=True),
]
)
judge_model = dict( judge_model = dict(
abbr='GPT4-Turbo', abbr='GPT4-Turbo',
type=OpenAIAllesAPIN, path='gpt-4-1106-preview', type=OpenAIAllesAPIN,
path='gpt-4-1106-preview',
key='xxxx', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well key='xxxx', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
url='xxxx', url='xxxx',
meta_template=api_meta_template, meta_template=api_meta_template,
...@@ -59,27 +76,21 @@ judge_model = dict( ...@@ -59,27 +76,21 @@ judge_model = dict(
max_out_len=2048, max_out_len=2048,
max_seq_len=2048, max_seq_len=2048,
batch_size=8, batch_size=8,
temperature = 0 temperature=0,
) )
## ------------- Evaluation Configuration ## ------------- Evaluation Configuration
eval = dict( eval = dict(
partitioner=dict( partitioner=dict(
type=SubjectiveNaivePartitioner, type=SubjectiveNaivePartitioner, mode='singlescore', models=models
mode='singlescore',
models = models
), ),
runner=dict( runner=dict(
type=LocalRunner, type=LocalRunner,
max_num_workers=2, max_num_workers=2,
task=dict( task=dict(type=SubjectiveEvalTask, judge_cfg=judge_model),
type=SubjectiveEvalTask, ),
judge_cfg=judge_model
)),
) )
summarizer = dict( summarizer = dict(type=AlignmentBenchSummarizer, judge_type='general')
type=AlignmentBenchSummarizer, judge_type = 'general'
)
work_dir = 'outputs/alignment_bench/' work_dir = 'outputs/alignment_bench/'
configs/eval_subjective_alpacaeval.py
View file @ 32b5948f
from mmengine.config import read_base from mmengine.config import read_base
with read_base(): with read_base():
from .models.qwen.hf_qwen_7b_chat import models as hf_qwen_7b_chat
from .models.qwen.hf_qwen_14b_chat import models as hf_qwen_14b_chat
from .models.chatglm.hf_chatglm3_6b import models as hf_chatglm3_6b
from .models.baichuan.hf_baichuan2_7b_chat import models as hf_baichuan2_7b
from .models.hf_internlm.hf_internlm_chat_7b import models as hf_internlm_chat_7b
from .models.hf_internlm.hf_internlm_chat_20b import models as hf_internlm_chat_20b
from .datasets.subjective.alpaca_eval.alpacav1_judgeby_gpt4 import subjective_datasets as alpacav1 from .datasets.subjective.alpaca_eval.alpacav1_judgeby_gpt4 import subjective_datasets as alpacav1
from .datasets.subjective.alpaca_eval.alpacav2_judgeby_gpt4 import subjective_datasets as alpacav2 from .datasets.subjective.alpaca_eval.alpacav2_judgeby_gpt4 import subjective_datasets as alpacav2
datasets = [*alpacav2]
from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3 from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3
from opencompass.models.openai_api import OpenAI, OpenAIAllesAPIN from opencompass.models.openai_api import OpenAI, OpenAIAllesAPIN
from opencompass.partitioners import NaivePartitioner, SizePartitioner from opencompass.partitioners import NaivePartitioner, SizePartitioner
...@@ -22,18 +15,59 @@ from opencompass.tasks import OpenICLInferTask ...@@ -22,18 +15,59 @@ from opencompass.tasks import OpenICLInferTask
from opencompass.tasks.subjective_eval import SubjectiveEvalTask from opencompass.tasks.subjective_eval import SubjectiveEvalTask
from opencompass.summarizers import AlpacaSummarizer from opencompass.summarizers import AlpacaSummarizer
models = [*hf_qwen_7b_chat, *hf_chatglm3_6b]
api_meta_template = dict( api_meta_template = dict(
round=[ round=[
dict(role='HUMAN', api_role='HUMAN'), dict(role='HUMAN', api_role='HUMAN'),
dict(role='BOT', api_role='BOT', generate=True) dict(role='BOT', api_role='BOT', generate=True),
],
reserved_roles=[
dict(role='SYSTEM', api_role='SYSTEM'),
], ],
reserved_roles=[dict(role='SYSTEM', api_role='SYSTEM')],
) )
# -------------Inference Stage ----------------------------------------
# For subjective evaluation, we often set do sample for models
models = [
dict(
type=HuggingFaceChatGLM3,
abbr='chatglm3-6b-hf',
path='THUDM/chatglm3-6b',
tokenizer_path='THUDM/chatglm3-6b',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True,
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
),
generation_kwargs=dict(
do_sample=True,
),
meta_template=api_meta_template,
max_out_len=2048,
max_seq_len=4096,
batch_size=1,
run_cfg=dict(num_gpus=1, num_procs=1),
)
]
datasets = [*alpacav2]
gpt4 = dict(
abbr='gpt4-turbo',
type=OpenAI,
path='gpt-4-1106-preview',
key='', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
meta_template=api_meta_template,
query_per_second=1,
max_out_len=2048,
max_seq_len=4096,
batch_size=4,
retry=20,
temperature=1,
) # Re-inference gpt4's predictions or you can choose to use the pre-commited gpt4's predictions
infer = dict( infer = dict(
partitioner=dict(type=NaivePartitioner), partitioner=dict(type=NaivePartitioner),
runner=dict( runner=dict(
...@@ -41,12 +75,17 @@ infer = dict( ...@@ -41,12 +75,17 @@ infer = dict(
partition='llmeval', partition='llmeval',
quotatype='auto', quotatype='auto',
max_num_workers=256, max_num_workers=256,
task=dict(type=OpenICLInferTask)), task=dict(type=OpenICLInferTask),
),
) )
# -------------Evalation Stage ----------------------------------------
## ------------- JudgeLLM Configuration
judge_model = dict( judge_model = dict(
abbr='GPT4-Turbo', abbr='GPT4-Turbo',
type=OpenAI, path='gpt-4-1106-preview', type=OpenAI,
path='gpt-4-1106-preview',
key='', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well key='', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
meta_template=api_meta_template, meta_template=api_meta_template,
query_per_second=1, query_per_second=1,
...@@ -54,29 +93,22 @@ judge_model = dict( ...@@ -54,29 +93,22 @@ judge_model = dict(
max_seq_len=4096, max_seq_len=4096,
batch_size=2, batch_size=2,
retry=20, retry=20,
temperature = 0 temperature=0,
) )
## ------------- Evaluation Configuration
eval = dict( eval = dict(
partitioner=dict( partitioner=dict(
type=SubjectiveSizePartitioner, type=SubjectiveSizePartitioner, max_task_size=1000, mode='m2n', base_models=[gpt4], compare_models=models
max_task_size=1000,
mode='m2n',
base_models = [*hf_chatglm3_6b],
compare_models = [*hf_qwen_7b_chat]
), ),
runner=dict( runner=dict(
type=SlurmSequentialRunner, type=SlurmSequentialRunner,
partition='llmeval', partition='llmeval',
quotatype='auto', quotatype='auto',
max_num_workers=256, max_num_workers=256,
task=dict( task=dict(type=SubjectiveEvalTask, judge_cfg=judge_model),
type=SubjectiveEvalTask, ),
judge_cfg=judge_model
)),
) )
work_dir = 'outputs/alpaca/' work_dir = 'outputs/alpaca/'
summarizer = dict( summarizer = dict(type=AlpacaSummarizer, judge_type='v2')
type=AlpacaSummarizer, judge_type='v2'
)
\ No newline at end of file
configs/eval_subjective_compassarena.py
View file @ 32b5948f
from os import getenv as gv from os import getenv as gv
from opencompass.models import HuggingFaceCausalLM from opencompass.models import HuggingFaceCausalLM
from mmengine.config import read_base from mmengine.config import read_base
with read_base(): with read_base():
from .models.chatglm.hf_chatglm3_6b_32k import models as chatglm3_6b_32k_model
from .datasets.subjective.compassarena.compassarena_compare import subjective_datasets from .datasets.subjective.compassarena.compassarena_compare import subjective_datasets
from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3, OpenAI from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3, OpenAI
...@@ -16,27 +16,49 @@ from opencompass.tasks import OpenICLInferTask ...@@ -16,27 +16,49 @@ from opencompass.tasks import OpenICLInferTask
from opencompass.tasks.subjective_eval import SubjectiveEvalTask from opencompass.tasks.subjective_eval import SubjectiveEvalTask
from opencompass.summarizers import CompassArenaSummarizer from opencompass.summarizers import CompassArenaSummarizer
infer = dict(
#partitioner=dict(type=NaivePartitioner),
partitioner=dict(type=SizePartitioner, strategy='split', max_task_size=10000),
runner=dict(
type=SlurmSequentialRunner,
partition='llm_dev2',
quotatype='auto',
max_num_workers=256,
task=dict(type=OpenICLInferTask)),
)
api_meta_template = dict( api_meta_template = dict(
round=[ round=[
dict(role='HUMAN', api_role='HUMAN'), dict(role='HUMAN', api_role='HUMAN'),
dict(role='BOT', api_role='BOT', generate=True), dict(role='BOT', api_role='BOT', generate=True),
] ],
reserved_roles=[dict(role='SYSTEM', api_role='SYSTEM')],
) )
# -------------Inference Stage ----------------------------------------
# For subjective evaluation, we often set do sample for models
models = [
dict(
type=HuggingFaceChatGLM3,
abbr='chatglm3-6b-hf',
path='THUDM/chatglm3-6b',
tokenizer_path='THUDM/chatglm3-6b',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True,
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
),
generation_kwargs=dict(
do_sample=True,
),
meta_template=api_meta_template,
max_out_len=2048,
max_seq_len=4096,
batch_size=1,
run_cfg=dict(num_gpus=1, num_procs=1),
)
]
datasets = [*subjective_datasets]
gpt4 = dict( gpt4 = dict(
abbr='gpt4-turbo', abbr='gpt4-turbo',
type=OpenAI, path='gpt-4-1106-preview', type=OpenAI,
path='gpt-4-1106-preview',
key='', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well key='', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
meta_template=api_meta_template, meta_template=api_meta_template,
query_per_second=1, query_per_second=1,
...@@ -44,20 +66,27 @@ gpt4 = dict( ...@@ -44,20 +66,27 @@ gpt4 = dict(
max_seq_len=4096, max_seq_len=4096,
batch_size=4, batch_size=4,
retry=20, retry=20,
temperature = 1 temperature=1,
) ) # Re-inference gpt4's predictions or you can choose to use the pre-commited gpt4's predictions
models = [*chatglm3_6b_32k_model]
datasets = [*subjective_datasets]
work_dir = 'outputs/compass_arena_debug/' infer = dict(
partitioner=dict(type=SizePartitioner, strategy='split', max_task_size=10000),
runner=dict(
type=SlurmSequentialRunner,
partition='llm_dev2',
quotatype='auto',
max_num_workers=256,
task=dict(type=OpenICLInferTask),
),
)
# -------------Inferen Stage ---------------------------------------- # -------------Evalation Stage ----------------------------------------
## ------------- JudgeLLM Configuration
judge_model = dict( judge_model = dict(
abbr='GPT4-Turbo', abbr='GPT4-Turbo',
type=OpenAI, path='gpt-4-1106-preview', type=OpenAI,
path='gpt-4-1106-preview',
key='', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well key='', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
meta_template=api_meta_template, meta_template=api_meta_template,
query_per_second=1, query_per_second=1,
...@@ -65,7 +94,7 @@ judge_model = dict( ...@@ -65,7 +94,7 @@ judge_model = dict(
max_seq_len=4096, max_seq_len=4096,
batch_size=2, batch_size=2,
retry=20, retry=20,
temperature = 0 temperature=0,
) )
## ------------- Evaluation Configuration ## ------------- Evaluation Configuration
...@@ -75,22 +104,18 @@ eval = dict( ...@@ -75,22 +104,18 @@ eval = dict(
strategy='split', strategy='split',
max_task_size=10000, max_task_size=10000,
mode='m2n', mode='m2n',
base_models = [gpt4], base_models=[gpt4],
compare_models = [*chatglm3_6b_32k_model] compare_models=models,
), ),
runner=dict( runner=dict(
type=SlurmSequentialRunner, type=SlurmSequentialRunner,
partition='llm_dev2', partition='llm_dev2',
quotatype='auto', quotatype='auto',
max_num_workers=32, max_num_workers=32,
task=dict( task=dict(type=SubjectiveEvalTask, judge_cfg=judge_model),
type=SubjectiveEvalTask, ),
judge_cfg=judge_model
)),
) )
work_dir = 'outputs/compass_arena_debug/'
summarizer = dict( summarizer = dict(type=CompassArenaSummarizer, summary_type='half_add')
type=CompassArenaSummarizer,
summary_type='half_add'
)
\ No newline at end of file
configs/eval_subjective_corev2.py
View file @ 32b5948f
from mmengine.config import read_base from mmengine.config import read_base
with read_base(): with read_base():
from .models.qwen.hf_qwen_7b_chat import models as hf_qwen_7b_chat
from .models.qwen.hf_qwen_14b_chat import models as hf_qwen_14b_chat
from .models.chatglm.hf_chatglm3_6b import models as hf_chatglm3_6b
from .models.baichuan.hf_baichuan2_7b_chat import models as hf_baichuan2_7b
from .models.hf_internlm.hf_internlm2_chat_7b import models as internlm2_7b
from .models.hf_internlm.hf_internlm2_chat_20b import models as internlm2_20b
from .datasets.subjective.subjective_cmp.subjective_corev2 import subjective_datasets from .datasets.subjective.subjective_cmp.subjective_corev2 import subjective_datasets
datasets = [*subjective_datasets] from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3, OpenAI
from opencompass.models import HuggingFaceCausalLM, HuggingFace, OpenAI
from opencompass.partitioners import NaivePartitioner, SizePartitioner from opencompass.partitioners import NaivePartitioner, SizePartitioner
from opencompass.partitioners.sub_naive import SubjectiveNaivePartitioner from opencompass.partitioners.sub_naive import SubjectiveNaivePartitioner
from opencompass.partitioners.sub_size import SubjectiveSizePartitioner from opencompass.partitioners.sub_size import SubjectiveSizePartitioner
...@@ -19,80 +12,104 @@ from opencompass.runners import SlurmSequentialRunner ...@@ -19,80 +12,104 @@ from opencompass.runners import SlurmSequentialRunner
from opencompass.tasks import OpenICLInferTask from opencompass.tasks import OpenICLInferTask
from opencompass.tasks.subjective_eval import SubjectiveEvalTask from opencompass.tasks.subjective_eval import SubjectiveEvalTask
from opencompass.summarizers import Corev2Summarizer from opencompass.summarizers import Corev2Summarizer
models = [*internlm2_7b, *internlm2_20b]
api_meta_template = dict( api_meta_template = dict(
round=[ round=[
dict(role='HUMAN', api_role='HUMAN'), dict(role='HUMAN', api_role='HUMAN'),
dict(role='BOT', api_role='BOT', generate=True) dict(role='BOT', api_role='BOT', generate=True),
], ],
reserved_roles=[ reserved_roles=[
dict(role='SYSTEM', api_role='SYSTEM'), dict(role='SYSTEM', api_role='SYSTEM'),
], ],
) )
infer = dict( # -------------Inference Stage ----------------------------------------
partitioner=dict(type=SizePartitioner, max_task_size=500),
runner=dict(
type=SlurmSequentialRunner,
partition='llm_dev2',
quotatype='auto',
max_num_workers=256,
task=dict(type=OpenICLInferTask)),
)
_meta_template = dict(
round=[
dict(role="HUMAN", begin='\n<|im_start|>user\n', end='<|im_end|>'),
dict(role="BOT", begin="\n<|im_start|>assistant\n", end='<|im_end|>', generate=True),
],
)
judge_model = dict( # For subjective evaluation, we often set do sample for models
type=HuggingFaceCausalLM, models = [
abbr='qwen-7b-chat-hf', dict(
path="Qwen/Qwen-7B-Chat", type=HuggingFaceChatGLM3,
tokenizer_path='Qwen/Qwen-7B-Chat', abbr='chatglm3-6b-hf',
path='THUDM/chatglm3-6b',
tokenizer_path='THUDM/chatglm3-6b',
model_kwargs=dict( model_kwargs=dict(
device_map='auto', device_map='auto',
trust_remote_code=True trust_remote_code=True,
), ),
tokenizer_kwargs=dict( tokenizer_kwargs=dict(
padding_side='left', padding_side='left',
truncation_side='left', truncation_side='left',
trust_remote_code=True, trust_remote_code=True,
use_fast=False,), ),
pad_token_id=151643, generation_kwargs=dict(
do_sample=True,
),
meta_template=api_meta_template,
max_out_len=2048, max_out_len=2048,
max_seq_len=2048, max_seq_len=4096,
batch_size=8, batch_size=1,
meta_template=_meta_template,
run_cfg=dict(num_gpus=1, num_procs=1), run_cfg=dict(num_gpus=1, num_procs=1),
) )
]
datasets = [*subjective_datasets]
gpt4 = dict(
abbr='gpt4-turbo',
type=OpenAI,
path='gpt-4-1106-preview',
key='', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
meta_template=api_meta_template,
query_per_second=1,
max_out_len=2048,
max_seq_len=4096,
batch_size=4,
retry=20,
temperature=1,
) # Re-inference gpt4's predictions or you can choose to use the pre-commited gpt4's predictions
infer = dict(
partitioner=dict(type=SizePartitioner, max_task_size=500),
runner=dict(
type=SlurmSequentialRunner,
partition='llm_dev2',
quotatype='auto',
max_num_workers=256,
task=dict(type=OpenICLInferTask),
),
)
# -------------Evalation Stage ----------------------------------------
## ------------- JudgeLLM Configuration
judge_model = dict(
abbr='GPT4-Turbo',
type=OpenAI,
path='gpt-4-1106-preview',
key='', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
meta_template=api_meta_template,
query_per_second=1,
max_out_len=1024,
max_seq_len=4096,
batch_size=2,
retry=20,
temperature=0,
)
## ------------- Evaluation Configuration
eval = dict( eval = dict(
partitioner=dict( partitioner=dict(
type=SubjectiveSizePartitioner, type=SubjectiveSizePartitioner, mode='m2n', max_task_size=500, base_models=[gpt4], compare_models=models
mode='m2n',
max_task_size=500,
base_models = [*internlm2_7b],
compare_models = [*internlm2_20b]
), ),
runner=dict( runner=dict(
type=SlurmSequentialRunner, type=SlurmSequentialRunner,
partition='llm_dev2', partition='llm_dev2',
quotatype='auto', quotatype='auto',
max_num_workers=256, max_num_workers=256,
task=dict( task=dict(type=SubjectiveEvalTask, judge_cfg=judge_model),
type=SubjectiveEvalTask, ),
judge_cfg=judge_model
)),
) )
work_dir = 'outputs/corev2/'
summarizer = dict( summarizer = dict(type=Corev2Summarizer, match_method='smart')
type=Corev2Summarizer,
match_method='smart', work_dir = 'outputs/corev2/'
)
\ No newline at end of file
configs/eval_subjective_creationbench.py
View file @ 32b5948f
from mmengine.config import read_base from mmengine.config import read_base
with read_base(): with read_base():
from .models.qwen.hf_qwen_7b_chat import models as hf_qwen_7b_chat
from .models.qwen.hf_qwen_14b_chat import models as hf_qwen_14b_chat
from .models.chatglm.hf_chatglm3_6b import models as hf_chatglm3_6b
from .models.baichuan.hf_baichuan2_7b_chat import models as hf_baichuan2_7b
from .models.hf_internlm.hf_internlm_chat_20b import models as hf_internlm_chat_20b
from .models.judge_llm.auto_j.hf_autoj_eng_13b import models as hf_autoj
from .models.judge_llm.judgelm.hf_judgelm_33b_v1 import models as hf_judgelm
from .models.judge_llm.pandalm.hf_pandalm_7b_v1 import models as hf_pandalm
from .datasets.subjective.creationbench.creationbench_judgeby_gpt4_withref import subjective_datasets from .datasets.subjective.creationbench.creationbench_judgeby_gpt4_withref import subjective_datasets
datasets = [*subjective_datasets]
from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3 from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3
from opencompass.models.openai_api import OpenAIAllesAPIN from opencompass.models.openai_api import OpenAIAllesAPIN
from opencompass.partitioners import NaivePartitioner, SizePartitioner from opencompass.partitioners import NaivePartitioner, SizePartitioner
...@@ -23,10 +14,42 @@ from opencompass.tasks import OpenICLInferTask ...@@ -23,10 +14,42 @@ from opencompass.tasks import OpenICLInferTask
from opencompass.tasks.subjective_eval import SubjectiveEvalTask from opencompass.tasks.subjective_eval import SubjectiveEvalTask
from opencompass.summarizers import CreationBenchSummarizer from opencompass.summarizers import CreationBenchSummarizer
api_meta_template = dict(
round=[
dict(role='HUMAN', api_role='HUMAN'),
dict(role='BOT', api_role='BOT', generate=True),
]
)
# -------------Inferen Stage ---------------------------------------- # -------------Inference Stage ----------------------------------------
# For subjective evaluation, we often set do sample for models
models = [
dict(
type=HuggingFaceChatGLM3,
abbr='chatglm3-6b-hf',
path='THUDM/chatglm3-6b',
tokenizer_path='THUDM/chatglm3-6b',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True,
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
),
generation_kwargs=dict(
do_sample=True,
),
meta_template=api_meta_template,
max_out_len=2048,
max_seq_len=4096,
batch_size=1,
run_cfg=dict(num_gpus=1, num_procs=1),
)
]
models = [*hf_chatglm3_6b]#, *hf_chatglm3_6b, *hf_internlm_chat_20b, *hf_qwen_7b_chat, *hf_qwen_14b_chat] datasets = [*subjective_datasets]
infer = dict( infer = dict(
partitioner=dict(type=NaivePartitioner), partitioner=dict(type=NaivePartitioner),
...@@ -35,23 +58,17 @@ infer = dict( ...@@ -35,23 +58,17 @@ infer = dict(
partition='llmeval', partition='llmeval',
quotatype='auto', quotatype='auto',
max_num_workers=256, max_num_workers=256,
task=dict(type=OpenICLInferTask)), task=dict(type=OpenICLInferTask),
),
) )
# -------------Evalation Stage ---------------------------------------- # -------------Evalation Stage ----------------------------------------
## ------------- JudgeLLM Configuration ## ------------- JudgeLLM Configuration
api_meta_template = dict(
round=[
dict(role='HUMAN', api_role='HUMAN'),
dict(role='BOT', api_role='BOT', generate=True),
]
)
judge_model = dict( judge_model = dict(
abbr='GPT4-Turbo', abbr='GPT4-Turbo',
type=OpenAIAllesAPIN, path='gpt-4-1106-preview', type=OpenAIAllesAPIN,
path='gpt-4-1106-preview',
key='xxxx', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well key='xxxx', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
url='xxxx', url='xxxx',
meta_template=api_meta_template, meta_template=api_meta_template,
...@@ -59,27 +76,15 @@ judge_model = dict( ...@@ -59,27 +76,15 @@ judge_model = dict(
max_out_len=2048, max_out_len=2048,
max_seq_len=2048, max_seq_len=2048,
batch_size=8, batch_size=8,
temperature = 0 temperature=0,
) )
## ------------- Evaluation Configuration ## ------------- Evaluation Configuration
eval = dict( eval = dict(
partitioner=dict( partitioner=dict(type=SubjectiveNaivePartitioner, mode='singlescore', models=models),
type=SubjectiveNaivePartitioner, runner=dict(type=LocalRunner, max_num_workers=2, task=dict(type=SubjectiveEvalTask, judge_cfg=judge_model)),
mode='singlescore',
models = models
),
runner=dict(
type=LocalRunner,
max_num_workers=2,
task=dict(
type=SubjectiveEvalTask,
judge_cfg=judge_model
)),
) )
summarizer = dict( summarizer = dict(type=CreationBenchSummarizer, judge_type='general')
type=CreationBenchSummarizer, judge_type = 'general'
)
work_dir = 'outputs/creationbench/' work_dir = 'outputs/creationbench/'
configs/eval_subjective_judge_pandalm.py
View file @ 32b5948f
from mmengine.config import read_base from mmengine.config import read_base
with read_base(): with read_base():
from .models.qwen.hf_qwen_7b_chat import models as hf_qwen_7b_chat
from .models.qwen.hf_qwen_14b_chat import models as hf_qwen_14b_chat
from .models.chatglm.hf_chatglm3_6b import models as hf_chatglm3_6b
from .models.baichuan.hf_baichuan2_7b_chat import models as hf_baichuan2_7b
from .models.hf_internlm.hf_internlm_chat_20b import models as hf_internlm_chat_20b
from .datasets.subjective.alignbench.alignbench_judgeby_critiquellm import subjective_datasets from .datasets.subjective.alignbench.alignbench_judgeby_critiquellm import subjective_datasets
datasets = [*subjective_datasets]
from opencompass.models import HuggingFaceCausalLM, HuggingFace, OpenAIAllesAPIN, HuggingFaceChatGLM3 from opencompass.models import HuggingFaceCausalLM, HuggingFace, OpenAIAllesAPIN, HuggingFaceChatGLM3
from opencompass.partitioners import NaivePartitioner from opencompass.partitioners import NaivePartitioner
from opencompass.partitioners.sub_naive import SubjectiveNaivePartitioner from opencompass.partitioners.sub_naive import SubjectiveNaivePartitioner
...@@ -18,10 +12,42 @@ from opencompass.tasks import OpenICLInferTask ...@@ -18,10 +12,42 @@ from opencompass.tasks import OpenICLInferTask
from opencompass.tasks.subjective_eval import SubjectiveEvalTask from opencompass.tasks.subjective_eval import SubjectiveEvalTask
from opencompass.summarizers import AlignmentBenchSummarizer from opencompass.summarizers import AlignmentBenchSummarizer
api_meta_template = dict(
round=[
dict(role='HUMAN', api_role='HUMAN'),
dict(role='BOT', api_role='BOT', generate=True),
]
)
# -------------Inferen Stage ---------------------------------------- # -------------Inference Stage ----------------------------------------
# For subjective evaluation, we often set do sample for models
models = [
dict(
type=HuggingFaceChatGLM3,
abbr='chatglm3-6b-hf',
path='THUDM/chatglm3-6b',
tokenizer_path='THUDM/chatglm3-6b',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True,
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
),
generation_kwargs=dict(
do_sample=True,
),
meta_template=api_meta_template,
max_out_len=2048,
max_seq_len=4096,
batch_size=1,
run_cfg=dict(num_gpus=1, num_procs=1),
)
]
models = [*hf_baichuan2_7b]#, *hf_chatglm3_6b, *hf_internlm_chat_20b, *hf_qwen_7b_chat, *hf_qwen_14b_chat] datasets = [*subjective_datasets]
infer = dict( infer = dict(
partitioner=dict(type=NaivePartitioner), partitioner=dict(type=NaivePartitioner),
...@@ -30,55 +56,37 @@ infer = dict( ...@@ -30,55 +56,37 @@ infer = dict(
partition='llmeval', partition='llmeval',
quotatype='auto', quotatype='auto',
max_num_workers=256, max_num_workers=256,
task=dict(type=OpenICLInferTask)), task=dict(type=OpenICLInferTask),
),
) )
# -------------Evalation Stage ---------------------------------------- # -------------Evalation Stage ----------------------------------------
## ------------- JudgeLLM Configuration ## ------------- JudgeLLM Configuration
api_meta_template = dict(
round=[
dict(role='HUMAN', api_role='HUMAN'),
dict(role='BOT', api_role='BOT', generate=True),
]
)
judge_model = dict( judge_model = dict(
type=HuggingFaceCausalLM, type=HuggingFaceCausalLM,
abbr='pandalm-7b-v1-hf', abbr='pandalm-7b-v1-hf',
path="WeOpenML/PandaLM-7B-v1", path='WeOpenML/PandaLM-7B-v1',
tokenizer_path='WeOpenML/PandaLM-7B-v1', tokenizer_path='WeOpenML/PandaLM-7B-v1',
tokenizer_kwargs=dict(padding_side='left', tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left', truncation_side='left',
trust_remote_code=True, trust_remote_code=True,
use_fast=False,), use_fast=False,
),
max_out_len=512, max_out_len=512,
max_seq_len=2048, max_seq_len=2048,
batch_size=8, batch_size=8,
model_kwargs=dict(device_map='auto', trust_remote_code=True), model_kwargs=dict(device_map='auto', trust_remote_code=True),
run_cfg=dict(num_gpus=1, num_procs=1), run_cfg=dict(num_gpus=1, num_procs=1),
) )
## ------------- Evaluation Configuration ## ------------- Evaluation Configuration
eval = dict( eval = dict(
partitioner=dict( partitioner=dict(type=SubjectiveNaivePartitioner, mode='singlescore', models=models),
type=SubjectiveNaivePartitioner, runner=dict(type=LocalRunner, max_num_workers=2, task=dict(type=SubjectiveEvalTask, judge_cfg=judge_model)),
mode='singlescore',
models = [*hf_baichuan2_7b]
),
runner=dict(
type=LocalRunner,
max_num_workers=2,
task=dict(
type=SubjectiveEvalTask,
judge_cfg=judge_model
)),
) )
summarizer = dict( summarizer = dict(type=AlignmentBenchSummarizer)
type=AlignmentBenchSummarizer,
)
work_dir = 'outputs/pandalm' work_dir = 'outputs/pandalm'
configs/eval_subjective_mtbench.py
View file @ 32b5948f
from mmengine.config import read_base from mmengine.config import read_base
with read_base(): with read_base():
from .models.qwen.hf_qwen_7b_chat import models as hf_qwen_7b_chat
from .models.qwen.hf_qwen_14b_chat import models as hf_qwen_14b_chat
from .models.chatglm.hf_chatglm3_6b import models as hf_chatglm3_6b
from .models.baichuan.hf_baichuan2_7b_chat import models as hf_baichuan2_7b
from .models.hf_internlm.hf_internlm_chat_20b import models as hf_internlm_chat_20b
from .models.judge_llm.auto_j.hf_autoj_eng_13b import models as hf_autoj
from .models.judge_llm.judgelm.hf_judgelm_33b_v1 import models as hf_judgelm
from .models.judge_llm.pandalm.hf_pandalm_7b_v1 import models as hf_pandalm
from .datasets.subjective.multiround.mtbench_single_judge import subjective_datasets from .datasets.subjective.multiround.mtbench_single_judge import subjective_datasets
#from .datasets.subjective.multiround.mtbench_pair_judge import subjective_datasets
datasets = [*subjective_datasets] # from .datasets.subjective.multiround.mtbench_pair_judge import subjective_datasets
from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3 from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3
from opencompass.models.openai_api import OpenAIAllesAPIN from opencompass.models.openai_api import OpenAIAllesAPIN
...@@ -24,10 +16,43 @@ from opencompass.tasks import OpenICLInferTask ...@@ -24,10 +16,43 @@ from opencompass.tasks import OpenICLInferTask
from opencompass.tasks.subjective_eval import SubjectiveEvalTask from opencompass.tasks.subjective_eval import SubjectiveEvalTask
from opencompass.summarizers import MTBenchSummarizer from opencompass.summarizers import MTBenchSummarizer
api_meta_template = dict(
round=[
dict(role='HUMAN', api_role='HUMAN'),
dict(role='BOT', api_role='BOT', generate=True),
]
)
# -------------Inference Stage ----------------------------------------
# For subjective evaluation, we often set do sample for models
models = [
dict(
type=HuggingFaceChatGLM3,
abbr='chatglm3-6b-hf',
path='THUDM/chatglm3-6b',
tokenizer_path='THUDM/chatglm3-6b',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True,
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
),
generation_kwargs=dict(
do_sample=True,
),
meta_template=api_meta_template,
max_out_len=2048,
max_seq_len=4096,
batch_size=1,
run_cfg=dict(num_gpus=1, num_procs=1),
)
]
# -------------Inferen Stage ---------------------------------------- datasets = [*subjective_datasets]
models = [*hf_chatglm3_6b, *hf_qwen_7b_chat]
infer = dict( infer = dict(
partitioner=dict(type=SizePartitioner, max_task_size=100), partitioner=dict(type=SizePartitioner, max_task_size=100),
runner=dict( runner=dict(
...@@ -35,82 +60,45 @@ infer = dict( ...@@ -35,82 +60,45 @@ infer = dict(
partition='llmeval', partition='llmeval',
quotatype='auto', quotatype='auto',
max_num_workers=256, max_num_workers=256,
task=dict(type=OpenICLInferTask)), task=dict(type=OpenICLInferTask),
),
) )
# -------------Evalation Stage ---------------------------------------- # -------------Evalation Stage ----------------------------------------
## ------------- JudgeLLM Configuration ## ------------- JudgeLLM Configuration
api_meta_template = dict(
round=[
dict(role='HUMAN', api_role='HUMAN'),
dict(role='BOT', api_role='BOT', generate=True),
]
)
judge_model = dict( judge_model = dict(
abbr='GPT4-Turbo', abbr='GPT4-Turbo',
type=OpenAIAllesAPIN, path='gpt-4-1106-preview', type=OpenAIAllesAPIN,
path='gpt-4-0613',
key='xxxx', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well key='xxxx', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
url='xxxx', url='xxxx',
meta_template=api_meta_template, meta_template=api_meta_template,
query_per_second=16, query_per_second=16,
max_out_len=2048, max_out_len=1024,
max_seq_len=2048, max_seq_len=2048,
batch_size=8, batch_size=8,
temperature = 0 temperature=0,
) )
## ------------- Evaluation Configuration ## ------------- Evaluation Configuration
''' # ## pair evaluation
## pair evaluation # eval = dict(
eval = dict( # partitioner=dict(
partitioner=dict( # type=SubjectiveSizePartitioner, max_task_size=100, mode='m2n', base_models=[gpt4], compare_models=models
type=SubjectiveSizePartitioner, # ),
max_task_size=100, # runner=dict(type=LocalRunner, max_num_workers=32, task=dict(type=SubjectiveEvalTask, judge_cfg=judge_model)),
mode='m2n', # )
base_models = [*hf_chatglm3_6b, ],
compare_models = models
),
runner=dict(
type=SlurmSequentialRunner,
partition='llmeval',
quotatype='auto',
max_num_workers=32,
task=dict(
type=SubjectiveEvalTask,
judge_cfg=judge_model
)),
)
summarizer = dict( # summarizer = dict(type=MTBenchSummarizer, judge_type='pair')
type=MTBenchSummarizer, judge_type='pair'
)
'''
## single evaluation ## single evaluation
eval = dict( eval = dict(
partitioner=dict( partitioner=dict(type=SubjectiveSizePartitioner, max_task_size=100, mode='singlescore', models=models),
type=SubjectiveSizePartitioner, runner=dict(type=LocalRunner, max_num_workers=32, task=dict(type=SubjectiveEvalTask, judge_cfg=judge_model)),
max_task_size=100,
mode='singlescore',
models = models
),
runner=dict(
type=SlurmSequentialRunner,
partition='llmeval',
quotatype='auto',
max_num_workers=32,
task=dict(
type=SubjectiveEvalTask,
judge_cfg=judge_model
)),
) )
summarizer = dict( summarizer = dict(type=MTBenchSummarizer, judge_type='single')
type=MTBenchSummarizer, judge_type='single'
)
work_dir = 'outputs/mtbench/' work_dir = 'outputs/mtbench/'
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment