eval_subjective_mtbench.py 2.87 KB
Newer Older
bittersweet1999's avatar
bittersweet1999 committed
1
from mmengine.config import read_base
2

bittersweet1999's avatar
bittersweet1999 committed
3
with read_base():
4
    from .datasets.subjective.multiround.mtbench_single_judge_diff_temp import subjective_datasets
bittersweet1999's avatar
bittersweet1999 committed
5

bittersweet1999's avatar
bittersweet1999 committed
6
from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3, OpenAI
bittersweet1999's avatar
bittersweet1999 committed
7
8
9
10
11
12
13
14
15
16
from opencompass.models.openai_api import OpenAIAllesAPIN
from opencompass.partitioners import NaivePartitioner, SizePartitioner
from opencompass.partitioners.sub_naive import SubjectiveNaivePartitioner
from opencompass.partitioners.sub_size import SubjectiveSizePartitioner
from opencompass.runners import LocalRunner
from opencompass.runners import SlurmSequentialRunner
from opencompass.tasks import OpenICLInferTask
from opencompass.tasks.subjective_eval import SubjectiveEvalTask
from opencompass.summarizers import MTBenchSummarizer

17
18
api_meta_template = dict(
    round=[
19
        dict(role='SYSTEM', api_role='SYSTEM'),
20
21
22
23
        dict(role='HUMAN', api_role='HUMAN'),
        dict(role='BOT', api_role='BOT', generate=True),
    ]
)
bittersweet1999's avatar
bittersweet1999 committed
24

25
26
27
28
29
30
_meta_template = dict(
    round=[
        dict(role="HUMAN", begin='\n<|im_start|>user\n', end='<|im_end|>'),
        dict(role="BOT", begin="\n<|im_start|>assistant\n", end='<|im_end|>', generate=True),
    ],
)
31
32
33
34
# -------------Inference Stage ----------------------------------------
# For subjective evaluation, we often set do sample for models
models = [
    dict(
35
36
37
38
        type=HuggingFaceCausalLM,
        abbr='qwen-7b-chat-hf',
        path="Qwen/Qwen-7B-Chat",
        tokenizer_path='Qwen/Qwen-7B-Chat',
39
40
        model_kwargs=dict(
            device_map='auto',
41
            trust_remote_code=True
42
43
44
45
46
        ),
        tokenizer_kwargs=dict(
            padding_side='left',
            truncation_side='left',
            trust_remote_code=True,
47
            use_fast=False,
48
        ),
49
50
51
52
53
        pad_token_id=151643,
        max_out_len=100,
        max_seq_len=2048,
        batch_size=8,
        meta_template=_meta_template,
54
        run_cfg=dict(num_gpus=1, num_procs=1),
55
        end_str='<|im_end|>',
56
57
58
59
    )
]

datasets = [*subjective_datasets]
bittersweet1999's avatar
bittersweet1999 committed
60
61
62
63

# -------------Evalation Stage ----------------------------------------

## ------------- JudgeLLM Configuration
64
judge_models = [dict(
65
    abbr='GPT4-Turbo',
bittersweet1999's avatar
bittersweet1999 committed
66
    type=OpenAI,
67
    path='gpt-4-0613', # To compare with the official leaderboard, please use gpt4-0613
68
69
70
    key='xxxx',  # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
    meta_template=api_meta_template,
    query_per_second=16,
71
    max_out_len=2048,
72
73
74
    max_seq_len=2048,
    batch_size=8,
    temperature=0,
75
)]
bittersweet1999's avatar
bittersweet1999 committed
76
77
78

## single evaluation
eval = dict(
79
80
    partitioner=dict(type=SubjectiveSizePartitioner, strategy='split', max_task_size=10000, mode='singlescore', models=models, judge_models=judge_models),
    runner=dict(type=LocalRunner, max_num_workers=32, task=dict(type=SubjectiveEvalTask)),
bittersweet1999's avatar
bittersweet1999 committed
81
82
)

83
summarizer = dict(type=MTBenchSummarizer, judge_type='single')
bittersweet1999's avatar
bittersweet1999 committed
84
85

work_dir = 'outputs/mtbench/'