medium.py 2.73 KB
Newer Older
Leymore's avatar
Leymore committed
1
2
3
4
5
6
7
8
9
from mmengine.config import read_base

with read_base():
    from .groups.agieval import agieval_summary_groups
    from .groups.mmlu import mmlu_summary_groups
    from .groups.ceval import ceval_summary_groups
    from .groups.bbh import bbh_summary_groups
    from .groups.GaokaoBench import GaokaoBench_summary_groups
    from .groups.flores import flores_summary_groups
Leymore's avatar
Leymore committed
10
    from .groups.jigsaw_multilingual import jigsaw_multilingual_summary_groups
Leymore's avatar
Leymore committed
11
12
13

summarizer = dict(
    dataset_abbrs = [
Leymore's avatar
Leymore committed
14
15
        '--------- 考试 Exam ---------', # category
        # 'Mixed', # subcategory
Leymore's avatar
Leymore committed
16
        "ceval",
Leymore's avatar
Leymore committed
17
18
        'agieval',
        'mmlu',
Leymore's avatar
Leymore committed
19
        "GaokaoBench",
Leymore's avatar
Leymore committed
20
21
22
23
24
25
26
27
        'ARC-c',
        '--------- 语言 Language ---------', # category
        # '字词释义', # subcategory
        'WiC',
        'summedits',
        # '成语习语', # subcategory
        'chid-dev',
        # '语义相似度', # subcategory
Leymore's avatar
Leymore committed
28
29
        'afqmc-dev',
        'bustm-dev',
Leymore's avatar
Leymore committed
30
        # '指代消解', # subcategory
Leymore's avatar
Leymore committed
31
        'cluewsc-dev',
Leymore's avatar
Leymore committed
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
        'WSC',
        'winogrande',
        # '翻译', # subcategory
        'flores_100',
        '--------- 知识 Knowledge ---------', # category
        # '知识问答', # subcategory
        'BoolQ',
        'commonsense_qa',
        'nq',
        'triviaqa',
        # '多语种问答', # subcategory
        '--------- 推理 Reasoning ---------', # category
        # '文本蕴含', # subcategory
        'cmnli',
        'ocnli',
Leymore's avatar
Leymore committed
47
48
49
50
51
        'ocnli_fc-dev',
        'AX_b',
        'AX_g',
        'CB',
        'RTE',
Leymore's avatar
Leymore committed
52
53
54
        # '常识推理', # subcategory
        'story_cloze',
        'COPA',
Leymore's avatar
Leymore committed
55
56
57
58
59
        'ReCoRD',
        'hellaswag',
        'piqa',
        'siqa',
        'strategyqa',
Leymore's avatar
Leymore committed
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
        # '数学推理', # subcategory
        'math',
        'gsm8k',
        # '定理应用', # subcategory
        'TheoremQA',
        # '代码', # subcategory
        'openai_humaneval',
        'mbpp',
        # '综合推理', # subcategory
        "bbh",
        '--------- 理解 Understanding ---------', # category
        # '阅读理解', # subcategory
        'C3',
        'CMRC_dev',
        'DRCD_dev',
        'MultiRC',
        'race-middle',
        'race-high',
Leymore's avatar
Leymore committed
78
        'openbookqa_fact',
Leymore's avatar
Leymore committed
79
80
81
82
83
84
85
86
87
88
        # '内容总结', # subcategory
        'csl_dev',
        'lcsts',
        'Xsum',
        # '内容分析', # subcategory
        'eprstmt-dev',
        'lambada',
        'tnews-dev',
        '--------- 安全 Safety ---------', # category
        # '偏见', # subcategory
Leymore's avatar
Leymore committed
89
90
91
92
93
94
95
96
        'crows_pairs',
    ],
    summary_groups=sum([v for k, v in locals().items() if k.endswith("_summary_groups")], []),
    prompt_db=dict(
        database_path='configs/datasets/log.json',
        config_dir='configs/datasets',
        blacklist='.promptignore'),
)