Skip to content

GitLab

"vscode:/vscode.git/clone" did not exist on "5913609168e6f691185f1b7b214e517d67dd5075"
Switch branch/tag
Find file Normal viewHistoryPermalink
medium.py 2.2 KB
Newer Older
Leymore's avatar
Add release contribution  
Leymore committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88 
from mmengine.config import read_base

with read_base():
    from .groups.agieval import agieval_summary_groups
    from .groups.mmlu import mmlu_summary_groups
    from .groups.ceval import ceval_summary_groups
    from .groups.bbh import bbh_summary_groups
    from .groups.GaokaoBench import GaokaoBench_summary_groups
    from .groups.flores import flores_summary_groups

summarizer = dict(
    dataset_abbrs = [
        '--- Exam ---',
        'agieval',
        'mmlu-all-set',
        "ceval",
        "GaokaoBench",
        "bbh",
        '--- Coding ---',
        'openai_humaneval',
        'mbpp',
        '--- ChineseUniversal ---',
        'C3',
        'CMRC_dev',
        'DRCD_dev',
        'afqmc-dev',
        'cmnli',
        'ocnli',
        'bustm-dev',
        'chid-dev',
        'cluewsc-dev',
        'csl_dev',
        'eprstmt-dev',
        'ocnli_fc-dev',
        'tnews-dev',
        'lcsts',
        '--- Completion ---',
        'lambada',
        'story_cloze',
        '--- EnglishUniversal ---',
        'AX_b',
        'AX_g',
        'BoolQ',
        'CB',
        'COPA',
        'MultiRC',
        'RTE',
        'ReCoRD',
        'WiC',
        'WSC',
        'race-high',
        'race-middle',
        '--- NLG ---',
        'Xsum',
        '--- Reasoning ---',
        'gsm8k',
        'summedits',
        'math',
        'TheoremQA',
        '--- QA ---',
        'hellaswag',
        'ARC-e',
        'ARC-c',
        'commonsense_qa',
        'piqa',
        'siqa',
        'strategyqa',
        'winogrande',
        'openbookqa',
        'openbookqa_fact',
        'nq',
        'triviaqa',
        '--- Translation ---',
        'flores_100_Indo-European-Germanic_English',
        'flores_100_English_Indo-European-Germanic',
        'flores_100_Indo-European-Romance_English',
        'flores_100_English_Indo-European-Romance',
        'flores_100_zho_simpl-eng',
        'flores_100_eng-zho_simpl',
        '--- Security ---',
        'crows_pairs',
    ],
    summary_groups=sum([v for k, v in locals().items() if k.endswith("_summary_groups")], []),
    prompt_db=dict(
        database_path='configs/datasets/log.json',
        config_dir='configs/datasets',
        blacklist='.promptignore'),
)