__init__.py 4.07 KB
Newer Older
&'s avatar
& committed
1
2
from pprint import pprint

&'s avatar
& committed
3
4
import sacrebleu

Jason Phang's avatar
Jason Phang committed
5
6
from . import superglue
from . import glue
Leo Gao's avatar
Leo Gao committed
7
from . import arc
Leo Gao's avatar
Leo Gao committed
8
from . import race
Leo Gao's avatar
Leo Gao committed
9
from . import webqs
Leo Gao's avatar
Leo Gao committed
10
from . import anli
11
from . import wsc273
Charles Foster's avatar
Charles Foster committed
12
from . import winogrande
Charles Foster's avatar
Charles Foster committed
13
from . import quac
Charles Foster's avatar
Charles Foster committed
14
from . import hellaswag
Charles Foster's avatar
Charles Foster committed
15
from . import openbookqa
Charles Foster's avatar
Charles Foster committed
16
from . import squad
17
from . import naturalqs
18
from . import sat
19
from . import arithmetic
Leo Gao's avatar
Leo Gao committed
20
from . import lambada
Jon Tow's avatar
Jon Tow committed
21
from . import race 
Leo Gao's avatar
Leo Gao committed
22
from . import piqa
23
from . import triviaqa
jeffhsu3's avatar
jeffhsu3 committed
24
from . import pubmedqa
25
from . import sciq
26
from . import webqs
27
from . import qa4mre
&'s avatar
& committed
28
from . import translation
29
30
from . import headqa
from . import mathqa
Jason Phang's avatar
gpt3  
Jason Phang committed
31

&'s avatar
& committed
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
########################################
# Translation tasks
########################################

# 6 total
gpt3_translation_benchmarks = {
    "wmt14": ['en-fr', 'fr-en'],  # French
    "wmt16": ['en-ro', 'ro-en', 'de-en', 'en-de'],  # German, Romanian
}


# 28 total
selected_translation_benchmarks = {
    **gpt3_translation_benchmarks,
    "wmt20": sacrebleu.get_langpairs_for_testset("wmt20"),
    "iwslt17": ['en-ar', 'ar-en']  # Arabic
}

# 319 total
all_translation_benchmarks = {
    ts: sacrebleu.get_langpairs_for_testset(ts)
    for ts in sacrebleu.get_available_testsets()
}


########################################
# All tasks
########################################


Jason Phang's avatar
Jason Phang committed
62
TASK_REGISTRY = {
Jason Phang's avatar
multirc  
Jason Phang committed
63
    # GLUE
Jason Phang's avatar
Jason Phang committed
64
65
    "cola": glue.CoLA,
    "mnli": glue.MNLI,
Jason Phang's avatar
Jason Phang committed
66
    "mnli_mismatched": glue.MNLIMismatched,
Jason Phang's avatar
Jason Phang committed
67
68
69
70
    "mrpc": glue.MRPC,
    "rte": glue.RTE,
    "qnli": glue.QNLI,
    "qqp": glue.QQP,
71
    #"stsb": glue.STSB, # not implemented yet
Jason Phang's avatar
Jason Phang committed
72
73
    "sst": glue.SST,
    "wnli": glue.WNLI,
Jason Phang's avatar
multirc  
Jason Phang committed
74
    # SuperGLUE
Jason Phang's avatar
Jason Phang committed
75
    "boolq": superglue.BoolQ,
thefazzer's avatar
thefazzer committed
76
    "cb": superglue.CommitmentBank,
Jason Phang's avatar
Jason Phang committed
77
    "copa": superglue.Copa,
Jason Phang's avatar
multirc  
Jason Phang committed
78
    "multirc": superglue.MultiRC,
Jason Phang's avatar
Jason Phang committed
79
    "record": superglue.ReCoRD,
Jason Phang's avatar
Jason Phang committed
80
    "wic": superglue.WordsInContext,
Jason Phang's avatar
wsc  
Jason Phang committed
81
    "wsc": superglue.SGWinogradSchemaChallenge,
82
    
Jason Phang's avatar
multirc  
Jason Phang committed
83
    # Order by benchmark/genre?
84

Leo Gao's avatar
Leo Gao committed
85
    "lambada": lambada.LAMBADA,
Leo Gao's avatar
Leo Gao committed
86
    "piqa": piqa.PiQA,
Leo Gao's avatar
Leo Gao committed
87

88
    # Science related
jeffhsu3's avatar
jeffhsu3 committed
89
    "pubmedqa" : pubmedqa.Pubmed_QA,
90
    "sciq" : sciq.SciQ,
91
92
93
94
    #"qa4mre" : qa4mre.QA4MRE,
    "qa4mre_2011" : qa4mre.QA4MRE_2011,
    "qa4mre_2012" : qa4mre.QA4MRE_2012,
    "qa4mre_2013" : qa4mre.QA4MRE_2013,
jeffhsu3's avatar
jeffhsu3 committed
95

Leo Gao's avatar
Leo Gao committed
96
    #"triviaqa": triviaqa.TriviaQA,
Jonathan Tow's avatar
Jonathan Tow committed
97
98
    "arc_easy": arc.ARCEasy,
    "arc_challenge": arc.ARCChallenge,
99
    # "quac": quac.QuAC, # not implemented yet
100
    "hellaswag": hellaswag.HellaSwag, # not implemented yet
Jon Tow's avatar
Jon Tow committed
101
    "openbookqa": openbookqa.OpenBookQA,
102
103
    # "sat": sat.SATAnalogies, # not implemented yet
    # "squad": squad.SQuAD, # not implemented yet
Jon Tow's avatar
Jon Tow committed
104
    "race": race.RACE,
105
    # "naturalqs": naturalqs.NaturalQs, # not implemented yet
106
107
    "headqa": headqa.HeadQA,
    "mathqa": mathqa.MathQA,
108
    "webqs": webqs.WebQs,
109
    "wsc273": wsc273.WinogradSchemaChallenge273,
110
    "winogrande": winogrande.Winogrande,
Jonathan Tow's avatar
Jonathan Tow committed
111
112
113
    "anli_r1": anli.ANLIRound1,
    "anli_r2": anli.ANLIRound2,
    "anli_r3": anli.ANLIRound3,
114
115
116
117
118
119
120
121
122
123
124
    # arithmetic
    "arithmetic_2da": arithmetic.Arithmetic2DPlus,
    "arithmetic_2ds": arithmetic.Arithmetic2DMinus,
    "arithmetic_3da": arithmetic.Arithmetic3DPlus,
    "arithmetic_3ds": arithmetic.Arithmetic3DMinus,
    "arithmetic_4da": arithmetic.Arithmetic4DPlus,
    "arithmetic_4ds": arithmetic.Arithmetic4DMinus,
    "arithmetic_5da": arithmetic.Arithmetic5DPlus,
    "arithmetic_5ds": arithmetic.Arithmetic5DMinus,
    "arithmetic_2dm": arithmetic.Arithmetic2DMultiplication,
    "arithmetic_1dc": arithmetic.Arithmetic1DComposite,
&'s avatar
& committed
125
126
    # TODO Perhaps make these groups of tasks
    #   e.g. anli, arithmetic, openai_translations, harness_translations
&'s avatar
...  
& committed
127
128

    # e.g. wmt14-fr-en
&'s avatar
& committed
129
130
131
    **translation.create_tasks_from_benchmarks(gpt3_translation_benchmarks),
    # chef's selection, mostly wmt20
    **translation.create_tasks_from_benchmarks(selected_translation_benchmarks),
Jason Phang's avatar
Jason Phang committed
132
}
Jason Phang's avatar
gpt3  
Jason Phang committed
133
134


Jason Phang's avatar
Jason Phang committed
135
ALL_TASKS = sorted(list(TASK_REGISTRY))
Jason Phang's avatar
Jason Phang committed
136
137


Jason Phang's avatar
cleanup  
Jason Phang committed
138
def get_task(task_name):
&'s avatar
& committed
139
140
141
142
143
144
    try:
        return TASK_REGISTRY[task_name]
    except KeyError as e:
        print("Available tasks:")
        pprint(TASK_REGISTRY)
        raise KeyError(f"Missing task {task_name}")
Jason Phang's avatar
cleanup  
Jason Phang committed
145
146
147
148
149
150
151


def get_task_dict(task_name_list):
    return {
        task_name: get_task(task_name)()
        for task_name in task_name_list
    }