superglue.py 12.1 KB
Newer Older
Jason Phang's avatar
Jason Phang committed
1
2
3
4
5
"""
To-do:
    - WSC requires free-form generation
    - ReCoRD
"""
Jason Phang's avatar
Jason Phang committed
6
import numpy as np
7
8
import sklearn
import transformers.data.metrics.squad_metrics as squad_metrics
Jason Phang's avatar
Jason Phang committed
9
from . common import HFTask, yesno
&'s avatar
& committed
10
11
from lm_eval.base import rf
from ..metrics import mean, acc_all, metric_max_over_ground_truths
Leo Gao's avatar
Fix  
Leo Gao committed
12
from ..utils import general_detokenize
Jason Phang's avatar
Jason Phang committed
13

Jason Phang's avatar
Jason Phang committed
14

15
class BoolQ(HFTask):
16
    VERSION = 1
Leo Gao's avatar
Leo Gao committed
17
18
    DATASET_PATH = "super_glue"
    DATASET_NAME = "boolq"
Jason Phang's avatar
Jason Phang committed
19
20
21
22
23
24
25
26

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
27
        return False
Jason Phang's avatar
Jason Phang committed
28

Leo Gao's avatar
Update  
Leo Gao committed
29
    def doc_to_text(self, doc):
30
        return f"{doc['passage']}\nQuestion: {doc['question']}?\nAnswer:"
31
32
33
34
35
36

    def should_decontaminate(self):
        return True

    def doc_to_decontamination_query(self, doc):
        return doc['passage']
Leo Gao's avatar
Update  
Leo Gao committed
37
38
    
    def doc_to_target(self, doc):
39
        return " " + yesno(doc['label']) 
Jason Phang's avatar
Jason Phang committed
40

41
    def construct_requests(self, doc, ctx):
Leo Gao's avatar
Update  
Leo Gao committed
42

43
        ll_yes, _ = rf.loglikelihood(ctx, ' yes')
Jason Phang's avatar
Jason Phang committed
44
        ll_no, _ = rf.loglikelihood(ctx, ' no')
Leo Gao's avatar
Update  
Leo Gao committed
45
46
47
48
49
50
51
52
53

        return ll_yes, ll_no

    def process_results(self, doc, results):
        ll_yes, ll_no = results
        gold = doc["label"]

        acc = 1. if (ll_yes > ll_no) == gold else 0.

54
55
56
57
58
59
60
61
62
63
64
65
66
        return {
            "acc": acc
        }
    
    def higher_is_better(self):
        return {
            "acc": True
        }
    
    def aggregation(self):
        return {
            "acc": mean
        }
Jason Phang's avatar
Jason Phang committed
67

Jason Phang's avatar
Jason Phang committed
68

69
class CommitmentBank(HFTask):
thomasw21's avatar
thomasw21 committed
70
    VERSION = 1
Leo Gao's avatar
Leo Gao committed
71
72
    DATASET_PATH = "super_glue"
    DATASET_NAME = "cb"
Jason Phang's avatar
Jason Phang committed
73
74
75
76
77
78
79
80

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
81
        return False
Jason Phang's avatar
Jason Phang committed
82

83
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
84
        return "{}\nQuestion: {}. True, False or Neither?\nAnswer:".format(
Jason Phang's avatar
Jason Phang committed
85
86
87
            doc["premise"],
            doc["hypothesis"],
        )
88

thefazzer's avatar
thefazzer committed
89
    def doc_to_target(self, doc):
90
91
92
        # True = entailment
        # False = contradiction
        # Neither = neutral
thomasw21's avatar
Fix CB  
thomasw21 committed
93
        return " {}".format({0: "True", 1: "False", 2: "Neither"}[doc["label"]])
Jason Phang's avatar
Jason Phang committed
94

thefazzer's avatar
thefazzer committed
95
    def construct_requests(self, doc, ctx):
Leo Gao's avatar
Leo Gao committed
96
97
        ll_true, _ = rf.loglikelihood(ctx, ' True')
        ll_false, _ = rf.loglikelihood(ctx, ' False')
thomasw21's avatar
Fix CB  
thomasw21 committed
98
        ll_neither, _ = rf.loglikelihood(ctx, ' Neither')
99

thomasw21's avatar
Fix CB  
thomasw21 committed
100
        return ll_true, ll_false, ll_neither
thefazzer's avatar
thefazzer committed
101
102
103

    def process_results(self, doc, results):
        gold = doc["label"]
thefazzer's avatar
thefazzer committed
104
105
        pred = np.argmax(results)
        acc = 1. if pred == gold else 0.
Jason Phang's avatar
Jason Phang committed
106

thefazzer's avatar
thefazzer committed
107
        return {
thefazzer's avatar
thefazzer committed
108
109
            "acc": acc,
            "f1": (pred, gold)
thefazzer's avatar
thefazzer committed
110
111
112
113
        }
    
    def higher_is_better(self):
        return {
114
115
            "acc": True,
            "f1": True
thefazzer's avatar
thefazzer committed
116
        }
Jason Phang's avatar
Jason Phang committed
117
118
119
120
121
122
123
124
125
126
127

    @classmethod
    def cb_multi_fi(cls, items):
        preds, golds = zip(*items)
        preds = np.array(preds)
        golds = np.array(golds)
        f11 = sklearn.metrics.f1_score(y_true=golds == 0, y_pred=preds == 0)
        f12 = sklearn.metrics.f1_score(y_true=golds == 1, y_pred=preds == 1)
        f13 = sklearn.metrics.f1_score(y_true=golds == 2, y_pred=preds == 2)
        avg_f1 = mean([f11, f12, f13])
        return avg_f1
thefazzer's avatar
thefazzer committed
128
129
130
    
    def aggregation(self):
        return {
thefazzer's avatar
thefazzer committed
131
            "acc": mean,
Jason Phang's avatar
Jason Phang committed
132
            "f1": self.cb_multi_fi,
thefazzer's avatar
thefazzer committed
133
        }
Jason Phang's avatar
Jason Phang committed
134

Jason Phang's avatar
Jason Phang committed
135

136
class Copa(HFTask):
Leo Gao's avatar
Leo Gao committed
137
    VERSION = 0
Leo Gao's avatar
Leo Gao committed
138
139
    DATASET_PATH = "super_glue"
    DATASET_NAME = "copa"
Jason Phang's avatar
Jason Phang committed
140
141
142
143
144
145
146
147

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
148
        return False
Jason Phang's avatar
Jason Phang committed
149

150
    def doc_to_text(self, doc):
Jason Phang's avatar
Jason Phang committed
151
        # Drop the period
Jason Phang's avatar
Jason Phang committed
152
153
154
155
        connector = {
            "cause": "because",
            "effect": "therefore",
        }[doc["question"]]
156
        return doc["premise"].strip()[:-1] + f" {connector}"
Jason Phang's avatar
Jason Phang committed
157

thefazzer's avatar
thefazzer committed
158
    def doc_to_target(self, doc):
159
160
        correct_choice = doc["choice1"] if doc["label"] == 0 else doc["choice2"]
        # Connect the sentences
161
        return " " + self.convert_choice(correct_choice)
thefazzer's avatar
thefazzer committed
162
163

    def construct_requests(self, doc, ctx):
thefazzer's avatar
thefazzer committed
164
165
        choice1 = " " + self.convert_choice(doc["choice1"])
        choice2 = " " + self.convert_choice(doc["choice2"])
thefazzer's avatar
thefazzer committed
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
        
        ll_choice1, _ = rf.loglikelihood(ctx, choice1)
        ll_choice2, _ = rf.loglikelihood(ctx, choice2)

        return ll_choice1, ll_choice2

    def process_results(self, doc, results):
        gold = doc["label"]
        pred = np.argmax(results)
        acc = 1. if pred == gold else 0.

        return {
            "acc": acc
        }
    
    def higher_is_better(self):
        return {
            "acc": True
        }
    
    def aggregation(self):
        return {
            "acc": mean
        }
Jason Phang's avatar
Jason Phang committed
190
191
192
193
194
195

    @staticmethod
    def convert_choice(choice):
        return choice[0].lower() + choice[1:]


196
class MultiRC(HFTask):
197
    VERSION = 1
Leo Gao's avatar
Leo Gao committed
198
199
    DATASET_PATH = "super_glue"
    DATASET_NAME = "multirc"
Jason Phang's avatar
multirc  
Jason Phang committed
200
201
202
203
204
205
206
207

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
208
        return False
Jason Phang's avatar
multirc  
Jason Phang committed
209

210
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
211
        return f"{doc['paragraph']}\nQuestion: {doc['question']}\nAnswer:"
212
213

    def doc_to_target(self, doc):
Leo Gao's avatar
Leo Gao committed
214
        return " " + self.format_answer(answer=doc["answer"], label=doc["label"])
Jason Phang's avatar
multirc  
Jason Phang committed
215
216
217

    @staticmethod
    def format_answer(answer, label):
Leo Gao's avatar
Fix  
Leo Gao committed
218
        label_str = "yes" if label else "no"
thomasw21's avatar
thomasw21 committed
219
        return f"{answer}\nIs the answer correct? {label_str}"
Jason Phang's avatar
multirc  
Jason Phang committed
220

thefazzer's avatar
thefazzer committed
221
222
223
224
225
226
227
228
229
230
    def construct_requests(self, doc, ctx):
        true_choice = self.format_answer(answer=doc["answer"], label=True)
        false_choice = self.format_answer(answer=doc["answer"], label=False)
        
        ll_true_choice, _ = rf.loglikelihood(ctx, f' {true_choice}')
        ll_false_choice, _ = rf.loglikelihood(ctx, f' {false_choice}')

        return ll_true_choice, ll_false_choice

    def process_results(self, doc, results):
thomasw21's avatar
thomasw21 committed
231
232
        ll_true_choice, ll_false_choice = results
        pred = ll_true_choice > ll_false_choice
Jason Phang's avatar
multirc  
Jason Phang committed
233
        return {
thefazzer's avatar
thefazzer committed
234
235
236
237
238
239
240
241
242
243
244
            "acc": (pred, doc)
        }
    
    def higher_is_better(self):
        return {
            "acc": True
        }
    
    def aggregation(self):
        return {
            "acc": acc_all
Jason Phang's avatar
multirc  
Jason Phang committed
245
246
        }

Jason Phang's avatar
Jason Phang committed
247
248

class ReCoRD(HFTask):
Leo Gao's avatar
Leo Gao committed
249
    VERSION = 0
Jason Phang's avatar
Jason Phang committed
250
251
252
253
254
255
256
257
258
259
    DATASET_PATH = "super_glue"
    DATASET_NAME = "record"

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
Leo Gao's avatar
Leo Gao committed
260
        return False
Jason Phang's avatar
Jason Phang committed
261
262
263
264

    def training_docs(self):
        # In ReCoRD, each doc manifests multiple "examples" in the context of few shot example packing.
        # Each doc consists of multiple answer candidates, each of which is scored yes/no.
265
266
267
        if self._training_docs is None:
            self._training_docs = []
            for doc in self.data["train"]:
Jason Phang's avatar
Jason Phang committed
268
                self._training_docs.append(self._process_doc(doc))
269
270
271
        return self._training_docs

    def validation_docs(self):
Jason Phang's avatar
Jason Phang committed
272
273
274
275
276
277
278
279
280
281
282
283
        # See: training_docs
        for doc in self.data["validation"]:
            yield self._process_doc(doc)

    @classmethod
    def _process_doc(cls, doc):
        return {
            "passage": doc["passage"],
            "query": doc["query"],
            "entities": sorted(list(set(doc["entities"]))),
            "answers": sorted(list(set(doc["answers"]))),
        }
Jason Phang's avatar
Jason Phang committed
284
285
286
287
288
289
290
291
292
293
294
295
296

    def doc_to_text(self, doc):
        initial_text, *highlights = doc["passage"].strip().split("\n@highlight\n")
        text = initial_text + "\n\n"
        for highlight in highlights:
            text += f"  - {highlight}.\n"
        return text

    @classmethod
    def format_answer(cls, query, entity):
        return f'  - {query}'.replace("@placeholder", entity)

    def doc_to_target(self, doc):
Jason Phang's avatar
Jason Phang committed
297
298
        # We only output the first correct entity in a doc
        return self.format_answer(query=doc["query"], entity=doc["answers"][0])
Jason Phang's avatar
Jason Phang committed
299
300
301
302

    def construct_requests(self, doc, ctx):
        requests = [
            rf.loglikelihood(ctx, self.format_answer(query=doc["query"], entity=entity))
Jason Phang's avatar
Jason Phang committed
303
            for entity in doc["entities"]
Jason Phang's avatar
Jason Phang committed
304
305
306
307
308
309
310
311
        ]
        return requests

    def process_results(self, doc, results):
        # ReCoRD's evaluation is actually deceptively simple:
        # - Pick the maximum likelihood prediction entity
        # - Evaluate the accuracy and token F1 PER EXAMPLE
        # - Average over all examples
Jason Phang's avatar
Jason Phang committed
312
        max_idx = np.argmax(np.array([result[0] for result in results]))
Leo Gao's avatar
Leo Gao committed
313

Jason Phang's avatar
Jason Phang committed
314
        prediction = doc["entities"][max_idx]
Jason Phang's avatar
Jason Phang committed
315
        gold_label_set = doc["answers"]
Jason Phang's avatar
Jason Phang committed
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
        f1 = metric_max_over_ground_truths(squad_metrics.compute_f1, prediction, gold_label_set)
        em = metric_max_over_ground_truths(squad_metrics.compute_exact, prediction, gold_label_set)

        return {
            "f1": f1,
            "em": em,
        }

    def higher_is_better(self):
        return {
            "f1": True,
            "em": True,
        }

    def aggregation(self):
        return {
            "f1": mean,
            "em": mean,
        }


337
class WordsInContext(HFTask):
Leo Gao's avatar
Leo Gao committed
338
    VERSION = 0
Leo Gao's avatar
Leo Gao committed
339
340
    DATASET_PATH = "super_glue"
    DATASET_NAME = "wic"
Jason Phang's avatar
Jason Phang committed
341
342
343
344
345
346
347
348

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
349
        return False
Jason Phang's avatar
Jason Phang committed
350

351
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
352
353
        return "Sentence 1: {}\nSentence 2: {}\nQuestion: Is the word '{}' used in the same way in the" \
               " two sentences above?\nAnswer:".format(
Jason Phang's avatar
Jason Phang committed
354
355
356
357
                    doc["sentence1"],
                    doc["sentence2"],
                    doc["sentence1"][doc["start1"]:doc["end1"]],
                )
358
359
360

    def doc_to_target(self, doc):
        return " {}".format({0: "no", 1: "yes"}[doc["label"]])
Jason Phang's avatar
Jason Phang committed
361

Jason Phang's avatar
Jason Phang committed
362
363
364
365
366
    def construct_requests(self, doc, ctx):
        ll_yes, _ = rf.loglikelihood(ctx, ' yes')
        ll_no, _ = rf.loglikelihood(ctx, ' no')

        return ll_yes, ll_no
367

Jason Phang's avatar
Jason Phang committed
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
    def process_results(self, doc, results):
        ll_yes, ll_no = results
        gold = doc["label"]

        acc = 1. if (ll_yes > ll_no) == gold else 0.

        return {
            "acc": acc
        }

    def higher_is_better(self):
        return {
            "acc": True
        }

    def aggregation(self):
        return {
            "acc": mean
        }
Jason Phang's avatar
Jason Phang committed
387
388


389
class SGWinogradSchemaChallenge(HFTask):
Leo Gao's avatar
Leo Gao committed
390
    VERSION = 0
Jason Phang's avatar
wsc  
Jason Phang committed
391
392
    # Note: This implementation differs from Fig G.32 because this is the SuperGLUE,
    #       binary version of the task.
Leo Gao's avatar
Leo Gao committed
393
394
    DATASET_PATH = "super_glue"
    DATASET_NAME = "wsc"
Jason Phang's avatar
Jason Phang committed
395
396
397
398
399
400
401
402

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
403
        return False
Jason Phang's avatar
Jason Phang committed
404
405
406
407

    def training_docs(self):
        if self.has_training_docs():
            if self._training_docs is None:
Jason Phang's avatar
Jason Phang committed
408
                # GPT-3 Paper's format only uses positive examples for fewshot "training"
Jason Phang's avatar
Jason Phang committed
409
410
                self._training_docs = [
                    doc for doc in
Jason Phang's avatar
Jason Phang committed
411
                    self.data["train"]
Jason Phang's avatar
Jason Phang committed
412
413
414
415
                    if doc["label"]
                ]
            return self._training_docs

416
    def doc_to_text(self, doc):
Jason Phang's avatar
Jason Phang committed
417
        raw_passage = doc["text"]
Jonathan Tow's avatar
Jonathan Tow committed
418
419
420
        # NOTE: HuggingFace span indices are word-based not character-based.
        pre = " ".join(raw_passage.split()[:doc["span2_index"]])
        post = raw_passage[len(pre) + len(doc["span2_text"]) + 1:]
Leo Gao's avatar
Leo Gao committed
421
        passage = general_detokenize(pre + " *{}*".format(doc['span2_text']) + post)
Jason Phang's avatar
wsc  
Jason Phang committed
422
        noun = doc["span1_text"]
Jason Phang's avatar
Jason Phang committed
423
424
425
        pronoun = doc["span2_text"]
        text = (
            f"Passage: {passage}\n"
Jason Phang's avatar
wsc  
Jason Phang committed
426
            + f"Question: In the passage above, does the pronoun \"*{pronoun}*\" refer to \"*{noun}*\"?\n"
Jason Phang's avatar
Jason Phang committed
427
428
429
430
            + "Answer:"
        )
        return text

431
    def doc_to_target(self, doc):
Leo Gao's avatar
Leo Gao committed
432
        return " " + yesno(doc['label'])
433

Leo Gao's avatar
Leo Gao committed
434
    def construct_requests(self, doc, ctx):
Jason Phang's avatar
wsc  
Jason Phang committed
435
436
437
438
439

        ll_yes, _ = rf.loglikelihood(ctx, ' yes')
        ll_no, _ = rf.loglikelihood(ctx, ' no')

        return ll_yes, ll_no
440

Jason Phang's avatar
Jason Phang committed
441
    def process_results(self, doc, results):
Jason Phang's avatar
wsc  
Jason Phang committed
442
443
444
445
446
447
448
449
        ll_yes, ll_no = results
        gold = doc["label"]

        acc = 1. if (ll_yes > ll_no) == gold else 0.

        return {
            "acc": acc
        }
Anish Thite's avatar
Anish Thite committed
450

Leo Gao's avatar
Leo Gao committed
451
    def higher_is_better(self):
Jason Phang's avatar
Jason Phang committed
452
453
454
455
456
457
458
459
        return {
            "acc": True
        }

    def aggregation(self):
        return {
            "acc": mean
        }