superglue.py 14.9 KB
Newer Older
Jason Phang's avatar
Jason Phang committed
1
2
3
4
5
"""
To-do:
    - WSC requires free-form generation
    - ReCoRD
"""
Jason Phang's avatar
Jason Phang committed
6
import numpy as np
Jason Phang's avatar
Jason Phang committed
7
from . common import HFTask, yesno
&'s avatar
& committed
8
9
from lm_eval.base import rf
from ..metrics import mean, acc_all, metric_max_over_ground_truths
Jason Phang's avatar
Jason Phang committed
10
11
import sklearn
import transformers.data.metrics.squad_metrics as squad_metrics
Leo Gao's avatar
Fix  
Leo Gao committed
12
from ..utils import general_detokenize
Jason Phang's avatar
Jason Phang committed
13

Jason Phang's avatar
Jason Phang committed
14

15
class BoolQ(HFTask):
Leo Gao's avatar
Leo Gao committed
16
17
    DATASET_PATH = "super_glue"
    DATASET_NAME = "boolq"
Jason Phang's avatar
Jason Phang committed
18
19
20
21
22
23
24
25
26
27
28

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
        return True

    def fewshot_description(self):
29
        # TODO: figure out actual description
Jason Phang's avatar
Jason Phang committed
30
31
        return "Read the following passages and answer each question with a yes or a no."

Leo Gao's avatar
Update  
Leo Gao committed
32
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
33
        return f"{doc['passage']}\nQuestion: {doc['question']}\nAnswer:"
Leo Gao's avatar
Update  
Leo Gao committed
34
35
    
    def doc_to_target(self, doc):
36
        return " " + yesno(doc['label']) 
Jason Phang's avatar
Jason Phang committed
37

38
    def construct_requests(self, doc, ctx):
Leo Gao's avatar
Update  
Leo Gao committed
39

40
        ll_yes, _ = rf.loglikelihood(ctx, ' yes')
Jason Phang's avatar
Jason Phang committed
41
        ll_no, _ = rf.loglikelihood(ctx, ' no')
Leo Gao's avatar
Update  
Leo Gao committed
42
43
44
45
46
47
48
49
50

        return ll_yes, ll_no

    def process_results(self, doc, results):
        ll_yes, ll_no = results
        gold = doc["label"]

        acc = 1. if (ll_yes > ll_no) == gold else 0.

51
52
53
54
55
56
57
58
59
60
61
62
63
        return {
            "acc": acc
        }
    
    def higher_is_better(self):
        return {
            "acc": True
        }
    
    def aggregation(self):
        return {
            "acc": mean
        }
Jason Phang's avatar
Jason Phang committed
64

Jason Phang's avatar
Jason Phang committed
65

66
class CommitmentBank(HFTask):
Leo Gao's avatar
Leo Gao committed
67
68
    DATASET_PATH = "super_glue"
    DATASET_NAME = "cb"
Jason Phang's avatar
Jason Phang committed
69
70
71
72
73
74
75
76
77
78

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
        return True

thefazzer's avatar
thefazzer committed
79
    def fewshot_description(self):
80
        # TODO: figure out actual description
Jason Phang's avatar
Jason Phang committed
81
82
        return "Given a premise and a hypothesis, classify whether the author of the premise is committed" \
            "to the truth of the hypothesis. The three possible labels are true, false or neither."
thefazzer's avatar
thefazzer committed
83

84
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
85
        return "{}\nQuestion: {}. True, False or Neither?\nAnswer:".format(
Jason Phang's avatar
Jason Phang committed
86
87
88
            doc["premise"],
            doc["hypothesis"],
        )
89

thefazzer's avatar
thefazzer committed
90
    def doc_to_target(self, doc):
91
92
93
        # True = entailment
        # False = contradiction
        # Neither = neutral
Leo Gao's avatar
Leo Gao committed
94
        return " {}".format({0: "True", 1: "Neither", 2: "False"}[doc["label"]])
Jason Phang's avatar
Jason Phang committed
95

thefazzer's avatar
thefazzer committed
96
    def construct_requests(self, doc, ctx):
Leo Gao's avatar
Leo Gao committed
97
98
99
        ll_true, _ = rf.loglikelihood(ctx, ' True')
        ll_neither, _ = rf.loglikelihood(ctx, ' Neither')
        ll_false, _ = rf.loglikelihood(ctx, ' False')
100

thefazzer's avatar
thefazzer committed
101
102
103
104
        return ll_true, ll_neither, ll_false

    def process_results(self, doc, results):
        gold = doc["label"]
thefazzer's avatar
thefazzer committed
105
106
        pred = np.argmax(results)
        acc = 1. if pred == gold else 0.
Jason Phang's avatar
Jason Phang committed
107

thefazzer's avatar
thefazzer committed
108
        return {
thefazzer's avatar
thefazzer committed
109
110
            "acc": acc,
            "f1": (pred, gold)
thefazzer's avatar
thefazzer committed
111
112
113
114
        }
    
    def higher_is_better(self):
        return {
115
116
            "acc": True,
            "f1": True
thefazzer's avatar
thefazzer committed
117
        }
Jason Phang's avatar
Jason Phang committed
118
119
120
121
122
123
124
125
126
127
128

    @classmethod
    def cb_multi_fi(cls, items):
        preds, golds = zip(*items)
        preds = np.array(preds)
        golds = np.array(golds)
        f11 = sklearn.metrics.f1_score(y_true=golds == 0, y_pred=preds == 0)
        f12 = sklearn.metrics.f1_score(y_true=golds == 1, y_pred=preds == 1)
        f13 = sklearn.metrics.f1_score(y_true=golds == 2, y_pred=preds == 2)
        avg_f1 = mean([f11, f12, f13])
        return avg_f1
thefazzer's avatar
thefazzer committed
129
130
131
    
    def aggregation(self):
        return {
thefazzer's avatar
thefazzer committed
132
            "acc": mean,
Jason Phang's avatar
Jason Phang committed
133
            "f1": self.cb_multi_fi,
thefazzer's avatar
thefazzer committed
134
        }
Jason Phang's avatar
Jason Phang committed
135

Jason Phang's avatar
Jason Phang committed
136

137
class Copa(HFTask):
Leo Gao's avatar
Leo Gao committed
138
139
    DATASET_PATH = "super_glue"
    DATASET_NAME = "copa"
Jason Phang's avatar
Jason Phang committed
140
141
142
143
144
145
146
147
148
149

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
        return True

thefazzer's avatar
thefazzer committed
150
    def fewshot_description(self):
151
        # TODO: figure out actual description
Jason Phang's avatar
Jason Phang committed
152
153
        return "Given a premise and one alternative with a causal relation to the premise and another without," \
            "choose the more plausible alternative"
thefazzer's avatar
thefazzer committed
154

155
    def doc_to_text(self, doc):
Jason Phang's avatar
Jason Phang committed
156
        # Drop the period
Jason Phang's avatar
Jason Phang committed
157
158
159
160
        connector = {
            "cause": "because",
            "effect": "therefore",
        }[doc["question"]]
161
        return doc["premise"].strip()[:-1] + f" {connector}"
Jason Phang's avatar
Jason Phang committed
162

thefazzer's avatar
thefazzer committed
163
    def doc_to_target(self, doc):
164
165
        correct_choice = doc["choice1"] if doc["label"] == 0 else doc["choice2"]
        # Connect the sentences
166
        return " " + self.convert_choice(correct_choice)
thefazzer's avatar
thefazzer committed
167
168

    def construct_requests(self, doc, ctx):
thefazzer's avatar
thefazzer committed
169
170
        choice1 = " " + self.convert_choice(doc["choice1"])
        choice2 = " " + self.convert_choice(doc["choice2"])
thefazzer's avatar
thefazzer committed
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
        
        ll_choice1, _ = rf.loglikelihood(ctx, choice1)
        ll_choice2, _ = rf.loglikelihood(ctx, choice2)

        return ll_choice1, ll_choice2

    def process_results(self, doc, results):
        gold = doc["label"]
        pred = np.argmax(results)
        acc = 1. if pred == gold else 0.

        return {
            "acc": acc
        }
    
    def higher_is_better(self):
        return {
            "acc": True
        }
    
    def aggregation(self):
        return {
            "acc": mean
        }
Jason Phang's avatar
Jason Phang committed
195
196
197
198
199
200

    @staticmethod
    def convert_choice(choice):
        return choice[0].lower() + choice[1:]


201
class MultiRC(HFTask):
Leo Gao's avatar
Leo Gao committed
202
203
    DATASET_PATH = "super_glue"
    DATASET_NAME = "multirc"
Jason Phang's avatar
multirc  
Jason Phang committed
204
205
206
207
208
209
210
211
212
213
214

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
        return True

    def fewshot_description(self):
215
        # TODO: figure out actual description
Jason Phang's avatar
multirc  
Jason Phang committed
216
217
        return "READING COMPREHENSION ANSWER KEY"

218
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
219
        return f"{doc['paragraph']}\nQuestion: {doc['question']}\nAnswer:"
220
221

    def doc_to_target(self, doc):
Leo Gao's avatar
Leo Gao committed
222
        return " " + self.format_answer(answer=doc["answer"], label=doc["label"])
Jason Phang's avatar
multirc  
Jason Phang committed
223
224
225

    @staticmethod
    def format_answer(answer, label):
Leo Gao's avatar
Fix  
Leo Gao committed
226
        label_str = "yes" if label else "no"
Leo Gao's avatar
Leo Gao committed
227
        return f"{label_str}, {answer}"
Jason Phang's avatar
multirc  
Jason Phang committed
228

thefazzer's avatar
thefazzer committed
229
230
231
232
233
234
235
236
237
238
239
    def construct_requests(self, doc, ctx):
        true_choice = self.format_answer(answer=doc["answer"], label=True)
        false_choice = self.format_answer(answer=doc["answer"], label=False)
        
        ll_true_choice, _ = rf.loglikelihood(ctx, f' {true_choice}')
        ll_false_choice, _ = rf.loglikelihood(ctx, f' {false_choice}')

        return ll_true_choice, ll_false_choice

    def process_results(self, doc, results):
        pred = np.argmax(results)
Jason Phang's avatar
multirc  
Jason Phang committed
240
        return {
thefazzer's avatar
thefazzer committed
241
242
243
244
245
246
247
248
249
250
251
            "acc": (pred, doc)
        }
    
    def higher_is_better(self):
        return {
            "acc": True
        }
    
    def aggregation(self):
        return {
            "acc": acc_all
Jason Phang's avatar
multirc  
Jason Phang committed
252
253
        }

Jason Phang's avatar
Jason Phang committed
254
255
256
257
258
259
260
261
262
263
264
265

class ReCoRD(HFTask):
    DATASET_PATH = "super_glue"
    DATASET_NAME = "record"

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
Leo Gao's avatar
Leo Gao committed
266
        return False
Jason Phang's avatar
Jason Phang committed
267

268
269
270
271
    def fewshot_description(self):
        # TODO: figure out actual description
        return ""

Jason Phang's avatar
Jason Phang committed
272
273
274
    def training_docs(self):
        # In ReCoRD, each doc manifests multiple "examples" in the context of few shot example packing.
        # Each doc consists of multiple answer candidates, each of which is scored yes/no.
Jason Phang's avatar
Jason Phang committed
275
        # Hence, we create one "doc" for each (context + passage, answer) pair.
Jason Phang's avatar
Jason Phang committed
276
        # Moreover, we only use the correct answers for context packing
277
278
279
280
281
282
283
284
285
286
287
288
289
        if self._training_docs is None:
            self._training_docs = []
            for doc in self.data["train"]:
                for entity in list(set(doc["entities"])):
                    self._training_docs.append({
                        "passage": doc["passage"],
                        "query": doc["query"],
                        "entity": entity,
                        "label": entity in doc["answers"],
                    })
        return self._training_docs

    def validation_docs(self):
Jason Phang's avatar
Jason Phang committed
290
291
292
293
294
295
296
297
298
        for example_idx, doc in enumerate(self.data["validation"]):
            for entity in sorted(list(set(doc["entities"]))):
                yield {
                    "passage": doc["passage"],
                    "query": doc["query"],
                    "entity": entity,
                    "label": entity in doc["answers"],
                    "example_idx": example_idx,
                }
Jason Phang's avatar
Jason Phang committed
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315

    def doc_to_text(self, doc):
        initial_text, *highlights = doc["passage"].strip().split("\n@highlight\n")
        text = initial_text + "\n\n"
        for highlight in highlights:
            text += f"  - {highlight}.\n"
        return text

    @classmethod
    def format_answer(cls, query, entity):
        return f'  - {query}'.replace("@placeholder", entity)

    def doc_to_target(self, doc):
        return self.format_answer(query=doc["query"], entity=doc["entity"])

    def construct_requests(self, doc, ctx):
        requests = [
Jason Phang's avatar
Jason Phang committed
316
            rf.loglikelihood(ctx, self.format_answer(query=doc["query"], entity=doc["entity"]))
Jason Phang's avatar
Jason Phang committed
317
318
319
320
        ]
        return requests

    def process_results(self, doc, results):
Jason Phang's avatar
Jason Phang committed
321
322
323
324
325
326
327
328
        # We defer the actual meat of ReCoRD's evaluation until we start collating the results across "docs"
        assert len(results) == 1
        scoring_info = {
            "example_idx": doc["example_idx"],
            "pred_score": results[0][0],
            "entity": doc["entity"],
            "label": doc["label"],
        }
Jason Phang's avatar
Jason Phang committed
329
        return {
Jason Phang's avatar
Jason Phang committed
330
331
            "f1": scoring_info,
            "em": scoring_info,
Jason Phang's avatar
Jason Phang committed
332
333
334
335
336
337
338
339
340
341
        }

    def higher_is_better(self):
        return {
            "f1": True,
            "em": True,
        }

    def aggregation(self):
        return {
Jason Phang's avatar
Jason Phang committed
342
343
            "f1": self.record_eval_em,
            "em": self.record_eval_f1,
Jason Phang's avatar
Jason Phang committed
344
345
        }

Jason Phang's avatar
Jason Phang committed
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
    @classmethod
    def record_eval_aggregation(cls, items, scoring_function):
        # ReCoRD's evaluation is actually deceptively simple:
        # - Pick the maximum likelihood prediction entity
        # - Evaluate the accuracy and token F1 PER EXAMPLE
        # - Average over all examples

        # Reconstruct an example_idx -> example results mapping
        # (remember, each example spans multiple docs)
        example_dict = {}
        for item in items:
            example_idx = item["example_idx"]
            if example_idx not in example_dict:
                example_dict[example_idx] = []
            example_dict[example_idx].append(item)

        # Compute score for each example
        score_list = []
        for example in example_dict.values():
            max_idx = int(np.argmax(np.array([result["pred_score"] for result in example])))
            entities = [result["entity"] for result in example]
            prediction = entities[max_idx]
            gold_label_set = list(set(result["entity"] for result in example if result["label"]))
            if not gold_label_set:
                # When we limit the number of docs processed, some examples may not have any valid answers.
                # We skip these example.
                continue
            per_example_score = metric_max_over_ground_truths(scoring_function, prediction, gold_label_set)
            score_list.append(per_example_score)
        return np.mean(score_list)

    @classmethod
    def record_eval_em(cls, items):
        return cls.record_eval_aggregation(items, scoring_function=squad_metrics.compute_exact)

    @classmethod
    def record_eval_f1(cls, items):
        return cls.record_eval_aggregation(items, scoring_function=squad_metrics.compute_f1)

Jason Phang's avatar
Jason Phang committed
385

386
class WordsInContext(HFTask):
Leo Gao's avatar
Leo Gao committed
387
388
    DATASET_PATH = "super_glue"
    DATASET_NAME = "wic"
Jason Phang's avatar
Jason Phang committed
389
390
391
392
393
394
395
396
397
398

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
        return True

399
400
401
402
    def fewshot_description(self):
        # TODO: figure out actual description
        return ""

403
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
404
405
        return "Sentence 1: {}\nSentence 2: {}\nQuestion: Is the word '{}' used in the same way in the" \
               " two sentences above?\nAnswer:".format(
Jason Phang's avatar
Jason Phang committed
406
407
408
409
                    doc["sentence1"],
                    doc["sentence2"],
                    doc["sentence1"][doc["start1"]:doc["end1"]],
                )
410
411
412

    def doc_to_target(self, doc):
        return " {}".format({0: "no", 1: "yes"}[doc["label"]])
Jason Phang's avatar
Jason Phang committed
413

Jason Phang's avatar
Jason Phang committed
414
415
416
417
418
    def construct_requests(self, doc, ctx):
        ll_yes, _ = rf.loglikelihood(ctx, ' yes')
        ll_no, _ = rf.loglikelihood(ctx, ' no')

        return ll_yes, ll_no
419

Jason Phang's avatar
Jason Phang committed
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
    def process_results(self, doc, results):
        ll_yes, ll_no = results
        gold = doc["label"]

        acc = 1. if (ll_yes > ll_no) == gold else 0.

        return {
            "acc": acc
        }

    def higher_is_better(self):
        return {
            "acc": True
        }

    def aggregation(self):
        return {
            "acc": mean
        }
Jason Phang's avatar
Jason Phang committed
439
440


441
class SGWinogradSchemaChallenge(HFTask):
Jason Phang's avatar
wsc  
Jason Phang committed
442
443
    # Note: This implementation differs from Fig G.32 because this is the SuperGLUE,
    #       binary version of the task.
Leo Gao's avatar
Leo Gao committed
444
445
    DATASET_PATH = "super_glue"
    DATASET_NAME = "wsc"
Jason Phang's avatar
Jason Phang committed
446
447
448
449
450
451
452
453
454
455
456
457
458

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
        return True

    def training_docs(self):
        if self.has_training_docs():
            if self._training_docs is None:
Jason Phang's avatar
Jason Phang committed
459
                # GPT-3 Paper's format only uses positive examples for fewshot "training"
Jason Phang's avatar
Jason Phang committed
460
461
                self._training_docs = [
                    doc for doc in
Jason Phang's avatar
Jason Phang committed
462
                    self.data["train"]
Jason Phang's avatar
Jason Phang committed
463
464
465
466
467
468
469
470
471
472
                    if doc["label"]
                ]
            return self._training_docs

    def fewshot_description(self):
        return "Final Exam with Answer Key\n" \
           "Instructions: Please carefully read the following passages. " \
           "For each passage, you must identify which noun the pronoun marked in *bold*" \
           " refers to.\n====="

473
    def doc_to_text(self, doc):
Jason Phang's avatar
Jason Phang committed
474
        raw_passage = doc["text"]
Jonathan Tow's avatar
Jonathan Tow committed
475
476
477
        # NOTE: HuggingFace span indices are word-based not character-based.
        pre = " ".join(raw_passage.split()[:doc["span2_index"]])
        post = raw_passage[len(pre) + len(doc["span2_text"]) + 1:]
Leo Gao's avatar
Leo Gao committed
478
        passage = general_detokenize(pre + " *{}*".format(doc['span2_text']) + post)
Jason Phang's avatar
wsc  
Jason Phang committed
479
        noun = doc["span1_text"]
Jason Phang's avatar
Jason Phang committed
480
481
482
        pronoun = doc["span2_text"]
        text = (
            f"Passage: {passage}\n"
Jason Phang's avatar
wsc  
Jason Phang committed
483
            + f"Question: In the passage above, does the pronoun \"*{pronoun}*\" refer to \"*{noun}*\"?\n"
Jason Phang's avatar
Jason Phang committed
484
485
486
487
            + "Answer:"
        )
        return text

488
    def doc_to_target(self, doc):
Leo Gao's avatar
Leo Gao committed
489
        return " " + yesno(doc['label'])
490

Leo Gao's avatar
Leo Gao committed
491
    def construct_requests(self, doc, ctx):
Jason Phang's avatar
wsc  
Jason Phang committed
492
493
494
495
496

        ll_yes, _ = rf.loglikelihood(ctx, ' yes')
        ll_no, _ = rf.loglikelihood(ctx, ' no')

        return ll_yes, ll_no
497

Jason Phang's avatar
Jason Phang committed
498
    def process_results(self, doc, results):
Jason Phang's avatar
wsc  
Jason Phang committed
499
500
501
502
503
504
505
506
        ll_yes, ll_no = results
        gold = doc["label"]

        acc = 1. if (ll_yes > ll_no) == gold else 0.

        return {
            "acc": acc
        }
Anish Thite's avatar
Anish Thite committed
507

Leo Gao's avatar
Leo Gao committed
508
    def higher_is_better(self):
Jason Phang's avatar
Jason Phang committed
509
510
511
512
513
514
515
516
        return {
            "acc": True
        }

    def aggregation(self):
        return {
            "acc": mean
        }