glue.py 16 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
"""
GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding
https://openreview.net/pdf?id=rJ4km2R5t7

The General Language Understanding Evaluation (GLUE) benchmark is a collection of
resources for training, evaluating, and analyzing natural language understanding
systems. GLUE consists of:
- A benchmark of nine sentence- or sentence-pair language understanding tasks built
on established existing datasets and selected to cover a diverse range of dataset
sizes, text genres, and degrees of difficulty, and
- A diagnostic dataset designed to evaluate and analyze model performance with
respect to a wide range of linguistic phenomena found in natural language.

Homepage: https://gluebenchmark.com/
15
16
"""
import numpy as np
17
from lm_eval.base import PromptSourceTask, rf, Task
Jonathan Tow's avatar
Jonathan Tow committed
18
19
from lm_eval.metrics import mean, matthews_corrcoef, f1_score, yesno
from lm_eval.utils import general_detokenize
20

21

22
23
# TODO(jon-tow): Add citations for the individual datasets/tasks that make up GLUE.
_CITATION = """
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
@inproceedings{wang-etal-2018-glue,
    title = "{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding",
    author = "Wang, Alex  and
      Singh, Amanpreet  and
      Michael, Julian  and
      Hill, Felix  and
      Levy, Omer  and
      Bowman, Samuel",
    booktitle = "Proceedings of the 2018 {EMNLP} Workshop {B}lackbox{NLP}: Analyzing and Interpreting Neural Networks for {NLP}",
    month = nov,
    year = "2018",
    address = "Brussels, Belgium",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/W18-5446",
    doi = "10.18653/v1/W18-5446",
    pages = "353--355",
    abstract = "Human ability to understand language is \textit{general, flexible, and robust}. In contrast, most NLU models above the word level are designed for a specific task and struggle with out-of-domain data. If we aspire to develop models with understanding beyond the detection of superficial correspondences between inputs and outputs, then it is critical to develop a unified model that can execute a range of linguistic tasks across different domains. To facilitate research in this direction, we present the General Language Understanding Evaluation (GLUE, gluebenchmark.com): a benchmark of nine diverse NLU tasks, an auxiliary dataset for probing models for understanding of specific linguistic phenomena, and an online platform for evaluating and comparing models. For some benchmark tasks, training data is plentiful, but for others it is limited or does not match the genre of the test set. GLUE thus favors models that can represent linguistic knowledge in a way that facilitates sample-efficient learning and effective knowledge-transfer across tasks. While none of the datasets in GLUE were created from scratch for the benchmark, four of them feature privately-held test data, which is used to ensure that the benchmark is used fairly. We evaluate baselines that use ELMo (Peters et al., 2018), a powerful transfer learning technique, as well as state-of-the-art sentence representation models. The best models still achieve fairly low absolute scores. Analysis with our diagnostic dataset yields similarly weak performance over all phenomena tested, with some exceptions.",
}
"""
43

Jonathan Tow's avatar
Jonathan Tow committed
44
45

# Single-Sentence Tasks
Jason Phang's avatar
Jason Phang committed
46
47


Jonathan Tow's avatar
Jonathan Tow committed
48
class CoLA(Task):
Leo Gao's avatar
Leo Gao committed
49
    VERSION = 0
sdtblck's avatar
sdtblck committed
50
51
    DATASET_PATH = "glue"
    DATASET_NAME = "cola"
Jonathan Tow's avatar
Jonathan Tow committed
52

Jason Phang's avatar
checkin  
Jason Phang committed
53
54
55
56
57
58
59
    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
60
        return False
Jason Phang's avatar
checkin  
Jason Phang committed
61

Jonathan Tow's avatar
Jonathan Tow committed
62
63
64
65
66
67
68
69
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

    def validation_docs(self):
        return self.dataset["validation"]

70
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
71
        return "{}\nQuestion: Does this sentence make sense?\nAnswer:".format(doc["sentence"])
72
73

    def doc_to_target(self, doc):
Leo Gao's avatar
Leo Gao committed
74
        return " {}".format({1: "yes", 0: "no"}[doc["label"]])
Jason Phang's avatar
checkin  
Jason Phang committed
75

Jonathan Tow's avatar
Jonathan Tow committed
76
    def construct_requests(self, doc, ctx):
Leo Gao's avatar
Leo Gao committed
77
78
        ll_true, _ = rf.loglikelihood(ctx, " yes")
        ll_false, _ = rf.loglikelihood(ctx, " no")
Jonathan Tow's avatar
Jonathan Tow committed
79
        return ll_true, ll_false
80

Jonathan Tow's avatar
Jonathan Tow committed
81
82
83
84
85
86
87
    def process_results(self, doc, results):
        ll_true, ll_false = results
        pred = ll_true > ll_false
        gold = doc["label"]
        return {
            "mcc": (gold, pred)
        }
88

Jonathan Tow's avatar
Jonathan Tow committed
89
    def higher_is_better(self):
Jason Phang's avatar
checkin  
Jason Phang committed
90
        return {
Jonathan Tow's avatar
Jonathan Tow committed
91
92
93
94
95
96
97
98
99
            "mcc": True
        }

    def aggregation(self):
        return {
            "mcc": matthews_corrcoef
        }


Jonathan Tow's avatar
Jonathan Tow committed
100
class SST(Task):
Leo Gao's avatar
Leo Gao committed
101
    VERSION = 0
Jonathan Tow's avatar
Jonathan Tow committed
102
103
104
105
106
107
108
109
110
111
    DATASET_PATH = "glue"
    DATASET_NAME = "sst2"

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
112
        return False
Jonathan Tow's avatar
Jonathan Tow committed
113

Jonathan Tow's avatar
Jonathan Tow committed
114
115
116
117
118
119
120
121
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

    def validation_docs(self):
        return self.dataset["validation"]

Jonathan Tow's avatar
Jonathan Tow committed
122
    def doc_to_text(self, doc):
Leo Gao's avatar
Fix  
Leo Gao committed
123
        return "{}\nQuestion: Is this sentence positive or negative?\nAnswer:".format(
Leo Gao's avatar
Leo Gao committed
124
            general_detokenize(doc["sentence"]),
Jonathan Tow's avatar
Jonathan Tow committed
125
126
127
        )

    def doc_to_target(self, doc):
Leo Gao's avatar
Fix  
Leo Gao committed
128
        return " {}".format({1: "positive", 0: "negative"}[doc["label"]])
Jonathan Tow's avatar
Jonathan Tow committed
129
130

    def construct_requests(self, doc, ctx):
Leo Gao's avatar
Fix  
Leo Gao committed
131
132
        ll_positive, _ = rf.loglikelihood(ctx, " positive")
        ll_negative, _ = rf.loglikelihood(ctx, " negative")
Jonathan Tow's avatar
Jonathan Tow committed
133
134
135
136
137
138
139
140
141
142
143
144
145
        return ll_positive, ll_negative

    def process_results(self, doc, results):
        ll_positive, ll_negative = results
        pred = ll_positive > ll_negative
        gold = doc["label"]
        return {
            "acc": pred == gold
        }

    def higher_is_better(self):
        return {
            "acc": True
Jason Phang's avatar
checkin  
Jason Phang committed
146
147
        }

Jonathan Tow's avatar
Jonathan Tow committed
148
149
150
151
152
153
154
155
    def aggregation(self):
        return {
            "acc": mean
        }


# Inference Tasks

Jason Phang's avatar
checkin  
Jason Phang committed
156

Jonathan Tow's avatar
Jonathan Tow committed
157
class MNLI(Task):
Leo Gao's avatar
Leo Gao committed
158
    VERSION = 0
sdtblck's avatar
sdtblck committed
159
160
    DATASET_PATH = "glue"
    DATASET_NAME = "mnli"
Jason Phang's avatar
Jason Phang committed
161

Jason Phang's avatar
checkin  
Jason Phang committed
162
163
164
165
166
167
168
    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
169
        return False
Jason Phang's avatar
checkin  
Jason Phang committed
170

Jonathan Tow's avatar
Jonathan Tow committed
171
172
173
174
175
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

Jason Phang's avatar
checkin  
Jason Phang committed
176
177
    def validation_docs(self):
        if self.has_validation_docs():
Jonathan Tow's avatar
Jonathan Tow committed
178
            return self.dataset["validation_matched"]
Jason Phang's avatar
checkin  
Jason Phang committed
179
180
181

    def test_docs(self):
        if self.has_test_docs():
Jonathan Tow's avatar
Jonathan Tow committed
182
            return self.dataset["test_matched"]
Jason Phang's avatar
checkin  
Jason Phang committed
183

184
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
185
        return "{}\nQuestion: {} True, False or Neither?\nAnswer:".format(
Jason Phang's avatar
Jason Phang committed
186
            doc["premise"],
Leo Gao's avatar
Fix  
Leo Gao committed
187
            doc["hypothesis"].strip() + ('' if doc["hypothesis"].strip().endswith('.') else '.'),
Jason Phang's avatar
checkin  
Jason Phang committed
188
        )
189
190
191
192
193
194

    def doc_to_target(self, doc):
        # True = entailment
        # False = contradiction
        # Neither = neutral
        return " {}".format({0: "True", 1: "Neither", 2: "False"}[doc["label"]])
Jason Phang's avatar
checkin  
Jason Phang committed
195

Jonathan Tow's avatar
Jonathan Tow committed
196
197
198
199
200
    def construct_requests(self, doc, ctx):
        ll_true, _ = rf.loglikelihood(ctx, " True")
        ll_neither, _ = rf.loglikelihood(ctx, " Neither")
        ll_false, _ = rf.loglikelihood(ctx, " False")
        return ll_true, ll_neither, ll_false
201

Jonathan Tow's avatar
Jonathan Tow committed
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
    def process_results(self, doc, results):
        gold = doc["label"]
        pred = np.argmax(results)
        return {
            "acc": pred == gold
        }

    def higher_is_better(self):
        return {
            "acc": True
        }

    def aggregation(self):
        return {
            "acc": mean
        }
Jason Phang's avatar
checkin  
Jason Phang committed
218
219


Jason Phang's avatar
Jason Phang committed
220
class MNLIMismatched(MNLI):
Leo Gao's avatar
Leo Gao committed
221
    VERSION = 0
Jason Phang's avatar
Jason Phang committed
222
223
224

    def validation_docs(self):
        if self.has_validation_docs():
Jonathan Tow's avatar
Jonathan Tow committed
225
            return self.dataset["validation_mismatched"]
Jason Phang's avatar
Jason Phang committed
226
227
228

    def test_docs(self):
        if self.has_test_docs():
Jonathan Tow's avatar
Jonathan Tow committed
229
            return self.dataset["test_mismatched"]
Jason Phang's avatar
Jason Phang committed
230
231


Jonathan Tow's avatar
Jonathan Tow committed
232
class QNLI(Task):
Leo Gao's avatar
Leo Gao committed
233
    VERSION = 0
sdtblck's avatar
sdtblck committed
234
    DATASET_PATH = "glue"
Jonathan Tow's avatar
Jonathan Tow committed
235
    DATASET_NAME = "qnli"
Jason Phang's avatar
Jason Phang committed
236
237
238
239
240
241
242
243

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
244
        return False
Jason Phang's avatar
Jason Phang committed
245

Jonathan Tow's avatar
Jonathan Tow committed
246
247
248
249
250
251
252
253
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

    def validation_docs(self):
        return self.dataset["validation"]

Jonathan Tow's avatar
Jonathan Tow committed
254
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
255
        return "{}\n{}\nQuestion: Does this response answer the question?\nAnswer:".format(
Jonathan Tow's avatar
Jonathan Tow committed
256
257
258
259
260
261
262
            doc["question"],
            doc["sentence"],
        )

    def doc_to_target(self, doc):
        # True = entailment
        # False = not entailment
Leo Gao's avatar
Fix  
Leo Gao committed
263
        return " {}".format({0: "yes", 1: "no"}[doc["label"]])
Jonathan Tow's avatar
Jonathan Tow committed
264
265

    def construct_requests(self, doc, ctx):
Leo Gao's avatar
Fix  
Leo Gao committed
266
267
        ll_yes, _ = rf.loglikelihood(ctx, " yes")
        ll_no, _ = rf.loglikelihood(ctx, " no")
Jonathan Tow's avatar
Jonathan Tow committed
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
        return ll_yes, ll_no

    def process_results(self, doc, results):
        ll_yes, ll_no = results
        pred = ll_no > ll_yes
        gold = doc["label"]
        return {
            "acc": pred == gold
        }

    def higher_is_better(self):
        return {
            "acc": True
        }

    def aggregation(self):
        return {
            "acc": mean
        }


289
class WNLI(PromptSourceTask):
thomasw21's avatar
thomasw21 committed
290
    VERSION = 1
Jonathan Tow's avatar
Jonathan Tow committed
291
292
293
294
295
296
297
298
299
300
    DATASET_PATH = "glue"
    DATASET_NAME = "wnli"

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
301
        return False
Jason Phang's avatar
Jason Phang committed
302

Jonathan Tow's avatar
Jonathan Tow committed
303
    def training_docs(self):
304
305
306
307
        # if self._training_docs is None:
        #     self._training_docs = list()
        # return self._training_docs
        return self.dataset["train"]
Jonathan Tow's avatar
Jonathan Tow committed
308
309
310
311

    def validation_docs(self):
        return self.dataset["validation"]

Jonathan Tow's avatar
Jonathan Tow committed
312
313
314
315
316
317
318
319
320
    def higher_is_better(self):
        return {
            "acc": True
        }

    def aggregation(self):
        return {
            "acc": mean
        }
321

Jason Phang's avatar
Jason Phang committed
322

323
class RTE(PromptSourceTask):
Leo Gao's avatar
Leo Gao committed
324
    VERSION = 0
sdtblck's avatar
sdtblck committed
325
326
    DATASET_PATH = "glue"
    DATASET_NAME = "rte"
Jason Phang's avatar
checkin  
Jason Phang committed
327
328
329
330
331
332
333
334

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
335
        return False
Jason Phang's avatar
checkin  
Jason Phang committed
336

Jonathan Tow's avatar
Jonathan Tow committed
337
338
339
340
341
342
343
344
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

    def validation_docs(self):
        return self.dataset["validation"]

345
346
347
348
349
350
351
    # def process_results(self, doc, results):
    #     ll_true, ll_false = results
    #     pred = ll_false > ll_true
    #     gold = doc["label"]
    #     return {
    #         "acc": pred == gold
    #     }
Jonathan Tow's avatar
Jonathan Tow committed
352
353
354
355
356

    def higher_is_better(self):
        return {
            "acc": True
        }
Jason Phang's avatar
Jason Phang committed
357

Jonathan Tow's avatar
Jonathan Tow committed
358
359
360
361
    def aggregation(self):
        return {
            "acc": mean
        }
Jason Phang's avatar
Jason Phang committed
362

Jonathan Tow's avatar
Jonathan Tow committed
363
364
365
366

# Similarity and Paraphrase Tasks


Jonathan Tow's avatar
Jonathan Tow committed
367
class MRPC(Task):
Leo Gao's avatar
Leo Gao committed
368
    VERSION = 0
sdtblck's avatar
sdtblck committed
369
    DATASET_PATH = "glue"
Jonathan Tow's avatar
Jonathan Tow committed
370
    DATASET_NAME = "mrpc"
Jason Phang's avatar
Jason Phang committed
371
372
373
374
375
376
377
378

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
Leo Gao's avatar
Leo Gao committed
379
        return False
Jason Phang's avatar
Jason Phang committed
380

Jonathan Tow's avatar
Jonathan Tow committed
381
382
383
384
385
386
387
388
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

    def validation_docs(self):
        return self.dataset["validation"]

389
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
390
391
392
        return "Sentence 1: {}\nSentence 2: {}\nQuestion: Do both sentences mean the same thing?\nAnswer:".format(
            general_detokenize(doc["sentence1"]),
            general_detokenize(doc["sentence2"]),
Jason Phang's avatar
Jason Phang committed
393
        )
394
395

    def doc_to_target(self, doc):
Jonathan Tow's avatar
Jonathan Tow committed
396
        return " {}".format(yesno(doc["label"]))
Jason Phang's avatar
Jason Phang committed
397

Jonathan Tow's avatar
Jonathan Tow committed
398
    def construct_requests(self, doc, ctx):
Leo Gao's avatar
Fix  
Leo Gao committed
399
400
        ll_yes, _ = rf.loglikelihood(ctx, " yes")
        ll_no, _ = rf.loglikelihood(ctx, " no")
Jonathan Tow's avatar
Jonathan Tow committed
401
        return ll_yes, ll_no
402

Jonathan Tow's avatar
Jonathan Tow committed
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
    def process_results(self, doc, results):
        ll_yes, ll_no = results
        gold = doc["label"]
        pred = ll_yes > ll_no
        return {
            "acc": pred == gold,
            "f1": (gold, pred),
        }

    def higher_is_better(self):
        return {
            "acc": True,
            "f1": True
        }

    def aggregation(self):
        return {
            "acc": mean,
            "f1": f1_score
        }
Jason Phang's avatar
Jason Phang committed
423
424


Jonathan Tow's avatar
Jonathan Tow committed
425
class QQP(Task):
Leo Gao's avatar
Leo Gao committed
426
    VERSION = 0
sdtblck's avatar
sdtblck committed
427
428
    DATASET_PATH = "glue"
    DATASET_NAME = "qqp"
Jason Phang's avatar
Jason Phang committed
429
430
431
432
433
434
435
436

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
Leo Gao's avatar
Leo Gao committed
437
        return False
Jason Phang's avatar
Jason Phang committed
438

Jonathan Tow's avatar
Jonathan Tow committed
439
440
441
442
443
444
445
446
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

    def validation_docs(self):
        return self.dataset["validation"]

447
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
448
        return "Question 1: {}\nQuestion 2: {}\nQuestion: Do both questions ask the same thing?\nAnswer:".format(
Jason Phang's avatar
Jason Phang committed
449
450
451
            doc["question1"],
            doc["question2"],
        )
452
453
454

    def doc_to_target(self, doc):
        return " {}".format(yesno(doc["label"]))
Jason Phang's avatar
Jason Phang committed
455

Jonathan Tow's avatar
Jonathan Tow committed
456
457
458
459
    def construct_requests(self, doc, ctx):
        ll_yes, _ = rf.loglikelihood(ctx, " yes")
        ll_no, _ = rf.loglikelihood(ctx, " no")
        return ll_yes, ll_no
460

Jonathan Tow's avatar
Jonathan Tow committed
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
    def process_results(self, doc, results):
        ll_yes, ll_no = results
        gold = doc["label"]
        pred = ll_yes > ll_no
        return {
            "acc": pred == gold,
            "f1": (gold, pred),
        }

    def higher_is_better(self):
        return {
            "acc": True,
            "f1": True
        }

    def aggregation(self):
        return {
            "acc": mean,
            "f1": f1_score
        }
Jason Phang's avatar
Jason Phang committed
481
482


Jonathan Tow's avatar
Jonathan Tow committed
483
class STSB(Task):
Leo Gao's avatar
Leo Gao committed
484
    VERSION = 0
sdtblck's avatar
sdtblck committed
485
486
    DATASET_PATH = "glue"
    DATASET_NAME = "stsb"
Jason Phang's avatar
Jason Phang committed
487
488
489
490
491
492
493
494
495
496

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
        return True

Jonathan Tow's avatar
Jonathan Tow committed
497
498
499
500
501
502
503
504
505
506
507
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

    def validation_docs(self):
        return self.dataset["validation"]

    def test_docs(self):
        return self.dataset["test"]

508
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
509
        return "sentence 1: {}\nsentence 2: {}\nAnswer:".format(
Jason Phang's avatar
Jason Phang committed
510
511
512
            doc["sentence1"],
            doc["sentence2"],
        )
513
514
515

    def doc_to_target(self, doc):
        return " {}".format(doc["label"])
Jason Phang's avatar
Jason Phang committed
516

Leo Gao's avatar
Leo Gao committed
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
    def construct_requests(self, doc, ctx):
        """ Uses RequestFactory to construct Requests and returns an iterable of 
        Requests which will be sent to the LM.

        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param ctx: str
            The context string, generated by fewshot_context. This includes the natural 
            language description, as well as the few shot examples, and the question
            part of the document for `doc`. 
        """
        # TODO: implement evaluation.
        raise NotImplementedError('Evaluation not implemented')
    
    def process_results(self, doc, results):
        """Take a single document and the LM results and evaluates, returning a 
        dict where keys are the names of submetrics and values are the values of 
        the metric for that one document

        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param results:
            The results of the requests created in construct_requests.
        """
        # TODO: implement evaluation.
        raise NotImplementedError('Evaluation not implemented')

    def aggregation(self):
        """
        :returns: {str: [float] -> float}
            A dictionary where keys are the names of submetrics and values are 
            functions that aggregate a list of metrics
        """
        # TODO: implement evaluation.
        raise NotImplementedError('Evaluation not implemented')

    def higher_is_better(self):
        """
        :returns: {str: bool}
            A dictionary where keys are the names of submetrics and values are 
            whether a higher value of the submetric is better
        """
        # TODO: implement evaluation.
        raise NotImplementedError('Evaluation not implemented')