glue.py 13.4 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
"""
GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding
https://openreview.net/pdf?id=rJ4km2R5t7

The General Language Understanding Evaluation (GLUE) benchmark is a collection of
resources for training, evaluating, and analyzing natural language understanding
systems. GLUE consists of:
- A benchmark of nine sentence- or sentence-pair language understanding tasks built
on established existing datasets and selected to cover a diverse range of dataset
sizes, text genres, and degrees of difficulty, and
- A diagnostic dataset designed to evaluate and analyze model performance with
respect to a wide range of linguistic phenomena found in natural language.

Homepage: https://gluebenchmark.com/
15
16
"""
import numpy as np
17
from lm_eval.base import PromptSourceTask, rf, Task
Jonathan Tow's avatar
Jonathan Tow committed
18
19
from lm_eval.metrics import mean, matthews_corrcoef, f1_score, yesno
from lm_eval.utils import general_detokenize
20

21

22
23
# TODO(jon-tow): Add citations for the individual datasets/tasks that make up GLUE.
_CITATION = """
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
@inproceedings{wang-etal-2018-glue,
    title = "{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding",
    author = "Wang, Alex  and
      Singh, Amanpreet  and
      Michael, Julian  and
      Hill, Felix  and
      Levy, Omer  and
      Bowman, Samuel",
    booktitle = "Proceedings of the 2018 {EMNLP} Workshop {B}lackbox{NLP}: Analyzing and Interpreting Neural Networks for {NLP}",
    month = nov,
    year = "2018",
    address = "Brussels, Belgium",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/W18-5446",
    doi = "10.18653/v1/W18-5446",
    pages = "353--355",
    abstract = "Human ability to understand language is \textit{general, flexible, and robust}. In contrast, most NLU models above the word level are designed for a specific task and struggle with out-of-domain data. If we aspire to develop models with understanding beyond the detection of superficial correspondences between inputs and outputs, then it is critical to develop a unified model that can execute a range of linguistic tasks across different domains. To facilitate research in this direction, we present the General Language Understanding Evaluation (GLUE, gluebenchmark.com): a benchmark of nine diverse NLU tasks, an auxiliary dataset for probing models for understanding of specific linguistic phenomena, and an online platform for evaluating and comparing models. For some benchmark tasks, training data is plentiful, but for others it is limited or does not match the genre of the test set. GLUE thus favors models that can represent linguistic knowledge in a way that facilitates sample-efficient learning and effective knowledge-transfer across tasks. While none of the datasets in GLUE were created from scratch for the benchmark, four of them feature privately-held test data, which is used to ensure that the benchmark is used fairly. We evaluate baselines that use ELMo (Peters et al., 2018), a powerful transfer learning technique, as well as state-of-the-art sentence representation models. The best models still achieve fairly low absolute scores. Analysis with our diagnostic dataset yields similarly weak performance over all phenomena tested, with some exceptions.",
}
"""
43

Jonathan Tow's avatar
Jonathan Tow committed
44
45

# Single-Sentence Tasks
Jason Phang's avatar
Jason Phang committed
46
47


jon-tow's avatar
jon-tow committed
48
class CoLA(PromptSourceTask):
Leo Gao's avatar
Leo Gao committed
49
    VERSION = 0
sdtblck's avatar
sdtblck committed
50
51
    DATASET_PATH = "glue"
    DATASET_NAME = "cola"
Jonathan Tow's avatar
Jonathan Tow committed
52

Jason Phang's avatar
checkin  
Jason Phang committed
53
54
55
56
57
58
59
    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
60
        return False
Jason Phang's avatar
checkin  
Jason Phang committed
61

Jonathan Tow's avatar
Jonathan Tow committed
62
63
64
65
66
67
68
69
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

    def validation_docs(self):
        return self.dataset["validation"]

jon-tow's avatar
jon-tow committed
70
71
72
73
74
75
76
77
78
79
80
    def process_results(self, doc, results): 
        answer_choices_list = self.prompt.get_answer_choices_list(doc)
        pred = np.argmax(results)
        target = answer_choices_list.index(self.doc_to_target(doc).strip())
        print("*" * 80)
        print(f"DOC: {doc}")
        print(f"TEXT: {self.doc_to_text(doc)}")
        print(f"STRING TARGET: {self.doc_to_target(doc)} END TARGET")
        print(f"TARGET: {target} END TARGET")
        print(f"PRED: {pred}")
        print("*" * 80)
81

Jonathan Tow's avatar
Jonathan Tow committed
82
        return {
jon-tow's avatar
jon-tow committed
83
            "mcc": (target, pred)
Jonathan Tow's avatar
Jonathan Tow committed
84
        }
85

Jonathan Tow's avatar
Jonathan Tow committed
86
    def higher_is_better(self):
Jason Phang's avatar
checkin  
Jason Phang committed
87
        return {
Jonathan Tow's avatar
Jonathan Tow committed
88
89
90
91
92
93
94
95
96
            "mcc": True
        }

    def aggregation(self):
        return {
            "mcc": matthews_corrcoef
        }


jon-tow's avatar
jon-tow committed
97
class SST(PromptSourceTask):
Leo Gao's avatar
Leo Gao committed
98
    VERSION = 0
Jonathan Tow's avatar
Jonathan Tow committed
99
100
101
102
103
104
105
106
107
108
    DATASET_PATH = "glue"
    DATASET_NAME = "sst2"

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
109
        return False
Jonathan Tow's avatar
Jonathan Tow committed
110

Jonathan Tow's avatar
Jonathan Tow committed
111
112
113
114
115
116
117
118
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

    def validation_docs(self):
        return self.dataset["validation"]

Jonathan Tow's avatar
Jonathan Tow committed
119
120
121
    def higher_is_better(self):
        return {
            "acc": True
Jason Phang's avatar
checkin  
Jason Phang committed
122
123
        }

Jonathan Tow's avatar
Jonathan Tow committed
124
125
126
127
128
129
130
131
    def aggregation(self):
        return {
            "acc": mean
        }


# Inference Tasks

Jason Phang's avatar
checkin  
Jason Phang committed
132

jon-tow's avatar
jon-tow committed
133
class MNLI(PromptSourceTask):
Leo Gao's avatar
Leo Gao committed
134
    VERSION = 0
sdtblck's avatar
sdtblck committed
135
136
    DATASET_PATH = "glue"
    DATASET_NAME = "mnli"
Jason Phang's avatar
Jason Phang committed
137

Jason Phang's avatar
checkin  
Jason Phang committed
138
139
140
141
142
143
144
    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
145
        return False
Jason Phang's avatar
checkin  
Jason Phang committed
146

Jonathan Tow's avatar
Jonathan Tow committed
147
148
149
150
151
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

Jason Phang's avatar
checkin  
Jason Phang committed
152
153
    def validation_docs(self):
        if self.has_validation_docs():
Jonathan Tow's avatar
Jonathan Tow committed
154
            return self.dataset["validation_matched"]
Jason Phang's avatar
checkin  
Jason Phang committed
155
156
157

    def test_docs(self):
        if self.has_test_docs():
Jonathan Tow's avatar
Jonathan Tow committed
158
            return self.dataset["test_matched"]
Jason Phang's avatar
checkin  
Jason Phang committed
159

Jonathan Tow's avatar
Jonathan Tow committed
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
    def process_results(self, doc, results):
        gold = doc["label"]
        pred = np.argmax(results)
        return {
            "acc": pred == gold
        }

    def higher_is_better(self):
        return {
            "acc": True
        }

    def aggregation(self):
        return {
            "acc": mean
        }
Jason Phang's avatar
checkin  
Jason Phang committed
176
177


Jason Phang's avatar
Jason Phang committed
178
class MNLIMismatched(MNLI):
Leo Gao's avatar
Leo Gao committed
179
    VERSION = 0
Jason Phang's avatar
Jason Phang committed
180
181
182

    def validation_docs(self):
        if self.has_validation_docs():
Jonathan Tow's avatar
Jonathan Tow committed
183
            return self.dataset["validation_mismatched"]
Jason Phang's avatar
Jason Phang committed
184
185
186

    def test_docs(self):
        if self.has_test_docs():
Jonathan Tow's avatar
Jonathan Tow committed
187
            return self.dataset["test_mismatched"]
Jason Phang's avatar
Jason Phang committed
188
189


Jonathan Tow's avatar
Jonathan Tow committed
190
class QNLI(Task):
Leo Gao's avatar
Leo Gao committed
191
    VERSION = 0
sdtblck's avatar
sdtblck committed
192
    DATASET_PATH = "glue"
Jonathan Tow's avatar
Jonathan Tow committed
193
    DATASET_NAME = "qnli"
Jason Phang's avatar
Jason Phang committed
194
195
196
197
198
199
200
201

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
202
        return False
Jason Phang's avatar
Jason Phang committed
203

Jonathan Tow's avatar
Jonathan Tow committed
204
205
206
207
208
209
210
211
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

    def validation_docs(self):
        return self.dataset["validation"]

Jonathan Tow's avatar
Jonathan Tow committed
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
    def process_results(self, doc, results):
        ll_yes, ll_no = results
        pred = ll_no > ll_yes
        gold = doc["label"]
        return {
            "acc": pred == gold
        }

    def higher_is_better(self):
        return {
            "acc": True
        }

    def aggregation(self):
        return {
            "acc": mean
        }


231
class WNLI(PromptSourceTask):
thomasw21's avatar
thomasw21 committed
232
    VERSION = 1
Jonathan Tow's avatar
Jonathan Tow committed
233
234
235
236
237
238
239
240
241
242
    DATASET_PATH = "glue"
    DATASET_NAME = "wnli"

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
243
        return False
Jason Phang's avatar
Jason Phang committed
244

Jonathan Tow's avatar
Jonathan Tow committed
245
    def training_docs(self):
246
247
248
249
        # if self._training_docs is None:
        #     self._training_docs = list()
        # return self._training_docs
        return self.dataset["train"]
Jonathan Tow's avatar
Jonathan Tow committed
250
251
252
253

    def validation_docs(self):
        return self.dataset["validation"]

Jonathan Tow's avatar
Jonathan Tow committed
254
255
256
257
258
259
260
261
262
    def higher_is_better(self):
        return {
            "acc": True
        }

    def aggregation(self):
        return {
            "acc": mean
        }
263

Jason Phang's avatar
Jason Phang committed
264

265
class RTE(PromptSourceTask):
Leo Gao's avatar
Leo Gao committed
266
    VERSION = 0
sdtblck's avatar
sdtblck committed
267
268
    DATASET_PATH = "glue"
    DATASET_NAME = "rte"
Jason Phang's avatar
checkin  
Jason Phang committed
269
270
271
272
273
274
275
276

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
277
        return False
Jason Phang's avatar
checkin  
Jason Phang committed
278

Jonathan Tow's avatar
Jonathan Tow committed
279
280
281
282
283
284
285
286
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

    def validation_docs(self):
        return self.dataset["validation"]

Jonathan Tow's avatar
Jonathan Tow committed
287
288
289
290
    def higher_is_better(self):
        return {
            "acc": True
        }
Jason Phang's avatar
Jason Phang committed
291

Jonathan Tow's avatar
Jonathan Tow committed
292
293
294
295
    def aggregation(self):
        return {
            "acc": mean
        }
Jason Phang's avatar
Jason Phang committed
296

Jonathan Tow's avatar
Jonathan Tow committed
297
298
299
300

# Similarity and Paraphrase Tasks


Jonathan Tow's avatar
Jonathan Tow committed
301
class MRPC(Task):
Leo Gao's avatar
Leo Gao committed
302
    VERSION = 0
sdtblck's avatar
sdtblck committed
303
    DATASET_PATH = "glue"
Jonathan Tow's avatar
Jonathan Tow committed
304
    DATASET_NAME = "mrpc"
Jason Phang's avatar
Jason Phang committed
305
306
307
308
309
310
311
312

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
Leo Gao's avatar
Leo Gao committed
313
        return False
Jason Phang's avatar
Jason Phang committed
314

Jonathan Tow's avatar
Jonathan Tow committed
315
316
317
318
319
320
321
322
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

    def validation_docs(self):
        return self.dataset["validation"]

Jonathan Tow's avatar
Jonathan Tow committed
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
    def process_results(self, doc, results):
        ll_yes, ll_no = results
        gold = doc["label"]
        pred = ll_yes > ll_no
        return {
            "acc": pred == gold,
            "f1": (gold, pred),
        }

    def higher_is_better(self):
        return {
            "acc": True,
            "f1": True
        }

    def aggregation(self):
        return {
            "acc": mean,
            "f1": f1_score
        }
Jason Phang's avatar
Jason Phang committed
343
344


Jonathan Tow's avatar
Jonathan Tow committed
345
class QQP(Task):
Leo Gao's avatar
Leo Gao committed
346
    VERSION = 0
sdtblck's avatar
sdtblck committed
347
348
    DATASET_PATH = "glue"
    DATASET_NAME = "qqp"
Jason Phang's avatar
Jason Phang committed
349
350
351
352
353
354
355
356

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
Leo Gao's avatar
Leo Gao committed
357
        return False
Jason Phang's avatar
Jason Phang committed
358

Jonathan Tow's avatar
Jonathan Tow committed
359
360
361
362
363
364
365
366
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

    def validation_docs(self):
        return self.dataset["validation"]

367
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
368
        return "Question 1: {}\nQuestion 2: {}\nQuestion: Do both questions ask the same thing?\nAnswer:".format(
Jason Phang's avatar
Jason Phang committed
369
370
371
            doc["question1"],
            doc["question2"],
        )
372
373
374

    def doc_to_target(self, doc):
        return " {}".format(yesno(doc["label"]))
Jason Phang's avatar
Jason Phang committed
375

Jonathan Tow's avatar
Jonathan Tow committed
376
377
378
379
    def construct_requests(self, doc, ctx):
        ll_yes, _ = rf.loglikelihood(ctx, " yes")
        ll_no, _ = rf.loglikelihood(ctx, " no")
        return ll_yes, ll_no
380

Jonathan Tow's avatar
Jonathan Tow committed
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
    def process_results(self, doc, results):
        ll_yes, ll_no = results
        gold = doc["label"]
        pred = ll_yes > ll_no
        return {
            "acc": pred == gold,
            "f1": (gold, pred),
        }

    def higher_is_better(self):
        return {
            "acc": True,
            "f1": True
        }

    def aggregation(self):
        return {
            "acc": mean,
            "f1": f1_score
        }
Jason Phang's avatar
Jason Phang committed
401
402


Jonathan Tow's avatar
Jonathan Tow committed
403
class STSB(Task):
Leo Gao's avatar
Leo Gao committed
404
    VERSION = 0
sdtblck's avatar
sdtblck committed
405
406
    DATASET_PATH = "glue"
    DATASET_NAME = "stsb"
Jason Phang's avatar
Jason Phang committed
407
408
409
410
411
412
413
414
415
416

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
        return True

Jonathan Tow's avatar
Jonathan Tow committed
417
418
419
420
421
422
423
424
425
426
427
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

    def validation_docs(self):
        return self.dataset["validation"]

    def test_docs(self):
        return self.dataset["test"]

428
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
429
        return "sentence 1: {}\nsentence 2: {}\nAnswer:".format(
Jason Phang's avatar
Jason Phang committed
430
431
432
            doc["sentence1"],
            doc["sentence2"],
        )
433
434
435

    def doc_to_target(self, doc):
        return " {}".format(doc["label"])
Jason Phang's avatar
Jason Phang committed
436

Leo Gao's avatar
Leo Gao committed
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
    def construct_requests(self, doc, ctx):
        """ Uses RequestFactory to construct Requests and returns an iterable of 
        Requests which will be sent to the LM.

        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param ctx: str
            The context string, generated by fewshot_context. This includes the natural 
            language description, as well as the few shot examples, and the question
            part of the document for `doc`. 
        """
        # TODO: implement evaluation.
        raise NotImplementedError('Evaluation not implemented')
    
    def process_results(self, doc, results):
        """Take a single document and the LM results and evaluates, returning a 
        dict where keys are the names of submetrics and values are the values of 
        the metric for that one document

        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param results:
            The results of the requests created in construct_requests.
        """
        # TODO: implement evaluation.
        raise NotImplementedError('Evaluation not implemented')

    def aggregation(self):
        """
        :returns: {str: [float] -> float}
            A dictionary where keys are the names of submetrics and values are 
            functions that aggregate a list of metrics
        """
        # TODO: implement evaluation.
        raise NotImplementedError('Evaluation not implemented')

    def higher_is_better(self):
        """
        :returns: {str: bool}
            A dictionary where keys are the names of submetrics and values are 
            whether a higher value of the submetric is better
        """
        # TODO: implement evaluation.
        raise NotImplementedError('Evaluation not implemented')