glue.py 12.3 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
"""
GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding
https://openreview.net/pdf?id=rJ4km2R5t7

The General Language Understanding Evaluation (GLUE) benchmark is a collection of
resources for training, evaluating, and analyzing natural language understanding
systems. GLUE consists of:
- A benchmark of nine sentence- or sentence-pair language understanding tasks built
on established existing datasets and selected to cover a diverse range of dataset
sizes, text genres, and degrees of difficulty, and
- A diagnostic dataset designed to evaluate and analyze model performance with
respect to a wide range of linguistic phenomena found in natural language.

Homepage: https://gluebenchmark.com/
15
16
"""
import numpy as np
17
from lm_eval.base import PromptSourceTask, rf, Task
Jonathan Tow's avatar
Jonathan Tow committed
18
19
from lm_eval.metrics import mean, matthews_corrcoef, f1_score, yesno
from lm_eval.utils import general_detokenize
20

21

22
23
# TODO(jon-tow): Add citations for the individual datasets/tasks that make up GLUE.
_CITATION = """
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
@inproceedings{wang-etal-2018-glue,
    title = "{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding",
    author = "Wang, Alex  and
      Singh, Amanpreet  and
      Michael, Julian  and
      Hill, Felix  and
      Levy, Omer  and
      Bowman, Samuel",
    booktitle = "Proceedings of the 2018 {EMNLP} Workshop {B}lackbox{NLP}: Analyzing and Interpreting Neural Networks for {NLP}",
    month = nov,
    year = "2018",
    address = "Brussels, Belgium",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/W18-5446",
    doi = "10.18653/v1/W18-5446",
    pages = "353--355",
    abstract = "Human ability to understand language is \textit{general, flexible, and robust}. In contrast, most NLU models above the word level are designed for a specific task and struggle with out-of-domain data. If we aspire to develop models with understanding beyond the detection of superficial correspondences between inputs and outputs, then it is critical to develop a unified model that can execute a range of linguistic tasks across different domains. To facilitate research in this direction, we present the General Language Understanding Evaluation (GLUE, gluebenchmark.com): a benchmark of nine diverse NLU tasks, an auxiliary dataset for probing models for understanding of specific linguistic phenomena, and an online platform for evaluating and comparing models. For some benchmark tasks, training data is plentiful, but for others it is limited or does not match the genre of the test set. GLUE thus favors models that can represent linguistic knowledge in a way that facilitates sample-efficient learning and effective knowledge-transfer across tasks. While none of the datasets in GLUE were created from scratch for the benchmark, four of them feature privately-held test data, which is used to ensure that the benchmark is used fairly. We evaluate baselines that use ELMo (Peters et al., 2018), a powerful transfer learning technique, as well as state-of-the-art sentence representation models. The best models still achieve fairly low absolute scores. Analysis with our diagnostic dataset yields similarly weak performance over all phenomena tested, with some exceptions.",
}
"""
43

Jonathan Tow's avatar
Jonathan Tow committed
44
45

# Single-Sentence Tasks
Jason Phang's avatar
Jason Phang committed
46
47


jon-tow's avatar
jon-tow committed
48
class CoLA(PromptSourceTask):
Leo Gao's avatar
Leo Gao committed
49
    VERSION = 0
sdtblck's avatar
sdtblck committed
50
51
    DATASET_PATH = "glue"
    DATASET_NAME = "cola"
Jonathan Tow's avatar
Jonathan Tow committed
52

Jason Phang's avatar
checkin  
Jason Phang committed
53
54
55
56
57
58
59
    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
60
        return False
Jason Phang's avatar
checkin  
Jason Phang committed
61

Jonathan Tow's avatar
Jonathan Tow committed
62
63
64
65
66
67
68
69
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

    def validation_docs(self):
        return self.dataset["validation"]

70
    def process_results(self, doc, results):
jon-tow's avatar
jon-tow committed
71
72
73
        answer_choices_list = self.prompt.get_answer_choices_list(doc)
        pred = np.argmax(results)
        target = answer_choices_list.index(self.doc_to_target(doc).strip())
74
        return {"mcc": (target, pred)}
75

Jonathan Tow's avatar
Jonathan Tow committed
76
    def higher_is_better(self):
77
        return {"mcc": True}
Jonathan Tow's avatar
Jonathan Tow committed
78
79

    def aggregation(self):
80
        return {"mcc": matthews_corrcoef}
Jonathan Tow's avatar
Jonathan Tow committed
81
82


jon-tow's avatar
jon-tow committed
83
class SST(PromptSourceTask):
Leo Gao's avatar
Leo Gao committed
84
    VERSION = 0
Jonathan Tow's avatar
Jonathan Tow committed
85
86
87
88
89
90
91
92
93
94
    DATASET_PATH = "glue"
    DATASET_NAME = "sst2"

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
95
        return False
Jonathan Tow's avatar
Jonathan Tow committed
96

Jonathan Tow's avatar
Jonathan Tow committed
97
98
99
100
101
102
103
104
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

    def validation_docs(self):
        return self.dataset["validation"]

Jonathan Tow's avatar
Jonathan Tow committed
105
106
107

# Inference Tasks

Jason Phang's avatar
checkin  
Jason Phang committed
108

jon-tow's avatar
jon-tow committed
109
class MNLI(PromptSourceTask):
Leo Gao's avatar
Leo Gao committed
110
    VERSION = 0
sdtblck's avatar
sdtblck committed
111
112
    DATASET_PATH = "glue"
    DATASET_NAME = "mnli"
Jason Phang's avatar
Jason Phang committed
113

Jason Phang's avatar
checkin  
Jason Phang committed
114
115
116
117
118
119
120
    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
121
        return False
Jason Phang's avatar
checkin  
Jason Phang committed
122

Jonathan Tow's avatar
Jonathan Tow committed
123
124
125
126
127
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

Jason Phang's avatar
checkin  
Jason Phang committed
128
129
    def validation_docs(self):
        if self.has_validation_docs():
Jonathan Tow's avatar
Jonathan Tow committed
130
            return self.dataset["validation_matched"]
Jason Phang's avatar
checkin  
Jason Phang committed
131
132
133

    def test_docs(self):
        if self.has_test_docs():
Jonathan Tow's avatar
Jonathan Tow committed
134
            return self.dataset["test_matched"]
Jason Phang's avatar
checkin  
Jason Phang committed
135
136


Jason Phang's avatar
Jason Phang committed
137
class MNLIMismatched(MNLI):
Leo Gao's avatar
Leo Gao committed
138
    VERSION = 0
Jason Phang's avatar
Jason Phang committed
139
140
141

    def validation_docs(self):
        if self.has_validation_docs():
Jonathan Tow's avatar
Jonathan Tow committed
142
            return self.dataset["validation_mismatched"]
Jason Phang's avatar
Jason Phang committed
143
144
145

    def test_docs(self):
        if self.has_test_docs():
Jonathan Tow's avatar
Jonathan Tow committed
146
            return self.dataset["test_mismatched"]
Jason Phang's avatar
Jason Phang committed
147
148


Jonathan Tow's avatar
Jonathan Tow committed
149
class QNLI(Task):
Leo Gao's avatar
Leo Gao committed
150
    VERSION = 0
sdtblck's avatar
sdtblck committed
151
    DATASET_PATH = "glue"
Jonathan Tow's avatar
Jonathan Tow committed
152
    DATASET_NAME = "qnli"
Jason Phang's avatar
Jason Phang committed
153
154
155
156
157
158
159
160

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
161
        return False
Jason Phang's avatar
Jason Phang committed
162

Jonathan Tow's avatar
Jonathan Tow committed
163
164
165
166
167
168
169
170
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

    def validation_docs(self):
        return self.dataset["validation"]

Jonathan Tow's avatar
Jonathan Tow committed
171
172
173
174
    def process_results(self, doc, results):
        ll_yes, ll_no = results
        pred = ll_no > ll_yes
        gold = doc["label"]
175
        return {"acc": pred == gold}
Jonathan Tow's avatar
Jonathan Tow committed
176
177

    def higher_is_better(self):
178
        return {"acc": True}
Jonathan Tow's avatar
Jonathan Tow committed
179
180

    def aggregation(self):
181
        return {"acc": mean}
Jonathan Tow's avatar
Jonathan Tow committed
182
183


184
class WNLI(PromptSourceTask):
thomasw21's avatar
thomasw21 committed
185
    VERSION = 1
Jonathan Tow's avatar
Jonathan Tow committed
186
187
188
189
190
191
192
193
194
195
    DATASET_PATH = "glue"
    DATASET_NAME = "wnli"

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
196
        return False
Jason Phang's avatar
Jason Phang committed
197

Jonathan Tow's avatar
Jonathan Tow committed
198
    def training_docs(self):
199
200
201
202
        # if self._training_docs is None:
        #     self._training_docs = list()
        # return self._training_docs
        return self.dataset["train"]
Jonathan Tow's avatar
Jonathan Tow committed
203
204
205
206

    def validation_docs(self):
        return self.dataset["validation"]

Jonathan Tow's avatar
Jonathan Tow committed
207
    def higher_is_better(self):
208
        return {"acc": True}
Jonathan Tow's avatar
Jonathan Tow committed
209
210

    def aggregation(self):
211
        return {"acc": mean}
212

Jason Phang's avatar
Jason Phang committed
213

214
class RTE(PromptSourceTask):
Leo Gao's avatar
Leo Gao committed
215
    VERSION = 0
sdtblck's avatar
sdtblck committed
216
217
    DATASET_PATH = "glue"
    DATASET_NAME = "rte"
Jason Phang's avatar
checkin  
Jason Phang committed
218
219
220
221
222
223
224
225

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
226
        return False
Jason Phang's avatar
checkin  
Jason Phang committed
227

Jonathan Tow's avatar
Jonathan Tow committed
228
229
230
231
232
233
234
235
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

    def validation_docs(self):
        return self.dataset["validation"]

Jonathan Tow's avatar
Jonathan Tow committed
236
    def higher_is_better(self):
237
        return {"acc": True}
Jason Phang's avatar
Jason Phang committed
238

Jonathan Tow's avatar
Jonathan Tow committed
239
    def aggregation(self):
240
        return {"acc": mean}
Jason Phang's avatar
Jason Phang committed
241

Jonathan Tow's avatar
Jonathan Tow committed
242
243
244
245

# Similarity and Paraphrase Tasks


Jonathan Tow's avatar
Jonathan Tow committed
246
class MRPC(Task):
Leo Gao's avatar
Leo Gao committed
247
    VERSION = 0
sdtblck's avatar
sdtblck committed
248
    DATASET_PATH = "glue"
Jonathan Tow's avatar
Jonathan Tow committed
249
    DATASET_NAME = "mrpc"
Jason Phang's avatar
Jason Phang committed
250
251
252
253
254
255
256
257

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
Leo Gao's avatar
Leo Gao committed
258
        return False
Jason Phang's avatar
Jason Phang committed
259

Jonathan Tow's avatar
Jonathan Tow committed
260
261
262
263
264
265
266
267
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

    def validation_docs(self):
        return self.dataset["validation"]

Jonathan Tow's avatar
Jonathan Tow committed
268
269
270
271
272
273
274
275
276
277
    def process_results(self, doc, results):
        ll_yes, ll_no = results
        gold = doc["label"]
        pred = ll_yes > ll_no
        return {
            "acc": pred == gold,
            "f1": (gold, pred),
        }

    def higher_is_better(self):
278
        return {"acc": True, "f1": True}
Jonathan Tow's avatar
Jonathan Tow committed
279
280

    def aggregation(self):
281
        return {"acc": mean, "f1": f1_score}
Jason Phang's avatar
Jason Phang committed
282
283


Jonathan Tow's avatar
Jonathan Tow committed
284
class QQP(Task):
Leo Gao's avatar
Leo Gao committed
285
    VERSION = 0
sdtblck's avatar
sdtblck committed
286
287
    DATASET_PATH = "glue"
    DATASET_NAME = "qqp"
Jason Phang's avatar
Jason Phang committed
288
289
290
291
292
293
294
295

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
Leo Gao's avatar
Leo Gao committed
296
        return False
Jason Phang's avatar
Jason Phang committed
297

Jonathan Tow's avatar
Jonathan Tow committed
298
299
300
301
302
303
304
305
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

    def validation_docs(self):
        return self.dataset["validation"]

306
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
307
        return "Question 1: {}\nQuestion 2: {}\nQuestion: Do both questions ask the same thing?\nAnswer:".format(
Jason Phang's avatar
Jason Phang committed
308
309
310
            doc["question1"],
            doc["question2"],
        )
311
312
313

    def doc_to_target(self, doc):
        return " {}".format(yesno(doc["label"]))
Jason Phang's avatar
Jason Phang committed
314

Jonathan Tow's avatar
Jonathan Tow committed
315
316
317
318
    def construct_requests(self, doc, ctx):
        ll_yes, _ = rf.loglikelihood(ctx, " yes")
        ll_no, _ = rf.loglikelihood(ctx, " no")
        return ll_yes, ll_no
319

Jonathan Tow's avatar
Jonathan Tow committed
320
321
322
323
324
325
326
327
328
329
    def process_results(self, doc, results):
        ll_yes, ll_no = results
        gold = doc["label"]
        pred = ll_yes > ll_no
        return {
            "acc": pred == gold,
            "f1": (gold, pred),
        }

    def higher_is_better(self):
330
        return {"acc": True, "f1": True}
Jonathan Tow's avatar
Jonathan Tow committed
331
332

    def aggregation(self):
333
        return {"acc": mean, "f1": f1_score}
Jason Phang's avatar
Jason Phang committed
334
335


Jonathan Tow's avatar
Jonathan Tow committed
336
class STSB(Task):
Leo Gao's avatar
Leo Gao committed
337
    VERSION = 0
sdtblck's avatar
sdtblck committed
338
339
    DATASET_PATH = "glue"
    DATASET_NAME = "stsb"
Jason Phang's avatar
Jason Phang committed
340
341
342
343
344
345
346
347
348
349

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
        return True

Jonathan Tow's avatar
Jonathan Tow committed
350
351
352
353
354
355
356
357
358
359
360
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

    def validation_docs(self):
        return self.dataset["validation"]

    def test_docs(self):
        return self.dataset["test"]

361
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
362
        return "sentence 1: {}\nsentence 2: {}\nAnswer:".format(
Jason Phang's avatar
Jason Phang committed
363
364
365
            doc["sentence1"],
            doc["sentence2"],
        )
366
367
368

    def doc_to_target(self, doc):
        return " {}".format(doc["label"])
Jason Phang's avatar
Jason Phang committed
369

Leo Gao's avatar
Leo Gao committed
370
    def construct_requests(self, doc, ctx):
371
        """Uses RequestFactory to construct Requests and returns an iterable of
Leo Gao's avatar
Leo Gao committed
372
373
374
375
376
        Requests which will be sent to the LM.

        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param ctx: str
377
            The context string, generated by fewshot_context. This includes the natural
Leo Gao's avatar
Leo Gao committed
378
            language description, as well as the few shot examples, and the question
379
            part of the document for `doc`.
Leo Gao's avatar
Leo Gao committed
380
381
        """
        # TODO: implement evaluation.
382
383
        raise NotImplementedError("Evaluation not implemented")

Leo Gao's avatar
Leo Gao committed
384
    def process_results(self, doc, results):
385
386
        """Take a single document and the LM results and evaluates, returning a
        dict where keys are the names of submetrics and values are the values of
Leo Gao's avatar
Leo Gao committed
387
388
389
390
391
392
393
394
        the metric for that one document

        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param results:
            The results of the requests created in construct_requests.
        """
        # TODO: implement evaluation.
395
        raise NotImplementedError("Evaluation not implemented")
Leo Gao's avatar
Leo Gao committed
396
397
398
399

    def aggregation(self):
        """
        :returns: {str: [float] -> float}
400
            A dictionary where keys are the names of submetrics and values are
Leo Gao's avatar
Leo Gao committed
401
402
403
            functions that aggregate a list of metrics
        """
        # TODO: implement evaluation.
404
        raise NotImplementedError("Evaluation not implemented")
Leo Gao's avatar
Leo Gao committed
405
406
407
408

    def higher_is_better(self):
        """
        :returns: {str: bool}
409
            A dictionary where keys are the names of submetrics and values are
Leo Gao's avatar
Leo Gao committed
410
411
412
            whether a higher value of the submetric is better
        """
        # TODO: implement evaluation.
413
        raise NotImplementedError("Evaluation not implemented")