glue.py 12 KB
Newer Older
Jason Phang's avatar
checkin  
Jason Phang committed
1
import numpy as np
&'s avatar
& committed
2
3
from lm_eval.base import rf
from ..metrics import mean, matthews_corrcoef, f1_score
Jonathan Tow's avatar
Jonathan Tow committed
4
from . common import HFTask, yesno
Leo Gao's avatar
Leo Gao committed
5
from ..utils import general_detokenize
Jonathan Tow's avatar
Jonathan Tow committed
6
7

# Single-Sentence Tasks
Jason Phang's avatar
Jason Phang committed
8
9


sdtblck's avatar
sdtblck committed
10
class CoLA(HFTask):
Leo Gao's avatar
Leo Gao committed
11
    VERSION = 0
sdtblck's avatar
sdtblck committed
12
13
    DATASET_PATH = "glue"
    DATASET_NAME = "cola"
Jonathan Tow's avatar
Jonathan Tow committed
14

Jason Phang's avatar
checkin  
Jason Phang committed
15
16
17
18
19
20
21
    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
22
        return False
Jason Phang's avatar
checkin  
Jason Phang committed
23

24
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
25
        return "{}\nQuestion: Does this sentence make sense?\nAnswer:".format(doc["sentence"])
26
27

    def doc_to_target(self, doc):
Leo Gao's avatar
Leo Gao committed
28
        return " {}".format({1: "yes", 0: "no"}[doc["label"]])
Jason Phang's avatar
checkin  
Jason Phang committed
29

Jonathan Tow's avatar
Jonathan Tow committed
30
    def construct_requests(self, doc, ctx):
Leo Gao's avatar
Leo Gao committed
31
32
        ll_true, _ = rf.loglikelihood(ctx, " yes")
        ll_false, _ = rf.loglikelihood(ctx, " no")
Jonathan Tow's avatar
Jonathan Tow committed
33
        return ll_true, ll_false
34

Jonathan Tow's avatar
Jonathan Tow committed
35
36
37
38
39
40
41
    def process_results(self, doc, results):
        ll_true, ll_false = results
        pred = ll_true > ll_false
        gold = doc["label"]
        return {
            "mcc": (gold, pred)
        }
42

Jonathan Tow's avatar
Jonathan Tow committed
43
    def higher_is_better(self):
Jason Phang's avatar
checkin  
Jason Phang committed
44
        return {
Jonathan Tow's avatar
Jonathan Tow committed
45
46
47
48
49
50
51
52
53
54
            "mcc": True
        }

    def aggregation(self):
        return {
            "mcc": matthews_corrcoef
        }


class SST(HFTask):
Leo Gao's avatar
Leo Gao committed
55
    VERSION = 0
Jonathan Tow's avatar
Jonathan Tow committed
56
57
58
59
60
61
62
63
64
65
    DATASET_PATH = "glue"
    DATASET_NAME = "sst2"

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
66
        return False
Jonathan Tow's avatar
Jonathan Tow committed
67
68

    def doc_to_text(self, doc):
Leo Gao's avatar
Fix  
Leo Gao committed
69
        return "{}\nQuestion: Is this sentence positive or negative?\nAnswer:".format(
Leo Gao's avatar
Leo Gao committed
70
            general_detokenize(doc["sentence"]),
Jonathan Tow's avatar
Jonathan Tow committed
71
72
73
        )

    def doc_to_target(self, doc):
Leo Gao's avatar
Fix  
Leo Gao committed
74
        return " {}".format({1: "positive", 0: "negative"}[doc["label"]])
Jonathan Tow's avatar
Jonathan Tow committed
75
76

    def construct_requests(self, doc, ctx):
Leo Gao's avatar
Fix  
Leo Gao committed
77
78
        ll_positive, _ = rf.loglikelihood(ctx, " positive")
        ll_negative, _ = rf.loglikelihood(ctx, " negative")
Jonathan Tow's avatar
Jonathan Tow committed
79
80
81
82
83
84
85
86
87
88
89
90
91
        return ll_positive, ll_negative

    def process_results(self, doc, results):
        ll_positive, ll_negative = results
        pred = ll_positive > ll_negative
        gold = doc["label"]
        return {
            "acc": pred == gold
        }

    def higher_is_better(self):
        return {
            "acc": True
Jason Phang's avatar
checkin  
Jason Phang committed
92
93
        }

Jonathan Tow's avatar
Jonathan Tow committed
94
95
96
97
98
99
100
101
    def aggregation(self):
        return {
            "acc": mean
        }


# Inference Tasks

Jason Phang's avatar
checkin  
Jason Phang committed
102

sdtblck's avatar
sdtblck committed
103
class MNLI(HFTask):
Leo Gao's avatar
Leo Gao committed
104
    VERSION = 0
sdtblck's avatar
sdtblck committed
105
106
    DATASET_PATH = "glue"
    DATASET_NAME = "mnli"
Jason Phang's avatar
Jason Phang committed
107

Jason Phang's avatar
checkin  
Jason Phang committed
108
109
110
111
112
113
114
    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
115
        return False
Jason Phang's avatar
checkin  
Jason Phang committed
116
117
118

    def validation_docs(self):
        if self.has_validation_docs():
sdtblck's avatar
sdtblck committed
119
            return self.data["validation_matched"]
Jason Phang's avatar
checkin  
Jason Phang committed
120
121
122

    def test_docs(self):
        if self.has_test_docs():
sdtblck's avatar
sdtblck committed
123
            return self.data["test_matched"]
Jason Phang's avatar
checkin  
Jason Phang committed
124

125
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
126
        return "{}\nQuestion: {} True, False or Neither?\nAnswer:".format(
Jason Phang's avatar
Jason Phang committed
127
            doc["premise"],
Leo Gao's avatar
Fix  
Leo Gao committed
128
            doc["hypothesis"].strip() + ('' if doc["hypothesis"].strip().endswith('.') else '.'),
Jason Phang's avatar
checkin  
Jason Phang committed
129
        )
130
131
132
133
134
135

    def doc_to_target(self, doc):
        # True = entailment
        # False = contradiction
        # Neither = neutral
        return " {}".format({0: "True", 1: "Neither", 2: "False"}[doc["label"]])
Jason Phang's avatar
checkin  
Jason Phang committed
136

Jonathan Tow's avatar
Jonathan Tow committed
137
138
139
140
141
    def construct_requests(self, doc, ctx):
        ll_true, _ = rf.loglikelihood(ctx, " True")
        ll_neither, _ = rf.loglikelihood(ctx, " Neither")
        ll_false, _ = rf.loglikelihood(ctx, " False")
        return ll_true, ll_neither, ll_false
142

Jonathan Tow's avatar
Jonathan Tow committed
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
    def process_results(self, doc, results):
        gold = doc["label"]
        pred = np.argmax(results)
        return {
            "acc": pred == gold
        }

    def higher_is_better(self):
        return {
            "acc": True
        }

    def aggregation(self):
        return {
            "acc": mean
        }
Jason Phang's avatar
checkin  
Jason Phang committed
159
160


Jason Phang's avatar
Jason Phang committed
161
class MNLIMismatched(MNLI):
Leo Gao's avatar
Leo Gao committed
162
    VERSION = 0
Jason Phang's avatar
Jason Phang committed
163
164
165
166
167
168
169
170
171
172

    def validation_docs(self):
        if self.has_validation_docs():
            return self.data["validation_mismatched"]

    def test_docs(self):
        if self.has_test_docs():
            return self.data["test_mismatched"]


Jonathan Tow's avatar
Jonathan Tow committed
173
class QNLI(HFTask):
Leo Gao's avatar
Leo Gao committed
174
    VERSION = 0
sdtblck's avatar
sdtblck committed
175
    DATASET_PATH = "glue"
Jonathan Tow's avatar
Jonathan Tow committed
176
    DATASET_NAME = "qnli"
Jason Phang's avatar
Jason Phang committed
177
178
179
180
181
182
183
184

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
185
        return False
Jason Phang's avatar
Jason Phang committed
186

Jonathan Tow's avatar
Jonathan Tow committed
187
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
188
        return "{}\n{}\nQuestion: Does this response answer the question?\nAnswer:".format(
Jonathan Tow's avatar
Jonathan Tow committed
189
190
191
192
193
194
195
            doc["question"],
            doc["sentence"],
        )

    def doc_to_target(self, doc):
        # True = entailment
        # False = not entailment
Leo Gao's avatar
Fix  
Leo Gao committed
196
        return " {}".format({0: "yes", 1: "no"}[doc["label"]])
Jonathan Tow's avatar
Jonathan Tow committed
197
198

    def construct_requests(self, doc, ctx):
Leo Gao's avatar
Fix  
Leo Gao committed
199
200
        ll_yes, _ = rf.loglikelihood(ctx, " yes")
        ll_no, _ = rf.loglikelihood(ctx, " no")
Jonathan Tow's avatar
Jonathan Tow committed
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
        return ll_yes, ll_no

    def process_results(self, doc, results):
        ll_yes, ll_no = results
        pred = ll_no > ll_yes
        gold = doc["label"]
        return {
            "acc": pred == gold
        }

    def higher_is_better(self):
        return {
            "acc": True
        }

    def aggregation(self):
        return {
            "acc": mean
        }


class WNLI(HFTask):
thomasw21's avatar
thomasw21 committed
223
    VERSION = 1
Jonathan Tow's avatar
Jonathan Tow committed
224
225
226
227
228
229
230
231
232
233
    DATASET_PATH = "glue"
    DATASET_NAME = "wnli"

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
234
        return False
Jason Phang's avatar
Jason Phang committed
235

236
    def doc_to_text(self, doc):
thomasw21's avatar
thomasw21 committed
237
        return "{}\nQuestion: {} True or False?\nAnswer:".format(
Jason Phang's avatar
Jason Phang committed
238
239
240
            doc["sentence1"],
            doc["sentence2"],
        )
241
242

    def doc_to_target(self, doc):
Jonathan Tow's avatar
Jonathan Tow committed
243
        # True = entailment
thomasw21's avatar
thomasw21 committed
244
245
        # False = not_entailment
        return " {}".format({0: "False", 1: "True"}[doc["label"]])
Jason Phang's avatar
Jason Phang committed
246

Jonathan Tow's avatar
Jonathan Tow committed
247
248
249
    def construct_requests(self, doc, ctx):
        ll_true, _ = rf.loglikelihood(ctx, " True")
        ll_false, _ = rf.loglikelihood(ctx, " False")
thomasw21's avatar
thomasw21 committed
250
        return ll_true, ll_false
Jonathan Tow's avatar
Jonathan Tow committed
251
252

    def process_results(self, doc, results):
thomasw21's avatar
thomasw21 committed
253
254
        ll_true, ll_false = results
        pred = ll_true > ll_false
Jonathan Tow's avatar
Jonathan Tow committed
255
256
257
258
259
260
261
262
263
264
265
266
267
268
        gold = doc["label"]
        return {
            "acc": pred == gold
        }

    def higher_is_better(self):
        return {
            "acc": True
        }

    def aggregation(self):
        return {
            "acc": mean
        }
269

Jason Phang's avatar
Jason Phang committed
270

sdtblck's avatar
sdtblck committed
271
class RTE(HFTask):
Leo Gao's avatar
Leo Gao committed
272
    VERSION = 0
sdtblck's avatar
sdtblck committed
273
274
    DATASET_PATH = "glue"
    DATASET_NAME = "rte"
Jason Phang's avatar
checkin  
Jason Phang committed
275
276
277
278
279
280
281
282

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
283
        return False
Jason Phang's avatar
checkin  
Jason Phang committed
284

285
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
286
        return "{}\nQuestion: {} True or False?\nAnswer:".format(
Jason Phang's avatar
checkin  
Jason Phang committed
287
288
289
            doc["sentence1"],
            doc["sentence2"],
        )
290
291
292
293
294

    def doc_to_target(self, doc):
        # 0 = entailment
        # 1 = not_entailment
        return " {}".format({0: "True", 1: "False"}[doc["label"]])
Jason Phang's avatar
checkin  
Jason Phang committed
295

Jonathan Tow's avatar
Jonathan Tow committed
296
297
298
299
    def construct_requests(self, doc, ctx):
        ll_true, _ = rf.loglikelihood(ctx, " True")
        ll_false, _ = rf.loglikelihood(ctx, " False")
        return ll_true, ll_false
300

Jonathan Tow's avatar
Jonathan Tow committed
301
302
303
304
305
306
307
308
309
310
311
312
    def process_results(self, doc, results):
        ll_true, ll_false = results
        pred = ll_false > ll_true
        gold = doc["label"]
        return {
            "acc": pred == gold
        }

    def higher_is_better(self):
        return {
            "acc": True
        }
Jason Phang's avatar
Jason Phang committed
313

Jonathan Tow's avatar
Jonathan Tow committed
314
315
316
317
    def aggregation(self):
        return {
            "acc": mean
        }
Jason Phang's avatar
Jason Phang committed
318

Jonathan Tow's avatar
Jonathan Tow committed
319
320
321
322
323

# Similarity and Paraphrase Tasks


class MRPC(HFTask):
Leo Gao's avatar
Leo Gao committed
324
    VERSION = 0
sdtblck's avatar
sdtblck committed
325
    DATASET_PATH = "glue"
Jonathan Tow's avatar
Jonathan Tow committed
326
    DATASET_NAME = "mrpc"
Jason Phang's avatar
Jason Phang committed
327
328
329
330
331
332
333
334

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
Leo Gao's avatar
Leo Gao committed
335
        return False
Jason Phang's avatar
Jason Phang committed
336

337
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
338
339
340
        return "Sentence 1: {}\nSentence 2: {}\nQuestion: Do both sentences mean the same thing?\nAnswer:".format(
            general_detokenize(doc["sentence1"]),
            general_detokenize(doc["sentence2"]),
Jason Phang's avatar
Jason Phang committed
341
        )
342
343

    def doc_to_target(self, doc):
Jonathan Tow's avatar
Jonathan Tow committed
344
        return " {}".format(yesno(doc["label"]))
Jason Phang's avatar
Jason Phang committed
345

Jonathan Tow's avatar
Jonathan Tow committed
346
    def construct_requests(self, doc, ctx):
Leo Gao's avatar
Fix  
Leo Gao committed
347
348
        ll_yes, _ = rf.loglikelihood(ctx, " yes")
        ll_no, _ = rf.loglikelihood(ctx, " no")
Jonathan Tow's avatar
Jonathan Tow committed
349
        return ll_yes, ll_no
350

Jonathan Tow's avatar
Jonathan Tow committed
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
    def process_results(self, doc, results):
        ll_yes, ll_no = results
        gold = doc["label"]
        pred = ll_yes > ll_no
        return {
            "acc": pred == gold,
            "f1": (gold, pred),
        }

    def higher_is_better(self):
        return {
            "acc": True,
            "f1": True
        }

    def aggregation(self):
        return {
            "acc": mean,
            "f1": f1_score
        }
Jason Phang's avatar
Jason Phang committed
371
372


sdtblck's avatar
sdtblck committed
373
class QQP(HFTask):
Leo Gao's avatar
Leo Gao committed
374
    VERSION = 0
sdtblck's avatar
sdtblck committed
375
376
    DATASET_PATH = "glue"
    DATASET_NAME = "qqp"
Jason Phang's avatar
Jason Phang committed
377
378
379
380
381
382
383
384

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
Leo Gao's avatar
Leo Gao committed
385
        return False
Jason Phang's avatar
Jason Phang committed
386

387
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
388
        return "Question 1: {}\nQuestion 2: {}\nQuestion: Do both questions ask the same thing?\nAnswer:".format(
Jason Phang's avatar
Jason Phang committed
389
390
391
            doc["question1"],
            doc["question2"],
        )
392
393
394

    def doc_to_target(self, doc):
        return " {}".format(yesno(doc["label"]))
Jason Phang's avatar
Jason Phang committed
395

Jonathan Tow's avatar
Jonathan Tow committed
396
397
398
399
    def construct_requests(self, doc, ctx):
        ll_yes, _ = rf.loglikelihood(ctx, " yes")
        ll_no, _ = rf.loglikelihood(ctx, " no")
        return ll_yes, ll_no
400

Jonathan Tow's avatar
Jonathan Tow committed
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
    def process_results(self, doc, results):
        ll_yes, ll_no = results
        gold = doc["label"]
        pred = ll_yes > ll_no
        return {
            "acc": pred == gold,
            "f1": (gold, pred),
        }

    def higher_is_better(self):
        return {
            "acc": True,
            "f1": True
        }

    def aggregation(self):
        return {
            "acc": mean,
            "f1": f1_score
        }
Jason Phang's avatar
Jason Phang committed
421
422


sdtblck's avatar
sdtblck committed
423
class STSB(HFTask):
Leo Gao's avatar
Leo Gao committed
424
    VERSION = 0
sdtblck's avatar
sdtblck committed
425
426
    DATASET_PATH = "glue"
    DATASET_NAME = "stsb"
Jason Phang's avatar
Jason Phang committed
427
428
429
430
431
432
433
434
435
436

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
        return True

437
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
438
        return "sentence 1: {}\nsentence 2: {}\nAnswer:".format(
Jason Phang's avatar
Jason Phang committed
439
440
441
            doc["sentence1"],
            doc["sentence2"],
        )
442
443
444

    def doc_to_target(self, doc):
        return " {}".format(doc["label"])
Jason Phang's avatar
Jason Phang committed
445

Leo Gao's avatar
Leo Gao committed
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
    def construct_requests(self, doc, ctx):
        """ Uses RequestFactory to construct Requests and returns an iterable of 
        Requests which will be sent to the LM.

        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param ctx: str
            The context string, generated by fewshot_context. This includes the natural 
            language description, as well as the few shot examples, and the question
            part of the document for `doc`. 
        """
        # TODO: implement evaluation.
        raise NotImplementedError('Evaluation not implemented')
    
    def process_results(self, doc, results):
        """Take a single document and the LM results and evaluates, returning a 
        dict where keys are the names of submetrics and values are the values of 
        the metric for that one document

        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param results:
            The results of the requests created in construct_requests.
        """
        # TODO: implement evaluation.
        raise NotImplementedError('Evaluation not implemented')

    def aggregation(self):
        """
        :returns: {str: [float] -> float}
            A dictionary where keys are the names of submetrics and values are 
            functions that aggregate a list of metrics
        """
        # TODO: implement evaluation.
        raise NotImplementedError('Evaluation not implemented')

    def higher_is_better(self):
        """
        :returns: {str: bool}
            A dictionary where keys are the names of submetrics and values are 
            whether a higher value of the submetric is better
        """
        # TODO: implement evaluation.
        raise NotImplementedError('Evaluation not implemented')