glue.py 12.1 KB
Newer Older
Jason Phang's avatar
checkin  
Jason Phang committed
1
import numpy as np
&'s avatar
& committed
2
3
from lm_eval.base import rf
from ..metrics import mean, matthews_corrcoef, f1_score
Jonathan Tow's avatar
Jonathan Tow committed
4
from . common import HFTask, yesno
Leo Gao's avatar
Leo Gao committed
5
from ..utils import general_detokenize
Jonathan Tow's avatar
Jonathan Tow committed
6
7

# Single-Sentence Tasks
Jason Phang's avatar
Jason Phang committed
8
9


sdtblck's avatar
sdtblck committed
10
class CoLA(HFTask):
Leo Gao's avatar
Leo Gao committed
11
    VERSION = 0
sdtblck's avatar
sdtblck committed
12
13
    DATASET_PATH = "glue"
    DATASET_NAME = "cola"
Jonathan Tow's avatar
Jonathan Tow committed
14

Jason Phang's avatar
checkin  
Jason Phang committed
15
16
17
18
19
20
21
    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
22
        return False
Jason Phang's avatar
checkin  
Jason Phang committed
23

24
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
25
        return "{}\nQuestion: Does this sentence make sense?\nAnswer:".format(doc["sentence"])
26
27

    def doc_to_target(self, doc):
Leo Gao's avatar
Leo Gao committed
28
        return " {}".format({1: "yes", 0: "no"}[doc["label"]])
Jason Phang's avatar
checkin  
Jason Phang committed
29

Jonathan Tow's avatar
Jonathan Tow committed
30
    def construct_requests(self, doc, ctx):
Leo Gao's avatar
Leo Gao committed
31
32
        ll_true, _ = rf.loglikelihood(ctx, " yes")
        ll_false, _ = rf.loglikelihood(ctx, " no")
Jonathan Tow's avatar
Jonathan Tow committed
33
        return ll_true, ll_false
34

Jonathan Tow's avatar
Jonathan Tow committed
35
36
37
38
39
40
41
    def process_results(self, doc, results):
        ll_true, ll_false = results
        pred = ll_true > ll_false
        gold = doc["label"]
        return {
            "mcc": (gold, pred)
        }
42

Jonathan Tow's avatar
Jonathan Tow committed
43
    def higher_is_better(self):
Jason Phang's avatar
checkin  
Jason Phang committed
44
        return {
Jonathan Tow's avatar
Jonathan Tow committed
45
46
47
48
49
50
51
52
53
54
            "mcc": True
        }

    def aggregation(self):
        return {
            "mcc": matthews_corrcoef
        }


class SST(HFTask):
Leo Gao's avatar
Leo Gao committed
55
    VERSION = 0
Jonathan Tow's avatar
Jonathan Tow committed
56
57
58
59
60
61
62
63
64
65
    DATASET_PATH = "glue"
    DATASET_NAME = "sst2"

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
66
        return False
Jonathan Tow's avatar
Jonathan Tow committed
67
68

    def doc_to_text(self, doc):
Leo Gao's avatar
Fix  
Leo Gao committed
69
        return "{}\nQuestion: Is this sentence positive or negative?\nAnswer:".format(
Leo Gao's avatar
Leo Gao committed
70
            general_detokenize(doc["sentence"]),
Jonathan Tow's avatar
Jonathan Tow committed
71
72
73
        )

    def doc_to_target(self, doc):
Leo Gao's avatar
Fix  
Leo Gao committed
74
        return " {}".format({1: "positive", 0: "negative"}[doc["label"]])
Jonathan Tow's avatar
Jonathan Tow committed
75
76

    def construct_requests(self, doc, ctx):
Leo Gao's avatar
Fix  
Leo Gao committed
77
78
        ll_positive, _ = rf.loglikelihood(ctx, " positive")
        ll_negative, _ = rf.loglikelihood(ctx, " negative")
Jonathan Tow's avatar
Jonathan Tow committed
79
80
81
82
83
84
85
86
87
88
89
90
91
        return ll_positive, ll_negative

    def process_results(self, doc, results):
        ll_positive, ll_negative = results
        pred = ll_positive > ll_negative
        gold = doc["label"]
        return {
            "acc": pred == gold
        }

    def higher_is_better(self):
        return {
            "acc": True
Jason Phang's avatar
checkin  
Jason Phang committed
92
93
        }

Jonathan Tow's avatar
Jonathan Tow committed
94
95
96
97
98
99
100
101
    def aggregation(self):
        return {
            "acc": mean
        }


# Inference Tasks

Jason Phang's avatar
checkin  
Jason Phang committed
102

sdtblck's avatar
sdtblck committed
103
class MNLI(HFTask):
Leo Gao's avatar
Leo Gao committed
104
    VERSION = 0
sdtblck's avatar
sdtblck committed
105
106
    DATASET_PATH = "glue"
    DATASET_NAME = "mnli"
Jason Phang's avatar
Jason Phang committed
107

Jason Phang's avatar
checkin  
Jason Phang committed
108
109
110
111
112
113
114
    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
115
        return False
Jason Phang's avatar
checkin  
Jason Phang committed
116
117
118

    def validation_docs(self):
        if self.has_validation_docs():
sdtblck's avatar
sdtblck committed
119
            return self.data["validation_matched"]
Jason Phang's avatar
checkin  
Jason Phang committed
120
121
122

    def test_docs(self):
        if self.has_test_docs():
sdtblck's avatar
sdtblck committed
123
            return self.data["test_matched"]
Jason Phang's avatar
checkin  
Jason Phang committed
124

125
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
126
        return "{}\nQuestion: {} True, False or Neither?\nAnswer:".format(
Jason Phang's avatar
Jason Phang committed
127
            doc["premise"],
Leo Gao's avatar
Fix  
Leo Gao committed
128
            doc["hypothesis"].strip() + ('' if doc["hypothesis"].strip().endswith('.') else '.'),
Jason Phang's avatar
checkin  
Jason Phang committed
129
        )
130
131
132
133
134
135

    def doc_to_target(self, doc):
        # True = entailment
        # False = contradiction
        # Neither = neutral
        return " {}".format({0: "True", 1: "Neither", 2: "False"}[doc["label"]])
Jason Phang's avatar
checkin  
Jason Phang committed
136

Jonathan Tow's avatar
Jonathan Tow committed
137
138
139
140
141
    def construct_requests(self, doc, ctx):
        ll_true, _ = rf.loglikelihood(ctx, " True")
        ll_neither, _ = rf.loglikelihood(ctx, " Neither")
        ll_false, _ = rf.loglikelihood(ctx, " False")
        return ll_true, ll_neither, ll_false
142

Jonathan Tow's avatar
Jonathan Tow committed
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
    def process_results(self, doc, results):
        gold = doc["label"]
        pred = np.argmax(results)
        return {
            "acc": pred == gold
        }

    def higher_is_better(self):
        return {
            "acc": True
        }

    def aggregation(self):
        return {
            "acc": mean
        }
Jason Phang's avatar
checkin  
Jason Phang committed
159
160


Jason Phang's avatar
Jason Phang committed
161
class MNLIMismatched(MNLI):
Leo Gao's avatar
Leo Gao committed
162
    VERSION = 0
Jason Phang's avatar
Jason Phang committed
163
164
165
166
167
168
169
170
171
172

    def validation_docs(self):
        if self.has_validation_docs():
            return self.data["validation_mismatched"]

    def test_docs(self):
        if self.has_test_docs():
            return self.data["test_mismatched"]


Jonathan Tow's avatar
Jonathan Tow committed
173
class QNLI(HFTask):
Leo Gao's avatar
Leo Gao committed
174
    VERSION = 0
sdtblck's avatar
sdtblck committed
175
    DATASET_PATH = "glue"
Jonathan Tow's avatar
Jonathan Tow committed
176
    DATASET_NAME = "qnli"
Jason Phang's avatar
Jason Phang committed
177
178
179
180
181
182
183
184

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
185
        return False
Jason Phang's avatar
Jason Phang committed
186

Jonathan Tow's avatar
Jonathan Tow committed
187
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
188
        return "{}\n{}\nQuestion: Does this response answer the question?\nAnswer:".format(
Jonathan Tow's avatar
Jonathan Tow committed
189
190
191
192
193
194
195
            doc["question"],
            doc["sentence"],
        )

    def doc_to_target(self, doc):
        # True = entailment
        # False = not entailment
Leo Gao's avatar
Fix  
Leo Gao committed
196
        return " {}".format({0: "yes", 1: "no"}[doc["label"]])
Jonathan Tow's avatar
Jonathan Tow committed
197
198

    def construct_requests(self, doc, ctx):
Leo Gao's avatar
Fix  
Leo Gao committed
199
200
        ll_yes, _ = rf.loglikelihood(ctx, " yes")
        ll_no, _ = rf.loglikelihood(ctx, " no")
Jonathan Tow's avatar
Jonathan Tow committed
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
        return ll_yes, ll_no

    def process_results(self, doc, results):
        ll_yes, ll_no = results
        pred = ll_no > ll_yes
        gold = doc["label"]
        return {
            "acc": pred == gold
        }

    def higher_is_better(self):
        return {
            "acc": True
        }

    def aggregation(self):
        return {
            "acc": mean
        }


class WNLI(HFTask):
Leo Gao's avatar
Leo Gao committed
223
    VERSION = 0
Jonathan Tow's avatar
Jonathan Tow committed
224
225
226
227
228
229
230
231
232
233
    DATASET_PATH = "glue"
    DATASET_NAME = "wnli"

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
234
        return False
Jason Phang's avatar
Jason Phang committed
235

236
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
237
        return "{}\nQuestion: {} True, False or Neither?\nAnswer:".format(
Jason Phang's avatar
Jason Phang committed
238
239
240
            doc["sentence1"],
            doc["sentence2"],
        )
241
242

    def doc_to_target(self, doc):
Jonathan Tow's avatar
Jonathan Tow committed
243
244
245
246
        # True = entailment
        # False = contradiction
        # Neither = neutral
        return " {}".format({0: "True", 1: "Neither", 2: "False"}[doc["label"]])
Jason Phang's avatar
Jason Phang committed
247

Jonathan Tow's avatar
Jonathan Tow committed
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
    def construct_requests(self, doc, ctx):
        ll_true, _ = rf.loglikelihood(ctx, " True")
        ll_neither, _ = rf.loglikelihood(ctx, " Neither")
        ll_false, _ = rf.loglikelihood(ctx, " False")
        return ll_true, ll_neither, ll_false

    def process_results(self, doc, results):
        gold = doc["label"]
        pred = np.argmax(results)
        return {
            "acc": pred == gold
        }

    def higher_is_better(self):
        return {
            "acc": True
        }

    def aggregation(self):
        return {
            "acc": mean
        }
270

Jason Phang's avatar
Jason Phang committed
271

sdtblck's avatar
sdtblck committed
272
class RTE(HFTask):
Leo Gao's avatar
Leo Gao committed
273
    VERSION = 0
sdtblck's avatar
sdtblck committed
274
275
    DATASET_PATH = "glue"
    DATASET_NAME = "rte"
Jason Phang's avatar
checkin  
Jason Phang committed
276
277
278
279
280
281
282
283

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
284
        return False
Jason Phang's avatar
checkin  
Jason Phang committed
285

286
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
287
        return "{}\nQuestion: {} True or False?\nAnswer:".format(
Jason Phang's avatar
checkin  
Jason Phang committed
288
289
290
            doc["sentence1"],
            doc["sentence2"],
        )
291
292
293
294
295

    def doc_to_target(self, doc):
        # 0 = entailment
        # 1 = not_entailment
        return " {}".format({0: "True", 1: "False"}[doc["label"]])
Jason Phang's avatar
checkin  
Jason Phang committed
296

Jonathan Tow's avatar
Jonathan Tow committed
297
298
299
300
    def construct_requests(self, doc, ctx):
        ll_true, _ = rf.loglikelihood(ctx, " True")
        ll_false, _ = rf.loglikelihood(ctx, " False")
        return ll_true, ll_false
301

Jonathan Tow's avatar
Jonathan Tow committed
302
303
304
305
306
307
308
309
310
311
312
313
    def process_results(self, doc, results):
        ll_true, ll_false = results
        pred = ll_false > ll_true
        gold = doc["label"]
        return {
            "acc": pred == gold
        }

    def higher_is_better(self):
        return {
            "acc": True
        }
Jason Phang's avatar
Jason Phang committed
314

Jonathan Tow's avatar
Jonathan Tow committed
315
316
317
318
    def aggregation(self):
        return {
            "acc": mean
        }
Jason Phang's avatar
Jason Phang committed
319

Jonathan Tow's avatar
Jonathan Tow committed
320
321
322
323
324

# Similarity and Paraphrase Tasks


class MRPC(HFTask):
Leo Gao's avatar
Leo Gao committed
325
    VERSION = 0
sdtblck's avatar
sdtblck committed
326
    DATASET_PATH = "glue"
Jonathan Tow's avatar
Jonathan Tow committed
327
    DATASET_NAME = "mrpc"
Jason Phang's avatar
Jason Phang committed
328
329
330
331
332
333
334
335

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
Leo Gao's avatar
Leo Gao committed
336
        return False
Jason Phang's avatar
Jason Phang committed
337

338
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
339
340
341
        return "Sentence 1: {}\nSentence 2: {}\nQuestion: Do both sentences mean the same thing?\nAnswer:".format(
            general_detokenize(doc["sentence1"]),
            general_detokenize(doc["sentence2"]),
Jason Phang's avatar
Jason Phang committed
342
        )
343
344

    def doc_to_target(self, doc):
Jonathan Tow's avatar
Jonathan Tow committed
345
        return " {}".format(yesno(doc["label"]))
Jason Phang's avatar
Jason Phang committed
346

Jonathan Tow's avatar
Jonathan Tow committed
347
    def construct_requests(self, doc, ctx):
Leo Gao's avatar
Fix  
Leo Gao committed
348
349
        ll_yes, _ = rf.loglikelihood(ctx, " yes")
        ll_no, _ = rf.loglikelihood(ctx, " no")
Jonathan Tow's avatar
Jonathan Tow committed
350
        return ll_yes, ll_no
351

Jonathan Tow's avatar
Jonathan Tow committed
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
    def process_results(self, doc, results):
        ll_yes, ll_no = results
        gold = doc["label"]
        pred = ll_yes > ll_no
        return {
            "acc": pred == gold,
            "f1": (gold, pred),
        }

    def higher_is_better(self):
        return {
            "acc": True,
            "f1": True
        }

    def aggregation(self):
        return {
            "acc": mean,
            "f1": f1_score
        }
Jason Phang's avatar
Jason Phang committed
372
373


sdtblck's avatar
sdtblck committed
374
class QQP(HFTask):
Leo Gao's avatar
Leo Gao committed
375
    VERSION = 0
sdtblck's avatar
sdtblck committed
376
377
    DATASET_PATH = "glue"
    DATASET_NAME = "qqp"
Jason Phang's avatar
Jason Phang committed
378
379
380
381
382
383
384
385

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
Leo Gao's avatar
Leo Gao committed
386
        return False
Jason Phang's avatar
Jason Phang committed
387

388
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
389
        return "Question 1: {}\nQuestion 2: {}\nQuestion: Do both questions ask the same thing?\nAnswer:".format(
Jason Phang's avatar
Jason Phang committed
390
391
392
            doc["question1"],
            doc["question2"],
        )
393
394
395

    def doc_to_target(self, doc):
        return " {}".format(yesno(doc["label"]))
Jason Phang's avatar
Jason Phang committed
396

Jonathan Tow's avatar
Jonathan Tow committed
397
398
399
400
    def construct_requests(self, doc, ctx):
        ll_yes, _ = rf.loglikelihood(ctx, " yes")
        ll_no, _ = rf.loglikelihood(ctx, " no")
        return ll_yes, ll_no
401

Jonathan Tow's avatar
Jonathan Tow committed
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
    def process_results(self, doc, results):
        ll_yes, ll_no = results
        gold = doc["label"]
        pred = ll_yes > ll_no
        return {
            "acc": pred == gold,
            "f1": (gold, pred),
        }

    def higher_is_better(self):
        return {
            "acc": True,
            "f1": True
        }

    def aggregation(self):
        return {
            "acc": mean,
            "f1": f1_score
        }
Jason Phang's avatar
Jason Phang committed
422
423


sdtblck's avatar
sdtblck committed
424
class STSB(HFTask):
Leo Gao's avatar
Leo Gao committed
425
    VERSION = 0
sdtblck's avatar
sdtblck committed
426
427
    DATASET_PATH = "glue"
    DATASET_NAME = "stsb"
Jason Phang's avatar
Jason Phang committed
428
429
430
431
432
433
434
435
436
437

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
        return True

438
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
439
        return "sentence 1: {}\nsentence 2: {}\nAnswer:".format(
Jason Phang's avatar
Jason Phang committed
440
441
442
            doc["sentence1"],
            doc["sentence2"],
        )
443
444
445

    def doc_to_target(self, doc):
        return " {}".format(doc["label"])
Jason Phang's avatar
Jason Phang committed
446

Leo Gao's avatar
Leo Gao committed
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
    def construct_requests(self, doc, ctx):
        """ Uses RequestFactory to construct Requests and returns an iterable of 
        Requests which will be sent to the LM.

        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param ctx: str
            The context string, generated by fewshot_context. This includes the natural 
            language description, as well as the few shot examples, and the question
            part of the document for `doc`. 
        """
        # TODO: implement evaluation.
        raise NotImplementedError('Evaluation not implemented')
    
    def process_results(self, doc, results):
        """Take a single document and the LM results and evaluates, returning a 
        dict where keys are the names of submetrics and values are the values of 
        the metric for that one document

        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param results:
            The results of the requests created in construct_requests.
        """
        # TODO: implement evaluation.
        raise NotImplementedError('Evaluation not implemented')

    def aggregation(self):
        """
        :returns: {str: [float] -> float}
            A dictionary where keys are the names of submetrics and values are 
            functions that aggregate a list of metrics
        """
        # TODO: implement evaluation.
        raise NotImplementedError('Evaluation not implemented')

    def higher_is_better(self):
        """
        :returns: {str: bool}
            A dictionary where keys are the names of submetrics and values are 
            whether a higher value of the submetric is better
        """
        # TODO: implement evaluation.
        raise NotImplementedError('Evaluation not implemented')