glue.py 12.2 KB
Newer Older
Jason Phang's avatar
checkin  
Jason Phang committed
1
import numpy as np
&'s avatar
& committed
2
3
from lm_eval.base import rf
from ..metrics import mean, matthews_corrcoef, f1_score
Jonathan Tow's avatar
Jonathan Tow committed
4
from . common import HFTask, yesno
Leo Gao's avatar
Leo Gao committed
5
from ..utils import general_detokenize
Jonathan Tow's avatar
Jonathan Tow committed
6
7

# Single-Sentence Tasks
Jason Phang's avatar
Jason Phang committed
8
9


sdtblck's avatar
sdtblck committed
10
class CoLA(HFTask):
Leo Gao's avatar
Leo Gao committed
11
    VERSION = 0
sdtblck's avatar
sdtblck committed
12
13
    DATASET_PATH = "glue"
    DATASET_NAME = "cola"
Jonathan Tow's avatar
Jonathan Tow committed
14

Jason Phang's avatar
checkin  
Jason Phang committed
15
16
17
18
19
20
21
    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
22
        return False
Jason Phang's avatar
checkin  
Jason Phang committed
23

24
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
25
        return "{}\nQuestion: Does this sentence make sense?\nAnswer:".format(doc["sentence"])
26

27
28
29
30
31
32
    def should_decontaminate(self):
        return True

    def doc_to_decontamination_query(self, doc):
        return doc["sentence"]

33
    def doc_to_target(self, doc):
Leo Gao's avatar
Leo Gao committed
34
        return " {}".format({1: "yes", 0: "no"}[doc["label"]])
Jason Phang's avatar
checkin  
Jason Phang committed
35

Jonathan Tow's avatar
Jonathan Tow committed
36
    def construct_requests(self, doc, ctx):
Leo Gao's avatar
Leo Gao committed
37
38
        ll_true, _ = rf.loglikelihood(ctx, " yes")
        ll_false, _ = rf.loglikelihood(ctx, " no")
Jonathan Tow's avatar
Jonathan Tow committed
39
        return ll_true, ll_false
40

Jonathan Tow's avatar
Jonathan Tow committed
41
42
43
44
45
46
47
    def process_results(self, doc, results):
        ll_true, ll_false = results
        pred = ll_true > ll_false
        gold = doc["label"]
        return {
            "mcc": (gold, pred)
        }
48

Jonathan Tow's avatar
Jonathan Tow committed
49
    def higher_is_better(self):
Jason Phang's avatar
checkin  
Jason Phang committed
50
        return {
Jonathan Tow's avatar
Jonathan Tow committed
51
52
53
54
55
56
57
58
59
60
            "mcc": True
        }

    def aggregation(self):
        return {
            "mcc": matthews_corrcoef
        }


class SST(HFTask):
Leo Gao's avatar
Leo Gao committed
61
    VERSION = 0
Jonathan Tow's avatar
Jonathan Tow committed
62
63
64
65
66
67
68
69
70
71
    DATASET_PATH = "glue"
    DATASET_NAME = "sst2"

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
72
        return False
Jonathan Tow's avatar
Jonathan Tow committed
73
74

    def doc_to_text(self, doc):
Leo Gao's avatar
Fix  
Leo Gao committed
75
        return "{}\nQuestion: Is this sentence positive or negative?\nAnswer:".format(
Leo Gao's avatar
Leo Gao committed
76
            general_detokenize(doc["sentence"]),
Jonathan Tow's avatar
Jonathan Tow committed
77
78
79
        )

    def doc_to_target(self, doc):
Leo Gao's avatar
Fix  
Leo Gao committed
80
        return " {}".format({1: "positive", 0: "negative"}[doc["label"]])
Jonathan Tow's avatar
Jonathan Tow committed
81
82

    def construct_requests(self, doc, ctx):
Leo Gao's avatar
Fix  
Leo Gao committed
83
84
        ll_positive, _ = rf.loglikelihood(ctx, " positive")
        ll_negative, _ = rf.loglikelihood(ctx, " negative")
Jonathan Tow's avatar
Jonathan Tow committed
85
86
87
88
89
90
91
92
93
94
95
96
97
        return ll_positive, ll_negative

    def process_results(self, doc, results):
        ll_positive, ll_negative = results
        pred = ll_positive > ll_negative
        gold = doc["label"]
        return {
            "acc": pred == gold
        }

    def higher_is_better(self):
        return {
            "acc": True
Jason Phang's avatar
checkin  
Jason Phang committed
98
99
        }

Jonathan Tow's avatar
Jonathan Tow committed
100
101
102
103
104
105
106
107
    def aggregation(self):
        return {
            "acc": mean
        }


# Inference Tasks

Jason Phang's avatar
checkin  
Jason Phang committed
108

sdtblck's avatar
sdtblck committed
109
class MNLI(HFTask):
Leo Gao's avatar
Leo Gao committed
110
    VERSION = 0
sdtblck's avatar
sdtblck committed
111
112
    DATASET_PATH = "glue"
    DATASET_NAME = "mnli"
Jason Phang's avatar
Jason Phang committed
113

Jason Phang's avatar
checkin  
Jason Phang committed
114
115
116
117
118
119
120
    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
121
        return False
Jason Phang's avatar
checkin  
Jason Phang committed
122
123
124

    def validation_docs(self):
        if self.has_validation_docs():
sdtblck's avatar
sdtblck committed
125
            return self.data["validation_matched"]
Jason Phang's avatar
checkin  
Jason Phang committed
126
127
128

    def test_docs(self):
        if self.has_test_docs():
sdtblck's avatar
sdtblck committed
129
            return self.data["test_matched"]
Jason Phang's avatar
checkin  
Jason Phang committed
130

131
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
132
        return "{}\nQuestion: {} True, False or Neither?\nAnswer:".format(
Jason Phang's avatar
Jason Phang committed
133
            doc["premise"],
Leo Gao's avatar
Fix  
Leo Gao committed
134
            doc["hypothesis"].strip() + ('' if doc["hypothesis"].strip().endswith('.') else '.'),
Jason Phang's avatar
checkin  
Jason Phang committed
135
        )
136
137
138
139
140
141

    def doc_to_target(self, doc):
        # True = entailment
        # False = contradiction
        # Neither = neutral
        return " {}".format({0: "True", 1: "Neither", 2: "False"}[doc["label"]])
Jason Phang's avatar
checkin  
Jason Phang committed
142

Jonathan Tow's avatar
Jonathan Tow committed
143
144
145
146
147
    def construct_requests(self, doc, ctx):
        ll_true, _ = rf.loglikelihood(ctx, " True")
        ll_neither, _ = rf.loglikelihood(ctx, " Neither")
        ll_false, _ = rf.loglikelihood(ctx, " False")
        return ll_true, ll_neither, ll_false
148

Jonathan Tow's avatar
Jonathan Tow committed
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
    def process_results(self, doc, results):
        gold = doc["label"]
        pred = np.argmax(results)
        return {
            "acc": pred == gold
        }

    def higher_is_better(self):
        return {
            "acc": True
        }

    def aggregation(self):
        return {
            "acc": mean
        }
Jason Phang's avatar
checkin  
Jason Phang committed
165
166


Jason Phang's avatar
Jason Phang committed
167
class MNLIMismatched(MNLI):
Leo Gao's avatar
Leo Gao committed
168
    VERSION = 0
Jason Phang's avatar
Jason Phang committed
169
170
171
172
173
174
175
176
177
178

    def validation_docs(self):
        if self.has_validation_docs():
            return self.data["validation_mismatched"]

    def test_docs(self):
        if self.has_test_docs():
            return self.data["test_mismatched"]


Jonathan Tow's avatar
Jonathan Tow committed
179
class QNLI(HFTask):
Leo Gao's avatar
Leo Gao committed
180
    VERSION = 0
sdtblck's avatar
sdtblck committed
181
    DATASET_PATH = "glue"
Jonathan Tow's avatar
Jonathan Tow committed
182
    DATASET_NAME = "qnli"
Jason Phang's avatar
Jason Phang committed
183
184
185
186
187
188
189
190

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
191
        return False
Jason Phang's avatar
Jason Phang committed
192

Jonathan Tow's avatar
Jonathan Tow committed
193
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
194
        return "{}\n{}\nQuestion: Does this response answer the question?\nAnswer:".format(
Jonathan Tow's avatar
Jonathan Tow committed
195
196
197
198
199
200
201
            doc["question"],
            doc["sentence"],
        )

    def doc_to_target(self, doc):
        # True = entailment
        # False = not entailment
Leo Gao's avatar
Fix  
Leo Gao committed
202
        return " {}".format({0: "yes", 1: "no"}[doc["label"]])
Jonathan Tow's avatar
Jonathan Tow committed
203
204

    def construct_requests(self, doc, ctx):
Leo Gao's avatar
Fix  
Leo Gao committed
205
206
        ll_yes, _ = rf.loglikelihood(ctx, " yes")
        ll_no, _ = rf.loglikelihood(ctx, " no")
Jonathan Tow's avatar
Jonathan Tow committed
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
        return ll_yes, ll_no

    def process_results(self, doc, results):
        ll_yes, ll_no = results
        pred = ll_no > ll_yes
        gold = doc["label"]
        return {
            "acc": pred == gold
        }

    def higher_is_better(self):
        return {
            "acc": True
        }

    def aggregation(self):
        return {
            "acc": mean
        }


class WNLI(HFTask):
thomasw21's avatar
thomasw21 committed
229
    VERSION = 1
Jonathan Tow's avatar
Jonathan Tow committed
230
231
232
233
234
235
236
237
238
239
    DATASET_PATH = "glue"
    DATASET_NAME = "wnli"

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
240
        return False
Jason Phang's avatar
Jason Phang committed
241

242
    def doc_to_text(self, doc):
thomasw21's avatar
thomasw21 committed
243
        return "{}\nQuestion: {} True or False?\nAnswer:".format(
Jason Phang's avatar
Jason Phang committed
244
245
246
            doc["sentence1"],
            doc["sentence2"],
        )
247
248

    def doc_to_target(self, doc):
Jonathan Tow's avatar
Jonathan Tow committed
249
        # True = entailment
thomasw21's avatar
thomasw21 committed
250
251
        # False = not_entailment
        return " {}".format({0: "False", 1: "True"}[doc["label"]])
Jason Phang's avatar
Jason Phang committed
252

Jonathan Tow's avatar
Jonathan Tow committed
253
254
255
    def construct_requests(self, doc, ctx):
        ll_true, _ = rf.loglikelihood(ctx, " True")
        ll_false, _ = rf.loglikelihood(ctx, " False")
thomasw21's avatar
thomasw21 committed
256
        return ll_true, ll_false
Jonathan Tow's avatar
Jonathan Tow committed
257
258

    def process_results(self, doc, results):
thomasw21's avatar
thomasw21 committed
259
260
        ll_true, ll_false = results
        pred = ll_true > ll_false
Jonathan Tow's avatar
Jonathan Tow committed
261
262
263
264
265
266
267
268
269
270
271
272
273
274
        gold = doc["label"]
        return {
            "acc": pred == gold
        }

    def higher_is_better(self):
        return {
            "acc": True
        }

    def aggregation(self):
        return {
            "acc": mean
        }
275

Jason Phang's avatar
Jason Phang committed
276

sdtblck's avatar
sdtblck committed
277
class RTE(HFTask):
Leo Gao's avatar
Leo Gao committed
278
    VERSION = 0
sdtblck's avatar
sdtblck committed
279
280
    DATASET_PATH = "glue"
    DATASET_NAME = "rte"
Jason Phang's avatar
checkin  
Jason Phang committed
281
282
283
284
285
286
287
288

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
289
        return False
Jason Phang's avatar
checkin  
Jason Phang committed
290

291
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
292
        return "{}\nQuestion: {} True or False?\nAnswer:".format(
Jason Phang's avatar
checkin  
Jason Phang committed
293
294
295
            doc["sentence1"],
            doc["sentence2"],
        )
296
297
298
299
300

    def doc_to_target(self, doc):
        # 0 = entailment
        # 1 = not_entailment
        return " {}".format({0: "True", 1: "False"}[doc["label"]])
Jason Phang's avatar
checkin  
Jason Phang committed
301

Jonathan Tow's avatar
Jonathan Tow committed
302
303
304
305
    def construct_requests(self, doc, ctx):
        ll_true, _ = rf.loglikelihood(ctx, " True")
        ll_false, _ = rf.loglikelihood(ctx, " False")
        return ll_true, ll_false
306

Jonathan Tow's avatar
Jonathan Tow committed
307
308
309
310
311
312
313
314
315
316
317
318
    def process_results(self, doc, results):
        ll_true, ll_false = results
        pred = ll_false > ll_true
        gold = doc["label"]
        return {
            "acc": pred == gold
        }

    def higher_is_better(self):
        return {
            "acc": True
        }
Jason Phang's avatar
Jason Phang committed
319

Jonathan Tow's avatar
Jonathan Tow committed
320
321
322
323
    def aggregation(self):
        return {
            "acc": mean
        }
Jason Phang's avatar
Jason Phang committed
324

Jonathan Tow's avatar
Jonathan Tow committed
325
326
327
328
329

# Similarity and Paraphrase Tasks


class MRPC(HFTask):
Leo Gao's avatar
Leo Gao committed
330
    VERSION = 0
sdtblck's avatar
sdtblck committed
331
    DATASET_PATH = "glue"
Jonathan Tow's avatar
Jonathan Tow committed
332
    DATASET_NAME = "mrpc"
Jason Phang's avatar
Jason Phang committed
333
334
335
336
337
338
339
340

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
Leo Gao's avatar
Leo Gao committed
341
        return False
Jason Phang's avatar
Jason Phang committed
342

343
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
344
345
346
        return "Sentence 1: {}\nSentence 2: {}\nQuestion: Do both sentences mean the same thing?\nAnswer:".format(
            general_detokenize(doc["sentence1"]),
            general_detokenize(doc["sentence2"]),
Jason Phang's avatar
Jason Phang committed
347
        )
348
349

    def doc_to_target(self, doc):
Jonathan Tow's avatar
Jonathan Tow committed
350
        return " {}".format(yesno(doc["label"]))
Jason Phang's avatar
Jason Phang committed
351

Jonathan Tow's avatar
Jonathan Tow committed
352
    def construct_requests(self, doc, ctx):
Leo Gao's avatar
Fix  
Leo Gao committed
353
354
        ll_yes, _ = rf.loglikelihood(ctx, " yes")
        ll_no, _ = rf.loglikelihood(ctx, " no")
Jonathan Tow's avatar
Jonathan Tow committed
355
        return ll_yes, ll_no
356

Jonathan Tow's avatar
Jonathan Tow committed
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
    def process_results(self, doc, results):
        ll_yes, ll_no = results
        gold = doc["label"]
        pred = ll_yes > ll_no
        return {
            "acc": pred == gold,
            "f1": (gold, pred),
        }

    def higher_is_better(self):
        return {
            "acc": True,
            "f1": True
        }

    def aggregation(self):
        return {
            "acc": mean,
            "f1": f1_score
        }
Jason Phang's avatar
Jason Phang committed
377
378


sdtblck's avatar
sdtblck committed
379
class QQP(HFTask):
Leo Gao's avatar
Leo Gao committed
380
    VERSION = 0
sdtblck's avatar
sdtblck committed
381
382
    DATASET_PATH = "glue"
    DATASET_NAME = "qqp"
Jason Phang's avatar
Jason Phang committed
383
384
385
386
387
388
389
390

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
Leo Gao's avatar
Leo Gao committed
391
        return False
Jason Phang's avatar
Jason Phang committed
392

393
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
394
        return "Question 1: {}\nQuestion 2: {}\nQuestion: Do both questions ask the same thing?\nAnswer:".format(
Jason Phang's avatar
Jason Phang committed
395
396
397
            doc["question1"],
            doc["question2"],
        )
398
399
400

    def doc_to_target(self, doc):
        return " {}".format(yesno(doc["label"]))
Jason Phang's avatar
Jason Phang committed
401

Jonathan Tow's avatar
Jonathan Tow committed
402
403
404
405
    def construct_requests(self, doc, ctx):
        ll_yes, _ = rf.loglikelihood(ctx, " yes")
        ll_no, _ = rf.loglikelihood(ctx, " no")
        return ll_yes, ll_no
406

Jonathan Tow's avatar
Jonathan Tow committed
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
    def process_results(self, doc, results):
        ll_yes, ll_no = results
        gold = doc["label"]
        pred = ll_yes > ll_no
        return {
            "acc": pred == gold,
            "f1": (gold, pred),
        }

    def higher_is_better(self):
        return {
            "acc": True,
            "f1": True
        }

    def aggregation(self):
        return {
            "acc": mean,
            "f1": f1_score
        }
Jason Phang's avatar
Jason Phang committed
427
428


sdtblck's avatar
sdtblck committed
429
class STSB(HFTask):
Leo Gao's avatar
Leo Gao committed
430
    VERSION = 0
sdtblck's avatar
sdtblck committed
431
432
    DATASET_PATH = "glue"
    DATASET_NAME = "stsb"
Jason Phang's avatar
Jason Phang committed
433
434
435
436
437
438
439
440
441
442

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
        return True

443
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
444
        return "sentence 1: {}\nsentence 2: {}\nAnswer:".format(
Jason Phang's avatar
Jason Phang committed
445
446
447
            doc["sentence1"],
            doc["sentence2"],
        )
448
449
450

    def doc_to_target(self, doc):
        return " {}".format(doc["label"])
Jason Phang's avatar
Jason Phang committed
451

Leo Gao's avatar
Leo Gao committed
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
    def construct_requests(self, doc, ctx):
        """ Uses RequestFactory to construct Requests and returns an iterable of 
        Requests which will be sent to the LM.

        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param ctx: str
            The context string, generated by fewshot_context. This includes the natural 
            language description, as well as the few shot examples, and the question
            part of the document for `doc`. 
        """
        # TODO: implement evaluation.
        raise NotImplementedError('Evaluation not implemented')
    
    def process_results(self, doc, results):
        """Take a single document and the LM results and evaluates, returning a 
        dict where keys are the names of submetrics and values are the values of 
        the metric for that one document

        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param results:
            The results of the requests created in construct_requests.
        """
        # TODO: implement evaluation.
        raise NotImplementedError('Evaluation not implemented')

    def aggregation(self):
        """
        :returns: {str: [float] -> float}
            A dictionary where keys are the names of submetrics and values are 
            functions that aggregate a list of metrics
        """
        # TODO: implement evaluation.
        raise NotImplementedError('Evaluation not implemented')

    def higher_is_better(self):
        """
        :returns: {str: bool}
            A dictionary where keys are the names of submetrics and values are 
            whether a higher value of the submetric is better
        """
        # TODO: implement evaluation.
        raise NotImplementedError('Evaluation not implemented')