winogrande.py 3.58 KB
Newer Older
Charles Foster's avatar
Charles Foster committed
1
import numpy as np
2
from . common import HFTask
&'s avatar
& committed
3
4
from lm_eval.base import rf
from ..metrics import mean
5
6
7
8
9
10
11

"""
This evaluation of Winogrande uses partial evaluation as described by
Trinh & Le in Simple Method for Commonsense Reasoning (2018).
Reference: https://arxiv.org/abs/1806.02847
"""

Charles Foster's avatar
Charles Foster committed
12
13

class Winogrande(HFTask):
Leo Gao's avatar
Leo Gao committed
14
    VERSION = 0
Charles Foster's avatar
Charles Foster committed
15
16
17
    DATASET_PATH = "winogrande"
    DATASET_NAME = "winogrande_xl"

18
19
    answer_to_num = {'1': 0, '2': 1}

Charles Foster's avatar
Charles Foster committed
20
21
22
23
24
25
26
    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
27
        return False
Charles Foster's avatar
Charles Foster committed
28

29
30
    def doc_to_text(self, doc):
        return self.partial_context(doc, doc["option" + doc["answer"]])
Charles Foster's avatar
Charles Foster committed
31

32
    @classmethod
33
34
    def partial_context(cls, doc, option):
        # Substitute the pronoun in the sentence with the specified option
35
36
        # and ignore everything after.
        pronoun_loc = doc["sentence"].index("_")
37
38
39
        return doc["sentence"][:pronoun_loc] + option

    def doc_to_target(self, doc):
Leo Gao's avatar
Leo Gao committed
40
        return self.partial_target(doc)
41
42
43
44
45

    @classmethod
    def partial_target(cls, doc):
        # The target is everything after the document specified pronoun.
        pronoun_loc = doc["sentence"].index("_") + 1
Leo Gao's avatar
Leo Gao committed
46
        return " " + doc["sentence"][pronoun_loc:].strip()
47

Leo Gao's avatar
Leo Gao committed
48
    def construct_requests(self, doc, ctx):
49
        """Uses RequestFactory to construct Requests and returns an iterable of
Leo Gao's avatar
Leo Gao committed
50
        Requests which will be sent to the LM.
51

Leo Gao's avatar
Leo Gao committed
52
53
54
        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param ctx: str
55
            The context string, generated by fewshot_context. This includes the natural
Leo Gao's avatar
Leo Gao committed
56
            language description, as well as the few shot examples, and the question
57
            part of the document for `doc`.
Leo Gao's avatar
Leo Gao committed
58
        """
59
        target = self.partial_target(doc)
60
        lls = []
61
62
63
        for option in [doc["option1"], doc["option2"]]:
            partial_ctx = self.partial_context(doc, option)
            full_ctx = self.append_context(ctx, partial_ctx)
64
65
            lls.append(rf.loglikelihood(full_ctx, target)[0])
        return lls
66
67
68

    @classmethod
    def append_context(cls, ctx, partial_ctx):
69
        ctx = ctx.split("\n\n")  # Each fewshot context is on its own new line.
70
71
        ctx.pop()  # Remove the correct context put in by `doc_to_text`.
        return "\n\n".join([*ctx, partial_ctx]) if ctx else partial_ctx
72

Leo Gao's avatar
Leo Gao committed
73
    def process_results(self, doc, results):
74
75
        """Take a single document and the LM results and evaluates, returning a
        dict where keys are the names of submetrics and values are the values of
Leo Gao's avatar
Leo Gao committed
76
77
78
79
80
81
82
        the metric for that one document

        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param results:
            The results of the requests created in construct_requests.
        """
83
        return {
84
            "acc": np.argmax(results) == self.answer_to_num[doc["answer"]]
85
        }
Leo Gao's avatar
Leo Gao committed
86
87
88
89

    def aggregation(self):
        """
        :returns: {str: [float] -> float}
90
            A dictionary where keys are the names of submetrics and values are
Leo Gao's avatar
Leo Gao committed
91
92
            functions that aggregate a list of metrics
        """
93
94
95
        return {
            "acc": mean
        }
Leo Gao's avatar
Leo Gao committed
96
97
98
99

    def higher_is_better(self):
        """
        :returns: {str: bool}
100
            A dictionary where keys are the names of submetrics and values are
Leo Gao's avatar
Leo Gao committed
101
102
            whether a higher value of the submetric is better
        """
103
104
105
        return {
            "acc": True
        }