winogrande.py 4.63 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
"""
WinoGrande: An Adversarial Winograd Schema Challenge at Scale
https://arxiv.org/pdf/1907.10641.pdf

WinoGrande is a collection of 44k problems, inspired by Winograd Schema Challenge
(Levesque, Davis, and Morgenstern 2011), but adjusted to improve the scale and
robustness against the dataset-specific bias. Formulated as a fill-in-a-blank
task with binary options, the goal is to choose the right option for a given
sentence which requires commonsense reasoning.

NOTE: This evaluation of Winogrande uses partial evaluation as described by
Trinh & Le in Simple Method for Commonsense Reasoning (2018). 
See: https://arxiv.org/abs/1806.02847

Homepage: https://leaderboard.allenai.org/winogrande/submissions/public
16
17
"""
import numpy as np
Jonathan Tow's avatar
Jonathan Tow committed
18
19
from lm_eval.base import rf, Task
from lm_eval.metrics import mean
20

21
22

_CITATION = """
23
24
25
26
27
28
29
@article{sakaguchi2019winogrande,
    title={WinoGrande: An Adversarial Winograd Schema Challenge at Scale},
    author={Sakaguchi, Keisuke and Bras, Ronan Le and Bhagavatula, Chandra and Choi, Yejin},
    journal={arXiv preprint arXiv:1907.10641},
    year={2019}
}
"""
30

Charles Foster's avatar
Charles Foster committed
31

Jonathan Tow's avatar
Jonathan Tow committed
32
class Winogrande(Task):
Leo Gao's avatar
Leo Gao committed
33
    VERSION = 0
Charles Foster's avatar
Charles Foster committed
34
35
36
    DATASET_PATH = "winogrande"
    DATASET_NAME = "winogrande_xl"

37
38
    answer_to_num = {'1': 0, '2': 1}

Charles Foster's avatar
Charles Foster committed
39
40
41
42
43
44
45
    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
46
        return False
Charles Foster's avatar
Charles Foster committed
47

Jonathan Tow's avatar
Jonathan Tow committed
48
49
50
51
52
53
54
55
    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

    def validation_docs(self):
        return self.dataset["validation"]

56
57
    def doc_to_text(self, doc):
        return self.partial_context(doc, doc["option" + doc["answer"]])
Charles Foster's avatar
Charles Foster committed
58

59
    @classmethod
60
61
    def partial_context(cls, doc, option):
        # Substitute the pronoun in the sentence with the specified option
62
63
        # and ignore everything after.
        pronoun_loc = doc["sentence"].index("_")
64
65
66
        return doc["sentence"][:pronoun_loc] + option

    def doc_to_target(self, doc):
Leo Gao's avatar
Leo Gao committed
67
        return self.partial_target(doc)
68
69
70
71
72

    @classmethod
    def partial_target(cls, doc):
        # The target is everything after the document specified pronoun.
        pronoun_loc = doc["sentence"].index("_") + 1
Leo Gao's avatar
Leo Gao committed
73
        return " " + doc["sentence"][pronoun_loc:].strip()
74

Leo Gao's avatar
Leo Gao committed
75
    def construct_requests(self, doc, ctx):
76
        """Uses RequestFactory to construct Requests and returns an iterable of
Leo Gao's avatar
Leo Gao committed
77
        Requests which will be sent to the LM.
78

Leo Gao's avatar
Leo Gao committed
79
80
81
        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param ctx: str
82
            The context string, generated by fewshot_context. This includes the natural
Leo Gao's avatar
Leo Gao committed
83
            language description, as well as the few shot examples, and the question
84
            part of the document for `doc`.
Leo Gao's avatar
Leo Gao committed
85
        """
86
        target = self.partial_target(doc)
87
        lls = []
88
89
90
        for option in [doc["option1"], doc["option2"]]:
            partial_ctx = self.partial_context(doc, option)
            full_ctx = self.append_context(ctx, partial_ctx)
91
92
            lls.append(rf.loglikelihood(full_ctx, target)[0])
        return lls
93
94
95

    @classmethod
    def append_context(cls, ctx, partial_ctx):
96
        ctx = ctx.split("\n\n")  # Each fewshot context is on its own new line.
97
98
        ctx.pop()  # Remove the correct context put in by `doc_to_text`.
        return "\n\n".join([*ctx, partial_ctx]) if ctx else partial_ctx
99

Leo Gao's avatar
Leo Gao committed
100
    def process_results(self, doc, results):
101
102
        """Take a single document and the LM results and evaluates, returning a
        dict where keys are the names of submetrics and values are the values of
Leo Gao's avatar
Leo Gao committed
103
104
105
106
107
108
109
        the metric for that one document

        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param results:
            The results of the requests created in construct_requests.
        """
110
        return {
111
            "acc": np.argmax(results) == self.answer_to_num[doc["answer"]]
112
        }
Leo Gao's avatar
Leo Gao committed
113
114
115
116

    def aggregation(self):
        """
        :returns: {str: [float] -> float}
117
            A dictionary where keys are the names of submetrics and values are
Leo Gao's avatar
Leo Gao committed
118
119
            functions that aggregate a list of metrics
        """
120
121
122
        return {
            "acc": mean
        }
Leo Gao's avatar
Leo Gao committed
123
124
125
126

    def higher_is_better(self):
        """
        :returns: {str: bool}
127
            A dictionary where keys are the names of submetrics and values are
Leo Gao's avatar
Leo Gao committed
128
129
            whether a higher value of the submetric is better
        """
130
131
132
        return {
            "acc": True
        }