anli.py 4.45 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
"""
Adversarial NLI: A New Benchmark for Natural Language Understanding
https://arxiv.org/pdf/1910.14599.pdf

Adversarial NLI (ANLI) is a dataset collected via an iterative, adversarial
human-and-model-in-the-loop procedure. It consists of three rounds that progressively
increase in difficulty and complexity, and each question-answer includes annotator-
provided explanations.

Homepage: "https://github.com/facebookresearch/anli"
11
12
"""
import numpy as np
Jonathan Tow's avatar
Jonathan Tow committed
13
14
from lm_eval.base import rf, Task
from lm_eval.metrics import mean
15

16
17

_CITATION = """
18
19
20
21
22
23
24
25
26
27
28
29
30
@inproceedings{nie-etal-2020-adversarial,
    title = "Adversarial {NLI}: A New Benchmark for Natural Language Understanding",
    author = "Nie, Yixin  and
      Williams, Adina  and
      Dinan, Emily  and
      Bansal, Mohit  and
      Weston, Jason  and
      Kiela, Douwe",
    booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics",
    year = "2020",
    publisher = "Association for Computational Linguistics",
}
"""
Leo Gao's avatar
Leo Gao committed
31

32

Jonathan Tow's avatar
Jonathan Tow committed
33
class ANLIBase(Task):
Leo Gao's avatar
Leo Gao committed
34
    VERSION = 0
Leo Gao's avatar
Leo Gao committed
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
    DATASET_PATH = "anli"
    DATASET_NAME = None
    SPLIT = None

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
        return True

    def training_docs(self):
        if self.has_training_docs():
            if self._training_docs is None:
Jonathan Tow's avatar
Jonathan Tow committed
51
                self._training_docs = list(self.dataset["train_r" + str(self.SPLIT)])
Leo Gao's avatar
Leo Gao committed
52
53
54
55
            return self._training_docs

    def validation_docs(self):
        if self.has_validation_docs():
Jonathan Tow's avatar
Jonathan Tow committed
56
            return self.dataset["dev_r" + str(self.SPLIT)]
Leo Gao's avatar
Leo Gao committed
57
58
59

    def test_docs(self):
        if self.has_test_docs():
Jonathan Tow's avatar
Jonathan Tow committed
60
            return self.dataset["test_r" + str(self.SPLIT)]
Leo Gao's avatar
Leo Gao committed
61

62
    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
63
        # OA does this a bit weirdly: they prepend "anli 1:  anli 1:  " to the beginning
bzantium's avatar
bzantium committed
64
65
        # of the prompt (yes, repeating it!). also, " True, False, or Neither?" is directly
        # appended onto the question, with no "Answer:" or even a newline. Do we *really*
Leo Gao's avatar
Leo Gao committed
66
        # want to do it exactly as OA did?
bzantium's avatar
bzantium committed
67
68
69
70
71
72
73
74
75
76
77
78
        return (
            doc["premise"]
            + "\nQuestion: "
            + doc["hypothesis"]
            + " True, False, or Neither?\nAnswer:"
        )

    def should_decontaminate(self):
        return True

    def doc_to_decontamination_query(self, doc):
        return doc["premise"]
Leo Gao's avatar
Leo Gao committed
79

80
    def doc_to_target(self, doc):
Jonathan Tow's avatar
Jonathan Tow committed
81
82
83
        # True = entailment
        # False = contradiction
        # Neither = neutral
bzantium's avatar
bzantium committed
84
        return " " + ["True", "Neither", "False"][doc["label"]]
Leo Gao's avatar
Leo Gao committed
85

Leo Gao's avatar
Leo Gao committed
86
    def construct_requests(self, doc, ctx):
bzantium's avatar
bzantium committed
87
        """Uses RequestFactory to construct Requests and returns an iterable of
Leo Gao's avatar
Leo Gao committed
88
89
90
91
92
        Requests which will be sent to the LM.

        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param ctx: str
bzantium's avatar
bzantium committed
93
            The context string, generated by fewshot_context. This includes the natural
Leo Gao's avatar
Leo Gao committed
94
            language description, as well as the few shot examples, and the question
bzantium's avatar
bzantium committed
95
            part of the document for `doc`.
Leo Gao's avatar
Leo Gao committed
96
        """
bzantium's avatar
bzantium committed
97
98
99
        ll_true, _ = rf.loglikelihood(ctx, " True")
        ll_neither, _ = rf.loglikelihood(ctx, " Neither")
        ll_false, _ = rf.loglikelihood(ctx, " False")
Jonathan Tow's avatar
Jonathan Tow committed
100
        return ll_true, ll_neither, ll_false
bzantium's avatar
bzantium committed
101

Leo Gao's avatar
Leo Gao committed
102
    def process_results(self, doc, results):
bzantium's avatar
bzantium committed
103
104
        """Take a single document and the LM results and evaluates, returning a
        dict where keys are the names of submetrics and values are the values of
Leo Gao's avatar
Leo Gao committed
105
106
107
108
109
110
111
        the metric for that one document

        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param results:
            The results of the requests created in construct_requests.
        """
Jonathan Tow's avatar
Jonathan Tow committed
112
113
        gold = doc["label"]
        pred = np.argmax(results)
bzantium's avatar
bzantium committed
114
        return {"acc": pred == gold}
Leo Gao's avatar
Leo Gao committed
115
116
117
118

    def aggregation(self):
        """
        :returns: {str: [float] -> float}
bzantium's avatar
bzantium committed
119
            A dictionary where keys are the names of submetrics and values are
Leo Gao's avatar
Leo Gao committed
120
121
            functions that aggregate a list of metrics
        """
bzantium's avatar
bzantium committed
122
        return {"acc": mean}
Leo Gao's avatar
Leo Gao committed
123
124
125
126

    def higher_is_better(self):
        """
        :returns: {str: bool}
bzantium's avatar
bzantium committed
127
            A dictionary where keys are the names of submetrics and values are
Leo Gao's avatar
Leo Gao committed
128
129
            whether a higher value of the submetric is better
        """
bzantium's avatar
bzantium committed
130
        return {"acc": True}
Leo Gao's avatar
Leo Gao committed
131

Jonathan Tow's avatar
Jonathan Tow committed
132

Leo Gao's avatar
Leo Gao committed
133
134
135
class ANLIRound1(ANLIBase):
    SPLIT = 1

Jonathan Tow's avatar
Jonathan Tow committed
136

Leo Gao's avatar
Leo Gao committed
137
138
139
class ANLIRound2(ANLIBase):
    SPLIT = 2

Jonathan Tow's avatar
Jonathan Tow committed
140

Leo Gao's avatar
Leo Gao committed
141
class ANLIRound3(ANLIBase):
Jonathan Tow's avatar
Jonathan Tow committed
142
    SPLIT = 3