drop.py 10.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
"""
DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs
https://aclanthology.org/attachments/N19-1246.Supplementary.pdf

DROP is a QA dataset which tests comprehensive understanding of paragraphs. In 
this crowdsourced, adversarially-created, 96k question-answering benchmark, a 
system must resolve multiple references in a question, map them onto a paragraph,
and perform discrete operations over them (such as addition, counting, or sorting).

Homepage: https://allenai.org/data/drop

Acknowledgement: This implementation is based on the official evaluation for `DROP`:
https://github.com/allenai/allennlp-reading-comprehension/blob/master/allennlp_rc/eval/drop_eval.py

@misc{dua2019drop,
      title={DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs}, 
      author={Dheeru Dua and Yizhong Wang and Pradeep Dasigi and Gabriel Stanovsky and Sameer Singh and Matt Gardner},
      year={2019},
      eprint={1903.00161},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}
"""
Anish Thite's avatar
Anish Thite committed
24
import json
Jon Tow's avatar
Jon Tow committed
25
26
import numpy as np
import re
27
import string
Jon Tow's avatar
Jon Tow committed
28
29
30
31
from best_download import download_file
from scipy.optimize import linear_sum_assignment
from lm_eval.base import Task, rf
from lm_eval.metrics import mean
Anish Thite's avatar
Anish Thite committed
32
from pathlib import Path
Jon Tow's avatar
Jon Tow committed
33
34
from zipfile import ZipFile

silentv0x's avatar
silentv0x committed
35
_ARTICLES = re.compile(r"\b(a|an|the)\b", re.UNICODE)
Anish Thite's avatar
Anish Thite committed
36

37

38
class DROP(Task):
Leo Gao's avatar
Leo Gao committed
39
    VERSION = 1
40
    DATASET_PATH = Path("data/drop")
Jon Tow's avatar
Jon Tow committed
41
42

    def download(self):
43
44
        if self.DATASET_PATH.exists():
            return
Jun Shern Chan's avatar
Jun Shern Chan committed
45
        Path.mkdir(self.DATASET_PATH, parents=True)
46
47
48
        url = "https://s3-us-west-2.amazonaws.com/allennlp/datasets/drop/drop_dataset.zip"
        checksum = "39d2278a29fd729de301b111a45f434c24834f40df8f4ff116d864589e3249d6"
        zip_path = self.DATASET_PATH / "drop_dataset.zip"
49
        download_file(url, local_file=str(zip_path), expected_checksum=checksum)
50
51
        with ZipFile(zip_path, "r") as zip:
            zip.extractall(self.DATASET_PATH)
52

Anish Thite's avatar
Anish Thite committed
53
54
    def has_training_docs(self):
        return True
Jon Tow's avatar
Jon Tow committed
55

Anish Thite's avatar
Anish Thite committed
56
57
58
59
60
61
    def has_validation_docs(self):
        return True

    def has_test_docs(self):
        return False

Jon Tow's avatar
Jon Tow committed
62
63
64
65
    def _load_docs(self, docs):
        for doc in docs:
            for qa in doc["qa_pairs"]:
                yield {
Jon Tow's avatar
Jon Tow committed
66
                    "id": qa["query_id"],
Jon Tow's avatar
Jon Tow committed
67
68
                    "passage": doc["passage"],
                    "question": qa["question"],
silentv0x's avatar
silentv0x committed
69
                    "answers": self.get_answers(qa),
Jon Tow's avatar
Jon Tow committed
70
                }
Anish Thite's avatar
Anish Thite committed
71

Jon Tow's avatar
Jon Tow committed
72
    @classmethod
silentv0x's avatar
silentv0x committed
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
    def get_answers(cls, qa):
        answers = []
        answers_set = set()

        candidates = [qa["answer"]] + qa.get("validated_answers", [])
        for candidate in candidates:
            answer = cls.parse_answer(candidate)
            if answer in answers_set:
                continue
            answers_set.add(answer)
            answers.append(answer)

        return answers

    @classmethod
    def parse_answer(cls, answer):
        # NOTE: Everything is returned as a tuple for uniformity and hashability.
        if answer["number"] != "":
            return (str(answer["number"]),)
        if answer["spans"] != []:
            return tuple(answer["spans"])
        return (" ".join([answer["date"]["day"],
                          answer["date"]["month"],
                          answer["date"]["year"]]).strip(),)
Jon Tow's avatar
Jon Tow committed
97
98

    def training_docs(self):
99
        docs = json.load(open(self.DATASET_PATH / "drop_dataset" / "drop_dataset_train.json"))
Jon Tow's avatar
Jon Tow committed
100
        return self._load_docs([docs[k] for k in docs.keys()])
Anish Thite's avatar
Anish Thite committed
101
102

    def validation_docs(self):
103
        docs = json.load(open(self.DATASET_PATH / "drop_dataset" / "drop_dataset_dev.json"))
Jon Tow's avatar
Jon Tow committed
104
105
106
107
108
109
        return self._load_docs([docs[k] for k in docs.keys()])

    def doc_to_text(self, doc):
        return f"Passage: {doc['passage']}\nQuestion: {doc['question']}\nAnswer:"

    def doc_to_target(self, doc):
silentv0x's avatar
silentv0x committed
110
        return " " + ", ".join(doc["answers"][0])
Anish Thite's avatar
Anish Thite committed
111

Leo Gao's avatar
Leo Gao committed
112
    def construct_requests(self, doc, ctx):
Jon Tow's avatar
Jon Tow committed
113
        """Uses RequestFactory to construct Requests and returns an iterable of
Leo Gao's avatar
Leo Gao committed
114
        Requests which will be sent to the LM.
115

Jon Tow's avatar
Jon Tow committed
116
        :param doc:
Leo Gao's avatar
Leo Gao committed
117
            The document as returned from training_docs, validation_docs, or test_docs.
Jon Tow's avatar
Jon Tow committed
118
        :param ctx: str
Jon Tow's avatar
Jon Tow committed
119
            The context string, generated by fewshot_context. This includes the natural
Leo Gao's avatar
Leo Gao committed
120
            language description, as well as the few shot examples, and the question
Jon Tow's avatar
Jon Tow committed
121
            part of the document for `doc`.
Leo Gao's avatar
Leo Gao committed
122
        """
silentv0x's avatar
silentv0x committed
123
        conts = [rf.greedy_until(ctx, ["."])]
Jon Tow's avatar
Jon Tow committed
124
125
        return conts

Leo Gao's avatar
Leo Gao committed
126
    def process_results(self, doc, results):
Jon Tow's avatar
Jon Tow committed
127
128
        """Take a single document and the LM results and evaluates, returning a
        dict where keys are the names of submetrics and values are the values of
Leo Gao's avatar
Leo Gao committed
129
130
        the metric for that one document

Jon Tow's avatar
Jon Tow committed
131
        :param doc:
Jon Tow's avatar
Jon Tow committed
132
            The document as returned from training_docs, validation_docs, or test_docs.
Leo Gao's avatar
Leo Gao committed
133
134
135
        :param results:
            The results of the requests created in construct_requests.
        """
136
        preds, golds = results, doc["answers"]
silentv0x's avatar
silentv0x committed
137
138
139
140
141
142
143
        max_em = 0
        max_f1 = 0
        for gold_answer in golds:
            exact_match, f1_score = self.get_metrics(preds, gold_answer)
            if gold_answer[0].strip():
                max_em = max(max_em, exact_match)
                max_f1 = max(max_f1, f1_score)
Jon Tow's avatar
Jon Tow committed
144
        return {
silentv0x's avatar
silentv0x committed
145
146
            "em": max_em,
            "f1": max_f1
Jon Tow's avatar
Jon Tow committed
147
        }
Jon Tow's avatar
Jon Tow committed
148

silentv0x's avatar
silentv0x committed
149
150
151
152
153
154
155
    def get_metrics(self, predicted, gold):
        """
        Takes a predicted answer and a gold answer (that are both either a string or a list of
        strings), and returns exact match and the DROP F1 metric for the prediction.  If you are
        writing a script for evaluating objects in memory (say, the output of predictions during
        validation, or while training), this is the function you want to call, after using
        :func:`answer_json_to_strings` when reading the gold answer from the released data file.
156
        """
silentv0x's avatar
silentv0x committed
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
        predicted_bags = self._answer_to_bags(predicted)
        gold_bags = self._answer_to_bags(gold)

        if set(predicted_bags[0]) == set(gold_bags[0]) and len(predicted_bags[0]) == len(gold_bags[0]):
            exact_match = 1.0
        else:
            exact_match = 0.0

        f1_per_bag = self._align_bags(predicted_bags[1], gold_bags[1])
        f1 = np.mean(f1_per_bag)
        f1 = round(f1, 2)
        return exact_match, f1

    def _answer_to_bags(self, answer):
        if isinstance(answer, (list, tuple)):
            raw_spans = answer
        else:
            raw_spans = [answer]
        normalized_spans = []
        token_bags = []
        for raw_span in raw_spans:
            normalized_span = self._normalize(raw_span)
            normalized_spans.append(normalized_span)
            token_bags.append(set(normalized_span.split()))
        return normalized_spans, token_bags

    def _align_bags(self, predicted, gold):
        """
        Takes gold and predicted answer sets and first finds the optimal 1-1 alignment
        between them and gets maximum metric values over all the answers.
        """
        scores = np.zeros([len(gold), len(predicted)])
        for gold_index, gold_item in enumerate(gold):
            for pred_index, pred_item in enumerate(predicted):
                if self._match_numbers_if_present(gold_item, pred_item):
                    scores[gold_index, pred_index] = self._compute_f1(pred_item, gold_item)
Jon Tow's avatar
Jon Tow committed
193
        row_ind, col_ind = linear_sum_assignment(-scores)
silentv0x's avatar
silentv0x committed
194
195

        max_scores = np.zeros([max(len(gold), len(predicted))])
Jon Tow's avatar
Jon Tow committed
196
197
198
199
        for row, column in zip(row_ind, col_ind):
            max_scores[row] = max(max_scores[row], scores[row, column])
        return max_scores

silentv0x's avatar
silentv0x committed
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
    def _compute_f1(self, predicted_bag, gold_bag):
        intersection = len(gold_bag.intersection(predicted_bag))
        if not predicted_bag:
            precision = 1.0
        else:
            precision = intersection / float(len(predicted_bag))
        if not gold_bag:
            recall = 1.0
        else:
            recall = intersection / float(len(gold_bag))
        f1 = (
            (2 * precision * recall) / (precision + recall)
            if not (precision == 0.0 and recall == 0.0)
            else 0.0
        )
Jon Tow's avatar
Jon Tow committed
215
216
        return f1

silentv0x's avatar
silentv0x committed
217
218
219
220
221
222
223
224
225
226
    def _match_numbers_if_present(self, gold_bag, predicted_bag):
        gold_numbers = set()
        predicted_numbers = set()
        for word in gold_bag:
            if self._is_number(word):
                gold_numbers.add(word)
        for word in predicted_bag:
            if self._is_number(word):
                predicted_numbers.add(word)
        if (not gold_numbers) or gold_numbers.intersection(predicted_numbers):
227
228
229
230
231
232
233
234
235
            return True
        return False

    def _is_number(self, text):
        try:
            float(text)
            return True
        except ValueError:
            return False
Jon Tow's avatar
Jon Tow committed
236

silentv0x's avatar
silentv0x committed
237
238
    def _remove_articles(self, text):
        return _ARTICLES.sub(" ", text)
239

silentv0x's avatar
silentv0x committed
240
241
    def _white_space_fix(self, text):
        return " ".join(text.split())
242

silentv0x's avatar
silentv0x committed
243
244
245
246
247
248
    def _remove_punc(self, text):
        exclude = set(string.punctuation)
        if not self._is_number(text):
            return "".join(ch for ch in text if ch not in exclude)
        else:
            return text
249

silentv0x's avatar
silentv0x committed
250
251
    def _fix_number(self, text):
        return str(float(text)) if self._is_number(text) else text
252

silentv0x's avatar
Bug fix  
silentv0x committed
253
    def _tokenize(self, text):
silentv0x's avatar
silentv0x committed
254
        return re.split(" |-", text)
255

silentv0x's avatar
silentv0x committed
256
    def _normalize(self, answer):
257
        tokens = [
silentv0x's avatar
silentv0x committed
258
259
            self._white_space_fix(self._remove_articles(self._fix_number(self._remove_punc(token.lower()))))
            for token in self._tokenize(answer)
260
        ]
Jon Tow's avatar
Fixes  
Jon Tow committed
261
        tokens = [token for token in tokens if token.strip()]
Jon Tow's avatar
Jon Tow committed
262
263
        normalized = " ".join(tokens).strip()
        return normalized
Leo Gao's avatar
Leo Gao committed
264
265
266
267

    def aggregation(self):
        """
        :returns: {str: [float] -> float}
Jon Tow's avatar
Jon Tow committed
268
269
            A dictionary where keys are the names of submetrics and values are
            functions that aggregate a list of metrics
Leo Gao's avatar
Leo Gao committed
270
        """
Jon Tow's avatar
Jon Tow committed
271
272
273
274
        return {
            "em": mean,
            "f1": mean
        }
Leo Gao's avatar
Leo Gao committed
275
276
277
278

    def higher_is_better(self):
        """
        :returns: {str: bool}
Jon Tow's avatar
Jon Tow committed
279
280
            A dictionary where keys are the names of submetrics and values are
            whether a higher value of the submetric is better
Leo Gao's avatar
Leo Gao committed
281
        """
Jon Tow's avatar
Jon Tow committed
282
283
284
285
        return {
            "em": True,
            "f1": True
        }