qasper.py 7.61 KB
Newer Older
bzantium's avatar
bzantium committed
1
"""
2
3
4
A Dataset of Information-Seeking Questions and Answers Anchored in Research Papers
https://arxiv.org/abs/2105.03011

5
6
7
8
9
10
11
QASPER is a dataset of 5,049 questions over 1,585 Natural Language Processing papers.
Each question is written by an NLP practitioner who read only the title and abstract
of the corresponding paper, and the question seeks information present in the full
text. The questions are then answered by a separate set of NLP practitioners who also
provide supporting evidence to answers.

Homepage: https://allenai.org/data/qasper
12
13
14
15
"""
from collections import Counter
import re
import string
Jonathan Tow's avatar
Jonathan Tow committed
16
from lm_eval.base import rf, Task
17
from lm_eval.metrics import f1_score, mean
18

19
20

_CITATION = """
21
@article{DBLP:journals/corr/abs-2105-03011,
22
    author    = {Pradeep Dasigi and
23
24
25
26
27
               Kyle Lo and
               Iz Beltagy and
               Arman Cohan and
               Noah A. Smith and
               Matt Gardner},
28
    title     = {A Dataset of Information-Seeking Questions and Answers Anchored in
29
               Research Papers},
30
31
32
33
34
35
36
37
38
    journal   = {CoRR},
    volume    = {abs/2105.03011},
    year      = {2021},
    url       = {https://arxiv.org/abs/2105.03011},
    eprinttype = {arXiv},
    eprint    = {2105.03011},
    timestamp = {Fri, 14 May 2021 12:13:30 +0200},
    biburl    = {https://dblp.org/rec/journals/corr/abs-2105-03011.bib},
    bibsource = {dblp computer science bibliography, https://dblp.org}
39
40
}
"""
41
42


43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
def normalize_answer(s):
    """
    Taken from the official evaluation script for v1.1 of the SQuAD dataset.
    Lower text and remove punctuation, articles and extra whitespace.
    """

    def remove_articles(text):
        return re.sub(r"\b(a|an|the)\b", " ", text)

    def white_space_fix(text):
        return " ".join(text.split())

    def remove_punc(text):
        exclude = set(string.punctuation)
        return "".join(ch for ch in text if ch not in exclude)

    def lower(text):
        return text.lower()

    return white_space_fix(remove_articles(remove_punc(lower(s))))


def categorise_answer(answer_blob):
    if answer_blob["unanswerable"]:
        answer = "unanswerable"
        answer_type = "unanswerable"
        return answer, answer_type
    elif answer_blob["yes_no"]:
71
        answer = "yes"
72
73
74
75
76
77
78
79
        answer_type = "bool"
        return answer, answer_type
    elif answer_blob["free_form_answer"]:
        answer = answer_blob["free_form_answer"]
        answer_type = "free form answer"
        return answer, answer_type
    elif answer_blob["extractive_spans"]:
        answer = answer_blob["extractive_spans"]
Stephen Hogg's avatar
Stephen Hogg committed
80
        answer_type = "extractive_spans"
81
82
        return answer, answer_type
    elif answer_blob["yes_no"] is False:
83
        answer = "no"
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
        answer_type = "bool"
        return answer, answer_type


def token_f1_score(prediction, ground_truth):
    """
    Taken from the official evaluation script for v1.1 of the SQuAD dataset.
    """
    prediction_tokens = normalize_answer(prediction).split()
    ground_truth_tokens = normalize_answer(ground_truth).split()
    common = Counter(prediction_tokens) & Counter(ground_truth_tokens)
    num_same = sum(common.values())
    if num_same == 0:
        return 0
    precision = 1.0 * num_same / len(prediction_tokens)
    recall = 1.0 * num_same / len(ground_truth_tokens)
    f1 = (2 * precision * recall) / (precision + recall)
    return f1


Jonathan Tow's avatar
Jonathan Tow committed
104
class QASPER(Task):
105
106
107
108
    VERSION = 0
    DATASET_PATH = "qasper"
    DATASET_NAME = None

Jonathan Tow's avatar
Jonathan Tow committed
109
110
111
112
113
114
115
116
117
    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
        return False

118
119
120
121
122
123
124
125
126
127
128
    def doc_to_text(self, doc):
        return (
            "TITLE: "
            + doc["title"]
            + "\n"
            + "ABSTRACT: "
            + doc["abstract"]
            + "\n\n"
            + "Q: "
            + doc["question"]
            + "\n\n"
Stephen Hogg's avatar
Stephen Hogg committed
129
            + "A:"
130
131
132
        )

    def doc_to_target(self, doc):
Stephen Hogg's avatar
Stephen Hogg committed
133
134
135
136
        answer = doc["answer"]
        if isinstance(answer, list):
            answer = ", ".join(answer)
        return " " + answer
137
138

    def training_docs(self):
Jonathan Tow's avatar
Jonathan Tow committed
139
        for doc in self.dataset["train"]:
Jon Tow's avatar
Jon Tow committed
140
            yield from self._process_doc(doc)
141
142

    def validation_docs(self):
Jonathan Tow's avatar
Jonathan Tow committed
143
        for doc in self.dataset["validation"]:
Jon Tow's avatar
Jon Tow committed
144
            yield from self._process_doc(doc)
145

Jon Tow's avatar
Jon Tow committed
146
    def _process_doc(self, doc):
147
148
149
150
151
152
        """Given a `doc`, flatten it out so that each JSON blob
        contains exactly one question and one answer. Logic taken from
        the reference implementation available at
        https://github.com/allenai/qasper-led-baseline/blob/main/scripts/evaluator.py
        """
        obs_list = []
153
154
155
156
157
158
159
160
161
162
163
164
        for question, answer_list in zip(doc["qas"]["question"], doc["qas"]["answers"]):
            for answer_blob in answer_list["answer"]:
                answer, answer_type = categorise_answer(answer_blob)
                obs_list.append(
                    {
                        "title": doc["title"],
                        "abstract": doc["abstract"],
                        "question": question,
                        "answer": answer,
                        "answer_type": answer_type,
                    }
                )
165
166
167
        return obs_list

    def process_results(self, doc, results):
Stephen Hogg's avatar
Stephen Hogg committed
168
169
        # TODO: Calculate a score for extractive spans once a request type for generating
        # extractive spans is available
170
171
172
173
        if not results:
            return {}
        elif len(results) == 1:
            [res] = results
Stephen Hogg's avatar
Stephen Hogg committed
174
        elif len(results) == 2:
175
            [ll_yes, ll_no] = results
176

Stephen Hogg's avatar
Stephen Hogg committed
177
178
179
180
        # TODO: Handle unanswerability first
        # unanswerable_gold = doc["answer_type"] == "unanswerable"
        # unanswerable_pred = exp(logprob_unanswerable)
        # res_dict["f1_unanswerable"] = (unanswerable_gold, unanswerable_pred)
181

182
        res_dict = {}
183
184
185
186
        # Handle yes/no questions
        if doc["answer_type"] == "bool":
            gold = 1 if doc["answer"] == "yes" else 0
            pred = ll_yes > ll_no
Stephen Hogg's avatar
Stephen Hogg committed
187
            res_dict["f1_yesno"] = (gold, pred)
188
189
190

        # Handle completions
        if doc["answer_type"] == "free form answer":
Stephen Hogg's avatar
Stephen Hogg committed
191
            res_dict["f1_abstractive"] = token_f1_score(res, doc["answer"])
192

193
194
195
        # TODO: Handle extraction
        # if doc["answer_type"] == "extractive_spans":
        #     res_dict["f1_extractive"] = 0
196
197
198
        return res_dict

    def aggregation(self):
Stephen Hogg's avatar
Stephen Hogg committed
199
200
201
202
        return {
            "f1_yesno": f1_score,
            "f1_abstractive": mean,
        }
203
204
205
206
207
208
209
210
211
212
213
214

    def construct_requests(self, doc, ctx):
        """Uses RequestFactory to construct Requests and returns an iterable of
        Requests which will be sent to the LM.

        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param ctx: str
            The context string, generated by fewshot_context. This includes the natural
            language description, as well as the few shot examples, and the question
            part of the document for `doc`.
        """
215
216
        # unanswerable = rf.loglikelihood(ctx, " " + "unanswerable")
        if doc["answer_type"] in ("free form answer"):
bzantium's avatar
bzantium committed
217
            return [rf.greedy_until(ctx, {"until": ["\n"]})]
218
219
220
        elif doc["answer_type"] in ("bool"):
            ll_yes, _ = rf.loglikelihood(ctx, " yes")
            ll_no, _ = rf.loglikelihood(ctx, " no")
221
            return [ll_yes, ll_no]
222
        else:
223
            return []
224
225
226
227
228
229
230

    def higher_is_better(self):
        """
        :returns: {str: bool}
            A dictionary where keys are the names of submetrics and values are
            whether a higher value of the submetric is better
        """
Stephen Hogg's avatar
Stephen Hogg committed
231
232
233
234
        return {
            "f1_yesno": True,
            "f1_abstractive": True,
        }