qasper.py 7.49 KB
Newer Older
1
2
3
4
""" 
A Dataset of Information-Seeking Questions and Answers Anchored in Research Papers
https://arxiv.org/abs/2105.03011

5
6
7
8
9
10
11
QASPER is a dataset of 5,049 questions over 1,585 Natural Language Processing papers.
Each question is written by an NLP practitioner who read only the title and abstract
of the corresponding paper, and the question seeks information present in the full
text. The questions are then answered by a separate set of NLP practitioners who also
provide supporting evidence to answers.

Homepage: https://allenai.org/data/qasper
12
13
14
15
16
17
18
19
20
"""
from collections import Counter
from math import exp
import random
import re
import string
from lm_eval.base import rf
from lm_eval.metrics import f1_score, mean
from .common import HFTask
21

22
23

_CITATION = """
24
@article{DBLP:journals/corr/abs-2105-03011,
25
    author    = {Pradeep Dasigi and
26
27
28
29
30
               Kyle Lo and
               Iz Beltagy and
               Arman Cohan and
               Noah A. Smith and
               Matt Gardner},
31
    title     = {A Dataset of Information-Seeking Questions and Answers Anchored in
32
               Research Papers},
33
34
35
36
37
38
39
40
41
    journal   = {CoRR},
    volume    = {abs/2105.03011},
    year      = {2021},
    url       = {https://arxiv.org/abs/2105.03011},
    eprinttype = {arXiv},
    eprint    = {2105.03011},
    timestamp = {Fri, 14 May 2021 12:13:30 +0200},
    biburl    = {https://dblp.org/rec/journals/corr/abs-2105-03011.bib},
    bibsource = {dblp computer science bibliography, https://dblp.org}
42
43
}
"""
44
45


46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
def normalize_answer(s):
    """
    Taken from the official evaluation script for v1.1 of the SQuAD dataset.
    Lower text and remove punctuation, articles and extra whitespace.
    """

    def remove_articles(text):
        return re.sub(r"\b(a|an|the)\b", " ", text)

    def white_space_fix(text):
        return " ".join(text.split())

    def remove_punc(text):
        exclude = set(string.punctuation)
        return "".join(ch for ch in text if ch not in exclude)

    def lower(text):
        return text.lower()

    return white_space_fix(remove_articles(remove_punc(lower(s))))


def categorise_answer(answer_blob):
    if answer_blob["unanswerable"]:
        answer = "unanswerable"
        answer_type = "unanswerable"
        return answer, answer_type
    elif answer_blob["yes_no"]:
74
        answer = "yes"
75
76
77
78
79
80
81
82
        answer_type = "bool"
        return answer, answer_type
    elif answer_blob["free_form_answer"]:
        answer = answer_blob["free_form_answer"]
        answer_type = "free form answer"
        return answer, answer_type
    elif answer_blob["extractive_spans"]:
        answer = answer_blob["extractive_spans"]
Stephen Hogg's avatar
Stephen Hogg committed
83
        answer_type = "extractive_spans"
84
85
        return answer, answer_type
    elif answer_blob["yes_no"] is False:
86
        answer = "no"
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
        answer_type = "bool"
        return answer, answer_type


def token_f1_score(prediction, ground_truth):
    """
    Taken from the official evaluation script for v1.1 of the SQuAD dataset.
    """
    prediction_tokens = normalize_answer(prediction).split()
    ground_truth_tokens = normalize_answer(ground_truth).split()
    common = Counter(prediction_tokens) & Counter(ground_truth_tokens)
    num_same = sum(common.values())
    if num_same == 0:
        return 0
    precision = 1.0 * num_same / len(prediction_tokens)
    recall = 1.0 * num_same / len(ground_truth_tokens)
    f1 = (2 * precision * recall) / (precision + recall)
    return f1


107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
class QASPER(HFTask):
    VERSION = 0
    DATASET_PATH = "qasper"
    DATASET_NAME = None

    def doc_to_text(self, doc):
        return (
            "TITLE: "
            + doc["title"]
            + "\n"
            + "ABSTRACT: "
            + doc["abstract"]
            + "\n\n"
            + "Q: "
            + doc["question"]
            + "\n\n"
Stephen Hogg's avatar
Stephen Hogg committed
123
            + "A:"
124
125
126
        )

    def doc_to_target(self, doc):
Stephen Hogg's avatar
Stephen Hogg committed
127
128
129
130
        answer = doc["answer"]
        if isinstance(answer, list):
            answer = ", ".join(answer)
        return " " + answer
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146

    def training_docs(self):
        for doc in self.data["train"]:
            yield from self.process_doc(doc)

    def validation_docs(self):
        for doc in self.data["train"]:
            yield from self.process_doc(doc)

    def process_doc(self, doc):
        """Given a `doc`, flatten it out so that each JSON blob
        contains exactly one question and one answer. Logic taken from
        the reference implementation available at
        https://github.com/allenai/qasper-led-baseline/blob/main/scripts/evaluator.py
        """
        obs_list = []
147
148
149
150
151
152
153
154
155
156
157
158
        for question, answer_list in zip(doc["qas"]["question"], doc["qas"]["answers"]):
            for answer_blob in answer_list["answer"]:
                answer, answer_type = categorise_answer(answer_blob)
                obs_list.append(
                    {
                        "title": doc["title"],
                        "abstract": doc["abstract"],
                        "question": question,
                        "answer": answer,
                        "answer_type": answer_type,
                    }
                )
159
160
161
        return obs_list

    def process_results(self, doc, results):
Stephen Hogg's avatar
Stephen Hogg committed
162
163
        # TODO: Calculate a score for extractive spans once a request type for generating
        # extractive spans is available
164
165
166
167
        if not results:
            return {}
        elif len(results) == 1:
            [res] = results
Stephen Hogg's avatar
Stephen Hogg committed
168
        elif len(results) == 2:
169
            [ll_yes, ll_no] = results
170

Stephen Hogg's avatar
Stephen Hogg committed
171
172
173
174
        # TODO: Handle unanswerability first
        # unanswerable_gold = doc["answer_type"] == "unanswerable"
        # unanswerable_pred = exp(logprob_unanswerable)
        # res_dict["f1_unanswerable"] = (unanswerable_gold, unanswerable_pred)
175

176
        res_dict = {}
177
178
179
180
        # Handle yes/no questions
        if doc["answer_type"] == "bool":
            gold = 1 if doc["answer"] == "yes" else 0
            pred = ll_yes > ll_no
Stephen Hogg's avatar
Stephen Hogg committed
181
            res_dict["f1_yesno"] = (gold, pred)
182
183
184

        # Handle completions
        if doc["answer_type"] == "free form answer":
Stephen Hogg's avatar
Stephen Hogg committed
185
            res_dict["f1_abstractive"] = token_f1_score(res, doc["answer"])
186

187
188
189
        # TODO: Handle extraction
        # if doc["answer_type"] == "extractive_spans":
        #     res_dict["f1_extractive"] = 0
190
191
192
        return res_dict

    def aggregation(self):
Stephen Hogg's avatar
Stephen Hogg committed
193
194
195
196
        return {
            "f1_yesno": f1_score,
            "f1_abstractive": mean,
        }
197
198
199
200
201
202
203
204
205
206
207
208

    def construct_requests(self, doc, ctx):
        """Uses RequestFactory to construct Requests and returns an iterable of
        Requests which will be sent to the LM.

        :param doc:
            The document as returned from training_docs, validation_docs, or test_docs.
        :param ctx: str
            The context string, generated by fewshot_context. This includes the natural
            language description, as well as the few shot examples, and the question
            part of the document for `doc`.
        """
209
210
211
        # unanswerable = rf.loglikelihood(ctx, " " + "unanswerable")
        if doc["answer_type"] in ("free form answer"):
            return [rf.greedy_until(ctx, ["\n"])]
212
213
214
        elif doc["answer_type"] in ("bool"):
            ll_yes, _ = rf.loglikelihood(ctx, " yes")
            ll_no, _ = rf.loglikelihood(ctx, " no")
215
            return [ll_yes, ll_no]
216
        else:
217
            return []
218
219
220
221
222
223
224

    def higher_is_better(self):
        """
        :returns: {str: bool}
            A dictionary where keys are the names of submetrics and values are
            whether a higher value of the submetric is better
        """
Stephen Hogg's avatar
Stephen Hogg committed
225
226
227
228
        return {
            "f1_yesno": True,
            "f1_abstractive": True,
        }