qa4mre.py 2.16 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
"""
QA4MRE 2011-2013: Overview of Question Answering for Machine Reading Evaluation
https://www.cs.cmu.edu/~./hovy/papers/13CLEF-QA4MRE.pdf

The (English only) QA4MRE challenge which was run as a Lab at CLEF 2011-2013.
The main objective of this exercise is to develop a methodology for evaluating 
Machine Reading systems through Question Answering and Reading Comprehension 
Tests. Systems should be able to extract knowledge from large volumes of text 
and use this knowledge to answer questions. Four different tasks have been
organized during these years: Main Task, Processing Modality and Negation for
Machine Reading, Machine Reading of Biomedical Texts about Alzheimer's disease,
and Entrance Exam.

Homepage: http://nlp.uned.es/clef-qa/repository/qa4mre.php
"""
16
17
from lm_eval.base import MultipleChoiceTask

18

19
20
21
22
23
24
25
26
27
28
_CITATION = """
@inproceedings{Peas2013QA4MRE2O,
    title={QA4MRE 2011-2013: Overview of Question Answering for Machine Reading Evaluation},
    author={Anselmo Pe{\~n}as and Eduard H. Hovy and Pamela Forner and {\'A}lvaro Rodrigo and Richard F. E. Sutcliffe and Roser Morante},
    booktitle={CLEF},
    year={2013}
}
"""


29
class QA4MRE(MultipleChoiceTask):
Leo Gao's avatar
Leo Gao committed
30
    VERSION = 0
Jonathan Tow's avatar
Jonathan Tow committed
31
32
    DATASET_PATH = "qa4mre"
    DATASET_NAME = None
33
34
35
36
37
38
39
40
41
42

    def has_training_docs(self):
        return False

    def has_validation_docs(self):
        return False

    def has_test_docs(self):
        return True

Jonathan Tow's avatar
Jonathan Tow committed
43
44
    def test_docs(self):
        # `qa4mre` only has train data so we use it for the test docs.
Jon Tow's avatar
Jon Tow committed
45
        return map(self._process_doc, self.dataset["train"])
Jonathan Tow's avatar
Jonathan Tow committed
46

Jon Tow's avatar
Jon Tow committed
47
    def _process_doc(self, doc):
Jonathan Tow's avatar
Jonathan Tow committed
48
        choices = doc["answer_options"]["answer_str"]
49
        out_doc = {
Jonathan Tow's avatar
Jonathan Tow committed
50
51
52
53
            "source": doc["document_str"].strip().replace("\'", "'"),
            "query": doc["question_str"],
            "choices": choices,
            "gold": int(doc["correct_answer_id"]) - 1,
54
55
56
57
        }
        return out_doc

    def doc_to_text(self, doc):
Leo Gao's avatar
Leo Gao committed
58
        return "{}\nQuestion: {}\nAnswer:".format(doc["source"], doc["query"])
59

Jonathan Tow's avatar
Jonathan Tow committed
60

61
class QA4MRE_2011(QA4MRE):
Jonathan Tow's avatar
Jonathan Tow committed
62
63
    DATASET_NAME = "2011.main.EN"

64
65

class QA4MRE_2012(QA4MRE):
Jonathan Tow's avatar
Jonathan Tow committed
66
67
    DATASET_NAME = "2012.main.EN"

68
69

class QA4MRE_2013(QA4MRE):
Jonathan Tow's avatar
Jonathan Tow committed
70
    DATASET_NAME = "2013.main.EN"