arc.py 2.59 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
"""
Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge
https://arxiv.org/pdf/1803.05457.pdf

The ARC dataset consists of 7,787 science exam questions drawn from a variety
of sources, including science questions provided under license by a research
partner affiliated with AI2. These are text-only, English language exam questions
that span several grade levels as indicated in the files. Each question has a
multiple choice structure (typically 4 answer options). The questions are sorted
into a Challenge Set of 2,590 “hard” questions (those that both a retrieval and
a co-occurrence method fail to answer correctly) and an Easy Set of 5,197 questions.

Homepage: https://allenai.org/data/arc
14
15
"""
from lm_eval.base import MultipleChoiceTask
16

17
18

_CITATION = """
19
20
21
22
23
24
25
26
@article{Clark2018ThinkYH,
  title={Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge},
  author={Peter Clark and Isaac Cowhey and Oren Etzioni and Tushar Khot and Ashish Sabharwal and Carissa Schoenick and Oyvind Tafjord},
  journal={ArXiv},
  year={2018},
  volume={abs/1803.05457}
}
"""
Leo Gao's avatar
Leo Gao committed
27

Jonathan Tow's avatar
Jonathan Tow committed
28

Jonathan Tow's avatar
Jonathan Tow committed
29
class ARCEasy(MultipleChoiceTask):
Leo Gao's avatar
Leo Gao committed
30
    VERSION = 0
Leo Gao's avatar
Leo Gao committed
31
32
    DATASET_PATH = "ai2_arc"
    DATASET_NAME = "ARC-Easy"
Leo Gao's avatar
Leo Gao committed
33
34
35
36
37
38
39
40
41
42

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
        return True

Jonathan Tow's avatar
Jonathan Tow committed
43
44
    def training_docs(self):
        if self._training_docs is None:
Jon Tow's avatar
Jon Tow committed
45
            self._training_docs = list(map(self._process_doc, self.dataset["train"]))
Jon Tow's avatar
Jon Tow committed
46
        return self._training_docs
Jonathan Tow's avatar
Jonathan Tow committed
47
48

    def validation_docs(self):
Jon Tow's avatar
Jon Tow committed
49
        return map(self._process_doc, self.dataset["validation"])
Jonathan Tow's avatar
Jonathan Tow committed
50
51

    def test_docs(self):
Jon Tow's avatar
Jon Tow committed
52
        return map(self._process_doc, self.dataset["test"])
Jonathan Tow's avatar
Jonathan Tow committed
53

Jon Tow's avatar
Jon Tow committed
54
    def _process_doc(self, doc):
55
56
57
58
59
60
61
62
63
64
65
        # NOTE: Some `doc["answerKey"]`s are in numeric string format being one
        # of {'1', '2', '3', '4', '5'}. We map them back to letters.
        num_to_letter = {"1": "A", "2": "B", "3": "C", "4": "D", "5": "E"}
        doc["answerKey"] = num_to_letter.get(doc["answerKey"], doc["answerKey"])
        out_doc = {
            "id": doc["id"],
            "query": "Question: " + doc["question"] + "\nAnswer:",
            "choices": doc["choices"]["text"],
            "gold": ["A", "B", "C", "D", "E"].index(doc["answerKey"]),
        }
        return out_doc
Leo Gao's avatar
Leo Gao committed
66

67
68
    def doc_to_text(self, doc):
        return doc["query"]
Jonathan Tow's avatar
Jonathan Tow committed
69

bzantium's avatar
bzantium committed
70
71
72
73
74
75
    def should_decontaminate(self):
        return True

    def doc_to_decontamination_query(self, doc):
        return doc["query"]

Leo Gao's avatar
Leo Gao committed
76
77

class ARCChallenge(ARCEasy):
Leo Gao's avatar
Leo Gao committed
78
    DATASET_PATH = "ai2_arc"
Leo Gao's avatar
Leo Gao committed
79
    DATASET_NAME = "ARC-Challenge"