Implement ARC

lm_eval/tasks/__init__.py

 from . import superglue
 from . import glue
+from . import arc
 
 TASK_REGISTRY = {
     "cola": glue.CoLA,
@@ -16,6 +17,8 @@ TASK_REGISTRY = {
     "copa": superglue.Copa,
     "wic": superglue.WordsInContext,
     "wsc": superglue.WinogradSchemaChallenge,
+    "arc_easy": arc.ARCEasy,
+    "arc_challenge": arc.ARCChallenge,
 }
 
 

lm_eval/tasks/arc.py

+from . common import HFNLPTask
+
+class ARCEasy(HFNLPTask):
+    NLP_PATH = "ai2_arc"
+    NLP_NAME = "ARC-Easy"
+
+    def has_training_docs(self):
+        return True
+
+    def has_validation_docs(self):
+        return True
+
+    def has_test_docs(self):
+        return True
+
+    def fewshot_description(self):
+        # TODO: figure out description
+        return ""
+
+    def doc_to_text(self, doc, include_target=True):
+        print(doc)
+        q = "Question: " + doc['question'] + '\n'
+        a = "Answer:" + (" " + doc['choices']['text'][doc['choices']['label'].index(doc['answerKey'])]) if include_target else ""
+        return q + a
+
+    def evaluate(self, docs, lm, provide_description, num_fewshot):
+        # TODO: implement
+        raise NotImplementedError()
+
+class ARCChallenge(ARCEasy):
+    NLP_PATH = "ai2_arc"
+    NLP_NAME = "ARC-Challenge"
\ No newline at end of file