Merge pull request #706 from guijinSON/master

add haerae

lm_eval/tasks/__init__.py

@@ -64,6 +64,7 @@ from . import mgsm
 from . import scrolls
 from . import ceval
 from . import csatqa
+from . import haerae
 
 ########################################
 # Translation tasks
@@ -325,6 +326,12 @@ TASK_REGISTRY = {
     "csatqa_rcss": csatqa.RCSS,
     "csatqa_rch": csatqa.RCH,
     "csatqa_li": csatqa.LI,
+    "haerae_hi": haerae.HI,
+    "haerae_kgk":haerae.KGK,
+    "haerae_lw":haerae.LW,
+    "haerae_rc":haerae.RC,
+    "haerae_rw":haerae.RW,
+    "haerae_sn":haerae.SN,
     # Requires manual download
     # Requires manual download of data.
     # "storycloze_2016": storycloze.StoryCloze2016,

lm_eval/tasks/csatqa.py

@@ -22,14 +22,14 @@ class CSATQA(MultipleChoiceTask):
 ### Context: {doc["context"]}
 ### Question: {doc["question"]}
 ### Options:
-(1) {doc['option#1']}\n(2) {doc["option#2"]}\n(3) {doc["option#3"]}\n(4) {doc['option#4']}\n(5) {doc['option$5']}
+(1) {doc['option#1']}\n(2) {doc["option#2"]}\n(3) {doc["option#3"]}\n(4) {doc['option#4']}\n(5) {doc['option#5']}
 ### Answer: 주어진 문제의 정답은"""
 
         choices = [doc["option#1"], doc["option#2"], doc["option#3"], doc["option#4"], doc["option#5"]]
         out_doc = {
             "question": instruction,
             "choices": ["(1)", "(2)","(3)","(4)","(5)"],
-            "gold": int(doc['gold']),
+            "gold": int(doc['gold'])-1,
         }
         return out_doc
 

lm_eval/tasks/haerae.py

+from lm_eval.base import MultipleChoiceTask
+
+
+class Haerae(MultipleChoiceTask):
+    VERSION = 0
+    DATASET_PATH = "amphora/haerae_bench"
+
+    def has_training_docs(self):
+        return False
+
+    def has_validation_docs(self):
+        return False
+
+    def has_test_docs(self):
+        return True
+
+    def test_docs(self):
+        return map(self._process_doc, self.dataset["test"])
+    
+    def _process_doc(self, doc):
+        choices = [doc["o1"], doc["o2"], doc["o3"], doc["o4"]]
+        if doc.get("o5") is not None:
+            choices.append(doc["o5"])
+        out_doc = {
+            "query": doc["query"],
+            "choices": choices,
+            "gold": int(doc['gold'])-1,
+        }
+        return out_doc
+
+    def doc_to_text(self, doc):
+        return doc["query"]
+
+
+class HI(Haerae):
+    DATASET_NAME = "HI"
+
+
+class KGK(Haerae):
+    DATASET_NAME = "KGK"
+
+
+class LW(Haerae):
+    DATASET_NAME = "LW"
+
+
+class RC(Haerae):
+    DATASET_NAME = "RC"
+
+
+class RW(Haerae):
+    DATASET_NAME = "RW"
+
+
+class SN(Haerae):
+    DATASET_NAME = "SN"