Error in --num_fewshot option for K-MMLU Evaluation Harness (#1178)

* update kmmlu default formatting

* Update _default_kmmlu_yaml

* Delete lm_eval/tasks/kmmlu/utils.py

lm_eval/tasks/kmmlu/_default_kmmlu_yaml

@@ -6,10 +6,9 @@ validation_split: dev
 test_split: test
 fewshot_split: dev
 output_type: multiple_choice
-process_docs: !function utils.process_docs
-doc_to_text: "{{question}}"
-doc_to_choice: "{{choices}}"
-doc_to_target: "{{gold}}"
+doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답："
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: "{{['A', 'B', 'C', 'D'][answer-1]}}"
 metric_list:
   - metric: acc
     aggregation: mean
@@ -18,4 +17,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  version: 1.0
+  version: 1.1

lm_eval/tasks/kmmlu/utils.py

-import datasets
-
-
-def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
-    def _process_doc(doc):
-        instruction = (
-            f"다음을 읽고 정답으로 알맞은 것을 고르시요.\n"
-            f"### Question: {doc['question']}\n"
-            f"### Options:\n"
-            f"(1) {doc['A']}\n(2) {doc['B']}\n(3) {doc['C']}\n(4) {doc['D']}\n"
-            f"### Answer: 주어진 문제의 정답은"
-        )
-        out_doc = {
-            "question": instruction,
-            "choices": ["(1)", "(2)", "(3)", "(4)"],
-            "gold": int(doc["answer"]) - 1,
-        }
-        return out_doc
-
-    return dataset.map(_process_doc)