Merge pull request #1089 from h-albert-lee/K-MMLU

Add kmmlu evaluation to tasks

lm_eval/tasks/kmmlu/kmmlu_public_safety.yaml

+"dataset_name": "Public Safety"
+"include": "_default_kmmlu_yaml"
+"task": "kmmlu_public_safety"

lm_eval/tasks/kmmlu/kmmlu_railway_and_automotive_engineering.yaml

+"dataset_name": "Railway and Automotive Engineering"
+"include": "_default_kmmlu_yaml"
+"task": "kmmlu_railway_and_automotive_engineering"

lm_eval/tasks/kmmlu/kmmlu_real_estate.yaml

+"dataset_name": "Real Estate"
+"include": "_default_kmmlu_yaml"
+"task": "kmmlu_real_estate"

lm_eval/tasks/kmmlu/kmmlu_refrigerating_machinery.yaml

+"dataset_name": "Refrigerating Machinery"
+"include": "_default_kmmlu_yaml"
+"task": "kmmlu_refrigerating_machinery"

lm_eval/tasks/kmmlu/kmmlu_social_welfare.yaml

+"dataset_name": "Social Welfare"
+"include": "_default_kmmlu_yaml"
+"task": "kmmlu_social_welfare"

lm_eval/tasks/kmmlu/kmmlu_taxation.yaml

+"dataset_name": "Taxation"
+"include": "_default_kmmlu_yaml"
+"task": "kmmlu_taxation"

lm_eval/tasks/kmmlu/kmmlu_telecommunications_and_wireless_technology.yaml

+"dataset_name": "Telecommunications and Wireless Technology"
+"include": "_default_kmmlu_yaml"
+"task": "kmmlu_telecommunications_and_wireless_technology"

lm_eval/tasks/kmmlu/utils.py

+import datasets
+
+
+def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
+    def _process_doc(doc):
+        instruction = (
+            f"다음을 읽고 정답으로 알맞은 것을 고르시요.\n"
+            f"### Question: {doc['question']}\n"
+            f"### Options:\n"
+            f"(1) {doc['option#1']}\n(2) {doc['option#2']}\n(3) {doc['option#3']}\n(4) {doc['option#4']}\n"
+            f"### Answer: 주어진 문제의 정답은"
+        )
+        out_doc = {
+            "question": instruction,
+            "choices": ["(1)", "(2)", "(3)", "(4)"],
+            "gold": int(doc["answer"]) - 1,
+        }
+        return out_doc
+
+    return dataset.map(_process_doc)