implementing kmmlu

lm_eval/tasks/kmmlu/kmmlu_public safety.yaml

+dataset_name: Public Safety
+include: _default_kmmlu_yaml
+task: kmmlu_public safety

lm_eval/tasks/kmmlu/kmmlu_railway and automotive engineering.yaml

+dataset_name: Railway and Automotive Engineering
+include: _default_kmmlu_yaml
+task: kmmlu_railway and automotive engineering

lm_eval/tasks/kmmlu/kmmlu_real estate.yaml

+dataset_name: Real Estate
+include: _default_kmmlu_yaml
+task: kmmlu_real estate

lm_eval/tasks/kmmlu/kmmlu_refrigerating machinery.yaml

+dataset_name: Refrigerating Machinery
+include: _default_kmmlu_yaml
+task: kmmlu_refrigerating machinery

lm_eval/tasks/kmmlu/kmmlu_social welfare.yaml

+dataset_name: Social Welfare
+include: _default_kmmlu_yaml
+task: kmmlu_social welfare

lm_eval/tasks/kmmlu/kmmlu_taxation.yaml

+dataset_name: Taxation
+include: _default_kmmlu_yaml
+task: kmmlu_taxation

lm_eval/tasks/kmmlu/kmmlu_telecommunications and wireless technology.yaml

+dataset_name: Telecommunications and Wireless Technology
+include: _default_kmmlu_yaml
+task: kmmlu_telecommunications and wireless technology

lm_eval/tasks/kmmlu/utils.py

+import datasets
+
+def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
+    def _process_doc(doc):
+        instruction = f"""다음을 읽고 정답으로 알맞은 것을 고르시요.
+            ### Context: {doc["context"]}
+            ### Question: {doc["question"]}
+            ### Options:
+            (1) {doc['option#1']}\n(2) {doc["option#2"]}\n(3) {doc["option#3"]}\n(4) {doc['option#4']}}
+            ### Answer: 주어진 문제의 정답은"""
+        out_doc = {
+            "question": instruction,
+            "choices": ["(1)", "(2)", "(3)", "(4)"],
+            "gold": int(doc["gold"]) - 1,
+        }
+        return out_doc
+
+    return dataset.map(_process_doc)