Add kmmlu multiple-choice(accuracy) task (#2849)

lm_eval/tasks/kmmlu/README.md

@@ -32,6 +32,7 @@ Homepage: https://huggingface.co/datasets/HAERAE-HUB/KMMLU
 #### Tasks
 
 The following tasks evaluate subjects in the KMMLU dataset
+- `kmmlu_{subject_english}`
 - `kmmlu_direct_{subject_english}`
 
 The following tasks evaluate subjects in the KMMLU-Hard dataset

lm_eval/tasks/kmmlu/default/_default_kmmlu_yaml

+dataset_path: HAERAE-HUB/KMMLU
+output_type: multiple_choice
+test_split: test
+fewshot_split: dev
+doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답："
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: "{{answer-1}}"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 2.0

lm_eval/tasks/kmmlu/default/_kmmlu_applied_science.yaml

+group: kmmlu_applied_science
+task:
+  - kmmlu_applied_science_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 2.0

lm_eval/tasks/kmmlu/default/_kmmlu_default.yaml

+group: kmmlu
+task:
+  - kmmlu_stem
+  - kmmlu_other
+  - kmmlu_applied_science
+  - kmmlu_humss
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 2.0

lm_eval/tasks/kmmlu/default/_kmmlu_humss.yaml

+group: kmmlu_humss
+task:
+  - kmmlu_humss_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 2.0

lm_eval/tasks/kmmlu/default/_kmmlu_other.yaml

+group: kmmlu_other
+task:
+  - kmmlu_other_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 2.0

lm_eval/tasks/kmmlu/default/_kmmlu_stem.yaml

+group: kmmlu_stem
+task:
+  - kmmlu_stem_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 2.0

lm_eval/tasks/kmmlu/default/kmmlu_accounting.yaml

+dataset_name: Accounting
+include: _default_kmmlu_yaml
+task: kmmlu_accounting
+tag: kmmlu_humss_tasks

lm_eval/tasks/kmmlu/default/kmmlu_agricultural_sciences.yaml

+dataset_name: Agricultural-Sciences
+include: _default_kmmlu_yaml
+task: kmmlu_agricultural_sciences
+tag: kmmlu_other_tasks

lm_eval/tasks/kmmlu/default/kmmlu_aviation_engineering_and_maintenance.yaml

+dataset_name: Aviation-Engineering-and-Maintenance
+include: _default_kmmlu_yaml
+task: kmmlu_aviation_engineering_and_maintenance
+tag: kmmlu_applied_science_tasks

lm_eval/tasks/kmmlu/default/kmmlu_biology.yaml

+dataset_name: Biology
+include: _default_kmmlu_yaml
+task: kmmlu_biology
+tag: kmmlu_stem_tasks

lm_eval/tasks/kmmlu/default/kmmlu_chemical_engineering.yaml

+dataset_name: Chemical-Engineering
+include: _default_kmmlu_yaml
+task: kmmlu_chemical_engineering
+tag: kmmlu_stem_tasks

lm_eval/tasks/kmmlu/default/kmmlu_chemistry.yaml

+dataset_name: Chemistry
+include: _default_kmmlu_yaml
+task: kmmlu_chemistry
+tag: kmmlu_stem_tasks

lm_eval/tasks/kmmlu/default/kmmlu_civil_engineering.yaml

+dataset_name: Civil-Engineering
+include: _default_kmmlu_yaml
+task: kmmlu_civil_engineering
+tag: kmmlu_stem_tasks

lm_eval/tasks/kmmlu/default/kmmlu_computer_science.yaml

+dataset_name: Computer-Science
+include: _default_kmmlu_yaml
+task: kmmlu_computer_science
+tag: kmmlu_stem_tasks

lm_eval/tasks/kmmlu/default/kmmlu_construction.yaml

+dataset_name: Construction
+include: _default_kmmlu_yaml
+task: kmmlu_construction
+tag: kmmlu_other_tasks

lm_eval/tasks/kmmlu/default/kmmlu_criminal_law.yaml

+dataset_name: Criminal-Law
+include: _default_kmmlu_yaml
+task: kmmlu_criminal_law
+tag: kmmlu_humss_tasks

lm_eval/tasks/kmmlu/default/kmmlu_ecology.yaml

+dataset_name: Ecology
+include: _default_kmmlu_yaml
+task: kmmlu_ecology
+tag: kmmlu_stem_tasks

lm_eval/tasks/kmmlu/default/kmmlu_economics.yaml

+dataset_name: Economics
+include: _default_kmmlu_yaml
+task: kmmlu_economics
+tag: kmmlu_humss_tasks

lm_eval/tasks/kmmlu/default/kmmlu_education.yaml

+dataset_name: Education
+include: _default_kmmlu_yaml
+task: kmmlu_education
+tag: kmmlu_humss_tasks