merged main

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_mechanical_engineering.yaml

+dataset_name: mechanical_engineering
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_mechanical_engineering

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_nondestructive_testing.yaml

+dataset_name: nondestructive_testing
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_nondestructive_testing

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_patent.yaml

+dataset_name: patent
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_patent

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_political_science_and_sociology.yaml

+dataset_name: political_science_and_sociology
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_political_science_and_sociology

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_psychology.yaml

+dataset_name: psychology
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_psychology

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_public_safety.yaml

+dataset_name: public_safety
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_public_safety

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_railway_and_automotive_engineering.yaml

+dataset_name: railway_and_automotive_engineering
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_railway_and_automotive_engineering

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_real_estate.yaml

+dataset_name: real_estate
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_real_estate

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_refrigerating_machinery.yaml

+dataset_name: refrigerating_machinery
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_refrigerating_machinery

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_social_welfare.yaml

+dataset_name: social_welfare
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_social_welfare

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_taxation.yaml

+dataset_name: taxation
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_taxation

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_telecommunications_and_wireless_technology.yaml

+dataset_name: telecommunications_and_wireless_technology
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_telecommunications_and_wireless_technology

lm_eval/tasks/kmmlu/_default_kmmlu_yaml → lm_eval/tasks/kmmlu/hard/_hard_kmmlu_yaml

-group: kmmlu
-dataset_path: HAERAE-HUB/K-MMLU-Preview
+group:
+    - kmmlu
+    - kmmlu_hard
+dataset_path: HAERAE-HUB/KMMLU-HARD
 output_type: multiple_choice
-training_split: train
-validation_split: dev
 test_split: test
 fewshot_split: dev
-output_type: multiple_choice
 doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답："
 doc_to_choice: ["A", "B", "C", "D"]
-doc_to_target: "{{['A', 'B', 'C', 'D'][answer-1]}}"
+doc_to_target: "{{answer-1}}"
 metric_list:
   - metric: acc
     aggregation: mean
@@ -17,4 +16,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  version: 1.1
+  version: 2.0

lm_eval/tasks/kmmlu/hard/kmmlu_hard_accounting.yaml

+dataset_name: accounting
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_accounting

lm_eval/tasks/kmmlu/hard/kmmlu_hard_agricultural_sciences.yaml

+dataset_name: agricultural_sciences
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_agricultural_sciences

lm_eval/tasks/kmmlu/hard/kmmlu_hard_aviation_engineering_and_maintenance.yaml

+dataset_name: aviation_engineering_and_maintenance
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_aviation_engineering_and_maintenance

lm_eval/tasks/kmmlu/hard/kmmlu_hard_biology.yaml

+dataset_name: biology
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_biology

lm_eval/tasks/kmmlu/hard/kmmlu_hard_chemical_engineering.yaml

+dataset_name: chemical_engineering
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_chemical_engineering

lm_eval/tasks/kmmlu/hard/kmmlu_hard_chemistry.yaml

+dataset_name: chemistry
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_chemistry

lm_eval/tasks/kmmlu/hard/kmmlu_hard_civil_engineering.yaml

+dataset_name: civil_engineering
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_civil_engineering