Merge branch 'main' into weight_by_size

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_telecommunications_and_wireless_technology.yaml

+dataset_name: telecommunications_and_wireless_technology
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_telecommunications_and_wireless_technology

lm_eval/tasks/kmmlu/_default_kmmlu_yaml → lm_eval/tasks/kmmlu/hard/_hard_kmmlu_yaml

-group: kmmlu
-dataset_path: HAERAE-HUB/K-MMLU-Preview
+group:
+    - kmmlu
+    - kmmlu_hard
+dataset_path: HAERAE-HUB/KMMLU-HARD
 output_type: multiple_choice
-training_split: train
-validation_split: dev
 test_split: test
 fewshot_split: dev
-output_type: multiple_choice
 doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답："
 doc_to_choice: ["A", "B", "C", "D"]
-doc_to_target: "{{['A', 'B', 'C', 'D'][answer-1]}}"
+doc_to_target: "{{answer-1}}"
 metric_list:
   - metric: acc
     aggregation: mean
@@ -17,4 +16,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  version: 1.1
+  version: 2.0

lm_eval/tasks/kmmlu/hard/kmmlu_hard_accounting.yaml

+dataset_name: accounting
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_accounting

lm_eval/tasks/kmmlu/hard/kmmlu_hard_agricultural_sciences.yaml

+dataset_name: agricultural_sciences
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_agricultural_sciences

lm_eval/tasks/kmmlu/hard/kmmlu_hard_aviation_engineering_and_maintenance.yaml

+dataset_name: aviation_engineering_and_maintenance
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_aviation_engineering_and_maintenance

lm_eval/tasks/kmmlu/hard/kmmlu_hard_biology.yaml

+dataset_name: biology
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_biology

lm_eval/tasks/kmmlu/hard/kmmlu_hard_chemical_engineering.yaml

+dataset_name: chemical_engineering
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_chemical_engineering

lm_eval/tasks/kmmlu/hard/kmmlu_hard_chemistry.yaml

+dataset_name: chemistry
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_chemistry

lm_eval/tasks/kmmlu/hard/kmmlu_hard_civil_engineering.yaml

+dataset_name: civil_engineering
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_civil_engineering

lm_eval/tasks/kmmlu/hard/kmmlu_hard_computer_science.yaml

+dataset_name: computer_science
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_computer_science

lm_eval/tasks/kmmlu/hard/kmmlu_hard_construction.yaml

+dataset_name: construction
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_construction

lm_eval/tasks/kmmlu/hard/kmmlu_hard_criminal_law.yaml

+dataset_name: criminal_law
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_criminal_law

lm_eval/tasks/kmmlu/hard/kmmlu_hard_ecology.yaml

+dataset_name: ecology
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_ecology

lm_eval/tasks/kmmlu/hard/kmmlu_hard_economics.yaml

+dataset_name: economics
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_economics

lm_eval/tasks/kmmlu/hard/kmmlu_hard_education.yaml

+dataset_name: education
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_education

lm_eval/tasks/kmmlu/hard/kmmlu_hard_electrical_engineering.yaml

+dataset_name: electrical_engineering
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_electrical_engineering

lm_eval/tasks/kmmlu/hard/kmmlu_hard_electronics_engineering.yaml

+dataset_name: electronics_engineering
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_electronics_engineering

lm_eval/tasks/kmmlu/hard/kmmlu_hard_energy_management.yaml

+dataset_name: energy_management
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_energy_management

lm_eval/tasks/kmmlu/hard/kmmlu_hard_environmental_science.yaml

+dataset_name: environmental_science
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_environmental_science

lm_eval/tasks/kmmlu/hard/kmmlu_hard_fashion.yaml

+dataset_name: fashion
+include: _hard_kmmlu_yaml
+task: kmmlu_hard_fashion