solved merge conflict

lm_eval/tasks/ifeval/instructions_util.py

@@ -17,7 +17,6 @@
 import functools
 import random
 import re
-from typing import List
 
 import immutabledict
 import nltk

lm_eval/tasks/kmmlu/_default_kmmlu_yaml

@@ -6,10 +6,9 @@ validation_split: dev
 test_split: test
 fewshot_split: dev
 output_type: multiple_choice
-process_docs: !function utils.process_docs
-doc_to_text: "{{question}}"
-doc_to_choice: "{{choices}}"
-doc_to_target: "{{gold}}"
+doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답："
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: "{{['A', 'B', 'C', 'D'][answer-1]}}"
 metric_list:
   - metric: acc
     aggregation: mean
@@ -18,4 +17,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 0.0
+  version: 1.1

lm_eval/tasks/kmmlu/kmmlu_agricultural_sciences.yaml

-"dataset_name": "Agricultural Sciences"
+"dataset_name": "Agricultural-Sciences"
 "include": "_default_kmmlu_yaml"
 "task": "kmmlu_agricultural_sciences"

lm_eval/tasks/kmmlu/kmmlu_aviation_engineering_and_maintenance.yaml

-"dataset_name": "Aviation Engineering and Maintenance"
+"dataset_name": "Aviation-Engineering-and-Maintenance"
 "include": "_default_kmmlu_yaml"
 "task": "kmmlu_aviation_engineering_and_maintenance"

lm_eval/tasks/kmmlu/kmmlu_chemical_engineering.yaml

-"dataset_name": "Chemical Engineering"
+"dataset_name": "Chemical-Engineering"
 "include": "_default_kmmlu_yaml"
 "task": "kmmlu_chemical_engineering"

lm_eval/tasks/kmmlu/kmmlu_civil_engineering.yaml

-"dataset_name": "Civil Engineering"
+"dataset_name": "Civil-Engineering"
 "include": "_default_kmmlu_yaml"
 "task": "kmmlu_civil_engineering"

lm_eval/tasks/kmmlu/kmmlu_computer_science.yaml

-"dataset_name": "Computer Science"
+"dataset_name": "Computer-Science"
 "include": "_default_kmmlu_yaml"
 "task": "kmmlu_computer_science"

lm_eval/tasks/kmmlu/kmmlu_criminal_law.yaml

-"dataset_name": "Criminal Law"
+"dataset_name": "Criminal-Law"
 "include": "_default_kmmlu_yaml"
 "task": "kmmlu_criminal_law"

lm_eval/tasks/kmmlu/kmmlu_electrical_engineering.yaml

-"dataset_name": "Electrical Engineering"
+"dataset_name": "Electrical-Engineering"
 "include": "_default_kmmlu_yaml"
 "task": "kmmlu_electrical_engineering"

lm_eval/tasks/kmmlu/kmmlu_electronics_engineering.yaml

-"dataset_name": "Electronics Engineering"
+"dataset_name": "Electronics-Engineering"
 "include": "_default_kmmlu_yaml"
 "task": "kmmlu_electronics_engineering"

lm_eval/tasks/kmmlu/kmmlu_energy_management.yaml

-"dataset_name": "Energy Management"
+"dataset_name": "Energy-Management"
 "include": "_default_kmmlu_yaml"
 "task": "kmmlu_energy_management"

lm_eval/tasks/kmmlu/kmmlu_environmental_science.yaml

-"dataset_name": "Environmental Science"
+"dataset_name": "Environmental-Science"
 "include": "_default_kmmlu_yaml"
 "task": "kmmlu_environmental_science"

lm_eval/tasks/kmmlu/kmmlu_food_processing.yaml

-"dataset_name": "Food Processing"
+"dataset_name": "Food-Processing"
 "include": "_default_kmmlu_yaml"
 "task": "kmmlu_food_processing"

lm_eval/tasks/kmmlu/kmmlu_gas_technology_and_engineering.yaml

-"dataset_name": "Gas Technology and Engineering"
+"dataset_name": "Gas-Technology-and-Engineering"
 "include": "_default_kmmlu_yaml"
 "task": "kmmlu_gas_technology_and_engineering"

lm_eval/tasks/kmmlu/kmmlu_general_physics.yaml

-"dataset_name": "General Physics"
-"include": "_default_kmmlu_yaml"
-"task": "kmmlu_general_physics"

lm_eval/tasks/kmmlu/kmmlu_industrial_engineer.yaml

-"dataset_name": "Industrial Engineer"
+"dataset_name": "Industrial-Engineer"
 "include": "_default_kmmlu_yaml"
 "task": "kmmlu_industrial_engineer"

lm_eval/tasks/kmmlu/kmmlu_information_technology.yaml

-"dataset_name": "Information Technology"
+"dataset_name": "Information-Technology"
 "include": "_default_kmmlu_yaml"
 "task": "kmmlu_information_technology"

lm_eval/tasks/kmmlu/kmmlu_interior_architecture_and_design.yaml

-"dataset_name": "Interior Architecture and Design"
+"dataset_name": "Interior-Architecture-and-Design"
 "include": "_default_kmmlu_yaml"
 "task": "kmmlu_interior_architecture_and_design"

lm_eval/tasks/kmmlu/kmmlu_korean_language.yaml

-"dataset_name": "Korean Language"
-"include": "_default_kmmlu_yaml"
-"task": "kmmlu_korean_language"

lm_eval/tasks/kmmlu/kmmlu_machine_design_and_manufacturing.yaml

-"dataset_name": "Machine Design and Manufacturing"
+"dataset_name": "Machine-Design-and-Manufacturing"
 "include": "_default_kmmlu_yaml"
 "task": "kmmlu_machine_design_and_manufacturing"