merged main

lm_eval/tasks/kmmlu/direct/_direct_kmmlu_yaml

+group:
+    - kmmlu
+    - kmmlu_direct
+dataset_path: HAERAE-HUB/KMMLU
+output_type: generate_until
+test_split: test
+fewshot_split: dev
+doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답："
+doc_to_target: "{{['A', 'B', 'C', 'D'][answer-1]}}"
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+    regexes_to_ignore:
+          - " "
+generation_kwargs:
+  until:
+    - "Q:"
+    - "\n\n"
+    - "</s>"
+    - "."
+  do_sample: false
+  temperature: 0.0
+metadata:
+  version: 2.0

lm_eval/tasks/kmmlu/direct/kmmlu_direct_accounting.yaml

+dataset_name: Accounting
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_accounting

lm_eval/tasks/kmmlu/direct/kmmlu_direct_agricultural_sciences.yaml

+dataset_name: Agricultural-Sciences
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_agricultural_sciences

lm_eval/tasks/kmmlu/direct/kmmlu_direct_aviation_engineering_and_maintenance.yaml

+dataset_name: Aviation-Engineering-and-Maintenance
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_aviation_engineering_and_maintenance

lm_eval/tasks/kmmlu/direct/kmmlu_direct_biology.yaml

+dataset_name: Biology
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_biology

lm_eval/tasks/kmmlu/direct/kmmlu_direct_chemical_engineering.yaml

+dataset_name: Chemical-Engineering
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_chemical_engineering

lm_eval/tasks/kmmlu/direct/kmmlu_direct_chemistry.yaml

+dataset_name: Chemistry
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_chemistry

lm_eval/tasks/kmmlu/direct/kmmlu_direct_civil_engineering.yaml

+dataset_name: Civil-Engineering
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_civil_engineering

lm_eval/tasks/kmmlu/direct/kmmlu_direct_computer_science.yaml

+dataset_name: Computer-Science
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_computer_science

lm_eval/tasks/kmmlu/direct/kmmlu_direct_construction.yaml

+dataset_name: Construction
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_construction

lm_eval/tasks/kmmlu/direct/kmmlu_direct_criminal_law.yaml

+dataset_name: Criminal-Law
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_criminal_law

lm_eval/tasks/kmmlu/direct/kmmlu_direct_ecology.yaml

+dataset_name: Ecology
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_ecology

lm_eval/tasks/kmmlu/direct/kmmlu_direct_economics.yaml

+dataset_name: Economics
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_economics

lm_eval/tasks/kmmlu/direct/kmmlu_direct_education.yaml

+dataset_name: Education
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_education

lm_eval/tasks/kmmlu/direct/kmmlu_direct_electrical_engineering.yaml

+dataset_name: Electrical-Engineering
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_electrical_engineering

lm_eval/tasks/kmmlu/direct/kmmlu_direct_electronics_engineering.yaml

+dataset_name: Electronics-Engineering
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_electronics_engineering

lm_eval/tasks/kmmlu/direct/kmmlu_direct_energy_management.yaml

+dataset_name: Energy-Management
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_energy_management

lm_eval/tasks/kmmlu/direct/kmmlu_direct_environmental_science.yaml

+dataset_name: Environmental-Science
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_environmental_science

lm_eval/tasks/kmmlu/direct/kmmlu_direct_fashion.yaml

+dataset_name: Fashion
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_fashion

lm_eval/tasks/kmmlu/direct/kmmlu_direct_food_processing.yaml

+dataset_name: Food-Processing
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_food_processing