merged main

lm_eval/tasks/kmmlu/direct/kmmlu_direct_railway_and_automotive_engineering.yaml

+dataset_name: Railway-and-Automotive-Engineering
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_railway_and_automotive_engineering

lm_eval/tasks/kmmlu/direct/kmmlu_direct_real_estate.yaml

+dataset_name: Real-Estate
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_real_estate

lm_eval/tasks/kmmlu/direct/kmmlu_direct_refrigerating_machinery.yaml

+dataset_name: Refrigerating-Machinery
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_refrigerating_machinery

lm_eval/tasks/kmmlu/direct/kmmlu_direct_social_welfare.yaml

+dataset_name: Social-Welfare
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_social_welfare

lm_eval/tasks/kmmlu/direct/kmmlu_direct_taxation.yaml

+dataset_name: Taxation
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_taxation

lm_eval/tasks/kmmlu/direct/kmmlu_direct_telecommunications_and_wireless_technology.yaml

+dataset_name: Telecommunications-and-Wireless-Technology
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_telecommunications_and_wireless_technology

lm_eval/tasks/kmmlu/direct_hard/_direct_hard_kmmlu_yaml

+group:
+    - kmmlu
+    - kmmlu_hard_direct
+dataset_path: HAERAE-HUB/KMMLU-HARD
+output_type: generate_until
+test_split: test
+fewshot_split: dev
+doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답："
+doc_to_target: "{{['A', 'B', 'C', 'D'][answer-1]}}"
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+    regexes_to_ignore:
+          - " "
+generation_kwargs:
+  until:
+    - "Q:"
+    - "\n\n"
+    - "</s>"
+    - "."
+  do_sample: false
+  temperature: 0.0
+metadata:
+  version: 2.0

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_accounting.yaml

+dataset_name: accounting
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_accounting

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_agricultural_sciences.yaml

+dataset_name: agricultural_sciences
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_agricultural_sciences

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_aviation_engineering_and_maintenance.yaml

+dataset_name: aviation_engineering_and_maintenance
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_aviation_engineering_and_maintenance

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_biology.yaml

+dataset_name: biology
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_biology

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_chemical_engineering.yaml

+dataset_name: chemical_engineering
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_chemical_engineering

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_chemistry.yaml

+dataset_name: chemistry
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_chemistry

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_civil_engineering.yaml

+dataset_name: civil_engineering
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_civil_engineering

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_computer_science.yaml

+dataset_name: computer_science
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_computer_science

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_construction.yaml

+dataset_name: construction
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_construction

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_criminal_law.yaml

+dataset_name: criminal_law
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_criminal_law

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_ecology.yaml

+dataset_name: ecology
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_ecology

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_economics.yaml

+dataset_name: economics
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_economics

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_education.yaml

+dataset_name: education
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_education