Merge branch 'main' into weight_by_size

lm_eval/tasks/kmmlu/direct/kmmlu_direct_marketing.yaml

+dataset_name: Marketing
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_marketing

lm_eval/tasks/kmmlu/direct/kmmlu_direct_materials_engineering.yaml

+dataset_name: Materials-Engineering
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_materials_engineering

lm_eval/tasks/kmmlu/direct/kmmlu_direct_math.yaml

+dataset_name: Math
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_math

lm_eval/tasks/kmmlu/direct/kmmlu_direct_mechanical_engineering.yaml

+dataset_name: Mechanical-Engineering
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_mechanical_engineering

lm_eval/tasks/kmmlu/direct/kmmlu_direct_nondestructive_testing.yaml

+dataset_name: Nondestructive-Testing
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_nondestructive_testing

lm_eval/tasks/kmmlu/direct/kmmlu_direct_patent.yaml

+dataset_name: Patent
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_patent

lm_eval/tasks/kmmlu/direct/kmmlu_direct_political_science_and_sociology.yaml

+dataset_name: Political-Science-and-Sociology
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_political_science_and_sociology

lm_eval/tasks/kmmlu/direct/kmmlu_direct_psychology.yaml

+dataset_name: Psychology
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_psychology

lm_eval/tasks/kmmlu/direct/kmmlu_direct_public_safety.yaml

+dataset_name: Public-Safety
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_public_safety

lm_eval/tasks/kmmlu/direct/kmmlu_direct_railway_and_automotive_engineering.yaml

+dataset_name: Railway-and-Automotive-Engineering
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_railway_and_automotive_engineering

lm_eval/tasks/kmmlu/direct/kmmlu_direct_real_estate.yaml

+dataset_name: Real-Estate
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_real_estate

lm_eval/tasks/kmmlu/direct/kmmlu_direct_refrigerating_machinery.yaml

+dataset_name: Refrigerating-Machinery
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_refrigerating_machinery

lm_eval/tasks/kmmlu/direct/kmmlu_direct_social_welfare.yaml

+dataset_name: Social-Welfare
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_social_welfare

lm_eval/tasks/kmmlu/direct/kmmlu_direct_taxation.yaml

+dataset_name: Taxation
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_taxation

lm_eval/tasks/kmmlu/direct/kmmlu_direct_telecommunications_and_wireless_technology.yaml

+dataset_name: Telecommunications-and-Wireless-Technology
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_telecommunications_and_wireless_technology

lm_eval/tasks/kmmlu/direct_hard/_direct_hard_kmmlu_yaml

+group:
+    - kmmlu
+    - kmmlu_hard_direct
+dataset_path: HAERAE-HUB/KMMLU-HARD
+output_type: generate_until
+test_split: test
+fewshot_split: dev
+doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답："
+doc_to_target: "{{['A', 'B', 'C', 'D'][answer-1]}}"
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+    regexes_to_ignore:
+          - " "
+generation_kwargs:
+  until:
+    - "Q:"
+    - "\n\n"
+    - "</s>"
+    - "."
+  do_sample: false
+  temperature: 0.0
+metadata:
+  version: 2.0

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_accounting.yaml

+dataset_name: accounting
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_accounting

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_agricultural_sciences.yaml

+dataset_name: agricultural_sciences
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_agricultural_sciences

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_aviation_engineering_and_maintenance.yaml

+dataset_name: aviation_engineering_and_maintenance
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_aviation_engineering_and_maintenance

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_biology.yaml

+dataset_name: biology
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_biology