resolved merge conflict

lm_eval/tasks/kmmlu/kmmlu_patent.yaml

+"dataset_name": "Patent"
+"include": "_default_kmmlu_yaml"
+"task": "kmmlu_patent"

lm_eval/tasks/kmmlu/kmmlu_political_science_and_sociology.yaml

+"dataset_name": "Political Science and Sociology"
+"include": "_default_kmmlu_yaml"
+"task": "kmmlu_political_science_and_sociology"

lm_eval/tasks/kmmlu/kmmlu_psychology.yaml

+"dataset_name": "Psychology"
+"include": "_default_kmmlu_yaml"
+"task": "kmmlu_psychology"

lm_eval/tasks/kmmlu/kmmlu_public_safety.yaml

+"dataset_name": "Public Safety"
+"include": "_default_kmmlu_yaml"
+"task": "kmmlu_public_safety"

lm_eval/tasks/kmmlu/kmmlu_railway_and_automotive_engineering.yaml

+"dataset_name": "Railway and Automotive Engineering"
+"include": "_default_kmmlu_yaml"
+"task": "kmmlu_railway_and_automotive_engineering"

lm_eval/tasks/kmmlu/kmmlu_real_estate.yaml

+"dataset_name": "Real Estate"
+"include": "_default_kmmlu_yaml"
+"task": "kmmlu_real_estate"

lm_eval/tasks/kmmlu/kmmlu_refrigerating_machinery.yaml

+"dataset_name": "Refrigerating Machinery"
+"include": "_default_kmmlu_yaml"
+"task": "kmmlu_refrigerating_machinery"

lm_eval/tasks/kmmlu/kmmlu_social_welfare.yaml

+"dataset_name": "Social Welfare"
+"include": "_default_kmmlu_yaml"
+"task": "kmmlu_social_welfare"

lm_eval/tasks/kmmlu/kmmlu_taxation.yaml

+"dataset_name": "Taxation"
+"include": "_default_kmmlu_yaml"
+"task": "kmmlu_taxation"

lm_eval/tasks/kmmlu/kmmlu_telecommunications_and_wireless_technology.yaml

+"dataset_name": "Telecommunications and Wireless Technology"
+"include": "_default_kmmlu_yaml"
+"task": "kmmlu_telecommunications_and_wireless_technology"

lm_eval/tasks/kmmlu/utils.py

+import datasets
+
+
+def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
+    def _process_doc(doc):
+        instruction = (
+            f"다음을 읽고 정답으로 알맞은 것을 고르시요.\n"
+            f"### Question: {doc['question']}\n"
+            f"### Options:\n"
+            f"(1) {doc['option#1']}\n(2) {doc['option#2']}\n(3) {doc['option#3']}\n(4) {doc['option#4']}\n"
+            f"### Answer: 주어진 문제의 정답은"
+        )
+        out_doc = {
+            "question": instruction,
+            "choices": ["(1)", "(2)", "(3)", "(4)"],
+            "gold": int(doc["answer"]) - 1,
+        }
+        return out_doc
+
+    return dataset.map(_process_doc)

lm_eval/tasks/mathqa/alternative_worlds/output_variation/styles.py

@@ -2,6 +2,7 @@ import re
 import string
 from functools import partial
 
+
 def parse_choices(doc):
     choices = [
         c[4:].rstrip(" ,")
@@ -9,6 +10,7 @@ def parse_choices(doc):
     ]
     return choices
 
+
 def doc_to_text_base(alphabet, style, doc):
     choices = parse_choices(doc)
     num = len(choices)
@@ -24,10 +26,12 @@ def doc_to_text_base(alphabet, style, doc):
     )
     return doc_to_text
 
+
 # Full continuation
 def choice_A(doc):
     return parse_choices(doc)
 
+
 # Letters only
 def choice_B(alphabet, style, doc):
     choices = parse_choices(doc)
@@ -37,6 +41,7 @@ def choice_B(alphabet, style, doc):
         letter_list = [letter.replace("\t", "") for letter in letter_list]
     return letter_list
 
+
 # Letters + Full continuation
 def choice_C(alphabet, style, doc):
     choices = parse_choices(doc)
@@ -46,6 +51,7 @@ def choice_C(alphabet, style, doc):
         letter_list = [letter + " " for letter in letter_list]
     return [letter + choice for letter, choice in zip(letter_list, choices)]
 
+
 template_01 = partial(doc_to_text_base, string.ascii_lowercase, "({})")
 choice_01a = choice_A
 choice_01b = partial(choice_B, string.ascii_lowercase, "({})")

lm_eval/tasks/minerva_math/minerva_math_algebra.yaml

@@ -19,5 +19,6 @@ metric_list:
   - metric: exact_match
     aggregation: mean
     higher_is_better: true
+num_fewshot: 0
 metadata:
   - version: 0.0

lm_eval/tasks/mmlu/alternative_worlds/mmlu_prompt_variation.yaml

@@ -15,4 +15,3 @@ task:
   # - mmlu_alt_pv_03_other
   # - mmlu_alt_pv_03_social_sciences
   # - mmlu_alt_pv_03_humanities
-

lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu_flan_cot_fewshot_template_yaml

@@ -15,6 +15,7 @@ generation_kwargs:
     - "</s>"
   do_sample: false
   temperature: 0.0
+num_fewshot: 0
 metric_list:
   - metric: exact_match
     aggregation: mean

lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu_flan_cot_zeroshot_template_yaml

@@ -15,6 +15,7 @@ generation_kwargs:
     - "</s>"
   do_sample: false
   temperature: 0.0
+num_fewshot: 0
 metric_list:
   - metric: exact_match
     aggregation: mean

lm_eval/tasks/sciq/alternative_worlds/output_variation/style_01/a.yaml

@@ -2,4 +2,4 @@ include: ../_sciq_alt_yaml
 group: sciq_alt_ov_01
 task: sciq_alt_ov_01a
 doc_to_text: !function ../styles.template_01
-doc_to_choice: !function ../styles.choice_01a
\ No newline at end of file
+doc_to_choice: !function ../styles.choice_01a

lm_eval/tasks/sciq/alternative_worlds/output_variation/style_01/b.yaml

@@ -2,4 +2,4 @@ include: ../_sciq_alt_yaml
 group: sciq_alt_ov_01
 task: sciq_alt_ov_01b
 doc_to_text: !function ../styles.template_01
-doc_to_choice: !function ../styles.choice_01b
\ No newline at end of file
+doc_to_choice: !function ../styles.choice_01b

lm_eval/tasks/sciq/alternative_worlds/output_variation/style_01/c.yaml

@@ -2,4 +2,4 @@ include: ../_sciq_alt_yaml
 group: sciq_alt_ov_01
 task: sciq_alt_ov_01c
 doc_to_text: !function ../styles.template_01
-doc_to_choice: !function ../styles.choice_01c
\ No newline at end of file
+doc_to_choice: !function ../styles.choice_01c

lm_eval/tasks/sciq/alternative_worlds/output_variation/style_02/a.yaml

@@ -2,4 +2,4 @@ include: ../_sciq_alt_yaml
 group: sciq_alt_ov_02
 task: sciq_alt_ov_02a
 doc_to_text: !function ../styles.template_02
-doc_to_choice: !function ../styles.choice_02a
\ No newline at end of file
+doc_to_choice: !function ../styles.choice_02a