removed unused files for now

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_marketing.yaml

+"dataset_name": "marketing"
+"description": "The following are multiple choice questions (with answers) about marketing.\n\
+  \n"
+"group": "mmlu_alt_ov_05a_other_generative"
+"include": "_template_yaml"
+"task": "mmlu_alt_ov_05a_marketing_generative"

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_medical_genetics.yaml

+"dataset_name": "medical_genetics"
+"description": "The following are multiple choice questions (with answers) about medical\
+  \ genetics.\n\n"
+"group": "mmlu_alt_ov_05a_other_generative"
+"include": "_template_yaml"
+"task": "mmlu_alt_ov_05a_medical_genetics_generative"

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_miscellaneous.yaml

+"dataset_name": "miscellaneous"
+"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\
+  \n"
+"group": "mmlu_alt_ov_05a_other_generative"
+"include": "_template_yaml"
+"task": "mmlu_alt_ov_05a_miscellaneous_generative"

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_moral_disputes.yaml

+"dataset_name": "moral_disputes"
+"description": "The following are multiple choice questions (with answers) about moral\
+  \ disputes.\n\n"
+"group": "mmlu_alt_ov_05a_humanities_generative"
+"include": "_template_yaml"
+"task": "mmlu_alt_ov_05a_moral_disputes_generative"

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_moral_scenarios.yaml

+"dataset_name": "moral_scenarios"
+"description": "The following are multiple choice questions (with answers) about moral\
+  \ scenarios.\n\n"
+"group": "mmlu_alt_ov_05a_humanities_generative"
+"include": "_template_yaml"
+"task": "mmlu_alt_ov_05a_moral_scenarios_generative"

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_nutrition.yaml

+"dataset_name": "nutrition"
+"description": "The following are multiple choice questions (with answers) about nutrition.\n\
+  \n"
+"group": "mmlu_alt_ov_05a_other_generative"
+"include": "_template_yaml"
+"task": "mmlu_alt_ov_05a_nutrition_generative"

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_philosophy.yaml

+"dataset_name": "philosophy"
+"description": "The following are multiple choice questions (with answers) about philosophy.\n\
+  \n"
+"group": "mmlu_alt_ov_05a_humanities_generative"
+"include": "_template_yaml"
+"task": "mmlu_alt_ov_05a_philosophy_generative"

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_prehistory.yaml

+"dataset_name": "prehistory"
+"description": "The following are multiple choice questions (with answers) about prehistory.\n\
+  \n"
+"group": "mmlu_alt_ov_05a_humanities_generative"
+"include": "_template_yaml"
+"task": "mmlu_alt_ov_05a_prehistory_generative"

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_professional_accounting.yaml

+"dataset_name": "professional_accounting"
+"description": "The following are multiple choice questions (with answers) about professional\
+  \ accounting.\n\n"
+"group": "mmlu_alt_ov_05a_other_generative"
+"include": "_template_yaml"
+"task": "mmlu_alt_ov_05a_professional_accounting_generative"

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_professional_law.yaml

+"dataset_name": "professional_law"
+"description": "The following are multiple choice questions (with answers) about professional\
+  \ law.\n\n"
+"group": "mmlu_alt_ov_05a_humanities_generative"
+"include": "_template_yaml"
+"task": "mmlu_alt_ov_05a_professional_law_generative"

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_professional_medicine.yaml

+"dataset_name": "professional_medicine"
+"description": "The following are multiple choice questions (with answers) about professional\
+  \ medicine.\n\n"
+"group": "mmlu_alt_ov_05a_other_generative"
+"include": "_template_yaml"
+"task": "mmlu_alt_ov_05a_professional_medicine_generative"

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_professional_psychology.yaml

+"dataset_name": "professional_psychology"
+"description": "The following are multiple choice questions (with answers) about professional\
+  \ psychology.\n\n"
+"group": "mmlu_alt_ov_05a_social_sciences_generative"
+"include": "_template_yaml"
+"task": "mmlu_alt_ov_05a_professional_psychology_generative"

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_public_relations.yaml

+"dataset_name": "public_relations"
+"description": "The following are multiple choice questions (with answers) about public\
+  \ relations.\n\n"
+"group": "mmlu_alt_ov_05a_social_sciences_generative"
+"include": "_template_yaml"
+"task": "mmlu_alt_ov_05a_public_relations_generative"

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_security_studies.yaml

+"dataset_name": "security_studies"
+"description": "The following are multiple choice questions (with answers) about security\
+  \ studies.\n\n"
+"group": "mmlu_alt_ov_05a_social_sciences_generative"
+"include": "_template_yaml"
+"task": "mmlu_alt_ov_05a_security_studies_generative"

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_sociology.yaml

+"dataset_name": "sociology"
+"description": "The following are multiple choice questions (with answers) about sociology.\n\
+  \n"
+"group": "mmlu_alt_ov_05a_social_sciences_generative"
+"include": "_template_yaml"
+"task": "mmlu_alt_ov_05a_sociology_generative"

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_us_foreign_policy.yaml

+"dataset_name": "us_foreign_policy"
+"description": "The following are multiple choice questions (with answers) about us\
+  \ foreign policy.\n\n"
+"group": "mmlu_alt_ov_05a_social_sciences_generative"
+"include": "_template_yaml"
+"task": "mmlu_alt_ov_05a_us_foreign_policy_generative"

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_virology.yaml

+"dataset_name": "virology"
+"description": "The following are multiple choice questions (with answers) about virology.\n\
+  \n"
+"group": "mmlu_alt_ov_05a_other_generative"
+"include": "_template_yaml"
+"task": "mmlu_alt_ov_05a_virology_generative"

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_05_generative/a/mmlu_world_religions.yaml

+"dataset_name": "world_religions"
+"description": "The following are multiple choice questions (with answers) about world\
+  \ religions.\n\n"
+"group": "mmlu_alt_ov_05a_humanities_generative"
+"include": "_template_yaml"
+"task": "mmlu_alt_ov_05a_world_religions_generative"

lm_eval/tasks/mmlu/alternative_worlds/styles.py

@@ -14,11 +14,12 @@ def doc_to_text_base(alphabet, style, doc):
     else:
         choice_string = "{} {}"
 
-    doc_to_text = "\n\n".join(
+    doc_to_text = "\n".join(
         [
-            "Question: " + doc["question"].strip() + "\nAnswer:",
+            "Question: " + doc["question"].strip()
         ]
-        + [choice_string.format(i, j) for i, j in zip(letter_list, choices)]
+        + [" ".join(choice_string.format(i, j) for i, j in zip(letter_list, choices))]
+        + ["Answer:"]
     )
 
     return doc_to_text
@@ -28,6 +29,8 @@ def doc_to_text_base(alphabet, style, doc):
 def choice_A(doc):
     return doc["choices"]
 
+def answer_A(doc):
+    return doc["choices"][doc["answer"]]
 
 # Letters only
 def choice_B(alphabet, style, doc):
@@ -41,6 +44,16 @@ def choice_B(alphabet, style, doc):
 
     return letter_list
 
+def answer_B(alphabet, style, doc):
+
+    choices = doc["choices"]
+    num = len(choices)
+
+    letter_list = [style.format(letter) for letter in alphabet[0:num]]
+    if "\t" in style:
+        letter_list = [letter.replace("\t", "") for letter in letter_list]
+
+    return letter_list[doc["answer"]]
 
 # Letters + Full continuation
 def choice_C(alphabet, style, doc):
@@ -54,6 +67,17 @@ def choice_C(alphabet, style, doc):
 
     return [letter + choice for letter, choice in zip(letter_list, choices)]
 
+def answer_C(alphabet, style, doc):
+
+    choices = doc["choices"]
+    num = len(choices)
+
+    letter_list = [style.format(letter) for letter in alphabet[0:num]]
+    if "\t" not in style:
+        letter_list = [letter + " " for letter in letter_list]
+
+    return [letter + choice for letter, choice in zip(letter_list, choices)][doc["answer"]]
+
 
 template_01 = partial(doc_to_text_base, string.ascii_lowercase, "({})")
 choice_01a = choice_A
@@ -72,6 +96,7 @@ choice_04a = choice_A
 choice_04b = partial(choice_B, string.ascii_lowercase, "{}\t")
 choice_04c = partial(choice_C, string.ascii_lowercase, "{}\t")
 template_05 = partial(doc_to_text_base, string.ascii_uppercase, "({})")
+target_05a = answer_A
 choice_05a = choice_A
 choice_05b = partial(choice_B, string.ascii_uppercase, "({})")
 choice_05c = partial(choice_C, string.ascii_uppercase, "({})")

lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu_flan_cot_fewshot_template_yaml

 dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split
 validation_split: validation
 fewshot_split: dev
+fewshot_config:
+  sampler: first_n
 output_type: generate_until
 doc_to_text: "Q: {{question.strip()}}\n(A) {{choices[0]}} (B) {{choices[1]}} (C) {{choices[2]}} (D) {{choices[3]}}\nA: Let's think step by step."
 doc_to_target: "{{['(A)', '(B)', '(C)', '(D)'][answer]}}"
@@ -8,7 +10,7 @@ filter_list:
   - name: "get-answer"
     filter:
       - function: "regex"
-        regex_pattern: "(?<=The answer is )(.*)(?=.)"
+        regex_pattern: "(?<=answer is )(.*)(?=.)"
       - function: "take_first"
 generation_kwargs:
   until: