add style 01

lm_eval/tasks/mmlu/alternative_worlds/style_01/c/style_01_moral_scenarios.yaml

+"dataset_name": "moral_scenarios"
+"description": "The following are multiple choice questions (with answers) about moral scenarios.\n\n"
+"include": "_template_yaml"
+"task": "mmlu_style_01c_moral_scenarios"

lm_eval/tasks/mmlu/alternative_worlds/style_01/c/style_01_nutrition.yaml

+"dataset_name": "nutrition"
+"description": "The following are multiple choice questions (with answers) about nutrition.\n\n"
+"include": "_template_yaml"
+"task": "mmlu_style_01c_nutrition"

lm_eval/tasks/mmlu/alternative_worlds/style_01/c/style_01_philosophy.yaml

+"dataset_name": "philosophy"
+"description": "The following are multiple choice questions (with answers) about philosophy.\n\n"
+"include": "_template_yaml"
+"task": "mmlu_style_01c_philosophy"

lm_eval/tasks/mmlu/alternative_worlds/style_01/c/style_01_prehistory.yaml

+"dataset_name": "prehistory"
+"description": "The following are multiple choice questions (with answers) about prehistory.\n\n"
+"include": "_template_yaml"
+"task": "mmlu_style_01c_prehistory"

lm_eval/tasks/mmlu/alternative_worlds/style_01/c/style_01_professional_accounting.yaml

+"dataset_name": "professional_accounting"
+"description": "The following are multiple choice questions (with answers) about professional accounting.\n\n"
+"include": "_template_yaml"
+"task": "mmlu_style_01c_professional_accounting"

lm_eval/tasks/mmlu/alternative_worlds/style_01/c/style_01_professional_law.yaml

+"dataset_name": "professional_law"
+"description": "The following are multiple choice questions (with answers) about professional law.\n\n"
+"include": "_template_yaml"
+"task": "mmlu_style_01c_professional_law"

lm_eval/tasks/mmlu/alternative_worlds/style_01/c/style_01_professional_medicine.yaml

+"dataset_name": "professional_medicine"
+"description": "The following are multiple choice questions (with answers) about professional medicine.\n\n"
+"include": "_template_yaml"
+"task": "mmlu_style_01c_professional_medicine"

lm_eval/tasks/mmlu/alternative_worlds/style_01/c/style_01_professional_psychology.yaml

+"dataset_name": "professional_psychology"
+"description": "The following are multiple choice questions (with answers) about professional psychology.\n\n"
+"include": "_template_yaml"
+"task": "mmlu_style_01c_professional_psychology"

lm_eval/tasks/mmlu/alternative_worlds/style_01/c/style_01_public_relations.yaml

+"dataset_name": "public_relations"
+"description": "The following are multiple choice questions (with answers) about public relations.\n\n"
+"include": "_template_yaml"
+"task": "mmlu_style_01c_public_relations"

lm_eval/tasks/mmlu/alternative_worlds/style_01/c/style_01_security_studies.yaml

+"dataset_name": "security_studies"
+"description": "The following are multiple choice questions (with answers) about security studies.\n\n"
+"include": "_template_yaml"
+"task": "mmlu_style_01c_security_studies"

lm_eval/tasks/mmlu/alternative_worlds/style_01/c/style_01_sociology.yaml

+"dataset_name": "sociology"
+"description": "The following are multiple choice questions (with answers) about sociology.\n\n"
+"include": "_template_yaml"
+"task": "mmlu_style_01c_sociology"

lm_eval/tasks/mmlu/alternative_worlds/style_01/c/style_01_us_foreign_policy.yaml

+"dataset_name": "us_foreign_policy"
+"description": "The following are multiple choice questions (with answers) about us foreign policy.\n\n"
+"include": "_template_yaml"
+"task": "mmlu_style_01c_us_foreign_policy"

lm_eval/tasks/mmlu/alternative_worlds/style_01/c/style_01_virology.yaml

+"dataset_name": "virology"
+"description": "The following are multiple choice questions (with answers) about virology.\n\n"
+"include": "_template_yaml"
+"task": "mmlu_style_01c_virology"

lm_eval/tasks/mmlu/alternative_worlds/style_01/c/style_01_world_religions.yaml

+"dataset_name": "world_religions"
+"description": "The following are multiple choice questions (with answers) about world religions.\n\n"
+"include": "_template_yaml"
+"task": "mmlu_style_01c_world_religions"

lm_eval/tasks/mmlu/alternative_worlds/styles.py

+import string
+from functools import partial
+
+def doc_to_text_base(alphabet, style, doc):
+
+    choices = doc["choices"]["text"]
+    num = len(choices)
+
+    letter_list = [style.format(letter) for letter in alphabet[0:num]]
+
+    if "\t" in style:
+        choice_string = "{}{}"
+    else:
+        choice_string = "{} {}"
+
+    doc_to_text = "\n\n".join([
+        "Question: "+doc["question"].strip()+"\nAnswer:",
+        ] + [
+            choice_string.format(i,j) for i,j in zip(letter_list, choices)
+        ]
+    )
+
+    return doc_to_text
+
+# Full continuation
+def choice_A(doc):
+    return doc["choices"]["text"]
+
+# Letters only
+def choice_B(alphabet, style, doc):
+
+    choices = doc["choices"]["text"]
+    num = len(choices)
+
+    letter_list = [style.format(letter) for letter in alphabet[0:num]]
+    if "\t" in style:
+        letter_list = [letter.replace("\t","") for letter in letter_list]
+
+    return letter_list
+
+# Letters + Full continuation
+def choice_C(alphabet, style, doc):
+
+    choices = doc["choices"]["text"]
+    num = len(choices)
+
+    letter_list = [style.format(letter) for letter in alphabet[0:num]]
+    if "\t" not in style:
+        letter_list = [letter+" " for letter in letter_list]
+
+    return [letter+choice for letter, choice in zip(letter_list, choices)]
+
+template_01 = partial(doc_to_text_base, string.ascii_lowercase, "({})")
+choice_01a = choice_A
+choice_01b = partial(choice_B, string.ascii_lowercase, "({})")
+choice_01c = partial(choice_C, string.ascii_lowercase, "({})")
+template_02 = partial(doc_to_text_base, string.ascii_lowercase, "{})")
+choice_02a = choice_A
+choice_02b = partial(choice_B, string.ascii_lowercase, "{})")
+choice_02c = partial(choice_C, string.ascii_lowercase, "{})")
+template_03 = partial(doc_to_text_base, string.ascii_lowercase, "{}.")
+choice_03a = choice_A
+choice_03b = partial(choice_B, string.ascii_lowercase, "{}.")
+choice_03c = partial(choice_C, string.ascii_lowercase, "{}.")
+template_04 = partial(doc_to_text_base, string.ascii_lowercase, "{}\t")
+choice_04a = choice_A
+choice_04b = partial(choice_B, string.ascii_lowercase, "{}\t")
+choice_04c = partial(choice_C, string.ascii_lowercase, "{}\t")
+template_05 = partial(doc_to_text_base, string.ascii_uppercase, "({})")
+choice_05a = choice_A
+choice_05b = partial(choice_B, string.ascii_uppercase, "({})")
+choice_05c = partial(choice_C, string.ascii_uppercase, "({})")
+template_06 = partial(doc_to_text_base, string.ascii_uppercase, "{})")
+choice_06a = choice_A
+choice_06b = partial(choice_B, string.ascii_uppercase, "{})")
+choice_06c = partial(choice_C, string.ascii_uppercase, "{})")
+template_07 = partial(doc_to_text_base, string.ascii_uppercase, "{}.")
+choice_07a = choice_A
+choice_07b = partial(choice_B, string.ascii_uppercase, "{}.")
+choice_07c = partial(choice_C, string.ascii_uppercase, "{}.")
+template_08 = partial(doc_to_text_base, string.ascii_uppercase, "{}\t")
+choice_08a = choice_A
+choice_08b = partial(choice_B, string.ascii_uppercase, "{}\t")
+choice_08c = partial(choice_C, string.ascii_uppercase, "{}\t")
+
+