removed alt worlds prompts

lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_07/c.yaml

-include: ../_mathqa_alt_yaml
-group: mathqa_alt_ov_07
-task: mathqa_alt_ov_07c
-doc_to_text: !function ../styles.template_07
-doc_to_choice: !function ../styles.choice_07c

lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_08/a.yaml

-include: ../_mathqa_alt_yaml
-group: mathqa_alt_ov_08
-task: mathqa_alt_ov_08a
-doc_to_text: !function ../styles.template_08
-doc_to_choice: !function ../styles.choice_08a

lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_08/b.yaml

-include: ../_mathqa_alt_yaml
-group: mathqa_alt_ov_08
-task: mathqa_alt_ov_08b
-doc_to_text: !function ../styles.template_08
-doc_to_choice: !function ../styles.choice_08b

lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_08/c.yaml

-include: ../_mathqa_alt_yaml
-group: mathqa_alt_ov_08
-task: mathqa_alt_ov_08c
-doc_to_text: !function ../styles.template_08
-doc_to_choice: !function ../styles.choice_08c

lm_eval/tasks/mathqa/alternative_worlds/output_variation/styles.py

-import re
-import string
-from functools import partial
-
-
-def parse_choices(doc):
-    choices = [
-        c[4:].rstrip(" ,")
-        for c in re.findall(r"[abcd] \) .*?, |e \) .*?$", doc["options"])
-    ]
-    return choices
-
-
-def doc_to_text_base(alphabet, style, doc):
-    choices = parse_choices(doc)
-    num = len(choices)
-    letter_list = [style.format(letter) for letter in alphabet[0:num]]
-    if "\t" in style:
-        choice_string = "{}{}"
-    else:
-        choice_string = "{} {}"
-    doc_to_text = "\n".join(
-        ["Question: " + doc["Problem"]]
-        + [choice_string.format(i, j) for i, j in zip(letter_list, choices)]
-        + ["Answer:"]
-    )
-    return doc_to_text
-
-
-# Full continuation
-def choice_A(doc):
-    return parse_choices(doc)
-
-
-# Letters only
-def choice_B(alphabet, style, doc):
-    choices = parse_choices(doc)
-    num = len(choices)
-    letter_list = [style.format(letter) for letter in alphabet[0:num]]
-    if "\t" in style:
-        letter_list = [letter.replace("\t", "") for letter in letter_list]
-    return letter_list
-
-
-# Letters + Full continuation
-def choice_C(alphabet, style, doc):
-    choices = parse_choices(doc)
-    num = len(choices)
-    letter_list = [style.format(letter) for letter in alphabet[0:num]]
-    if "\t" not in style:
-        letter_list = [letter + " " for letter in letter_list]
-    return [letter + choice for letter, choice in zip(letter_list, choices)]
-
-
-template_01 = partial(doc_to_text_base, string.ascii_lowercase, "({})")
-choice_01a = choice_A
-choice_01b = partial(choice_B, string.ascii_lowercase, "({})")
-choice_01c = partial(choice_C, string.ascii_lowercase, "({})")
-template_02 = partial(doc_to_text_base, string.ascii_lowercase, "{})")
-choice_02a = choice_A
-choice_02b = partial(choice_B, string.ascii_lowercase, "{})")
-choice_02c = partial(choice_C, string.ascii_lowercase, "{})")
-template_03 = partial(doc_to_text_base, string.ascii_lowercase, "{}.")
-choice_03a = choice_A
-choice_03b = partial(choice_B, string.ascii_lowercase, "{}.")
-choice_03c = partial(choice_C, string.ascii_lowercase, "{}.")
-template_04 = partial(doc_to_text_base, string.ascii_lowercase, "{}\t")
-choice_04a = choice_A
-choice_04b = partial(choice_B, string.ascii_lowercase, "{}\t")
-choice_04c = partial(choice_C, string.ascii_lowercase, "{}\t")
-template_05 = partial(doc_to_text_base, string.ascii_uppercase, "({})")
-choice_05a = choice_A
-choice_05b = partial(choice_B, string.ascii_uppercase, "({})")
-choice_05c = partial(choice_C, string.ascii_uppercase, "({})")
-template_06 = partial(doc_to_text_base, string.ascii_uppercase, "{})")
-choice_06a = choice_A
-choice_06b = partial(choice_B, string.ascii_uppercase, "{})")
-choice_06c = partial(choice_C, string.ascii_uppercase, "{})")
-template_07 = partial(doc_to_text_base, string.ascii_uppercase, "{}.")
-choice_07a = choice_A
-choice_07b = partial(choice_B, string.ascii_uppercase, "{}.")
-choice_07c = partial(choice_C, string.ascii_uppercase, "{}.")
-template_08 = partial(doc_to_text_base, string.ascii_uppercase, "{}\t")
-choice_08a = choice_A
-choice_08b = partial(choice_B, string.ascii_uppercase, "{}\t")
-choice_08c = partial(choice_C, string.ascii_uppercase, "{}\t")

lm_eval/tasks/mathqa/alternative_worlds/prompt_variation/style_01.yaml

-group: mathqa_alt_pv
-task: mathqa_alt_pv_01
-dataset_path: math_qa
-output_type: multiple_choice
-training_split: train
-validation_split: validation
-test_split: test
-doc_to_text: "{{Problem}}"
-doc_to_target: "{{['a', 'b', 'c', 'd', 'e'].index(correct)}}"
-doc_to_choice: !function ../../utils.doc_to_choice
-should_decontaminate: true
-doc_to_decontamination_query: "{{Problem}}"
-metric_list:
-  - metric: acc
-  - metric: acc_norm
-  - metric: brier_score

lm_eval/tasks/mathqa/alternative_worlds/prompt_variation/style_02.yaml

-group: mathqa_alt_pv
-task: mathqa_alt_pv_02
-dataset_path: math_qa
-output_type: multiple_choice
-training_split: train
-validation_split: validation
-test_split: test
-doc_to_text: "Q: {{Problem}}\nA:"
-doc_to_target: "{{['a', 'b', 'c', 'd', 'e'].index(correct)}}"
-doc_to_choice: !function ../../utils.doc_to_choice
-should_decontaminate: true
-doc_to_decontamination_query: "Q: {{Problem}}\nA:"
-metric_list:
-  - metric: acc
-  - metric: acc_norm
-  - metric: brier_score

lm_eval/tasks/mathqa/alternative_worlds/prompt_variation/style_03.yaml

-group: mathqa_alt_pv
-task: mathqa_alt_pv_03
-dataset_path: math_qa
-output_type: multiple_choice
-training_split: train
-validation_split: validation
-test_split: test
-doc_to_text: "Question: {{Problem}}\nAnswer:"
-doc_to_target: "{{['a', 'b', 'c', 'd', 'e'].index(correct)}}"
-doc_to_choice: !function ../../utils.doc_to_choice
-should_decontaminate: true
-doc_to_decontamination_query: "Question: {{Problem}}\nAnswer:"
-metric_list:
-  - metric: acc
-  - metric: acc_norm
-  - metric: brier_score

lm_eval/tasks/mmlu/alternative_worlds/mmlu_output_variation.yaml

-group: mmlu_alt_ov
-task:
-  - mmlu_alt_ov_01
-  - mmlu_alt_ov_02
-  - mmlu_alt_ov_03
-  - mmlu_alt_ov_04
-  - mmlu_alt_ov_05
-  - mmlu_alt_ov_06
-  - mmlu_alt_ov_07
-  - mmlu_alt_ov_08

lm_eval/tasks/mmlu/alternative_worlds/mmlu_prompt_variation.yaml

-group: mmlu_alt_pv
-task:
-  - mmlu_alt_pv_01
-  - mmlu_alt_pv_02
-  - mmlu_alt_pv_03

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/_mmlu_ov_01.yaml

-group: mmlu_alt_ov_01
-task:
-  - mmlu_alt_ov_01a
-  - mmlu_alt_ov_01b
-  - mmlu_alt_ov_01c

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/_mmlu.yaml

-group: mmlu_alt_ov_01a
-task:
-  - mmlu_alt_ov_01a_stem
-  - mmlu_alt_ov_01a_other
-  - mmlu_alt_ov_01a_social_sciences
-  - mmlu_alt_ov_01a_humanities

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/_template_yaml

-dataset_path: hails/mmlu_no_train
-test_split: test
-fewshot_split: dev
-output_type: multiple_choice
-doc_to_text: !function ../../../styles.template_01
-doc_to_choice: !function ../../../styles.choice_01a
-doc_to_target: answer
-metric_list:
-  - metric: acc
-  - metric: acc_norm
-  - metric: brier_score

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_abstract_algebra.yaml

-"dataset_name": "abstract_algebra"
-"description": "The following are multiple choice questions (with answers) about abstract\
-  \ algebra.\n\n"
-"group": "mmlu_alt_ov_01a_stem"
-"include": "_template_yaml"
-"task": "mmlu_alt_ov_01a_abstract_algebra"

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_anatomy.yaml

-"dataset_name": "anatomy"
-"description": "The following are multiple choice questions (with answers) about anatomy.\n\
-  \n"
-"group": "mmlu_alt_ov_01a_stem"
-"include": "_template_yaml"
-"task": "mmlu_alt_ov_01a_anatomy"

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_astronomy.yaml

-"dataset_name": "astronomy"
-"description": "The following are multiple choice questions (with answers) about astronomy.\n\
-  \n"
-"group": "mmlu_alt_ov_01a_stem"
-"include": "_template_yaml"
-"task": "mmlu_alt_ov_01a_astronomy"

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_business_ethics.yaml

-"dataset_name": "business_ethics"
-"description": "The following are multiple choice questions (with answers) about business\
-  \ ethics.\n\n"
-"group": "mmlu_alt_ov_01a_other"
-"include": "_template_yaml"
-"task": "mmlu_alt_ov_01a_business_ethics"

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_clinical_knowledge.yaml

-"dataset_name": "clinical_knowledge"
-"description": "The following are multiple choice questions (with answers) about clinical\
-  \ knowledge.\n\n"
-"group": "mmlu_alt_ov_01a_other"
-"include": "_template_yaml"
-"task": "mmlu_alt_ov_01a_clinical_knowledge"

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_biology.yaml

-"dataset_name": "college_biology"
-"description": "The following are multiple choice questions (with answers) about college\
-  \ biology.\n\n"
-"group": "mmlu_alt_ov_01a_stem"
-"include": "_template_yaml"
-"task": "mmlu_alt_ov_01a_college_biology"

lm_eval/tasks/mmlu/alternative_worlds/output_variation/style_01/a/mmlu_college_chemistry.yaml

-"dataset_name": "college_chemistry"
-"description": "The following are multiple choice questions (with answers) about college\
-  \ chemistry.\n\n"
-"group": "mmlu_alt_ov_01a_stem"
-"include": "_template_yaml"
-"task": "mmlu_alt_ov_01a_college_chemistry"