added prompt and output variations

lm_eval/tasks/mathqa/alternative_worlds/style_06/b.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_06/b.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_06
-group_alias: style_06
-task: mathqa_06b
-task_alias: b
+group: mathqa_alt_ov_06
+task: mathqa_alt_ov_06b
 doc_to_text: !function ../styles.template_06
 doc_to_choice: !function ../styles.choice_06b

lm_eval/tasks/mathqa/alternative_worlds/style_06/c.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_06/c.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_06
-group_alias: style_06
-task: mathqa_06c
-task_alias: c
+group: mathqa_alt_ov_06
+task: mathqa_alt_ov_06c
 doc_to_text: !function ../styles.template_06
 doc_to_choice: !function ../styles.choice_06c

lm_eval/tasks/mathqa/alternative_worlds/style_07/a.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_07/a.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_07
-group_alias: style_07
-task: mathqa_07a
-task_alias: a
+group: mathqa_alt_ov_07
+task: mathqa_alt_ov_07a
 doc_to_text: !function ../styles.template_07
 doc_to_choice: !function ../styles.choice_07a

lm_eval/tasks/mathqa/alternative_worlds/style_07/b.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_07/b.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_07
-group_alias: style_07
-task: mathqa_07b
-task_alias: b
+group: mathqa_alt_ov_07
+task: mathqa_alt_ov_07b
 doc_to_text: !function ../styles.template_07
 doc_to_choice: !function ../styles.choice_07b

lm_eval/tasks/mathqa/alternative_worlds/style_07/c.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_07/c.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_07
-group_alias: style_07
-task: mathqa_07c
-task_alias: c
+group: mathqa_alt_ov_07
+task: mathqa_alt_ov_07c
 doc_to_text: !function ../styles.template_07
 doc_to_choice: !function ../styles.choice_07c

lm_eval/tasks/mathqa/alternative_worlds/style_08/a.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_08/a.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_08
-group_alias: style_08
-task: mathqa_08a
-task_alias: a
+group: mathqa_alt_ov_08
+task: mathqa_alt_ov_08a
 doc_to_text: !function ../styles.template_08
 doc_to_choice: !function ../styles.choice_08a

lm_eval/tasks/mathqa/alternative_worlds/style_08/b.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_08/b.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_08
-group_alias: style_08
-task: mathqa_08b
-task_alias: b
+group: mathqa_alt_ov_08
+task: mathqa_alt_ov_08b
 doc_to_text: !function ../styles.template_08
 doc_to_choice: !function ../styles.choice_08b

lm_eval/tasks/mathqa/alternative_worlds/style_08/c.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_08/c.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_08
-group_alias: style_08
-task: mathqa_08c
-task_alias: c
+group: mathqa_alt_ov_08
+task: mathqa_alt_ov_08c
 doc_to_text: !function ../styles.template_08
 doc_to_choice: !function ../styles.choice_08c

lm_eval/tasks/mathqa/alternative_worlds/styles.py → lm_eval/tasks/mathqa/alternative_worlds/output_variation/styles.py

@@ -2,7 +2,6 @@ import re
 import string
 from functools import partial
 
-
 def parse_choices(doc):
     choices = [
         c[4:].rstrip(" ,")
@@ -10,58 +9,43 @@ def parse_choices(doc):
     ]
     return choices
 
-
 def doc_to_text_base(alphabet, style, doc):
-
     choices = parse_choices(doc)
-
     num = len(choices)
-
     letter_list = [style.format(letter) for letter in alphabet[0:num]]
-
     if "\t" in style:
         choice_string = "{}{}"
     else:
         choice_string = "{} {}"
-    doc_to_text = "\n\n".join(
-        [doc["Problem"]]
+    doc_to_text = "\n".join(
+        ["Question: " + doc["Problem"]]
         + [choice_string.format(i, j) for i, j in zip(letter_list, choices)]
+        + ["Answer:"]
     )
-
     return doc_to_text
 
-
 # Full continuation
 def choice_A(doc):
     return parse_choices(doc)
 
-
 # Letters only
 def choice_B(alphabet, style, doc):
-
     choices = parse_choices(doc)
     num = len(choices)
-
     letter_list = [style.format(letter) for letter in alphabet[0:num]]
     if "\t" in style:
         letter_list = [letter.replace("\t", "") for letter in letter_list]
-
     return letter_list
 
-
 # Letters + Full continuation
 def choice_C(alphabet, style, doc):
-
     choices = parse_choices(doc)
     num = len(choices)
-
     letter_list = [style.format(letter) for letter in alphabet[0:num]]
     if "\t" not in style:
         letter_list = [letter + " " for letter in letter_list]
-
     return [letter + choice for letter, choice in zip(letter_list, choices)]
 
-
 template_01 = partial(doc_to_text_base, string.ascii_lowercase, "({})")
 choice_01a = choice_A
 choice_01b = partial(choice_B, string.ascii_lowercase, "({})")

lm_eval/tasks/mathqa/alternative_worlds/prompt_variation/style_01.yaml

+group: mathqa_alt_pv
+task: mathqa_alt_pv_01
+dataset_path: math_qa
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+test_split: test
+doc_to_text: "{{Problem}}"
+doc_to_target: "{{['a', 'b', 'c', 'd', 'e'].index(correct)}}"
+doc_to_choice: !function ../../utils.doc_to_choice
+should_decontaminate: true
+doc_to_decontamination_query: "{{Problem}}"
+metric_list:
+  - metric: acc
+  - metric: acc_norm
+  - metric: brier_score

lm_eval/tasks/mathqa/alternative_worlds/prompt_variation/style_02.yaml

+group: mathqa_alt_pv
+task: mathqa_alt_pv_02
+dataset_path: math_qa
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+test_split: test
+doc_to_text: "Q: {{Problem}}\nA:"
+doc_to_target: "{{['a', 'b', 'c', 'd', 'e'].index(correct)}}"
+doc_to_choice: !function ../../utils.doc_to_choice
+should_decontaminate: true
+doc_to_decontamination_query: "Q: {{Problem}}\nA:"
+metric_list:
+  - metric: acc
+  - metric: acc_norm
+  - metric: brier_score

lm_eval/tasks/mathqa/alternative_worlds/prompt_variation/style_03.yaml

+group: mathqa_alt_pv
+task: mathqa_alt_pv_03
+dataset_path: math_qa
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+test_split: test
+doc_to_text: "Question: {{Problem}}\nAnswer:"
+doc_to_target: "{{['a', 'b', 'c', 'd', 'e'].index(correct)}}"
+doc_to_choice: !function ../../utils.doc_to_choice
+should_decontaminate: true
+doc_to_decontamination_query: "Question: {{Problem}}\nAnswer:"
+metric_list:
+  - metric: acc
+  - metric: acc_norm
+  - metric: brier_score