added prompt and output variations

lm_eval/tasks/mathqa/alternative_worlds/mathqa_alt.yaml

-group: mathqa_alt
-task:
-  - mathqa_01
-  - mathqa_02
-  - mathqa_03
-  - mathqa_04
-  - mathqa_05
-  - mathqa_06
-  - mathqa_07
-  - mathqa_08

lm_eval/tasks/mathqa/alternative_worlds/README.md → lm_eval/tasks/mathqa/alternative_worlds/output_variation/README.md

 
-
 Investigate affect of letter options
 - (A)
 - A)
@@ -9,7 +8,6 @@ Investigate affect of letter options
 - a)
 - a.
 - a\t
-
 Answer types:
 - letters only
     - original option

lm_eval/tasks/mathqa/alternative_worlds/_mathqa_alt_yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/_mathqa_alt_yaml

@@ -6,10 +6,5 @@ test_split: test
 doc_to_target: "{{['a', 'b', 'c', 'd', 'e'].index(correct)}}"
 metric_list:
   - metric: acc
-    aggregation: mean
-    higher_is_better: true
   - metric: acc_norm
-    aggregation: mean
-    higher_is_better: true
   - metric: brier_score
-    higher_is_better: false

lm_eval/tasks/mathqa/alternative_worlds/output_variation/mathqa_alt.yaml

+group: mathqa_alt_ov
+task:
+  - mathqa_alt_ov_01
+  - mathqa_alt_ov_02
+  - mathqa_alt_ov_03
+  - mathqa_alt_ov_04
+  - mathqa_alt_ov_05
+  - mathqa_alt_ov_06
+  - mathqa_alt_ov_07
+  - mathqa_alt_ov_08

lm_eval/tasks/mathqa/alternative_worlds/style_01/a.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_01/a.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_01
-group_alias: style_01
-task: mathqa_01a
-task_alias: a
+group: mathqa_alt_ov_01
+task: mathqa_alt_ov_01a
 doc_to_text: !function ../styles.template_01
 doc_to_choice: !function ../styles.choice_01a

lm_eval/tasks/mathqa/alternative_worlds/style_01/b.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_01/b.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_01
-group_alias: style_01
-task: mathqa_01b
-task_alias: b
+group: mathqa_alt_ov_01
+task: mathqa_alt_ov_01b
 doc_to_text: !function ../styles.template_01
 doc_to_choice: !function ../styles.choice_01b

lm_eval/tasks/mathqa/alternative_worlds/style_01/c.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_01/c.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_01
-group_alias: style_01
-task: mathqa_01c
-task_alias: c
+group: mathqa_alt_ov_01
+task: mathqa_alt_ov_01c
 doc_to_text: !function ../styles.template_01
 doc_to_choice: !function ../styles.choice_01c

lm_eval/tasks/mathqa/alternative_worlds/style_02/a.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_02/a.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_02
-group_alias: style_02
-task: mathqa_02a
-task_alias: a
+group: mathqa_alt_ov_02
+task: mathqa_alt_ov_02a
 doc_to_text: !function ../styles.template_02
 doc_to_choice: !function ../styles.choice_02a

lm_eval/tasks/mathqa/alternative_worlds/style_02/b.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_02/b.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_02
-group_alias: style_02
-task: mathqa_02b
-task_alias: b
+group: mathqa_alt_ov_02
+task: mathqa_alt_ov_02b
 doc_to_text: !function ../styles.template_02
 doc_to_choice: !function ../styles.choice_02b

lm_eval/tasks/mathqa/alternative_worlds/style_02/c.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_02/c.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_02
-group_alias: style_02
-task: mathqa_02c
-task_alias: c
+group: mathqa_alt_ov_02
+task: mathqa_alt_ov_02c
 doc_to_text: !function ../styles.template_02
 doc_to_choice: !function ../styles.choice_02c

lm_eval/tasks/mathqa/alternative_worlds/style_03/a.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_03/a.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_03
-group_alias: style_03
-task: mathqa_03a
-task_alias: a
+group: mathqa_alt_ov_03
+task: mathqa_alt_ov_03a
 doc_to_text: !function ../styles.template_03
 doc_to_choice: !function ../styles.choice_03a

lm_eval/tasks/mathqa/alternative_worlds/style_03/b.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_03/b.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_03
-group_alias: style_03
-task: mathqa_03b
-task_alias: b
+group: mathqa_alt_ov_03
+task: mathqa_alt_ov_03b
 doc_to_text: !function ../styles.template_03
 doc_to_choice: !function ../styles.choice_03b

lm_eval/tasks/mathqa/alternative_worlds/style_03/c.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_03/c.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_03
-group_alias: style_03
-task: mathqa_03c
-task_alias: c
+group: mathqa_alt_ov_03
+task: mathqa_alt_ov_03c
 doc_to_text: !function ../styles.template_03
 doc_to_choice: !function ../styles.choice_03c

lm_eval/tasks/mathqa/alternative_worlds/style_04/a.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_04/a.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_04
-group_alias: style_04
-task: mathqa_04a
-task_alias: a
+group: mathqa_alt_ov_04
+task: mathqa_alt_ov_04a
 doc_to_text: !function ../styles.template_04
 doc_to_choice: !function ../styles.choice_04a

lm_eval/tasks/mathqa/alternative_worlds/style_04/b.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_04/b.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_04
-group_alias: style_04
-task: mathqa_04b
-task_alias: b
+group: mathqa_alt_ov_04
+task: mathqa_alt_ov_04b
 doc_to_text: !function ../styles.template_04
 doc_to_choice: !function ../styles.choice_04b

lm_eval/tasks/mathqa/alternative_worlds/style_04/c.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_04/c.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_04
-group_alias: style_04
-task: mathqa_04c
-task_alias: c
+group: mathqa_alt_ov_04
+task: mathqa_alt_ov_04c
 doc_to_text: !function ../styles.template_04
 doc_to_choice: !function ../styles.choice_04c

lm_eval/tasks/mathqa/alternative_worlds/style_05/a.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_05/a.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_05
-group_alias: style_05
-task: mathqa_05a
-task_alias: a
+group: mathqa_alt_ov_05
+task: mathqa_alt_ov_05a
 doc_to_text: !function ../styles.template_05
 doc_to_choice: !function ../styles.choice_05a

lm_eval/tasks/mathqa/alternative_worlds/style_05/b.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_05/b.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_05
-group_alias: style_05
-task: mathqa_05b
-task_alias: b
+group: mathqa_alt_ov_05
+task: mathqa_alt_ov_05b
 doc_to_text: !function ../styles.template_05
 doc_to_choice: !function ../styles.choice_05b

lm_eval/tasks/mathqa/alternative_worlds/style_05/c.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_05/c.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_05
-group_alias: style_05
-task: mathqa_05c
-task_alias: c
+group: mathqa_alt_ov_05
+task: mathqa_alt_ov_05c
 doc_to_text: !function ../styles.template_05
 doc_to_choice: !function ../styles.choice_05c

lm_eval/tasks/mathqa/alternative_worlds/style_06/a.yaml → lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_06/a.yaml

 include: ../_mathqa_alt_yaml
-group: mathqa_06
-group_alias: style_06
-task: mathqa_06a
-task_alias: a
+group: mathqa_alt_ov_06
+task: mathqa_alt_ov_06a
 doc_to_text: !function ../styles.template_06
 doc_to_choice: !function ../styles.choice_06a