added alternative prompts for mathqa

lm_eval/tasks/mathqa/alternative_worlds/README.md

+
+
+Investigate affect of letter options
+- (A)
+- A)
+- A.
+- A\t
+- (a)
+- a)
+- a.
+- a\t
+
+Answer types:
+- letters only
+    - original option
+    - just letter
+- letters + continuation
+    - original option 
+    - just letter
+- continuation
\ No newline at end of file

lm_eval/tasks/mathqa/alternative_worlds/_mathqa_alt_yaml

+dataset_path: math_qa
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+test_split: test
+doc_to_target: "{{['a', 'b', 'c', 'd', 'e'].index(correct)}}"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: brier_score
+    higher_is_better: false

lm_eval/tasks/mathqa/alternative_worlds/mathqa_alt.yaml

+group: mathqa_alt
+task:
+  - mathqa_01
+  - mathqa_02
+  - mathqa_03
+  - mathqa_04
+  - mathqa_05
+  - mathqa_06
+  - mathqa_07
+  - mathqa_08

lm_eval/tasks/mathqa/alternative_worlds/style_01/a.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_01
+group_alias: style_01 
+task: mathqa_01a
+task_alias: a
+doc_to_text: !function ../styles.template_01
+doc_to_choice: !function ../styles.choice_01a
\ No newline at end of file

lm_eval/tasks/mathqa/alternative_worlds/style_01/b.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_01
+group_alias: style_01 
+task: mathqa_01b
+task_alias: b
+doc_to_text: !function ../styles.template_01
+doc_to_choice: !function ../styles.choice_01b
\ No newline at end of file

lm_eval/tasks/mathqa/alternative_worlds/style_01/c.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_01
+group_alias: style_01 
+task: mathqa_01c
+task_alias: c
+doc_to_text: !function ../styles.template_01
+doc_to_choice: !function ../styles.choice_01c
\ No newline at end of file

lm_eval/tasks/mathqa/alternative_worlds/style_02/a.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_02
+group_alias: style_02 
+task: mathqa_02a
+task_alias: a
+doc_to_text: !function ../styles.template_02
+doc_to_choice: !function ../styles.choice_02a
\ No newline at end of file

lm_eval/tasks/mathqa/alternative_worlds/style_02/b.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_02
+group_alias: style_02 
+task: mathqa_02b
+task_alias: b
+doc_to_text: !function ../styles.template_02
+doc_to_choice: !function ../styles.choice_02b
\ No newline at end of file

lm_eval/tasks/mathqa/alternative_worlds/style_02/c.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_02
+group_alias: style_02 
+task: mathqa_02c
+task_alias: c
+doc_to_text: !function ../styles.template_02
+doc_to_choice: !function ../styles.choice_02c
\ No newline at end of file

lm_eval/tasks/mathqa/alternative_worlds/style_03/a.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_03
+group_alias: style_03 
+task: mathqa_03a
+task_alias: a
+doc_to_text: !function ../styles.template_03
+doc_to_choice: !function ../styles.choice_03a
\ No newline at end of file

lm_eval/tasks/mathqa/alternative_worlds/style_03/b.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_03
+group_alias: style_03 
+task: mathqa_03b
+task_alias: b
+doc_to_text: !function ../styles.template_03
+doc_to_choice: !function ../styles.choice_03b
\ No newline at end of file

lm_eval/tasks/mathqa/alternative_worlds/style_03/c.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_03
+group_alias: style_03 
+task: mathqa_03c
+task_alias: c
+doc_to_text: !function ../styles.template_03
+doc_to_choice: !function ../styles.choice_03c
\ No newline at end of file

lm_eval/tasks/mathqa/alternative_worlds/style_04/a.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_04
+group_alias: style_04 
+task: mathqa_04a
+task_alias: a
+doc_to_text: !function ../styles.template_04
+doc_to_choice: !function ../styles.choice_04a
\ No newline at end of file

lm_eval/tasks/mathqa/alternative_worlds/style_04/b.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_04
+group_alias: style_04 
+task: mathqa_04b
+task_alias: b
+doc_to_text: !function ../styles.template_04
+doc_to_choice: !function ../styles.choice_04b
\ No newline at end of file

lm_eval/tasks/mathqa/alternative_worlds/style_04/c.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_04
+group_alias: style_04 
+task: mathqa_04c
+task_alias: c
+doc_to_text: !function ../styles.template_04
+doc_to_choice: !function ../styles.choice_04c
\ No newline at end of file

lm_eval/tasks/mathqa/alternative_worlds/style_05/a.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_05
+group_alias: style_05 
+task: mathqa_05a
+task_alias: a
+doc_to_text: !function ../styles.template_05
+doc_to_choice: !function ../styles.choice_05a
\ No newline at end of file

lm_eval/tasks/mathqa/alternative_worlds/style_05/b.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_05
+group_alias: style_05 
+task: mathqa_05b
+task_alias: b
+doc_to_text: !function ../styles.template_05
+doc_to_choice: !function ../styles.choice_05b
\ No newline at end of file

lm_eval/tasks/mathqa/alternative_worlds/style_05/c.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_05
+group_alias: style_05 
+task: mathqa_05c
+task_alias: c
+doc_to_text: !function ../styles.template_05
+doc_to_choice: !function ../styles.choice_05c
\ No newline at end of file

lm_eval/tasks/mathqa/alternative_worlds/style_06/a.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_06
+group_alias: style_06 
+task: mathqa_06a
+task_alias: a
+doc_to_text: !function ../styles.template_06
+doc_to_choice: !function ../styles.choice_06a
\ No newline at end of file

lm_eval/tasks/mathqa/alternative_worlds/style_06/b.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_06
+group_alias: style_06 
+task: mathqa_06b
+task_alias: b
+doc_to_text: !function ../styles.template_06
+doc_to_choice: !function ../styles.choice_06b
\ No newline at end of file