added alternative prompts for hellaswag

lm_eval/tasks/hellaswag/alternative_worlds/README.md

+
+
+Investigate affect of letter options
+- (A)
+- A)
+- A.
+- A\t
+- (a)
+- a)
+- a.
+- a\t
+
+Answer types:
+- letters only
+    - original option
+    - just letter
+- letters + continuation
+    - original option 
+    - just letter
+- continuation
\ No newline at end of file

lm_eval/tasks/hellaswag/alternative_worlds/_hellaswag_alt_yaml

+dataset_path: Rowan/hellaswag
+dataset_name: null
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+test_split: null
+process_docs: !function ../utils.process_docs
+doc_to_text: "{{query}}"
+doc_to_target: "{{label}}"
+doc_to_choice: "{{choices}}"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: brier_score
+    higher_is_better: false

lm_eval/tasks/hellaswag/alternative_worlds/hellaswag_alt.yaml

+group: hellaswag_alt
+task:
+  - hellaswag_01
+  - hellaswag_02
+  - hellaswag_03
+  - hellaswag_04
+  - hellaswag_05
+  - hellaswag_06
+  - hellaswag_07
+  - hellaswag_08

lm_eval/tasks/hellaswag/alternative_worlds/style_01/a.yaml

+include: ../_hellaswag_alt_yaml
+group: hellaswag_01
+group_alias: style_01 
+task: hellaswag_01a
+task_alias: a
+doc_to_text: !function ../styles.template_01
+doc_to_choice: !function ../styles.choice_01a
\ No newline at end of file

lm_eval/tasks/hellaswag/alternative_worlds/style_01/b.yaml

+include: ../_hellaswag_alt_yaml
+group: hellaswag_01
+group_alias: style_01 
+task: hellaswag_01b
+task_alias: b
+doc_to_text: !function ../styles.template_01
+doc_to_choice: !function ../styles.choice_01b
\ No newline at end of file

lm_eval/tasks/hellaswag/alternative_worlds/style_01/c.yaml

+include: ../_hellaswag_alt_yaml
+group: hellaswag_01
+group_alias: style_01 
+task: hellaswag_01c
+task_alias: c
+doc_to_text: !function ../styles.template_01
+doc_to_choice: !function ../styles.choice_01c
\ No newline at end of file

lm_eval/tasks/hellaswag/alternative_worlds/style_02/a.yaml

+include: ../_hellaswag_alt_yaml
+group: hellaswag_02
+group_alias: style_02 
+task: hellaswag_02a
+task_alias: a
+doc_to_text: !function ../styles.template_02
+doc_to_choice: !function ../styles.choice_02a
\ No newline at end of file

lm_eval/tasks/hellaswag/alternative_worlds/style_02/b.yaml

+include: ../_hellaswag_alt_yaml
+group: hellaswag_02
+group_alias: style_02 
+task: hellaswag_02b
+task_alias: b
+doc_to_text: !function ../styles.template_02
+doc_to_choice: !function ../styles.choice_02b
\ No newline at end of file

lm_eval/tasks/hellaswag/alternative_worlds/style_02/c.yaml

+include: ../_hellaswag_alt_yaml
+group: hellaswag_02
+group_alias: style_02 
+task: hellaswag_02c
+task_alias: c
+doc_to_text: !function ../styles.template_02
+doc_to_choice: !function ../styles.choice_02c
\ No newline at end of file

lm_eval/tasks/hellaswag/alternative_worlds/style_03/a.yaml

+include: ../_hellaswag_alt_yaml
+group: hellaswag_03
+group_alias: style_03 
+task: hellaswag_03a
+task_alias: a
+doc_to_text: !function ../styles.template_03
+doc_to_choice: !function ../styles.choice_03a
\ No newline at end of file

lm_eval/tasks/hellaswag/alternative_worlds/style_03/b.yaml

+include: ../_hellaswag_alt_yaml
+group: hellaswag_03
+group_alias: style_03 
+task: hellaswag_03b
+task_alias: b
+doc_to_text: !function ../styles.template_03
+doc_to_choice: !function ../styles.choice_03b
\ No newline at end of file

lm_eval/tasks/hellaswag/alternative_worlds/style_03/c.yaml

+include: ../_hellaswag_alt_yaml
+group: hellaswag_03
+group_alias: style_03 
+task: hellaswag_03c
+task_alias: c
+doc_to_text: !function ../styles.template_03
+doc_to_choice: !function ../styles.choice_03c
\ No newline at end of file

lm_eval/tasks/hellaswag/alternative_worlds/style_04/a.yaml

+include: ../_hellaswag_alt_yaml
+group: hellaswag_04
+group_alias: style_04 
+task: hellaswag_04a
+task_alias: a
+doc_to_text: !function ../styles.template_04
+doc_to_choice: !function ../styles.choice_04a
\ No newline at end of file

lm_eval/tasks/hellaswag/alternative_worlds/style_04/b.yaml

+include: ../_hellaswag_alt_yaml
+group: hellaswag_04
+group_alias: style_04 
+task: hellaswag_04b
+task_alias: b
+doc_to_text: !function ../styles.template_04
+doc_to_choice: !function ../styles.choice_04b
\ No newline at end of file

lm_eval/tasks/hellaswag/alternative_worlds/style_04/c.yaml

+include: ../_hellaswag_alt_yaml
+group: hellaswag_04
+group_alias: style_04 
+task: hellaswag_04c
+task_alias: c
+doc_to_text: !function ../styles.template_04
+doc_to_choice: !function ../styles.choice_04c
\ No newline at end of file

lm_eval/tasks/hellaswag/alternative_worlds/style_05/a.yaml

+include: ../_hellaswag_alt_yaml
+group: hellaswag_05
+group_alias: style_05 
+task: hellaswag_05a
+task_alias: a
+doc_to_text: !function ../styles.template_05
+doc_to_choice: !function ../styles.choice_05a
\ No newline at end of file

lm_eval/tasks/hellaswag/alternative_worlds/style_05/b.yaml

+include: ../_hellaswag_alt_yaml
+group: hellaswag_05
+group_alias: style_05 
+task: hellaswag_05b
+task_alias: b
+doc_to_text: !function ../styles.template_05
+doc_to_choice: !function ../styles.choice_05b
\ No newline at end of file

lm_eval/tasks/hellaswag/alternative_worlds/style_05/c.yaml

+include: ../_hellaswag_alt_yaml
+group: hellaswag_05
+group_alias: style_05 
+task: hellaswag_05c
+task_alias: c
+doc_to_text: !function ../styles.template_05
+doc_to_choice: !function ../styles.choice_05c
\ No newline at end of file

lm_eval/tasks/hellaswag/alternative_worlds/style_06/a.yaml

+include: ../_hellaswag_alt_yaml
+group: hellaswag_06
+group_alias: style_06 
+task: hellaswag_06a
+task_alias: a
+doc_to_text: !function ../styles.template_06
+doc_to_choice: !function ../styles.choice_06a
\ No newline at end of file

lm_eval/tasks/hellaswag/alternative_worlds/style_06/b.yaml

+include: ../_hellaswag_alt_yaml
+group: hellaswag_06
+group_alias: style_06 
+task: hellaswag_06b
+task_alias: b
+doc_to_text: !function ../styles.template_06
+doc_to_choice: !function ../styles.choice_06b
\ No newline at end of file