add boolq

lm_eval/tasks/super_glue/boolq/alternative_worlds/README.md

+
+
+Investigate affect of letter options
+- (A)
+- A)
+- A.
+- A\t
+- (a)
+- a)
+- a.
+- a\t
+
+Answer types:
+- letters only
+    - original option
+    - just letter
+- letters + continuation
+    - original option
+    - just letter
+- continuation

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/_boolq_alt_ov_yaml

+dataset_path: super_glue
+dataset_name: boolq
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+doc_to_text: "{{passage}}\nQuestion: {{question}}?\nAnswer:"
+doc_to_target: label
+doc_to_choice: ["no", "yes"]
+should_decontaminate: true
+doc_to_decontamination_query: passage
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+  - metric: brier_score
+    aggregation: brier_score
+    higher_is_better: false
+metadata:
+  - version: 1.0

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/boolq_ov.yaml

+group: boolq_alt_ov
+task:
+  - boolq_alt_ov_01
+  - boolq_alt_ov_02
+  - boolq_alt_ov_03
+  - boolq_alt_ov_04
+  - boolq_alt_ov_05
+  - boolq_alt_ov_06
+  - boolq_alt_ov_07
+  - boolq_alt_ov_08

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_01/a.yaml

+include: ../_boolq_alt_ov_yaml
+group: boolq_alt_ov_01
+task: boolq_alt_ov_01a
+doc_to_text: !function ../styles.template_01
+doc_to_choice: !function ../styles.choice_01a
+doc_to_decontamination_query: !function ../styles.template_01

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_01/b.yaml

+include: ../_boolq_alt_ov_yaml
+group: boolq_alt_ov_01
+task: boolq_alt_ov_01b
+doc_to_text: !function ../styles.template_01
+doc_to_choice: !function ../styles.choice_01b
+doc_to_decontamination_query: !function ../styles.template_01

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_01/c.yaml

+include: ../_boolq_alt_ov_yaml
+group: boolq_alt_ov_01
+task: boolq_alt_ov_01c
+doc_to_text: !function ../styles.template_01
+doc_to_choice: !function ../styles.choice_01c
+doc_to_decontamination_query: !function ../styles.template_01

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_02/a.yaml

+include: ../_boolq_alt_ov_yaml
+group: boolq_alt_ov_02
+task: boolq_alt_ov_02a
+doc_to_text: !function ../styles.template_02
+doc_to_choice: !function ../styles.choice_02a
+doc_to_decontamination_query: !function ../styles.template_02

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_02/b.yaml

+include: ../_boolq_alt_ov_yaml
+group: boolq_alt_ov_02
+task: boolq_alt_ov_02b
+doc_to_text: !function ../styles.template_02
+doc_to_choice: !function ../styles.choice_02b
+doc_to_decontamination_query: !function ../styles.template_02

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_02/c.yaml

+include: ../_boolq_alt_ov_yaml
+group: boolq_alt_ov_02
+task: boolq_alt_ov_02c
+doc_to_text: !function ../styles.template_02
+doc_to_choice: !function ../styles.choice_02c
+doc_to_decontamination_query: !function ../styles.template_02

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_03/a.yaml

+include: ../_boolq_alt_ov_yaml
+group: boolq_alt_ov_03
+task: boolq_alt_ov_03a
+doc_to_text: !function ../styles.template_03
+doc_to_choice: !function ../styles.choice_03a
+doc_to_decontamination_query: !function ../styles.template_03

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_03/b.yaml

+include: ../_boolq_alt_ov_yaml
+group: boolq_alt_ov_03
+task: boolq_alt_ov_03b
+doc_to_text: !function ../styles.template_03
+doc_to_choice: !function ../styles.choice_03b
+doc_to_decontamination_query: !function ../styles.template_03

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_03/c.yaml

+include: ../_boolq_alt_ov_yaml
+group: boolq_alt_ov_03
+task: boolq_alt_ov_03c
+doc_to_text: !function ../styles.template_03
+doc_to_choice: !function ../styles.choice_03c
+doc_to_decontamination_query: !function ../styles.template_03

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_04/a.yaml

+include: ../_boolq_alt_ov_yaml
+group: boolq_alt_ov_04
+task: boolq_alt_ov_04a
+doc_to_text: !function ../styles.template_04
+doc_to_choice: !function ../styles.choice_04a
+doc_to_decontamination_query: !function ../styles.template_04

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_04/b.yaml

+include: ../_boolq_alt_ov_yaml
+group: boolq_alt_ov_04
+task: boolq_alt_ov_04b
+doc_to_text: !function ../styles.template_04
+doc_to_choice: !function ../styles.choice_04b
+doc_to_decontamination_query: !function ../styles.template_04

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_04/c.yaml

+include: ../_boolq_alt_ov_yaml
+group: boolq_alt_ov_04
+task: boolq_alt_ov_04c
+doc_to_text: !function ../styles.template_04
+doc_to_choice: !function ../styles.choice_04c
+doc_to_decontamination_query: !function ../styles.template_04

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_05/a.yaml

+include: ../_boolq_alt_ov_yaml
+group: boolq_alt_ov_05
+task: boolq_alt_ov_05a
+doc_to_text: !function ../styles.template_05
+doc_to_choice: !function ../styles.choice_05a
+doc_to_decontamination_query: !function ../styles.template_05

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_05/b.yaml

+include: ../_boolq_alt_ov_yaml
+group: boolq_alt_ov_05
+task: boolq_alt_ov_05b
+doc_to_text: !function ../styles.template_05
+doc_to_choice: !function ../styles.choice_05b
+doc_to_decontamination_query: !function ../styles.template_05

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_05/c.yaml

+include: ../_boolq_alt_ov_yaml
+group: boolq_alt_ov_05
+task: boolq_alt_ov_05c
+doc_to_text: !function ../styles.template_05
+doc_to_choice: !function ../styles.choice_05c
+doc_to_decontamination_query: !function ../styles.template_05

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_06/a.yaml

+include: ../_boolq_alt_ov_yaml
+group: boolq_alt_ov_06
+task: boolq_alt_ov_06a
+doc_to_text: !function ../styles.template_06
+doc_to_choice: !function ../styles.choice_06a
+doc_to_decontamination_query: !function ../styles.template_06

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_06/b.yaml

+include: ../_boolq_alt_ov_yaml
+group: boolq_alt_ov_06
+task: boolq_alt_ov_06b
+doc_to_text: !function ../styles.template_06
+doc_to_choice: !function ../styles.choice_06b
+doc_to_decontamination_query: !function ../styles.template_06