removed alt worlds prompts

lm_eval/tasks/arc/alternative_worlds/arc_challenge/README.md

-
-
-Investigate affect of letter options
-- (A)
-- A)
-- A.
-- A\t
-- (a)
-- a)
-- a.
-- a\t
-
-Answer types:
-- letters only
-    - original option
-    - just letter
-- letters + continuation
-    - original option
-    - just letter
-- continuation

lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/_arc_challenge_alt_yaml

-group:
-  - ai2_arc
-dataset_path: ai2_arc
-dataset_name: ARC-Challenge
-output_type: multiple_choice
-training_split: train
-validation_split: validation
-test_split: test
-doc_to_text: "Question: {{question}}\nAnswer:"
-doc_to_target: "{{choices.label.index(answerKey)}}"
-doc_to_choice: "{{choices.text}}"
-should_decontaminate: true
-doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
-metric_list:
-  - metric: acc
-    aggregation: mean
-    higher_is_better: true
-  - metric: acc_norm
-    aggregation: mean
-    higher_is_better: true
-  - metric: brier_score
-    aggregation: brier_score
-    higher_is_better: false

lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/arc_challenge_alt.yaml

-group: arc_challenge_alt_ov
-task:
-  - arc_challenge_alt_ov_01
-  - arc_challenge_alt_ov_02
-  - arc_challenge_alt_ov_03
-  - arc_challenge_alt_ov_04
-  - arc_challenge_alt_ov_05
-  - arc_challenge_alt_ov_06
-  - arc_challenge_alt_ov_07
-  - arc_challenge_alt_ov_08

lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_01/a.yaml

-include: ../_arc_challenge_alt_yaml
-group: arc_challenge_alt_ov_01
-task: arc_challenge_alt_ov_01a
-doc_to_text: !function ../styles.template_01
-doc_to_choice: !function ../styles.choice_01a
-doc_to_decontamination_query: !function ../styles.template_01

lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_01/b.yaml

-include: ../_arc_challenge_alt_yaml
-group: arc_challenge_alt_ov_01
-task: arc_challenge_alt_ov_01b
-doc_to_text: !function ../styles.template_01
-doc_to_choice: !function ../styles.choice_01b
-doc_to_decontamination_query: !function ../styles.template_01

lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_01/c.yaml

-include: ../_arc_challenge_alt_yaml
-group: arc_challenge_alt_ov_01
-task: arc_challenge_alt_ov_01c
-doc_to_text: !function ../styles.template_01
-doc_to_choice: !function ../styles.choice_01c
-doc_to_decontamination_query: !function ../styles.template_01

lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_02/a.yaml

-include: ../_arc_challenge_alt_yaml
-group: arc_challenge_alt_ov_02
-task: arc_challenge_alt_ov_02a
-doc_to_text: !function ../styles.template_02
-doc_to_choice: !function ../styles.choice_02a
-doc_to_decontamination_query: !function ../styles.template_02

lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_02/b.yaml

-include: ../_arc_challenge_alt_yaml
-group: arc_challenge_alt_ov_02
-task: arc_challenge_alt_ov_02b
-doc_to_text: !function ../styles.template_02
-doc_to_choice: !function ../styles.choice_02b
-doc_to_decontamination_query: !function ../styles.template_02

lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_02/c.yaml

-include: ../_arc_challenge_alt_yaml
-group: arc_challenge_alt_ov_02
-task: arc_challenge_alt_ov_02c
-doc_to_text: !function ../styles.template_02
-doc_to_choice: !function ../styles.choice_02c
-doc_to_decontamination_query: !function ../styles.template_02

lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_03/a.yaml

-include: ../_arc_challenge_alt_yaml
-group: arc_challenge_alt_ov_03
-task: arc_challenge_alt_ov_03a
-doc_to_text: !function ../styles.template_03
-doc_to_choice: !function ../styles.choice_03a
-doc_to_decontamination_query: !function ../styles.template_03

lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_03/b.yaml

-include: ../_arc_challenge_alt_yaml
-group: arc_challenge_alt_ov_03
-task: arc_challenge_alt_ov_03b
-doc_to_text: !function ../styles.template_03
-doc_to_choice: !function ../styles.choice_03b
-doc_to_decontamination_query: !function ../styles.template_03

lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_03/c.yaml

-include: ../_arc_challenge_alt_yaml
-group: arc_challenge_alt_ov_03
-task: arc_challenge_alt_ov_03c
-doc_to_text: !function ../styles.template_03
-doc_to_choice: !function ../styles.choice_03c
-doc_to_decontamination_query: !function ../styles.template_03

lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_04/a.yaml

-include: ../_arc_challenge_alt_yaml
-group: arc_challenge_alt_ov_04
-task: arc_challenge_alt_ov_04a
-doc_to_text: !function ../styles.template_04
-doc_to_choice: !function ../styles.choice_04a
-doc_to_decontamination_query: !function ../styles.template_04

lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_04/b.yaml

-include: ../_arc_challenge_alt_yaml
-group: arc_challenge_alt_ov_04
-task: arc_challenge_alt_ov_04b
-doc_to_text: !function ../styles.template_04
-doc_to_choice: !function ../styles.choice_04b
-doc_to_decontamination_query: !function ../styles.template_04

lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_04/c.yaml

-include: ../_arc_challenge_alt_yaml
-group: arc_challenge_alt_ov_04
-task: arc_challenge_alt_ov_04c
-doc_to_text: !function ../styles.template_04
-doc_to_choice: !function ../styles.choice_04c
-doc_to_decontamination_query: !function ../styles.template_04

lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_05/a.yaml

-include: ../_arc_challenge_alt_yaml
-group: arc_challenge_alt_ov_05
-task: arc_challenge_alt_ov_05a
-doc_to_text: !function ../styles.template_05
-doc_to_choice: !function ../styles.choice_05a
-doc_to_decontamination_query: !function ../styles.template_05

lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_05/b.yaml

-include: ../_arc_challenge_alt_yaml
-group: arc_challenge_alt_ov_05
-task: arc_challenge_alt_ov_05b
-doc_to_text: !function ../styles.template_05
-doc_to_choice: !function ../styles.choice_05b
-doc_to_decontamination_query: !function ../styles.template_05

lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_05/c.yaml

-include: ../_arc_challenge_alt_yaml
-group: arc_challenge_alt_ov_05
-task: arc_challenge_alt_ov_05c
-doc_to_text: !function ../styles.template_05
-doc_to_choice: !function ../styles.choice_05c
-doc_to_decontamination_query: !function ../styles.template_05

lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_06/a.yaml

-include: ../_arc_challenge_alt_yaml
-group: arc_challenge_alt_ov_06
-task: arc_challenge_alt_ov_06a
-doc_to_text: !function ../styles.template_06
-doc_to_choice: !function ../styles.choice_06a
-doc_to_decontamination_query: !function ../styles.template_06

lm_eval/tasks/arc/alternative_worlds/arc_challenge/output_variation/style_06/b.yaml

-include: ../_arc_challenge_alt_yaml
-group: arc_challenge_alt_ov_06
-task: arc_challenge_alt_ov_06b
-doc_to_text: !function ../styles.template_06
-doc_to_choice: !function ../styles.choice_06b
-doc_to_decontamination_query: !function ../styles.template_06