made prompt and output variations

lm_eval/tasks/arc/alternative_worlds/style_06/c.yaml → lm_eval/tasks/arc/alternative_worlds/output_variation/style_06/c.yaml

 include: ../_arc_easy_alt_yaml
-group: arc_easy_06
-group_alias: style_06
-task: arc_easy_06c
-task_alias: c
+group: arc_easy_alt_ov_06
+task: arc_easy_alt_ov_06c
 doc_to_text: !function ../styles.template_06
 doc_to_choice: !function ../styles.choice_06c
+doc_to_decontamination_query: !function ../styles.template_06
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_07/a.yaml → lm_eval/tasks/arc/alternative_worlds/output_variation/style_07/a.yaml

 include: ../_arc_easy_alt_yaml
-group: arc_easy_07
-group_alias: style_07
-task: arc_easy_07a
-task_alias: a
+group: arc_easy_alt_ov_07
+task: arc_easy_alt_ov_07a
 doc_to_text: !function ../styles.template_07
 doc_to_choice: !function ../styles.choice_07a
+doc_to_decontamination_query: !function ../styles.template_07
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_07/b.yaml → lm_eval/tasks/arc/alternative_worlds/output_variation/style_07/b.yaml

 include: ../_arc_easy_alt_yaml
-group: arc_easy_07
-group_alias: style_07
-task: arc_easy_07b
-task_alias: b
+group: arc_easy_alt_ov_07
+task: arc_easy_alt_ov_07b
 doc_to_text: !function ../styles.template_07
 doc_to_choice: !function ../styles.choice_07b
+doc_to_decontamination_query: !function ../styles.template_07
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_07/c.yaml → lm_eval/tasks/arc/alternative_worlds/output_variation/style_07/c.yaml

 include: ../_arc_easy_alt_yaml
-group: arc_easy_07
-group_alias: style_07
-task: arc_easy_07c
-task_alias: c
+group: arc_easy_alt_ov_07
+task: arc_easy_alt_ov_07c
 doc_to_text: !function ../styles.template_07
 doc_to_choice: !function ../styles.choice_07c
+doc_to_decontamination_query: !function ../styles.template_07
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_08/a.yaml → lm_eval/tasks/arc/alternative_worlds/output_variation/style_08/a.yaml

 include: ../_arc_easy_alt_yaml
-group: arc_easy_08
-group_alias: style_08
-task: arc_easy_08a
-task_alias: a
+group: arc_easy_alt_ov_08
+task: arc_easy_alt_ov_08a
 doc_to_text: !function ../styles.template_08
 doc_to_choice: !function ../styles.choice_08a
+doc_to_decontamination_query: !function ../styles.template_08
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_08/b.yaml → lm_eval/tasks/arc/alternative_worlds/output_variation/style_08/b.yaml

 include: ../_arc_easy_alt_yaml
-group: arc_easy_08
-group_alias: style_08
-task: arc_easy_08b
-task_alias: b
+group: arc_easy_alt_ov_08
+task: arc_easy_alt_ov_08b
 doc_to_text: !function ../styles.template_08
 doc_to_choice: !function ../styles.choice_08b
+doc_to_decontamination_query: !function ../styles.template_08
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_08/c.yaml → lm_eval/tasks/arc/alternative_worlds/output_variation/style_08/c.yaml

 include: ../_arc_easy_alt_yaml
-group: arc_easy_08
-group_alias: style_08
-task: arc_easy_08c
-task_alias: c
+group: arc_easy_alt_ov_08
+task: arc_easy_alt_ov_08c
 doc_to_text: !function ../styles.template_08
 doc_to_choice: !function ../styles.choice_08c
+doc_to_decontamination_query: !function ../styles.template_08
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/styles.py → lm_eval/tasks/arc/alternative_worlds/output_variation/styles.py

@@ -14,11 +14,12 @@ def doc_to_text_base(alphabet, style, doc):
     else:
         choice_string = "{} {}"
 
-    doc_to_text = "\n\n".join(
+    doc_to_text = "\n".join(
         [
-            "Question: " + doc["question"] + "\nAnswer:",
+            "Question: " + doc["question"],
         ]
         + [choice_string.format(i, j) for i, j in zip(letter_list, choices)]
+        + ["Answer:"]
     )
 
     return doc_to_text

lm_eval/tasks/arc/alternative_worlds/prompt_variation/_arc_easy_alt_yaml

+group:
+  - ai2_arc
+task: arc_easy
+dataset_path: ai2_arc
+dataset_name: ARC-Easy
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+test_split: test
+doc_to_text: "Question: {{question}}\nAnswer:"
+doc_to_target: "{{choices.label.index(answerKey)}}"
+doc_to_choice: "{{choices.text}}"
+should_decontaminate: true
+doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+  - metric: brier_score
+    aggregation: brier_score
+    higher_is_better: false

lm_eval/tasks/arc/alternative_worlds/prompt_variation/style_01.yaml

+include: _arc_easy_alt_yaml
+group: arc_easy_alt_pv
+task: arc_easy_alt_pv_01
+doc_to_text: "{{question}}"
+doc_to_decontamination_query: "{{question}}"

lm_eval/tasks/arc/alternative_worlds/prompt_variation/style_02.yaml

+include: _arc_easy_alt_yaml
+group: arc_easy_alt_pv
+task: arc_easy_alt_pv_02
+doc_to_text: "Q: {{question}}\nA:"
+doc_to_decontamination_query: "Q: {{question}}\nA:"
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/prompt_variation/style_03.yaml

+include: _arc_easy_alt_yaml
+group: arc_easy_alt_pv
+task: arc_easy_alt_pv_03
+doc_to_text: "Question: {{question}}\nAnswer:"
+doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
\ No newline at end of file