merged latest and added altworld files

lm_eval/tasks/arc/alternative_worlds/style_07/b.yaml

+include: ../_arc_easy_alt_yaml
+group: arc_easy_07
+group_alias: style_07 
+task: arc_easy_07b
+task_alias: b
+doc_to_text: !function ../styles.template_07
+doc_to_choice: !function ../styles.choice_07b
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_07/c.yaml

+include: ../_arc_easy_alt_yaml
+group: arc_easy_07
+group_alias: style_07 
+task: arc_easy_07c
+task_alias: c
+doc_to_text: !function ../styles.template_07
+doc_to_choice: !function ../styles.choice_07c
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_08/a.yaml

+include: ../_arc_easy_alt_yaml
+group: arc_easy_08
+group_alias: style_08 
+task: arc_easy_08a
+task_alias: a
+doc_to_text: !function ../styles.template_08
+doc_to_choice: !function ../styles.choice_08a
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_08/b.yaml

+include: ../_arc_easy_alt_yaml
+group: arc_easy_08
+group_alias: style_08 
+task: arc_easy_08b
+task_alias: b
+doc_to_text: !function ../styles.template_08
+doc_to_choice: !function ../styles.choice_08b
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/style_08/c.yaml

+include: ../_arc_easy_alt_yaml
+group: arc_easy_08
+group_alias: style_08 
+task: arc_easy_08c
+task_alias: c
+doc_to_text: !function ../styles.template_08
+doc_to_choice: !function ../styles.choice_08c
\ No newline at end of file

lm_eval/tasks/arc/alternative_worlds/styles.py

+import string
+from functools import partial
+
+def doc_to_text_base(alphabet, style, doc):
+
+    choices = doc["choices"]["text"]
+    num = len(choices)
+
+    letter_list = [style.format(letter) for letter in alphabet[0:num]]
+
+    if "\t" in style:
+        choice_string = "{}{}"
+    else:
+        choice_string = "{} {}"
+
+    doc_to_text = "\n\n".join([
+        "Question: "+doc["question"]+"\nAnswer:",
+        ] + [
+            choice_string.format(i,j) for i,j in zip(letter_list, choices)
+        ]
+    )
+
+    return doc_to_text
+
+# Full continuation
+def choice_A(doc):
+    return doc["choices"]["text"]
+
+# Letters only
+def choice_B(alphabet, style, doc):
+
+    choices = doc["choices"]["text"]
+    num = len(choices)
+
+    letter_list = [style.format(letter) for letter in alphabet[0:num]]
+    if "\t" in style:
+        letter_list = [letter.replace("\t","") for letter in letter_list]
+
+    return letter_list
+
+# Letters + Full continuation
+def choice_C(alphabet, style, doc):
+
+    choices = doc["choices"]["text"]
+    num = len(choices)
+
+    letter_list = [style.format(letter) for letter in alphabet[0:num]]
+    if "\t" not in style:
+        letter_list = [letter+" " for letter in letter_list]
+
+    return [letter+choice for letter, choice in zip(letter_list, choices)]
+
+template_01 = partial(doc_to_text_base, string.ascii_lowercase, "({})")
+choice_01a = choice_A
+choice_01b = partial(choice_B, string.ascii_lowercase, "({})")
+choice_01c = partial(choice_C, string.ascii_lowercase, "({})")
+template_02 = partial(doc_to_text_base, string.ascii_lowercase, "{})")
+choice_02a = choice_A
+choice_02b = partial(choice_B, string.ascii_lowercase, "{})")
+choice_02c = partial(choice_C, string.ascii_lowercase, "{})")
+template_03 = partial(doc_to_text_base, string.ascii_lowercase, "{}.")
+choice_03a = choice_A
+choice_03b = partial(choice_B, string.ascii_lowercase, "{}.")
+choice_03c = partial(choice_C, string.ascii_lowercase, "{}.")
+template_04 = partial(doc_to_text_base, string.ascii_lowercase, "{}\t")
+choice_04a = choice_A
+choice_04b = partial(choice_B, string.ascii_lowercase, "{}\t")
+choice_04c = partial(choice_C, string.ascii_lowercase, "{}\t")
+template_05 = partial(doc_to_text_base, string.ascii_uppercase, "({})")
+choice_05a = choice_A
+choice_05b = partial(choice_B, string.ascii_uppercase, "({})")
+choice_05c = partial(choice_C, string.ascii_uppercase, "({})")
+template_06 = partial(doc_to_text_base, string.ascii_uppercase, "{})")
+choice_06a = choice_A
+choice_06b = partial(choice_B, string.ascii_uppercase, "{})")
+choice_06c = partial(choice_C, string.ascii_uppercase, "{})")
+template_07 = partial(doc_to_text_base, string.ascii_uppercase, "{}.")
+choice_07a = choice_A
+choice_07b = partial(choice_B, string.ascii_uppercase, "{}.")
+choice_07c = partial(choice_C, string.ascii_uppercase, "{}.")
+template_08 = partial(doc_to_text_base, string.ascii_uppercase, "{}\t")
+choice_08a = choice_A
+choice_08b = partial(choice_B, string.ascii_uppercase, "{}\t")
+choice_08c = partial(choice_C, string.ascii_uppercase, "{}\t")
+
+

lm_eval/tasks/arc/arc_easy.yaml

@@ -19,3 +19,5 @@ metric_list:
   - metric: acc_norm
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/arithmetic/alternative_worlds/arithmetic_alt.yaml

 group: arithmetic_alt
 task:
+  - arithmetic_alt_00
   - arithmetic_alt_01
   - arithmetic_alt_02
   - arithmetic_alt_03

lm_eval/tasks/arithmetic/alternative_worlds/style_00/_template_00_yaml

+include: ../_template_yaml
+group: arithmetic_alt_00
+group_alias: arithmetic (Style 00)
+dataset_path: EleutherAI/arithmetic
+output_type: loglikelihood
+validation_split: validation
+test_split: null
+doc_to_text: !function ../utils.style_00
+doc_to_target: "{{completion}}"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: brier_score
+    higher_is_better: false
\ No newline at end of file

lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_1dc.yaml

+include: _template_00_yaml
+task: arithmetic_1dc_alt_00
+dataset_name: arithmetic_1dc
+task_alias: 1dc
\ No newline at end of file

lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_2da.yaml

+include: _template_00_yaml
+task: arithmetic_2da_alt_00
+dataset_name: arithmetic_2da
+task_alias: 2da
\ No newline at end of file

lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_2dm.yaml

+include: _template_00_yaml
+task: arithmetic_2dm_alt_00
+dataset_name: arithmetic_2dm
+task_alias: 2dm
\ No newline at end of file

lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_2ds.yaml

+include: _template_00_yaml
+task: arithmetic_2ds_alt_00
+dataset_name: arithmetic_2ds
+task_alias: 2ds
\ No newline at end of file

lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_3da.yaml

+include: _template_00_yaml
+task: arithmetic_3da_alt_00
+dataset_name: arithmetic_3da
+task_alias: 3da
\ No newline at end of file

lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_3ds.yaml

+include: _template_00_yaml
+task: arithmetic_3ds_alt_00
+dataset_name: arithmetic_3ds
+task_alias: 3ds
\ No newline at end of file

lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_4da.yaml

+include: _template_00_yaml
+task: arithmetic_4da_alt_00
+dataset_name: arithmetic_4da
+task_alias: 4da
\ No newline at end of file

lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_4ds.yaml

+include: _template_00_yaml
+task: arithmetic_4ds_alt_00
+dataset_name: arithmetic_4ds
+task_alias: 4ds
\ No newline at end of file

lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_5da.yaml

+include: _template_00_yaml
+task: arithmetic_5da_alt_00
+dataset_name: arithmetic_5da
+task_alias: 5da
\ No newline at end of file

lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_5ds.yaml

+include: _template_00_yaml
+task: arithmetic_5ds_alt_00
+dataset_name: arithmetic_5ds
+task_alias: 5ds
\ No newline at end of file

lm_eval/tasks/arithmetic/alternative_worlds/utils.py

@@ -2,6 +2,10 @@ import re
 
 # Original Prompt
 # Question: What is (9 + 8) * 2? Answer:
+def style_00(docs):
+
+    # What is (9 + 8) * 2?
+    return docs["context"]
 
 def style_01(docs):