resolved conflict

lm_eval/tasks/hellaswag/alternative_worlds/style_08/c.yaml

+include: ../_hellaswag_alt_yaml
+group: hellaswag_08
+group_alias: style_08
+task: hellaswag_08c
+task_alias: c
+doc_to_text: !function ../styles.template_08
+doc_to_choice: !function ../styles.choice_08c

lm_eval/tasks/hellaswag/alternative_worlds/styles.py

+import string
+from functools import partial
+
+
+def doc_to_text_base(alphabet, style, doc):
+
+    choices = doc["choices"]
+    num = len(choices)
+
+    letter_list = [style.format(letter) for letter in alphabet[0:num]]
+
+    if "\t" in style:
+        choice_string = "{}{}"
+    else:
+        choice_string = "{} {}"
+
+    doc_to_text = "\n\n".join(
+        [
+            doc["query"] + "...",
+            " What is the most appropriate continuation?",
+        ]
+        + [choice_string.format(i, j) for i, j in zip(letter_list, choices)]
+    )
+
+    return doc_to_text
+
+
+# Full continuation
+def choice_A(doc):
+    return doc["choices"]
+
+
+# Letters only
+def choice_B(alphabet, style, doc):
+
+    choices = doc["choices"]
+    num = len(choices)
+
+    letter_list = [style.format(letter) for letter in alphabet[0:num]]
+    if "\t" in style:
+        letter_list = [letter.replace("\t", "") for letter in letter_list]
+
+    return letter_list
+
+
+# Letters + Full continuation
+def choice_C(alphabet, style, doc):
+
+    choices = doc["choices"]
+    num = len(choices)
+
+    letter_list = [style.format(letter) for letter in alphabet[0:num]]
+    if "\t" not in style:
+        letter_list = [letter + " " for letter in letter_list]
+
+    return [letter + choice for letter, choice in zip(letter_list, doc["choices"])]
+
+
+template_01 = partial(doc_to_text_base, string.ascii_lowercase, "({})")
+choice_01a = choice_A
+choice_01b = partial(choice_B, string.ascii_lowercase, "({})")
+choice_01c = partial(choice_C, string.ascii_lowercase, "({})")
+template_02 = partial(doc_to_text_base, string.ascii_lowercase, "{})")
+choice_02a = choice_A
+choice_02b = partial(choice_B, string.ascii_lowercase, "{})")
+choice_02c = partial(choice_C, string.ascii_lowercase, "{})")
+template_03 = partial(doc_to_text_base, string.ascii_lowercase, "{}.")
+choice_03a = choice_A
+choice_03b = partial(choice_B, string.ascii_lowercase, "{}.")
+choice_03c = partial(choice_C, string.ascii_lowercase, "{}.")
+template_04 = partial(doc_to_text_base, string.ascii_lowercase, "{}\t")
+choice_04a = choice_A
+choice_04b = partial(choice_B, string.ascii_lowercase, "{}\t")
+choice_04c = partial(choice_C, string.ascii_lowercase, "{}\t")
+template_05 = partial(doc_to_text_base, string.ascii_uppercase, "({})")
+choice_05a = choice_A
+choice_05b = partial(choice_B, string.ascii_uppercase, "({})")
+choice_05c = partial(choice_C, string.ascii_uppercase, "({})")
+template_06 = partial(doc_to_text_base, string.ascii_uppercase, "{})")
+choice_06a = choice_A
+choice_06b = partial(choice_B, string.ascii_uppercase, "{})")
+choice_06c = partial(choice_C, string.ascii_uppercase, "{})")
+template_07 = partial(doc_to_text_base, string.ascii_uppercase, "{}.")
+choice_07a = choice_A
+choice_07b = partial(choice_B, string.ascii_uppercase, "{}.")
+choice_07c = partial(choice_C, string.ascii_uppercase, "{}.")
+template_08 = partial(doc_to_text_base, string.ascii_uppercase, "{}\t")
+choice_08a = choice_A
+choice_08b = partial(choice_B, string.ascii_uppercase, "{}\t")
+choice_08c = partial(choice_C, string.ascii_uppercase, "{}\t")

lm_eval/tasks/hellaswag/hellaswag.yaml

 group:
   - multiple_choice
 task: hellaswag
-dataset_path: hellaswag
+dataset_path: Rowan/hellaswag
 dataset_name: null
 output_type: multiple_choice
 training_split: train

lm_eval/tasks/mathqa/alternative_worlds/output_variation/README.md

+
+Investigate affect of letter options
+- (A)
+- A)
+- A.
+- A\t
+- (a)
+- a)
+- a.
+- a\t
+Answer types:
+- letters only
+    - original option
+    - just letter
+- letters + continuation
+    - original option
+    - just letter
+- continuation

lm_eval/tasks/mathqa/alternative_worlds/output_variation/_mathqa_alt_yaml

+dataset_path: math_qa
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+test_split: test
+doc_to_target: "{{['a', 'b', 'c', 'd', 'e'].index(correct)}}"
+metric_list:
+  - metric: acc
+  - metric: acc_norm
+  - metric: brier_score

lm_eval/tasks/mathqa/alternative_worlds/output_variation/mathqa_alt.yaml

+group: mathqa_alt_ov
+task:
+  - mathqa_alt_ov_01
+  - mathqa_alt_ov_02
+  - mathqa_alt_ov_03
+  - mathqa_alt_ov_04
+  - mathqa_alt_ov_05
+  - mathqa_alt_ov_06
+  - mathqa_alt_ov_07
+  - mathqa_alt_ov_08

lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_01/a.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_alt_ov_01
+task: mathqa_alt_ov_01a
+doc_to_text: !function ../styles.template_01
+doc_to_choice: !function ../styles.choice_01a

lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_01/b.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_alt_ov_01
+task: mathqa_alt_ov_01b
+doc_to_text: !function ../styles.template_01
+doc_to_choice: !function ../styles.choice_01b

lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_01/c.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_alt_ov_01
+task: mathqa_alt_ov_01c
+doc_to_text: !function ../styles.template_01
+doc_to_choice: !function ../styles.choice_01c

lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_02/a.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_alt_ov_02
+task: mathqa_alt_ov_02a
+doc_to_text: !function ../styles.template_02
+doc_to_choice: !function ../styles.choice_02a

lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_02/b.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_alt_ov_02
+task: mathqa_alt_ov_02b
+doc_to_text: !function ../styles.template_02
+doc_to_choice: !function ../styles.choice_02b

lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_02/c.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_alt_ov_02
+task: mathqa_alt_ov_02c
+doc_to_text: !function ../styles.template_02
+doc_to_choice: !function ../styles.choice_02c

lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_03/a.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_alt_ov_03
+task: mathqa_alt_ov_03a
+doc_to_text: !function ../styles.template_03
+doc_to_choice: !function ../styles.choice_03a

lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_03/b.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_alt_ov_03
+task: mathqa_alt_ov_03b
+doc_to_text: !function ../styles.template_03
+doc_to_choice: !function ../styles.choice_03b

lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_03/c.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_alt_ov_03
+task: mathqa_alt_ov_03c
+doc_to_text: !function ../styles.template_03
+doc_to_choice: !function ../styles.choice_03c

lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_04/a.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_alt_ov_04
+task: mathqa_alt_ov_04a
+doc_to_text: !function ../styles.template_04
+doc_to_choice: !function ../styles.choice_04a

lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_04/b.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_alt_ov_04
+task: mathqa_alt_ov_04b
+doc_to_text: !function ../styles.template_04
+doc_to_choice: !function ../styles.choice_04b

lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_04/c.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_alt_ov_04
+task: mathqa_alt_ov_04c
+doc_to_text: !function ../styles.template_04
+doc_to_choice: !function ../styles.choice_04c

lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_05/a.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_alt_ov_05
+task: mathqa_alt_ov_05a
+doc_to_text: !function ../styles.template_05
+doc_to_choice: !function ../styles.choice_05a

lm_eval/tasks/mathqa/alternative_worlds/output_variation/style_05/b.yaml

+include: ../_mathqa_alt_yaml
+group: mathqa_alt_ov_05
+task: mathqa_alt_ov_05b
+doc_to_text: !function ../styles.template_05
+doc_to_choice: !function ../styles.choice_05b