add piqa

lm_eval/tasks/piqa/alternative_worlds/output_variation/style_06/c.yaml

+include: ../_piqa_alt_ov_yaml
+group: piqa_alt_ov_06
+task: piqa_alt_ov_06c
+doc_to_text: !function ../styles.template_06
+doc_to_choice: !function ../styles.choice_06c
+doc_to_decontamination_query: !function ../styles.template_06

lm_eval/tasks/piqa/alternative_worlds/output_variation/style_07/a.yaml

+include: ../_piqa_alt_ov_yaml
+group: piqa_alt_ov_07
+task: piqa_alt_ov_07a
+doc_to_text: !function ../styles.template_07
+doc_to_choice: !function ../styles.choice_07a
+doc_to_decontamination_query: !function ../styles.template_07

lm_eval/tasks/piqa/alternative_worlds/output_variation/style_07/b.yaml

+include: ../_piqa_alt_ov_yaml
+group: piqa_alt_ov_07
+task: piqa_alt_ov_07b
+doc_to_text: !function ../styles.template_07
+doc_to_choice: !function ../styles.choice_07b
+doc_to_decontamination_query: !function ../styles.template_07

lm_eval/tasks/piqa/alternative_worlds/output_variation/style_07/c.yaml

+include: ../_piqa_alt_ov_yaml
+group: piqa_alt_ov_07
+task: piqa_alt_ov_07c
+doc_to_text: !function ../styles.template_07
+doc_to_choice: !function ../styles.choice_07c
+doc_to_decontamination_query: !function ../styles.template_07

lm_eval/tasks/piqa/alternative_worlds/output_variation/style_08/a.yaml

+include: ../_piqa_alt_ov_yaml
+group: piqa_alt_ov_08
+task: piqa_alt_ov_08a
+doc_to_text: !function ../styles.template_08
+doc_to_choice: !function ../styles.choice_08a
+doc_to_decontamination_query: !function ../styles.template_08

lm_eval/tasks/piqa/alternative_worlds/output_variation/style_08/b.yaml

+include: ../_piqa_alt_ov_yaml
+group: piqa_alt_ov_08
+task: piqa_alt_ov_08b
+doc_to_text: !function ../styles.template_08
+doc_to_choice: !function ../styles.choice_08b
+doc_to_decontamination_query: !function ../styles.template_08

lm_eval/tasks/piqa/alternative_worlds/output_variation/style_08/c.yaml

+include: ../_piqa_alt_ov_yaml
+group: piqa_alt_ov_08
+task: piqa_alt_ov_08c
+doc_to_text: !function ../styles.template_08
+doc_to_choice: !function ../styles.choice_08c
+doc_to_decontamination_query: !function ../styles.template_08

lm_eval/tasks/piqa/alternative_worlds/output_variation/styles.py

+import string
+from functools import partial
+
+
+def doc_to_text_base(alphabet, style, doc):
+
+    choices = [doc["sol1"], doc["sol2"]]
+    num = len(choices)
+
+    letter_list = [style.format(letter) for letter in alphabet[0:num]]
+
+    if "\t" in style:
+        choice_string = "{}{}"
+    else:
+        choice_string = "{} {}"
+
+    doc_to_text = "\n".join(
+        [
+            "Question: " + doc["goal"],
+        ]
+        + [choice_string.format(i, j) for i, j in zip(letter_list, choices)]
+        + ["Answer:"]
+    )
+
+    return doc_to_text
+
+
+# Full continuation
+def choice_A(doc):
+    return doc["choices"]["text"]
+
+
+# Letters only
+def choice_B(alphabet, style, doc):
+
+    choices = [doc["sol1"], doc["sol2"]]
+    num = len(choices)
+
+    letter_list = [style.format(letter) for letter in alphabet[0:num]]
+    if "\t" in style:
+        letter_list = [letter.replace("\t", "") for letter in letter_list]
+
+    return letter_list
+
+
+# Letters + Full continuation
+def choice_C(alphabet, style, doc):
+
+    choices = [doc["sol1"], doc["sol2"]]
+    num = len(choices)
+
+    letter_list = [style.format(letter) for letter in alphabet[0:num]]
+    if "\t" not in style:
+        letter_list = [letter + " " for letter in letter_list]
+
+    return [letter + choice for letter, choice in zip(letter_list, choices)]
+
+
+template_01 = partial(doc_to_text_base, string.ascii_lowercase, "({})")
+choice_01a = choice_A
+choice_01b = partial(choice_B, string.ascii_lowercase, "({})")
+choice_01c = partial(choice_C, string.ascii_lowercase, "({})")
+template_02 = partial(doc_to_text_base, string.ascii_lowercase, "{})")
+choice_02a = choice_A
+choice_02b = partial(choice_B, string.ascii_lowercase, "{})")
+choice_02c = partial(choice_C, string.ascii_lowercase, "{})")
+template_03 = partial(doc_to_text_base, string.ascii_lowercase, "{}.")
+choice_03a = choice_A
+choice_03b = partial(choice_B, string.ascii_lowercase, "{}.")
+choice_03c = partial(choice_C, string.ascii_lowercase, "{}.")
+template_04 = partial(doc_to_text_base, string.ascii_lowercase, "{}\t")
+choice_04a = choice_A
+choice_04b = partial(choice_B, string.ascii_lowercase, "{}\t")
+choice_04c = partial(choice_C, string.ascii_lowercase, "{}\t")
+template_05 = partial(doc_to_text_base, string.ascii_uppercase, "({})")
+choice_05a = choice_A
+choice_05b = partial(choice_B, string.ascii_uppercase, "({})")
+choice_05c = partial(choice_C, string.ascii_uppercase, "({})")
+template_06 = partial(doc_to_text_base, string.ascii_uppercase, "{})")
+choice_06a = choice_A
+choice_06b = partial(choice_B, string.ascii_uppercase, "{})")
+choice_06c = partial(choice_C, string.ascii_uppercase, "{})")
+template_07 = partial(doc_to_text_base, string.ascii_uppercase, "{}.")
+choice_07a = choice_A
+choice_07b = partial(choice_B, string.ascii_uppercase, "{}.")
+choice_07c = partial(choice_C, string.ascii_uppercase, "{}.")
+template_08 = partial(doc_to_text_base, string.ascii_uppercase, "{}\t")
+choice_08a = choice_A
+choice_08b = partial(choice_B, string.ascii_uppercase, "{}\t")
+choice_08c = partial(choice_C, string.ascii_uppercase, "{}\t")

lm_eval/tasks/piqa/alternative_worlds/prompt_variation/_piqa_yaml

+# dataset_path: ai2_arc
+# dataset_name: ARC-Easy
+# output_type: multiple_choice
+# training_split: train
+# validation_split: validation
+# test_split: test
+# doc_to_text: "Question: {{question}}\nAnswer:"
+# doc_to_target: "{{choices.label.index(answerKey)}}"
+# doc_to_choice: "{{choices.text}}"
+# should_decontaminate: true
+# doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
+# metric_list:
+#   - metric: acc
+#     aggregation: mean
+#     higher_is_better: true
+#   - metric: acc_norm
+#     aggregation: mean
+#     higher_is_better: true
+#   - metric: brier_score
+#     aggregation: brier_score
+#     higher_is_better: false
+
+
+
+dataset_path: piqa
+dataset_name: null
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+test_split: null
+doc_to_text: "Question: {{goal}}\nAnswer:"
+doc_to_target: label
+doc_to_choice: "{{[sol1, sol2]}}"
+should_decontaminate: true
+doc_to_decontamination_query: goal
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+  - metric: brier_score
+    aggregation: brier_score
+    higher_is_better: false
+metadata:
+  - version: 1.0

lm_eval/tasks/piqa/alternative_worlds/prompt_variation/style_01.yaml

+include: _piqa_yaml
+group: piqa_alt_pv
+task: piqa_alt_pv_01
+doc_to_text: "{{goal}}"
+doc_to_decontamination_query: "{{goal}}"

lm_eval/tasks/piqa/alternative_worlds/prompt_variation/style_02.yaml

+include: _piqa_yaml
+group: piqa_alt_pv
+task: piqa_alt_pv_02
+doc_to_text: "Q: {{goal}}\nA:"
+doc_to_decontamination_query: "Q: {{goal}}\nA:"

lm_eval/tasks/piqa/alternative_worlds/prompt_variation/style_03.yaml

+include: _piqa_yaml
+group: piqa_alt_pv
+task: piqa_alt_pv_03
+doc_to_text: "Question: {{goal}}\nAnswer:"
+doc_to_decontamination_query: "Question: {{goal}}\nAnswer:"