removed unused files for now

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_05/b.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_05
-task: boolq_alt_ov_05b
-doc_to_text: !function ../styles.template_05
-doc_to_choice: !function ../styles.choice_05b
-doc_to_decontamination_query: !function ../styles.template_05

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_05/c.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_05
-task: boolq_alt_ov_05c
-doc_to_text: !function ../styles.template_05
-doc_to_choice: !function ../styles.choice_05c
-doc_to_decontamination_query: !function ../styles.template_05

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_06/a.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_06
-task: boolq_alt_ov_06a
-doc_to_text: !function ../styles.template_06
-doc_to_choice: !function ../styles.choice_06a
-doc_to_decontamination_query: !function ../styles.template_06

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_06/b.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_06
-task: boolq_alt_ov_06b
-doc_to_text: !function ../styles.template_06
-doc_to_choice: !function ../styles.choice_06b
-doc_to_decontamination_query: !function ../styles.template_06

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_06/c.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_06
-task: boolq_alt_ov_06c
-doc_to_text: !function ../styles.template_06
-doc_to_choice: !function ../styles.choice_06c
-doc_to_decontamination_query: !function ../styles.template_06

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_07/a.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_07
-task: boolq_alt_ov_07a
-doc_to_text: !function ../styles.template_07
-doc_to_choice: !function ../styles.choice_07a
-doc_to_decontamination_query: !function ../styles.template_07

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_07/b.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_07
-task: boolq_alt_ov_07b
-doc_to_text: !function ../styles.template_07
-doc_to_choice: !function ../styles.choice_07b
-doc_to_decontamination_query: !function ../styles.template_07

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_07/c.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_07
-task: boolq_alt_ov_07c
-doc_to_text: !function ../styles.template_07
-doc_to_choice: !function ../styles.choice_07c
-doc_to_decontamination_query: !function ../styles.template_07

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_08/a.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_08
-task: boolq_alt_ov_08a
-doc_to_text: !function ../styles.template_08
-doc_to_choice: !function ../styles.choice_08a
-doc_to_decontamination_query: !function ../styles.template_08

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_08/b.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_08
-task: boolq_alt_ov_08b
-doc_to_text: !function ../styles.template_08
-doc_to_choice: !function ../styles.choice_08b
-doc_to_decontamination_query: !function ../styles.template_08

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/style_08/c.yaml

-include: ../_boolq_alt_ov_yaml
-group: boolq_alt_ov_08
-task: boolq_alt_ov_08c
-doc_to_text: !function ../styles.template_08
-doc_to_choice: !function ../styles.choice_08c
-doc_to_decontamination_query: !function ../styles.template_08

lm_eval/tasks/super_glue/boolq/alternative_worlds/output_variation/styles.py

-import string
-from functools import partial
-
-
-def doc_to_text_base(alphabet, style, doc):
-
-    choices = ["no", "yes"]
-    num = len(choices)
-
-    letter_list = [style.format(letter) for letter in alphabet[0:num]]
-
-    if "\t" in style:
-        choice_string = "{}{}"
-    else:
-        choice_string = "{} {}"
-
-    doc_to_text = "\n".join(
-        [doc["passage"]]
-        + [
-            "Question: " + doc["question"],
-        ]
-        + [choice_string.format(i, j) for i, j in zip(letter_list, choices)]
-        + ["Answer:"]
-    )
-
-    return doc_to_text
-
-
-# Full continuation
-def choice_A(doc):
-    return doc["choices"]["text"]
-
-
-# Letters only
-def choice_B(alphabet, style, doc):
-
-    choices = ["no", "yes"]
-    num = len(choices)
-
-    letter_list = [style.format(letter) for letter in alphabet[0:num]]
-    if "\t" in style:
-        letter_list = [letter.replace("\t", "") for letter in letter_list]
-
-    return letter_list
-
-
-# Letters + Full continuation
-def choice_C(alphabet, style, doc):
-
-    choices = ["no", "yes"]
-    num = len(choices)
-
-    letter_list = [style.format(letter) for letter in alphabet[0:num]]
-    if "\t" not in style:
-        letter_list = [letter + " " for letter in letter_list]
-
-    return [letter + choice for letter, choice in zip(letter_list, choices)]
-
-
-template_01 = partial(doc_to_text_base, string.ascii_lowercase, "({})")
-choice_01a = choice_A
-choice_01b = partial(choice_B, string.ascii_lowercase, "({})")
-choice_01c = partial(choice_C, string.ascii_lowercase, "({})")
-template_02 = partial(doc_to_text_base, string.ascii_lowercase, "{})")
-choice_02a = choice_A
-choice_02b = partial(choice_B, string.ascii_lowercase, "{})")
-choice_02c = partial(choice_C, string.ascii_lowercase, "{})")
-template_03 = partial(doc_to_text_base, string.ascii_lowercase, "{}.")
-choice_03a = choice_A
-choice_03b = partial(choice_B, string.ascii_lowercase, "{}.")
-choice_03c = partial(choice_C, string.ascii_lowercase, "{}.")
-template_04 = partial(doc_to_text_base, string.ascii_lowercase, "{}\t")
-choice_04a = choice_A
-choice_04b = partial(choice_B, string.ascii_lowercase, "{}\t")
-choice_04c = partial(choice_C, string.ascii_lowercase, "{}\t")
-template_05 = partial(doc_to_text_base, string.ascii_uppercase, "({})")
-choice_05a = choice_A
-choice_05b = partial(choice_B, string.ascii_uppercase, "({})")
-choice_05c = partial(choice_C, string.ascii_uppercase, "({})")
-template_06 = partial(doc_to_text_base, string.ascii_uppercase, "{})")
-choice_06a = choice_A
-choice_06b = partial(choice_B, string.ascii_uppercase, "{})")
-choice_06c = partial(choice_C, string.ascii_uppercase, "{})")
-template_07 = partial(doc_to_text_base, string.ascii_uppercase, "{}.")
-choice_07a = choice_A
-choice_07b = partial(choice_B, string.ascii_uppercase, "{}.")
-choice_07c = partial(choice_C, string.ascii_uppercase, "{}.")
-template_08 = partial(doc_to_text_base, string.ascii_uppercase, "{}\t")
-choice_08a = choice_A
-choice_08b = partial(choice_B, string.ascii_uppercase, "{}\t")
-choice_08c = partial(choice_C, string.ascii_uppercase, "{}\t")

lm_eval/tasks/super_glue/boolq/alternative_worlds/prompt_variation/_piqa_yaml

-dataset_path: super_glue
-dataset_name: boolq
-output_type: multiple_choice
-training_split: train
-validation_split: validation
-doc_to_text: "{{passage}}\nQuestion: {{question}}?\nAnswer:"
-doc_to_target: label
-doc_to_choice: ["no", "yes"]
-should_decontaminate: true
-doc_to_decontamination_query: passage
-  - metric: acc
-    aggregation: mean
-    higher_is_better: true
-  - metric: acc_norm
-    aggregation: mean
-    higher_is_better: true
-  - metric: brier_score
-    aggregation: brier_score
-    higher_is_better: false
-metadata:
-  - version: 1.0

lm_eval/tasks/super_glue/boolq/alternative_worlds/prompt_variation/style_01.yaml

-include: _boolq_yaml
-group: boolq_alt_pv
-task: boolq_alt_pv_01
-doc_to_text: "{{passage}}\n{{question}}?"
-doc_to_decontamination_query: "{{passage}}\n{{question}}?"

lm_eval/tasks/super_glue/boolq/alternative_worlds/prompt_variation/style_02.yaml

-include: _boolq_yaml
-group: boolq_alt_pv
-task: boolq_alt_pv_02
-doc_to_text: "{{passage}}\nQ: {{question}}?\nA:"
-doc_to_decontamination_query: "{{passage}}\nQ: {{question}}?\nA:"

lm_eval/tasks/super_glue/boolq/alternative_worlds/prompt_variation/style_03.yaml

-include: _boolq_yaml
-group: boolq_alt_pv
-task: boolq_alt_pv_03
-doc_to_text: "{{passage}}\nQuestion: {{question}}?\nAnswer:"
-doc_to_decontamination_query: "{{passage}}\nQuestion: {{question}}?\nAnswer:"

lm_eval/tasks/truthfulqa/alternative_worlds/prompt_variation/style_01/truthfulqa_mc1.yaml

-task: truthfulqa_mc1_alt_pv_01
-dataset_path: truthful_qa
-dataset_name: multiple_choice
-output_type: multiple_choice
-training_split: null
-validation_split: validation
-test_split: null
-num_fewshot: 0
-doc_to_text: "{{question}}"
-doc_to_target: 0
-doc_to_choice: "{{mc1_targets.choices}}"
-should_decontaminate: True
-doc_to_decontamination_query: question
-metric_list:
-  - metric: acc
-    aggregation: mean
-    higher_is_better: true

lm_eval/tasks/truthfulqa/alternative_worlds/prompt_variation/style_01/truthfulqa_mc2.yaml

-include: truthfulqa_mc1.yaml
-task: truthfulqa_mc2_alt_pv_01
-doc_to_target: 0
-doc_to_choice: "{{mc2_targets.choices}}"
-process_results: !function ../../../utils.process_results_mc2
-should_decontaminate: True
-doc_to_decontamination_query: question
-metric_list:
-  - metric: acc
-    aggregation: mean
-    higher_is_better: true

lm_eval/tasks/truthfulqa/alternative_worlds/prompt_variation/style_02/truthfulqa_mc1.yaml

-task: truthfulqa_mc1_alt_pv_02
-dataset_path: truthful_qa
-dataset_name: multiple_choice
-output_type: multiple_choice
-training_split: null
-validation_split: validation
-test_split: null
-num_fewshot: 0
-doc_to_choice: "Q: {{mc2_targets.choices}}\nA:"
-doc_to_target: 0
-doc_to_choice: "{{mc1_targets.choices}}"
-should_decontaminate: True
-doc_to_decontamination_query: question
-metric_list:
-  - metric: acc
-    aggregation: mean
-    higher_is_better: true

lm_eval/tasks/truthfulqa/alternative_worlds/prompt_variation/style_02/truthfulqa_mc2.yaml

-include: truthfulqa_mc1.yaml
-task: truthfulqa_mc2_alt_pv_02
-doc_to_target: 0
-doc_to_choice: "{{mc2_targets.choices}}"
-process_results: !function ../../../utils.process_results_mc2
-should_decontaminate: True
-doc_to_decontamination_query: question
-metric_list:
-  - metric: acc
-    aggregation: mean
-    higher_is_better: true