Merge branch 'big-refactor' into headqa

lm_eval/tasks/pubmedqa/preprocess_pubmedqa.py

+def doc_to_text(doc):
+    ctxs = "\n".join(doc["context"]["contexts"])
+    return "Abstract: {}\nQuestion: {}\nAnswer:".format(
+        ctxs, doc["question"], doc["final_decision"]
+    )
+
+def doc_to_target(doc):
+    return " {}".format(doc["final_decision"])
+
+def gold_alias(doc):
+    dict_to_label = {
+        'yes': 0,
+        'no': 1,
+        'maybe': 2
+    }
+    return dict_to_label[doc["final_decision"]]
\ No newline at end of file

lm_eval/tasks/pubmedqa/pubmedqa.yaml

+group:
+  - multiple_choice
+task: pubmed_qa
+dataset_path: pubmed_qa
+dataset_name: pqa_labeled
+output_type: multiple_choice
+training_split: null
+validation_split: null
+test_split: train
+template_aliases: "{% set answer_choices = ['yes', 'no', 'maybe'] %}{% set gold = final_decision %}"
+doc_to_text: !function preprocess_pubmedqa.doc_to_text
+doc_to_target: !function preprocess_pubmedqa.doc_to_target
+gold_alias: !function preprocess_pubmedqa.gold_alias
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
\ No newline at end of file

lm_eval/tasks/qa4mre/preprocess_qa4mre.py

+def qa4mre_process(doc):
+    return int(doc["correct_answer_id"]) - 1
+
+
+def doc_to_target(doc):
+    return doc["answer_options"]["answer_str"][qa4mre_process(doc)]

lm_eval/tasks/qa4mre/qa4mre_2011.yaml

+group:
+  - multiple_choice
+task: qa4mre_2011
+dataset_path: qa4mre
+dataset_name: 2011.main.EN
+output_type: multiple_choice
+test_split: train
+template_aliases: "{% set answer_choices = answer_options['answer_str'] %}"
+doc_to_text: "{{document_str.strip()}}\nQuestion: {{question_str}}\nChoices:\n- {{answer_choices|join('\n- ')}}\nAnswer:"
+doc_to_target: !function preprocess_qa4mre.doc_to_target
+gold_alias: !function preprocess_qa4mre.qa4mre_process
+should_decontaminate: true
+doc_to_decontamination_query: "{{document_str.strip()}} + ' ' + {{question_str}}"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true

lm_eval/tasks/qa4mre/qa4mre_2012.yaml

+group:
+  - multiple_choice
+task: qa4mre_2012
+dataset_path: qa4mre
+dataset_name: 2012.main.EN
+output_type: multiple_choice
+test_split: train
+template_aliases: "{% set answer_choices = answer_options['answer_str'] %}"
+doc_to_text: "{{document_str.strip()}}\nQuestion: {{question_str}}\nChoices:\n- {{answer_choices|join('\n- ')}}\nAnswer:"
+doc_to_target: !function preprocess_qa4mre.doc_to_target
+gold_alias: !function preprocess_qa4mre.qa4mre_process
+should_decontaminate: true
+doc_to_decontamination_query: "{{document_str.strip()}} + ' ' + {{question_str}}"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true

lm_eval/tasks/qa4mre/qa4mre_2013.yaml

+group:
+  - multiple_choice
+task: qa4mre_2013
+dataset_path: qa4mre
+dataset_name: 2013.main.EN
+output_type: multiple_choice
+test_split: train
+template_aliases: "{% set answer_choices = answer_options['answer_str'] %}"
+doc_to_text: "{{document_str.strip()}}\nQuestion: {{question_str}}\nChoices:\n- {{answer_choices|join('\n- ')}}\nAnswer:"
+doc_to_target: !function preprocess_qa4mre.doc_to_target
+gold_alias: !function preprocess_qa4mre.qa4mre_process
+should_decontaminate: true
+doc_to_decontamination_query: "{{document_str.strip()}} + ' ' + {{question_str}}"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true

lm_eval/tasks/race/preprocess_race.py

+import ast 
+
+def process_ast(string):
+    return ast.literal_eval(string)
+
+def last_problem(doc):
+    return process_ast(doc["problems"])[-1]
+
+def get_answer_option(problem):
+    letter_to_num = {"A": 0, "B": 1, "C": 2, "D": 3}
+    answer = letter_to_num[problem["answer"]]
+    return problem["options"][answer]
+
+def create_choices(doc):
+    problem = last_problem(doc)
+    choices = [problem["options"][i] for i in range(4)]
+    return choices
+
+def doc_to_text(doc):
+    text = "Article: " + doc["article"] + "\n\n"
+    for problem in process_ast(doc["problems"])[:-1]:
+        if problem["question"][-6:] == "  _  .":
+            text += (
+                problem["question"][-5:] + get_answer_option(problem) + "\n"
+            )
+        else:
+            question = "Question: " + problem["question"] + "\n"
+            answer = "Answer: " + get_answer_option(problem) + "\n"
+            text += question + answer
+    text += last_problem(doc)["question"]
+    return text
+
+def doc_to_target(doc):
+    letter_to_num = {"A": 0, "B": 1, "C": 2, "D": 3}
+    answer = letter_to_num[last_problem(doc)["answer"]]
+    return answer

lm_eval/tasks/race/race.yaml

+group:
+  - multiple_choice
+task: race
+dataset_path: EleutherAI/race
+dataset_name: high
+output_type: multiple_choice
+test_split: test
+create_choices: !function preprocess_race.create_choices
+doc_to_text: !function preprocess_race.doc_to_text
+doc_to_target: !function preprocess_race.doc_to_target
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true

lm_eval/tasks/super_glue/copa/default.yaml

 group:
-  - super-glue-lm-eval-v1-
+  - super-glue-lm-eval-v1
 task: "copa"
 dataset_path: super_glue
 dataset_name: copa

lm_eval/tasks/swag/swag.yaml

+group:
+  - multiple_choice
+task: swag
+dataset_path: swag
+dataset_name: regular
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+test_split: null
+template_aliases: "{% set answer_choices = [ending0, ending1, ending2, ending3] %}{% set gold = label %}"
+doc_to_text: "{{startphrase}}"
+doc_to_target: "{{answer_choices[gold]}}"
+gold_alias: "{{gold}}"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
\ No newline at end of file

lm_eval/tasks/winogrande/preprocess_winogrande.py

+def partial_context(doc, option):
+    # Substitute the pronoun in the sentence with the specified option
+    # and ignore everything after.
+    pronoun_loc = doc["sentence"].index("_")
+    return doc["sentence"][:pronoun_loc] + option
+
+def partial_target(doc):
+    # The target is everything after the document specified pronoun.
+    pronoun_loc = doc["sentence"].index("_") + 1
+    return doc["sentence"][pronoun_loc:].strip()
+
+def create_choices(doc):
+    choices = []
+    for option in [doc["option1"], doc["option2"]]:
+        partial_ctx = partial_context(doc, option)
+        choices.append(partial_ctx)
+    return choices
+
+def gold_alias(doc):
+    answer_to_num = {"1": 0, "2": 1}
+    return answer_to_num[doc['answer']]
\ No newline at end of file

lm_eval/tasks/winogrande/winogrande.yaml

+task: winogrande
+dataset_path: winogrande
+dataset_name: winogrande_xl
+output_type: winograd_schema
+training_split: train
+validation_split: validation
+doc_to_target: !function preprocess_winogrande.partial_target
+doc_to_text: "{{sentence}}"
+create_choices: !function preprocess_winogrande.create_choices
+gold_alias: !function preprocess_winogrande.gold_alias
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true