Merge pull request #1 from EleutherAI/qa4mre

List out answer choices in QA4MRE

lm_eval/tasks/qa4mre/preprocess_qa4mre.py

 def qa4mre_process(doc):
     return int(doc["correct_answer_id"]) - 1
+
+
+def doc_to_target(doc):
+    return doc["answer_options"]["answer_str"][qa4mre_process(doc)]

lm_eval/tasks/qa4mre/qa4mre_2011.yaml

@@ -6,8 +6,9 @@ dataset_name: 2011.main.EN
 output_type: multiple_choice
 test_split: train
 template_aliases: "{% set answer_choices = answer_options['answer_str'] %}"
-doc_to_text: "{{document_str.strip()}}\nQuestion: {{question_str}}\nAnswer:"
-doc_to_target: !function preprocess_qa4mre.qa4mre_process
+doc_to_text: "{{document_str.strip()}}\nQuestion: {{question_str}}\nChoices:\n- {{answer_choices|join('\n- ')}}\nAnswer:"
+doc_to_target: !function preprocess_qa4mre.doc_to_target
+gold_alias: !function preprocess_qa4mre.qa4mre_process
 should_decontaminate: true
 doc_to_decontamination_query: "{{document_str.strip()}} + ' ' + {{question_str}}"
 metric_list:

lm_eval/tasks/qa4mre/qa4mre_2012.yaml

@@ -6,8 +6,9 @@ dataset_name: 2012.main.EN
 output_type: multiple_choice
 test_split: train
 template_aliases: "{% set answer_choices = answer_options['answer_str'] %}"
-doc_to_text: "{{document_str.strip()}}\nQuestion: {{question_str}}\nAnswer:"
-doc_to_target: !function preprocess_qa4mre.qa4mre_process
+doc_to_text: "{{document_str.strip()}}\nQuestion: {{question_str}}\nChoices:\n- {{answer_choices|join('\n- ')}}\nAnswer:"
+doc_to_target: !function preprocess_qa4mre.doc_to_target
+gold_alias: !function preprocess_qa4mre.qa4mre_process
 should_decontaminate: true
 doc_to_decontamination_query: "{{document_str.strip()}} + ' ' + {{question_str}}"
 metric_list:

lm_eval/tasks/qa4mre/qa4mre_2013.yaml

@@ -6,8 +6,9 @@ dataset_name: 2013.main.EN
 output_type: multiple_choice
 test_split: train
 template_aliases: "{% set answer_choices = answer_options['answer_str'] %}"
-doc_to_text: "{{document_str.strip()}}\nQuestion: {{question_str}}\nAnswer:"
-doc_to_target: !function preprocess_qa4mre.qa4mre_process
+doc_to_text: "{{document_str.strip()}}\nQuestion: {{question_str}}\nChoices:\n- {{answer_choices|join('\n- ')}}\nAnswer:"
+doc_to_target: !function preprocess_qa4mre.doc_to_target
+gold_alias: !function preprocess_qa4mre.qa4mre_process
 should_decontaminate: true
 doc_to_decontamination_query: "{{document_str.strip()}} + ' ' + {{question_str}}"
 metric_list: