Merge pull request #651 from farzanehnakhaee70/add-qa4mre-config

[refactor] Add qa4mre config

lm_eval/tasks/qa4mre/preprocess_qa4mre.py

+def qa4mre_process(doc):
+    return int(doc["correct_answer_id"]) - 1
+
+
+def doc_to_target(doc):
+    return doc["answer_options"]["answer_str"][qa4mre_process(doc)]

lm_eval/tasks/qa4mre/qa4mre_2011.yaml

+group:
+  - multiple_choice
+task: qa4mre_2011
+dataset_path: qa4mre
+dataset_name: 2011.main.EN
+output_type: multiple_choice
+test_split: train
+template_aliases: "{% set answer_choices = answer_options['answer_str'] %}"
+doc_to_text: "{{document_str.strip()}}\nQuestion: {{question_str}}\nChoices:\n- {{answer_choices|join('\n- ')}}\nAnswer:"
+doc_to_target: !function preprocess_qa4mre.doc_to_target
+gold_alias: !function preprocess_qa4mre.qa4mre_process
+should_decontaminate: true
+doc_to_decontamination_query: "{{document_str.strip()}} + ' ' + {{question_str}}"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true

lm_eval/tasks/qa4mre/qa4mre_2012.yaml

+group:
+  - multiple_choice
+task: qa4mre_2012
+dataset_path: qa4mre
+dataset_name: 2012.main.EN
+output_type: multiple_choice
+test_split: train
+template_aliases: "{% set answer_choices = answer_options['answer_str'] %}"
+doc_to_text: "{{document_str.strip()}}\nQuestion: {{question_str}}\nChoices:\n- {{answer_choices|join('\n- ')}}\nAnswer:"
+doc_to_target: !function preprocess_qa4mre.doc_to_target
+gold_alias: !function preprocess_qa4mre.qa4mre_process
+should_decontaminate: true
+doc_to_decontamination_query: "{{document_str.strip()}} + ' ' + {{question_str}}"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true

lm_eval/tasks/qa4mre/qa4mre_2013.yaml

+group:
+  - multiple_choice
+task: qa4mre_2013
+dataset_path: qa4mre
+dataset_name: 2013.main.EN
+output_type: multiple_choice
+test_split: train
+template_aliases: "{% set answer_choices = answer_options['answer_str'] %}"
+doc_to_text: "{{document_str.strip()}}\nQuestion: {{question_str}}\nChoices:\n- {{answer_choices|join('\n- ')}}\nAnswer:"
+doc_to_target: !function preprocess_qa4mre.doc_to_target
+gold_alias: !function preprocess_qa4mre.qa4mre_process
+should_decontaminate: true
+doc_to_decontamination_query: "{{document_str.strip()}} + ' ' + {{question_str}}"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true