qa4mre_2012.yaml

group:
  - multiple_choice
task: qa4mre_2012
dataset_path: qa4mre
dataset_name: 2012.main.EN
output_type: multiple_choice
test_split: train
template_aliases: "{% set answer_choices = answer_options['answer_str'] %}"
doc_to_text: "{{document_str.strip()}}\nQuestion: {{question_str}}\nChoices:\n- {{answer_choices|join('\n- ')}}\nAnswer:"
doc_to_target: !function preprocess_qa4mre.doc_to_target
gold_alias: !function preprocess_qa4mre.qa4mre_process
should_decontaminate: true
doc_to_decontamination_query: "{{document_str.strip()}} + ' ' + {{question_str}}"
metric_list:
  - metric: acc
    aggregation: mean
    higher_is_better: true
  - metric: acc_norm
    aggregation: mean
    higher_is_better: true