headqa_en.yaml

group:
  - headqa
task: headqa_en
dataset_path: EleutherAI/headqa
dataset_name: en
output_type: multiple_choice
training_split: train
validation_split: validation
test_split: test
doc_to_text: "Question: {{qtext}}\nAnswer:"
doc_to_target: "{{ra - 1}}"
doc_to_choice: "{{answers|map(attribute='atext')|list}}" # this will be cast to an int.
should_decontaminate: true
doc_to_decontamination_query: query
metric_list:
  - metric: acc
    aggregation: mean
    higher_is_better: true
  - metric: acc_norm
    aggregation: mean
    higher_is_better: true
metadata:
  version: 1.0