mutual.yaml

task: mutual
dataset_path: "EleutherAI/mutual"
dataset_name: mutual
output_type: multiple_choice
training_split: train
validation_split: validation
doc_to_text: "{{article}}"
doc_to_target: "{{['A', 'B', 'C', 'D'].index(answers)}}"
doc_to_choice: "{{options}}"
process_docs: !function utils.process_docs
process_results: !function utils.process_results
should_decontaminate: true
doc_to_decontamination_query: "{{article}}"
metric_list:
  - metric: r@1
    aggregation: mean
    higher_is_better: true
  - metric: r@2
    aggregation: mean
    higher_is_better: true
  - metric: mrr
    aggregation: mean
    higher_is_better: true