group:
  - mmlu
  - afrimmlu
  - afrimmlu_direct
task: null
dataset_path: masakhane/afrimmlu
dataset_name: null
output_type: multiple_choice
validation_split: validation
test_split: test
fewshot_split: validation
doc_to_text: !function utils.doc_to_text 
doc_to_target: "{{['A', 'B', 'C', 'D'].index(answer)}}"
doc_to_choice: !function utils.doc_to_choice
should_decontaminate: true
doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
metric_list:
  - metric: f1 
    aggregation: !function utils.weighted_f1_score 
    # aggregation: mean
    average: weighted 
    hf_evaluate: true 
    higher_is_better: True 
    ignore_case: true
    ignore_punctuation: true
    regexes_to_ignore:
      - ","
      - "\\$"
  - metric: acc
    aggregation: mean
    higher_is_better: true
    ignore_case: true
    ignore_punctuation: true
    regexes_to_ignore:
      - ","
      - "\\$"
metadata:
  version: 1.0