default.yaml

group:
  - super-glue-lm-eval-v1
task: winogrande
dataset_path: winogrande
dataset_name: winogrande_xl
output_type: multiple_choice
should_decontaminate: true
doc_to_decontamination_query: "{{sentence}}"
training_split: train
validation_split: validation
metric_list:
  - metric: exact_match
    aggregation: mean
    higher_is_better: true
    ignore_case: true
    ignore_punctuation: true