group: t0_eval
task:
  # Coreference Resolution
  - dataset_path: super_glue
    dataset_name: wsc.fixed
    use_prompt: promptsource:*
    training_split: train
    validation_split: validation
    metric_list:
      - metric: exact_match
        aggregation: mean
        higher_is_better: true
        ignore_case: true
        ignore_punctuation: true
  # Coreference Resolution
  - dataset_path: winogrande
    dataset_name: winogrande_xl
    use_prompt: promptsource:*
    training_split: train
    validation_split: validation
    metric_list:
      - metric: exact_match
        aggregation: mean
        higher_is_better: true
        ignore_case: true
        ignore_punctuation: true
  # Natural Language Inference
  - dataset_path: super_glue
    dataset_name: cb
    use_prompt: promptsource:*
    training_split: train
    validation_split: validation
    output_type: greedy_until
    metric_list:
      - metric: exact_match
        aggregation: mean
        higher_is_better: true
        ignore_case: true
        ignore_punctuation: true
  - dataset_path: super_glue
    dataset_name: rte
    use_prompt: promptsource:*
    training_split: train
    validation_split: validation
    metric_list:
      - metric: exact_match
        aggregation: mean
        higher_is_better: true
        ignore_case: true
        ignore_punctuation: true
  - task: anli_r1
    dataset_path: anli
    use_prompt: promptsource:*
    training_split: train_r1
    validation_split: dev_r1
    metric_list:
      - metric: exact_match
        aggregation: mean
        higher_is_better: true
        ignore_case: true
        ignore_punctuation: true
  - task: anli_r2
    dataset_path: anli
    use_prompt: promptsource:*
    training_split: train_r2
    validation_split: dev_r2
    metric_list:
      - metric: exact_match
        aggregation: mean
        higher_is_better: true
        ignore_case: true
        ignore_punctuation: true
  - task: anli_r3
    dataset_path: anli
    use_prompt: promptsource:*
    training_split: train_r3
    validation_split: dev_r3
    metric_list:
      - metric: exact_match
        aggregation: mean
        higher_is_better: true
        ignore_case: true
        ignore_punctuation: true
  # Sentence Completion
  - dataset_path: super_glue
    dataset_name: copa
    use_prompt: promptsource:*
    training_split: train
    validation_split: validation
    metric_list:
      - metric: exact_match
        aggregation: mean
        higher_is_better: true
        ignore_case: true
        ignore_punctuation: true
  # Natural Language Inference
  - dataset_path: hellaswag
    use_prompt: promptsource:*
    training_split: train
    validation_split: validation
    metric_list:
      - metric: exact_match
        aggregation: mean
        higher_is_better: true
        ignore_case: true
        ignore_punctuation: true
  # Word Sense Disambiguation
  - dataset_path: super_glue
    dataset_name: wic
    use_prompt: promptsource:*
    training_split: train
    validation_split: validation
    metric_list:
      - metric: exact_match
        aggregation: mean
        higher_is_better: true
        ignore_case: true
        ignore_punctuation: true
