template.yaml

dataset_path: super_glue
dataset_name: cb
training_split: train
validation_split: validation
template:
  - name: can we infer
  reference: Webson & Pavlick 2021
  doc_to_text: 'Suppose {{premise}} Can we infer that "{{hypothesis}}"? Yes, no, or maybe?'
  doc_to_target: '{% set answer_choices = ['Yes', 'No', 'Maybe'] %}{{answer_choices[label]}}'
  - name: based on the previous passage
  reference: "Adapted from the BoolQ prompts in Schick & Sch\xFCtze 2021."
  doc_to_text: '{{premise}} Based on the previous passage, is it true that "{{hypothesis}}"? Yes, no, or maybe?'
  doc_to_target: '{% set answer_choices = ['Yes', 'No', 'Maybe'] %}{{answer_choices[label]}}'
  - name: claim true/false/inconclusive
  reference: Sanh et al. 2021
  doc_to_text: '{{premise}} Based on that information, is the claim: "{{hypothesis}}" {{"true"}}, {{"false"}}, or {{"inconclusive"}}?'
  doc_to_target: '{% set answer_choices = ['True', 'False', 'Inconclusive'] %}{{answer_choices[label]}}'
metric_list:
  - metric: exact_match
    aggregation: mean
    higher_is_better: true
    ignore_case: true
    ignore_punctuation: true