openbookqa_generative.yaml

task: openbookqa_generative
dataset_path: openbookqa
dataset_name: main
output_type: generate_until
training_split: train
validation_split: validation
test_split: test
doc_to_text: |-
  Given the following question and four candidate answers (A, B, C and D), choose the best answer.

  Question: {{question_stem}}
  A. {{choices.text[0]}}
  B. {{choices.text[1]}}
  C. {{choices.text[2]}}
  D. {{choices.text[3]}}
  Your response should end with "The best answer is [the_answer_letter]" where the [the_answer_letter] is one of choice letters, A, B, C or D.
doc_to_target: answerKey
should_decontaminate: true
filter_list:
  - name: "strict-match"
    filter:
      - function: "multi_choice_regex"
      - function: "take_first"
gen_kwargs:
  until: []
  max_gen_toks: 10
metric_list:
  - metric: exact_match
    aggregation: mean
    higher_is_better: true
metadata:
  version: 1.0