group: lama_negated_yes_no
dataset_path: lintang/lama_primed_negated
output_type: multiple_choice
# description: "Answer the following question with either a Yes or a No. "
doc_to_text: "Question: {{masked_negations[0]|replace('[MASK]', obj_label)}} Yes or No?\n\nAnswer:"
doc_to_target: "No"
doc_to_choice: ["Yes", "No"]
should_decontaminate: true
doc_to_decontamination_query: "Question: {{masked_negations|replace('[MASK]', obj_label)}} Yes or No?\n\nAnswer:"
metric_list:
  - metric: acc
    aggregation: mean
    higher_is_better: true
  - metric: acc_norm
    aggregation: mean
    higher_is_better: true
metadata:
  version: 1.0
