Commit b9e55312 authored by lintangsutawika's avatar lintangsutawika
Browse files

add alt prompts for truthfulqa

parent edb21d21
task: truthfulqa_mc1_alt_pv_01
dataset_path: truthful_qa
dataset_name: multiple_choice
output_type: multiple_choice
training_split: null
validation_split: validation
test_split: null
num_fewshot: 0
doc_to_text: "{{question}}"
doc_to_target: 0
doc_to_choice: "{{mc1_targets.choices}}"
should_decontaminate: True
doc_to_decontamination_query: question
metric_list:
- metric: acc
aggregation: mean
higher_is_better: true
include: truthfulqa_mc1.yaml
task: truthfulqa_mc2_alt_pv_01
doc_to_target: 0
doc_to_choice: "{{mc2_targets.choices}}"
process_results: !function ../../../utils.process_results_mc2
should_decontaminate: True
doc_to_decontamination_query: question
metric_list:
- metric: acc
aggregation: mean
higher_is_better: true
task: truthfulqa_mc1_alt_pv_02
dataset_path: truthful_qa
dataset_name: multiple_choice
output_type: multiple_choice
training_split: null
validation_split: validation
test_split: null
num_fewshot: 0
doc_to_choice: "Q: {{mc2_targets.choices}}\nA:"
doc_to_target: 0
doc_to_choice: "{{mc1_targets.choices}}"
should_decontaminate: True
doc_to_decontamination_query: question
metric_list:
- metric: acc
aggregation: mean
higher_is_better: true
include: truthfulqa_mc1.yaml
task: truthfulqa_mc2_alt_pv_02
doc_to_target: 0
doc_to_choice: "{{mc2_targets.choices}}"
process_results: !function ../../../utils.process_results_mc2
should_decontaminate: True
doc_to_decontamination_query: question
metric_list:
- metric: acc
aggregation: mean
higher_is_better: true
task: truthfulqa_mc1_alt_pv_03
dataset_path: truthful_qa
dataset_name: multiple_choice
output_type: multiple_choice
training_split: null
validation_split: validation
test_split: null
num_fewshot: 0
doc_to_choice: "Question: {{mc2_targets.choices}}\nAnswer:"
doc_to_target: 0
doc_to_choice: "{{mc1_targets.choices}}"
should_decontaminate: True
doc_to_decontamination_query: question
metric_list:
- metric: acc
aggregation: mean
higher_is_better: true
include: truthfulqa_mc1.yaml
task: truthfulqa_mc2_alt_pv_03
doc_to_target: 0
doc_to_choice: "{{mc2_targets.choices}}"
process_results: !function ../../../utils.process_results_mc2
should_decontaminate: True
doc_to_decontamination_query: question
metric_list:
- metric: acc
aggregation: mean
higher_is_better: true
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment