Commit bbb8386c authored by lintangsutawika's avatar lintangsutawika
Browse files

removed alt worlds prompts

parent 3e5e9da2
dataset_path: ai2_arc
dataset_name: ARC-Easy
output_type: multiple_choice
training_split: train
validation_split: validation
test_split: test
doc_to_text: "Question: {{question}}\nAnswer:"
doc_to_target: "{{choices.label.index(answerKey)}}"
doc_to_choice: "{{choices.text}}"
should_decontaminate: true
doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
metric_list:
- metric: acc
aggregation: mean
higher_is_better: true
- metric: acc_norm
aggregation: mean
higher_is_better: true
- metric: brier_score
aggregation: brier_score
higher_is_better: false
include: _arc_easy_alt_yaml
group: arc_easy_alt_pv
task: arc_easy_alt_pv_01
doc_to_text: "{{question}}"
doc_to_decontamination_query: "{{question}}"
include: _arc_easy_alt_yaml
group: arc_easy_alt_pv
task: arc_easy_alt_pv_02
doc_to_text: "Q: {{question}}\nA:"
doc_to_decontamination_query: "Q: {{question}}\nA:"
include: _arc_easy_alt_yaml
group: arc_easy_alt_pv
task: arc_easy_alt_pv_03
doc_to_text: "Question: {{question}}\nAnswer:"
doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
dataset_path: EleutherAI/arithmetic
dataset_name: arithmetic_1dc
output_type: loglikelihood
validation_split: validation
test_split: null
doc_to_text: "{{context}}"
doc_to_target: "{{completion}}"
metric_list:
- metric: acc
aggregation: mean
higher_is_better: true
- metric: brier_score
higher_is_better: false
group: arithmetic_alt
task:
- arithmetic_alt_00
- arithmetic_alt_01
- arithmetic_alt_02
- arithmetic_alt_03
- arithmetic_alt_04
- arithmetic_alt_05
include: ../_template_yaml
group: arithmetic_alt_00
group_alias: arithmetic (Style 00)
dataset_path: EleutherAI/arithmetic
output_type: loglikelihood
validation_split: validation
test_split: null
doc_to_text: !function ../utils.style_00
doc_to_target: "{{completion}}"
metric_list:
- metric: acc
aggregation: mean
higher_is_better: true
- metric: brier_score
higher_is_better: false
include: _template_00_yaml
task: arithmetic_1dc_alt_00
dataset_name: arithmetic_1dc
task_alias: 1dc
include: _template_00_yaml
task: arithmetic_2da_alt_00
dataset_name: arithmetic_2da
task_alias: 2da
include: _template_00_yaml
task: arithmetic_2dm_alt_00
dataset_name: arithmetic_2dm
task_alias: 2dm
include: _template_00_yaml
task: arithmetic_2ds_alt_00
dataset_name: arithmetic_2ds
task_alias: 2ds
include: _template_00_yaml
task: arithmetic_3da_alt_00
dataset_name: arithmetic_3da
task_alias: 3da
include: _template_00_yaml
task: arithmetic_3ds_alt_00
dataset_name: arithmetic_3ds
task_alias: 3ds
include: _template_00_yaml
task: arithmetic_4da_alt_00
dataset_name: arithmetic_4da
task_alias: 4da
include: _template_00_yaml
task: arithmetic_4ds_alt_00
dataset_name: arithmetic_4ds
task_alias: 4ds
include: _template_00_yaml
task: arithmetic_5da_alt_00
dataset_name: arithmetic_5da
task_alias: 5da
include: _template_00_yaml
task: arithmetic_5ds_alt_00
dataset_name: arithmetic_5ds
task_alias: 5ds
include: ../_template_yaml
group: arithmetic_alt_01
group_alias: arithmetic (Style 01)
dataset_path: EleutherAI/arithmetic
output_type: loglikelihood
validation_split: validation
test_split: null
doc_to_text: !function ../utils.style_01
doc_to_target: "{{completion}}"
metric_list:
- metric: acc
aggregation: mean
higher_is_better: true
- metric: brier_score
higher_is_better: false
include: _template_01_yaml
task: arithmetic_1dc_alt_01
dataset_name: arithmetic_1dc
task_alias: 1dc
include: _template_01_yaml
task: arithmetic_2da_alt_01
dataset_name: arithmetic_2da
task_alias: 2da
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment