Commit 97e3c9fe authored by lintangsutawika's avatar lintangsutawika
Browse files

added lama primed and negated dataset

parent 7852985b
include: _misprimed_right_wrong_template_yaml
task: lama_misprimed_google_re_right_wrong
dataset_name: GoogleRE
test_split: high_ranked
include: _misprimed_right_wrong_template_yaml
task: lama_misprimed_squad_right_wrong
dataset_name: SQUAD
test_split: high_ranked
include: _misprimed_right_wrong_template_yaml
task: lama_misprimed_trex_right_wrong
dataset_name: TREx
test_split: high_ranked
include: _negated_right_wrong_template_yaml
task: lama_negated_conceptnet_right_wrong
dataset_name: ConceptNet
test_split: high_ranked
include: _negated_right_wrong_template_yaml
task: lama_negated_google_re_right_wrong
dataset_name: GoogleRE
test_split: high_ranked
include: _negated_right_wrong_template_yaml
task: lama_negated_squad_right_wrong
dataset_name: SQUAD
test_split: high_ranked
include: _negated_right_wrong_template_yaml
task: lama_negated_trex_right_wrong
dataset_name: TREx
test_split: high_ranked
include: _normal_right_wrong_template_yaml
task: lama_normal_conceptnet_right_wrong
dataset_name: ConceptNet
test_split: high_ranked
include: _normal_right_wrong_template_yaml
task: lama_normal_google_re_right_wrong
dataset_name: GoogleRE
test_split: high_ranked
include: _normal_right_wrong_template_yaml
task: lama_normal_squad_right_wrong
dataset_name: SQUAD
test_split: high_ranked
include: _normal_right_wrong_template_yaml
task: lama_normal_trex_right_wrong
dataset_name: TREx
test_split: high_ranked
group: lama_misprimed_true_false
dataset_path: lintang/lama_primed_negated
output_type: multiple_choice
# description: "Answer the following question with either True or False. "
doc_to_text: "Question: {{masked_misprimed[0]|replace('[MASK]', obj_label)}} True or False?\n\nAnswer:"
doc_to_target: "True"
doc_to_choice: ["True", "False"]
should_decontaminate: true
doc_to_decontamination_query: "Question: {{masked_misprimed[0]|replace('[MASK]', obj_label)}} True or False?\n\nAnswer:"
metric_list:
- metric: acc
aggregation: mean
higher_is_better: true
- metric: acc_norm
aggregation: mean
higher_is_better: true
metadata:
version: 1.0
group: lama_negated_true_false
dataset_path: lintang/lama_primed_negated
output_type: multiple_choice
# description: "Answer the following question with either True or False. "
doc_to_text: "Question: {{masked_negations[0]|replace('[MASK]', obj_label)}} True or False?\n\nAnswer:"
doc_to_target: "False"
doc_to_choice: ["True", "False"]
should_decontaminate: true
doc_to_decontamination_query: "Question: {{masked_negations[0]|replace('[MASK]', obj_label)}} True or False?\n\nAnswer:"
metric_list:
- metric: acc
aggregation: mean
higher_is_better: true
- metric: acc_norm
aggregation: mean
higher_is_better: true
metadata:
version: 1.0
group: lama_normal_true_false
dataset_path: lintang/lama_primed_negated
output_type: multiple_choice
# description: "Answer the following question with either True or False. "
doc_to_text: "Question: {{masked_sentences[0]|replace('[MASK]', obj_label)}} True or False?\n\nAnswer:"
doc_to_target: "True"
doc_to_choice: ["True", "False"]
should_decontaminate: true
doc_to_decontamination_query: "Question: {{masked_sentences[0]|replace('[MASK]', obj_label)}} True or False?\n\nAnswer:"
metric_list:
- metric: acc
aggregation: mean
higher_is_better: true
- metric: acc_norm
aggregation: mean
higher_is_better: true
metadata:
version: 1.0
include: _misprimed_true_false_template_yaml
task: lama_misprimed_conceptnet_true_false
dataset_name: ConceptNet
test_split: high_ranked
include: _misprimed_true_false_template_yaml
task: lama_misprimed_google_re_true_false
dataset_name: GoogleRE
test_split: high_ranked
include: _misprimed_true_false_template_yaml
task: lama_misprimed_squad_true_false
dataset_name: SQUAD
test_split: high_ranked
include: _misprimed_true_false_template_yaml
task: lama_misprimed_trex_true_false
dataset_name: TREx
test_split: high_ranked
include: _negated_true_false_template_yaml
task: lama_negated_conceptnet_true_false
dataset_name: ConceptNet
test_split: high_ranked
include: _negated_true_false_template_yaml
task: lama_negated_google_re_true_false
dataset_name: GoogleRE
test_split: high_ranked
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment