updated yamls

lm_eval/tasks/gsm8k/gsm8k.yaml

@@ -21,7 +21,7 @@ metric_list:
       - ","
       - "\\$"
       - ".*### "
-delimiter: "\n\n"
+fewshot_delimiter: "\n\n"
 generation_kwargs:
   until:
     - "\n\n"

lm_eval/tasks/openbookqa/openbookqa.yaml

@@ -7,11 +7,11 @@ output_type: multiple_choice
 training_split: train
 validation_split: validation
 test_split: test
-template_aliases: "{% set answer_choices = choices['text'] %}{% set gold = choices.label.index(answerKey.lstrip()) %}" # set the list of possible answer choices, and set what this doc's gold answer is (set what ds column used, and what)
-doc_to_text: "{{question_stem}}"
-doc_to_target: "{{gold}}" # this will be cast to an int.
+doc_to_text: question_stem
+doc_to_target: "{{choices['text'][choices.label.index(answerKey.lstrip())]}}"
+doc_to_choice: "{{choices.text}}"
 should_decontaminate: true
-doc_to_decontamination_query: "{{question_stem}}"
+doc_to_decontamination_query: question_stem
 metric_list:
   - metric: acc
     aggregation: mean

lm_eval/tasks/piqa/piqa.yaml

@@ -7,10 +7,9 @@ output_type: multiple_choice
 training_split: train
 validation_split: validation
 test_split: null
-template_aliases: "{% set question = goal %}{% set answer_choices = [sol1, sol2] %}{% set gold = label %}" # set the list of possible answer choices, and set what this doc's gold label idx is
 doc_to_text: "Question: {{question}}\nAnswer:"
-doc_to_target: "{{answer_choices[gold]}}"
-gold_alias: "{{gold}}" # this will be cast to an int.
+doc_to_target: label
+doc_to_choice: "{{[sol1, sol2]}}"
 metric_list:
   - metric: acc
     aggregation: mean

lm_eval/tasks/prost/corypaik_prost.yaml

@@ -5,10 +5,9 @@ dataset_path: corypaik/prost
 dataset_name: null
 output_type: multiple_choice
 test_split: test
-template_aliases: "{% set answer_choices = [A, B, C, D] %}{% set gold = label %}" # set the list of possible answer choices, and set what this doc's gold answer is (set what ds column used, and what)
 doc_to_text: "{{context}}\nQuestion: {{ex_question}}\nAnswer:"
-doc_to_target: "{{answer_choices[gold]}}"
-gold_alias: "{{gold}}" # this will be cast to an int.
+doc_to_target: label
+doc_to_choice: [A, B, C, D]
 should_decontaminate: true
 doc_to_decontamination_query: "{{context}}\nQuestion: {{ex_question}}\nAnswer:"
 metric_list:

lm_eval/tasks/pubmedqa/pubmedqa.yaml

@@ -7,11 +7,10 @@ output_type: multiple_choice
 training_split: null
 validation_split: null
 test_split: train
-template_aliases: "{% set answer_choices = ['yes', 'no', 'maybe'] %}{% set gold = final_decision %}"
 doc_to_text: !function preprocess_pubmedqa.doc_to_text
-doc_to_target: !function preprocess_pubmedqa.doc_to_target
-gold_alias: !function preprocess_pubmedqa.gold_alias
+doc_to_target: final_decision
+doc_to_choice: ["yes", "no", "maybe"]
 metric_list:
   - metric: acc
     aggregation: mean
-    higher_is_better: true
+    higher_is_better: true
\ No newline at end of file

lm_eval/tasks/race/race.yaml

@@ -5,9 +5,9 @@ dataset_path: EleutherAI/race
 dataset_name: high
 output_type: multiple_choice
 test_split: test
-create_choices: !function preprocess_race.create_choices
 doc_to_text: !function preprocess_race.doc_to_text
 doc_to_target: !function preprocess_race.doc_to_target
+doc_to_choice: !function preprocess_race.doc_to_choice
 metric_list:
   - metric: acc
     aggregation: mean

lm_eval/tasks/super_glue/wic/default.yaml

@@ -7,8 +7,7 @@ output_type: multiple_choice
 training_split: train
 validation_split: validation
 doc_to_text: !function utils.doc_to_text
-doc_to_target: !function utils.doc_to_target
-gold_alias: "{{label}}" # this will be cast to an int.
-template_aliases: "{% set answer_choices = ['no', 'yes'] %}"
+doc_to_target: label
+doc_to_choice: ['no', 'yes']
 metric_list:
   - metric: acc

lm_eval/tasks/super_glue/wic/utils.py

@@ -7,7 +7,3 @@ def doc_to_text(doc):
             doc["sentence1"][doc["start1"] : doc["end1"]],
         )
     )
-
-
-def doc_to_target(doc):
-    return " {}".format({0: "no", 1: "yes"}[doc["label"]])

lm_eval/tasks/super_glue/wsc/t5-prompt.yaml

@@ -6,7 +6,8 @@ dataset_name: wsc
 training_split: train
 validation_split: validation
 doc_to_text: !function "preprocess_wsc.doc_to_text"
-doc_to_target: "{% set answer_choices = ['False', 'True'] %}{{answer_choices[label]}}"
+doc_to_target: label
+doc_to_choice: ['False', 'True']
 metric_list:
   - metric: exact_match
     aggregation: mean