modified yamls

lm_eval/tasks/super_glue/cb/default.yaml

@@ -7,9 +7,8 @@ output_type: multiple_choice
 training_split: train
 validation_split: validation
 doc_to_text: "{{premise}}\nQuestion: {{hypothesis}}. True, False, or Neither?\nAnswer:"
-doc_to_target: "{{answer_choices[label]}}"
-gold_alias: "{{label}}" # this will be cast to an int.
-template_aliases: "{% set answer_choices = ['True', 'False', 'Neither'] %}"
+doc_to_target: label
+doc_to_choice: ['True', 'False', 'Neither']
 metric_list:
   - metric: acc
   - metric: f1

lm_eval/tasks/swag/swag.yaml

@@ -7,14 +7,13 @@ output_type: multiple_choice
 training_split: train
 validation_split: validation
 test_split: null
-template_aliases: "{% set answer_choices = [ending0, ending1, ending2, ending3] %}{% set gold = label %}"
-doc_to_text: "{{startphrase}}"
-doc_to_target: "{{answer_choices[gold]}}"
-gold_alias: "{{gold}}"
+doc_to_text: startphrase
+doc_to_target: label
+doc_to_choice: "{{[ending0, ending1, ending2, ending3]}}"
 metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
   - metric: acc_norm
     aggregation: mean
-    higher_is_better: true
\ No newline at end of file
+    higher_is_better: true