adjustments for super_glue

lm_eval/api/task.py

@@ -23,6 +23,7 @@ from lm_eval.api.filter import FilterEnsemble
 from lm_eval.logger import eval_logger
 from lm_eval.prompts import get_prompt
 from lm_eval.filters import build_filter_ensemble
+from lm_eval.metrics import metric_max_over_ground_truths
 from lm_eval.api.metrics import (
     mean,
     weighted_perplexity,

lm_eval/tasks/super_glue/boolq/default.yaml

@@ -8,7 +8,7 @@ training_split: train
 validation_split: validation
 doc_to_text: "{{passage}}\nQuestion: {{question}}?\nAnswer:"
 doc_to_target: label
-doc_to_choice: {0: "no", 1: "yes"}
+doc_to_choice: ["no", "yes"]
 should_decontaminate: true
 doc_to_decontamination_query: passage
 metric_list:

lm_eval/tasks/super_glue/multirc/default.yaml

@@ -8,6 +8,6 @@ training_split: train
 validation_split: validation
 doc_to_text: "{{paragraph}}\nQuestion: {{question}}\nAnswer:"
 doc_to_target: label
-doc_to_choice: "[\"{{answer}}\\nIs the answer correct? yes\", \"{{answer}}\\nIs the answer correct? no\"]"
+doc_to_choice: "['''{{answer}}\\nIs the answer correct? yes''', '''{{answer}}\\nIs the answer correct? no''']"
 metric_list:
   - metric: acc

lm_eval/tasks/super_glue/record/default.yaml

-group:
-  - super-glue-lm-eval-v1
+# group:
+#   - super-glue-lm-eval-v1
 task: record
 dataset_path: super_glue
 dataset_name: record