superglue add yamls

lm_eval/tasks/super_glue/copa/default.yaml

 group:
   - super-glue-lm-eval-v1
-task: "copa"
+task: copa
 dataset_path: super_glue
 dataset_name: copa
 output_type: multiple_choice
@@ -8,7 +8,6 @@ training_split: train
 validation_split: validation
 doc_to_text: !function utils.doc_to_text
 doc_to_target: !function utils.doc_to_target
-gold_alias: "{{label}}" # this will be cast to an int.
-template_aliases: "{% set answer_choices = [{{doc.choice1}}, 'b'] %} {{answer_choices}}"
+doc_to_choice: !function utils.doc_to_choice
 metric_list:
   - metric: acc

lm_eval/tasks/super_glue/copa/utils.py

@@ -15,3 +15,7 @@ def doc_to_target(doc):
     correct_choice = doc["choice1"] if doc["label"] == 0 else doc["choice2"]
     # Connect the sentences
     return " " + convert_choice(correct_choice)
+
+
+def doc_to_choice(doc):
+    return [" " + convert_choice(doc["choice1"]), " " + convert_choice(doc["choice2"])]

lm_eval/tasks/super_glue/multirc/default.yaml

+group:
+  - super-glue-lm-eval-v1
+task: multirc
+dataset_path: super_glue
+dataset_name: multirc
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+doc_to_text: "{{paragraph}}\nQuestion: {{question}}\nAnswer:"
+doc_to_target: label
+doc_to_choice: "[\"{{answer}}\\nIs the answer correct? yes\", \"{{answer}}\\nIs the answer correct? no\"]"
+metric_list:
+  - metric: acc

lm_eval/tasks/super_glue/record/default.yaml

+group:
+  - super-glue-lm-eval-v1
+task: record
+dataset_path: super_glue
+dataset_name: record
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+doc_to_text: !function util.doc_to_text
+doc_to_target: "{{answers}}"
+doc_to_choice: "{{entities}}"
+metric_list:
+  - metric: f1
+  - metric: em

lm_eval/tasks/super_glue/record/promptsource-01.yaml

-include: promptsource-00.yaml
-group:
-  - super-glue-promptsource
-task: "Add sentence after after (continuation choices)"
-use_prompt: "promptsource:Add sentence after after (continuation choices)"

lm_eval/tasks/super_glue/record/promptsource-02.yaml

-include: promptsource-00.yaml
-group:
-  - super-glue-promptsource
-task: "Can you figure out…"
-use_prompt: "promptsource:Can you figure out…"

lm_eval/tasks/super_glue/record/util.py

+def doc_to_text(doc):
+    initial_text, *highlights = doc["passage"].strip().split("\n@highlight\n")
+    text = initial_text + "\n\n"
+    for highlight in highlights:
+        text += f"  - {highlight}.\n"
+    return text
+
+
+def format_answer(query, entity):
+    return f"  - {query}".replace("@placeholder", entity)
+
+
+def doc_to_target(doc):
+    # We only output the first correct entity in a doc
+    return format_answer(query=doc["query"], entity=doc["answers"][0])

lm_eval/tasks/super_glue/wic/default.yaml

@@ -6,7 +6,7 @@ dataset_name: wic
 output_type: multiple_choice
 training_split: train
 validation_split: validation
-doc_to_text: !function utils.doc_to_text
+doc_to_text: "Sentence 1: {{sentence1}}\nSentence 2: {{sentence2}}\nQuestion: Is the word '{{sentence1[start1:end1]}}' used in the same way in the two sentences above?\nAnswer:"
 doc_to_target: label
 doc_to_choice: ['no', 'yes']
 metric_list:

lm_eval/tasks/super_glue/wic/utils.py

-def doc_to_text(doc):
-    return (
-        "Sentence 1: {}\nSentence 2: {}\nQuestion: Is the word '{}' used in the same way in the"
-        " two sentences above?\nAnswer:".format(
-            doc["sentence1"],
-            doc["sentence2"],
-            doc["sentence1"][doc["start1"] : doc["end1"]],
-        )
-    )

lm_eval/tasks/super_glue/wsc/default.yaml

+group:
+  - super-glue-lm-eval-v1
+task: wsc
+dataset_path: super_glue
+dataset_name: wsc
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+doc_to_text: !function preprocess_wsc.default_doc_to_text
+doc_to_target: label
+doc_to_choice: ['no', 'yes']
+metric_list:
+  - metric: acc