add winogrande

lm_eval/tasks/winogrande/default.yaml

+group:
+  - super-glue-lm-eval-v1
+task: winogrande
+dataset_path: winogrande
+dataset_name: winogrande_xl
+output_type: multiple_choice
+should_decontaminate: true
+doc_to_decontamination_query: "{{sentence}}"
+training_split: train
+validation_split: validation
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true

lm_eval/tasks/winogrande/preprocess.py

+import re
+from lm_eval.utils import general_detokenize
+
+
+def partial_context(doc, option):
+    # Substitute the pronoun in the sentence with the specified option
+    # and ignore everything after.
+    pronoun_loc = doc["sentence"].index("_")
+    return doc["sentence"][:pronoun_loc] + option
+
+
+def partial_target(doc):
+    # The target is everything after the document specified pronoun.
+    pronoun_loc = doc["sentence"].index("_") + 1
+    return " " + doc["sentence"][pronoun_loc:].strip()