Merge branch 'master' into json-task

lm_eval/tasks/__init__.py

@@ -53,6 +53,13 @@ from . import storycloze
 from . import toxigen
 from . import crowspairs
 from . import json
+from . import xcopa
+from . import bigbench
+from . import xstorycloze
+from . import xwinograd
+from . import pawsx
+from . import xnli
+from . import mgsm
 
 ########################################
 # Translation tasks
@@ -311,6 +318,13 @@ TASK_REGISTRY = {
     # "storycloze_2016": storycloze.StoryCloze2016,
     # "storycloze_2018": storycloze.StoryCloze2018,
     # "sat": sat.SATAnalogies,
+    **xcopa.construct_tasks(),
+    **bigbench.create_all_tasks(),
+    **xstorycloze.create_all_tasks(),
+    **xwinograd.create_all_tasks(),
+    **pawsx.construct_tasks(),
+    **xnli.construct_tasks(),
+    **mgsm.construct_tasks(),
 }
 
 

lm_eval/tasks/coqa.py

@@ -141,7 +141,7 @@ class CoQA(Task):
             language description, as well as the few shot examples, and the question
             part of the document for `doc`.
         """
-        cont_request = rf.greedy_until(ctx, ["\nQ:"])
+        cont_request = rf.greedy_until(ctx, {"until": ["\nQ:"]})
         return cont_request
 
     def process_results(self, doc, results):