Merge pull request #172 from EleutherAI/lambada_cloze

add cloze variant of lambada task

lm_eval/tasks/__init__.py

@@ -35,6 +35,7 @@ from . import unscramble
 from . import logiqa
 from . import hendrycks_test
 from . import hendrycks_math
+from . import lambada_cloze
 
 ########################################
 # Translation tasks
@@ -91,6 +92,7 @@ TASK_REGISTRY = {
     "coqa": coqa.CoQA,
     "drop": drop.DROP,
     "lambada": lambada.LAMBADA,
+    "lambada_cloze": lambada_cloze.LAMBADA_cloze,
     "piqa": piqa.PiQA,
 
     # Science related

lm_eval/tasks/lambada_cloze.py

+import json
+from lm_eval.base import Task, rf
+from lm_eval.metrics import mean, perplexity
+from lm_eval.utils import sh
+from lm_eval.tasks.lambada import LAMBADA
+from best_download import download_file
+
+
+class LAMBADA_cloze(LAMBADA):
+    def doc_to_text(self, doc):
+        return doc['text'].rsplit(' ', 1)[0] + " ____. ->"
+
+    def doc_to_target(self, doc):
+        return " " + doc['text'].rsplit(' ', 1)[1]
+    
+    def fewshot_description(self):
+        return "Fill in blank:\n"