Merge pull request #39 from EleutherAI/add_lambada

add_lambada

lm_eval/tasks/common.py

@@ -4,14 +4,14 @@ import random
 from ..base import Dataset
 
 
-class HFNLPTask(Dataset):
-    NLP_PATH = None
-    NLP_NAME = None
+class HFTask(Dataset):
+    DATASET_PATH = None
+    DATASET_NAME = None
 
     def __init__(self):
         super().__init__()
         self._training_docs = None
-        self.data = datasets.load_dataset(path=self.NLP_PATH, name=self.NLP_NAME)
+        self.data = datasets.load_dataset(path=self.DATASET_PATH, name=self.DATASET_NAME)
 
     def has_training_docs(self):
         """Whether the task has a training set"""

lm_eval/tasks/glue.py

@@ -2,8 +2,7 @@ import numpy as np
 from scipy.stats import pearsonr, spearmanr
 from sklearn.metrics import f1_score, matthews_corrcoef
 from tqdm import auto as tqdm_lib
-from . common import HFNLPTask, simple_accuracy_metric, yesno
-
+from . common import HFTask, simple_accuracy_metric, yesno
 
 def get_accuracy_and_f1(preds, golds):
     golds = np.array(golds)
@@ -22,10 +21,10 @@ def get_accuracy_and_f1(preds, golds):
     }
 
 
-class CoLA(HFNLPTask):
-    NLP_PATH = "glue"
-    NLP_NAME = "cola"
-
+class CoLA(HFTask):
+    DATASET_PATH = "glue"
+    DATASET_NAME = "cola"
+    
     def has_training_docs(self):
         return True
 
@@ -64,9 +63,9 @@ class CoLA(HFNLPTask):
         }
 
 
-class MNLI(HFNLPTask):
-    NLP_PATH = "glue"
-    NLP_NAME = "mnli"
+class MNLI(HFTask):
+    DATASET_PATH = "glue"
+    DATASET_NAME = "mnli"
 
     def has_training_docs(self):
         return True
@@ -79,11 +78,11 @@ class MNLI(HFNLPTask):
 
     def validation_docs(self):
         if self.has_validation_docs():
-            return self._load_nlp_dataset()["validation_matched"]
+            return self.data["validation_matched"]
 
     def test_docs(self):
         if self.has_test_docs():
-            return self._load_nlp_dataset()["test_matched"]
+            return self.data["test_matched"]
 
     def doc_to_text(self, doc, include_target=True):
         text = "{}\nquestion:\t{}\tTrue, False or Neither?\nanswer:".format(
@@ -115,9 +114,9 @@ class MNLI(HFNLPTask):
         return simple_accuracy_metric(preds=preds, golds=golds)
 
 
-class MRPC(HFNLPTask):
-    NLP_PATH = "glue"
-    NLP_NAME = "mrpc"
+class MRPC(HFTask):
+    DATASET_PATH = "glue"
+    DATASET_NAME = "mrpc"
 
     def has_training_docs(self):
         return True
@@ -152,10 +151,10 @@ class MRPC(HFNLPTask):
             preds.append(lm.loglikelihood(ctx, 'yes') > lm.loglikelihood(ctx, 'no'))
         return get_accuracy_and_f1(preds=preds, golds=golds)
 
-
-class RTE(HFNLPTask):
-    NLP_PATH = "glue"
-    NLP_NAME = "rte"
+      
+class RTE(HFTask):
+    DATASET_PATH = "glue"
+    DATASET_NAME = "rte"
 
     def has_training_docs(self):
         return True
@@ -190,9 +189,9 @@ class RTE(HFNLPTask):
         return simple_accuracy_metric(preds=preds, golds=golds)
 
 
-class QNLI(HFNLPTask):
-    NLP_PATH = "glue"
-    NLP_NAME = "qnli"
+class QNLI(HFTask):
+    DATASET_PATH = "glue"
+    DATASET_NAME = "qnli"
 
     def has_training_docs(self):
         return True
@@ -227,9 +226,9 @@ class QNLI(HFNLPTask):
         return simple_accuracy_metric(preds=preds, golds=golds)
 
 
-class QQP(HFNLPTask):
-    NLP_PATH = "glue"
-    NLP_NAME = "qqp"
+class QQP(HFTask):
+    DATASET_PATH = "glue"
+    DATASET_NAME = "qqp"
 
     def has_training_docs(self):
         return True
@@ -265,9 +264,9 @@ class QQP(HFNLPTask):
         return get_accuracy_and_f1(preds=preds, golds=golds)
 
 
-class STSB(HFNLPTask):
-    NLP_PATH = "glue"
-    NLP_NAME = "stsb"
+class STSB(HFTask):
+    DATASET_PATH = "glue"
+    DATASET_NAME = "stsb"
 
     def has_training_docs(self):
         return True
@@ -322,9 +321,9 @@ class STSB(HFNLPTask):
         }
 
 
-class SST(HFNLPTask):
-    NLP_PATH = "glue"
-    NLP_NAME = "sst2"
+class SST(HFTask):
+    DATASET_PATH = "glue"
+    DATASET_NAME = "sst2"
 
     def has_training_docs(self):
         return True
@@ -359,10 +358,10 @@ class SST(HFNLPTask):
         return simple_accuracy_metric(preds=preds, golds=golds)
 
 
-class WNLI(HFNLPTask):
-    NLP_PATH = "glue"
-    NLP_NAME = "wnli"
-
+class WNLI(HFTask):
+    DATASET_PATH = "glue"
+    DATASET_NAME = "wnli"
+    
     def has_training_docs(self):
         return True
 

lm_eval/tasks/lambada.py

+from lm_eval.base import Dataset
+from lm_eval.utils import sh
+import json
+import requests
+import ftfy
+
+
+class Lambada(Dataset):
+
+    def download(self):
+        sh("mkdir -p data/lambada")
+        with open("data/lambada/lambada_test.json", 'w') as f:
+            req = requests.get("https://storage.googleapis.com/gpt-2/data/lambada_test.jsonl")
+            req.raise_for_status()
+            jsons = [json.loads(l) for l in req.iter_lines()]
+            texts = [ftfy.fix_text(j['text'], normalization='NFKC') for j in jsons]
+            json.dump(texts, f)
+
+    def has_training_docs(self):
+        return False
+
+    def has_validation_docs(self):
+        return False
+
+    def has_test_docs(self):
+        return True
+
+    def training_docs(self):
+        pass
+
+    def validation_docs(self):
+        pass
+
+    def load_doc(self, myjson):
+        return [doc['text'] for doc in myjson]
+
+    def test_docs(self):
+        myjson = json.load(open("data/lambada/lambada_test.json"))
+        return self.load_doc(myjson)
+
+    def doc_to_text(self, doc, include_target=True):
+        pass
+
+    def evaluate(self, docs, lm, provide_description, num_fewshot):
+        pass
\ No newline at end of file