Merge pull request #56 from anishthite/master

add rte text

lm_eval/tasks/lambada.py

@@ -6,7 +6,8 @@ import ftfy
 
 
 class Lambada(Dataset):
-
+    def __init__(self):
+        self.download()
     def download(self):
         sh("mkdir -p data/lambada")
         with open("data/lambada/lambada_test.json", 'w') as f:
@@ -32,14 +33,16 @@ class Lambada(Dataset):
         pass
 
     def load_doc(self, myjson):
-        return [doc['text'] for doc in myjson]
+        return [doc for doc in myjson]
 
     def test_docs(self):
         myjson = json.load(open("data/lambada/lambada_test.json"))
         return self.load_doc(myjson)
 
     def doc_to_text(self, doc, include_target=True):
-        pass
+        #TODO: check if this is how OA does it
+        #label = doc[]
+        return doc
 
     def evaluate(self, docs, lm, provide_description, num_fewshot):
         pass
\ No newline at end of file

lm_eval/tasks/superglue.py

@@ -282,3 +282,25 @@ class SGWinogradSchemaChallenge(HFTask):
             )
             preds.append(1 if generated == to_predict else 0)
         return simple_accuracy_metric(preds=preds, golds=golds)
+
+class RTE(HFTask):
+    DATASET_PATH = "super_glue"
+    DATASET_NAME = "rte"
+
+    def fewshot_description(self):
+        #TODO: implement
+        pass
+
+    def doc_to_text(self, doc, include_target=True):
+        if include_target:
+            if doc['label'] == 0:
+                answer = 'True'
+            else:
+                answer = 'False'
+            return ''.join([doc['premise'], '\nquestion: ',doc['hypothesis'], ' True or False?\nanswer: ', answer])
+        else:
+            return ''.join([doc['premise'], '\nquestion: ',doc['hypothesis'], ' True or False?\nanswer: '])
+    def evaluate(self, docs, lm, provide_description, num_fewshot):
+        #TODO: 
+        pass
+