Merge pull request #53 from anishthite/master

Update drop to be consistent with gpt3 paper

lm_eval/tasks/coqa.py

@@ -5,8 +5,10 @@ from ..utils import sh
 
 
 class CoQA(Dataset):
-
+    def __init__(self):
+        self.download()
     def download(self):
+        #TODO: don't download if files already there
         sh("""
             mkdir -p data/coqa 
             wget http://downloads.cs.stanford.edu/nlp/data/coqa/coqa-train-v1.0.json -O data/coqa/coqa-train-v1.0.json
@@ -17,48 +19,31 @@ class CoQA(Dataset):
         return True
 
     def has_validation_docs(self):
-        return False
+        return True
 
     def has_test_docs(self):
         return False
 
     def training_docs(self):
-        myjson = json.load(open('data/coqa/coqa-train-v1.0.json'))['data']
-        return self.load_doc(myjson)
+        return json.load(open('data/coqa/coqa-train-v1.0.json'))['data']
 
     def validation_docs(self):
-        pass
+        return  json.load(open('data/coqa/coqa-dev-v1.0.json'))['data']  
 
     def test_docs(self):
-        myjson = json.load(open('data/coqa/coqa-dev-v1.0.json'))['data']    
-        return self.load_doc(myjson)
-    
-    def fewshot_examples(self, k):
-        traindocs = list(self.training_docs())
-        random.seed(123)
-        random.shuffle(traindocs)
-
-        return traindocs[:k]
+        pass   
     
     def fewshot_description(self):
         pass
-
-    def load_doc(self, myjson):
-        docs = []
-        for item in myjson:
-            new_instance = [item['story']]
-            qa_pairs = zip(item['questions'], item['answers'])
-            for pair in qa_pairs:
-                new_instance.append('\n')
-                new_instance.append(''.join(['Q: ',pair[0]['input_text']]))
-                new_instance.append(''.join(['A: ',pair[1]['input_text']]))
-            docs.append(new_instance)  
-        return docs
     
     def doc_to_text(self, doc, include_target=True):
-        text = '\n<|endoftext|>\n'.join(['\n'.join(instance) for instance in doc])
-        text = text + '\n<|endoftext|>'
-        return text
+        text = [doc['story']]
+        for pair in zip(doc['questions'], doc['answers']):
+            text.append('\n\n')
+            text.append(''.join(['Q: ',pair[0]['input_text'], '\n\n']))
+            text.append(''.join(['A: ',pair[1]['input_text']]))
+
+        return ''.join(text)
 
     def evaluate(self, docs, lm):
         pass

lm_eval/tasks/drop.py

@@ -10,6 +10,9 @@ from ..base import Dataset
 class DROP(Dataset):
     DATAFOLDER = Path(__file__).parent / "../../data/drop"
     
+    def __init__(self):
+        self.download()
+
     def has_training_docs(self):
         """Whether the task has a training set"""
         return True
@@ -35,10 +38,10 @@ class DROP(Dataset):
         pass
     
     def doc_to_text(self, doc, include_target=True):
-        doctext = "Passage: {}\n\n".format(doc["passage"])
+        doctext = "Passage: {}\n".format(doc["passage"])
         qa_texts = []
         for pair in doc["qa_pairs"]:
-            text = ''.join(['Q: ', pair['question'],'\nA: '])
+            text = ''.join(['Question: ', pair['question'],'\nAnswer: '])
             if include_target:
                 def get_answer(ans_dict):
                     if ans_dict['number'] != '':
@@ -52,7 +55,7 @@ class DROP(Dataset):
                                      ans_dict['date']['year']]).strip() 
                 text = ''.join([text, get_answer(pair['answer'])])
             qa_texts.append(text)
-        return ''.join([doctext, '\n\n'.join(qa_texts)])
+        return ''.join([doctext, '\n'.join(qa_texts)])
             
     
     def evaluate(self, docs, lm, provide_description, num_fewshot):

lm_eval/tasks/storycloze.py

+import json
+import random
+from lm_eval.base import Dataset
+from ..utils import sh
+import csv
+
+class StoryCloze(Dataset):
+    def __init__(self):
+        self.download()
+    def download(self):
+        #TODO: replace with Eye link
+        pass
+
+    def has_training_docs(self):
+        return False
+
+    def has_validation_docs(self):
+        return True
+
+    def has_test_docs(self):
+        return True
+
+    def training_docs(self):
+        pass
+
+    def load_doc(self, filename):
+        with open(filename, newline='') as file:
+            filereader = csv.reader(file)
+            return list(filereader)
+                
+
+    def validation_docs(self):
+        return  self.load_doc("data/storycloze/cloze_test_val__winter2018-cloze_test_ALL_val - 1 - 1.csv")
+
+    def test_docs(self):
+        return self.load_doc("data/storycloze/cloze_test_test__winter2018-cloze_test_ALL_test - 1.csv")
+
+    
+    def fewshot_description(self):
+        pass
+    
+    def doc_to_text(self, doc, include_target=True):
+        return ' '.join([*doc[1:5],doc[int(doc[-1])-4]]) 
+
+    def evaluate(self, docs, lm):
+        pass
+

lm_eval/tasks/triviaqa.py

+import json
+import random
+from lm_eval.base import Dataset
+from ..utils import sh
+
+class TriviaQA(Dataset):
+    def __init__(self):
+        self.download()
+    def download(self):
+        #pass
+        #TODO: don't download if files already there
+        sh("""
+           mkdir -p data/triviaqa
+           wget http://nlp.cs.washington.edu/triviaqa/data/triviaqa-unfiltered.tar.gz -O data/triviaqa/trivia_qa-unfiltered.tar.gz
+           tar -xf data/triviaqa/trivia_qa-unfiltered.tar.gz
+           mv triviaqa-unfiltered/ data/triviaqa/
+           """)
+
+    def has_training_docs(self):
+        return True
+
+    def has_validation_docs(self):
+        return True
+
+    def has_test_docs(self):
+        return True
+
+    def training_docs(self):
+        return json.load(open('data/triviaqa/triviaqa-unfiltered/unfiltered-web-train.json'))['Data']
+
+    def validation_docs(self):
+        return  json.load(open('data/triviaqa/triviaqa-unfiltered/unfiltered-web-dev.json'))['Data']
+
+    def test_docs(self):
+        return  json.load(open('data/triviaqa/triviaqa-unfiltered/unfiltered-web-test.json'))['Data']     
+    
+    def fewshot_description(self):
+        pass
+    
+    def doc_to_text(self, doc, include_target=True):
+        return ''.join(['Q: ', doc['Question'], '\n\n','A: ', doc['Answer']['Aliases'][0]])
+
+    def evaluate(self, docs, lm):
+        pass
+