Merge pull request #195 from EleutherAI/sdtblck-patch-1

Change generate until sequence in coqa to be \nQ:

lm_eval/tasks/coqa.py

@@ -8,7 +8,7 @@ from best_download import download_file
 
 
 class CoQA(Task):
-    VERSION = 0
+    VERSION = 1
 
     def download(self):
         coqa_train_filepath = 'data/coqa/coqa-train-v1.0.json'
@@ -115,7 +115,7 @@ class CoQA(Task):
             language description, as well as the few shot examples, and the question
             part of the document for `doc`. 
         """
-        cont_request = rf.greedy_until(ctx, ['\n'])
+        cont_request = rf.greedy_until(ctx, ['\nQ:'])
         return cont_request
 
     def process_results(self, doc, results):
@@ -130,7 +130,7 @@ class CoQA(Task):
         """
         turn_id = len(doc["questions"])
         gold_list = self.get_answers(doc, turn_id)
-        pred = results[0]
+        pred = results[0].strip().split('\n')[0]
 
         scores = self.compute_scores(gold_list, pred)
 

tests/testdata/coqa-v1-greedy_until

+57581470b921435d40da97872bb1cfda6ecf963ccc4b0240a3b04e3fea8c8e3a
\ No newline at end of file

tests/testdata/coqa-v1-res.json

+{"results": {"coqa": {"em": 0.0, "em_stderr": 0.0, "f1": 0.0, "f1_stderr": 0.0}}, "versions": {"coqa": 1}}
\ No newline at end of file