Fix training docs caching

lm_eval/tasks/arc.py

@@ -42,8 +42,8 @@ class ARCEasy(MultipleChoiceTask):
 
     def training_docs(self):
         if self._training_docs is None:
-            self._training_docs = list(self.dataset["train"])
-        return map(self._convert_standard, self._training_docs)
+            self._training_docs = list(map(self._convert_standard, self.dataset["train"]))
+        return self._training_docs
 
     def validation_docs(self):
         return map(self._convert_standard, self.dataset["validation"])

lm_eval/tasks/drop.py

@@ -52,7 +52,9 @@ class DROP(Task):
         return False
 
     def training_docs(self):
-        return map(self._convert_standard, self.dataset["train"])
+        if self._training_docs is None:
+            self._training_docs = list(map(self._convert_standard, self.dataset["train"]))
+        return self._training_docs
 
     def validation_docs(self):
         return map(self._convert_standard, self.dataset["validation"])

lm_eval/tasks/headqa.py

@@ -38,8 +38,8 @@ class HeadQABase(MultipleChoiceTask):
 
     def training_docs(self):
         if self._training_docs is None:
-            self._training_docs = list(self.dataset["train"])
-        return map(self._convert_standard, self._training_docs)
+            self._training_docs = list(map(self._convert_standard, self.dataset["train"]))
+        return self._training_docs
 
     def validation_docs(self):
         return map(self._convert_standard, self.dataset["validation"])

lm_eval/tasks/hellaswag.py

@@ -43,8 +43,8 @@ class HellaSwag(MultipleChoiceTask):
 
     def training_docs(self):
         if self._training_docs is None:
-            self._training_docs = list(self.dataset["train"])
-        return map(self._convert_standard, self._training_docs)
+            self._training_docs = list(map(self._convert_standard, self.dataset["train"]))
+        return self._training_docs
 
     def validation_docs(self):
         return map(self._convert_standard, self.dataset["validation"])

lm_eval/tasks/mathqa.py

@@ -40,8 +40,8 @@ class MathQA(MultipleChoiceTask):
 
     def training_docs(self):
         if self._training_docs is None:
-            self._training_docs = list(self.dataset["train"])
-        return map(self._convert_standard, self._training_docs)
+            self._training_docs = list(map(self._convert_standard, self.dataset["train"]))
+        return self._training_docs
 
     def validation_docs(self):
         return map(self._convert_standard, self.dataset["validation"])

lm_eval/tasks/openbookqa.py

@@ -43,8 +43,8 @@ class OpenBookQA(MultipleChoiceTask):
 
     def training_docs(self):
         if self._training_docs is None:
-            self._training_docs = list(self.dataset["train"])
-        return map(self._convert_standard, self._training_docs)
+            self._training_docs = list(map(self._convert_standard, self.dataset["train"]))
+        return self._training_docs
 
     def validation_docs(self):
         return map(self._convert_standard, self.dataset["validation"])

lm_eval/tasks/piqa.py

@@ -42,8 +42,8 @@ class PiQA(MultipleChoiceTask):
 
     def training_docs(self):
         if self._training_docs is None:
-            self._training_docs = list(self.dataset["train"])
-        return map(self._convert_standard, self._training_docs)
+            self._training_docs = list(map(self._convert_standard, self.dataset["train"]))
+        return self._training_docs
 
     def validation_docs(self):
         return map(self._convert_standard, self.dataset["validation"])

lm_eval/tasks/quac.py

@@ -40,7 +40,9 @@ class QuAC(Task):
         return False
 
     def training_docs(self):
-        return map(self._convert_standard, self.dataset["train"])
+        if self._training_docs is None:
+            self._training_docs = list(map(self._convert_standard, self.dataset["train"]))
+        return self._training_docs
 
     def validation_docs(self):
         return map(self._convert_standard, self.dataset["validation"])

lm_eval/tasks/sciq.py

@@ -38,8 +38,8 @@ class SciQ(MultipleChoiceTask):
 
     def training_docs(self):
         if self._training_docs is None:
-            self._training_docs = list(self.dataset["train"])
-        return map(self._convert_standard, self._training_docs)
+            self._training_docs = list(map(self._convert_standard, self.dataset["train"]))
+        return self._training_docs
 
     def validation_docs(self):
         return map(self._convert_standard, self.dataset["validation"])