remove seeding

lm_eval/base.py

@@ -80,7 +80,6 @@ class Dataset(abc.ABC):
         traindocs = list(self.training_docs())
         random.seed(123)
         random.shuffle(traindocs)
-
         return traindocs[:k]
 
     @abc.abstractmethod
@@ -107,7 +106,8 @@ class Dataset(abc.ABC):
         return ""
 
     def fewshot_context(self, doc, num_fewshot, provide_description):
-        description = (self.fewshot_description() + "\n\n") if provide_description else ""
+        raw_description = self.fewshot_description()
+        description = (raw_description + "\n\n") if provide_description and raw_description else ""
         labeled_examples = "\n\n".join(
             map(self.doc_to_text, self.fewshot_examples(k=num_fewshot))
         ) + "\n\n"

lm_eval/tasks/common.py

-import abc
 import nlp
 import numpy as np
+import random
 from ..base import Dataset
 
 
@@ -23,6 +23,12 @@ class NLP_TASK(Dataset):
         if self.has_test_docs():
             return self._load_nlp_dataset()["test"]
 
+    def fewshot_examples(self, k):
+        training_docs = self.training_docs()
+        n = len(training_docs)
+        indices = random.sample(range(n), k)
+        return [training_docs[i] for i in indices]
+
 
 def simple_accuracy_metric(preds, golds):
     acc = float((np.array(preds) == np.array(golds)).mean())

lm_eval/tasks/glue.py

@@ -177,7 +177,7 @@ class RTE(NLP_TASK):
             doc["sentence2"],
         )
         if include_target:
-            text += " {}".format({1: "True", 0: "False"}[doc["label"]])
+            text += " {}".format({0: "True", 1: "False"}[doc["label"]])
         return text
 
     def evaluate(self, docs, lm, provide_description, num_fewshot):
@@ -189,7 +189,7 @@ class RTE(NLP_TASK):
                 provide_description=provide_description,
                 num_fewshot=num_fewshot,
             )
-            preds.append(lm.loglikelihood(ctx, ' True') > lm.loglikelihood(ctx, ' False'))
+            preds.append(lm.loglikelihood(ctx, ' False') > lm.loglikelihood(ctx, ' True'))
         return simple_accuracy_metric(preds=preds, golds=golds)
 
 

lm_eval/tasks/superglue.py

@@ -4,7 +4,7 @@ from . import TASK_REGISTRY
 
 @TASK_REGISTRY.register("boolq")
 class BoolQ(NLP_TASK):
-    NLP_PATH = "superglue"
+    NLP_PATH = "super_glue"
     NLP_NAME = "boolq"
 
     def has_training_docs(self):
@@ -21,10 +21,10 @@ class BoolQ(NLP_TASK):
 
     def doc_to_text(self, doc, include_target=True):
         return f"{doc['passage']}\nquestion: {doc['question']}\nanswer: " \
-            + (yesno(doc['answer']) if include_target else "")
+            + (yesno(doc['label']) if include_target else "")
 
     def evaluate(self, docs, lm, provide_description, num_fewshot):
-        golds = [doc["answer"] for doc in docs]
+        golds = [doc["label"] for doc in docs]
         preds = []
         for doc in docs:
             ctx = self.fewshot_context(