Deterministic fewshot_experiments

6a78fdaf · Leo Gao · GitHub · 283b2c5e · 6a78fdaf
Unverified Commit 6a78fdaf authored Mar 25, 2021 by Leo Gao Committed by GitHub Mar 25, 2021
Hide whitespace changes
Inline Side-by-side

Showing with 3 additions and 1 deletion

lm_eval/base.py lm_eval/base.py +3 -1

No files found.
--- a/lm_eval/base.py
+++ b/lm_eval/base.py
@@ -117,7 +117,9 @@ class Task(abc.ABC):
    def fewshot_examples(self, k):
        if self._training_docs is None:
            self._training_docs = list(self.training_docs())
-        return random.sample(self._training_docs, k)
+        rnd = random.Random()
+        rnd.seed(42)
+        return rnd.sample(self._training_docs, k)

    @abc.abstractmethod
    def doc_to_text(self, doc):