wsc273: change random

d42af30d · Leo Gao · GitHub · dc1a17b1 · d42af30d
Unverified Commit d42af30d authored Mar 25, 2021 by Leo Gao Committed by GitHub Mar 25, 2021
Hide whitespace changes
Inline Side-by-side

Showing with 3 additions and 1 deletion

lm_eval/tasks/wsc273.py lm_eval/tasks/wsc273.py +3 -1

No files found.
--- a/lm_eval/tasks/wsc273.py
+++ b/lm_eval/tasks/wsc273.py
@@ -59,7 +59,9 @@ class WinogradSchemaChallenge273(HFTask):
    def fewshot_examples(self, k):
        # NOTE: `super().fewshot_examples` samples from training docs which are
        # not available for this test-set-only dataset.
-        return random.sample(list(self.test_docs()), k)
+        rnd = random.Random()
+        rnd.seed(42)
+        return rnd.sample(list(self.test_docs()), k)

    def doc_to_text(self, doc):
        return self.partial_context(doc, doc["options"][doc["label"]])