Enforce `rnd` args with assertions

lm_eval/base.py

@@ -471,13 +471,13 @@ class Task(abc.ABC):
             Not implemented, and this option is deprecated and will be removed in a future version in favor of a different description providing method
         :param rnd: random.Random
             The pseudo-random number generator used to randomly sample examples.
-            WARNING: If you do not provide a `rnd` arg, a default `random.Random`
-            object will be created and seeded with this Task's name attribute, `__name__`.
+            WARNING: This is currently a required arg although it's optionalized with a default `None`.
         :param description: str
             The task's description that will be prepended to the fewshot examples.
         :returns: str
             The fewshot context.
         """
+        assert rnd is not None, "A `random.Random` generator argument must be provided to `rnd`"
         assert not provide_description, (
             "The `provide_description` arg will be removed in future versions. To prepend "
             "a custom description to the context, supply the corresponding string via the "
@@ -489,11 +489,6 @@ class Task(abc.ABC):
 
         description = description + "\n\n" if description else ""
 
-        # TODO (jon-tow): Remove this default `rand` behaviour after `provide_description` is removed and remove the respective `rand` arg warning in the docs above.
-        if rnd is None:
-            rnd = random.Random()
-            rnd.seed(self.__name__)
-
         if num_fewshot == 0:
             labeled_examples = ""
         else:
@@ -567,6 +562,7 @@ class PerplexityTask(Task, abc.ABC):
 
     def fewshot_context(self, doc, num_fewshot, provide_description=None, rnd=None, description=None):
         assert num_fewshot == 0
+        assert rnd is not None, "A `random.Random` generator argument must be provided to `rnd`"
         assert not provide_description, (
             "The `provide_description` arg will be removed in future versions. To prepend "
             "a custom description to the context, supply the corresponding string via the  "
@@ -576,11 +572,6 @@ class PerplexityTask(Task, abc.ABC):
             # nudge people to not specify it at all
             print("WARNING: provide_description is deprecated and will be removed in a future version in favor of description_dict")
 
-        # TODO (jon-tow): Remove this default `rand` behaviour after `provide_description` is removed and remove the respective `rand` arg warning in the docs above.
-        if rnd is None:
-            rnd = random.Random()
-            rnd.seed(self.__name__)
-
         return ""
 
     def higher_is_better(self):