Shuffle order of docs in evaluator

lm_eval/evaluator.py

 import collections
 import itertools
+import random
 
 
 def evaluate(lm, task_dict, provide_description, num_fewshot, limit):
@@ -29,7 +30,13 @@ def evaluate(lm, task_dict, provide_description, num_fewshot, limit):
         elif task.has_test_docs():
             task_doc_func = task.test_docs
 
-        for doc_id, doc in enumerate(itertools.islice(task_doc_func(), 0, limit)):
+        # deterministically shuffle docs and chop off the first `limit` because sometimes docs are in some kind of order
+        task_docs = list(task_doc_func())
+        rnd = random.Random()
+        rnd.seed(42)
+        rnd.shuffle(docs)
+
+        for doc_id, doc in enumerate(itertools.islice(task_docs, 0, limit)):
             docs[(task_name, doc_id)] = doc
 
             ctx = task.fewshot_context(
@@ -40,7 +47,7 @@ def evaluate(lm, task_dict, provide_description, num_fewshot, limit):
 
             reqs = task.construct_requests(doc, ctx)
             if not isinstance(reqs, (list, tuple)): reqs = [reqs]
-
+            
             for i, req in enumerate(reqs):
                 requests[req.type].append(req)
                 # i: index in requests for a single task instance