Change to test before val

lm_eval/evaluator.py

@@ -23,12 +23,12 @@ def evaluate(lm, task_dict, provide_description, num_fewshot, limit):
 
     # get lists of each type of requeste
     for task_name, task in task_dict_items:
-        #default to validation doc, fall back to test doc if validation unavailable
-        # TODO: the val-fallback-to-test system isn't final, we should revisit it at some point
-        if task.has_validation_docs():
-            task_doc_func = task.validation_docs
-        elif task.has_test_docs():
+        #default to test doc, fall back to val doc if validation unavailable
+        # TODO: the test-fallback-to-val system isn't final, we should revisit it at some point
+        if task.has_test_docs():
             task_doc_func = task.test_docs
+        elif task.has_validation_docs():
+            task_doc_func = task.validation_docs
 
         # deterministically shuffle docs and chop off the first `limit` because sometimes docs are in some kind of order
         task_docs = list(task_doc_func())