Fix some things for tests

lm_eval/base.py

@@ -351,7 +351,7 @@ class PerplexityTask(Task, abc.ABC):
 req_ret_lens = {
     'loglikelihood': 2,
     'greedy_until': None,
-    'loglikelihood_perplexity': 1,
+    'loglikelihood_perplexity': None,
 }
 
 import os

lm_eval/tasks/pile.py

@@ -16,6 +16,7 @@ class PilePerplexityTask(PerplexityTask, abc.ABC):
     TEST_PATH = 'data/pile/test.jsonl.zst'
 
     def download(self):
+        # TODO: separate pile val/test out by component so we don't have to scan the entire file once per set
         os.makedirs("data/pile/", exist_ok=True)
         if not os.path.exists(self.VAL_PATH):
             download_file("https://the-eye.eu/public/AI/pile/val.jsonl.zst", self.VAL_PATH)

tests/test_evaluator.py

@@ -35,7 +35,7 @@ def test_evaluator(taskname, Task):
         res = []
         random.seed(42)
         for _ in reqs:
-            res.append((-random.random(),))
+            res.append(-random.random())
 
         return res