Merge pull request #206 from jon-tow/mutual-fix

Fix `MuTual` loglikelihood request bug

lm_eval/tasks/mutual.py

@@ -21,7 +21,7 @@ from best_download import download_file
 
 
 class MuTualBase(Task):
-    VERSION = 0
+    VERSION = 1
     BASE_PATH = Path("data/mutual")
     DATASET_NAME = None
     CHOICES = ['A', 'B', 'C', 'D']
@@ -55,7 +55,7 @@ class MuTualBase(Task):
         return False
 
     def _load_docs(self, path):
-        for file in path.iterdir():
+        for file in sorted(path.iterdir()):
             if file.suffix != ".txt":
                 continue
             with open(file, 'r', encoding='utf-8') as f:
@@ -83,7 +83,7 @@ class MuTualBase(Task):
     def construct_requests(self, doc, ctx):
         lls = []
         for option in doc["options"]:
-            lls.append(rf.loglikelihood(ctx, f" {self.detokenize(option)}"))
+            lls.append(rf.loglikelihood(ctx, f" {self.detokenize(option)}")[0])
         return lls
 
     def detokenize(self, text):

tests/testdata/mutual-v1-loglikelihood

+f759213a28f0412510bf1a24c9cab0dae64bdee902d42a26225295445e7779db
\ No newline at end of file

tests/testdata/mutual-v1-res.json

+{"results": {"mutual": {"mrr": 0.5023513920240772, "mrr_stderr": 0.009501864812936679, "r@1": 0.22460496613995484, "r@1_stderr": 0.014028122493992806, "r@2": 0.4706546275395034, "r@2_stderr": 0.016778343895001414}}, "versions": {"mutual": 1}}
\ No newline at end of file

tests/testdata/mutual_plus-v1-loglikelihood

+b846bb9db109535f59a93d1ce340cf09f68bdf4fed5b8decd168784220fe07fa
\ No newline at end of file

tests/testdata/mutual_plus-v1-res.json

+{"results": {"mutual_plus": {"mrr": 0.5275583145221953, "mrr_stderr": 0.009940894824430708, "r@1": 0.26297968397291194, "r@1_stderr": 0.01479889176605113, "r@2": 0.5, "r@2_stderr": 0.01680731613632036}}, "versions": {"mutual_plus": 1}}
\ No newline at end of file