Merge pull request #234 from EleutherAI/race_fix

Pin datasets to 1.15.1 to fix RACE

lm_eval/tasks/race.py

@@ -22,6 +22,8 @@ class RACE(HFTask):
     cache = {}
     letter_to_num = {'A': 0, 'B': 1, 'C': 2, 'D': 3}
 
+    assert datasets.__version__ == "1.15.1", "RACE requires datasets==1.15.1!"
+
     def has_training_docs(self):
         return True
 

setup.py

@@ -22,7 +22,7 @@ setuptools.setup(
     install_requires=[
         "black==20.8b1",
         "best_download>=0.0.6",
-        "datasets>=1.2.1",
+        "datasets==1.15.1",
         "click>=7.1",
         "scikit-learn>=0.24.1",
         "torch>=1.7",