fix squad breakage due to HF change

lm_eval/tasks/squad.py

@@ -4,6 +4,7 @@ from lm_eval.base import rf
 from lm_eval.metrics import f1_score, mean
 from . common import HFTask
 from functools import partial
+from packaging import version
 
 
 def _squad_metric(predictions, references):
@@ -18,10 +19,13 @@ def _squad_agg(key, items):
 
 
 class SQuAD2(HFTask):
-    VERSION = 0
+    VERSION = 1
     DATASET_PATH = "squad_v2"
     DATASET_NAME = None
 
+    # HF changed squad on us so we have to make sure we aren't running the old one
+    assert version.parse(datasets.__version__) >= version.parse("1.11.0"), "datasets v1.11.0 or later required for SQuAD"
+
     def has_training_docs(self):
         return True
 

tests/testdata/squad2-v1-greedy_until

+e17e3d85c1d5adaf2d6b4b752c4babc2e0b3a6e144e6de70cb3b2287e85109b8
\ No newline at end of file

tests/testdata/squad2-v1-loglikelihood

+f5da6173402b274dc89130755c222c6ca6b2a3bacaaa4e4ab07be9322b7bad65
\ No newline at end of file

tests/testdata/squad2-v1-res.json

+{"results": {"squad2": {"HasAns_exact": 0.0, "HasAns_f1": 0.0, "NoAns_exact": 0.0, "NoAns_f1": 0.0, "best_exact": 50.07159100480081, "best_f1": 50.07159100480081, "exact": 0.0, "f1": 0.0}}, "versions": {"squad2": 1}}
\ No newline at end of file