Fix for https://github.com/EleutherAI/lm-evaluation-harness/issues/1383 (#1384)

Fixes https://github.com/EleutherAI/lm-evaluation-harness/issues/1383

If this is okay, it will need to be propagated to SCROLLS

lm_eval/tasks/squadv2/task.py

@@ -19,7 +19,7 @@ from math import exp
 from functools import partial
 from packaging import version
 
-from lm_eval.api.task import Task
+from lm_eval.api.task import ConfigurableTask
 from lm_eval.api.instance import Instance
 
 _CITATION = """
@@ -46,11 +46,14 @@ def _squad_agg(key, items):
     return _squad_metric(predictions=predictions, references=references).get(key, 0)
 
 
-class SQuAD2(Task):
+class SQuAD2(ConfigurableTask):
     VERSION = 3
     DATASET_PATH = "squad_v2"
     DATASET_NAME = None
 
+    def __init__(self):
+        super().__init__(config={'metadata': {'version': self.VERSION}})
+
     # HF changed squad on us so we have to make sure we aren't running the old one
     assert version.parse(datasets.__version__) >= version.parse(
         "1.11.0"