Rename task

lm_eval/tasks/__init__.py

@@ -107,7 +107,7 @@ TASK_REGISTRY = {
     "hellaswag": hellaswag.HellaSwag, # not implemented yet
     "openbookqa": openbookqa.OpenBookQA,
     # "sat": sat.SATAnalogies, # not implemented yet
-    "squad": squad.SQuAD,
+    "squad2": squad.SQuAD2,
     "race": race.RACE,
     # "naturalqs": naturalqs.NaturalQs, # not implemented yet
     "headqa": headqa.HeadQA,

lm_eval/tasks/squad.py

@@ -17,7 +17,8 @@ def _squad_agg(key, items):
     return _squad_metric(predictions=predictions, references=references)[key]
 
 
-class SQuAD(HFTask):
+
+class SQuAD2(HFTask):
     DATASET_PATH = "squad_v2"
     DATASET_NAME = None