fix tests

lm_eval/api/task.py

@@ -1215,7 +1215,7 @@ class ConfigurableTask(Task):
                 arguments = [(ctx, f"{target_delimiter}{cont}") for cont in choices]
 
             # TODO: we should raise a warning telling users this will at most ~2x runtime.
-            if "acc_mutual_info" in [m.metric_name for m in self.metric_list]:
+            if "acc_mutual_info" in [m.metric_name for m in self.config._metric_list]:
                 # if we are calculating multiple choice accuracy
                 # using mutual information instead of raw loglikelihood as metric, need unconditional lls.
 

lm_eval/config/task.py

@@ -120,7 +120,7 @@ class FewshotConfig:
 @dataclass
 class TaskConfig(dict):
     # task naming/registry
-    task: str
+    task: Optional[str] = None
     task_alias: Optional[str] = None
     tag: Optional[Union[str, list]] = None
     # HF dataset options.

tests/test_tasks.py

@@ -46,8 +46,12 @@ def limit() -> int:
     return 10
 
 
-# Tests
-class BaseTasks:
+@pytest.mark.parametrize(
+    "task_class",
+    task_class(get_new_tasks_else_default()),
+    ids=lambda x: f"{x.config.task}",
+)
+class TestBaseTasks:
     """
     Base class for testing tasks
     """
@@ -161,7 +165,7 @@ class BaseTasks:
     task_class(get_new_tasks_else_default()),
     ids=lambda x: f"{x.config.task}",
 )
-class TestNewTasksElseDefault(BaseTasks):
+class TestNewTasksElseDefault(TestBaseTasks):
     """
     Test class parameterized with a list of new/modified tasks
     (or a set of default tasks if none have been modified)
@@ -175,7 +179,7 @@ class TestNewTasksElseDefault(BaseTasks):
     ),
     ids=lambda x: f"{x.config.task}",
 )
-class TestUnitxtTasks(BaseTasks):
+class TestUnitxtTasks(TestBaseTasks):
     """
     Test class for Unitxt tasks parameterized with a small custom
     task as described here: