update task_id to be updateable and uses group:task format

lm_eval/api/task.py

@@ -134,8 +134,8 @@ class ConfigurableGroup(abc.ABC):
         config: Optional[dict] = None,
     ) -> None:
         # Create a unique identifier ID
-        self._task_id = shortuuid.uuid()[:8]
         self._config = GroupConfig(**config)
+        self._task_id = self._config.group
 
     @property
     def group(self):
@@ -155,7 +155,11 @@ class ConfigurableGroup(abc.ABC):
 
     @property
     def task_id(self) -> Any:
-        return "-".join((self.group_name, self._task_id))
+        return self._task_id
+
+    @task_id.setter
+    def task_id(self, value):
+        self._task_id = value
 
     @property
     def group_name(self) -> Any:
@@ -804,6 +808,10 @@ class Task(abc.ABC):
     def task_id(self) -> Any:
         return self._task_id
 
+    @task_id.setter
+    def task_id(self, value):
+        self._task_id = value
+
 
 class ConfigurableTask(Task):
     VERSION = "Yaml"
@@ -1644,10 +1652,6 @@ class ConfigurableTask(Task):
     def get_config(self, key: str) -> Any:
         return getattr(self._config, key, None)
 
-    @property
-    def task_id(self) -> Any:
-        return "-".join((self.task_name, self._task_id))
-
     @property
     def task_name(self) -> Any:
         return getattr(self.config, "task", None)

lm_eval/evaluator_utils.py

@@ -128,13 +128,19 @@ class TaskOutput:
         )
 
 
-def get_task_list(task_dict: dict) -> List[TaskOutput]:
+def get_task_list(task_dict: dict, task_root=None) -> List[TaskOutput]:
     outputs = []
     for task_name, task_obj in task_dict.items():
+        if isinstance(task_name, str):
+            prefix_name = task_name
+        else:
+            prefix_name = task_name.task_id
+
         if isinstance(task_obj, dict):
-            _outputs = get_task_list(task_obj)
+            _outputs = get_task_list(task_obj, task_root=prefix_name)
             outputs.extend(_outputs)
         else:
+            task_obj.task_id = f"{task_root}:{task_obj.task_id}"
             task_output = TaskOutput.from_taskdict(task_name, task_obj)
             outputs.append(task_output)
 

lm_eval/tasks/__init__.py

@@ -153,6 +153,10 @@ class TaskManager:
                 task_object = config["class"](config=config)
             else:
                 task_object = ConfigurableTask(config=config)
+
+            if task != task_object.task_id:
+                task_object.task_id = task
+
             return {task: task_object}
 
         def _get_group_and_subtask_from_config(config):