fixed issues related to printing alias of group and updated yaml

lm_eval/evaluator_utils.py

@@ -164,7 +164,10 @@ def get_sample_size(task, limit: Optional[int]) -> Union[int, None]:
 
 
 def prepare_print_tasks(
-    task_hierarchy: dict, results: dict, tab=0
+    task_hierarchy: dict,
+    results: dict,
+    tab=0,
+    group_tab=0,
 ) -> Tuple[dict, dict]:
     """
     @param task_hierarchy: Dictionary representing the group hierarchy of tasks. Each key is a group name and its
@@ -197,17 +200,20 @@ def prepare_print_tasks(
         results_agg[group_name]["alias"] = tab_string + group_name
 
     if len(task_list) > 0:
+        if " " not in results[group_name]:
+            group_tab_string = " " * group_tab + "- " if group_tab > 0 else ""
             groups_agg[group_name] = results[group_name].copy()
-        # groups_agg[group_name]["tab"] = tab
+            group_tab += 1
+
             if "samples" in groups_agg[group_name]:
                 groups_agg[group_name].pop("samples")
 
             if "alias" in groups_agg[group_name]:
                 groups_agg[group_name]["alias"] = (
-                tab_string + groups_agg[group_name]["alias"]
+                    group_tab_string + groups_agg[group_name]["alias"]
                 )
             else:
-            groups_agg[group_name]["alias"] = tab_string + group_name
+                groups_agg[group_name]["alias"] = group_tab_string + group_name
 
         for task_name in task_list:
             if task_name in task_hierarchy:
@@ -222,7 +228,7 @@ def prepare_print_tasks(
                 }
 
             _results_agg, _groups_agg = prepare_print_tasks(
-                _task_hierarchy, results, tab + 1
+                _task_hierarchy, results, tab + 1, group_tab
             )
             results_agg = {**results_agg, **_results_agg}
             groups_agg = {**groups_agg, **_groups_agg}

lm_eval/tasks/__init__.py

@@ -178,7 +178,6 @@ class TaskManager:
                     yaml_path = self._get_yaml_path(group_name)
 
                     if (update_config is not None) and ("group_alias" in update_config):
-                        group_name = update_config["group_alias"]
                         update_config.pop("group_alias")
 
         if isinstance(name_or_config, dict):
@@ -240,8 +239,9 @@ class TaskManager:
 
         all_subtasks = {}
         if parent_name is not None:
-            # all_subtasks = {group_name: (parent_name, None)}
             parent_group_config = self._get_config(parent_name)
+            if "group_alias" in parent_group_config:
+                parent_name = parent_group_config["group_alias"]
             all_subtasks = {group_name: (parent_name, parent_group_config)}
 
         fn = partial(

lm_eval/tasks/benchmarks/flan/flan_held_in.yaml

@@ -4,6 +4,8 @@ task:
   # ANLI R1
   - group: anli_r1_flan
     group_alias: ANLI R1
+    group_config:
+      aggregate_metric: True
     task:
       - task: anli_r1
         task_alias: prompt-0
@@ -53,6 +55,8 @@ task:
   # ANLI R2
   - group: anli_r2_flan
     group_alias: ANLI R2
+    group_config:
+      aggregate_metric: True
     task:
       - task: anli_r2
         task_alias: prompt-0
@@ -102,6 +106,8 @@ task:
   # ANLI R3
   - group: anli_r3_flan
     group_alias: ANLI R3
+    group_config:
+      aggregate_metric: True
     task:
       - task: anli_r3
         task_alias: prompt-0
@@ -151,6 +157,8 @@ task:
   # Arc Easy
   - group: arc_easy_flan
     group_alias: Arc Easy
+    group_config:
+      aggregate_metric: True
     task:
       - task: arc_easy
         task_alias: prompt-0
@@ -190,6 +198,8 @@ task:
   # Arc Challenge
   - group: arc_challenge_flan
     group_alias: Arc Challenge
+    group_config:
+      aggregate_metric: True
     task:
       - task: arc_challenge
         task_alias: prompt-0
@@ -229,6 +239,8 @@ task:
   # BoolQ
   - group: boolq_flan
     group_alias: BoolQ
+    group_config:
+      aggregate_metric: True
     task:
       - task: boolq
         task_alias: prompt-0
@@ -283,6 +295,8 @@ task:
   # RTE
   - group: rte_flan
     group_alias: RTE
+    group_config:
+      aggregate_metric: True
     task:
       - task: rte
         task_alias: prompt-0