Merge branch 'group-agg-rework' of...

Merge branch 'group-agg-rework' of https://github.com/EleutherAI/lm-evaluation-harness into group-agg-rework

lm_eval/evaluator.py

@@ -541,15 +541,13 @@ def evaluate(
                     task_hierarchy = {}
 
                 for group_or_task, group_or_task_info in task_dict.items():
+                    
+                    # Convert to string
                     if isinstance(group_or_task, ConfigurableGroup):
                         group_config = group_or_task.config
                         group_or_task = group_or_task.group
-                        show_group_table = (
-                            show_group_table | group_config["aggregate_metric"]
-                        )
-                        if group_config["aggregate_metric"] is False:
-                            results[group_or_task][" "] = " "
-                            continue
+                    else:
+                        group_config = None
 
                     if isinstance(group_or_task_info, ConfigurableTask):
                         if task_root:
@@ -570,6 +568,14 @@ def evaluate(
                                 task_hierarchy.get(group_or_task, [])
                             )
 
+                        if (group_config is not None) and (group_config["aggregate_metric"] is False):
+                            results[group_or_task][" "] = " "
+                            continue
+
+                        show_group_table = (
+                            show_group_table | group_config["aggregate_metric"]
+                        )
+
                         task_list = _task_hierarchy[group_or_task]
                         metric_list = list(
                             {

lm_eval/tasks/__init__.py

@@ -192,7 +192,9 @@ class TaskManager:
                     }
                     subtask_list = self._get_tasklist(name)
                     if subtask_list == -1:
-                        subtask_list = self._get_config(name)["task"]
+                        group_config = self._get_config(name)
+                        subtask_list = group_config["task"]
+                    group_name = ConfigurableGroup(config=group_config)
                 else:
                     if self._name_is_registered(name):
                         base_task_config = self._get_config(name)

lm_eval/tasks/truthfulqa/_truthfulqa.yaml

+group: truthfulqa
+group_alias: Truthful QA
+task:
+  - truthfulqa_gen
+  - truthfulqa_mc1
+  - truthfulqa_mc2

lm_eval/tasks/truthfulqa/truthfulqa_gen.yaml

-group:
-  - truthfulqa
 task: truthfulqa_gen
 dataset_path: truthful_qa
 dataset_name: generation

lm_eval/tasks/truthfulqa/truthfulqa_mc1.yaml

-group:
-  - truthfulqa
 task: truthfulqa_mc1
 dataset_path: truthful_qa
 dataset_name: multiple_choice