format fix

lm_eval/evaluator.py

@@ -584,7 +584,6 @@ def evaluate(
                     task_aggregation_list = {}
 
                 for group_or_task, group_or_task_info in task_dict.items():
-
                     # Convert to string
                     if isinstance(group_or_task, ConfigurableGroup):
                         group_config = group_or_task.config

lm_eval/evaluator_utils.py

@@ -232,13 +232,13 @@ def prepare_print_tasks(
     for task_or_group_name, task_or_group_obj in task_dict.items():
         tab_string = " " * task_depth + "- " if task_depth > 0 else ""
         if isinstance(task_or_group_name, ConfigurableGroup):
-            string_name = task_or_group_name.group_name
+            # string_name = task_or_group_name.group_name
             name = task_or_group_name.task_id
             from_configurable_group = True
         elif isinstance(task_or_group_name, str):
             name = task_or_group_name
             if isinstance(task_or_group_obj, ConfigurableTask):
-                string_name = task_or_group_obj.task_name
+                # string_name = task_or_group_obj.task_name
                 name = task_or_group_obj.task_id
             from_configurable_group = False
 

tests/testyamls/test-01.yaml

@@ -3,12 +3,12 @@ group_alias: test 1
 task:
   - piqa # string task
   - ai2_arc # string tag
-  # - task: super-glue-lm-eval-v1 # Should this be spread out?
-  #   num_fewshot: 3
+  - task: super-glue-lm-eval-v1 # Should this be spread out?
+    num_fewshot: 3
   - task: swag # dict registered task
     num_fewshot: 2
-  # - task: mmlu
-  #   num_fewshot: 5
+  - task: mmlu
+    num_fewshot: 5
   - group: nli-tasks # dict group
     task:
       - anli
@@ -17,31 +17,29 @@ task:
     num_fewshot: 4
     metric_list:
       - metric: brier_score
-    aggregate_metric: true
-    
-  # - task: sciq # dict registered task duplicate
-  #   task_alias: sciq 2-shot
-  #   num_fewshot: 2
-  # - task: sciq # dict registered task duplicate
-  #   task_alias: sciq 4-shot
-  #   num_fewshot: 4
-  # - task: sciq # dict registered task duplicate
-  #   task_alias: sciq 6-shot
-  #   num_fewshot: 6
-  # - task: siqa_custom # dict task
-  #   dataset_path: social_i_qa
-  #   dataset_name: null
-  #   output_type: multiple_choice
-  #   training_split: train
-  #   validation_split: validation
-  #   doc_to_text: "Question: {{context}} {{question}}\nAnswer:"
-  #   target_delimiter: " "
-  #   doc_to_choice:
-  #     - "{{answerA}}"
-  #     - "{{answerB}}"
-  #     - "{{answerC}}"
-  #   doc_to_target: "{{ (label|int) - 1 }}"
-  #   metric_list:
-  #     - metric: acc
-  #       aggregation: mean
-  #       higher_is_better: true
+  - task: sciq # dict registered task duplicate
+    task_alias: sciq 2-shot
+    num_fewshot: 2
+  - task: sciq # dict registered task duplicate
+    task_alias: sciq 4-shot
+    num_fewshot: 4
+  - task: sciq # dict registered task duplicate
+    task_alias: sciq 6-shot
+    num_fewshot: 6
+  - task: siqa_custom # dict task
+    dataset_path: social_i_qa
+    dataset_name: null
+    output_type: multiple_choice
+    training_split: train
+    validation_split: validation
+    doc_to_text: "Question: {{context}} {{question}}\nAnswer:"
+    target_delimiter: " "
+    doc_to_choice:
+      - "{{answerA}}"
+      - "{{answerB}}"
+      - "{{answerC}}"
+    doc_to_target: "{{ (label|int) - 1 }}"
+    metric_list:
+      - metric: acc
+        aggregation: mean
+        higher_is_better: true