fixes how num_fewshot from config is read

lm_eval/api/task.py

@@ -81,7 +81,7 @@ class TaskConfig(dict):
     fewshot_delimiter: str = "\n\n"
     fewshot_config: dict = None
     # runtime configuration options
-    num_fewshot: int = 0
+    num_fewshot: int = -1
     # scoring options
     metric_list: list = None
     output_type: str = "generate_until"

lm_eval/evaluator.py

@@ -138,7 +138,7 @@ def simple_evaluate(
                 eval_logger.info(
                     f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored."
                 )
-            elif config["num_fewshot"] > 0:
+            else:
                 default_num_fewshot = config["num_fewshot"]
                 eval_logger.warning(
                     f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}"
@@ -624,6 +624,9 @@ def evaluate(
             else:
                 groups_agg[group]["alias"] = tab_string + group
 
+        for group_name, task_list in task_hierarchy.items():
+            num_fewshot[group_name] = num_fewshot[task_list[0]]
+
         results_dict = {
             "results": dict(results_agg.items()),
             **({"groups": dict(groups_agg.items())} if bool(groups_agg) else {}),