Use num_fewshot set in yaml and show warning if it's being overwritten by argparse

lm_eval/api/task.py

@@ -130,6 +130,9 @@ class TaskConfig(dict):
     def __getitem__(self, item):
         return getattr(self, item)
 
+    def __setitem__(self, item, value):
+        return setattr(self, item, value)
+
     def to_dict(self):
         """dumps the current config as a dictionary object, as a printable format.
         null fields will not be printed.

lm_eval/evaluator.py

@@ -35,7 +35,7 @@ def simple_evaluate(
     model,
     model_args=None,
     tasks=[],
-    num_fewshot=0,
+    num_fewshot=None,
     batch_size=None,
     max_batch_size=None,
     device=None,
@@ -112,7 +112,17 @@ def simple_evaluate(
             + "_rank" + str(lm.rank) + ".db",
         )
 
-    task_dict = lm_eval.tasks.get_task_dict(tasks, num_fewshot=num_fewshot)
+    task_dict = lm_eval.tasks.get_task_dict(tasks)
+    for task_name in task_dict.keys():
+        config = task_dict[task_name]._config
+        if num_fewshot is not None:
+            if config["num_fewshot"] > 0:
+                default_num_fewshot = config["num_fewshot"]
+                eval_logger.warning(
+                    f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}"
+                )
+
+            task_dict[task_name]._config.__setitem__("num_fewshot", num_fewshot)
 
     if check_integrity:
         run_task_tests(task_list=tasks)
@@ -134,7 +144,6 @@ def simple_evaluate(
             if isinstance(model, str)
             else model.model.config._name_or_path,
             "model_args": model_args,
-            "num_fewshot": num_fewshot,
             "batch_size": batch_size,
             "batch_sizes": list(lm.batch_sizes.values())
             if hasattr(lm, "batch_sizes")
@@ -169,8 +178,6 @@ def evaluate(
         Language Model
     :param task_dict: dict[str, Task]
         Dictionary of tasks. Tasks will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.
-    :param num_fewshot: int
-        Number of examples in few-shot context
     :param limit: int, optional
         Limit the number of examples per task (only use this for testing)
     :param bootstrap_iters:
@@ -359,6 +366,7 @@ def evaluate(
         for (task_name, key, metric), items in vals.items():
             task = task_dict[task_name]
             results[task_name][metric + "," + key] = task.aggregation()[metric](items)
+            # results[task_name]['num_fewshot'] = configs[task_name]
 
             # hotfix: bleu, chrf, ter seem to be really expensive to bootstrap
             # so we run them less iterations. still looking for a cleaner way to do this

lm_eval/utils.py

@@ -265,9 +265,19 @@ def make_table(result_dict):
 
     md_writer = MarkdownTableWriter()
     latex_writer = LatexTableWriter()
-    md_writer.headers = ["Task", "Version", "Filter", "Metric", "Value", "", "Stderr"]
+    md_writer.headers = [
+        "Task",
+        "Fewshot",
+        "Version",
+        "Filter",
+        "Metric",
+        "Value",
+        "",
+        "Stderr",
+    ]
     latex_writer.headers = [
         "Task",
+        "Fewshot",
         "Version",
         "Filter",
         "Metric",
@@ -280,6 +290,7 @@ def make_table(result_dict):
 
     for k, dic in result_dict["results"].items():
         version = result_dict["versions"][k]
+        n = str(result_dict["configs"][k]["num_fewshot"])
         for (mf), v in dic.items():
             m, _, f = mf.partition(",")
             if m.endswith("_stderr"):
@@ -287,10 +298,11 @@ def make_table(result_dict):
 
             if m + "_stderr" + "," + f in dic:
                 se = dic[m + "_stderr" + "," + f]
-                values.append([k, version, f, m, "%.4f" % v, "±", "%.4f" % se])
+                values.append([k, n, version, f, m, "%.4f" % v, "±", "%.4f" % se])
             else:
-                values.append([k, version, f, m, "%.4f" % v, "", ""])
+                values.append([k, n, version, f, m, "%.4f" % v, "", ""])
             k = ""
+            n = ""
             version = ""
     md_writer.value_matrix = values
     latex_writer.value_matrix = values

main.py

@@ -28,7 +28,7 @@ def parse_args():
     parser.add_argument(
         "--num_fewshot",
         type=int,
-        default=0,
+        default=None,
         help="Number of examples in few-shot context",
     )
     parser.add_argument("--batch_size", type=int, default=1)  # TODO: only integers