can include paths in main

lm_eval/api/task.py

@@ -70,6 +70,7 @@ class TaskConfig(dict):
     doc_to_target: Union[Callable, str] = None
     doc_to_choice: Union[Callable, str, dict, list] = None
     gold_alias: Union[Callable, str] = None
+    process_results: Union[Callable, str] = None
     use_prompt: str = None
     description: str = ""
     target_delimiter: str = " "
@@ -884,8 +885,8 @@ class ConfigurableTask(Task):
 
     def process_results(self, doc, results):
 
-        # if callable(self._config.process_results):
-        #     return self._config.process_results(doc, results)
+        if callable(self._config.process_results):
+            return self._config.process_results(doc, results)
 
         result_dict = {}
         use_metric = list(self._metric_fn_list.keys())

main.py

@@ -9,6 +9,7 @@ import logging
 from lm_eval import evaluator, utils
 from lm_eval.api.registry import ALL_TASKS
 from lm_eval.logger import eval_logger
+from lm_eval.tasks import include_task_folder
 
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
@@ -30,6 +31,7 @@ def parse_args():
         help="Maximal batch size to try with --batch_size auto",
     )
     parser.add_argument("--device", type=str, default=None)
+    parser.add_argument("--include_path", default=None)
     parser.add_argument("--output_path", default=None)
     parser.add_argument(
         "--limit",
@@ -56,6 +58,10 @@ def main():
             "REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT."
         )
 
+    if args.include_path is not None:
+        eval_logger.info(f"Including path: {args.include_path}")
+        include_task_folder(args.include_path)
+
     if args.tasks is None:
         task_names = ALL_TASKS
     else: