error set to DEBUG

lm_eval/__main__.py

@@ -98,9 +98,9 @@ def parse_eval_args() -> argparse.Namespace:
         help="Additional path to include if there are external tasks to include.",
     )
     parser.add_argument(
-        "--verbose",
-        type=bool,
-        default=False,
+        "--verbosity",
+        type=str,
+        default="INFO",
         help="Log error when tasks are not registered.",
     )
     return parser.parse_args()
@@ -112,6 +112,7 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
         # we allow for args to be passed externally, else we parse them ourselves
         args = parse_eval_args()
 
+    eval_logger.setLevel(getattr(logging, f"{args.verbosity}"))
     os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
     if args.limit:
@@ -173,7 +174,6 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
         assert args.output_path, "Specify --output_path"
 
     eval_logger.info(f"Selected Tasks: {task_names}")
-    eval_logger.verbose = args.verbose
 
     results = evaluator.simple_evaluate(
         model=args.model,

lm_eval/tasks/__init__.py

@@ -4,7 +4,7 @@ from typing import List, Union, Dict
 
 from lm_eval import utils
 from lm_eval import prompts
-from lm_eval.logger import eval_logger
+# from lm_eval.logger import eval_logger
 from lm_eval.api.task import TaskConfig, Task, ConfigurableTask
 from lm_eval.api.registry import (
     register_task,
@@ -14,6 +14,9 @@ from lm_eval.api.registry import (
     ALL_TASKS,
 )
 
+import logging
+
+eval_logger = logging.getLogger('lm-eval')
 
 def register_configurable_task(config: Dict[str, str]) -> int:
     SubClass = type(
@@ -139,18 +142,15 @@ def include_task_folder(task_dir: str, register_task: bool = True) -> None:
                                 register_configurable_group(config, yaml_path)
 
                 except Exception as error:
-                    if eval_logger.verbose:
-                        import traceback
-
-                        eval_logger.warning(
-                            "Failed to load config in\n"
-                            f"                                 {yaml_path}\n"
-                            "                                 Config will not be added to registry\n"
-                            f"                                 Error: {error}\n"
-                            f"                                 Traceback: {traceback.format_exc()}"
-                        )
-                    else:
-                        eval_logger.warning("Yaml failed to register {yaml_path}\n")
+                    import traceback
+
+                    eval_logger.debug(
+                        "Failed to load config in\n"
+                        f"                                 {yaml_path}\n"
+                        "                                 Config will not be added to registry\n"
+                        f"                                 Error: {error}\n"
+                        f"                                 Traceback: {traceback.format_exc()}"
+                    )
     return 0