added initialize_task and updated where eval_logger is imported from

lm_eval/__main__.py

@@ -9,12 +9,9 @@ import numpy as np
 from pathlib import Path
 from typing import Union
 
-logging.basicConfig(
-    format="%(asctime)s,%(msecs)03d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s",
-    datefmt="%Y-%m-%d:%H:%M:%S",
-    level=logging.INFO,
-)
-
+from lm_eval import evaluator, utils
+from lm_eval.tasks import initialize_tasks, include_path
+from lm_eval.api.registry import ALL_TASKS
 
 def _handle_non_serializable(o):
     if isinstance(o, np.int64) or isinstance(o, np.int32):
@@ -121,14 +118,12 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
         # we allow for args to be passed externally, else we parse them ourselves
         args = parse_eval_args()
 
-    eval_logger = logging.getLogger("lm-eval")
+    eval_logger = utils.eval_logger
     eval_logger.setLevel(getattr(logging, f"{args.verbosity}"))
     eval_logger.info(f"Verbosity set to {args.verbosity}")
     os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
-    from lm_eval import evaluator, utils
-    from lm_eval.tasks import include_path
-    from lm_eval.api.registry import ALL_TASKS
+    initialize_tasks(args.verbosity)
 
     if args.limit:
         eval_logger.warning(

lm_eval/evaluator.py

@@ -20,10 +20,9 @@ from lm_eval.utils import (
     make_table,
     create_iterator,
     get_git_commit_hash,
+    eval_logger
 )
 
-eval_logger = logging.getLogger("lm-eval")
-
 
 @positional_deprecated
 def simple_evaluate(

lm_eval/models/anthropic_llms.py

@@ -2,9 +2,10 @@ from lm_eval.api.model import LM
 from lm_eval.api.registry import register_model
 from tqdm import tqdm
 import time
-from lm_eval.logger import eval_logger
+from lm_eval import utils
 from typing import List, Any, Tuple
 
+eval_logger = utils.eval_logger
 
 def anthropic_completion(
     client,  #: anthropic.Anthropic,

lm_eval/models/huggingface.py

@@ -16,7 +16,6 @@ from pathlib import Path
 import torch.nn.functional as F
 
 from lm_eval import utils
-from lm_eval.logger import eval_logger
 from lm_eval.api.model import LM
 from lm_eval.api.registry import register_model
 
@@ -25,6 +24,7 @@ from lm_eval.utils import MultiTokenEOSCriteria, stop_sequences_criteria
 from accelerate import Accelerator, find_executable_batch_size, DistributedType
 from typing import List, Optional, Union
 
+eval_logger = utils.eval_logger
 
 def _get_accelerate_args(
     device_map_option: Optional[str] = "auto",

lm_eval/prompts/__init__.py

@@ -3,7 +3,7 @@ import ast
 
 from typing import Dict
 from lm_eval import utils
-from lm_eval.logger import eval_logger
+from lm_eval.utils import eval_logger
 
 # Prompt library.
 # Stores prompts in a dictionary indexed by 2 levels:

lm_eval/tasks/__init__.py

@@ -14,8 +14,7 @@ from lm_eval.api.registry import (
 )
 
 import logging
-
-eval_logger = logging.getLogger("lm-eval")
+eval_logger = utils.eval_logger
 
 
 def register_configurable_task(config: Dict[str, str]) -> int:
@@ -168,8 +167,12 @@ def include_path(task_dir):
     return 0
 
 
-task_dir = os.path.dirname(os.path.abspath(__file__)) + "/"
-include_path(task_dir)
+def initialize_tasks(verbosity="INFO"):
+
+    eval_logger.setLevel(getattr(logging, f"{verbosity}"))
+
+    task_dir = os.path.dirname(os.path.abspath(__file__)) + "/"
+    include_path(task_dir)
 
 
 def get_task(task_name, config):

lm_eval/tasks/minerva_math/utils.py

 import datasets
 import re
 import signal
-from lm_eval.logger import eval_logger
+from lm_eval.utils import eval_logger
 from typing import Optional, List, Dict
 
 try:

lm_eval/tasks/realtoxicityprompts/metric.py

@@ -3,7 +3,7 @@ import json
 import requests
 import numpy as np
 
-from lm_eval.logger import eval_logger
+from lm_eval.utils import eval_logger
 
 
 def toxicity_perspective_api(references, predictions, **kwargs):

lm_eval/utils.py

@@ -21,8 +21,14 @@ from itertools import islice
 
 import logging
 
+logging.basicConfig(
+    format="%(asctime)s,%(msecs)03d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s",
+    datefmt="%Y-%m-%d:%H:%M:%S",
+    level=logging.INFO,
+)
 eval_logger = logging.getLogger("lm-eval")
 
+SPACING = " " * 47
 
 def escaped_split(text, sep_char, maxsplit=-1):
     """Split text into a list on occurrences of the given separation