Merge branch 'recursive-groups' of...

Merge branch 'recursive-groups' of https://github.com/EleutherAI/lm-evaluation-harness into t5v2-alt-plus

lm_eval/tasks/__init__.py

@@ -96,7 +96,7 @@ class TaskManager(abc.ABC):
     def _get_config(self, name):
         assert name in self.ALL_TASKS
         yaml_path = self._get_yaml_path(name)
-        return utils.load_yaml_config(yaml_path)
+        return utils.load_yaml_config("full", yaml_path)
 
     def _get_tasklist(self, name):
         assert self._name_is_task(name) == False
@@ -196,7 +196,7 @@ class TaskManager(abc.ABC):
             for f in file_list:
                 if f.endswith(".yaml"):
                     yaml_path = os.path.join(root, f)
-                    config = utils.simple_load_yaml_config(yaml_path)
+                    config = utils.load_yaml_config("simple", yaml_path)
                     if set(config.keys()) == set(PYTHON_TASK_KEYS):
                         # This is a python class config
                         tasks_and_groups[config["task"]] = {

lm_eval/utils.py

@@ -480,18 +480,10 @@ def get_git_commit_hash():
     return git_hash
 
 
-def simple_load_yaml_config(yaml_path=None, yaml_config=None, yaml_dir=None):
-    def ignore_constructor(loader, node):
+def ignore_constructor(loader, node):
     return node
 
-    yaml.add_constructor("!function", ignore_constructor)
-    with open(yaml_path, "rb") as file:
-        yaml_config = yaml.full_load(file)
-    return yaml_config
-
-
-def load_yaml_config(yaml_path=None, yaml_config=None, yaml_dir=None):
-    def import_function(loader, node):
+def import_function(loader, node):
     function_name = loader.construct_scalar(node)
     yaml_path = os.path.dirname(loader.name)
 
@@ -509,8 +501,16 @@ def load_yaml_config(yaml_path=None, yaml_config=None, yaml_dir=None):
     function = getattr(module, function_name)
     return function
 
+
+def load_yaml_config(mode="simple", yaml_path=None, yaml_config=None, yaml_dir=None):
+
+    if mode == "simple":
+        constuctor_fn = ignore_constructor
+    elif mode == "full":
+        constuctor_fn = import_function
+
     # Add the import_function constructor to the YAML loader
-    yaml.add_constructor("!function", import_function)
+    yaml.add_constructor("!function", constuctor_fn)
     if yaml_config is None:
         with open(yaml_path, "rb") as file:
             yaml_config = yaml.full_load(file)
@@ -538,7 +538,7 @@ def load_yaml_config(yaml_path=None, yaml_config=None, yaml_dir=None):
                 path = os.path.join(yaml_dir, path)
 
             try:
-                included_yaml_config = load_yaml_config(path)
+                included_yaml_config = load_yaml_config(mode=mode, yaml_path=path)
                 final_yaml_config.update(included_yaml_config)
             except Exception as ex:
                 # If failed to load, ignore