Skip to content

GitLab

"src/vscode:/vscode.git/clone" did not exist on "a808c1f44a755a8f770040e95de45511f090819f"
Commit 7e186d53 authored by lintangsutawika's avatar lintangsutawika
Browse files

add method to have 1 yaml with multiple tasks

parent 88486e57
Hide whitespace changes
Inline Side-by-side
Showing with 104 additions and 66 deletions
lm_eval/tasks/__init__.py
View file @ 7e186d53
...@@ -128,6 +128,13 @@ class TaskManager: ...@@ -128,6 +128,13 @@ class TaskManager:
raise ValueError raise ValueError
return self.task_index[name]["yaml_path"] return self.task_index[name]["yaml_path"]
def _get_idx(self, name):
if name not in self.task_index:
raise ValueError
if "idx" not in self.task_index[name]:
return None
return self.task_index[name]["idx"]
def _get_config(self, name): def _get_config(self, name):
if name not in self.task_index: if name not in self.task_index:
raise ValueError raise ValueError
...@@ -135,6 +142,9 @@ class TaskManager: ...@@ -135,6 +142,9 @@ class TaskManager:
if yaml_path == -1: if yaml_path == -1:
return {} return {}
else: else:
idx = self._get_idx(name)
if idx is not None:
return utils.load_yaml_config(yaml_path, mode="full")[idx]
return utils.load_yaml_config(yaml_path, mode="full") return utils.load_yaml_config(yaml_path, mode="full")
def _get_tasklist(self, name): def _get_tasklist(self, name):
...@@ -373,76 +383,89 @@ class TaskManager: ...@@ -373,76 +383,89 @@ class TaskManager:
for f in file_list: for f in file_list:
if f.endswith(".yaml"): if f.endswith(".yaml"):
yaml_path = os.path.join(root, f) yaml_path = os.path.join(root, f)
config = utils.load_yaml_config(yaml_path, mode="simple") config_list = utils.load_yaml_config(yaml_path, mode="simple")
if self._config_is_python_task(config): use_idx = False
# This is a python class config if isinstance(config_list, dict):
tasks_and_groups[config["task"]] = { config_list = [config_list]
"type": "python_task", else:
"yaml_path": yaml_path, use_idx = True
}
elif self._config_is_group(config): for idx, config in enumerate(config_list):
# This is a group config if self._config_is_python_task(config):
tasks_and_groups[config["group"]] = { # This is a python class config
"type": "group", tasks_and_groups[config["task"]] = {
"task": -1, # This signals that "type": "python_task",
# we don't need to know "yaml_path": yaml_path,
# the task list for indexing }
# as it can be loaded elif self._config_is_group(config):
# when called. # This is a group config
"yaml_path": yaml_path, tasks_and_groups[config["group"]] = {
} "type": "group",
"task": -1, # This signals that
# we don't need to know
# the task list for indexing
# as it can be loaded
# when called.
"yaml_path": yaml_path,
}
# # Registered the level 1 tasks from a group config # # Registered the level 1 tasks from a group config
# for config in config["task"]: # for config in config["task"]:
# if isinstance(config, dict) and self._config_is_task(config): # if isinstance(config, dict) and self._config_is_task(config):
# task = config["task"] # task = config["task"]
# tasks_and_groups[task] = { # tasks_and_groups[task] = {
# "type": "task", # "type": "task",
# "yaml_path": yaml_path, # "yaml_path": yaml_path,
# } # }
elif self._config_is_task(config): elif self._config_is_task(config):
# This is a task config # This is a task config
task = config["task"] task = config["task"]
tasks_and_groups[task] = { tasks_and_groups[task] = {
"type": "task", "type": "task",
"yaml_path": yaml_path, "yaml_path": yaml_path,
} }
# TODO: remove group in next release if use_idx:
for attr in ["tag", "group"]: tasks_and_groups[task] = {
if attr in config: **tasks_and_groups[task],
if attr == "group" and print_info: **{"idx": idx},
self.logger.info( }
"`group` and `group_alias` keys in tasks' configs will no longer be used in the next release of lm-eval. "
"`tag` will be used to allow to call a collection of tasks just like `group`. " # TODO: remove group in next release
"`group` will be removed in order to not cause confusion with the new ConfigurableGroup " for attr in ["tag", "group"]:
"which will be the offical way to create groups with addition of group-wide configuations." if attr in config:
) if attr == "group" and print_info:
print_info = False
# attr = "tag"
attr_list = config[attr]
if isinstance(attr_list, str):
attr_list = [attr_list]
for tag in attr_list:
if tag not in tasks_and_groups:
tasks_and_groups[tag] = {
"type": "tag",
"task": [task],
"yaml_path": -1,
}
elif tasks_and_groups[tag]["type"] != "tag":
self.logger.info( self.logger.info(
f"The tag {tag} is already registered as a group, this tag will not be registered. " "`group` and `group_alias` keys in tasks' configs will no longer be used in the next release of lm-eval. "
"This may affect tasks you want to call." "`tag` will be used to allow to call a collection of tasks just like `group`. "
"`group` will be removed in order to not cause confusion with the new ConfigurableGroup "
"which will be the offical way to create groups with addition of group-wide configuations."
) )
break print_info = False
else: # attr = "tag"
tasks_and_groups[tag]["task"].append(task)
else: attr_list = config[attr]
self.logger.debug(f"File {f} in {root} could not be loaded") if isinstance(attr_list, str):
attr_list = [attr_list]
for tag in attr_list:
if tag not in tasks_and_groups:
tasks_and_groups[tag] = {
"type": "tag",
"task": [task],
"yaml_path": -1,
}
elif tasks_and_groups[tag]["type"] != "tag":
self.logger.info(
f"The tag {tag} is already registered as a group, this tag will not be registered. "
"This may affect tasks you want to call."
)
break
else:
tasks_and_groups[tag]["task"].append(task)
else:
self.logger.debug(f"File {f} in {root} could not be loaded")
return tasks_and_groups return tasks_and_groups
......
lm_eval/utils.py
View file @ 7e186d53
...@@ -431,6 +431,9 @@ def load_yaml_config(yaml_path=None, yaml_config=None, yaml_dir=None, mode="full ...@@ -431,6 +431,9 @@ def load_yaml_config(yaml_path=None, yaml_config=None, yaml_dir=None, mode="full
with open(yaml_path, "rb") as file: with open(yaml_path, "rb") as file:
yaml_config = yaml.full_load(file) yaml_config = yaml.full_load(file)
if isinstance(yaml_config, list):
return [load_yaml_config(yaml_path, config) for config in yaml_config]
if yaml_dir is None: if yaml_dir is None:
yaml_dir = os.path.dirname(yaml_path) yaml_dir = os.path.dirname(yaml_path)
......
tests/testyamls/test-tasklist.yaml 0 → 100644
View file @ 7e186d53
# Arc Easy 2-shots
- task: arc_easy-2-shot
include: lm_eval/tasks/arc/arc_easy.yaml
num_fewshot: 2
# Arc Easy 4-shots
- task: arc_easy-4-shot
include: lm_eval/tasks/arc/arc_easy.yaml
num_fewshot: 4
# Arc Easy 8-shots
- task: arc_easy-8-shot
include: lm_eval/tasks/arc/arc_easy.yaml
num_fewshot: 8
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment