Skip to content

GitLab

Commit 7e186d53 authored by lintangsutawika's avatar lintangsutawika
Browse files

add method to have 1 yaml with multiple tasks

parent 88486e57
Hide whitespace changes
Inline Side-by-side
Showing with 104 additions and 66 deletions
lm_eval/tasks/__init__.py
View file @ 7e186d53
......@@ -128,6 +128,13 @@ class TaskManager:
raise ValueError
return self.task_index[name]["yaml_path"]
def _get_idx(self, name):
if name not in self.task_index:
raise ValueError
if "idx" not in self.task_index[name]:
return None
return self.task_index[name]["idx"]
def _get_config(self, name):
if name not in self.task_index:
raise ValueError
......@@ -135,6 +142,9 @@ class TaskManager:
if yaml_path == -1:
return {}
else:
idx = self._get_idx(name)
if idx is not None:
return utils.load_yaml_config(yaml_path, mode="full")[idx]
return utils.load_yaml_config(yaml_path, mode="full")
def _get_tasklist(self, name):
......@@ -373,76 +383,89 @@ class TaskManager:
for f in file_list:
if f.endswith(".yaml"):
yaml_path = os.path.join(root, f)
config = utils.load_yaml_config(yaml_path, mode="simple")
if self._config_is_python_task(config):
# This is a python class config
tasks_and_groups[config["task"]] = {
"type": "python_task",
"yaml_path": yaml_path,
}
elif self._config_is_group(config):
# This is a group config
tasks_and_groups[config["group"]] = {
"type": "group",
"task": -1, # This signals that
# we don't need to know
# the task list for indexing
# as it can be loaded
# when called.
"yaml_path": yaml_path,
}
config_list = utils.load_yaml_config(yaml_path, mode="simple")
use_idx = False
if isinstance(config_list, dict):
config_list = [config_list]
else:
use_idx = True
for idx, config in enumerate(config_list):
if self._config_is_python_task(config):
# This is a python class config
tasks_and_groups[config["task"]] = {
"type": "python_task",
"yaml_path": yaml_path,
}
elif self._config_is_group(config):
# This is a group config
tasks_and_groups[config["group"]] = {
"type": "group",
"task": -1, # This signals that
# we don't need to know
# the task list for indexing
# as it can be loaded
# when called.
"yaml_path": yaml_path,
}
# # Registered the level 1 tasks from a group config
# for config in config["task"]:
# if isinstance(config, dict) and self._config_is_task(config):
# task = config["task"]
# tasks_and_groups[task] = {
# "type": "task",
# "yaml_path": yaml_path,
# }
elif self._config_is_task(config):
# This is a task config
task = config["task"]
tasks_and_groups[task] = {
"type": "task",
"yaml_path": yaml_path,
}
# # Registered the level 1 tasks from a group config
# for config in config["task"]:
# if isinstance(config, dict) and self._config_is_task(config):
# task = config["task"]
# tasks_and_groups[task] = {
# "type": "task",
# "yaml_path": yaml_path,
# }
elif self._config_is_task(config):
# This is a task config
task = config["task"]
tasks_and_groups[task] = {
"type": "task",
"yaml_path": yaml_path,
}
# TODO: remove group in next release
for attr in ["tag", "group"]:
if attr in config:
if attr == "group" and print_info:
self.logger.info(
"`group` and `group_alias` keys in tasks' configs will no longer be used in the next release of lm-eval. "
"`tag` will be used to allow to call a collection of tasks just like `group`. "
"`group` will be removed in order to not cause confusion with the new ConfigurableGroup "
"which will be the offical way to create groups with addition of group-wide configuations."
)
print_info = False
# attr = "tag"
attr_list = config[attr]
if isinstance(attr_list, str):
attr_list = [attr_list]
for tag in attr_list:
if tag not in tasks_and_groups:
tasks_and_groups[tag] = {
"type": "tag",
"task": [task],
"yaml_path": -1,
}
elif tasks_and_groups[tag]["type"] != "tag":
if use_idx:
tasks_and_groups[task] = {
**tasks_and_groups[task],
**{"idx": idx},
}
# TODO: remove group in next release
for attr in ["tag", "group"]:
if attr in config:
if attr == "group" and print_info:
self.logger.info(
f"The tag {tag} is already registered as a group, this tag will not be registered. "
"This may affect tasks you want to call."
"`group` and `group_alias` keys in tasks' configs will no longer be used in the next release of lm-eval. "
"`tag` will be used to allow to call a collection of tasks just like `group`. "
"`group` will be removed in order to not cause confusion with the new ConfigurableGroup "
"which will be the offical way to create groups with addition of group-wide configuations."
)
break
else:
tasks_and_groups[tag]["task"].append(task)
else:
self.logger.debug(f"File {f} in {root} could not be loaded")
print_info = False
# attr = "tag"
attr_list = config[attr]
if isinstance(attr_list, str):
attr_list = [attr_list]
for tag in attr_list:
if tag not in tasks_and_groups:
tasks_and_groups[tag] = {
"type": "tag",
"task": [task],
"yaml_path": -1,
}
elif tasks_and_groups[tag]["type"] != "tag":
self.logger.info(
f"The tag {tag} is already registered as a group, this tag will not be registered. "
"This may affect tasks you want to call."
)
break
else:
tasks_and_groups[tag]["task"].append(task)
else:
self.logger.debug(f"File {f} in {root} could not be loaded")
return tasks_and_groups
......
lm_eval/utils.py
View file @ 7e186d53
......@@ -431,6 +431,9 @@ def load_yaml_config(yaml_path=None, yaml_config=None, yaml_dir=None, mode="full
with open(yaml_path, "rb") as file:
yaml_config = yaml.full_load(file)
if isinstance(yaml_config, list):
return [load_yaml_config(yaml_path, config) for config in yaml_config]
if yaml_dir is None:
yaml_dir = os.path.dirname(yaml_path)
......
tests/testyamls/test-tasklist.yaml 0 → 100644
View file @ 7e186d53
# Arc Easy 2-shots
- task: arc_easy-2-shot
include: lm_eval/tasks/arc/arc_easy.yaml
num_fewshot: 2
# Arc Easy 4-shots
- task: arc_easy-4-shot
include: lm_eval/tasks/arc/arc_easy.yaml
num_fewshot: 4
# Arc Easy 8-shots
- task: arc_easy-8-shot
include: lm_eval/tasks/arc/arc_easy.yaml
num_fewshot: 8
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment