Skip to content

GitLab

Commit 48aefc70 authored by lintangsutawika's avatar lintangsutawika
Browse files

iron out bugs

parent 702d86aa
Show whitespace changes
Inline Side-by-side
Showing with 6 additions and 7 deletions
lm_eval/tasks/__init__.py
View file @ 48aefc70
...@@ -45,7 +45,7 @@ class TaskManager(abc.ABC): ...@@ -45,7 +45,7 @@ class TaskManager(abc.ABC):
self.verbosity = verbosity self.verbosity = verbosity
self.include_path = include_path self.include_path = include_path
self.eval_logger.setLevel(getattr(logging, f"{verbosity}")) self.logger = eval_logger.setLevel(getattr(logging, f"{verbosity}"))
self.ALL_TASKS = self.initialize_tasks( self.ALL_TASKS = self.initialize_tasks(
include_path=include_path include_path=include_path
...@@ -66,9 +66,8 @@ class TaskManager(abc.ABC): ...@@ -66,9 +66,8 @@ class TaskManager(abc.ABC):
return ALL_TASKS return ALL_TASKS
@property
def all_tasks(self): def all_tasks(self):
return sorted(self.ALL_TASKS.keys()) return sorted(list(self.ALL_TASKS.keys()))
def _load_individual_task_or_group(self, task_name_or_config: Union[str, dict] = None) -> ConfigurableTask: def _load_individual_task_or_group(self, task_name_or_config: Union[str, dict] = None) -> ConfigurableTask:
...@@ -93,13 +92,13 @@ class TaskManager(abc.ABC): ...@@ -93,13 +92,13 @@ class TaskManager(abc.ABC):
for task_or_config in subtask_list: for task_or_config in subtask_list:
if isinstance(task_or_config, str): if isinstance(task_or_config, str):
all_subtasks[task_or_config] = (group_name, None) all_subtasks[task_or_config] = (group_name, None)
task_object = self._load_individual_task_or_group(self.ALL_TASKS, task_name_or_config=task_or_config) task_object = self._load_individual_task_or_group(task_name_or_config=task_or_config)
elif isinstance(task_or_config, dict): elif isinstance(task_or_config, dict):
if "group" in task_or_config: if "group" in task_or_config:
all_subtasks[task_or_config["group"]] = (group_name, None) all_subtasks[task_or_config["group"]] = (group_name, None)
elif "task" in task_or_config: elif "task" in task_or_config:
all_subtasks[task_or_config["task"]] = (group_name, None) all_subtasks[task_or_config["task"]] = (group_name, None)
task_object = self._load_individual_task_or_group(self.ALL_TASKS, task_name_or_config=task_or_config) task_object = self._load_individual_task_or_group(task_name_or_config=task_or_config)
if isinstance(task_object, dict): if isinstance(task_object, dict):
all_subtasks = {**task_object, **all_subtasks} all_subtasks = {**task_object, **all_subtasks}
...@@ -118,10 +117,10 @@ class TaskManager(abc.ABC): ...@@ -118,10 +117,10 @@ class TaskManager(abc.ABC):
all_subtasks = {} all_subtasks = {}
for task_or_config in subtask_list: for task_or_config in subtask_list:
if isinstance(task_or_config, str): if isinstance(task_or_config, str):
task_object = self._load_individual_task_or_group(self.ALL_TASKS, task_name_or_config=task_or_config) task_object = self._load_individual_task_or_group(task_name_or_config=task_or_config)
task_name = task_or_config task_name = task_or_config
elif isinstance(task_or_config, dict): elif isinstance(task_or_config, dict):
task_object = self._load_individual_task_or_group(self.ALL_TASKS, task_name_or_config=task_or_config) task_object = self._load_individual_task_or_group(task_name_or_config=task_or_config)
if isinstance(task_object, dict): if isinstance(task_object, dict):
all_subtasks = {**task_object, **all_subtasks} all_subtasks = {**task_object, **all_subtasks}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment