__init__.py 4.95 KB
Newer Older
1
import os
lintangsutawika's avatar
lintangsutawika committed
2
from typing import List, Union
&'s avatar
& committed
3

4
from .arc import *
lintangsutawika's avatar
lintangsutawika committed
5

6
from lm_eval import utils
lintangsutawika's avatar
lintangsutawika committed
7
from lm_eval.logger import eval_logger
8
9
10
11
12
from lm_eval.api.task import TaskConfig, Task, ConfigurableTask
from lm_eval.api.register import (
    register_task,
    register_group,
    task_registry,
lintangsutawika's avatar
lintangsutawika committed
13
14
    group_registry,
)
lintangsutawika's avatar
lintangsutawika committed
15

16

17
18
def get_task_name_from_config(task_config):
    return "configurable_{dataset_path}_{dataset_name}".format(**task_config)
19
20


21
22
23
24
25
26
27
task_dir = os.path.dirname(os.path.abspath(__file__)) + "/"
for root, subdirs, file_list in os.walk(task_dir):
    if (subdirs == []) and (len(file_list) > 0):
        for file in file_list:
            if "yaml" in file:
                yaml_path = os.path.join(root, file)
                try:
28
                    config = utils.load_yaml_config(yaml_path)
29
30

                    SubClass = type(
lintangsutawika's avatar
lintangsutawika committed
31
                        config["task"] + "ConfigurableTask",
32
                        (ConfigurableTask,),
lintangsutawika's avatar
lintangsutawika committed
33
                        {"CONFIG": TaskConfig(**config)},
34
35
                    )

lintangsutawika's avatar
lintangsutawika committed
36
                    if "task" in config:
37
                        task_name = "{}:{}".format(
lintangsutawika's avatar
lintangsutawika committed
38
39
                            get_task_name_from_config(config), config["task"]
                        )
40
41
                        register_task(task_name)(SubClass)

lintangsutawika's avatar
lintangsutawika committed
42
43
                    if "group" in config:
                        for group in config["group"]:
44
                            register_group(group)(SubClass)
45
46
                except Exception as err:
                    print(f"Unexpected {err=}, {type(err)=}")
47
48
49

TASK_REGISTRY = task_registry
GROUP_REGISTRY = group_registry
lintangsutawika's avatar
lintangsutawika committed
50
ALL_TASKS = sorted(list(TASK_REGISTRY.keys()) + list(GROUP_REGISTRY.keys()))
lintangsutawika's avatar
lintangsutawika committed
51

lintangsutawika's avatar
lintangsutawika committed
52

53
54
def get_task(task_name, config):
    try:
55
        return TASK_REGISTRY[task_name](config=config)
56
    except KeyError:
lintangsutawika's avatar
lintangsutawika committed
57
58
        eval_logger.info("Available tasks:")
        eval_logger.info(TASK_REGISTRY)
lintangsutawika's avatar
lintangsutawika committed
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
        raise KeyError(f"Missing task {task_name}")


def get_task_name_from_object(task_object):
    for name, class_ in TASK_REGISTRY.items():
        if class_ is task_object:
            return name

    # TODO: scrap this
    # this gives a mechanism for non-registered tasks to have a custom name anyways when reporting
    return (
        task_object.EVAL_HARNESS_NAME
        if hasattr(task_object, "EVAL_HARNESS_NAME")
        else type(task_object).__name__
    )


# TODO: pass num_fewshot and other cmdline overrides in a better way
lintangsutawika's avatar
lintangsutawika committed
77
78
79
def get_task_dict(task_name_list: List[Union[str, dict, Task]], **kwargs):

    config = {**kwargs}
80
81
82
83
84
85
86
87
88
89
90
91
92

    task_name_from_registry_dict = {}
    task_name_from_config_dict = {}
    task_name_from_object_dict = {}

    for task_element in task_name_list:
        if isinstance(task_element, str):

            if task_element in GROUP_REGISTRY:
                for task_name in GROUP_REGISTRY[task_element]:
                    if task_name not in task_name_from_registry_dict:
                        task_name_from_registry_dict = {
                            **task_name_from_registry_dict,
lintangsutawika's avatar
lintangsutawika committed
93
94
                            task_name: get_task(task_name=task_name, config=config),
                        }
95
            else:
96
                task_name = task_element
97
98
99
                if task_name not in task_name_from_registry_dict:
                    task_name_from_registry_dict = {
                        **task_name_from_registry_dict,
lintangsutawika's avatar
lintangsutawika committed
100
101
                        task_name: get_task(task_name=task_element, config=config),
                    }
102
103

        elif isinstance(task_element, dict):
104
            task_element.update(config)
105
106
107
            task_name_from_config_dict = {
                **task_name_from_config_dict,
                get_task_name_from_config(task_element): ConfigurableTask(
108
                    config=task_element
lintangsutawika's avatar
lintangsutawika committed
109
                ),
110
111
112
113
114
115
            }

        elif isinstance(task_element, Task):

            task_name_from_object_dict = {
                **task_name_from_object_dict,
lintangsutawika's avatar
lintangsutawika committed
116
                get_task_name_from_object(task_element): task_element,
117
            }
lintangsutawika's avatar
lintangsutawika committed
118

119
120
121
122
123
    # task_name_from_registry_dict = {
    #     task_name: get_task(
    #         task_name=task_name,
    #         task_config=config
    #     )
lintangsutawika's avatar
lintangsutawika committed
124
    #     for group_name in task_name_list for task_name in GROUP_REGISTRY[group_name]
125
126
127
128
129
130
131
132
133
134
    #     if (isinstance(group_name, str)) and (group_name in GROUP_REGISTRY)
    # }
    # task_name_from_config_dict = {
    #     get_task_name_from_config(task_config): ConfigurableTask(
    #         config=task_config
    #     )
    #     for task_config in task_name_list
    #     if isinstance(task_config, dict)
    # }
    # # TODO: Do we still need this?
lintangsutawika's avatar
lintangsutawika committed
135
136
137
138
139
    # task_name_from_object_dict = {
    #     get_task_name_from_object(task_object): task_object
    #     for task_object in task_name_list
    #     if isinstance(task_object, Task)
    # }
140

lintangsutawika's avatar
lintangsutawika committed
141
142
143
    assert set(task_name_from_registry_dict.keys()).isdisjoint(
        set(task_name_from_object_dict.keys())
    )
lintangsutawika's avatar
lintangsutawika committed
144
145
146
    return {
        **task_name_from_registry_dict,
        **task_name_from_config_dict,
147
        **task_name_from_object_dict,
lintangsutawika's avatar
lintangsutawika committed
148
    }