evaluator.py 11.9 KB
Newer Older
Leo Gao's avatar
Leo Gao committed
1
2
import collections
import itertools
3
import numpy as np
Leo Gao's avatar
Leo Gao committed
4
import random
Leo Gao's avatar
Leo Gao committed
5
import lm_eval.metrics
6
7
8
import lm_eval.models
import lm_eval.tasks
import lm_eval.base
Stephen Hogg's avatar
Stephen Hogg committed
9
from lm_eval.utils import positional_deprecated, run_task_tests
10

Fabrizio Milo's avatar
Fabrizio Milo committed
11

12
@positional_deprecated
Fabrizio Milo's avatar
Fabrizio Milo committed
13
14
15
16
17
18
19
20
21
22
23
24
25
26
def simple_evaluate(
    model,
    model_args=None,
    tasks=[],
    num_fewshot=0,
    batch_size=None,
    device=None,
    no_cache=False,
    limit=None,
    bootstrap_iters=100000,
    description_dict=None,
    check_integrity=False,
    decontamination_ngrams_path=None,
):
27

28
    """Instantiate and evaluate a model on a list of tasks.
29

30
31
32
    :param model: Union[str, LM]
        Name of model or LM object, see lm_eval.models.get_model
    :param model_args: Optional[str]
Fabrizio Milo's avatar
Fabrizio Milo committed
33
        String arguments for each model class, see LM.create_from_arg_string.
34
35
        Ignored if `model` argument is a LM object.
    :param tasks: list[Union[str, Task]]
Leo Gao's avatar
Leo Gao committed
36
        List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.
37
38
39
40
41
    :param num_fewshot: int
        Number of examples in few-shot context
    :param batch_size: int, optional
        Batch size for model
    :param device: str, optional
42
        PyTorch device (e.g. "cpu" or "cuda:0") for running models
43
    :param no_cache: bool
Leo Gao's avatar
Leo Gao committed
44
        Whether or not to cache
45
46
47
48
    :param limit: int, optional
        Limit the number of examples per task (only use this for testing)
    :param bootstrap_iters:
        Number of iterations for bootstrap statistics
Jonathan Tow's avatar
Jonathan Tow committed
49
    :param description_dict: dict[str, str]
Fabrizio Milo's avatar
Fabrizio Milo committed
50
        Dictionary of custom task descriptions of the form: `task_name: description`
Stephen Hogg's avatar
Stephen Hogg committed
51
52
    :param check_integrity: bool
        Whether to run the relevant part of the test suite for the tasks
53
    :return
54
        Dictionary of results
55
    """
56
57
58
    random.seed(1234)
    np.random.seed(1234)

59
60
61
    assert tasks != [], "No tasks specified"

    if isinstance(model, str):
Fabrizio Milo's avatar
Fabrizio Milo committed
62
63
64
65
66
        if model_args is None:
            model_args = ""
        lm = lm_eval.models.get_model(model).create_from_arg_string(
            model_args, {"batch_size": batch_size, "device": device}
        )
67
68
69
    else:
        assert isinstance(model, lm_eval.base.LM)
        lm = model
70
71

    if not no_cache:
72
        lm = lm_eval.base.CachingLM(
Fabrizio Milo's avatar
Fabrizio Milo committed
73
74
75
76
77
78
            lm,
            "lm_cache/"
            + model
            + "_"
            + model_args.replace("=", "-").replace(",", "_").replace("/", "-")
            + ".db",
79
        )
Fabrizio Milo's avatar
Fabrizio Milo committed
80

81
    task_dict = lm_eval.tasks.get_task_dict(tasks)
Jonathan Tow's avatar
Merge  
Jonathan Tow committed
82

Stephen Hogg's avatar
Stephen Hogg committed
83
    if check_integrity:
84
        run_task_tests(task_list=tasks)
Stephen Hogg's avatar
Stephen Hogg committed
85

86
87
88
89
90
    results = evaluate(
        lm=lm,
        task_dict=task_dict,
        num_fewshot=num_fewshot,
        limit=limit,
91
        description_dict=description_dict,
Fabrizio Milo's avatar
Fabrizio Milo committed
92
        decontamination_ngrams_path=decontamination_ngrams_path,
93
    )
94
95
96
97
98
99
100
101
102
103

    # add info about the model and few shot config
    results["config"] = {
        "model": model,
        "model_args": model_args,
        "num_fewshot": num_fewshot,
        "batch_size": batch_size,
        "device": device,
        "no_cache": no_cache,
        "limit": limit,
104
        "bootstrap_iters": bootstrap_iters,
Fabrizio Milo's avatar
Fabrizio Milo committed
105
        "description_dict": description_dict,
106
107
108
    }

    return results
Leo Gao's avatar
Leo Gao committed
109

Fabrizio Milo's avatar
Fabrizio Milo committed
110

111
decontaminate_suffix = "_decontaminate"
Leo Gao's avatar
Leo Gao committed
112

Fabrizio Milo's avatar
Fabrizio Milo committed
113

114
@positional_deprecated
Fabrizio Milo's avatar
Fabrizio Milo committed
115
116
117
118
119
120
121
122
123
124
def evaluate(
    lm,
    task_dict,
    provide_description=None,
    num_fewshot=0,
    limit=None,
    bootstrap_iters=100000,
    description_dict=None,
    decontamination_ngrams_path=None,
):
125
126
127
128
129
    """Instantiate and evaluate a model on a list of tasks.

    :param lm: obj
        Language Model
    :param task_dict: dict[str, Task]
Leo Gao's avatar
Leo Gao committed
130
        Dictionary of tasks. Tasks will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.
131
    :param provide_description: bool
Leo Gao's avatar
Leo Gao committed
132
        Not implemented, and this option is deprecated and will be removed in a future version in favor of a different description providing method
133
134
135
136
137
138
    :param num_fewshot: int
        Number of examples in few-shot context
    :param limit: int, optional
        Limit the number of examples per task (only use this for testing)
    :param bootstrap_iters:
        Number of iterations for bootstrap statistics
Jonathan Tow's avatar
Jonathan Tow committed
139
    :param description_dict: dict[str, str]
Fabrizio Milo's avatar
Fabrizio Milo committed
140
        Dictionary of custom task descriptions of the form: `task_name: description`
141
142
143
    :return
        Dictionary of results
    """
Leo Gao's avatar
Leo Gao committed
144
145
    # TODO: completely refactor this entire function to not be a huge mess, ideally breaking it down into smaller pieces

146
147
    # TODO: todo: implement proper description-providing system
    assert not provide_description  # not implemented.
Leo Gao's avatar
Leo Gao committed
148
149
    if provide_description is not None:
        # nudge people to not specify it at all
Fabrizio Milo's avatar
Fabrizio Milo committed
150
151
152
        print(
            "WARNING: provide_description is deprecated and will be removed in a future version in favor of description_dict"
        )
153

Leo Gao's avatar
Leo Gao committed
154
    decontaminate = decontamination_ngrams_path is not None
155

156
157
158
    task_dict_items = [
        (name, task)
        for name, task in task_dict.items()
Fabrizio Milo's avatar
Fabrizio Milo committed
159
        if (task.has_validation_docs() or task.has_test_docs())
160
    ]
Leo Gao's avatar
Leo Gao committed
161
162

    results = collections.defaultdict(dict)
Leo Gao's avatar
Leo Gao committed
163
    versions = collections.defaultdict(dict)
Leo Gao's avatar
Leo Gao committed
164
165
166
167

    requests = collections.defaultdict(list)
    requests_origin = collections.defaultdict(list)

Fabrizio Milo's avatar
Fabrizio Milo committed
168
    overlaps = collections.defaultdict(list)  # {task_name: contaminated_docs}
169

170
171
172
173
    # If we ever run into issues where the eval tasks don't fit in memory and we can't afford a machine with bigger
    # memory, we can always modify this plumbing to support that, but I didn't want to include it just yet because
    # over-engineering is bad (or we could make it write the requests to disk and then read them back out again
    #  - probably using an sqlite db because of all the moving parts we have
Leo Gao's avatar
Leo Gao committed
174
175
176
177

    # TODO: we need unit tests & sanity checks or something to ensure that the return of `validation_docs` is stable
    docs = {}

178
179
    docs_for_decontamination = collections.defaultdict(list)

180
    # get lists of each type of request
Leo Gao's avatar
Leo Gao committed
181
    for task_name, task in task_dict_items:
Leo Gao's avatar
Leo Gao committed
182
        versions[task_name] = task.VERSION
183
        # default to test doc, fall back to val doc if validation unavailable
Leo Gao's avatar
Leo Gao committed
184
185
        # TODO: the test-fallback-to-val system isn't final, we should revisit it at some point
        if task.has_test_docs():
Leo Gao's avatar
Leo Gao committed
186
            task_doc_func = task.test_docs
Fabrizio Milo's avatar
Fabrizio Milo committed
187
            task_set = "test"  # Required for caching in the decontamination
Leo Gao's avatar
Leo Gao committed
188
        elif task.has_validation_docs():
Fabrizio Milo's avatar
Fabrizio Milo committed
189
            task_set = "val"  # Required for caching in the decontamination
Leo Gao's avatar
Leo Gao committed
190
            task_doc_func = task.validation_docs
191
192
        else:
            raise RuntimeError("Task has neither test_docs nor validation_docs")
Leo Gao's avatar
Leo Gao committed
193

Leo Gao's avatar
Leo Gao committed
194
195
196
197
        # deterministically shuffle docs and chop off the first `limit` because sometimes docs are in some kind of order
        task_docs = list(task_doc_func())
        rnd = random.Random()
        rnd.seed(42)
Jason Phang's avatar
Jason Phang committed
198
        rnd.shuffle(task_docs)
Leo Gao's avatar
Leo Gao committed
199

Fabrizio Milo's avatar
Fabrizio Milo committed
200
201
202
203
204
        description = (
            description_dict[task_name]
            if description_dict and task_name in description_dict
            else ""
        )
205

Leo Gao's avatar
Leo Gao committed
206
        for doc_id, doc in enumerate(itertools.islice(task_docs, 0, limit)):
207
208

            if decontaminate and task.should_decontaminate():
Fabrizio Milo's avatar
Fabrizio Milo committed
209
210
211
                docs_for_decontamination[(task_name, task_set)].append(
                    task.doc_to_decontamination_query(doc)
                )
212

Leo Gao's avatar
Leo Gao committed
213
214
            docs[(task_name, doc_id)] = doc
            ctx = task.fewshot_context(
Fabrizio Milo's avatar
Fabrizio Milo committed
215
                doc=doc, num_fewshot=num_fewshot, rnd=rnd, description=description
Leo Gao's avatar
Leo Gao committed
216
217
            )
            reqs = task.construct_requests(doc, ctx)
218
219
            if not isinstance(reqs, (list, tuple)):
                reqs = [reqs]
Leo Gao's avatar
Leo Gao committed
220
            for i, req in enumerate(reqs):
Leo Gao's avatar
Leo Gao committed
221
                requests[req.request_type].append(req)
Leo Gao's avatar
Leo Gao committed
222
223
                # i: index in requests for a single task instance
                # doc_id: unique id that we can get back to a doc using `docs`
Leo Gao's avatar
Leo Gao committed
224
                requests_origin[req.request_type].append((i, task_name, doc, doc_id))
Leo Gao's avatar
Leo Gao committed
225

226
227
    # Compare all tasks/sets at once to ensure a single training set scan
    if decontaminate:
228
        from lm_eval.decontamination.decontaminate import get_train_overlap
229
        print("Finding train/test overlap, please wait...")
Fabrizio Milo's avatar
Fabrizio Milo committed
230
231
232
        overlaps = get_train_overlap(
            docs_for_decontamination, decontamination_ngrams_path, limit
        )
233

Leo Gao's avatar
Leo Gao committed
234
235
236
237
238
    # all responses for each (task, doc)
    process_res_queue = collections.defaultdict(list)

    # execute each type of request
    for reqtype, reqs in requests.items():
239
240
241
242
        # TODO: right now, this code runs multiple separate LM requests for multiple Requests differing
        #       only in index. We could implement some kind of caching, but that would be more of a band-aid
        #       solution. we could also implement some kind of auto-grouping here;
        #       they should end up next to each other.
Leo Gao's avatar
Leo Gao committed
243

Leo Gao's avatar
Leo Gao committed
244
        print("Running", reqtype, "requests")
Leo Gao's avatar
Leo Gao committed
245
        resps = getattr(lm, reqtype)([req.args for req in reqs])
Fabrizio Milo's avatar
Fabrizio Milo committed
246
247
248
        resps = [
            x if req.index is None else x[req.index] for x, req in zip(resps, reqs)
        ]
Leo Gao's avatar
Leo Gao committed
249
250
251

        for resp, (i, task_name, doc, doc_id) in zip(resps, requests_origin[reqtype]):
            process_res_queue[(task_name, doc_id)].append((i, resp))
Fabrizio Milo's avatar
Fabrizio Milo committed
252

Leo Gao's avatar
Leo Gao committed
253
254
255
256
257
258
259
260
261
262
263
264
265
    vals = collections.defaultdict(list)

    # unpack results and sort back in order and return control to Task
    for (task_name, doc_id), requests in process_res_queue.items():
        requests.sort(key=lambda x: x[0])
        requests = [x[1] for x in requests]

        task = task_dict[task_name]
        doc = docs[(task_name, doc_id)]

        metrics = task.process_results(doc, requests)
        for metric, value in metrics.items():
            vals[(task_name, metric)].append(value)
266
267
268
269
270

            # Re-use the evaluation for the decontaminated set by just ignoring the overlaps
            if decontaminate and task_name in overlaps:
                if doc_id not in overlaps[task_name]:
                    vals[(task_name, metric + decontaminate_suffix)].append(value)
Fabrizio Milo's avatar
Fabrizio Milo committed
271

Leo Gao's avatar
Leo Gao committed
272
273
274
    # aggregate results
    for (task_name, metric), items in vals.items():
        task = task_dict[task_name]
Fabrizio Milo's avatar
Fabrizio Milo committed
275
        real_metric = metric  # key when looking up the metric with task.aggregation
276
        if metric.endswith(decontaminate_suffix):
Fabrizio Milo's avatar
Fabrizio Milo committed
277
278
279
            real_metric = metric.replace(
                decontaminate_suffix, ""
            )  # decontaminated still uses the same metric
280
        results[task_name][metric] = task.aggregation()[real_metric](items)
Leo Gao's avatar
Leo Gao committed
281

282
283
        # hotfix: bleu, chrf, ter seem to be really expensive to bootstrap
        # so we run them less iterations. still looking for a cleaner way to do this
284

285
        stderr = lm_eval.metrics.stderr_for_metric(
286
            metric=task.aggregation()[real_metric],
Fabrizio Milo's avatar
Fabrizio Milo committed
287
288
289
            bootstrap_iters=min(bootstrap_iters, 1000)
            if metric in ["bleu", "chrf", "ter"]
            else bootstrap_iters,
290
        )
Fabrizio Milo's avatar
Fabrizio Milo committed
291

Leo Gao's avatar
Leo Gao committed
292
293
        if stderr is not None:
            results[task_name][metric + "_stderr"] = stderr(items)
Fabrizio Milo's avatar
Fabrizio Milo committed
294
295

    return {"results": dict(results), "versions": dict(versions)}
296
297
298


def make_table(result_dict):
299
    """Generate table of results."""
300
301
302
303
304
305
306
307
308
309
310
311
    from pytablewriter import MarkdownTableWriter, LatexTableWriter

    md_writer = MarkdownTableWriter()
    latex_writer = LatexTableWriter()
    md_writer.headers = ["Task", "Version", "Metric", "Value", "", "Stderr"]
    latex_writer.headers = ["Task", "Version", "Metric", "Value", "", "Stderr"]

    values = []

    for k, dic in result_dict["results"].items():
        version = result_dict["versions"][k]
        for m, v in dic.items():
312
313
            if m.endswith("_stderr"):
                continue
314
315
316

            if m + "_stderr" in dic:
                se = dic[m + "_stderr"]
Fabrizio Milo's avatar
Fabrizio Milo committed
317
                values.append([k, version, m, "%.4f" % v, "±", "%.4f" % se])
318
            else:
Fabrizio Milo's avatar
Fabrizio Milo committed
319
                values.append([k, version, m, "%.4f" % v, "", ""])
320
321
322
323
324
325
326
327
            k = ""
            version = ""
    md_writer.value_matrix = values
    latex_writer.value_matrix = values

    # todo: make latex table look good
    # print(latex_writer.dumps())

328
    return md_writer.dumps()