evaluator.py 15.8 KB
Newer Older
Leo Gao's avatar
Leo Gao committed
1
2
import collections
import itertools
Leo Gao's avatar
Leo Gao committed
3
import random
4

Leo Gao's avatar
Leo Gao committed
5
import lm_eval.metrics
6
7
8
import lm_eval.models
import lm_eval.tasks
import lm_eval.base
Stephen Hogg's avatar
Stephen Hogg committed
9
from lm_eval.utils import positional_deprecated, run_task_tests
10
11
12
13
from lm_eval.models.gpt2 import HFLM

import numpy as np
import transformers
14

Fabrizio Milo's avatar
Fabrizio Milo committed
15

16
@positional_deprecated
Fabrizio Milo's avatar
Fabrizio Milo committed
17
18
19
20
21
22
def simple_evaluate(
    model,
    model_args=None,
    tasks=[],
    num_fewshot=0,
    batch_size=None,
23
    max_batch_size=None,
Fabrizio Milo's avatar
Fabrizio Milo committed
24
25
26
27
28
29
30
    device=None,
    no_cache=False,
    limit=None,
    bootstrap_iters=100000,
    description_dict=None,
    check_integrity=False,
    decontamination_ngrams_path=None,
31
32
    write_out=False,
    output_base_path=None,
Fabrizio Milo's avatar
Fabrizio Milo committed
33
):
34
    """Instantiate and evaluate a model on a list of tasks.
35

36
    :param model: Union[str, LM]
haileyschoelkopf's avatar
haileyschoelkopf committed
37
        Name of model, transformers.PreTrainedModel object, or LM object, see lm_eval.models.get_model
38
    :param model_args: Optional[str]
Fabrizio Milo's avatar
Fabrizio Milo committed
39
        String arguments for each model class, see LM.create_from_arg_string.
40
41
        Ignored if `model` argument is a LM object.
    :param tasks: list[Union[str, Task]]
Leo Gao's avatar
Leo Gao committed
42
        List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.
43
44
    :param num_fewshot: int
        Number of examples in few-shot context
45
    :param batch_size: int or str, optional
46
        Batch size for model
47
48
    :param max_batch_size: int, optional
        Maximal batch size to try with automatic batch size detection
49
    :param device: str, optional
50
        PyTorch device (e.g. "cpu" or "cuda:0") for running models
51
    :param no_cache: bool
Leo Gao's avatar
Leo Gao committed
52
        Whether or not to cache
53
54
    :param limit: int or float, optional
        Limit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples.
55
56
    :param bootstrap_iters:
        Number of iterations for bootstrap statistics
Jonathan Tow's avatar
Jonathan Tow committed
57
    :param description_dict: dict[str, str]
Fabrizio Milo's avatar
Fabrizio Milo committed
58
        Dictionary of custom task descriptions of the form: `task_name: description`
Stephen Hogg's avatar
Stephen Hogg committed
59
60
    :param check_integrity: bool
        Whether to run the relevant part of the test suite for the tasks
61
    :param write_out: bool
62
        If True, write details about prompts and logits to json for all tasks
63
    :param output_base_path: str, optional
64
        Directory to which detailed eval info will be written. Defaults to present working dir.
65
    :return
66
        Dictionary of results
67
    """
68
69
70
    random.seed(1234)
    np.random.seed(1234)

71
72
73
    assert tasks != [], "No tasks specified"

    if isinstance(model, str):
Fabrizio Milo's avatar
Fabrizio Milo committed
74
75
76
        if model_args is None:
            model_args = ""
        lm = lm_eval.models.get_model(model).create_from_arg_string(
77
            model_args, {"batch_size": batch_size, "max_batch_size": max_batch_size, "device": device}
Fabrizio Milo's avatar
Fabrizio Milo committed
78
        )
79
    elif isinstance(model, transformers.PreTrainedModel):
haileyschoelkopf's avatar
haileyschoelkopf committed
80
        lm = lm_eval.models.get_model("hf-causal")(
81
                pretrained=model,
haileyschoelkopf's avatar
haileyschoelkopf committed
82
                batch_size=batch_size,
83
                max_batch_size=max_batch_size,
84
85
                )
        no_cache = True
86
87
88
    else:
        assert isinstance(model, lm_eval.base.LM)
        lm = model
89
90

    if not no_cache:
91
        lm = lm_eval.base.CachingLM(
Fabrizio Milo's avatar
Fabrizio Milo committed
92
93
            lm,
            "lm_cache/"
94
            + (model if isinstance(model, str) else model.model.config._name_or_path)
Fabrizio Milo's avatar
Fabrizio Milo committed
95
96
97
            + "_"
            + model_args.replace("=", "-").replace(",", "_").replace("/", "-")
            + ".db",
98
        )
Fabrizio Milo's avatar
Fabrizio Milo committed
99

100
    task_dict = lm_eval.tasks.get_task_dict(tasks)
Jonathan Tow's avatar
Merge  
Jonathan Tow committed
101

Stephen Hogg's avatar
Stephen Hogg committed
102
    if check_integrity:
103
        run_task_tests(task_list=tasks)
Stephen Hogg's avatar
Stephen Hogg committed
104

105
106
107
108
109
    results = evaluate(
        lm=lm,
        task_dict=task_dict,
        num_fewshot=num_fewshot,
        limit=limit,
Niklas Muennighoff's avatar
Niklas Muennighoff committed
110
        bootstrap_iters=bootstrap_iters,
111
        description_dict=description_dict,
Fabrizio Milo's avatar
Fabrizio Milo committed
112
        decontamination_ngrams_path=decontamination_ngrams_path,
113
114
        write_out=write_out,
        output_base_path=output_base_path,
115
    )
116
117

    # add info about the model and few shot config
118
119
120
121
122
    model_name = None
    if isinstance(model, str):
        model_name = model
    elif isinstance(model, transformers.PreTrainedModel):
        model_name = "pretrained=" + model.config._name_or_path
123
    results["config"] = {
124
        "model": model_name,
125
126
127
        "model_args": model_args,
        "num_fewshot": num_fewshot,
        "batch_size": batch_size,
gk's avatar
gk committed
128
        "batch_sizes": list(lm.batch_sizes.values()) if hasattr(lm, "batch_sizes") else [],
129
130
131
        "device": device,
        "no_cache": no_cache,
        "limit": limit,
132
        "bootstrap_iters": bootstrap_iters,
Fabrizio Milo's avatar
Fabrizio Milo committed
133
        "description_dict": description_dict,
134
135
136
    }

    return results
Leo Gao's avatar
Leo Gao committed
137

Fabrizio Milo's avatar
Fabrizio Milo committed
138

139
decontaminate_suffix = "_decontaminate"
Leo Gao's avatar
Leo Gao committed
140

Fabrizio Milo's avatar
Fabrizio Milo committed
141

142
@positional_deprecated
Fabrizio Milo's avatar
Fabrizio Milo committed
143
144
145
146
147
148
149
150
151
def evaluate(
    lm,
    task_dict,
    provide_description=None,
    num_fewshot=0,
    limit=None,
    bootstrap_iters=100000,
    description_dict=None,
    decontamination_ngrams_path=None,
152
153
    write_out=False,
    output_base_path=None,
Fabrizio Milo's avatar
Fabrizio Milo committed
154
):
155
156
157
158
159
    """Instantiate and evaluate a model on a list of tasks.

    :param lm: obj
        Language Model
    :param task_dict: dict[str, Task]
Leo Gao's avatar
Leo Gao committed
160
        Dictionary of tasks. Tasks will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.
161
    :param provide_description: bool
Leo Gao's avatar
Leo Gao committed
162
        Not implemented, and this option is deprecated and will be removed in a future version in favor of a different description providing method
163
164
165
166
167
168
    :param num_fewshot: int
        Number of examples in few-shot context
    :param limit: int, optional
        Limit the number of examples per task (only use this for testing)
    :param bootstrap_iters:
        Number of iterations for bootstrap statistics
Jonathan Tow's avatar
Jonathan Tow committed
169
    :param description_dict: dict[str, str]
Fabrizio Milo's avatar
Fabrizio Milo committed
170
        Dictionary of custom task descriptions of the form: `task_name: description`
171
    :param write_out: bool
172
        If True, write all prompts, logits and metrics to json for offline analysis
173
    :param output_base_path: str, optional
174
        Directory to which detailed eval info will be written. Defaults to present working dir
175
176
177
    :return
        Dictionary of results
    """
Leo Gao's avatar
Leo Gao committed
178
179
    # TODO: completely refactor this entire function to not be a huge mess, ideally breaking it down into smaller pieces

180
181
    # TODO: todo: implement proper description-providing system
    assert not provide_description  # not implemented.
Leo Gao's avatar
Leo Gao committed
182
183
    if provide_description is not None:
        # nudge people to not specify it at all
Fabrizio Milo's avatar
Fabrizio Milo committed
184
185
186
        print(
            "WARNING: provide_description is deprecated and will be removed in a future version in favor of description_dict"
        )
187

Leo Gao's avatar
Leo Gao committed
188
    decontaminate = decontamination_ngrams_path is not None
189

190
191
192
    task_dict_items = [
        (name, task)
        for name, task in task_dict.items()
Fabrizio Milo's avatar
Fabrizio Milo committed
193
        if (task.has_validation_docs() or task.has_test_docs())
194
    ]
Leo Gao's avatar
Leo Gao committed
195
196

    results = collections.defaultdict(dict)
Leo Gao's avatar
Leo Gao committed
197
    versions = collections.defaultdict(dict)
Leo Gao's avatar
Leo Gao committed
198
199
200
201

    requests = collections.defaultdict(list)
    requests_origin = collections.defaultdict(list)

Fabrizio Milo's avatar
Fabrizio Milo committed
202
    overlaps = collections.defaultdict(list)  # {task_name: contaminated_docs}
203

204
205
206
207
    # If we ever run into issues where the eval tasks don't fit in memory and we can't afford a machine with bigger
    # memory, we can always modify this plumbing to support that, but I didn't want to include it just yet because
    # over-engineering is bad (or we could make it write the requests to disk and then read them back out again
    #  - probably using an sqlite db because of all the moving parts we have
Leo Gao's avatar
Leo Gao committed
208
209
210

    # TODO: we need unit tests & sanity checks or something to ensure that the return of `validation_docs` is stable
    docs = {}
Julen Etxaniz's avatar
Julen Etxaniz committed
211
    write_out_info = {}
Leo Gao's avatar
Leo Gao committed
212

213
214
    docs_for_decontamination = collections.defaultdict(list)

215
    # get lists of each type of request
Leo Gao's avatar
Leo Gao committed
216
    for task_name, task in task_dict_items:
Leo Gao's avatar
Leo Gao committed
217
        versions[task_name] = task.VERSION
218
        # default to test doc, fall back to val doc if validation unavailable
Leo Gao's avatar
Leo Gao committed
219
220
        # TODO: the test-fallback-to-val system isn't final, we should revisit it at some point
        if task.has_test_docs():
Leo Gao's avatar
Leo Gao committed
221
            task_doc_func = task.test_docs
Fabrizio Milo's avatar
Fabrizio Milo committed
222
            task_set = "test"  # Required for caching in the decontamination
Leo Gao's avatar
Leo Gao committed
223
        elif task.has_validation_docs():
Fabrizio Milo's avatar
Fabrizio Milo committed
224
            task_set = "val"  # Required for caching in the decontamination
Leo Gao's avatar
Leo Gao committed
225
            task_doc_func = task.validation_docs
226
227
        else:
            raise RuntimeError("Task has neither test_docs nor validation_docs")
Leo Gao's avatar
Leo Gao committed
228

Leo Gao's avatar
Leo Gao committed
229
230
231
232
        # deterministically shuffle docs and chop off the first `limit` because sometimes docs are in some kind of order
        task_docs = list(task_doc_func())
        rnd = random.Random()
        rnd.seed(42)
Jason Phang's avatar
Jason Phang committed
233
        rnd.shuffle(task_docs)
234
235
        print(f"Task: {task_name}; number of docs: {len(task_docs)}")

236
        if write_out:
237
            prompt_details = []
Leo Gao's avatar
Leo Gao committed
238

Fabrizio Milo's avatar
Fabrizio Milo committed
239
240
241
242
243
        description = (
            description_dict[task_name]
            if description_dict and task_name in description_dict
            else ""
        )
244
245
        if limit is not None:
            limit = int(len(task_docs) * limit) if limit < 1.0 else int(limit)
246

Leo Gao's avatar
Leo Gao committed
247
        for doc_id, doc in enumerate(itertools.islice(task_docs, 0, limit)):
248
            if decontaminate and task.should_decontaminate():
Fabrizio Milo's avatar
Fabrizio Milo committed
249
250
251
                docs_for_decontamination[(task_name, task_set)].append(
                    task.doc_to_decontamination_query(doc)
                )
252

Leo Gao's avatar
Leo Gao committed
253
254
            docs[(task_name, doc_id)] = doc
            ctx = task.fewshot_context(
Fabrizio Milo's avatar
Fabrizio Milo committed
255
                doc=doc, num_fewshot=num_fewshot, rnd=rnd, description=description
Leo Gao's avatar
Leo Gao committed
256
257
            )
            reqs = task.construct_requests(doc, ctx)
258

259
            if write_out:
260
261
262
263
264
265
266
267
268
                prompt_details.append({"doc_id": doc_id})

            # print the prompt for the first few documents
            if doc_id < 1:
                print(
                    f"Task: {task_name}; document {doc_id}; context prompt (starting on next line):\n{ctx}\n(end of prompt on previous line)"
                )
                print("Requests:", reqs)

269
270
            if not isinstance(reqs, (list, tuple)):
                reqs = [reqs]
Leo Gao's avatar
Leo Gao committed
271
            for i, req in enumerate(reqs):
Leo Gao's avatar
Leo Gao committed
272
                requests[req.request_type].append(req)
Leo Gao's avatar
Leo Gao committed
273
274
                # i: index in requests for a single task instance
                # doc_id: unique id that we can get back to a doc using `docs`
Leo Gao's avatar
Leo Gao committed
275
                requests_origin[req.request_type].append((i, task_name, doc, doc_id))
Leo Gao's avatar
Leo Gao committed
276

277
                if write_out:
278
279
280
281
                    prompt_details[-1][f"prompt_{i}"] = "".join(
                        (map(lambda x: "".join(x), req.args))
                    )

282
        if write_out:
Julen Etxaniz's avatar
Julen Etxaniz committed
283
            write_out_info[task_name] = prompt_details
284

285
286
    # Compare all tasks/sets at once to ensure a single training set scan
    if decontaminate:
287
        from lm_eval.decontamination.decontaminate import get_train_overlap
jon-tow's avatar
jon-tow committed
288

289
        print("Finding train/test overlap, please wait...")
Fabrizio Milo's avatar
Fabrizio Milo committed
290
291
292
        overlaps = get_train_overlap(
            docs_for_decontamination, decontamination_ngrams_path, limit
        )
293

Leo Gao's avatar
Leo Gao committed
294
295
296
297
298
    # all responses for each (task, doc)
    process_res_queue = collections.defaultdict(list)

    # execute each type of request
    for reqtype, reqs in requests.items():
299
300
301
302
        # TODO: right now, this code runs multiple separate LM requests for multiple Requests differing
        #       only in index. We could implement some kind of caching, but that would be more of a band-aid
        #       solution. we could also implement some kind of auto-grouping here;
        #       they should end up next to each other.
Leo Gao's avatar
Leo Gao committed
303

Leo Gao's avatar
Leo Gao committed
304
        print("Running", reqtype, "requests")
Leo Gao's avatar
Leo Gao committed
305
        resps = getattr(lm, reqtype)([req.args for req in reqs])
Fabrizio Milo's avatar
Fabrizio Milo committed
306
307
308
        resps = [
            x if req.index is None else x[req.index] for x, req in zip(resps, reqs)
        ]
Leo Gao's avatar
Leo Gao committed
309
310
311

        for resp, (i, task_name, doc, doc_id) in zip(resps, requests_origin[reqtype]):
            process_res_queue[(task_name, doc_id)].append((i, resp))
Fabrizio Milo's avatar
Fabrizio Milo committed
312

313
            if write_out:
Julen Etxaniz's avatar
Julen Etxaniz committed
314
                write_out_info[task_name][doc_id][f"logit_{i}"] = resp
315
316
                task = task_dict[task_name]
                if isinstance(task, lm_eval.base.MultipleChoiceTask):
Julen Etxaniz's avatar
Julen Etxaniz committed
317
                    write_out_info[task_name][doc_id]["truth"] = doc["gold"]
318
                elif isinstance(task, lm_eval.tasks.winogrande.Winogrande):
Julen Etxaniz's avatar
Julen Etxaniz committed
319
                    write_out_info[task_name][doc_id]["truth"] = task.answer_to_num[
320
321
322
                        doc["answer"]
                    ]
                else:
Julen Etxaniz's avatar
Julen Etxaniz committed
323
                    write_out_info[task_name][doc_id]["truth"] = task.doc_to_target(doc)
324

Leo Gao's avatar
Leo Gao committed
325
326
327
328
329
330
331
332
333
334
335
336
337
    vals = collections.defaultdict(list)

    # unpack results and sort back in order and return control to Task
    for (task_name, doc_id), requests in process_res_queue.items():
        requests.sort(key=lambda x: x[0])
        requests = [x[1] for x in requests]

        task = task_dict[task_name]
        doc = docs[(task_name, doc_id)]

        metrics = task.process_results(doc, requests)
        for metric, value in metrics.items():
            vals[(task_name, metric)].append(value)
338

339
            if write_out:
Julen Etxaniz's avatar
Julen Etxaniz committed
340
                write_out_info[task_name][doc_id][metric] = str(value)
341

342
343
344
345
            # Re-use the evaluation for the decontaminated set by just ignoring the overlaps
            if decontaminate and task_name in overlaps:
                if doc_id not in overlaps[task_name]:
                    vals[(task_name, metric + decontaminate_suffix)].append(value)
Fabrizio Milo's avatar
Fabrizio Milo committed
346

Leo Gao's avatar
Leo Gao committed
347
348
349
    # aggregate results
    for (task_name, metric), items in vals.items():
        task = task_dict[task_name]
Fabrizio Milo's avatar
Fabrizio Milo committed
350
        real_metric = metric  # key when looking up the metric with task.aggregation
351
        if metric.endswith(decontaminate_suffix):
Fabrizio Milo's avatar
Fabrizio Milo committed
352
353
354
            real_metric = metric.replace(
                decontaminate_suffix, ""
            )  # decontaminated still uses the same metric
355
        results[task_name][metric] = task.aggregation()[real_metric](items)
Leo Gao's avatar
Leo Gao committed
356

357
358
        # hotfix: bleu, chrf, ter seem to be really expensive to bootstrap
        # so we run them less iterations. still looking for a cleaner way to do this
359

360
        stderr = lm_eval.metrics.stderr_for_metric(
361
            metric=task.aggregation()[real_metric],
Fabrizio Milo's avatar
Fabrizio Milo committed
362
363
364
            bootstrap_iters=min(bootstrap_iters, 1000)
            if metric in ["bleu", "chrf", "ter"]
            else bootstrap_iters,
365
        )
Fabrizio Milo's avatar
Fabrizio Milo committed
366

Leo Gao's avatar
Leo Gao committed
367
368
        if stderr is not None:
            results[task_name][metric + "_stderr"] = stderr(items)
Fabrizio Milo's avatar
Fabrizio Milo committed
369

370
    if write_out:
371
372
373
        import json
        import pathlib

374
375
376
        output_base_path = (
            pathlib.Path(output_base_path)
            if output_base_path is not None
377
378
379
            else pathlib.Path(".")
        )
        try:
380
            output_base_path.mkdir(parents=True, exist_ok=False)
381
382
383
384
385
        except FileExistsError:
            pass

        for task_name, _ in task_dict_items:
            with open(
Julen Etxaniz's avatar
Julen Etxaniz committed
386
                output_base_path.joinpath(f"{task_name}_write_out_info.json"),
387
388
389
                "w",
                encoding="utf8",
            ) as fp:
Julen Etxaniz's avatar
Julen Etxaniz committed
390
                json.dump(write_out_info[task_name], fp, indent=4, ensure_ascii=False)
391

Fabrizio Milo's avatar
Fabrizio Milo committed
392
    return {"results": dict(results), "versions": dict(versions)}
393
394
395


def make_table(result_dict):
396
    """Generate table of results."""
397
398
399
400
401
402
403
404
405
406
407
408
    from pytablewriter import MarkdownTableWriter, LatexTableWriter

    md_writer = MarkdownTableWriter()
    latex_writer = LatexTableWriter()
    md_writer.headers = ["Task", "Version", "Metric", "Value", "", "Stderr"]
    latex_writer.headers = ["Task", "Version", "Metric", "Value", "", "Stderr"]

    values = []

    for k, dic in result_dict["results"].items():
        version = result_dict["versions"][k]
        for m, v in dic.items():
409
410
            if m.endswith("_stderr"):
                continue
411
412
413

            if m + "_stderr" in dic:
                se = dic[m + "_stderr"]
Fabrizio Milo's avatar
Fabrizio Milo committed
414
                values.append([k, version, m, "%.4f" % v, "±", "%.4f" % se])
415
            else:
Fabrizio Milo's avatar
Fabrizio Milo committed
416
                values.append([k, version, m, "%.4f" % v, "", ""])
417
418
419
420
421
422
423
424
            k = ""
            version = ""
    md_writer.value_matrix = values
    latex_writer.value_matrix = values

    # todo: make latex table look good
    # print(latex_writer.dumps())

425
    return md_writer.dumps()