evaluator.py 26.2 KB
Newer Older
Baber Abbasi's avatar
Baber Abbasi committed
1
import itertools
2
import json
3
import logging
Baber Abbasi's avatar
Baber Abbasi committed
4
import random
5
import time
6
7
from collections import defaultdict
from typing import TYPE_CHECKING, List, Optional, Union
Baber Abbasi's avatar
Baber Abbasi committed
8

9
import numpy as np
Baber Abbasi's avatar
Baber Abbasi committed
10
import torch
lintangsutawika's avatar
lintangsutawika committed
11

lintangsutawika's avatar
lintangsutawika committed
12
import lm_eval.api.metrics
lintangsutawika's avatar
lintangsutawika committed
13
import lm_eval.api.registry
Lintang Sutawika's avatar
Lintang Sutawika committed
14
import lm_eval.api.task
Baber Abbasi's avatar
Baber Abbasi committed
15
import lm_eval.models
16
from lm_eval.caching.cache import delete_cache
17
from lm_eval.evaluator_utils import (
Lintang Sutawika's avatar
Lintang Sutawika committed
18
    consolidate_group_results,
19
20
    consolidate_results,
    get_sample_size,
Lintang Sutawika's avatar
Lintang Sutawika committed
21
    get_subtask_list,
22
23
24
25
26
    get_task_list,
    prepare_print_tasks,
    print_writeout,
    run_task_tests,
)
KonradSzafer's avatar
KonradSzafer committed
27
from lm_eval.loggers import EvaluationTracker
28
from lm_eval.loggers.utils import add_env_info, add_tokenizer_info, get_git_commit_hash
Lintang Sutawika's avatar
Lintang Sutawika committed
29
30
31
32
from lm_eval.tasks import (
    TaskManager,
    get_task_dict,
)
33
34
35
36
37
38
39
from lm_eval.utils import (
    eval_logger,
    handle_non_serializable,
    hash_string,
    positional_deprecated,
    simple_parse_args_string,
)
40

Fabrizio Milo's avatar
Fabrizio Milo committed
41

42
43
if TYPE_CHECKING:
    from lm_eval.api.model import LM
Lintang Sutawika's avatar
Lintang Sutawika committed
44
    from lm_eval.api.task import Task
45
46


47
@positional_deprecated
Fabrizio Milo's avatar
Fabrizio Milo committed
48
49
def simple_evaluate(
    model,
50
51
    model_args: Optional[Union[str, dict]] = None,
    tasks: Optional[List[Union[str, dict, object]]] = None,
Baber Abbasi's avatar
Baber Abbasi committed
52
    num_fewshot: Optional[int] = None,
53
    batch_size: Optional[Union[int, str]] = None,
Baber Abbasi's avatar
Baber Abbasi committed
54
55
56
    max_batch_size: Optional[int] = None,
    device: Optional[str] = None,
    use_cache: Optional[str] = None,
57
58
59
    cache_requests: bool = False,
    rewrite_requests_cache: bool = False,
    delete_requests_cache: bool = False,
Baber Abbasi's avatar
Baber Abbasi committed
60
    limit: Optional[Union[int, float]] = None,
Ethan Smith's avatar
Ethan Smith committed
61
62
63
64
    bootstrap_iters: int = 100000,
    check_integrity: bool = False,
    write_out: bool = False,
    log_samples: bool = True,
KonradSzafer's avatar
KonradSzafer committed
65
66
    evaluation_tracker: Optional[EvaluationTracker] = None,
    system_instruction: Optional[str] = None,
67
    apply_chat_template: Union[bool, str] = False,
KonradSzafer's avatar
KonradSzafer committed
68
    fewshot_as_multiturn: bool = False,
69
70
    gen_kwargs: Optional[str] = None,
    task_manager: Optional[TaskManager] = None,
71
    verbosity: str = "INFO",
Baber Abbasi's avatar
Baber Abbasi committed
72
    predict_only: bool = False,
73
74
75
    random_seed: int = 0,
    numpy_random_seed: int = 1234,
    torch_random_seed: int = 1234,
76
    fewshot_random_seed: int = 1234,
Fabrizio Milo's avatar
Fabrizio Milo committed
77
):
78
    """Instantiate and evaluate a model on a list of tasks.
79

80
81
    :param model: Union[str, LM]
        Name of model or LM object, see lm_eval.models.get_model
82
83
    :param model_args: Optional[str, dict]
        String or dict arguments for each model class, see LM.create_from_arg_string and LM.create_from_arg_object.
84
        Ignored if `model` argument is a LM object.
85
    :param tasks: list[Union[str, dict, Task]]
Leo Gao's avatar
Leo Gao committed
86
        List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.
87
88
    :param num_fewshot: int
        Number of examples in few-shot context
89
    :param batch_size: int or str, optional
90
        Batch size for model
91
92
    :param max_batch_size: int, optional
        Maximal batch size to try with automatic batch size detection
93
    :param device: str, optional
94
        PyTorch device (e.g. "cpu" or "cuda:0") for running models
haileyschoelkopf's avatar
haileyschoelkopf committed
95
96
    :param use_cache: str, optional
        A path to a sqlite db file for caching model responses. `None` if not caching.
97
98
99
100
101
102
    :param cache_requests: bool, optional
        Speed up evaluation by caching the building of dataset requests. `None` if not caching.
    :param rewrite_requests_cache: bool, optional
        Rewrites all of the request cache if set to `True`. `None` if not desired.
    :param delete_requests_cache: bool, optional
        Deletes all of the request cache if set to `True`. `None` if not desired.
103
104
    :param limit: int or float, optional
        Limit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples.
105
    :param bootstrap_iters:
106
        Number of iterations for bootstrap statistics, used when calculating stderrs. set to 0 for no stderr calculations to be performed.
Stephen Hogg's avatar
Stephen Hogg committed
107
108
    :param check_integrity: bool
        Whether to run the relevant part of the test suite for the tasks
109
    :param write_out: bool
110
111
112
        If True, write out an example document and model input for checking task integrity
    :param log_samples: bool
        If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
KonradSzafer's avatar
KonradSzafer committed
113
114
    :param system_instruction: str
        System instruction to be applied to the prompt
115
116
117
118
119
    :param apply_chat_template: Union[bool, str]
        Specifies whether to apply a chat template to the prompt.
        - If set to True, the default chat template is applied.
        - If set to a string, applies the specified chat template by name.
        Defaults to False (no chat template applied).
KonradSzafer's avatar
KonradSzafer committed
120
121
    :param fewshot_as_multiturn: bool
        Whether to provide the fewshot examples as a multiturn conversation or a single user turn.
122
123
124
    :param gen_kwargs: str
        String arguments for model generation
        Ignored for all tasks with loglikelihood output_type
Baber Abbasi's avatar
Baber Abbasi committed
125
126
    :param predict_only: bool
        If true only model outputs will be generated and returned. Metrics will not be evaluated
127
128
129
130
131
132
    :param random_seed: int
        Random seed for python's random module. If set to None, the seed will not be set.
    :param numpy_random_seed: int
        Random seed for numpy. If set to None, the seed will not be set.
    :param torch_random_seed: int
        Random seed for torch. If set to None, the seed will not be set.
133
134
    :param fewshot_random_seed: int
        Random seed for fewshot sampler random generator. If set to None, the seed of generator will be set to None.
Baber Abbasi's avatar
Baber Abbasi committed
135

136
    :return
137
        Dictionary of results
138
    """
139
    eval_logger.setLevel(getattr(logging, f"{verbosity}"))
140
    start_date = time.time()
141

142
143
144
145
    if delete_requests_cache:
        eval_logger.info("Deleting requests cache...")
        delete_cache()

146
    seed_message = []
147
148
    if random_seed is not None:
        # See https://github.com/EleutherAI/lm-evaluation-harness/pull/1412
149
        seed_message.append(f"Setting random seed to {random_seed}")
150
151
152
        random.seed(random_seed)

    if numpy_random_seed is not None:
153
        seed_message.append(f"Setting numpy seed to {numpy_random_seed}")
154
155
156
        np.random.seed(numpy_random_seed)

    if torch_random_seed is not None:
157
        seed_message.append(f"Setting torch manual seed to {torch_random_seed}")
158
159
        torch.manual_seed(torch_random_seed)

160
161
162
    if seed_message:
        eval_logger.info(" | ".join(seed_message))

163
164
    if tasks is None:
        tasks = []
165
166
167
168
    if len(tasks) == 0:
        raise ValueError(
            "No tasks specified, or no tasks found. Please verify the task names."
        )
169

lintangsutawika's avatar
lintangsutawika committed
170
171
    if gen_kwargs is not None:
        gen_kwargs = simple_parse_args_string(gen_kwargs)
lintangsutawika's avatar
udate  
lintangsutawika committed
172
        eval_logger.warning(
173
174
            "generation_kwargs specified through cli, these settings will update set parameters in yaml tasks. "
            "Ensure 'do_sample=True' for non-greedy decoding!"
lintangsutawika's avatar
udate  
lintangsutawika committed
175
        )
lintangsutawika's avatar
lintangsutawika committed
176
177
178
        if gen_kwargs == "":
            gen_kwargs = None

179
    if isinstance(model, str):
Fabrizio Milo's avatar
Fabrizio Milo committed
180
        if model_args is None:
181
            eval_logger.warning("model_args not specified. Using defaults.")
Fabrizio Milo's avatar
Fabrizio Milo committed
182
            model_args = ""
183

184
        if isinstance(model_args, dict):
185
186
187
            eval_logger.info(
                f"Initializing {model} model, with arguments: {model_args}"
            )
188
189
190
191
192
193
194
195
196
197
            lm = lm_eval.api.registry.get_model(model).create_from_arg_obj(
                model_args,
                {
                    "batch_size": batch_size,
                    "max_batch_size": max_batch_size,
                    "device": device,
                },
            )

        else:
198
199
200
            eval_logger.info(
                f"Initializing {model} model, with arguments: {simple_parse_args_string(model_args)}"
            )
201
202
203
204
205
206
207
208
            lm = lm_eval.api.registry.get_model(model).create_from_arg_string(
                model_args,
                {
                    "batch_size": batch_size,
                    "max_batch_size": max_batch_size,
                    "device": device,
                },
            )
209
    else:
210
211
        if not isinstance(model, lm_eval.api.model.LM):
            raise TypeError
212
        eval_logger.info("Using pre-initialized model")
213
        lm = model
214

haileyschoelkopf's avatar
haileyschoelkopf committed
215
    if use_cache is not None:
216
        eval_logger.info(f"Using cache at {use_cache + '_rank' + str(lm.rank) + '.db'}")
haileyschoelkopf's avatar
haileyschoelkopf committed
217
218
219
220
221
        lm = lm_eval.api.model.CachingLM(
            lm,
            use_cache
            # each rank receives a different cache db.
            # necessary to avoid multiple writes to cache at once
222
223
224
            + "_rank"
            + str(lm.rank)
            + ".db",
haileyschoelkopf's avatar
haileyschoelkopf committed
225
226
        )

227
228
229
230
    if task_manager is None:
        task_manager = TaskManager(verbosity)

    task_dict = get_task_dict(tasks, task_manager)
Baber Abbasi's avatar
Baber Abbasi committed
231

Lintang Sutawika's avatar
Lintang Sutawika committed
232
233
234
235
236
237
238
239
240
241
    # helper function to recursively apply config overrides to leaf subtasks, skipping their constituent groups.
    # (setting of num_fewshot ; bypassing metric calculation ; setting fewshot seed)
    def _adjust_config(task_dict):
        adjusted_task_dict = {}
        for task_name, task_obj in task_dict.items():
            if isinstance(task_obj, dict):
                adjusted_task_dict = {
                    **adjusted_task_dict,
                    **{task_name: _adjust_config(task_obj)},
                }
242

243
            else:
Lintang Sutawika's avatar
Lintang Sutawika committed
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
                if task_obj.get_config("output_type") == "generate_until":
                    if gen_kwargs is not None:
                        task_obj.set_config(
                            key="generation_kwargs", value=gen_kwargs, update=True
                        )

                if predict_only:
                    eval_logger.info(
                        f"Processing {task_name} in output-only mode. Metrics will not be calculated!"
                    )
                    # we have to change the class properties post-hoc. This is pretty hacky.
                    task_obj.override_metric(metric_name="bypass")

                # override tasks' fewshot values to the provided num_fewshot arg value
                # except if tasks have it set to 0 manually in their configs--then we should never overwrite that
                if num_fewshot is not None:
                    if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0:
                        eval_logger.info(
                            f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored."
                        )
                    else:
                        eval_logger.warning(
                            f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}"
                        )
                        task_obj.set_config(key="num_fewshot", value=num_fewshot)
                else:
                    # if num_fewshot not provided, and the task does not define a default one, default to 0
                    if (
                        default_num_fewshot := task_obj.get_config("num_fewshot")
                    ) is None:
                        task_obj.set_config(key="num_fewshot", value=0)
                # fewshot_random_seed set for tasks, even with a default num_fewshot (e.g. in the YAML file)
                task_obj.set_fewshot_seed(seed=fewshot_random_seed)
                eval_logger.info(
                    f"Setting fewshot random generator seed to {fewshot_random_seed}"
Baber Abbasi's avatar
Baber Abbasi committed
279
                )
Lintang Sutawika's avatar
Lintang Sutawika committed
280
281
282
283
284
285

                adjusted_task_dict[task_name] = task_obj

        return adjusted_task_dict

    task_dict = _adjust_config(task_dict)
Jonathan Tow's avatar
Merge  
Jonathan Tow committed
286

Stephen Hogg's avatar
Stephen Hogg committed
287
    if check_integrity:
288
        run_task_tests(task_list=tasks)
Stephen Hogg's avatar
Stephen Hogg committed
289

Baber Abbasi's avatar
Baber Abbasi committed
290
291
292
293
294
295
    # hotfix: delete when chat_template fixed
    try:
        chat = lm.chat_template(apply_chat_template)
    except:  # noqa: E722
        chat = None

KonradSzafer's avatar
KonradSzafer committed
296
297
298
299
300
    if evaluation_tracker is not None:
        evaluation_tracker.general_config_tracker.log_experiment_args(
            model_source=model,
            model_args=model_args,
            system_instruction=system_instruction,
Baber Abbasi's avatar
Baber Abbasi committed
301
            chat_template=chat,
302
            fewshot_as_multiturn=fewshot_as_multiturn,
KonradSzafer's avatar
KonradSzafer committed
303
304
        )

305
306
307
308
    results = evaluate(
        lm=lm,
        task_dict=task_dict,
        limit=limit,
309
310
        cache_requests=cache_requests,
        rewrite_requests_cache=rewrite_requests_cache,
Niklas Muennighoff's avatar
Niklas Muennighoff committed
311
        bootstrap_iters=bootstrap_iters,
312
        write_out=write_out,
Lintang Sutawika's avatar
Lintang Sutawika committed
313
        log_samples=True if predict_only else log_samples,
KonradSzafer's avatar
KonradSzafer committed
314
315
316
        system_instruction=system_instruction,
        apply_chat_template=apply_chat_template,
        fewshot_as_multiturn=fewshot_as_multiturn,
317
        verbosity=verbosity,
318
    )
319

320
    if lm.rank == 0:
321
322
323
324
325
326
327
        if isinstance(model, str):
            model_name = model
        elif hasattr(model, "config") and hasattr(model.config, "_name_or_path"):
            model_name = model.config._name_or_path
        else:
            model_name = type(model).__name__

328
329
        # add info about the model and few shot config
        results["config"] = {
330
            "model": model_name,
331
332
            "model_args": model_args,
        }
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
        # add more detailed model info if available
        if isinstance(lm, lm_eval.models.huggingface.HFLM):
            results["config"].update(lm.get_model_info())
        # add info about execution
        results["config"].update(
            {
                "batch_size": batch_size,
                "batch_sizes": (
                    list(lm.batch_sizes.values()) if hasattr(lm, "batch_sizes") else []
                ),
                "device": device,
                "use_cache": use_cache,
                "limit": limit,
                "bootstrap_iters": bootstrap_iters,
                "gen_kwargs": gen_kwargs,
348
349
350
351
                "random_seed": random_seed,
                "numpy_seed": numpy_random_seed,
                "torch_seed": torch_random_seed,
                "fewshot_seed": fewshot_random_seed,
352
353
            }
        )
354
        results["git_hash"] = get_git_commit_hash()
355
        results["date"] = start_date
356
        add_env_info(results)  # additional environment info to results
achervyakov's avatar
achervyakov committed
357
        add_tokenizer_info(results, lm)  # additional info about tokenizer
358
359
360
        return results
    else:
        return None
361

Leo Gao's avatar
Leo Gao committed
362

363
@positional_deprecated
Fabrizio Milo's avatar
Fabrizio Milo committed
364
def evaluate(
365
    lm: "LM",
Fabrizio Milo's avatar
Fabrizio Milo committed
366
    task_dict,
Baber Abbasi's avatar
Baber Abbasi committed
367
    limit: Optional[int] = None,
368
369
    cache_requests: bool = False,
    rewrite_requests_cache: bool = False,
Baber Abbasi's avatar
Baber Abbasi committed
370
    bootstrap_iters: Optional[int] = 100000,
Ethan Smith's avatar
Ethan Smith committed
371
372
    write_out: bool = False,
    log_samples: bool = True,
KonradSzafer's avatar
KonradSzafer committed
373
    system_instruction: Optional[str] = None,
374
    apply_chat_template: Union[bool, str] = False,
KonradSzafer's avatar
KonradSzafer committed
375
    fewshot_as_multiturn: bool = False,
376
    verbosity: str = "INFO",
Fabrizio Milo's avatar
Fabrizio Milo committed
377
):
378
379
380
381
382
    """Instantiate and evaluate a model on a list of tasks.

    :param lm: obj
        Language Model
    :param task_dict: dict[str, Task]
haileyschoelkopf's avatar
haileyschoelkopf committed
383
        Dictionary of tasks. Tasks will be taken to have name type(task).config.task .
384
385
386
    :param limit: int, optional
        Limit the number of examples per task (only use this for testing)
    :param bootstrap_iters:
387
        Number of iterations for bootstrap statistics, used when calculating stderr. Set to 0 for skipping all stderr calculations.
388
    :param write_out: bool
389
390
391
        If True, write out an example document and model input for checking task integrity
    :param log_samples: bool
        If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
KonradSzafer's avatar
KonradSzafer committed
392
393
    :param system_instruction: str
        System instruction to be applied to the prompt
394
395
396
397
398
    :param apply_chat_template: Union[bool, str]
        Specifies whether to apply a chat template to the prompt.
        - If set to True, the default chat template is applied.
        - If set to a string, applies the specified chat template by name.
        Defaults to False (no chat template applied).
KonradSzafer's avatar
KonradSzafer committed
399
400
    :param fewshot_as_multiturn: bool
        Whether to provide the fewshot examples as a multiturn conversation or a single user turn.
401
402
403
    :return
        Dictionary of results
    """
404

405
    eval_logger.setLevel(getattr(logging, f"{verbosity}"))
406

407
    # tracks all Instances/requests a model must generate output on.
408
    requests = defaultdict(list)
409
410
    # stores the amount to pad out reqs per req. type so that
    # number of fwd passes per distributed rank is equal
411
    padding_requests = defaultdict(int)
412

413
    # get lists of group hierarchy and each type of request
Lintang Sutawika's avatar
Lintang Sutawika committed
414
    eval_tasks = get_task_list(task_dict)
415
    if not log_samples:
416
        if not all(
417
418
            "bypass" not in getattr(task_output.task, "_metric_fn_list", {}).keys()
            for task_output in eval_tasks
419
420
        ):
            raise ValueError("log_samples must be True for 'bypass' metric-only tasks")
421
422
423
    for task_output in eval_tasks:
        task: Task = task_output.task
        limit = get_sample_size(task, limit)
424
425
426
427
428
429
        task.build_all_requests(
            limit=limit,
            rank=lm.rank,
            world_size=lm.world_size,
            cache_requests=cache_requests,
            rewrite_requests_cache=rewrite_requests_cache,
KonradSzafer's avatar
KonradSzafer committed
430
            system_instruction=system_instruction,
431
            apply_chat_template=bool(apply_chat_template),
KonradSzafer's avatar
KonradSzafer committed
432
            fewshot_as_multiturn=fewshot_as_multiturn,
433
434
435
436
437
438
            chat_template=getattr(lm, "apply_chat_template")
            if apply_chat_template
            else None,
            tokenizer_name=getattr(lm, "tokenizer_name", "")
            if apply_chat_template
            else "",
439
        )
440
        eval_logger.debug(
441
            f"Task: {task_output.task_name}; number of requests on this rank: {len(task.instances)}"
haileyschoelkopf's avatar
haileyschoelkopf committed
442
443
        )
        if write_out:
444
            print_writeout(task)
445
        # aggregate Instances by LM method requested to get output.
lintangsutawika's avatar
lintangsutawika committed
446
447
448
        for instance in task.instances:
            reqtype = instance.request_type
            requests[reqtype].append(instance)
449
450

        if lm.world_size > 1:
451
452
453
454
            instances_rnk = torch.tensor(len(task._instances), device=lm.device)
            gathered_item = (
                lm.accelerator.gather(instances_rnk).cpu().detach().numpy().tolist()
            )
455
456
457
458
459
460
            # "multiple_choice" task types dispatch (several) "loglikelihood" request types
            reqtype = (
                "loglikelihood"
                if task.OUTPUT_TYPE == "multiple_choice"
                else task.OUTPUT_TYPE
            )
461
            # compute number of pseudo-batches to pad with (FSDP/DDP require even batches among ranks)
462
            numpad = max(gathered_item) - gathered_item[lm.rank]
463
464
            # todo: may not account for padding in cases like SquadV2 which has multiple req types
            padding_requests[reqtype] += numpad
465

466
    ### Run LM on inputs, get all outputs ###
Leo Gao's avatar
Leo Gao committed
467
468
    # execute each type of request
    for reqtype, reqs in requests.items():
469
        eval_logger.info(f"Running {reqtype} requests")
470
471
472
473
        # create `K` copies of each request `req` based off `K = req.repeats`
        cloned_reqs = []
        for req in reqs:
            cloned_reqs.extend([req] * req.repeats)
lintangsutawika's avatar
lintangsutawika committed
474

475
476
        if (lm.world_size > 1) and (padding_requests[reqtype] > 0):
            for _ in range(padding_requests[reqtype]):
477
478
                cloned_reqs.extend([req] * req.repeats)

479
480
481
482
483
484
485
        # run requests through model
        resps = getattr(lm, reqtype)(cloned_reqs)

        # put responses from model into a list of length K for each request.
        for x, req in zip(resps, cloned_reqs):
            req.resps.append(x)

486
487
        if lm.world_size > 1:
            lm.accelerator.wait_for_everyone()
488

489
490
    RANK = lm.rank
    WORLD_SIZE = lm.world_size
491
492
    ### Postprocess outputs ###
    # TODO: del model here, maybe (idea: allow user to specify device of e.g. reward model separately)
493
494
    for task_output in eval_tasks:
        task = task_output.task
495
496
        task.apply_filters()

497
498
        ### Collect values of metrics on all datapoints ###
        # # unpack results and sort back in order and return control to Task
haileyschoelkopf's avatar
haileyschoelkopf committed
499
        # TODO: make it possible to use a different metric per filter
500
        # Pre-process task.instances to group by doc_id
501
        instances_by_doc_id = defaultdict(list)
502
503
504
505
506
        for instance in task.instances:
            instances_by_doc_id[instance.doc_id].append(instance)
        # Sort instances within each group
        for instances in instances_by_doc_id.values():
            instances.sort(key=lambda x: x.idx)
haileyschoelkopf's avatar
haileyschoelkopf committed
507
        # iterate over different filters used
508
509
510
        for filter_key in task.instances[0].filtered_resps.keys():
            doc_iterator = task.doc_iterator(
                rank=RANK, limit=limit, world_size=WORLD_SIZE
511
            )
512
            for doc_id, doc in doc_iterator:
513
                requests = instances_by_doc_id[doc_id]
lintangsutawika's avatar
lintangsutawika committed
514
                metrics = task.process_results(
515
                    doc, [req.filtered_resps[filter_key] for req in requests]
lintangsutawika's avatar
lintangsutawika committed
516
                )
517
518
519
520
521
522
523
524
                if log_samples:
                    target = task.doc_to_target(doc)
                    example = {
                        "doc_id": doc_id,
                        "doc": doc,
                        "target": target,
                        "arguments": [req.args for req in requests],
                        "resps": [req.resps for req in requests],
525
526
527
                        "filtered_resps": [
                            req.filtered_resps[filter_key] for req in requests
                        ],
528
529
530
531
532
533
534
535
536
537
                        "doc_hash": hash_string(
                            json.dumps(
                                requests[0].doc,
                                indent=2,
                                default=handle_non_serializable,
                                ensure_ascii=False,
                            )
                        ),
                        "prompt_hash": hash_string(requests[0].arguments[0]),
                        "target_hash": hash_string(str(target)),
538
539
                    }
                    example.update(metrics)
540
                    task_output.logged_samples.append(example)
541
                for metric, value in metrics.items():
542
                    task_output.sample_metrics[(metric, filter_key)].append(value)
543

544
545
    if WORLD_SIZE > 1:
        # if multigpu, then gather data across all ranks to rank 0
546
        # first gather logged samples across all ranks
547
548
549
550
551
552
553
554
        for task_output in eval_tasks:
            if log_samples:
                # for task_name, task_samples in list(samples.items()):
                full_samples = [None] * WORLD_SIZE if RANK == 0 else None
                torch.distributed.gather_object(
                    obj=task_output.logged_samples,
                    object_gather_list=full_samples,
                    dst=0,
555
                )
556

557
558
559
560
                if RANK == 0:
                    task_output.logged_samples = list(
                        itertools.chain.from_iterable(full_samples)
                    )
561

562
563
564
565
566
567
568
            # then collect metrics across all ranks
            for metrics in task_output.sample_metrics:
                metric_list = [None] * WORLD_SIZE if RANK == 0 else None
                torch.distributed.gather_object(
                    obj=task_output.sample_metrics[metrics],
                    object_gather_list=metric_list,
                    dst=0,
569
                )
570
571
572
573
                if RANK == 0:
                    task_output.sample_metrics[metrics] = list(
                        itertools.chain.from_iterable(metric_list)
                    )
574

575
    if RANK == 0:
576
577
        ### Aggregate results over all datapoints ###
        # aggregate results ; run bootstrap CIs
578
579
        for task_output in eval_tasks:
            task_output.calculate_aggregate_metric(bootstrap_iters=bootstrap_iters)
580
581
582
583
584
585
586
587
        (
            results,
            samples,
            configs,
            versions,
            num_fewshot,
            higher_is_better,
        ) = consolidate_results(eval_tasks)
Fabrizio Milo's avatar
Fabrizio Milo committed
588

589
        ### Calculate group metrics ###
lintangsutawika's avatar
lintangsutawika committed
590
        if bool(results):
Lintang Sutawika's avatar
Lintang Sutawika committed
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
            results, versions, show_group_table, *_ = consolidate_group_results(
                results, versions, task_dict
            )

        results_agg, group_agg = prepare_print_tasks(task_dict, results)
        subtask_list = get_subtask_list(task_dict)

        # collect all higher_is_better values for metrics
        # in the group's subtasks.
        # TODO: clean this up ; unify with the below metric_list loop?
        _higher_is_better = {}
        for group, task_list in subtask_list.items():
            if (
                len(task_list) != 0
            ):  # subtask list will list "task_name": [] for solo tasks
606
607
608
609
                for task in task_list:
                    for m, h in higher_is_better[task].items():
                        if m not in _higher_is_better.keys():
                            _higher_is_better[m] = h
lintangsutawika's avatar
lintangsutawika committed
610

Lintang Sutawika's avatar
Lintang Sutawika committed
611
612
613
614
615
616
617
618
619
620
                        if (
                            m in _higher_is_better
                            and _higher_is_better[m] is not None
                            and _higher_is_better[m] != h
                        ):
                            eval_logger.warning(
                                f"Higher_is_better values for metric {m} in group {group} are not consistent. Defaulting to None."
                            )
                            _higher_is_better[m] = None
                higher_is_better[group] = _higher_is_better
621

622
        results_dict = {
623
            "results": dict(results_agg.items()),
Lintang Sutawika's avatar
Lintang Sutawika committed
624
625
626
627
628
629
            **(
                {"groups": dict(group_agg.items())}
                if (bool(group_agg) & show_group_table)
                else {}
            ),
            "group_subtasks": dict(reversed(subtask_list.items())),
630
631
            "configs": dict(sorted(configs.items())),
            "versions": dict(sorted(versions.items())),
632
            "n-shot": dict(sorted(num_fewshot.items())),
633
            "higher_is_better": dict(sorted(higher_is_better.items())),
634
635
636
            "n-samples": {
                task_output.task_name: {
                    "original": len(task_output.task.eval_docs),
KonradSzafer's avatar
KonradSzafer committed
637
638
639
640
                    "effective": min(
                        limit if limit else len(task_output.task.eval_docs),
                        len(task_output.task.eval_docs),
                    ),
641
642
643
                }
                for task_output in eval_tasks
            },
644
        }
645
646
647
648
        if log_samples:
            results_dict["samples"] = dict(samples)

        return results_dict
Fabrizio Milo's avatar
Fabrizio Milo committed
649

650
651
    else:
        return None
652
653
654
655


def request_caching_arg_to_dict(cache_requests: str) -> dict:
    request_caching_args = {
656
657
658
        "cache_requests": cache_requests in {"true", "refresh"},
        "rewrite_requests_cache": cache_requests == "refresh",
        "delete_requests_cache": cache_requests == "delete",
659
660
661
    }

    return request_caching_args