evaluator.py 25.5 KB
Newer Older
Baber Abbasi's avatar
Baber Abbasi committed
1
import itertools
2
import json
3
import logging
Baber Abbasi's avatar
Baber Abbasi committed
4
import random
5
import time
6
7
from collections import defaultdict
from typing import TYPE_CHECKING, List, Optional, Union
Baber Abbasi's avatar
Baber Abbasi committed
8

9
import numpy as np
Baber Abbasi's avatar
Baber Abbasi committed
10
import torch
lintangsutawika's avatar
lintangsutawika committed
11

lintangsutawika's avatar
lintangsutawika committed
12
import lm_eval.api.metrics
lintangsutawika's avatar
lintangsutawika committed
13
import lm_eval.api.registry
Baber Abbasi's avatar
Baber Abbasi committed
14
import lm_eval.models
15
from lm_eval.caching.cache import delete_cache
16
17
18
19
20
21
22
23
from lm_eval.evaluator_utils import (
    consolidate_results,
    get_sample_size,
    get_task_list,
    prepare_print_tasks,
    print_writeout,
    run_task_tests,
)
24
from lm_eval.logging.utils import add_env_info, get_git_commit_hash
Baber Abbasi's avatar
Baber Abbasi committed
25
from lm_eval.tasks import TaskManager, get_task_dict
26
27
28
29
30
31
32
from lm_eval.utils import (
    eval_logger,
    handle_non_serializable,
    hash_string,
    positional_deprecated,
    simple_parse_args_string,
)
33

Fabrizio Milo's avatar
Fabrizio Milo committed
34

35
36
37
38
39
if TYPE_CHECKING:
    from lm_eval.api.model import LM
    from lm_eval.tasks import Task


40
@positional_deprecated
Fabrizio Milo's avatar
Fabrizio Milo committed
41
42
def simple_evaluate(
    model,
43
44
    model_args: Optional[Union[str, dict]] = None,
    tasks: Optional[List[Union[str, dict, object]]] = None,
Baber Abbasi's avatar
Baber Abbasi committed
45
46
47
48
49
    num_fewshot: Optional[int] = None,
    batch_size: Optional[int] = None,
    max_batch_size: Optional[int] = None,
    device: Optional[str] = None,
    use_cache: Optional[str] = None,
50
51
52
    cache_requests: bool = False,
    rewrite_requests_cache: bool = False,
    delete_requests_cache: bool = False,
Baber Abbasi's avatar
Baber Abbasi committed
53
    limit: Optional[Union[int, float]] = None,
Ethan Smith's avatar
Ethan Smith committed
54
55
56
57
    bootstrap_iters: int = 100000,
    check_integrity: bool = False,
    write_out: bool = False,
    log_samples: bool = True,
Konrad's avatar
Konrad committed
58
    system_instruction: Optional[str] = None,
Konrad's avatar
Konrad committed
59
    apply_chat_template: bool = False,
Konrad's avatar
Konrad committed
60
    fewshot_as_multiturn: bool = False,
61
62
    gen_kwargs: Optional[str] = None,
    task_manager: Optional[TaskManager] = None,
63
    verbosity: str = "INFO",
Baber Abbasi's avatar
Baber Abbasi committed
64
    predict_only: bool = False,
65
66
67
    random_seed: int = 0,
    numpy_random_seed: int = 1234,
    torch_random_seed: int = 1234,
68
    fewshot_random_seed: int = 1234,
Fabrizio Milo's avatar
Fabrizio Milo committed
69
):
70
    """Instantiate and evaluate a model on a list of tasks.
71

72
73
    :param model: Union[str, LM]
        Name of model or LM object, see lm_eval.models.get_model
74
75
    :param model_args: Optional[str, dict]
        String or dict arguments for each model class, see LM.create_from_arg_string and LM.create_from_arg_object.
76
        Ignored if `model` argument is a LM object.
77
    :param tasks: list[Union[str, dict, Task]]
Leo Gao's avatar
Leo Gao committed
78
        List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.
79
80
    :param num_fewshot: int
        Number of examples in few-shot context
81
    :param batch_size: int or str, optional
82
        Batch size for model
83
84
    :param max_batch_size: int, optional
        Maximal batch size to try with automatic batch size detection
85
    :param device: str, optional
86
        PyTorch device (e.g. "cpu" or "cuda:0") for running models
haileyschoelkopf's avatar
haileyschoelkopf committed
87
88
    :param use_cache: str, optional
        A path to a sqlite db file for caching model responses. `None` if not caching.
89
90
91
92
93
94
    :param cache_requests: bool, optional
        Speed up evaluation by caching the building of dataset requests. `None` if not caching.
    :param rewrite_requests_cache: bool, optional
        Rewrites all of the request cache if set to `True`. `None` if not desired.
    :param delete_requests_cache: bool, optional
        Deletes all of the request cache if set to `True`. `None` if not desired.
95
96
    :param limit: int or float, optional
        Limit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples.
97
98
    :param bootstrap_iters:
        Number of iterations for bootstrap statistics
Stephen Hogg's avatar
Stephen Hogg committed
99
100
    :param check_integrity: bool
        Whether to run the relevant part of the test suite for the tasks
101
    :param write_out: bool
102
103
104
        If True, write out an example document and model input for checking task integrity
    :param log_samples: bool
        If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
Konrad's avatar
Konrad committed
105
106
    :param system_instruction: str
        System instruction to be applied to the prompt
Konrad's avatar
Konrad committed
107
108
    :param apply_chat_template: bool
        If True, apply chat template to the prompt
Konrad's avatar
Konrad committed
109
110
    :param fewshot_as_multiturn: bool
        Whether to provide the fewshot examples as a multiturn conversation or a single user turn.
111
112
113
    :param gen_kwargs: str
        String arguments for model generation
        Ignored for all tasks with loglikelihood output_type
Baber Abbasi's avatar
Baber Abbasi committed
114
115
    :param predict_only: bool
        If true only model outputs will be generated and returned. Metrics will not be evaluated
116
117
118
119
120
121
    :param random_seed: int
        Random seed for python's random module. If set to None, the seed will not be set.
    :param numpy_random_seed: int
        Random seed for numpy. If set to None, the seed will not be set.
    :param torch_random_seed: int
        Random seed for torch. If set to None, the seed will not be set.
122
123
    :param fewshot_random_seed: int
        Random seed for fewshot sampler random generator. If set to None, the seed of generator will be set to None.
Baber Abbasi's avatar
Baber Abbasi committed
124

125
    :return
126
        Dictionary of results
127
    """
128
    eval_logger.setLevel(getattr(logging, f"{verbosity}"))
129
    start_date = time.time()
130

131
132
133
134
    if delete_requests_cache:
        eval_logger.info("Deleting requests cache...")
        delete_cache()

135
    seed_message = []
136
137
    if random_seed is not None:
        # See https://github.com/EleutherAI/lm-evaluation-harness/pull/1412
138
        seed_message.append(f"Setting random seed to {random_seed}")
139
140
141
        random.seed(random_seed)

    if numpy_random_seed is not None:
142
        seed_message.append(f"Setting numpy seed to {numpy_random_seed}")
143
144
145
        np.random.seed(numpy_random_seed)

    if torch_random_seed is not None:
146
        seed_message.append(f"Setting torch manual seed to {torch_random_seed}")
147
148
        torch.manual_seed(torch_random_seed)

149
150
151
    if seed_message:
        eval_logger.info(" | ".join(seed_message))

152
153
    if tasks is None:
        tasks = []
154
155
156
157
    if len(tasks) == 0:
        raise ValueError(
            "No tasks specified, or no tasks found. Please verify the task names."
        )
158

lintangsutawika's avatar
lintangsutawika committed
159
160
    if gen_kwargs is not None:
        gen_kwargs = simple_parse_args_string(gen_kwargs)
lintangsutawika's avatar
udate  
lintangsutawika committed
161
        eval_logger.warning(
162
163
            "generation_kwargs specified through cli, these settings will update set parameters in yaml tasks. "
            "Ensure 'do_sample=True' for non-greedy decoding!"
lintangsutawika's avatar
udate  
lintangsutawika committed
164
        )
lintangsutawika's avatar
lintangsutawika committed
165
166
167
        if gen_kwargs == "":
            gen_kwargs = None

168
    if isinstance(model, str):
Fabrizio Milo's avatar
Fabrizio Milo committed
169
        if model_args is None:
170
            eval_logger.warning("model_args not specified. Using defaults.")
Fabrizio Milo's avatar
Fabrizio Milo committed
171
            model_args = ""
172

173
        if isinstance(model_args, dict):
174
175
176
            eval_logger.info(
                f"Initializing {model} model, with arguments: {model_args}"
            )
177
178
179
180
181
182
183
184
185
186
            lm = lm_eval.api.registry.get_model(model).create_from_arg_obj(
                model_args,
                {
                    "batch_size": batch_size,
                    "max_batch_size": max_batch_size,
                    "device": device,
                },
            )

        else:
187
188
189
            eval_logger.info(
                f"Initializing {model} model, with arguments: {simple_parse_args_string(model_args)}"
            )
190
191
192
193
194
195
196
197
            lm = lm_eval.api.registry.get_model(model).create_from_arg_string(
                model_args,
                {
                    "batch_size": batch_size,
                    "max_batch_size": max_batch_size,
                    "device": device,
                },
            )
198
    else:
199
200
        if not isinstance(model, lm_eval.api.model.LM):
            raise TypeError
201
        eval_logger.info("Using pre-initialized model")
202
        lm = model
203

haileyschoelkopf's avatar
haileyschoelkopf committed
204
    if use_cache is not None:
205
        eval_logger.info(f"Using cache at {use_cache + '_rank' + str(lm.rank) + '.db'}")
haileyschoelkopf's avatar
haileyschoelkopf committed
206
207
208
209
210
        lm = lm_eval.api.model.CachingLM(
            lm,
            use_cache
            # each rank receives a different cache db.
            # necessary to avoid multiple writes to cache at once
211
212
213
            + "_rank"
            + str(lm.rank)
            + ".db",
haileyschoelkopf's avatar
haileyschoelkopf committed
214
215
        )

216
217
218
219
    if task_manager is None:
        task_manager = TaskManager(verbosity)

    task_dict = get_task_dict(tasks, task_manager)
220
    for task_name in task_dict.keys():
lintangsutawika's avatar
lintangsutawika committed
221
        task_obj = task_dict[task_name]
222
        if isinstance(task_obj, tuple):
223
            _, task_obj = task_obj
224
225
            if task_obj is None:
                continue
lintangsutawika's avatar
lintangsutawika committed
226

Baber Abbasi's avatar
Baber Abbasi committed
227
228
        if task_obj.get_config("output_type") == "generate_until":
            if gen_kwargs is not None:
Baber Abbasi's avatar
Baber Abbasi committed
229
                task_obj.set_config(
Baber Abbasi's avatar
Baber Abbasi committed
230
231
232
                    key="generation_kwargs", value=gen_kwargs, update=True
                )

233
234
235
236
237
238
239
        if predict_only:
            log_samples = True
            eval_logger.info(
                f"Processing {task_name} in output-only mode. Metrics will not be calculated!"
            )
            # we have to change the class properties post-hoc. This is pretty hacky.
            task_obj.override_metric(metric_name="bypass")
240

241
242
        # override tasks' fewshot values to the provided num_fewshot arg value
        # except if tasks have it set to 0 manually in their configs--then we should never overwrite that
243
        if num_fewshot is not None:
Baber Abbasi's avatar
Baber Abbasi committed
244
            if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0:
245
246
247
                eval_logger.info(
                    f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored."
                )
248
            else:
Baber Abbasi's avatar
Baber Abbasi committed
249
250
251
                eval_logger.warning(
                    f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}"
                )
Baber Abbasi's avatar
Baber Abbasi committed
252
                task_obj.set_config(key="num_fewshot", value=num_fewshot)
253
254
255
256
            task_obj.set_fewshot_seed(seed=fewshot_random_seed)
            eval_logger.info(
                f"Setting fewshot random generator seed to {fewshot_random_seed}"
            )
257
258
259
260
        else:
            # if num_fewshot not provided, and the task does not define a default one, default to 0
            if (default_num_fewshot := task_obj.get_config("num_fewshot")) is None:
                task_obj.set_config(key="num_fewshot", value=0)
Jonathan Tow's avatar
Merge  
Jonathan Tow committed
261

Stephen Hogg's avatar
Stephen Hogg committed
262
    if check_integrity:
263
        run_task_tests(task_list=tasks)
Stephen Hogg's avatar
Stephen Hogg committed
264

265
266
267
268
    results = evaluate(
        lm=lm,
        task_dict=task_dict,
        limit=limit,
269
270
        cache_requests=cache_requests,
        rewrite_requests_cache=rewrite_requests_cache,
Niklas Muennighoff's avatar
Niklas Muennighoff committed
271
        bootstrap_iters=bootstrap_iters,
272
        write_out=write_out,
273
        log_samples=log_samples,
Konrad's avatar
Konrad committed
274
        system_instruction=system_instruction,
Konrad's avatar
Konrad committed
275
        apply_chat_template=apply_chat_template,
Konrad's avatar
Konrad committed
276
        fewshot_as_multiturn=fewshot_as_multiturn,
277
        verbosity=verbosity,
278
    )
279

280
    if lm.rank == 0:
281
282
283
284
285
286
287
        if isinstance(model, str):
            model_name = model
        elif hasattr(model, "config") and hasattr(model.config, "_name_or_path"):
            model_name = model.config._name_or_path
        else:
            model_name = type(model).__name__

288
289
        # add info about the model and few shot config
        results["config"] = {
290
            "model": model_name,
291
292
            "model_args": model_args,
        }
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
        # add more detailed model info if available
        if isinstance(lm, lm_eval.models.huggingface.HFLM):
            results["config"].update(lm.get_model_info())
        # add info about execution
        results["config"].update(
            {
                "batch_size": batch_size,
                "batch_sizes": (
                    list(lm.batch_sizes.values()) if hasattr(lm, "batch_sizes") else []
                ),
                "device": device,
                "use_cache": use_cache,
                "limit": limit,
                "bootstrap_iters": bootstrap_iters,
                "gen_kwargs": gen_kwargs,
308
309
310
311
                "random_seed": random_seed,
                "numpy_seed": numpy_random_seed,
                "torch_seed": torch_random_seed,
                "fewshot_seed": fewshot_random_seed,
312
313
            }
        )
314
        results["git_hash"] = get_git_commit_hash()
315
        results["date"] = start_date
316
        add_env_info(results)  # additional environment info to results
317
318
319
        return results
    else:
        return None
320

Leo Gao's avatar
Leo Gao committed
321

322
@positional_deprecated
Fabrizio Milo's avatar
Fabrizio Milo committed
323
def evaluate(
324
    lm: "LM",
Fabrizio Milo's avatar
Fabrizio Milo committed
325
    task_dict,
Baber Abbasi's avatar
Baber Abbasi committed
326
    limit: Optional[int] = None,
327
328
    cache_requests: bool = False,
    rewrite_requests_cache: bool = False,
Baber Abbasi's avatar
Baber Abbasi committed
329
    bootstrap_iters: Optional[int] = 100000,
Ethan Smith's avatar
Ethan Smith committed
330
331
    write_out: bool = False,
    log_samples: bool = True,
Konrad's avatar
Konrad committed
332
    system_instruction: Optional[str] = None,
Konrad's avatar
Konrad committed
333
    apply_chat_template: bool = False,
Konrad's avatar
Konrad committed
334
    fewshot_as_multiturn: bool = False,
335
    verbosity: str = "INFO",
Fabrizio Milo's avatar
Fabrizio Milo committed
336
):
337
338
339
340
341
    """Instantiate and evaluate a model on a list of tasks.

    :param lm: obj
        Language Model
    :param task_dict: dict[str, Task]
haileyschoelkopf's avatar
haileyschoelkopf committed
342
        Dictionary of tasks. Tasks will be taken to have name type(task).config.task .
343
344
345
346
    :param limit: int, optional
        Limit the number of examples per task (only use this for testing)
    :param bootstrap_iters:
        Number of iterations for bootstrap statistics
347
    :param write_out: bool
348
349
350
        If True, write out an example document and model input for checking task integrity
    :param log_samples: bool
        If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
Konrad's avatar
Konrad committed
351
352
    :param system_instruction: str
        System instruction to be applied to the prompt
Konrad's avatar
Konrad committed
353
354
    :param apply_chat_template: bool
        If True, apply chat template to the prompt
Konrad's avatar
Konrad committed
355
356
    :param fewshot_as_multiturn: bool
        Whether to provide the fewshot examples as a multiturn conversation or a single user turn.
357
358
359
    :return
        Dictionary of results
    """
360

361
    eval_logger.setLevel(getattr(logging, f"{verbosity}"))
362

363
    # tracks all Instances/requests a model must generate output on.
364
    requests = defaultdict(list)
365
366
    # stores the amount to pad out reqs per req. type so that
    # number of fwd passes per distributed rank is equal
367
    padding_requests = defaultdict(int)
368

369
370
371
    # get lists of group hierarchy and each type of request
    task_hierarchy, eval_tasks = get_task_list(task_dict)
    if not log_samples:
372
        if not all(
373
374
            "bypass" not in getattr(task_output.task, "_metric_fn_list", {}).keys()
            for task_output in eval_tasks
375
376
        ):
            raise ValueError("log_samples must be True for 'bypass' metric-only tasks")
377
378
379
    for task_output in eval_tasks:
        task: Task = task_output.task
        limit = get_sample_size(task, limit)
380
381
382
383
384
385
        task.build_all_requests(
            limit=limit,
            rank=lm.rank,
            world_size=lm.world_size,
            cache_requests=cache_requests,
            rewrite_requests_cache=rewrite_requests_cache,
Konrad's avatar
Konrad committed
386
            system_instruction=system_instruction,
Konrad's avatar
Konrad committed
387
            apply_chat_template=apply_chat_template,
Konrad's avatar
Konrad committed
388
            fewshot_as_multiturn=fewshot_as_multiturn,
Konrad's avatar
Konrad committed
389
            tokenizer=lm.tokenizer if hasattr(lm, "tokenizer") else None,
390
        )
391
        eval_logger.debug(
392
            f"Task: {task_output.task_name}; number of requests on this rank: {len(task.instances)}"
haileyschoelkopf's avatar
haileyschoelkopf committed
393
394
        )
        if write_out:
395
            print_writeout(task)
396
        # aggregate Instances by LM method requested to get output.
lintangsutawika's avatar
lintangsutawika committed
397
398
399
        for instance in task.instances:
            reqtype = instance.request_type
            requests[reqtype].append(instance)
400
401

        if lm.world_size > 1:
402
403
404
405
            instances_rnk = torch.tensor(len(task._instances), device=lm.device)
            gathered_item = (
                lm.accelerator.gather(instances_rnk).cpu().detach().numpy().tolist()
            )
406
407
408
409
410
411
            # "multiple_choice" task types dispatch (several) "loglikelihood" request types
            reqtype = (
                "loglikelihood"
                if task.OUTPUT_TYPE == "multiple_choice"
                else task.OUTPUT_TYPE
            )
412
            # compute number of pseudo-batches to pad with (FSDP/DDP require even batches among ranks)
413
            numpad = max(gathered_item) - gathered_item[lm.rank]
414
415
            # todo: may not account for padding in cases like SquadV2 which has multiple req types
            padding_requests[reqtype] += numpad
416

417
    ### Run LM on inputs, get all outputs ###
Leo Gao's avatar
Leo Gao committed
418
419
    # execute each type of request
    for reqtype, reqs in requests.items():
420
        eval_logger.info(f"Running {reqtype} requests")
421
422
423
424
        # create `K` copies of each request `req` based off `K = req.repeats`
        cloned_reqs = []
        for req in reqs:
            cloned_reqs.extend([req] * req.repeats)
lintangsutawika's avatar
lintangsutawika committed
425

426
427
        if (lm.world_size > 1) and (padding_requests[reqtype] > 0):
            for _ in range(padding_requests[reqtype]):
428
429
                cloned_reqs.extend([req] * req.repeats)

430
431
432
433
434
435
436
        # run requests through model
        resps = getattr(lm, reqtype)(cloned_reqs)

        # put responses from model into a list of length K for each request.
        for x, req in zip(resps, cloned_reqs):
            req.resps.append(x)

437
438
        if lm.world_size > 1:
            lm.accelerator.wait_for_everyone()
439

440
441
    RANK = lm.rank
    WORLD_SIZE = lm.world_size
442
443
    ### Postprocess outputs ###
    # TODO: del model here, maybe (idea: allow user to specify device of e.g. reward model separately)
444
445
    for task_output in eval_tasks:
        task = task_output.task
446
447
        task.apply_filters()

448
449
        ### Collect values of metrics on all datapoints ###
        # # unpack results and sort back in order and return control to Task
haileyschoelkopf's avatar
haileyschoelkopf committed
450
        # TODO: make it possible to use a different metric per filter
451
        # Pre-process task.instances to group by doc_id
452
        instances_by_doc_id = defaultdict(list)
453
454
455
456
457
        for instance in task.instances:
            instances_by_doc_id[instance.doc_id].append(instance)
        # Sort instances within each group
        for instances in instances_by_doc_id.values():
            instances.sort(key=lambda x: x.idx)
haileyschoelkopf's avatar
haileyschoelkopf committed
458
        # iterate over different filters used
459
460
461
        for filter_key in task.instances[0].filtered_resps.keys():
            doc_iterator = task.doc_iterator(
                rank=RANK, limit=limit, world_size=WORLD_SIZE
462
            )
463
            for doc_id, doc in doc_iterator:
464
                requests = instances_by_doc_id[doc_id]
lintangsutawika's avatar
lintangsutawika committed
465
                metrics = task.process_results(
466
                    doc, [req.filtered_resps[filter_key] for req in requests]
lintangsutawika's avatar
lintangsutawika committed
467
                )
468
469
470
471
472
473
474
475
                if log_samples:
                    target = task.doc_to_target(doc)
                    example = {
                        "doc_id": doc_id,
                        "doc": doc,
                        "target": target,
                        "arguments": [req.args for req in requests],
                        "resps": [req.resps for req in requests],
476
477
478
                        "filtered_resps": [
                            req.filtered_resps[filter_key] for req in requests
                        ],
479
480
481
482
483
484
485
486
487
488
                        "doc_hash": hash_string(
                            json.dumps(
                                requests[0].doc,
                                indent=2,
                                default=handle_non_serializable,
                                ensure_ascii=False,
                            )
                        ),
                        "prompt_hash": hash_string(requests[0].arguments[0]),
                        "target_hash": hash_string(str(target)),
489
490
                    }
                    example.update(metrics)
491
                    task_output.logged_samples.append(example)
492
                for metric, value in metrics.items():
493
                    task_output.sample_metrics[(metric, filter_key)].append(value)
494

495
496
    if WORLD_SIZE > 1:
        # if multigpu, then gather data across all ranks to rank 0
497
        # first gather logged samples across all ranks
498
499
500
501
502
503
504
505
        for task_output in eval_tasks:
            if log_samples:
                # for task_name, task_samples in list(samples.items()):
                full_samples = [None] * WORLD_SIZE if RANK == 0 else None
                torch.distributed.gather_object(
                    obj=task_output.logged_samples,
                    object_gather_list=full_samples,
                    dst=0,
506
                )
507

508
509
510
511
                if RANK == 0:
                    task_output.logged_samples = list(
                        itertools.chain.from_iterable(full_samples)
                    )
512

513
514
515
516
517
518
519
            # then collect metrics across all ranks
            for metrics in task_output.sample_metrics:
                metric_list = [None] * WORLD_SIZE if RANK == 0 else None
                torch.distributed.gather_object(
                    obj=task_output.sample_metrics[metrics],
                    object_gather_list=metric_list,
                    dst=0,
520
                )
521
522
523
524
                if RANK == 0:
                    task_output.sample_metrics[metrics] = list(
                        itertools.chain.from_iterable(metric_list)
                    )
525

526
    if RANK == 0:
527
528
        ### Aggregate results over all datapoints ###
        # aggregate results ; run bootstrap CIs
529
530
531
532
533
        for task_output in eval_tasks:
            task_output.calculate_aggregate_metric(bootstrap_iters=bootstrap_iters)
        results, samples, configs, versions, num_fewshot = consolidate_results(
            eval_tasks
        )
Fabrizio Milo's avatar
Fabrizio Milo committed
534

535
        ### Calculate group metrics ###
lintangsutawika's avatar
lintangsutawika committed
536
        if bool(results):
537
            for group, task_list in reversed(task_hierarchy.items()):
538
539
540
541
542
543
                if len(task_list) == 0:
                    # task_hierarchy entries are either
                    # `group_name: [subtask1, subtask2, ...]`
                    # or `task_name: []`.
                    # we only want to operate on groups here.
                    continue
544
545
546
547
548
549
550
551
552
                metric_list = list(
                    {
                        key
                        for task in task_list
                        for key in results[task].keys()
                        if "_stderr" not in key and key not in ["alias", "samples"]
                    }
                )
                for metric in metric_list:
553
554
555
                    stderr = "_stderr,".join(metric.split(","))

                    # gather metrics, sizes, and stderrs from subtasks
Baber Abbasi's avatar
Baber Abbasi committed
556
                    metrics = [
557
558
559
                        results[task][metric]
                        for task in task_list
                        if metric in results[task]
Baber Abbasi's avatar
Baber Abbasi committed
560
                    ]  # TODO: copy?
561
562
563
564
565
566
567
568
569
570
                    stderrs = [
                        results[task][stderr]
                        for task in task_list
                        if stderr in results[task]
                    ]
                    sizes = [
                        results[task]["samples"]
                        for task in task_list
                        if metric in results[task]
                    ]
571
572

                    # compute group's pooled metric and stderr
Baber Abbasi's avatar
Baber Abbasi committed
573
574
575
                    results[group][
                        metric
                    ] = lm_eval.api.metrics.aggregate_subtask_metrics(metrics, sizes)
576
577
578
579
                    # TODO: calculate grouped metric using aggregation fn
                    if "N/A" in stderrs:
                        results[group][stderr] = "N/A"
                    else:
Baber Abbasi's avatar
Baber Abbasi committed
580
581
582
                        results[group][
                            stderr
                        ] = lm_eval.api.metrics.pooled_sample_stderr(stderrs, sizes)
583
584
585
586
587
                        # TODO: allow GroupConfigs to choose which variance formula is used, for back-compatibility
                        # To use the old (likely incorrect) variance formula, comment out the above and uncomment this line:
                        # results[group][stderr] = lm_eval.api.metrics.combined_sample_stderr(stderrs, sizes, metrics=metrics)

                    results[group]["samples"] = sum(sizes)
lintangsutawika's avatar
lintangsutawika committed
588

589
590
        results_agg = defaultdict(dict)
        groups_agg = defaultdict(dict)
Lintang Sutawika's avatar
Lintang Sutawika committed
591
592
593
594
595
596
597
598
599
600
        all_tasks_list = list(task_hierarchy.keys())
        while True:
            add_tasks_list = list(k for k in results_agg.keys())
            left_tasks_list = sorted(list(set(all_tasks_list) - set(add_tasks_list)))
            if len(left_tasks_list) == 0:
                break

            _task_hierarchy = {
                k: v for k, v in task_hierarchy.items() if k in left_tasks_list
            }
601
            _results_agg, _groups_agg = prepare_print_tasks(_task_hierarchy, results)
Lintang Sutawika's avatar
Lintang Sutawika committed
602
603
604

            results_agg = {**results_agg, **_results_agg}
            groups_agg = {**groups_agg, **_groups_agg}
lintangsutawika's avatar
lintangsutawika committed
605

606
        for group_name, task_list in task_hierarchy.items():
Baber Abbasi's avatar
Baber Abbasi committed
607
608
609
610
            if task_list:
                num_fewshot[group_name] = num_fewshot[
                    task_list[0]
                ]  # TODO: validate this
611

612
        results_dict = {
613
            "results": dict(results_agg.items()),
lintangsutawika's avatar
lintangsutawika committed
614
            **({"groups": dict(groups_agg.items())} if bool(groups_agg) else {}),
615
            "group_subtasks": dict(reversed(task_hierarchy.items())),
616
617
            "configs": dict(sorted(configs.items())),
            "versions": dict(sorted(versions.items())),
618
            "n-shot": dict(sorted(num_fewshot.items())),
619
620
621
            "n-samples": {
                task_output.task_name: {
                    "original": len(task_output.task.eval_docs),
KonradSzafer's avatar
KonradSzafer committed
622
623
624
625
                    "effective": min(
                        limit if limit else len(task_output.task.eval_docs),
                        len(task_output.task.eval_docs),
                    ),
626
627
628
                }
                for task_output in eval_tasks
            },
629
        }
630
631
632
633
        if log_samples:
            results_dict["samples"] = dict(samples)

        return results_dict
Fabrizio Milo's avatar
Fabrizio Milo committed
634

635
636
    else:
        return None
637
638
639
640


def request_caching_arg_to_dict(cache_requests: str) -> dict:
    request_caching_args = {
641
642
643
        "cache_requests": cache_requests in {"true", "refresh"},
        "rewrite_requests_cache": cache_requests == "refresh",
        "delete_requests_cache": cache_requests == "delete",
644
645
646
    }

    return request_caching_args