evaluator.py 27.8 KB
Newer Older
Baber Abbasi's avatar
Baber Abbasi committed
1
import itertools
2
import logging
Baber Abbasi's avatar
Baber Abbasi committed
3
import random
4
import time
5
6
from collections import defaultdict
from typing import TYPE_CHECKING, List, Optional, Union
Baber Abbasi's avatar
Baber Abbasi committed
7

8
import numpy as np
Baber Abbasi's avatar
Baber Abbasi committed
9
import torch
lintangsutawika's avatar
lintangsutawika committed
10

lintangsutawika's avatar
lintangsutawika committed
11
import lm_eval.api.metrics
lintangsutawika's avatar
lintangsutawika committed
12
import lm_eval.api.registry
13
import lm_eval.api.task
Baber Abbasi's avatar
Baber Abbasi committed
14
import lm_eval.models
15
from lm_eval.caching.cache import delete_cache
16
17
18
19
20
21
22
from lm_eval.evaluator_utils import (
    consolidate_results,
    get_sample_size,
    get_task_list,
    print_writeout,
    run_task_tests,
)
23
from lm_eval.logging_utils import add_env_info, get_git_commit_hash
24
25
26
27
28
29
from lm_eval.tasks import (
    ConfigurableGroup,
    ConfigurableTask,
    TaskManager,
    get_task_dict,
)
30
from lm_eval.utils import eval_logger, positional_deprecated, simple_parse_args_string
31

Fabrizio Milo's avatar
Fabrizio Milo committed
32

33
34
35
36
37
if TYPE_CHECKING:
    from lm_eval.api.model import LM
    from lm_eval.tasks import Task


38
@positional_deprecated
Fabrizio Milo's avatar
Fabrizio Milo committed
39
40
def simple_evaluate(
    model,
41
42
    model_args: Optional[Union[str, dict]] = None,
    tasks: Optional[List[Union[str, dict, object]]] = None,
Baber Abbasi's avatar
Baber Abbasi committed
43
44
45
46
47
    num_fewshot: Optional[int] = None,
    batch_size: Optional[int] = None,
    max_batch_size: Optional[int] = None,
    device: Optional[str] = None,
    use_cache: Optional[str] = None,
48
49
50
    cache_requests: bool = False,
    rewrite_requests_cache: bool = False,
    delete_requests_cache: bool = False,
Baber Abbasi's avatar
Baber Abbasi committed
51
    limit: Optional[Union[int, float]] = None,
Ethan Smith's avatar
Ethan Smith committed
52
53
54
55
    bootstrap_iters: int = 100000,
    check_integrity: bool = False,
    write_out: bool = False,
    log_samples: bool = True,
56
57
    gen_kwargs: Optional[str] = None,
    task_manager: Optional[TaskManager] = None,
58
    verbosity: str = "INFO",
Baber Abbasi's avatar
Baber Abbasi committed
59
    predict_only: bool = False,
60
61
62
    random_seed: int = 0,
    numpy_random_seed: int = 1234,
    torch_random_seed: int = 1234,
Fabrizio Milo's avatar
Fabrizio Milo committed
63
):
64
    """Instantiate and evaluate a model on a list of tasks.
65

66
67
    :param model: Union[str, LM]
        Name of model or LM object, see lm_eval.models.get_model
68
69
    :param model_args: Optional[str, dict]
        String or dict arguments for each model class, see LM.create_from_arg_string and LM.create_from_arg_object.
70
        Ignored if `model` argument is a LM object.
71
    :param tasks: list[Union[str, dict, Task]]
Leo Gao's avatar
Leo Gao committed
72
        List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.
73
74
    :param num_fewshot: int
        Number of examples in few-shot context
75
    :param batch_size: int or str, optional
76
        Batch size for model
77
78
    :param max_batch_size: int, optional
        Maximal batch size to try with automatic batch size detection
79
    :param device: str, optional
80
        PyTorch device (e.g. "cpu" or "cuda:0") for running models
haileyschoelkopf's avatar
haileyschoelkopf committed
81
82
    :param use_cache: str, optional
        A path to a sqlite db file for caching model responses. `None` if not caching.
83
84
85
86
87
88
    :param cache_requests: bool, optional
        Speed up evaluation by caching the building of dataset requests. `None` if not caching.
    :param rewrite_requests_cache: bool, optional
        Rewrites all of the request cache if set to `True`. `None` if not desired.
    :param delete_requests_cache: bool, optional
        Deletes all of the request cache if set to `True`. `None` if not desired.
89
90
    :param limit: int or float, optional
        Limit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples.
91
92
    :param bootstrap_iters:
        Number of iterations for bootstrap statistics
Stephen Hogg's avatar
Stephen Hogg committed
93
94
    :param check_integrity: bool
        Whether to run the relevant part of the test suite for the tasks
95
    :param write_out: bool
96
97
98
        If True, write out an example document and model input for checking task integrity
    :param log_samples: bool
        If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
99
100
101
    :param gen_kwargs: str
        String arguments for model generation
        Ignored for all tasks with loglikelihood output_type
Baber Abbasi's avatar
Baber Abbasi committed
102
103
    :param predict_only: bool
        If true only model outputs will be generated and returned. Metrics will not be evaluated
104
105
106
107
108
109
    :param random_seed: int
        Random seed for python's random module. If set to None, the seed will not be set.
    :param numpy_random_seed: int
        Random seed for numpy. If set to None, the seed will not be set.
    :param torch_random_seed: int
        Random seed for torch. If set to None, the seed will not be set.
Baber Abbasi's avatar
Baber Abbasi committed
110

111
    :return
112
        Dictionary of results
113
    """
114
    eval_logger.setLevel(getattr(logging, f"{verbosity}"))
115
    start_date = time.time()
116

117
118
119
120
    if delete_requests_cache:
        eval_logger.info("Deleting requests cache...")
        delete_cache()

121
    seed_message = []
122
123
    if random_seed is not None:
        # See https://github.com/EleutherAI/lm-evaluation-harness/pull/1412
124
        seed_message.append(f"Setting random seed to {random_seed}")
125
126
127
        random.seed(random_seed)

    if numpy_random_seed is not None:
128
        seed_message.append(f"Setting numpy seed to {numpy_random_seed}")
129
130
131
        np.random.seed(numpy_random_seed)

    if torch_random_seed is not None:
132
        seed_message.append(f"Setting torch manual seed to {torch_random_seed}")
133
134
        torch.manual_seed(torch_random_seed)

135
136
137
    if seed_message:
        eval_logger.info(" | ".join(seed_message))

138
139
    if tasks is None:
        tasks = []
140
141
142
143
    if len(tasks) == 0:
        raise ValueError(
            "No tasks specified, or no tasks found. Please verify the task names."
        )
144

lintangsutawika's avatar
lintangsutawika committed
145
146
    if gen_kwargs is not None:
        gen_kwargs = simple_parse_args_string(gen_kwargs)
lintangsutawika's avatar
udate  
lintangsutawika committed
147
        eval_logger.warning(
148
149
            "generation_kwargs specified through cli, these settings will update set parameters in yaml tasks. "
            "Ensure 'do_sample=True' for non-greedy decoding!"
lintangsutawika's avatar
udate  
lintangsutawika committed
150
        )
lintangsutawika's avatar
lintangsutawika committed
151
152
153
        if gen_kwargs == "":
            gen_kwargs = None

154
    if isinstance(model, str):
Fabrizio Milo's avatar
Fabrizio Milo committed
155
        if model_args is None:
156
            eval_logger.warning("model_args not specified. Using defaults.")
Fabrizio Milo's avatar
Fabrizio Milo committed
157
            model_args = ""
158
159
160
161
162
163
164
165
166
        if "pretrained" not in model_args and model in [
            "hf-auto",
            "hf",
            "huggingface",
            "vllm",
        ]:
            eval_logger.warning(
                "pretrained not specified. Using default pretrained=gpt2."
            )
167

168
        if isinstance(model_args, dict):
169
170
171
            eval_logger.info(
                f"Initializing {model} model, with arguments: {model_args}"
            )
172
173
174
175
176
177
178
179
180
181
            lm = lm_eval.api.registry.get_model(model).create_from_arg_obj(
                model_args,
                {
                    "batch_size": batch_size,
                    "max_batch_size": max_batch_size,
                    "device": device,
                },
            )

        else:
182
183
184
            eval_logger.info(
                f"Initializing {model} model, with arguments: {simple_parse_args_string(model_args)}"
            )
185
186
187
188
189
190
191
192
            lm = lm_eval.api.registry.get_model(model).create_from_arg_string(
                model_args,
                {
                    "batch_size": batch_size,
                    "max_batch_size": max_batch_size,
                    "device": device,
                },
            )
193
    else:
194
195
        if not isinstance(model, lm_eval.api.model.LM):
            raise TypeError
196
        eval_logger.info("Using pre-initialized model")
197
        lm = model
198

haileyschoelkopf's avatar
haileyschoelkopf committed
199
    if use_cache is not None:
200
        eval_logger.info(f"Using cache at {use_cache + '_rank' + str(lm.rank) + '.db'}")
haileyschoelkopf's avatar
haileyschoelkopf committed
201
202
203
204
205
        lm = lm_eval.api.model.CachingLM(
            lm,
            use_cache
            # each rank receives a different cache db.
            # necessary to avoid multiple writes to cache at once
206
207
208
            + "_rank"
            + str(lm.rank)
            + ".db",
haileyschoelkopf's avatar
haileyschoelkopf committed
209
210
        )

211
212
213
    if check_integrity:
        run_task_tests(task_list=tasks)

214
215
216
217
    if task_manager is None:
        task_manager = TaskManager(verbosity)

    task_dict = get_task_dict(tasks, task_manager)
Jonathan Tow's avatar
Merge  
Jonathan Tow committed
218

219
    def _adjust_config(task_dict):
220
221
222
223
224
        adjusted_task_dict = {}
        for task_name, task_obj in task_dict.items():
            if isinstance(task_obj, dict):
                adjusted_task_dict = {
                    **adjusted_task_dict,
225
                    **{task_name: _adjust_config(task_obj)},
226
                }
Stephen Hogg's avatar
Stephen Hogg committed
227

228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
            else:
                if task_obj.get_config("output_type") == "generate_until":
                    if gen_kwargs is not None:
                        task_obj.set_config(
                            key="generation_kwargs", value=gen_kwargs, update=True
                        )

                if predict_only:
                    eval_logger.info(
                        f"Processing {task_name} in output-only mode. Metrics will not be calculated!"
                    )
                    # we have to change the class properties post-hoc. This is pretty hacky.
                    task_obj.override_metric(metric_name="bypass")

                # override tasks' fewshot values to the provided num_fewshot arg value
                # except if tasks have it set to 0 manually in their configs--then we should never overwrite that
                if num_fewshot is not None:
                    if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0:
                        eval_logger.info(
                            f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored."
                        )
                    else:
                        eval_logger.warning(
                            f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}"
                        )
                        task_obj.set_config(key="num_fewshot", value=num_fewshot)
                else:
                    # if num_fewshot not provided, and the task does not define a default one, default to 0
256
257
258
                    if (
                        default_num_fewshot := task_obj.get_config("num_fewshot")
                    ) is None:
259
                        task_obj.set_config(key="num_fewshot", value=0)
260

261
262
263
264
265
                adjusted_task_dict[task_name] = task_obj

        return adjusted_task_dict

    task_dict = _adjust_config(task_dict)
266
267
268
269
    results = evaluate(
        lm=lm,
        task_dict=task_dict,
        limit=limit,
270
271
        cache_requests=cache_requests,
        rewrite_requests_cache=rewrite_requests_cache,
Niklas Muennighoff's avatar
Niklas Muennighoff committed
272
        bootstrap_iters=bootstrap_iters,
273
        write_out=write_out,
274
        log_samples=True if predict_only else log_samples,
275
        verbosity=verbosity,
276
    )
277

278
    if lm.rank == 0:
279
280
281
282
283
284
285
        if isinstance(model, str):
            model_name = model
        elif hasattr(model, "config") and hasattr(model.config, "_name_or_path"):
            model_name = model.config._name_or_path
        else:
            model_name = type(model).__name__

286
287
        # add info about the model and few shot config
        results["config"] = {
288
            "model": model_name,
289
290
            "model_args": model_args,
            "batch_size": batch_size,
291
292
293
            "batch_sizes": (
                list(lm.batch_sizes.values()) if hasattr(lm, "batch_sizes") else []
            ),
294
            "device": device,
haileyschoelkopf's avatar
haileyschoelkopf committed
295
            "use_cache": use_cache,
296
297
            "limit": limit,
            "bootstrap_iters": bootstrap_iters,
lintangsutawika's avatar
lintangsutawika committed
298
            "gen_kwargs": gen_kwargs,
299
        }
300
        results["git_hash"] = get_git_commit_hash()
301
        results["date"] = start_date
302
        add_env_info(results)  # additional environment info to results
303
304
305
        return results
    else:
        return None
306

Leo Gao's avatar
Leo Gao committed
307

308
@positional_deprecated
Fabrizio Milo's avatar
Fabrizio Milo committed
309
def evaluate(
310
    lm: "LM",
Fabrizio Milo's avatar
Fabrizio Milo committed
311
    task_dict,
Baber Abbasi's avatar
Baber Abbasi committed
312
    limit: Optional[int] = None,
313
314
    cache_requests: bool = False,
    rewrite_requests_cache: bool = False,
Baber Abbasi's avatar
Baber Abbasi committed
315
    bootstrap_iters: Optional[int] = 100000,
Ethan Smith's avatar
Ethan Smith committed
316
317
    write_out: bool = False,
    log_samples: bool = True,
318
    verbosity: str = "INFO",
Fabrizio Milo's avatar
Fabrizio Milo committed
319
):
320
321
322
323
324
    """Instantiate and evaluate a model on a list of tasks.

    :param lm: obj
        Language Model
    :param task_dict: dict[str, Task]
haileyschoelkopf's avatar
haileyschoelkopf committed
325
        Dictionary of tasks. Tasks will be taken to have name type(task).config.task .
326
327
328
329
    :param limit: int, optional
        Limit the number of examples per task (only use this for testing)
    :param bootstrap_iters:
        Number of iterations for bootstrap statistics
330
    :param write_out: bool
331
332
333
        If True, write out an example document and model input for checking task integrity
    :param log_samples: bool
        If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
334
335
336
    :return
        Dictionary of results
    """
337

338
    eval_logger.setLevel(getattr(logging, f"{verbosity}"))
339

340
    # tracks all Instances/requests a model must generate output on.
341
    requests = defaultdict(list)
342
343
    # stores the amount to pad out reqs per req. type so that
    # number of fwd passes per distributed rank is equal
344
    padding_requests = defaultdict(int)
345

346
    # get lists of group hierarchy and each type of request
347
    eval_tasks = get_task_list(task_dict)
348
    if not log_samples:
349
        if not all(
350
351
            "bypass" not in getattr(task_output.task, "_metric_fn_list", {}).keys()
            for task_output in eval_tasks
352
353
        ):
            raise ValueError("log_samples must be True for 'bypass' metric-only tasks")
354
355
356
    for task_output in eval_tasks:
        task: Task = task_output.task
        limit = get_sample_size(task, limit)
357
358
359
360
361
362
363
        task.build_all_requests(
            limit=limit,
            rank=lm.rank,
            world_size=lm.world_size,
            cache_requests=cache_requests,
            rewrite_requests_cache=rewrite_requests_cache,
        )
364
        eval_logger.debug(
365
            f"Task: {task_output.task_name}; number of requests on this rank: {len(task.instances)}"
haileyschoelkopf's avatar
haileyschoelkopf committed
366
367
368
        )

        if write_out:
369
            print_writeout(task)
370
        # aggregate Instances by LM method requested to get output.
lintangsutawika's avatar
lintangsutawika committed
371
372
373
        for instance in task.instances:
            reqtype = instance.request_type
            requests[reqtype].append(instance)
374
375

        if lm.world_size > 1:
376
377
378
379
            instances_rnk = torch.tensor(len(task._instances), device=lm.device)
            gathered_item = (
                lm.accelerator.gather(instances_rnk).cpu().detach().numpy().tolist()
            )
380
381
382
383
384
385
            # "multiple_choice" task types dispatch (several) "loglikelihood" request types
            reqtype = (
                "loglikelihood"
                if task.OUTPUT_TYPE == "multiple_choice"
                else task.OUTPUT_TYPE
            )
386
            # compute number of pseudo-batches to pad with (FSDP/DDP require even batches among ranks)
387
            numpad = max(gathered_item) - gathered_item[lm.rank]
388
389
            # todo: may not account for padding in cases like SquadV2 which has multiple req types
            padding_requests[reqtype] += numpad
390

391
    ### Run LM on inputs, get all outputs ###
Leo Gao's avatar
Leo Gao committed
392
393
    # execute each type of request
    for reqtype, reqs in requests.items():
394
        eval_logger.info(f"Running {reqtype} requests")
395
396
397
398
        # create `K` copies of each request `req` based off `K = req.repeats`
        cloned_reqs = []
        for req in reqs:
            cloned_reqs.extend([req] * req.repeats)
lintangsutawika's avatar
lintangsutawika committed
399

400
401
        if (lm.world_size > 1) and (padding_requests[reqtype] > 0):
            for _ in range(padding_requests[reqtype]):
402
403
                cloned_reqs.extend([req] * req.repeats)

404
405
406
407
408
409
410
        # run requests through model
        resps = getattr(lm, reqtype)(cloned_reqs)

        # put responses from model into a list of length K for each request.
        for x, req in zip(resps, cloned_reqs):
            req.resps.append(x)

411
412
        if lm.world_size > 1:
            lm.accelerator.wait_for_everyone()
413

414
415
    RANK = lm.rank
    WORLD_SIZE = lm.world_size
416
417
    ### Postprocess outputs ###
    # TODO: del model here, maybe (idea: allow user to specify device of e.g. reward model separately)
418
419
    for task_output in eval_tasks:
        task = task_output.task
420
421
        task.apply_filters()

422
423
        ### Collect values of metrics on all datapoints ###
        # # unpack results and sort back in order and return control to Task
haileyschoelkopf's avatar
haileyschoelkopf committed
424
        # TODO: make it possible to use a different metric per filter
425
        # Pre-process task.instances to group by doc_id
426
        instances_by_doc_id = defaultdict(list)
427
428
429
430
431
        for instance in task.instances:
            instances_by_doc_id[instance.doc_id].append(instance)
        # Sort instances within each group
        for instances in instances_by_doc_id.values():
            instances.sort(key=lambda x: x.idx)
haileyschoelkopf's avatar
haileyschoelkopf committed
432
        # iterate over different filters used
433
434
435
        for filter_key in task.instances[0].filtered_resps.keys():
            doc_iterator = task.doc_iterator(
                rank=RANK, limit=limit, world_size=WORLD_SIZE
436
            )
437
            for doc_id, doc in doc_iterator:
438
                requests = instances_by_doc_id[doc_id]
lintangsutawika's avatar
lintangsutawika committed
439
                metrics = task.process_results(
440
                    doc, [req.filtered_resps[filter_key] for req in requests]
lintangsutawika's avatar
lintangsutawika committed
441
                )
442
443
444
445
446
447
448
449
                if log_samples:
                    target = task.doc_to_target(doc)
                    example = {
                        "doc_id": doc_id,
                        "doc": doc,
                        "target": target,
                        "arguments": [req.args for req in requests],
                        "resps": [req.resps for req in requests],
450
451
452
                        "filtered_resps": [
                            req.filtered_resps[filter_key] for req in requests
                        ],
453
454
                    }
                    example.update(metrics)
455
                    task_output.logged_samples.append(example)
456
                for metric, value in metrics.items():
457
                    task_output.sample_metrics[(metric, filter_key)].append(value)
458

459
460
    if WORLD_SIZE > 1:
        # if multigpu, then gather data across all ranks to rank 0
461
        # first gather logged samples across all ranks
462
463
464
465
466
467
468
469
        for task_output in eval_tasks:
            if log_samples:
                # for task_name, task_samples in list(samples.items()):
                full_samples = [None] * WORLD_SIZE if RANK == 0 else None
                torch.distributed.gather_object(
                    obj=task_output.logged_samples,
                    object_gather_list=full_samples,
                    dst=0,
470
                )
471

472
473
474
475
                if RANK == 0:
                    task_output.logged_samples = list(
                        itertools.chain.from_iterable(full_samples)
                    )
476

477
478
479
480
481
482
483
            # then collect metrics across all ranks
            for metrics in task_output.sample_metrics:
                metric_list = [None] * WORLD_SIZE if RANK == 0 else None
                torch.distributed.gather_object(
                    obj=task_output.sample_metrics[metrics],
                    object_gather_list=metric_list,
                    dst=0,
484
                )
485
486
487
488
                if RANK == 0:
                    task_output.sample_metrics[metrics] = list(
                        itertools.chain.from_iterable(metric_list)
                    )
489

490
    if RANK == 0:
491
492
        ### Aggregate results over all datapoints ###
        # aggregate results ; run bootstrap CIs
493
494
495
496
497
        for task_output in eval_tasks:
            task_output.calculate_aggregate_metric(bootstrap_iters=bootstrap_iters)
        results, samples, configs, versions, num_fewshot = consolidate_results(
            eval_tasks
        )
Fabrizio Milo's avatar
Fabrizio Milo committed
498

499
        ### Calculate group metrics ###
lintangsutawika's avatar
lintangsutawika committed
500
        if bool(results):
501

502
503
504
505
506
507
508
            def process_group(
                results,
                task_dict,
                task_root=None,
                task_hierarchy=None,
                show_group_table=False,
            ):
509
510
511
512
513
514
515
                if task_root is None:
                    task_root = {}

                if task_hierarchy is None:
                    task_hierarchy = {}

                for group_or_task, group_or_task_info in task_dict.items():
516
517
518
519
520
521
522
523
524
525
                    if isinstance(group_or_task, ConfigurableGroup):
                        group_config = group_or_task.config
                        group_or_task = group_or_task.group
                        show_group_table = (
                            show_group_table | group_config["aggregate_metric"]
                        )
                        if group_config["aggregate_metric"] is False:
                            results[group_or_task][" "] = " "
                            continue

526
527
                    if isinstance(group_or_task_info, ConfigurableTask):
                        if task_root:
528
529
530
                            task_hierarchy.setdefault(task_root, []).append(
                                group_or_task
                            )
531
                    else:
532
533
534
535
536
537
538
                        results, _task_hierarchy, show_group_table = process_group(
                            results,
                            group_or_task_info,
                            group_or_task,
                            task_hierarchy,
                            show_group_table,
                        )
539
                        if task_root:
540
541
542
                            task_hierarchy.setdefault(task_root, []).extend(
                                task_hierarchy.get(group_or_task, [])
                            )
543
544
545
546
547
548
549

                        task_list = _task_hierarchy[group_or_task]
                        metric_list = list(
                            {
                                key
                                for task in task_list
                                for key in results[task].keys()
550
551
                                if "_stderr" not in key
                                and key not in ["alias", "samples"]
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
                            }
                        )
                        for metric in metric_list:
                            stderr = "_stderr,".join(metric.split(","))

                            # gather metrics, sizes, and stderrs from subtasks
                            metrics = [
                                results[task][metric]
                                for task in task_list
                                if metric in results[task]
                            ]  # TODO: copy?
                            stderrs = [
                                results[task][stderr]
                                for task in task_list
                                if stderr in results[task]
                            ]
                            sizes = [
                                results[task]["samples"]
                                for task in task_list
                                if metric in results[task]
                            ]

                            # compute group's pooled metric and stderr
575
                            results[group_or_task][
576
577
578
579
580
581
582
583
                                metric
                            ] = lm_eval.api.metrics.aggregate_subtask_metrics(
                                metrics,
                                sizes,
                                group_config["weight_by_size"],
                            )
                            # TODO: calculate grouped metric using aggregation fn
                            if "N/A" in stderrs:
584
                                results[group_or_task][stderr] = "N/A"
585
                            else:
586
                                results[group_or_task][
587
                                    stderr
588
589
590
                                ] = lm_eval.api.metrics.pooled_sample_stderr(
                                    stderrs, sizes
                                )
591
592
593
594
                                # TODO: allow GroupConfigs to choose which variance formula is used, for back-compatibility
                                # To use the old (likely incorrect) variance formula, comment out the above and uncomment this line:
                                # results[group][stderr] = lm_eval.api.metrics.combined_sample_stderr(stderrs, sizes, metrics=metrics)

595
                            results[group_or_task]["samples"] = sum(sizes)
596
597
                return results, task_hierarchy, show_group_table

598
599
600
601
            results, task_hierarchy, show_group_table = process_group(
                results, task_dict
            )

lintangsutawika's avatar
lintangsutawika committed
602
603
604
605
606
607
608
609
610
611
612
613
614
        def print_table(task_dict, results, task_depth=0, group_depth=0):
            """
            @param task_dict: Dictionary representing the group hierarchy of tasks. Each key is a group name and its
            value is a list of task names.
            @param results: Dictionary containing the results of each task. Each key is a
            group name and its value is a dictionary of task results.
            @param task_depth: The indentation level for printing the task
            hierarchy. Default is 0.
            @return: A tuple of two dictionaries: results_agg and groups_agg. results_agg contains
            aggregated results for each task, and groups_agg contains aggregated results for each group.

            Prepares the task hierarchy and aggregates the results for each task and group recursively for printing.
            """
615
            task_agg = defaultdict(dict)
lintangsutawika's avatar
lintangsutawika committed
616
            group_agg = defaultdict(dict)
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
            for task_or_group_name, task_or_group_obj in task_dict.items():
                tab_string = " " * task_depth + "- " if task_depth > 0 else ""
                if isinstance(task_or_group_name, ConfigurableGroup):
                    name = task_or_group_name.group
                    from_configurable_group = True
                elif isinstance(task_or_group_name, str):
                    name = task_or_group_name
                    from_configurable_group = False

                task_agg[name] = results[name].copy()
                if from_configurable_group:
                    if task_or_group_name.group_alias is not None:
                        alias = task_or_group_name.group_alias
                    else:
                        alias = name
                else:
                    if "alias" in task_agg[name]:
                        alias = task_agg[name]["alias"]
                    else:
                        alias = name

                task_agg[name]["alias"] = tab_string + alias
                if "samples" in task_agg[name]:
                    task_agg[name].pop("samples")

lintangsutawika's avatar
lintangsutawika committed
642
643
644
645
646
647
648
649
650
                if from_configurable_group and (" " not in results[name]):
                    group_tab_string = (
                        " " * group_depth + "- " if group_depth > 0 else ""
                    )
                    group_agg[name] = results[name].copy()
                    group_agg[name]["alias"] = group_tab_string + alias
                    if "samples" in group_agg[name]:
                        group_agg[name].pop("samples")

651
652
                if isinstance(task_or_group_obj, dict):
                    task_depth += 1
lintangsutawika's avatar
lintangsutawika committed
653
654
655
656
                    group_depth += 1
                    _task_agg, _group_agg = print_table(
                        task_or_group_obj, results, task_depth, group_depth
                    )
657
658
                    task_agg = {
                        **task_agg,
lintangsutawika's avatar
lintangsutawika committed
659
                        **_task_agg,
660
                    }
lintangsutawika's avatar
lintangsutawika committed
661
                    group_agg = {**group_agg, **_group_agg}
662
                    task_depth -= 1
lintangsutawika's avatar
lintangsutawika committed
663
664
                    group_depth -= 1
            return task_agg, group_agg
665

lintangsutawika's avatar
lintangsutawika committed
666
        results_agg, group_agg = print_table(task_dict, results)
667
        results_dict = {
668
            "results": dict(results_agg.items()),
lintangsutawika's avatar
lintangsutawika committed
669
670
671
672
673
            **(
                {"groups": dict(group_agg.items())}
                if (bool(group_agg) & show_group_table)
                else {}
            ),
674
            "group_subtasks": dict(reversed(task_hierarchy.items())),
675
676
            "configs": dict(sorted(configs.items())),
            "versions": dict(sorted(versions.items())),
677
            "n-shot": dict(sorted(num_fewshot.items())),
678
        }
679
680
681
682
        if log_samples:
            results_dict["samples"] = dict(samples)

        return results_dict
Fabrizio Milo's avatar
Fabrizio Milo committed
683

684
685
    else:
        return None
686
687
688
689


def request_caching_arg_to_dict(cache_requests: str) -> dict:
    request_caching_args = {
690
691
692
        "cache_requests": cache_requests in {"true", "refresh"},
        "rewrite_requests_cache": cache_requests == "refresh",
        "delete_requests_cache": cache_requests == "delete",
693
694
695
    }

    return request_caching_args