evaluator.py 24.6 KB
Newer Older
Baber Abbasi's avatar
Baber Abbasi committed
1
import itertools
2
import json
3
import logging
Baber Abbasi's avatar
Baber Abbasi committed
4
import random
5
import time
6
7
from collections import defaultdict
from typing import TYPE_CHECKING, List, Optional, Union
Baber Abbasi's avatar
Baber Abbasi committed
8

9
import numpy as np
Baber Abbasi's avatar
Baber Abbasi committed
10
import torch
lintangsutawika's avatar
lintangsutawika committed
11

lintangsutawika's avatar
lintangsutawika committed
12
import lm_eval.api.metrics
lintangsutawika's avatar
lintangsutawika committed
13
import lm_eval.api.registry
Baber Abbasi's avatar
Baber Abbasi committed
14
import lm_eval.models
15
from lm_eval.caching.cache import delete_cache
16
17
18
19
20
21
22
23
from lm_eval.evaluator_utils import (
    consolidate_results,
    get_sample_size,
    get_task_list,
    prepare_print_tasks,
    print_writeout,
    run_task_tests,
)
24
from lm_eval.logging.utils import add_env_info, get_git_commit_hash
Baber Abbasi's avatar
Baber Abbasi committed
25
from lm_eval.tasks import TaskManager, get_task_dict
26
27
28
29
30
31
32
from lm_eval.utils import (
    eval_logger,
    handle_non_serializable,
    hash_string,
    positional_deprecated,
    simple_parse_args_string,
)
33

Fabrizio Milo's avatar
Fabrizio Milo committed
34

35
36
37
38
39
if TYPE_CHECKING:
    from lm_eval.api.model import LM
    from lm_eval.tasks import Task


40
@positional_deprecated
Fabrizio Milo's avatar
Fabrizio Milo committed
41
42
def simple_evaluate(
    model,
43
44
    model_args: Optional[Union[str, dict]] = None,
    tasks: Optional[List[Union[str, dict, object]]] = None,
Baber Abbasi's avatar
Baber Abbasi committed
45
46
47
48
49
    num_fewshot: Optional[int] = None,
    batch_size: Optional[int] = None,
    max_batch_size: Optional[int] = None,
    device: Optional[str] = None,
    use_cache: Optional[str] = None,
50
51
52
    cache_requests: bool = False,
    rewrite_requests_cache: bool = False,
    delete_requests_cache: bool = False,
Baber Abbasi's avatar
Baber Abbasi committed
53
    limit: Optional[Union[int, float]] = None,
Ethan Smith's avatar
Ethan Smith committed
54
55
56
57
    bootstrap_iters: int = 100000,
    check_integrity: bool = False,
    write_out: bool = False,
    log_samples: bool = True,
Konrad's avatar
Konrad committed
58
    apply_chat_template: bool = False,
59
60
    gen_kwargs: Optional[str] = None,
    task_manager: Optional[TaskManager] = None,
61
    verbosity: str = "INFO",
Baber Abbasi's avatar
Baber Abbasi committed
62
    predict_only: bool = False,
63
64
65
    random_seed: int = 0,
    numpy_random_seed: int = 1234,
    torch_random_seed: int = 1234,
66
    fewshot_random_seed: int = 1234,
Fabrizio Milo's avatar
Fabrizio Milo committed
67
):
68
    """Instantiate and evaluate a model on a list of tasks.
69

70
71
    :param model: Union[str, LM]
        Name of model or LM object, see lm_eval.models.get_model
72
73
    :param model_args: Optional[str, dict]
        String or dict arguments for each model class, see LM.create_from_arg_string and LM.create_from_arg_object.
74
        Ignored if `model` argument is a LM object.
75
    :param tasks: list[Union[str, dict, Task]]
Leo Gao's avatar
Leo Gao committed
76
        List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.
77
78
    :param num_fewshot: int
        Number of examples in few-shot context
79
    :param batch_size: int or str, optional
80
        Batch size for model
81
82
    :param max_batch_size: int, optional
        Maximal batch size to try with automatic batch size detection
83
    :param device: str, optional
84
        PyTorch device (e.g. "cpu" or "cuda:0") for running models
haileyschoelkopf's avatar
haileyschoelkopf committed
85
86
    :param use_cache: str, optional
        A path to a sqlite db file for caching model responses. `None` if not caching.
87
88
89
90
91
92
    :param cache_requests: bool, optional
        Speed up evaluation by caching the building of dataset requests. `None` if not caching.
    :param rewrite_requests_cache: bool, optional
        Rewrites all of the request cache if set to `True`. `None` if not desired.
    :param delete_requests_cache: bool, optional
        Deletes all of the request cache if set to `True`. `None` if not desired.
93
94
    :param limit: int or float, optional
        Limit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples.
95
96
    :param bootstrap_iters:
        Number of iterations for bootstrap statistics
Stephen Hogg's avatar
Stephen Hogg committed
97
98
    :param check_integrity: bool
        Whether to run the relevant part of the test suite for the tasks
99
    :param write_out: bool
100
101
102
        If True, write out an example document and model input for checking task integrity
    :param log_samples: bool
        If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
Konrad's avatar
Konrad committed
103
104
    :param apply_chat_template: bool
        If True, apply chat template to the prompt
105
106
107
    :param gen_kwargs: str
        String arguments for model generation
        Ignored for all tasks with loglikelihood output_type
Baber Abbasi's avatar
Baber Abbasi committed
108
109
    :param predict_only: bool
        If true only model outputs will be generated and returned. Metrics will not be evaluated
110
111
112
113
114
115
    :param random_seed: int
        Random seed for python's random module. If set to None, the seed will not be set.
    :param numpy_random_seed: int
        Random seed for numpy. If set to None, the seed will not be set.
    :param torch_random_seed: int
        Random seed for torch. If set to None, the seed will not be set.
116
117
    :param fewshot_random_seed: int
        Random seed for fewshot sampler random generator. If set to None, the seed of generator will be set to None.
Baber Abbasi's avatar
Baber Abbasi committed
118

119
    :return
120
        Dictionary of results
121
    """
122
    eval_logger.setLevel(getattr(logging, f"{verbosity}"))
123
    start_date = time.time()
124

125
126
127
128
    if delete_requests_cache:
        eval_logger.info("Deleting requests cache...")
        delete_cache()

129
    seed_message = []
130
131
    if random_seed is not None:
        # See https://github.com/EleutherAI/lm-evaluation-harness/pull/1412
132
        seed_message.append(f"Setting random seed to {random_seed}")
133
134
135
        random.seed(random_seed)

    if numpy_random_seed is not None:
136
        seed_message.append(f"Setting numpy seed to {numpy_random_seed}")
137
138
139
        np.random.seed(numpy_random_seed)

    if torch_random_seed is not None:
140
        seed_message.append(f"Setting torch manual seed to {torch_random_seed}")
141
142
        torch.manual_seed(torch_random_seed)

143
144
145
    if seed_message:
        eval_logger.info(" | ".join(seed_message))

146
147
    if tasks is None:
        tasks = []
148
149
150
151
    if len(tasks) == 0:
        raise ValueError(
            "No tasks specified, or no tasks found. Please verify the task names."
        )
152

lintangsutawika's avatar
lintangsutawika committed
153
154
    if gen_kwargs is not None:
        gen_kwargs = simple_parse_args_string(gen_kwargs)
lintangsutawika's avatar
udate  
lintangsutawika committed
155
        eval_logger.warning(
156
157
            "generation_kwargs specified through cli, these settings will update set parameters in yaml tasks. "
            "Ensure 'do_sample=True' for non-greedy decoding!"
lintangsutawika's avatar
udate  
lintangsutawika committed
158
        )
lintangsutawika's avatar
lintangsutawika committed
159
160
161
        if gen_kwargs == "":
            gen_kwargs = None

162
    if isinstance(model, str):
Fabrizio Milo's avatar
Fabrizio Milo committed
163
        if model_args is None:
164
            eval_logger.warning("model_args not specified. Using defaults.")
Fabrizio Milo's avatar
Fabrizio Milo committed
165
            model_args = ""
166

167
        if isinstance(model_args, dict):
168
169
170
            eval_logger.info(
                f"Initializing {model} model, with arguments: {model_args}"
            )
171
172
173
174
175
176
177
178
179
180
            lm = lm_eval.api.registry.get_model(model).create_from_arg_obj(
                model_args,
                {
                    "batch_size": batch_size,
                    "max_batch_size": max_batch_size,
                    "device": device,
                },
            )

        else:
181
182
183
            eval_logger.info(
                f"Initializing {model} model, with arguments: {simple_parse_args_string(model_args)}"
            )
184
185
186
187
188
189
190
191
            lm = lm_eval.api.registry.get_model(model).create_from_arg_string(
                model_args,
                {
                    "batch_size": batch_size,
                    "max_batch_size": max_batch_size,
                    "device": device,
                },
            )
192
    else:
193
194
        if not isinstance(model, lm_eval.api.model.LM):
            raise TypeError
195
        eval_logger.info("Using pre-initialized model")
196
        lm = model
197

haileyschoelkopf's avatar
haileyschoelkopf committed
198
    if use_cache is not None:
199
        eval_logger.info(f"Using cache at {use_cache + '_rank' + str(lm.rank) + '.db'}")
haileyschoelkopf's avatar
haileyschoelkopf committed
200
201
202
203
204
        lm = lm_eval.api.model.CachingLM(
            lm,
            use_cache
            # each rank receives a different cache db.
            # necessary to avoid multiple writes to cache at once
205
206
207
            + "_rank"
            + str(lm.rank)
            + ".db",
haileyschoelkopf's avatar
haileyschoelkopf committed
208
209
        )

210
211
212
213
    if task_manager is None:
        task_manager = TaskManager(verbosity)

    task_dict = get_task_dict(tasks, task_manager)
214
    for task_name in task_dict.keys():
lintangsutawika's avatar
lintangsutawika committed
215
        task_obj = task_dict[task_name]
216
        if isinstance(task_obj, tuple):
217
            _, task_obj = task_obj
218
219
            if task_obj is None:
                continue
lintangsutawika's avatar
lintangsutawika committed
220

Baber Abbasi's avatar
Baber Abbasi committed
221
222
        if task_obj.get_config("output_type") == "generate_until":
            if gen_kwargs is not None:
Baber Abbasi's avatar
Baber Abbasi committed
223
                task_obj.set_config(
Baber Abbasi's avatar
Baber Abbasi committed
224
225
226
                    key="generation_kwargs", value=gen_kwargs, update=True
                )

227
228
229
230
231
232
233
        if predict_only:
            log_samples = True
            eval_logger.info(
                f"Processing {task_name} in output-only mode. Metrics will not be calculated!"
            )
            # we have to change the class properties post-hoc. This is pretty hacky.
            task_obj.override_metric(metric_name="bypass")
234

235
236
        # override tasks' fewshot values to the provided num_fewshot arg value
        # except if tasks have it set to 0 manually in their configs--then we should never overwrite that
237
        if num_fewshot is not None:
Baber Abbasi's avatar
Baber Abbasi committed
238
            if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0:
239
240
241
                eval_logger.info(
                    f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored."
                )
242
            else:
Baber Abbasi's avatar
Baber Abbasi committed
243
244
245
                eval_logger.warning(
                    f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}"
                )
Baber Abbasi's avatar
Baber Abbasi committed
246
                task_obj.set_config(key="num_fewshot", value=num_fewshot)
247
248
249
250
            task_obj.set_fewshot_seed(seed=fewshot_random_seed)
            eval_logger.info(
                f"Setting fewshot random generator seed to {fewshot_random_seed}"
            )
251
252
253
254
        else:
            # if num_fewshot not provided, and the task does not define a default one, default to 0
            if (default_num_fewshot := task_obj.get_config("num_fewshot")) is None:
                task_obj.set_config(key="num_fewshot", value=0)
Jonathan Tow's avatar
Merge  
Jonathan Tow committed
255

Stephen Hogg's avatar
Stephen Hogg committed
256
    if check_integrity:
257
        run_task_tests(task_list=tasks)
Stephen Hogg's avatar
Stephen Hogg committed
258

259
260
261
262
    results = evaluate(
        lm=lm,
        task_dict=task_dict,
        limit=limit,
263
264
        cache_requests=cache_requests,
        rewrite_requests_cache=rewrite_requests_cache,
Niklas Muennighoff's avatar
Niklas Muennighoff committed
265
        bootstrap_iters=bootstrap_iters,
266
        write_out=write_out,
267
        log_samples=log_samples,
Konrad's avatar
Konrad committed
268
        apply_chat_template=apply_chat_template,
269
        verbosity=verbosity,
270
    )
271

272
    if lm.rank == 0:
273
274
275
276
277
278
279
        if isinstance(model, str):
            model_name = model
        elif hasattr(model, "config") and hasattr(model.config, "_name_or_path"):
            model_name = model.config._name_or_path
        else:
            model_name = type(model).__name__

280
281
        # add info about the model and few shot config
        results["config"] = {
282
            "model": model_name,
283
284
            "model_args": model_args,
        }
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
        # add more detailed model info if available
        if isinstance(lm, lm_eval.models.huggingface.HFLM):
            results["config"].update(lm.get_model_info())
        # add info about execution
        results["config"].update(
            {
                "batch_size": batch_size,
                "batch_sizes": (
                    list(lm.batch_sizes.values()) if hasattr(lm, "batch_sizes") else []
                ),
                "device": device,
                "use_cache": use_cache,
                "limit": limit,
                "bootstrap_iters": bootstrap_iters,
                "gen_kwargs": gen_kwargs,
300
301
302
303
                "random_seed": random_seed,
                "numpy_seed": numpy_random_seed,
                "torch_seed": torch_random_seed,
                "fewshot_seed": fewshot_random_seed,
304
305
            }
        )
306
        results["git_hash"] = get_git_commit_hash()
307
        results["date"] = start_date
308
        add_env_info(results)  # additional environment info to results
309
310
311
        return results
    else:
        return None
312

Leo Gao's avatar
Leo Gao committed
313

314
@positional_deprecated
Fabrizio Milo's avatar
Fabrizio Milo committed
315
def evaluate(
316
    lm: "LM",
Fabrizio Milo's avatar
Fabrizio Milo committed
317
    task_dict,
Baber Abbasi's avatar
Baber Abbasi committed
318
    limit: Optional[int] = None,
319
320
    cache_requests: bool = False,
    rewrite_requests_cache: bool = False,
Baber Abbasi's avatar
Baber Abbasi committed
321
    bootstrap_iters: Optional[int] = 100000,
Ethan Smith's avatar
Ethan Smith committed
322
323
    write_out: bool = False,
    log_samples: bool = True,
Konrad's avatar
Konrad committed
324
    apply_chat_template: bool = False,
325
    verbosity: str = "INFO",
Fabrizio Milo's avatar
Fabrizio Milo committed
326
):
327
328
329
330
331
    """Instantiate and evaluate a model on a list of tasks.

    :param lm: obj
        Language Model
    :param task_dict: dict[str, Task]
haileyschoelkopf's avatar
haileyschoelkopf committed
332
        Dictionary of tasks. Tasks will be taken to have name type(task).config.task .
333
334
335
336
    :param limit: int, optional
        Limit the number of examples per task (only use this for testing)
    :param bootstrap_iters:
        Number of iterations for bootstrap statistics
337
    :param write_out: bool
338
339
340
        If True, write out an example document and model input for checking task integrity
    :param log_samples: bool
        If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
Konrad's avatar
Konrad committed
341
342
    :param apply_chat_template: bool
        If True, apply chat template to the prompt
343
344
345
    :return
        Dictionary of results
    """
346

347
    eval_logger.setLevel(getattr(logging, f"{verbosity}"))
348

349
    # tracks all Instances/requests a model must generate output on.
350
    requests = defaultdict(list)
351
352
    # stores the amount to pad out reqs per req. type so that
    # number of fwd passes per distributed rank is equal
353
    padding_requests = defaultdict(int)
354

355
356
357
    # get lists of group hierarchy and each type of request
    task_hierarchy, eval_tasks = get_task_list(task_dict)
    if not log_samples:
358
        if not all(
359
360
            "bypass" not in getattr(task_output.task, "_metric_fn_list", {}).keys()
            for task_output in eval_tasks
361
362
        ):
            raise ValueError("log_samples must be True for 'bypass' metric-only tasks")
363
364
365
    for task_output in eval_tasks:
        task: Task = task_output.task
        limit = get_sample_size(task, limit)
366
367
368
369
370
371
        task.build_all_requests(
            limit=limit,
            rank=lm.rank,
            world_size=lm.world_size,
            cache_requests=cache_requests,
            rewrite_requests_cache=rewrite_requests_cache,
Konrad's avatar
Konrad committed
372
373
            apply_chat_template=apply_chat_template,
            tokenizer=lm.tokenizer,
374
        )
375
        eval_logger.debug(
376
            f"Task: {task_output.task_name}; number of requests on this rank: {len(task.instances)}"
haileyschoelkopf's avatar
haileyschoelkopf committed
377
378
        )
        if write_out:
379
            print_writeout(task)
380
        # aggregate Instances by LM method requested to get output.
lintangsutawika's avatar
lintangsutawika committed
381
382
383
        for instance in task.instances:
            reqtype = instance.request_type
            requests[reqtype].append(instance)
384
385

        if lm.world_size > 1:
386
387
388
389
            instances_rnk = torch.tensor(len(task._instances), device=lm.device)
            gathered_item = (
                lm.accelerator.gather(instances_rnk).cpu().detach().numpy().tolist()
            )
390
391
392
393
394
395
            # "multiple_choice" task types dispatch (several) "loglikelihood" request types
            reqtype = (
                "loglikelihood"
                if task.OUTPUT_TYPE == "multiple_choice"
                else task.OUTPUT_TYPE
            )
396
            # compute number of pseudo-batches to pad with (FSDP/DDP require even batches among ranks)
397
            numpad = max(gathered_item) - gathered_item[lm.rank]
398
399
            # todo: may not account for padding in cases like SquadV2 which has multiple req types
            padding_requests[reqtype] += numpad
400

401
    ### Run LM on inputs, get all outputs ###
Leo Gao's avatar
Leo Gao committed
402
403
    # execute each type of request
    for reqtype, reqs in requests.items():
404
        eval_logger.info(f"Running {reqtype} requests")
405
406
407
408
        # create `K` copies of each request `req` based off `K = req.repeats`
        cloned_reqs = []
        for req in reqs:
            cloned_reqs.extend([req] * req.repeats)
lintangsutawika's avatar
lintangsutawika committed
409

410
411
        if (lm.world_size > 1) and (padding_requests[reqtype] > 0):
            for _ in range(padding_requests[reqtype]):
412
413
                cloned_reqs.extend([req] * req.repeats)

414
415
416
417
418
419
420
        # run requests through model
        resps = getattr(lm, reqtype)(cloned_reqs)

        # put responses from model into a list of length K for each request.
        for x, req in zip(resps, cloned_reqs):
            req.resps.append(x)

421
422
        if lm.world_size > 1:
            lm.accelerator.wait_for_everyone()
423

424
425
    RANK = lm.rank
    WORLD_SIZE = lm.world_size
426
427
    ### Postprocess outputs ###
    # TODO: del model here, maybe (idea: allow user to specify device of e.g. reward model separately)
428
429
    for task_output in eval_tasks:
        task = task_output.task
430
431
        task.apply_filters()

432
433
        ### Collect values of metrics on all datapoints ###
        # # unpack results and sort back in order and return control to Task
haileyschoelkopf's avatar
haileyschoelkopf committed
434
        # TODO: make it possible to use a different metric per filter
435
        # Pre-process task.instances to group by doc_id
436
        instances_by_doc_id = defaultdict(list)
437
438
439
440
441
        for instance in task.instances:
            instances_by_doc_id[instance.doc_id].append(instance)
        # Sort instances within each group
        for instances in instances_by_doc_id.values():
            instances.sort(key=lambda x: x.idx)
haileyschoelkopf's avatar
haileyschoelkopf committed
442
        # iterate over different filters used
443
444
445
        for filter_key in task.instances[0].filtered_resps.keys():
            doc_iterator = task.doc_iterator(
                rank=RANK, limit=limit, world_size=WORLD_SIZE
446
            )
447
            for doc_id, doc in doc_iterator:
448
                requests = instances_by_doc_id[doc_id]
lintangsutawika's avatar
lintangsutawika committed
449
                metrics = task.process_results(
450
                    doc, [req.filtered_resps[filter_key] for req in requests]
lintangsutawika's avatar
lintangsutawika committed
451
                )
452
453
454
455
456
457
458
459
                if log_samples:
                    target = task.doc_to_target(doc)
                    example = {
                        "doc_id": doc_id,
                        "doc": doc,
                        "target": target,
                        "arguments": [req.args for req in requests],
                        "resps": [req.resps for req in requests],
460
461
462
                        "filtered_resps": [
                            req.filtered_resps[filter_key] for req in requests
                        ],
463
464
465
466
467
468
469
470
471
472
                        "doc_hash": hash_string(
                            json.dumps(
                                requests[0].doc,
                                indent=2,
                                default=handle_non_serializable,
                                ensure_ascii=False,
                            )
                        ),
                        "prompt_hash": hash_string(requests[0].arguments[0]),
                        "target_hash": hash_string(str(target)),
473
474
                    }
                    example.update(metrics)
475
                    task_output.logged_samples.append(example)
476
                for metric, value in metrics.items():
477
                    task_output.sample_metrics[(metric, filter_key)].append(value)
478

479
480
    if WORLD_SIZE > 1:
        # if multigpu, then gather data across all ranks to rank 0
481
        # first gather logged samples across all ranks
482
483
484
485
486
487
488
489
        for task_output in eval_tasks:
            if log_samples:
                # for task_name, task_samples in list(samples.items()):
                full_samples = [None] * WORLD_SIZE if RANK == 0 else None
                torch.distributed.gather_object(
                    obj=task_output.logged_samples,
                    object_gather_list=full_samples,
                    dst=0,
490
                )
491

492
493
494
495
                if RANK == 0:
                    task_output.logged_samples = list(
                        itertools.chain.from_iterable(full_samples)
                    )
496

497
498
499
500
501
502
503
            # then collect metrics across all ranks
            for metrics in task_output.sample_metrics:
                metric_list = [None] * WORLD_SIZE if RANK == 0 else None
                torch.distributed.gather_object(
                    obj=task_output.sample_metrics[metrics],
                    object_gather_list=metric_list,
                    dst=0,
504
                )
505
506
507
508
                if RANK == 0:
                    task_output.sample_metrics[metrics] = list(
                        itertools.chain.from_iterable(metric_list)
                    )
509

510
    if RANK == 0:
511
512
        ### Aggregate results over all datapoints ###
        # aggregate results ; run bootstrap CIs
513
514
515
516
517
        for task_output in eval_tasks:
            task_output.calculate_aggregate_metric(bootstrap_iters=bootstrap_iters)
        results, samples, configs, versions, num_fewshot = consolidate_results(
            eval_tasks
        )
Fabrizio Milo's avatar
Fabrizio Milo committed
518

519
        ### Calculate group metrics ###
lintangsutawika's avatar
lintangsutawika committed
520
        if bool(results):
521
            for group, task_list in reversed(task_hierarchy.items()):
522
523
524
525
526
527
                if len(task_list) == 0:
                    # task_hierarchy entries are either
                    # `group_name: [subtask1, subtask2, ...]`
                    # or `task_name: []`.
                    # we only want to operate on groups here.
                    continue
528
529
530
531
532
533
534
535
536
                metric_list = list(
                    {
                        key
                        for task in task_list
                        for key in results[task].keys()
                        if "_stderr" not in key and key not in ["alias", "samples"]
                    }
                )
                for metric in metric_list:
537
538
539
                    stderr = "_stderr,".join(metric.split(","))

                    # gather metrics, sizes, and stderrs from subtasks
Baber Abbasi's avatar
Baber Abbasi committed
540
                    metrics = [
541
542
543
                        results[task][metric]
                        for task in task_list
                        if metric in results[task]
Baber Abbasi's avatar
Baber Abbasi committed
544
                    ]  # TODO: copy?
545
546
547
548
549
550
551
552
553
554
                    stderrs = [
                        results[task][stderr]
                        for task in task_list
                        if stderr in results[task]
                    ]
                    sizes = [
                        results[task]["samples"]
                        for task in task_list
                        if metric in results[task]
                    ]
555
556

                    # compute group's pooled metric and stderr
Baber Abbasi's avatar
Baber Abbasi committed
557
558
559
                    results[group][
                        metric
                    ] = lm_eval.api.metrics.aggregate_subtask_metrics(metrics, sizes)
560
561
562
563
                    # TODO: calculate grouped metric using aggregation fn
                    if "N/A" in stderrs:
                        results[group][stderr] = "N/A"
                    else:
Baber Abbasi's avatar
Baber Abbasi committed
564
565
566
                        results[group][
                            stderr
                        ] = lm_eval.api.metrics.pooled_sample_stderr(stderrs, sizes)
567
568
569
570
571
                        # TODO: allow GroupConfigs to choose which variance formula is used, for back-compatibility
                        # To use the old (likely incorrect) variance formula, comment out the above and uncomment this line:
                        # results[group][stderr] = lm_eval.api.metrics.combined_sample_stderr(stderrs, sizes, metrics=metrics)

                    results[group]["samples"] = sum(sizes)
lintangsutawika's avatar
lintangsutawika committed
572

573
574
        results_agg = defaultdict(dict)
        groups_agg = defaultdict(dict)
Lintang Sutawika's avatar
Lintang Sutawika committed
575
576
577
578
579
580
581
582
583
584
        all_tasks_list = list(task_hierarchy.keys())
        while True:
            add_tasks_list = list(k for k in results_agg.keys())
            left_tasks_list = sorted(list(set(all_tasks_list) - set(add_tasks_list)))
            if len(left_tasks_list) == 0:
                break

            _task_hierarchy = {
                k: v for k, v in task_hierarchy.items() if k in left_tasks_list
            }
585
            _results_agg, _groups_agg = prepare_print_tasks(_task_hierarchy, results)
Lintang Sutawika's avatar
Lintang Sutawika committed
586
587
588

            results_agg = {**results_agg, **_results_agg}
            groups_agg = {**groups_agg, **_groups_agg}
lintangsutawika's avatar
lintangsutawika committed
589

590
        for group_name, task_list in task_hierarchy.items():
Baber Abbasi's avatar
Baber Abbasi committed
591
592
593
594
            if task_list:
                num_fewshot[group_name] = num_fewshot[
                    task_list[0]
                ]  # TODO: validate this
595

596
        results_dict = {
597
            "results": dict(results_agg.items()),
lintangsutawika's avatar
lintangsutawika committed
598
            **({"groups": dict(groups_agg.items())} if bool(groups_agg) else {}),
599
            "group_subtasks": dict(reversed(task_hierarchy.items())),
600
601
            "configs": dict(sorted(configs.items())),
            "versions": dict(sorted(versions.items())),
602
            "n-shot": dict(sorted(num_fewshot.items())),
603
604
605
            "n-samples": {
                task_output.task_name: {
                    "original": len(task_output.task.eval_docs),
KonradSzafer's avatar
KonradSzafer committed
606
607
608
609
                    "effective": min(
                        limit if limit else len(task_output.task.eval_docs),
                        len(task_output.task.eval_docs),
                    ),
610
611
612
                }
                for task_output in eval_tasks
            },
613
        }
614
615
616
617
        if log_samples:
            results_dict["samples"] = dict(samples)

        return results_dict
Fabrizio Milo's avatar
Fabrizio Milo committed
618

619
620
    else:
        return None
621
622
623
624


def request_caching_arg_to_dict(cache_requests: str) -> dict:
    request_caching_args = {
625
626
627
        "cache_requests": cache_requests in {"true", "refresh"},
        "rewrite_requests_cache": cache_requests == "refresh",
        "delete_requests_cache": cache_requests == "delete",
628
629
630
    }

    return request_caching_args