evaluator.py 31.8 KB
Newer Older
Baber Abbasi's avatar
Baber Abbasi committed
1
import itertools
2
import json
3
import logging
4
import os
Baber Abbasi's avatar
Baber Abbasi committed
5
import random
6
import time
7
8
from collections import defaultdict
from typing import TYPE_CHECKING, List, Optional, Union
Baber Abbasi's avatar
Baber Abbasi committed
9

10
import numpy as np
Baber Abbasi's avatar
Baber Abbasi committed
11
import torch
lintangsutawika's avatar
lintangsutawika committed
12

lintangsutawika's avatar
lintangsutawika committed
13
import lm_eval.api.metrics
lintangsutawika's avatar
lintangsutawika committed
14
import lm_eval.api.registry
Lintang Sutawika's avatar
Lintang Sutawika committed
15
import lm_eval.api.task
Baber Abbasi's avatar
Baber Abbasi committed
16
import lm_eval.models
17
from lm_eval.caching.cache import delete_cache
18
from lm_eval.evaluator_utils import (
Lintang Sutawika's avatar
Lintang Sutawika committed
19
    consolidate_group_results,
20
21
    consolidate_results,
    get_sample_size,
Lintang Sutawika's avatar
Lintang Sutawika committed
22
    get_subtask_list,
23
24
25
26
27
    get_task_list,
    prepare_print_tasks,
    print_writeout,
    run_task_tests,
)
KonradSzafer's avatar
KonradSzafer committed
28
from lm_eval.loggers import EvaluationTracker
29
from lm_eval.loggers.utils import add_env_info, add_tokenizer_info, get_git_commit_hash
30
from lm_eval.tasks import TaskManager, get_task_dict
31
32
from lm_eval.utils import (
    handle_non_serializable,
33
    hash_dict_images,
34
35
    hash_string,
    positional_deprecated,
Baber Abbasi's avatar
Baber Abbasi committed
36
    setup_logging,
37
    simple_parse_args_string,
38
    wrap_text,
39
)
40

Fabrizio Milo's avatar
Fabrizio Milo committed
41

42
43
if TYPE_CHECKING:
    from lm_eval.api.model import LM
Lintang Sutawika's avatar
Lintang Sutawika committed
44
    from lm_eval.api.task import Task
45

Lintang Sutawika's avatar
Lintang Sutawika committed
46
47
eval_logger = logging.getLogger(__name__)

48

49
@positional_deprecated
Fabrizio Milo's avatar
Fabrizio Milo committed
50
51
def simple_evaluate(
    model,
52
53
    model_args: Optional[Union[str, dict]] = None,
    tasks: Optional[List[Union[str, dict, object]]] = None,
Baber Abbasi's avatar
Baber Abbasi committed
54
    num_fewshot: Optional[int] = None,
55
    batch_size: Optional[Union[int, str]] = None,
Baber Abbasi's avatar
Baber Abbasi committed
56
57
58
    max_batch_size: Optional[int] = None,
    device: Optional[str] = None,
    use_cache: Optional[str] = None,
59
60
61
    cache_requests: bool = False,
    rewrite_requests_cache: bool = False,
    delete_requests_cache: bool = False,
Baber Abbasi's avatar
Baber Abbasi committed
62
    limit: Optional[Union[int, float]] = None,
63
    samples: Optional[dict] = None,
Ethan Smith's avatar
Ethan Smith committed
64
65
66
67
    bootstrap_iters: int = 100000,
    check_integrity: bool = False,
    write_out: bool = False,
    log_samples: bool = True,
KonradSzafer's avatar
KonradSzafer committed
68
69
    evaluation_tracker: Optional[EvaluationTracker] = None,
    system_instruction: Optional[str] = None,
70
    apply_chat_template: Union[bool, str] = False,
KonradSzafer's avatar
KonradSzafer committed
71
    fewshot_as_multiturn: bool = False,
Baber Abbasi's avatar
Baber Abbasi committed
72
    gen_kwargs: Union[str, dict, None] = None,
73
    task_manager: Optional[TaskManager] = None,
Baber Abbasi's avatar
Baber Abbasi committed
74
    verbosity=None,
Baber Abbasi's avatar
Baber Abbasi committed
75
    predict_only: bool = False,
76
77
78
    random_seed: int = 0,
    numpy_random_seed: int = 1234,
    torch_random_seed: int = 1234,
79
    fewshot_random_seed: int = 1234,
Hojin Lee's avatar
Hojin Lee committed
80
    confirm_run_unsafe_code: bool = False,
Baber Abbasi's avatar
Baber Abbasi committed
81
    metadata: Optional[dict] = None,
Fabrizio Milo's avatar
Fabrizio Milo committed
82
):
83
    """Instantiate and evaluate a model on a list of tasks.
84

85
86
    :param model: Union[str, LM]
        Name of model or LM object, see lm_eval.models.get_model
87
88
    :param model_args: Optional[str, dict]
        String or dict arguments for each model class, see LM.create_from_arg_string and LM.create_from_arg_object.
89
        Ignored if `model` argument is a LM object.
90
    :param tasks: list[Union[str, dict, Task]]
Leo Gao's avatar
Leo Gao committed
91
        List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.
92
93
    :param num_fewshot: int
        Number of examples in few-shot context
94
    :param batch_size: int or str, optional
95
        Batch size for model
96
97
    :param max_batch_size: int, optional
        Maximal batch size to try with automatic batch size detection
98
    :param device: str, optional
99
        PyTorch device (e.g. "cpu" or "cuda:0") for running models
haileyschoelkopf's avatar
haileyschoelkopf committed
100
101
    :param use_cache: str, optional
        A path to a sqlite db file for caching model responses. `None` if not caching.
102
103
104
    :param cache_requests: bool, optional
        Speed up evaluation by caching the building of dataset requests. `None` if not caching.
    :param rewrite_requests_cache: bool, optional
Baber Abbasi's avatar
Baber Abbasi committed
105
        Rewrites all the request cache if set to `True`. `None` if not desired.
106
    :param delete_requests_cache: bool, optional
Baber Abbasi's avatar
Baber Abbasi committed
107
        Deletes all the request cache if set to `True`. `None` if not desired.
108
109
    :param limit: int or float, optional
        Limit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples.
110
111
    :param samples: dictionary, optional
        Dictionary indicating which examples should be tested in each task, e.g., {"mmlu_astronomy":[0,3,6],"mmlu_anatomy":[1,4,7,10]}.
112
    :param bootstrap_iters:
113
        Number of iterations for bootstrap statistics, used when calculating stderrs. set to 0 for no stderr calculations to be performed.
Stephen Hogg's avatar
Stephen Hogg committed
114
115
    :param check_integrity: bool
        Whether to run the relevant part of the test suite for the tasks
116
    :param write_out: bool
117
118
119
        If True, write out an example document and model input for checking task integrity
    :param log_samples: bool
        If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
KonradSzafer's avatar
KonradSzafer committed
120
121
    :param system_instruction: str
        System instruction to be applied to the prompt
122
123
124
125
126
    :param apply_chat_template: Union[bool, str]
        Specifies whether to apply a chat template to the prompt.
        - If set to True, the default chat template is applied.
        - If set to a string, applies the specified chat template by name.
        Defaults to False (no chat template applied).
KonradSzafer's avatar
KonradSzafer committed
127
128
    :param fewshot_as_multiturn: bool
        Whether to provide the fewshot examples as a multiturn conversation or a single user turn.
Baber Abbasi's avatar
Baber Abbasi committed
129
130
    :param gen_kwargs: dict or comma-separated string
        Arguments for model generation
131
        Ignored for all tasks with loglikelihood output_type
Baber Abbasi's avatar
Baber Abbasi committed
132
    :param verbosity: str
Lintang Sutawika's avatar
Lintang Sutawika committed
133
        Verbosity level for logging
Baber Abbasi's avatar
Baber Abbasi committed
134
135
    :param predict_only: bool
        If true only model outputs will be generated and returned. Metrics will not be evaluated
136
137
138
139
140
141
    :param random_seed: int
        Random seed for python's random module. If set to None, the seed will not be set.
    :param numpy_random_seed: int
        Random seed for numpy. If set to None, the seed will not be set.
    :param torch_random_seed: int
        Random seed for torch. If set to None, the seed will not be set.
142
143
    :param fewshot_random_seed: int
        Random seed for fewshot sampler random generator. If set to None, the seed of generator will be set to None.
Baber Abbasi's avatar
Baber Abbasi committed
144
145
146
    :param metadata: dict
        Additional metadata to be added to the task manager. Will get passed to the download function of the task.
    return
147
        Dictionary of results
148
    """
Baber Abbasi's avatar
Baber Abbasi committed
149
150
    if verbosity is not None:
        setup_logging(verbosity=verbosity)
151
    start_date = time.time()
152

153
154
155
156
157
    if limit is not None and samples is not None:
        raise ValueError(
            "Either 'limit' or 'samples' must be None, but both are not None."
        )

158
    _NEEDS_CHAT_TEMPLATE = ("inst", "chat")
159
    if (
160
161
162
163
        (
            isinstance(model_args, str)
            and any(kw in model_args.lower() for kw in _NEEDS_CHAT_TEMPLATE)
        )
164
165
        or (
            isinstance(model_args, dict)
166
167
168
169
            and any(
                any(kw in str(v).lower() for kw in _NEEDS_CHAT_TEMPLATE)
                for v in model_args.values()
            )
170
171
        )
    ) and not apply_chat_template:
Baber Abbasi's avatar
Baber Abbasi committed
172
        eval_logger.warning(
173
174
175
176
177
            wrap_text(
                f"""pretrained={model_args.get("pretrained") if isinstance(model_args, dict) else model_args} appears to be an
                instruct or chat variant but chat template is not applied.
                Recommend setting `apply_chat_template` (optionally `fewshot_as_multiturn`).""",
            )
Baber Abbasi's avatar
Baber Abbasi committed
178
179
        )

180
181
182
183
    if delete_requests_cache:
        eval_logger.info("Deleting requests cache...")
        delete_cache()

184
    seed_message = []
185
186
    if random_seed is not None:
        # See https://github.com/EleutherAI/lm-evaluation-harness/pull/1412
187
        seed_message.append(f"Setting random seed to {random_seed}")
188
189
190
        random.seed(random_seed)

    if numpy_random_seed is not None:
191
        seed_message.append(f"Setting numpy seed to {numpy_random_seed}")
192
193
194
        np.random.seed(numpy_random_seed)

    if torch_random_seed is not None:
195
        seed_message.append(f"Setting torch manual seed to {torch_random_seed}")
196
197
        torch.manual_seed(torch_random_seed)

198
199
200
    if fewshot_random_seed is not None:
        seed_message.append(f"Setting fewshot manual seed to {fewshot_random_seed}")

201
202
203
    if seed_message:
        eval_logger.info(" | ".join(seed_message))

204
205
    if tasks is None:
        tasks = []
206
207
208
209
    if len(tasks) == 0:
        raise ValueError(
            "No tasks specified, or no tasks found. Please verify the task names."
        )
210

lintangsutawika's avatar
lintangsutawika committed
211
    if gen_kwargs is not None:
Baber Abbasi's avatar
Baber Abbasi committed
212
213
        if isinstance(gen_kwargs, str):
            gen_kwargs = simple_parse_args_string(gen_kwargs)
lintangsutawika's avatar
udate  
lintangsutawika committed
214
        eval_logger.warning(
Baber Abbasi's avatar
Baber Abbasi committed
215
            f"generation_kwargs: {gen_kwargs} specified through cli, these settings will update set parameters in yaml tasks. "
216
            "Ensure 'do_sample=True' for non-greedy decoding!"
lintangsutawika's avatar
udate  
lintangsutawika committed
217
        )
Baber Abbasi's avatar
Baber Abbasi committed
218
        if not gen_kwargs:
lintangsutawika's avatar
lintangsutawika committed
219
220
            gen_kwargs = None

221
    if isinstance(model, str):
Fabrizio Milo's avatar
Fabrizio Milo committed
222
        if model_args is None:
223
            eval_logger.warning("model_args not specified. Using defaults.")
Fabrizio Milo's avatar
Fabrizio Milo committed
224
            model_args = ""
225

226
        if isinstance(model_args, dict):
227
228
229
            eval_logger.info(
                f"Initializing {model} model, with arguments: {model_args}"
            )
230
231
232
233
234
235
236
237
238
239
            lm = lm_eval.api.registry.get_model(model).create_from_arg_obj(
                model_args,
                {
                    "batch_size": batch_size,
                    "max_batch_size": max_batch_size,
                    "device": device,
                },
            )

        else:
240
            eval_logger.info(
241
242
243
                wrap_text(
                    f"Initializing {model} model, with arguments: {simple_parse_args_string(model_args)}"
                )
244
            )
245
246
247
248
249
250
251
252
            lm = lm_eval.api.registry.get_model(model).create_from_arg_string(
                model_args,
                {
                    "batch_size": batch_size,
                    "max_batch_size": max_batch_size,
                    "device": device,
                },
            )
253
    else:
254
        if not isinstance(model, lm_eval.api.model.LM):
255
256
257
            raise TypeError(
                f"The value of `model` passed to simple_evaluate() was of type {type(model)}, but is required to be a subclass of lm_eval.api.model.LM . This may be because you are passing an initialized Hugging Face PreTrainedModel without having wrapped it in `lm_eval.models.huggingface.HFLM(pretrained=my_model)` first."
            )
258
        eval_logger.info("Using pre-initialized model")
259
        lm = model
260

haileyschoelkopf's avatar
haileyschoelkopf committed
261
    if use_cache is not None:
262
        eval_logger.info(f"Using cache at {use_cache + '_rank' + str(lm.rank) + '.db'}")
haileyschoelkopf's avatar
haileyschoelkopf committed
263
264
265
266
267
        lm = lm_eval.api.model.CachingLM(
            lm,
            use_cache
            # each rank receives a different cache db.
            # necessary to avoid multiple writes to cache at once
268
269
270
            + "_rank"
            + str(lm.rank)
            + ".db",
haileyschoelkopf's avatar
haileyschoelkopf committed
271
272
        )

273
    if task_manager is None:
Baber Abbasi's avatar
Baber Abbasi committed
274
275
276
277
278
279
280
281
        metadata = (
            simple_parse_args_string(model_args)
            if isinstance(model_args, str)
            else model_args
            if isinstance(model_args, dict)
            else {}
        ) | (metadata or {})
        task_manager = TaskManager(metadata=metadata)
282

Baber Abbasi's avatar
Baber Abbasi committed
283
284
285
286
    task_dict = get_task_dict(
        tasks,
        task_manager,
    )
Baber Abbasi's avatar
Baber Abbasi committed
287

Lintang Sutawika's avatar
Lintang Sutawika committed
288
289
    # helper function to recursively apply config overrides to leaf subtasks, skipping their constituent groups.
    # (setting of num_fewshot ; bypassing metric calculation ; setting fewshot seed)
Baber's avatar
Baber committed
290
    def _adjust_config(task_dict: dict[str, "Task"]) -> dict[str, "Task"]:
Lintang Sutawika's avatar
Lintang Sutawika committed
291
292
293
294
295
296
297
        adjusted_task_dict = {}
        for task_name, task_obj in task_dict.items():
            if isinstance(task_obj, dict):
                adjusted_task_dict = {
                    **adjusted_task_dict,
                    **{task_name: _adjust_config(task_obj)},
                }
298

299
            else:
Lintang Sutawika's avatar
Lintang Sutawika committed
300
301
302
303
304
                if task_obj.get_config("output_type") == "generate_until":
                    if gen_kwargs is not None:
                        task_obj.set_config(
                            key="generation_kwargs", value=gen_kwargs, update=True
                        )
Baber Abbasi's avatar
Baber Abbasi committed
305
306
307
                    eval_logger.info(
                        f"{task_obj.config.task}: Using gen_kwargs: {task_obj.config.generation_kwargs}"
                    )
Lintang Sutawika's avatar
Lintang Sutawika committed
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341

                if predict_only:
                    eval_logger.info(
                        f"Processing {task_name} in output-only mode. Metrics will not be calculated!"
                    )
                    # we have to change the class properties post-hoc. This is pretty hacky.
                    task_obj.override_metric(metric_name="bypass")

                # override tasks' fewshot values to the provided num_fewshot arg value
                # except if tasks have it set to 0 manually in their configs--then we should never overwrite that
                if num_fewshot is not None:
                    if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0:
                        eval_logger.info(
                            f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored."
                        )
                    else:
                        eval_logger.warning(
                            f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}"
                        )
                        task_obj.set_config(key="num_fewshot", value=num_fewshot)
                else:
                    # if num_fewshot not provided, and the task does not define a default one, default to 0
                    if (
                        default_num_fewshot := task_obj.get_config("num_fewshot")
                    ) is None:
                        task_obj.set_config(key="num_fewshot", value=0)
                # fewshot_random_seed set for tasks, even with a default num_fewshot (e.g. in the YAML file)
                task_obj.set_fewshot_seed(seed=fewshot_random_seed)

                adjusted_task_dict[task_name] = task_obj

        return adjusted_task_dict

    task_dict = _adjust_config(task_dict)
Jonathan Tow's avatar
Merge  
Jonathan Tow committed
342

Stephen Hogg's avatar
Stephen Hogg committed
343
    if check_integrity:
344
        run_task_tests(task_list=tasks)
Stephen Hogg's avatar
Stephen Hogg committed
345

KonradSzafer's avatar
KonradSzafer committed
346
347
348
349
350
    if evaluation_tracker is not None:
        evaluation_tracker.general_config_tracker.log_experiment_args(
            model_source=model,
            model_args=model_args,
            system_instruction=system_instruction,
Baber Abbasi's avatar
Baber Abbasi committed
351
352
353
            chat_template=lm.chat_template(apply_chat_template)
            if apply_chat_template
            else None,
354
            fewshot_as_multiturn=fewshot_as_multiturn,
KonradSzafer's avatar
KonradSzafer committed
355
356
        )

357
358
359
360
    results = evaluate(
        lm=lm,
        task_dict=task_dict,
        limit=limit,
361
        samples=samples,
362
363
        cache_requests=cache_requests,
        rewrite_requests_cache=rewrite_requests_cache,
Niklas Muennighoff's avatar
Niklas Muennighoff committed
364
        bootstrap_iters=bootstrap_iters,
365
        write_out=write_out,
Lintang Sutawika's avatar
Lintang Sutawika committed
366
        log_samples=True if predict_only else log_samples,
KonradSzafer's avatar
KonradSzafer committed
367
368
369
        system_instruction=system_instruction,
        apply_chat_template=apply_chat_template,
        fewshot_as_multiturn=fewshot_as_multiturn,
Baber Abbasi's avatar
Baber Abbasi committed
370
        verbosity=verbosity,
Hojin Lee's avatar
Hojin Lee committed
371
        confirm_run_unsafe_code=confirm_run_unsafe_code,
372
    )
Baber Abbasi's avatar
Baber Abbasi committed
373
    if verbosity is not None:
Zeyuan Allen-Zhu's avatar
Zeyuan Allen-Zhu committed
374
        setup_logging(verbosity=verbosity)
375

376
    if lm.rank == 0:
377
378
379
380
381
382
383
        if isinstance(model, str):
            model_name = model
        elif hasattr(model, "config") and hasattr(model.config, "_name_or_path"):
            model_name = model.config._name_or_path
        else:
            model_name = type(model).__name__

384
385
        # add info about the model and few shot config
        results["config"] = {
386
            "model": model_name,
387
388
            "model_args": model_args,
        }
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
        # add more detailed model info if available
        if isinstance(lm, lm_eval.models.huggingface.HFLM):
            results["config"].update(lm.get_model_info())
        # add info about execution
        results["config"].update(
            {
                "batch_size": batch_size,
                "batch_sizes": (
                    list(lm.batch_sizes.values()) if hasattr(lm, "batch_sizes") else []
                ),
                "device": device,
                "use_cache": use_cache,
                "limit": limit,
                "bootstrap_iters": bootstrap_iters,
                "gen_kwargs": gen_kwargs,
404
405
406
407
                "random_seed": random_seed,
                "numpy_seed": numpy_random_seed,
                "torch_seed": torch_random_seed,
                "fewshot_seed": fewshot_random_seed,
408
409
            }
        )
410
        results["git_hash"] = get_git_commit_hash()
411
        results["date"] = start_date
412
        add_env_info(results)  # additional environment info to results
achervyakov's avatar
achervyakov committed
413
        add_tokenizer_info(results, lm)  # additional info about tokenizer
414
415
416
        return results
    else:
        return None
417

Leo Gao's avatar
Leo Gao committed
418

419
@positional_deprecated
Fabrizio Milo's avatar
Fabrizio Milo committed
420
def evaluate(
421
    lm: "LM",
Fabrizio Milo's avatar
Fabrizio Milo committed
422
    task_dict,
Baber Abbasi's avatar
Baber Abbasi committed
423
    limit: Optional[int] = None,
424
    samples: Optional[dict] = None,
425
426
    cache_requests: bool = False,
    rewrite_requests_cache: bool = False,
Baber Abbasi's avatar
Baber Abbasi committed
427
    bootstrap_iters: Optional[int] = 100000,
Ethan Smith's avatar
Ethan Smith committed
428
429
    write_out: bool = False,
    log_samples: bool = True,
KonradSzafer's avatar
KonradSzafer committed
430
    system_instruction: Optional[str] = None,
431
    apply_chat_template: Union[bool, str] = False,
KonradSzafer's avatar
KonradSzafer committed
432
    fewshot_as_multiturn: bool = False,
433
    verbosity: str = "INFO",
Hojin Lee's avatar
Hojin Lee committed
434
    confirm_run_unsafe_code: bool = False,
Fabrizio Milo's avatar
Fabrizio Milo committed
435
):
436
437
438
439
440
    """Instantiate and evaluate a model on a list of tasks.

    :param lm: obj
        Language Model
    :param task_dict: dict[str, Task]
haileyschoelkopf's avatar
haileyschoelkopf committed
441
        Dictionary of tasks. Tasks will be taken to have name type(task).config.task .
442
443
    :param limit: int, optional
        Limit the number of examples per task (only use this for testing)
444
445
    :param samples: dictionary, optional
        Dictionary indicating which examples should be tested in each task, e.g., {"mmlu_astronomy":[0,3,6],"mmlu_anatomy":[1,4,7,10]}.
Hojin Lee's avatar
Hojin Lee committed
446
447
448
449
    :param cache_requests: bool, optional
        Speed up evaluation by caching the building of dataset requests.
    :param rewrite_requests_cache: bool, optional
        Rewrites all the request cache if set to `True`.
450
    :param bootstrap_iters:
451
        Number of iterations for bootstrap statistics, used when calculating stderr. Set to 0 for skipping all stderr calculations.
452
    :param write_out: bool
453
454
455
        If True, write out an example document and model input for checking task integrity
    :param log_samples: bool
        If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
KonradSzafer's avatar
KonradSzafer committed
456
457
    :param system_instruction: str
        System instruction to be applied to the prompt
458
459
460
461
462
    :param apply_chat_template: Union[bool, str]
        Specifies whether to apply a chat template to the prompt.
        - If set to True, the default chat template is applied.
        - If set to a string, applies the specified chat template by name.
        Defaults to False (no chat template applied).
KonradSzafer's avatar
KonradSzafer committed
463
464
    :param fewshot_as_multiturn: bool
        Whether to provide the fewshot examples as a multiturn conversation or a single user turn.
Hojin Lee's avatar
Hojin Lee committed
465
466
467
468
    :param verbosity: str
        Verbosity level for logging
    :param confirm_run_unsafe_code: bool
        Whether to confirm running tasks marked as unsafe.
469
470
471
    :return
        Dictionary of results
    """
472

473
474
475
476
477
478
    if limit is not None and samples is not None:
        raise ValueError(
            "Either 'limit' or 'samples' must be None, but both are not None."
        )
    if samples is not None:
        eval_logger.info(f"Evaluating examples for tasks {list(samples.keys())}")
479
480
481
482
    if apply_chat_template:
        eval_logger.warning(
            "Chat template formatting change affects loglikelihood and multiple-choice tasks. See docs/chat-template-readme.md for details."
        )
483
    # tracks all Instances/requests a model must generate output on.
484
    requests = defaultdict(list)
485
486
    # stores the amount to pad out reqs per req. type so that
    # number of fwd passes per distributed rank is equal
487
    padding_requests = defaultdict(int)
488

489
    # get lists of group hierarchy and each type of request
Lintang Sutawika's avatar
Lintang Sutawika committed
490
    eval_tasks = get_task_list(task_dict)
491
    if not log_samples:
492
        if not all(
493
494
            "bypass" not in getattr(task_output.task, "_metric_fn_list", {}).keys()
            for task_output in eval_tasks
495
496
        ):
            raise ValueError("log_samples must be True for 'bypass' metric-only tasks")
497

Hojin Lee's avatar
Hojin Lee committed
498
499
500
    # validation checks:
    # 1.are we running multimodal task <-> non-multimodal model class, or vice-versa.
    # 2.are we running code that is marked as unsafe.
501
    incompatible_tasks = []
502
503
    for task_output in eval_tasks:
        task: Task = task_output.task
504

505
        if getattr(task, "MULTIMODAL", False) and not getattr(lm, "MULTIMODAL", False):
506
            incompatible_tasks.append(task_output.task_name)
Hojin Lee's avatar
Hojin Lee committed
507
508
509
510
        elif getattr(task, "UNSAFE_CODE", False) and not confirm_run_unsafe_code:
            raise ValueError(
                f"Attempted to run task: {task_output.task_name} which is marked as unsafe. Set confirm_run_unsafe_code=True to run this task."
            )
511
512
513
514
515
    if len(incompatible_tasks) > 0:
        if not getattr(lm, "MULTIMODAL", False):
            raise ValueError(
                f"Attempted to run tasks: {incompatible_tasks} which require multimodal input, but the selected model type does not currently implement this. Multimodal support is currently restricted to the ['hf-multimodal', 'vllm-vlm'] model type."
            )
Hojin Lee's avatar
Hojin Lee committed
516
    # end validation check
517

Chenjie Luo's avatar
Chenjie Luo committed
518
519
520
    # Cache the limit arg.
    limit_arg = limit
    limits = []
521
522
523
    for task_output in eval_tasks:
        task: Task = task_output.task

Chenjie Luo's avatar
Chenjie Luo committed
524
525
        limit = get_sample_size(task, limit_arg)
        limits.append(limit)
526
527
        task.build_all_requests(
            limit=limit,
528
529
530
            samples=samples.get(task_output.task_name, None)
            if samples is not None
            else samples,
531
532
533
534
            rank=lm.rank,
            world_size=lm.world_size,
            cache_requests=cache_requests,
            rewrite_requests_cache=rewrite_requests_cache,
KonradSzafer's avatar
KonradSzafer committed
535
            system_instruction=system_instruction,
536
            apply_chat_template=bool(apply_chat_template),
KonradSzafer's avatar
KonradSzafer committed
537
            fewshot_as_multiturn=fewshot_as_multiturn,
538
539
540
541
542
543
            chat_template=getattr(lm, "apply_chat_template")
            if apply_chat_template
            else None,
            tokenizer_name=getattr(lm, "tokenizer_name", "")
            if apply_chat_template
            else "",
544
        )
545
        eval_logger.debug(
546
            f"Task: {task_output.task_name}; number of requests on this rank: {len(task.instances)}"
haileyschoelkopf's avatar
haileyschoelkopf committed
547
548
        )
        if write_out:
549
            print_writeout(task)
550
        # aggregate Instances by LM method requested to get output.
lintangsutawika's avatar
lintangsutawika committed
551
552
553
        for instance in task.instances:
            reqtype = instance.request_type
            requests[reqtype].append(instance)
554
555

        if lm.world_size > 1:
556
557
558
559
            instances_rnk = torch.tensor(len(task._instances), device=lm.device)
            gathered_item = (
                lm.accelerator.gather(instances_rnk).cpu().detach().numpy().tolist()
            )
560
561
562
563
564
565
            # "multiple_choice" task types dispatch (several) "loglikelihood" request types
            reqtype = (
                "loglikelihood"
                if task.OUTPUT_TYPE == "multiple_choice"
                else task.OUTPUT_TYPE
            )
566
            # compute number of pseudo-batches to pad with (FSDP/DDP require even batches among ranks)
567
            numpad = max(gathered_item) - gathered_item[lm.rank]
568
569
            # todo: may not account for padding in cases like SquadV2 which has multiple req types
            padding_requests[reqtype] += numpad
570

571
    ### Run LM on inputs, get all outputs ###
Leo Gao's avatar
Leo Gao committed
572
573
    # execute each type of request
    for reqtype, reqs in requests.items():
574
        eval_logger.info(f"Running {reqtype} requests")
575
576
577
578
        # create `K` copies of each request `req` based off `K = req.repeats`
        cloned_reqs = []
        for req in reqs:
            cloned_reqs.extend([req] * req.repeats)
lintangsutawika's avatar
lintangsutawika committed
579

580
581
        if (lm.world_size > 1) and (padding_requests[reqtype] > 0):
            for _ in range(padding_requests[reqtype]):
582
583
                cloned_reqs.extend([req] * req.repeats)

584
585
586
587
588
589
590
        # run requests through model
        resps = getattr(lm, reqtype)(cloned_reqs)

        # put responses from model into a list of length K for each request.
        for x, req in zip(resps, cloned_reqs):
            req.resps.append(x)

591
592
        if lm.world_size > 1:
            lm.accelerator.wait_for_everyone()
593

594
595
    RANK = lm.rank
    WORLD_SIZE = lm.world_size
596
597
    ### Postprocess outputs ###
    # TODO: del model here, maybe (idea: allow user to specify device of e.g. reward model separately)
Chenjie Luo's avatar
Chenjie Luo committed
598
    for task_output, limit in zip(eval_tasks, limits):
599
        task = task_output.task
600
601
        task.apply_filters()

602
603
        ### Collect values of metrics on all datapoints ###
        # # unpack results and sort back in order and return control to Task
haileyschoelkopf's avatar
haileyschoelkopf committed
604
        # TODO: make it possible to use a different metric per filter
605
        # Pre-process task.instances to group by doc_id
606
        instances_by_doc_id = defaultdict(list)
607
608
609
610
611
        for instance in task.instances:
            instances_by_doc_id[instance.doc_id].append(instance)
        # Sort instances within each group
        for instances in instances_by_doc_id.values():
            instances.sort(key=lambda x: x.idx)
haileyschoelkopf's avatar
haileyschoelkopf committed
612
        # iterate over different filters used
613
        for filter_key in task.instances[0].filtered_resps.keys():
614
615
616
617
618
            indices = (
                samples.get(task_output.task_name, None)
                if samples is not None
                else None
            )
619
            doc_iterator = task.doc_iterator(
620
621
622
623
                rank=RANK,
                limit=limit,
                world_size=WORLD_SIZE,
                samples=indices,
624
            )
625
            for doc_id, doc in doc_iterator:
626
627
628
629
                if indices:
                    doc_id_true = indices[doc_id]
                else:
                    doc_id_true = doc_id
630
                requests = instances_by_doc_id[doc_id]
lintangsutawika's avatar
lintangsutawika committed
631
                metrics = task.process_results(
632
                    doc, [req.filtered_resps[filter_key] for req in requests]
lintangsutawika's avatar
lintangsutawika committed
633
                )
634
635
636
                if log_samples:
                    target = task.doc_to_target(doc)
                    example = {
637
                        "doc_id": doc_id_true,
638
639
640
641
                        "doc": doc,
                        "target": target,
                        "arguments": [req.args for req in requests],
                        "resps": [req.resps for req in requests],
642
643
644
                        "filtered_resps": [
                            req.filtered_resps[filter_key] for req in requests
                        ],
645
646
                        "filter": filter_key,
                        "metrics": list(metrics.keys()),
647
648
649
650
651
652
653
654
655
656
                        "doc_hash": hash_string(
                            json.dumps(
                                requests[0].doc,
                                indent=2,
                                default=handle_non_serializable,
                                ensure_ascii=False,
                            )
                        ),
                        "prompt_hash": hash_string(requests[0].arguments[0]),
                        "target_hash": hash_string(str(target)),
657
658
                    }
                    example.update(metrics)
659
                    task_output.logged_samples.append(example)
660
                for metric, value in metrics.items():
661
                    task_output.sample_metrics[(metric, filter_key)].append(value)
662

663
664
    if WORLD_SIZE > 1:
        # if multigpu, then gather data across all ranks to rank 0
665
        # first gather logged samples across all ranks
666
667
668
669
670
671
672
673
        for task_output in eval_tasks:
            if log_samples:
                # for task_name, task_samples in list(samples.items()):
                full_samples = [None] * WORLD_SIZE if RANK == 0 else None
                torch.distributed.gather_object(
                    obj=task_output.logged_samples,
                    object_gather_list=full_samples,
                    dst=0,
674
                )
675

676
677
678
679
                if RANK == 0:
                    task_output.logged_samples = list(
                        itertools.chain.from_iterable(full_samples)
                    )
680

681
682
683
684
685
686
687
            # then collect metrics across all ranks
            for metrics in task_output.sample_metrics:
                metric_list = [None] * WORLD_SIZE if RANK == 0 else None
                torch.distributed.gather_object(
                    obj=task_output.sample_metrics[metrics],
                    object_gather_list=metric_list,
                    dst=0,
688
                )
689
690
691
692
                if RANK == 0:
                    task_output.sample_metrics[metrics] = list(
                        itertools.chain.from_iterable(metric_list)
                    )
693

694
    if RANK == 0:
695
696
        ### Aggregate results over all datapoints ###
        # aggregate results ; run bootstrap CIs
697
698
        for task_output in eval_tasks:
            task_output.calculate_aggregate_metric(bootstrap_iters=bootstrap_iters)
699
700
701
702
703
704
705
706
        (
            results,
            samples,
            configs,
            versions,
            num_fewshot,
            higher_is_better,
        ) = consolidate_results(eval_tasks)
Fabrizio Milo's avatar
Fabrizio Milo committed
707

708
        ### Calculate group metrics ###
lintangsutawika's avatar
lintangsutawika committed
709
        if bool(results):
Lintang Sutawika's avatar
Lintang Sutawika committed
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
            results, versions, show_group_table, *_ = consolidate_group_results(
                results, versions, task_dict
            )

        results_agg, group_agg = prepare_print_tasks(task_dict, results)
        subtask_list = get_subtask_list(task_dict)

        # collect all higher_is_better values for metrics
        # in the group's subtasks.
        # TODO: clean this up ; unify with the below metric_list loop?
        _higher_is_better = {}
        for group, task_list in subtask_list.items():
            if (
                len(task_list) != 0
            ):  # subtask list will list "task_name": [] for solo tasks
725
726
727
728
                for task in task_list:
                    for m, h in higher_is_better[task].items():
                        if m not in _higher_is_better.keys():
                            _higher_is_better[m] = h
lintangsutawika's avatar
lintangsutawika committed
729

Lintang Sutawika's avatar
Lintang Sutawika committed
730
731
732
733
734
735
736
737
738
739
                        if (
                            m in _higher_is_better
                            and _higher_is_better[m] is not None
                            and _higher_is_better[m] != h
                        ):
                            eval_logger.warning(
                                f"Higher_is_better values for metric {m} in group {group} are not consistent. Defaulting to None."
                            )
                            _higher_is_better[m] = None
                higher_is_better[group] = _higher_is_better
740

741
        results_dict = {
742
            "results": dict(results_agg.items()),
Lintang Sutawika's avatar
Lintang Sutawika committed
743
744
745
746
747
748
            **(
                {"groups": dict(group_agg.items())}
                if (bool(group_agg) & show_group_table)
                else {}
            ),
            "group_subtasks": dict(reversed(subtask_list.items())),
749
750
            "configs": dict(sorted(configs.items())),
            "versions": dict(sorted(versions.items())),
751
            "n-shot": dict(sorted(num_fewshot.items())),
752
            "higher_is_better": dict(sorted(higher_is_better.items())),
753
754
755
            "n-samples": {
                task_output.task_name: {
                    "original": len(task_output.task.eval_docs),
KonradSzafer's avatar
KonradSzafer committed
756
757
758
759
                    "effective": min(
                        limit if limit else len(task_output.task.eval_docs),
                        len(task_output.task.eval_docs),
                    ),
760
                }
Chenjie Luo's avatar
Chenjie Luo committed
761
                for task_output, limit in zip(eval_tasks, limits)
762
            },
763
        }
764
        if log_samples:
765
766
767
768
            # default: hash images
            samples = (
                hash_dict_images(samples)
                if os.environ.get("LMEVAL_HASHMM", "1") != "0"
Baber Abbasi's avatar
Baber Abbasi committed
769
                and (hasattr(lm, "MULTIMODAL"))
770
771
                else samples
            )
772
773
774
            results_dict["samples"] = dict(samples)

        return results_dict
Fabrizio Milo's avatar
Fabrizio Milo committed
775

776
777
    else:
        return None
778
779
780
781


def request_caching_arg_to_dict(cache_requests: str) -> dict:
    request_caching_args = {
782
783
784
        "cache_requests": cache_requests in {"true", "refresh"},
        "rewrite_requests_cache": cache_requests == "refresh",
        "delete_requests_cache": cache_requests == "delete",
785
786
787
    }

    return request_caching_args