evaluator.py 29.1 KB
Newer Older
Baber Abbasi's avatar
Baber Abbasi committed
1
import itertools
2
import json
3
import logging
Baber Abbasi's avatar
Baber Abbasi committed
4
import random
5
import time
6
7
from collections import defaultdict
from typing import TYPE_CHECKING, List, Optional, Union
Baber Abbasi's avatar
Baber Abbasi committed
8

9
import numpy as np
Baber Abbasi's avatar
Baber Abbasi committed
10
import torch
lintangsutawika's avatar
lintangsutawika committed
11

lintangsutawika's avatar
lintangsutawika committed
12
import lm_eval.api.metrics
lintangsutawika's avatar
lintangsutawika committed
13
import lm_eval.api.registry
Lintang Sutawika's avatar
Lintang Sutawika committed
14
import lm_eval.api.task
Baber Abbasi's avatar
Baber Abbasi committed
15
import lm_eval.models
16
from lm_eval.caching.cache import delete_cache
17
from lm_eval.evaluator_utils import (
Lintang Sutawika's avatar
Lintang Sutawika committed
18
    consolidate_group_results,
19
20
    consolidate_results,
    get_sample_size,
Lintang Sutawika's avatar
Lintang Sutawika committed
21
    get_subtask_list,
22
23
24
25
26
    get_task_list,
    prepare_print_tasks,
    print_writeout,
    run_task_tests,
)
KonradSzafer's avatar
KonradSzafer committed
27
from lm_eval.loggers import EvaluationTracker
28
from lm_eval.loggers.utils import add_env_info, add_tokenizer_info, get_git_commit_hash
Lintang Sutawika's avatar
Lintang Sutawika committed
29
30
31
32
from lm_eval.tasks import (
    TaskManager,
    get_task_dict,
)
33
34
35
36
from lm_eval.utils import (
    handle_non_serializable,
    hash_string,
    positional_deprecated,
Baber Abbasi's avatar
Baber Abbasi committed
37
    setup_logging,
38
39
    simple_parse_args_string,
)
40

Fabrizio Milo's avatar
Fabrizio Milo committed
41

42
43
if TYPE_CHECKING:
    from lm_eval.api.model import LM
Lintang Sutawika's avatar
Lintang Sutawika committed
44
    from lm_eval.api.task import Task
45

Lintang Sutawika's avatar
Lintang Sutawika committed
46
47
eval_logger = logging.getLogger(__name__)

48

49
@positional_deprecated
Fabrizio Milo's avatar
Fabrizio Milo committed
50
51
def simple_evaluate(
    model,
52
53
    model_args: Optional[Union[str, dict]] = None,
    tasks: Optional[List[Union[str, dict, object]]] = None,
Baber Abbasi's avatar
Baber Abbasi committed
54
    num_fewshot: Optional[int] = None,
55
    batch_size: Optional[Union[int, str]] = None,
Baber Abbasi's avatar
Baber Abbasi committed
56
57
58
    max_batch_size: Optional[int] = None,
    device: Optional[str] = None,
    use_cache: Optional[str] = None,
59
60
61
    cache_requests: bool = False,
    rewrite_requests_cache: bool = False,
    delete_requests_cache: bool = False,
Baber Abbasi's avatar
Baber Abbasi committed
62
    limit: Optional[Union[int, float]] = None,
Ethan Smith's avatar
Ethan Smith committed
63
64
65
66
    bootstrap_iters: int = 100000,
    check_integrity: bool = False,
    write_out: bool = False,
    log_samples: bool = True,
KonradSzafer's avatar
KonradSzafer committed
67
68
    evaluation_tracker: Optional[EvaluationTracker] = None,
    system_instruction: Optional[str] = None,
69
    apply_chat_template: Union[bool, str] = False,
KonradSzafer's avatar
KonradSzafer committed
70
    fewshot_as_multiturn: bool = False,
71
72
    gen_kwargs: Optional[str] = None,
    task_manager: Optional[TaskManager] = None,
Lintang Sutawika's avatar
Lintang Sutawika committed
73
    verbostiy=None,
Baber Abbasi's avatar
Baber Abbasi committed
74
    predict_only: bool = False,
75
76
77
    random_seed: int = 0,
    numpy_random_seed: int = 1234,
    torch_random_seed: int = 1234,
78
    fewshot_random_seed: int = 1234,
Hojin Lee's avatar
Hojin Lee committed
79
    confirm_run_unsafe_code: bool = False,
Fabrizio Milo's avatar
Fabrizio Milo committed
80
):
81
    """Instantiate and evaluate a model on a list of tasks.
82

83
84
    :param model: Union[str, LM]
        Name of model or LM object, see lm_eval.models.get_model
85
86
    :param model_args: Optional[str, dict]
        String or dict arguments for each model class, see LM.create_from_arg_string and LM.create_from_arg_object.
87
        Ignored if `model` argument is a LM object.
88
    :param tasks: list[Union[str, dict, Task]]
Leo Gao's avatar
Leo Gao committed
89
        List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.
90
91
    :param num_fewshot: int
        Number of examples in few-shot context
92
    :param batch_size: int or str, optional
93
        Batch size for model
94
95
    :param max_batch_size: int, optional
        Maximal batch size to try with automatic batch size detection
96
    :param device: str, optional
97
        PyTorch device (e.g. "cpu" or "cuda:0") for running models
haileyschoelkopf's avatar
haileyschoelkopf committed
98
99
    :param use_cache: str, optional
        A path to a sqlite db file for caching model responses. `None` if not caching.
100
101
102
103
104
105
    :param cache_requests: bool, optional
        Speed up evaluation by caching the building of dataset requests. `None` if not caching.
    :param rewrite_requests_cache: bool, optional
        Rewrites all of the request cache if set to `True`. `None` if not desired.
    :param delete_requests_cache: bool, optional
        Deletes all of the request cache if set to `True`. `None` if not desired.
106
107
    :param limit: int or float, optional
        Limit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples.
108
    :param bootstrap_iters:
109
        Number of iterations for bootstrap statistics, used when calculating stderrs. set to 0 for no stderr calculations to be performed.
Stephen Hogg's avatar
Stephen Hogg committed
110
111
    :param check_integrity: bool
        Whether to run the relevant part of the test suite for the tasks
112
    :param write_out: bool
113
114
115
        If True, write out an example document and model input for checking task integrity
    :param log_samples: bool
        If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
KonradSzafer's avatar
KonradSzafer committed
116
117
    :param system_instruction: str
        System instruction to be applied to the prompt
118
119
120
121
122
    :param apply_chat_template: Union[bool, str]
        Specifies whether to apply a chat template to the prompt.
        - If set to True, the default chat template is applied.
        - If set to a string, applies the specified chat template by name.
        Defaults to False (no chat template applied).
KonradSzafer's avatar
KonradSzafer committed
123
124
    :param fewshot_as_multiturn: bool
        Whether to provide the fewshot examples as a multiturn conversation or a single user turn.
125
126
127
    :param gen_kwargs: str
        String arguments for model generation
        Ignored for all tasks with loglikelihood output_type
Lintang Sutawika's avatar
Lintang Sutawika committed
128
129
    :param verbostiy: str
        Verbosity level for logging
Baber Abbasi's avatar
Baber Abbasi committed
130
131
    :param predict_only: bool
        If true only model outputs will be generated and returned. Metrics will not be evaluated
132
133
134
135
136
137
    :param random_seed: int
        Random seed for python's random module. If set to None, the seed will not be set.
    :param numpy_random_seed: int
        Random seed for numpy. If set to None, the seed will not be set.
    :param torch_random_seed: int
        Random seed for torch. If set to None, the seed will not be set.
138
139
    :param fewshot_random_seed: int
        Random seed for fewshot sampler random generator. If set to None, the seed of generator will be set to None.
Baber Abbasi's avatar
Baber Abbasi committed
140

141
    :return
142
        Dictionary of results
143
    """
Lintang Sutawika's avatar
Lintang Sutawika committed
144
    if verbostiy is not None:
Baber Abbasi's avatar
Baber Abbasi committed
145
        setup_logging(verbosity=verbostiy)
146
    start_date = time.time()
147

Baber Abbasi's avatar
Baber Abbasi committed
148
149
150
151
152
153
154
    if isinstance(model_args, str) and (
        "instruct" in model_args and not apply_chat_template
    ):
        eval_logger.warning(
            "Instruct model detected, but chat template not applied. Recommend setting `apply_chat_template` (optionally `fewshot_as_multiturn`)."
        )

155
156
157
158
    if delete_requests_cache:
        eval_logger.info("Deleting requests cache...")
        delete_cache()

159
    seed_message = []
160
161
    if random_seed is not None:
        # See https://github.com/EleutherAI/lm-evaluation-harness/pull/1412
162
        seed_message.append(f"Setting random seed to {random_seed}")
163
164
165
        random.seed(random_seed)

    if numpy_random_seed is not None:
166
        seed_message.append(f"Setting numpy seed to {numpy_random_seed}")
167
168
169
        np.random.seed(numpy_random_seed)

    if torch_random_seed is not None:
170
        seed_message.append(f"Setting torch manual seed to {torch_random_seed}")
171
172
        torch.manual_seed(torch_random_seed)

173
174
175
    if fewshot_random_seed is not None:
        seed_message.append(f"Setting fewshot manual seed to {fewshot_random_seed}")

176
177
178
    if seed_message:
        eval_logger.info(" | ".join(seed_message))

179
180
    if tasks is None:
        tasks = []
181
182
183
184
    if len(tasks) == 0:
        raise ValueError(
            "No tasks specified, or no tasks found. Please verify the task names."
        )
185

lintangsutawika's avatar
lintangsutawika committed
186
187
    if gen_kwargs is not None:
        gen_kwargs = simple_parse_args_string(gen_kwargs)
lintangsutawika's avatar
udate  
lintangsutawika committed
188
        eval_logger.warning(
189
190
            "generation_kwargs specified through cli, these settings will update set parameters in yaml tasks. "
            "Ensure 'do_sample=True' for non-greedy decoding!"
lintangsutawika's avatar
udate  
lintangsutawika committed
191
        )
lintangsutawika's avatar
lintangsutawika committed
192
193
194
        if gen_kwargs == "":
            gen_kwargs = None

195
    if isinstance(model, str):
Fabrizio Milo's avatar
Fabrizio Milo committed
196
        if model_args is None:
197
            eval_logger.warning("model_args not specified. Using defaults.")
Fabrizio Milo's avatar
Fabrizio Milo committed
198
            model_args = ""
199

200
        if isinstance(model_args, dict):
201
202
203
            eval_logger.info(
                f"Initializing {model} model, with arguments: {model_args}"
            )
204
205
206
207
208
209
210
211
212
213
            lm = lm_eval.api.registry.get_model(model).create_from_arg_obj(
                model_args,
                {
                    "batch_size": batch_size,
                    "max_batch_size": max_batch_size,
                    "device": device,
                },
            )

        else:
214
215
216
            eval_logger.info(
                f"Initializing {model} model, with arguments: {simple_parse_args_string(model_args)}"
            )
217
218
219
220
221
222
223
224
            lm = lm_eval.api.registry.get_model(model).create_from_arg_string(
                model_args,
                {
                    "batch_size": batch_size,
                    "max_batch_size": max_batch_size,
                    "device": device,
                },
            )
225
    else:
226
        if not isinstance(model, lm_eval.api.model.LM):
227
228
229
            raise TypeError(
                f"The value of `model` passed to simple_evaluate() was of type {type(model)}, but is required to be a subclass of lm_eval.api.model.LM . This may be because you are passing an initialized Hugging Face PreTrainedModel without having wrapped it in `lm_eval.models.huggingface.HFLM(pretrained=my_model)` first."
            )
230
        eval_logger.info("Using pre-initialized model")
231
        lm = model
232

haileyschoelkopf's avatar
haileyschoelkopf committed
233
    if use_cache is not None:
234
        eval_logger.info(f"Using cache at {use_cache + '_rank' + str(lm.rank) + '.db'}")
haileyschoelkopf's avatar
haileyschoelkopf committed
235
236
237
238
239
        lm = lm_eval.api.model.CachingLM(
            lm,
            use_cache
            # each rank receives a different cache db.
            # necessary to avoid multiple writes to cache at once
240
241
242
            + "_rank"
            + str(lm.rank)
            + ".db",
haileyschoelkopf's avatar
haileyschoelkopf committed
243
244
        )

245
    if task_manager is None:
Lintang Sutawika's avatar
Lintang Sutawika committed
246
        task_manager = TaskManager()
247
248

    task_dict = get_task_dict(tasks, task_manager)
Baber Abbasi's avatar
Baber Abbasi committed
249

Lintang Sutawika's avatar
Lintang Sutawika committed
250
251
252
253
254
255
256
257
258
259
    # helper function to recursively apply config overrides to leaf subtasks, skipping their constituent groups.
    # (setting of num_fewshot ; bypassing metric calculation ; setting fewshot seed)
    def _adjust_config(task_dict):
        adjusted_task_dict = {}
        for task_name, task_obj in task_dict.items():
            if isinstance(task_obj, dict):
                adjusted_task_dict = {
                    **adjusted_task_dict,
                    **{task_name: _adjust_config(task_obj)},
                }
260

261
            else:
Lintang Sutawika's avatar
Lintang Sutawika committed
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
                if task_obj.get_config("output_type") == "generate_until":
                    if gen_kwargs is not None:
                        task_obj.set_config(
                            key="generation_kwargs", value=gen_kwargs, update=True
                        )

                if predict_only:
                    eval_logger.info(
                        f"Processing {task_name} in output-only mode. Metrics will not be calculated!"
                    )
                    # we have to change the class properties post-hoc. This is pretty hacky.
                    task_obj.override_metric(metric_name="bypass")

                # override tasks' fewshot values to the provided num_fewshot arg value
                # except if tasks have it set to 0 manually in their configs--then we should never overwrite that
                if num_fewshot is not None:
                    if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0:
                        eval_logger.info(
                            f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored."
                        )
                    else:
                        eval_logger.warning(
                            f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}"
                        )
                        task_obj.set_config(key="num_fewshot", value=num_fewshot)
                else:
                    # if num_fewshot not provided, and the task does not define a default one, default to 0
                    if (
                        default_num_fewshot := task_obj.get_config("num_fewshot")
                    ) is None:
                        task_obj.set_config(key="num_fewshot", value=0)
                # fewshot_random_seed set for tasks, even with a default num_fewshot (e.g. in the YAML file)
                task_obj.set_fewshot_seed(seed=fewshot_random_seed)

                adjusted_task_dict[task_name] = task_obj

        return adjusted_task_dict

    task_dict = _adjust_config(task_dict)
Jonathan Tow's avatar
Merge  
Jonathan Tow committed
301

Stephen Hogg's avatar
Stephen Hogg committed
302
    if check_integrity:
303
        run_task_tests(task_list=tasks)
Stephen Hogg's avatar
Stephen Hogg committed
304

KonradSzafer's avatar
KonradSzafer committed
305
306
307
308
309
    if evaluation_tracker is not None:
        evaluation_tracker.general_config_tracker.log_experiment_args(
            model_source=model,
            model_args=model_args,
            system_instruction=system_instruction,
Baber Abbasi's avatar
Baber Abbasi committed
310
311
312
            chat_template=lm.chat_template(apply_chat_template)
            if apply_chat_template
            else None,
313
            fewshot_as_multiturn=fewshot_as_multiturn,
KonradSzafer's avatar
KonradSzafer committed
314
315
        )

316
317
318
319
    results = evaluate(
        lm=lm,
        task_dict=task_dict,
        limit=limit,
320
321
        cache_requests=cache_requests,
        rewrite_requests_cache=rewrite_requests_cache,
Niklas Muennighoff's avatar
Niklas Muennighoff committed
322
        bootstrap_iters=bootstrap_iters,
323
        write_out=write_out,
Lintang Sutawika's avatar
Lintang Sutawika committed
324
        log_samples=True if predict_only else log_samples,
KonradSzafer's avatar
KonradSzafer committed
325
326
327
        system_instruction=system_instruction,
        apply_chat_template=apply_chat_template,
        fewshot_as_multiturn=fewshot_as_multiturn,
Lintang Sutawika's avatar
Lintang Sutawika committed
328
        verbosity=verbostiy,
Hojin Lee's avatar
Hojin Lee committed
329
        confirm_run_unsafe_code=confirm_run_unsafe_code,
330
    )
Lintang Sutawika's avatar
Lintang Sutawika committed
331
332
    if verbostiy is not None:
        lm_eval.setup_logging(verbosity=verbostiy)
333

334
    if lm.rank == 0:
335
336
337
338
339
340
341
        if isinstance(model, str):
            model_name = model
        elif hasattr(model, "config") and hasattr(model.config, "_name_or_path"):
            model_name = model.config._name_or_path
        else:
            model_name = type(model).__name__

342
343
        # add info about the model and few shot config
        results["config"] = {
344
            "model": model_name,
345
346
            "model_args": model_args,
        }
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
        # add more detailed model info if available
        if isinstance(lm, lm_eval.models.huggingface.HFLM):
            results["config"].update(lm.get_model_info())
        # add info about execution
        results["config"].update(
            {
                "batch_size": batch_size,
                "batch_sizes": (
                    list(lm.batch_sizes.values()) if hasattr(lm, "batch_sizes") else []
                ),
                "device": device,
                "use_cache": use_cache,
                "limit": limit,
                "bootstrap_iters": bootstrap_iters,
                "gen_kwargs": gen_kwargs,
362
363
364
365
                "random_seed": random_seed,
                "numpy_seed": numpy_random_seed,
                "torch_seed": torch_random_seed,
                "fewshot_seed": fewshot_random_seed,
366
367
            }
        )
368
        results["git_hash"] = get_git_commit_hash()
369
        results["date"] = start_date
370
        add_env_info(results)  # additional environment info to results
achervyakov's avatar
achervyakov committed
371
        add_tokenizer_info(results, lm)  # additional info about tokenizer
372
373
374
        return results
    else:
        return None
375

Leo Gao's avatar
Leo Gao committed
376

377
@positional_deprecated
Fabrizio Milo's avatar
Fabrizio Milo committed
378
def evaluate(
379
    lm: "LM",
Fabrizio Milo's avatar
Fabrizio Milo committed
380
    task_dict,
Baber Abbasi's avatar
Baber Abbasi committed
381
    limit: Optional[int] = None,
382
383
    cache_requests: bool = False,
    rewrite_requests_cache: bool = False,
Baber Abbasi's avatar
Baber Abbasi committed
384
    bootstrap_iters: Optional[int] = 100000,
Ethan Smith's avatar
Ethan Smith committed
385
386
    write_out: bool = False,
    log_samples: bool = True,
KonradSzafer's avatar
KonradSzafer committed
387
    system_instruction: Optional[str] = None,
388
    apply_chat_template: Union[bool, str] = False,
KonradSzafer's avatar
KonradSzafer committed
389
    fewshot_as_multiturn: bool = False,
390
    verbosity: str = "INFO",
Hojin Lee's avatar
Hojin Lee committed
391
    confirm_run_unsafe_code: bool = False,
Fabrizio Milo's avatar
Fabrizio Milo committed
392
):
393
394
395
396
397
    """Instantiate and evaluate a model on a list of tasks.

    :param lm: obj
        Language Model
    :param task_dict: dict[str, Task]
haileyschoelkopf's avatar
haileyschoelkopf committed
398
        Dictionary of tasks. Tasks will be taken to have name type(task).config.task .
399
400
    :param limit: int, optional
        Limit the number of examples per task (only use this for testing)
Hojin Lee's avatar
Hojin Lee committed
401
402
403
404
    :param cache_requests: bool, optional
        Speed up evaluation by caching the building of dataset requests.
    :param rewrite_requests_cache: bool, optional
        Rewrites all the request cache if set to `True`.
405
    :param bootstrap_iters:
406
        Number of iterations for bootstrap statistics, used when calculating stderr. Set to 0 for skipping all stderr calculations.
407
    :param write_out: bool
408
409
410
        If True, write out an example document and model input for checking task integrity
    :param log_samples: bool
        If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
KonradSzafer's avatar
KonradSzafer committed
411
412
    :param system_instruction: str
        System instruction to be applied to the prompt
413
414
415
416
417
    :param apply_chat_template: Union[bool, str]
        Specifies whether to apply a chat template to the prompt.
        - If set to True, the default chat template is applied.
        - If set to a string, applies the specified chat template by name.
        Defaults to False (no chat template applied).
KonradSzafer's avatar
KonradSzafer committed
418
419
    :param fewshot_as_multiturn: bool
        Whether to provide the fewshot examples as a multiturn conversation or a single user turn.
Hojin Lee's avatar
Hojin Lee committed
420
421
422
423
    :param verbosity: str
        Verbosity level for logging
    :param confirm_run_unsafe_code: bool
        Whether to confirm running tasks marked as unsafe.
424
425
426
    :return
        Dictionary of results
    """
427

428
429
430
431
432
    if apply_chat_template:
        eval_logger.warning(
            "Chat template formatting change affects loglikelihood and multiple-choice tasks. See docs/chat-template-readme.md for details."
        )

433
    # tracks all Instances/requests a model must generate output on.
434
    requests = defaultdict(list)
435
436
    # stores the amount to pad out reqs per req. type so that
    # number of fwd passes per distributed rank is equal
437
    padding_requests = defaultdict(int)
438

439
    # get lists of group hierarchy and each type of request
Lintang Sutawika's avatar
Lintang Sutawika committed
440
    eval_tasks = get_task_list(task_dict)
441
    if not log_samples:
442
        if not all(
443
444
            "bypass" not in getattr(task_output.task, "_metric_fn_list", {}).keys()
            for task_output in eval_tasks
445
446
        ):
            raise ValueError("log_samples must be True for 'bypass' metric-only tasks")
447

Hojin Lee's avatar
Hojin Lee committed
448
449
450
    # validation checks:
    # 1.are we running multimodal task <-> non-multimodal model class, or vice-versa.
    # 2.are we running code that is marked as unsafe.
451
    incompatible_tasks = []
452
453
    for task_output in eval_tasks:
        task: Task = task_output.task
454
455
456

        if getattr(lm, "MULTIMODAL", False) != getattr(task, "MULTIMODAL", False):
            incompatible_tasks.append(task_output.task_name)
Hojin Lee's avatar
Hojin Lee committed
457
458
459
460
        elif getattr(task, "UNSAFE_CODE", False) and not confirm_run_unsafe_code:
            raise ValueError(
                f"Attempted to run task: {task_output.task_name} which is marked as unsafe. Set confirm_run_unsafe_code=True to run this task."
            )
461
462
463
464
465
466
467
468
469
    if len(incompatible_tasks) > 0:
        if not getattr(lm, "MULTIMODAL", False):
            raise ValueError(
                f"Attempted to run tasks: {incompatible_tasks} which require multimodal input, but the selected model type does not currently implement this. Multimodal support is currently restricted to the ['hf-multimodal', 'vllm-vlm'] model type."
            )
        else:
            raise ValueError(
                f"Attempted to run tasks: {incompatible_tasks} which are text-only, but used a model type which only currently supports multimodal tasks."
            )
Hojin Lee's avatar
Hojin Lee committed
470
    # end validation check
471

Chenjie Luo's avatar
Chenjie Luo committed
472
473
474
    # Cache the limit arg.
    limit_arg = limit
    limits = []
475
476
477
    for task_output in eval_tasks:
        task: Task = task_output.task

Chenjie Luo's avatar
Chenjie Luo committed
478
479
        limit = get_sample_size(task, limit_arg)
        limits.append(limit)
480
481
482
483
484
485
        task.build_all_requests(
            limit=limit,
            rank=lm.rank,
            world_size=lm.world_size,
            cache_requests=cache_requests,
            rewrite_requests_cache=rewrite_requests_cache,
KonradSzafer's avatar
KonradSzafer committed
486
            system_instruction=system_instruction,
487
            apply_chat_template=bool(apply_chat_template),
KonradSzafer's avatar
KonradSzafer committed
488
            fewshot_as_multiturn=fewshot_as_multiturn,
489
490
491
492
493
494
            chat_template=getattr(lm, "apply_chat_template")
            if apply_chat_template
            else None,
            tokenizer_name=getattr(lm, "tokenizer_name", "")
            if apply_chat_template
            else "",
495
        )
496
        eval_logger.debug(
497
            f"Task: {task_output.task_name}; number of requests on this rank: {len(task.instances)}"
haileyschoelkopf's avatar
haileyschoelkopf committed
498
499
        )
        if write_out:
500
            print_writeout(task)
501
        # aggregate Instances by LM method requested to get output.
lintangsutawika's avatar
lintangsutawika committed
502
503
504
        for instance in task.instances:
            reqtype = instance.request_type
            requests[reqtype].append(instance)
505
506

        if lm.world_size > 1:
507
508
509
510
            instances_rnk = torch.tensor(len(task._instances), device=lm.device)
            gathered_item = (
                lm.accelerator.gather(instances_rnk).cpu().detach().numpy().tolist()
            )
511
512
513
514
515
516
            # "multiple_choice" task types dispatch (several) "loglikelihood" request types
            reqtype = (
                "loglikelihood"
                if task.OUTPUT_TYPE == "multiple_choice"
                else task.OUTPUT_TYPE
            )
517
            # compute number of pseudo-batches to pad with (FSDP/DDP require even batches among ranks)
518
            numpad = max(gathered_item) - gathered_item[lm.rank]
519
520
            # todo: may not account for padding in cases like SquadV2 which has multiple req types
            padding_requests[reqtype] += numpad
521

522
    ### Run LM on inputs, get all outputs ###
Leo Gao's avatar
Leo Gao committed
523
524
    # execute each type of request
    for reqtype, reqs in requests.items():
525
        eval_logger.info(f"Running {reqtype} requests")
526
527
528
529
        # create `K` copies of each request `req` based off `K = req.repeats`
        cloned_reqs = []
        for req in reqs:
            cloned_reqs.extend([req] * req.repeats)
lintangsutawika's avatar
lintangsutawika committed
530

531
532
        if (lm.world_size > 1) and (padding_requests[reqtype] > 0):
            for _ in range(padding_requests[reqtype]):
533
534
                cloned_reqs.extend([req] * req.repeats)

535
536
537
538
539
540
541
        # run requests through model
        resps = getattr(lm, reqtype)(cloned_reqs)

        # put responses from model into a list of length K for each request.
        for x, req in zip(resps, cloned_reqs):
            req.resps.append(x)

542
543
        if lm.world_size > 1:
            lm.accelerator.wait_for_everyone()
544

545
546
    RANK = lm.rank
    WORLD_SIZE = lm.world_size
547
548
    ### Postprocess outputs ###
    # TODO: del model here, maybe (idea: allow user to specify device of e.g. reward model separately)
Chenjie Luo's avatar
Chenjie Luo committed
549
    for task_output, limit in zip(eval_tasks, limits):
550
        task = task_output.task
551
552
        task.apply_filters()

553
554
        ### Collect values of metrics on all datapoints ###
        # # unpack results and sort back in order and return control to Task
haileyschoelkopf's avatar
haileyschoelkopf committed
555
        # TODO: make it possible to use a different metric per filter
556
        # Pre-process task.instances to group by doc_id
557
        instances_by_doc_id = defaultdict(list)
558
559
560
561
562
        for instance in task.instances:
            instances_by_doc_id[instance.doc_id].append(instance)
        # Sort instances within each group
        for instances in instances_by_doc_id.values():
            instances.sort(key=lambda x: x.idx)
haileyschoelkopf's avatar
haileyschoelkopf committed
563
        # iterate over different filters used
564
565
566
        for filter_key in task.instances[0].filtered_resps.keys():
            doc_iterator = task.doc_iterator(
                rank=RANK, limit=limit, world_size=WORLD_SIZE
567
            )
568
            for doc_id, doc in doc_iterator:
569
                requests = instances_by_doc_id[doc_id]
lintangsutawika's avatar
lintangsutawika committed
570
                metrics = task.process_results(
571
                    doc, [req.filtered_resps[filter_key] for req in requests]
lintangsutawika's avatar
lintangsutawika committed
572
                )
573
574
575
576
577
578
579
580
                if log_samples:
                    target = task.doc_to_target(doc)
                    example = {
                        "doc_id": doc_id,
                        "doc": doc,
                        "target": target,
                        "arguments": [req.args for req in requests],
                        "resps": [req.resps for req in requests],
581
582
583
                        "filtered_resps": [
                            req.filtered_resps[filter_key] for req in requests
                        ],
584
585
                        "filter": filter_key,
                        "metrics": list(metrics.keys()),
586
587
588
589
590
591
592
593
594
595
                        "doc_hash": hash_string(
                            json.dumps(
                                requests[0].doc,
                                indent=2,
                                default=handle_non_serializable,
                                ensure_ascii=False,
                            )
                        ),
                        "prompt_hash": hash_string(requests[0].arguments[0]),
                        "target_hash": hash_string(str(target)),
596
597
                    }
                    example.update(metrics)
598
                    task_output.logged_samples.append(example)
599
                for metric, value in metrics.items():
600
                    task_output.sample_metrics[(metric, filter_key)].append(value)
601

602
603
    if WORLD_SIZE > 1:
        # if multigpu, then gather data across all ranks to rank 0
604
        # first gather logged samples across all ranks
605
606
607
608
609
610
611
612
        for task_output in eval_tasks:
            if log_samples:
                # for task_name, task_samples in list(samples.items()):
                full_samples = [None] * WORLD_SIZE if RANK == 0 else None
                torch.distributed.gather_object(
                    obj=task_output.logged_samples,
                    object_gather_list=full_samples,
                    dst=0,
613
                )
614

615
616
617
618
                if RANK == 0:
                    task_output.logged_samples = list(
                        itertools.chain.from_iterable(full_samples)
                    )
619

620
621
622
623
624
625
626
            # then collect metrics across all ranks
            for metrics in task_output.sample_metrics:
                metric_list = [None] * WORLD_SIZE if RANK == 0 else None
                torch.distributed.gather_object(
                    obj=task_output.sample_metrics[metrics],
                    object_gather_list=metric_list,
                    dst=0,
627
                )
628
629
630
631
                if RANK == 0:
                    task_output.sample_metrics[metrics] = list(
                        itertools.chain.from_iterable(metric_list)
                    )
632

633
    if RANK == 0:
634
635
        ### Aggregate results over all datapoints ###
        # aggregate results ; run bootstrap CIs
636
637
        for task_output in eval_tasks:
            task_output.calculate_aggregate_metric(bootstrap_iters=bootstrap_iters)
638
639
640
641
642
643
644
645
        (
            results,
            samples,
            configs,
            versions,
            num_fewshot,
            higher_is_better,
        ) = consolidate_results(eval_tasks)
Fabrizio Milo's avatar
Fabrizio Milo committed
646

647
        ### Calculate group metrics ###
lintangsutawika's avatar
lintangsutawika committed
648
        if bool(results):
Lintang Sutawika's avatar
Lintang Sutawika committed
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
            results, versions, show_group_table, *_ = consolidate_group_results(
                results, versions, task_dict
            )

        results_agg, group_agg = prepare_print_tasks(task_dict, results)
        subtask_list = get_subtask_list(task_dict)

        # collect all higher_is_better values for metrics
        # in the group's subtasks.
        # TODO: clean this up ; unify with the below metric_list loop?
        _higher_is_better = {}
        for group, task_list in subtask_list.items():
            if (
                len(task_list) != 0
            ):  # subtask list will list "task_name": [] for solo tasks
664
665
666
667
                for task in task_list:
                    for m, h in higher_is_better[task].items():
                        if m not in _higher_is_better.keys():
                            _higher_is_better[m] = h
lintangsutawika's avatar
lintangsutawika committed
668

Lintang Sutawika's avatar
Lintang Sutawika committed
669
670
671
672
673
674
675
676
677
678
                        if (
                            m in _higher_is_better
                            and _higher_is_better[m] is not None
                            and _higher_is_better[m] != h
                        ):
                            eval_logger.warning(
                                f"Higher_is_better values for metric {m} in group {group} are not consistent. Defaulting to None."
                            )
                            _higher_is_better[m] = None
                higher_is_better[group] = _higher_is_better
679

680
        results_dict = {
681
            "results": dict(results_agg.items()),
Lintang Sutawika's avatar
Lintang Sutawika committed
682
683
684
685
686
687
            **(
                {"groups": dict(group_agg.items())}
                if (bool(group_agg) & show_group_table)
                else {}
            ),
            "group_subtasks": dict(reversed(subtask_list.items())),
688
689
            "configs": dict(sorted(configs.items())),
            "versions": dict(sorted(versions.items())),
690
            "n-shot": dict(sorted(num_fewshot.items())),
691
            "higher_is_better": dict(sorted(higher_is_better.items())),
692
693
694
            "n-samples": {
                task_output.task_name: {
                    "original": len(task_output.task.eval_docs),
KonradSzafer's avatar
KonradSzafer committed
695
696
697
698
                    "effective": min(
                        limit if limit else len(task_output.task.eval_docs),
                        len(task_output.task.eval_docs),
                    ),
699
                }
Chenjie Luo's avatar
Chenjie Luo committed
700
                for task_output, limit in zip(eval_tasks, limits)
701
            },
702
        }
703
704
705
706
        if log_samples:
            results_dict["samples"] = dict(samples)

        return results_dict
Fabrizio Milo's avatar
Fabrizio Milo committed
707

708
709
    else:
        return None
710
711
712
713


def request_caching_arg_to_dict(cache_requests: str) -> dict:
    request_caching_args = {
714
715
716
        "cache_requests": cache_requests in {"true", "refresh"},
        "rewrite_requests_cache": cache_requests == "refresh",
        "delete_requests_cache": cache_requests == "delete",
717
718
719
    }

    return request_caching_args