evaluator.py 30.8 KB
Newer Older
Baber Abbasi's avatar
Baber Abbasi committed
1
import itertools
2
import json
3
import logging
Baber Abbasi's avatar
Baber Abbasi committed
4
import random
5
import time
6
7
from collections import defaultdict
from typing import TYPE_CHECKING, List, Optional, Union
Baber Abbasi's avatar
Baber Abbasi committed
8

9
import numpy as np
Baber Abbasi's avatar
Baber Abbasi committed
10
import torch
lintangsutawika's avatar
lintangsutawika committed
11

lintangsutawika's avatar
lintangsutawika committed
12
import lm_eval.api.metrics
lintangsutawika's avatar
lintangsutawika committed
13
import lm_eval.api.registry
Lintang Sutawika's avatar
Lintang Sutawika committed
14
import lm_eval.api.task
Baber Abbasi's avatar
Baber Abbasi committed
15
import lm_eval.models
16
from lm_eval.caching.cache import delete_cache
17
from lm_eval.evaluator_utils import (
Lintang Sutawika's avatar
Lintang Sutawika committed
18
    consolidate_group_results,
19
20
    consolidate_results,
    get_sample_size,
Lintang Sutawika's avatar
Lintang Sutawika committed
21
    get_subtask_list,
22
23
24
25
26
    get_task_list,
    prepare_print_tasks,
    print_writeout,
    run_task_tests,
)
KonradSzafer's avatar
KonradSzafer committed
27
from lm_eval.loggers import EvaluationTracker
28
from lm_eval.loggers.utils import add_env_info, add_tokenizer_info, get_git_commit_hash
29
from lm_eval.tasks import TaskManager, get_task_dict
30
31
32
33
from lm_eval.utils import (
    handle_non_serializable,
    hash_string,
    positional_deprecated,
Baber Abbasi's avatar
Baber Abbasi committed
34
    setup_logging,
35
36
    simple_parse_args_string,
)
37

Fabrizio Milo's avatar
Fabrizio Milo committed
38

39
40
if TYPE_CHECKING:
    from lm_eval.api.model import LM
Lintang Sutawika's avatar
Lintang Sutawika committed
41
    from lm_eval.api.task import Task
42

Lintang Sutawika's avatar
Lintang Sutawika committed
43
44
eval_logger = logging.getLogger(__name__)

45

46
@positional_deprecated
Fabrizio Milo's avatar
Fabrizio Milo committed
47
48
def simple_evaluate(
    model,
49
50
    model_args: Optional[Union[str, dict]] = None,
    tasks: Optional[List[Union[str, dict, object]]] = None,
Baber Abbasi's avatar
Baber Abbasi committed
51
    num_fewshot: Optional[int] = None,
52
    batch_size: Optional[Union[int, str]] = None,
Baber Abbasi's avatar
Baber Abbasi committed
53
54
55
    max_batch_size: Optional[int] = None,
    device: Optional[str] = None,
    use_cache: Optional[str] = None,
56
57
58
    cache_requests: bool = False,
    rewrite_requests_cache: bool = False,
    delete_requests_cache: bool = False,
Baber Abbasi's avatar
Baber Abbasi committed
59
    limit: Optional[Union[int, float]] = None,
60
    samples: Optional[dict] = None,
Ethan Smith's avatar
Ethan Smith committed
61
62
63
64
    bootstrap_iters: int = 100000,
    check_integrity: bool = False,
    write_out: bool = False,
    log_samples: bool = True,
KonradSzafer's avatar
KonradSzafer committed
65
66
    evaluation_tracker: Optional[EvaluationTracker] = None,
    system_instruction: Optional[str] = None,
67
    apply_chat_template: Union[bool, str] = False,
KonradSzafer's avatar
KonradSzafer committed
68
    fewshot_as_multiturn: bool = False,
Baber Abbasi's avatar
Baber Abbasi committed
69
    gen_kwargs: Union[str, dict, None] = None,
70
    task_manager: Optional[TaskManager] = None,
Baber Abbasi's avatar
Baber Abbasi committed
71
    verbosity=None,
Baber Abbasi's avatar
Baber Abbasi committed
72
    predict_only: bool = False,
73
74
75
    random_seed: int = 0,
    numpy_random_seed: int = 1234,
    torch_random_seed: int = 1234,
76
    fewshot_random_seed: int = 1234,
Hojin Lee's avatar
Hojin Lee committed
77
    confirm_run_unsafe_code: bool = False,
Baber Abbasi's avatar
Baber Abbasi committed
78
    metadata: Optional[dict] = None,
Fabrizio Milo's avatar
Fabrizio Milo committed
79
):
80
    """Instantiate and evaluate a model on a list of tasks.
81

82
83
    :param model: Union[str, LM]
        Name of model or LM object, see lm_eval.models.get_model
84
85
    :param model_args: Optional[str, dict]
        String or dict arguments for each model class, see LM.create_from_arg_string and LM.create_from_arg_object.
86
        Ignored if `model` argument is a LM object.
87
    :param tasks: list[Union[str, dict, Task]]
Leo Gao's avatar
Leo Gao committed
88
        List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.
89
90
    :param num_fewshot: int
        Number of examples in few-shot context
91
    :param batch_size: int or str, optional
92
        Batch size for model
93
94
    :param max_batch_size: int, optional
        Maximal batch size to try with automatic batch size detection
95
    :param device: str, optional
96
        PyTorch device (e.g. "cpu" or "cuda:0") for running models
haileyschoelkopf's avatar
haileyschoelkopf committed
97
98
    :param use_cache: str, optional
        A path to a sqlite db file for caching model responses. `None` if not caching.
99
100
101
    :param cache_requests: bool, optional
        Speed up evaluation by caching the building of dataset requests. `None` if not caching.
    :param rewrite_requests_cache: bool, optional
Baber Abbasi's avatar
Baber Abbasi committed
102
        Rewrites all the request cache if set to `True`. `None` if not desired.
103
    :param delete_requests_cache: bool, optional
Baber Abbasi's avatar
Baber Abbasi committed
104
        Deletes all the request cache if set to `True`. `None` if not desired.
105
106
    :param limit: int or float, optional
        Limit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples.
107
108
    :param samples: dictionary, optional
        Dictionary indicating which examples should be tested in each task, e.g., {"mmlu_astronomy":[0,3,6],"mmlu_anatomy":[1,4,7,10]}.
109
    :param bootstrap_iters:
110
        Number of iterations for bootstrap statistics, used when calculating stderrs. set to 0 for no stderr calculations to be performed.
Stephen Hogg's avatar
Stephen Hogg committed
111
112
    :param check_integrity: bool
        Whether to run the relevant part of the test suite for the tasks
113
    :param write_out: bool
114
115
116
        If True, write out an example document and model input for checking task integrity
    :param log_samples: bool
        If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
KonradSzafer's avatar
KonradSzafer committed
117
118
    :param system_instruction: str
        System instruction to be applied to the prompt
119
120
121
122
123
    :param apply_chat_template: Union[bool, str]
        Specifies whether to apply a chat template to the prompt.
        - If set to True, the default chat template is applied.
        - If set to a string, applies the specified chat template by name.
        Defaults to False (no chat template applied).
KonradSzafer's avatar
KonradSzafer committed
124
125
    :param fewshot_as_multiturn: bool
        Whether to provide the fewshot examples as a multiturn conversation or a single user turn.
Baber Abbasi's avatar
Baber Abbasi committed
126
127
    :param gen_kwargs: dict or comma-separated string
        Arguments for model generation
128
        Ignored for all tasks with loglikelihood output_type
Baber Abbasi's avatar
Baber Abbasi committed
129
    :param verbosity: str
Lintang Sutawika's avatar
Lintang Sutawika committed
130
        Verbosity level for logging
Baber Abbasi's avatar
Baber Abbasi committed
131
132
    :param predict_only: bool
        If true only model outputs will be generated and returned. Metrics will not be evaluated
133
134
135
136
137
138
    :param random_seed: int
        Random seed for python's random module. If set to None, the seed will not be set.
    :param numpy_random_seed: int
        Random seed for numpy. If set to None, the seed will not be set.
    :param torch_random_seed: int
        Random seed for torch. If set to None, the seed will not be set.
139
140
    :param fewshot_random_seed: int
        Random seed for fewshot sampler random generator. If set to None, the seed of generator will be set to None.
Baber Abbasi's avatar
Baber Abbasi committed
141
142
    :param metadata: dict
        Additional metadata to be added to the task manager. Will get passed to the download function of the task.
Baber Abbasi's avatar
Baber Abbasi committed
143

Baber Abbasi's avatar
Baber Abbasi committed
144
    return
145
        Dictionary of results
146
    """
Baber Abbasi's avatar
Baber Abbasi committed
147
148
    if verbosity is not None:
        setup_logging(verbosity=verbosity)
149
    start_date = time.time()
150

151
152
153
154
155
    if limit is not None and samples is not None:
        raise ValueError(
            "Either 'limit' or 'samples' must be None, but both are not None."
        )

Baber Abbasi's avatar
Baber Abbasi committed
156
157
158
159
160
161
162
    if isinstance(model_args, str) and (
        "instruct" in model_args and not apply_chat_template
    ):
        eval_logger.warning(
            "Instruct model detected, but chat template not applied. Recommend setting `apply_chat_template` (optionally `fewshot_as_multiturn`)."
        )

163
164
165
166
    if delete_requests_cache:
        eval_logger.info("Deleting requests cache...")
        delete_cache()

167
    seed_message = []
168
169
    if random_seed is not None:
        # See https://github.com/EleutherAI/lm-evaluation-harness/pull/1412
170
        seed_message.append(f"Setting random seed to {random_seed}")
171
172
173
        random.seed(random_seed)

    if numpy_random_seed is not None:
174
        seed_message.append(f"Setting numpy seed to {numpy_random_seed}")
175
176
177
        np.random.seed(numpy_random_seed)

    if torch_random_seed is not None:
178
        seed_message.append(f"Setting torch manual seed to {torch_random_seed}")
179
180
        torch.manual_seed(torch_random_seed)

181
182
183
    if fewshot_random_seed is not None:
        seed_message.append(f"Setting fewshot manual seed to {fewshot_random_seed}")

184
185
186
    if seed_message:
        eval_logger.info(" | ".join(seed_message))

187
188
    if tasks is None:
        tasks = []
189
190
191
192
    if len(tasks) == 0:
        raise ValueError(
            "No tasks specified, or no tasks found. Please verify the task names."
        )
193

lintangsutawika's avatar
lintangsutawika committed
194
    if gen_kwargs is not None:
Baber Abbasi's avatar
Baber Abbasi committed
195
196
        if isinstance(gen_kwargs, str):
            gen_kwargs = simple_parse_args_string(gen_kwargs)
lintangsutawika's avatar
udate  
lintangsutawika committed
197
        eval_logger.warning(
Baber Abbasi's avatar
Baber Abbasi committed
198
            f"generation_kwargs: {gen_kwargs} specified through cli, these settings will update set parameters in yaml tasks. "
199
            "Ensure 'do_sample=True' for non-greedy decoding!"
lintangsutawika's avatar
udate  
lintangsutawika committed
200
        )
Baber Abbasi's avatar
Baber Abbasi committed
201
        if not gen_kwargs:
lintangsutawika's avatar
lintangsutawika committed
202
203
            gen_kwargs = None

204
    if isinstance(model, str):
Fabrizio Milo's avatar
Fabrizio Milo committed
205
        if model_args is None:
206
            eval_logger.warning("model_args not specified. Using defaults.")
Fabrizio Milo's avatar
Fabrizio Milo committed
207
            model_args = ""
208

209
        if isinstance(model_args, dict):
210
211
212
            eval_logger.info(
                f"Initializing {model} model, with arguments: {model_args}"
            )
213
214
215
216
217
218
219
220
221
222
            lm = lm_eval.api.registry.get_model(model).create_from_arg_obj(
                model_args,
                {
                    "batch_size": batch_size,
                    "max_batch_size": max_batch_size,
                    "device": device,
                },
            )

        else:
223
224
225
            eval_logger.info(
                f"Initializing {model} model, with arguments: {simple_parse_args_string(model_args)}"
            )
226
227
228
229
230
231
232
233
            lm = lm_eval.api.registry.get_model(model).create_from_arg_string(
                model_args,
                {
                    "batch_size": batch_size,
                    "max_batch_size": max_batch_size,
                    "device": device,
                },
            )
234
    else:
235
        if not isinstance(model, lm_eval.api.model.LM):
236
237
238
            raise TypeError(
                f"The value of `model` passed to simple_evaluate() was of type {type(model)}, but is required to be a subclass of lm_eval.api.model.LM . This may be because you are passing an initialized Hugging Face PreTrainedModel without having wrapped it in `lm_eval.models.huggingface.HFLM(pretrained=my_model)` first."
            )
239
        eval_logger.info("Using pre-initialized model")
240
        lm = model
241

haileyschoelkopf's avatar
haileyschoelkopf committed
242
    if use_cache is not None:
243
        eval_logger.info(f"Using cache at {use_cache + '_rank' + str(lm.rank) + '.db'}")
244
        lm = lm_eval.api.model.CachingLM(lm, use_cache)
haileyschoelkopf's avatar
haileyschoelkopf committed
245

246
    if task_manager is None:
Baber Abbasi's avatar
Baber Abbasi committed
247
248
249
250
251
252
253
254
        metadata = (
            simple_parse_args_string(model_args)
            if isinstance(model_args, str)
            else model_args
            if isinstance(model_args, dict)
            else {}
        ) | (metadata or {})
        task_manager = TaskManager(metadata=metadata)
255

Baber Abbasi's avatar
Baber Abbasi committed
256
257
258
259
    task_dict = get_task_dict(
        tasks,
        task_manager,
    )
Baber Abbasi's avatar
Baber Abbasi committed
260

Lintang Sutawika's avatar
Lintang Sutawika committed
261
262
263
264
265
266
267
268
269
270
    # helper function to recursively apply config overrides to leaf subtasks, skipping their constituent groups.
    # (setting of num_fewshot ; bypassing metric calculation ; setting fewshot seed)
    def _adjust_config(task_dict):
        adjusted_task_dict = {}
        for task_name, task_obj in task_dict.items():
            if isinstance(task_obj, dict):
                adjusted_task_dict = {
                    **adjusted_task_dict,
                    **{task_name: _adjust_config(task_obj)},
                }
271

272
            else:
Lintang Sutawika's avatar
Lintang Sutawika committed
273
274
275
276
277
                if task_obj.get_config("output_type") == "generate_until":
                    if gen_kwargs is not None:
                        task_obj.set_config(
                            key="generation_kwargs", value=gen_kwargs, update=True
                        )
Baber Abbasi's avatar
Baber Abbasi committed
278
279
280
                    eval_logger.info(
                        f"{task_obj.config.task}: Using gen_kwargs: {task_obj.config.generation_kwargs}"
                    )
Lintang Sutawika's avatar
Lintang Sutawika committed
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314

                if predict_only:
                    eval_logger.info(
                        f"Processing {task_name} in output-only mode. Metrics will not be calculated!"
                    )
                    # we have to change the class properties post-hoc. This is pretty hacky.
                    task_obj.override_metric(metric_name="bypass")

                # override tasks' fewshot values to the provided num_fewshot arg value
                # except if tasks have it set to 0 manually in their configs--then we should never overwrite that
                if num_fewshot is not None:
                    if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0:
                        eval_logger.info(
                            f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored."
                        )
                    else:
                        eval_logger.warning(
                            f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}"
                        )
                        task_obj.set_config(key="num_fewshot", value=num_fewshot)
                else:
                    # if num_fewshot not provided, and the task does not define a default one, default to 0
                    if (
                        default_num_fewshot := task_obj.get_config("num_fewshot")
                    ) is None:
                        task_obj.set_config(key="num_fewshot", value=0)
                # fewshot_random_seed set for tasks, even with a default num_fewshot (e.g. in the YAML file)
                task_obj.set_fewshot_seed(seed=fewshot_random_seed)

                adjusted_task_dict[task_name] = task_obj

        return adjusted_task_dict

    task_dict = _adjust_config(task_dict)
Jonathan Tow's avatar
Merge  
Jonathan Tow committed
315

Stephen Hogg's avatar
Stephen Hogg committed
316
    if check_integrity:
317
        run_task_tests(task_list=tasks)
Stephen Hogg's avatar
Stephen Hogg committed
318

KonradSzafer's avatar
KonradSzafer committed
319
320
321
322
323
    if evaluation_tracker is not None:
        evaluation_tracker.general_config_tracker.log_experiment_args(
            model_source=model,
            model_args=model_args,
            system_instruction=system_instruction,
Baber Abbasi's avatar
Baber Abbasi committed
324
325
326
            chat_template=lm.chat_template(apply_chat_template)
            if apply_chat_template
            else None,
327
            fewshot_as_multiturn=fewshot_as_multiturn,
KonradSzafer's avatar
KonradSzafer committed
328
329
        )

330
331
332
333
    results = evaluate(
        lm=lm,
        task_dict=task_dict,
        limit=limit,
334
        samples=samples,
335
336
        cache_requests=cache_requests,
        rewrite_requests_cache=rewrite_requests_cache,
Niklas Muennighoff's avatar
Niklas Muennighoff committed
337
        bootstrap_iters=bootstrap_iters,
338
        write_out=write_out,
Lintang Sutawika's avatar
Lintang Sutawika committed
339
        log_samples=True if predict_only else log_samples,
KonradSzafer's avatar
KonradSzafer committed
340
341
342
        system_instruction=system_instruction,
        apply_chat_template=apply_chat_template,
        fewshot_as_multiturn=fewshot_as_multiturn,
Baber Abbasi's avatar
Baber Abbasi committed
343
        verbosity=verbosity,
Hojin Lee's avatar
Hojin Lee committed
344
        confirm_run_unsafe_code=confirm_run_unsafe_code,
345
    )
Baber Abbasi's avatar
Baber Abbasi committed
346
    if verbosity is not None:
Zeyuan Allen-Zhu's avatar
Zeyuan Allen-Zhu committed
347
        setup_logging(verbosity=verbosity)
348

349
    if lm.rank == 0:
350
351
352
353
354
355
356
        if isinstance(model, str):
            model_name = model
        elif hasattr(model, "config") and hasattr(model.config, "_name_or_path"):
            model_name = model.config._name_or_path
        else:
            model_name = type(model).__name__

357
358
        # add info about the model and few shot config
        results["config"] = {
359
            "model": model_name,
360
361
            "model_args": model_args,
        }
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
        # add more detailed model info if available
        if isinstance(lm, lm_eval.models.huggingface.HFLM):
            results["config"].update(lm.get_model_info())
        # add info about execution
        results["config"].update(
            {
                "batch_size": batch_size,
                "batch_sizes": (
                    list(lm.batch_sizes.values()) if hasattr(lm, "batch_sizes") else []
                ),
                "device": device,
                "use_cache": use_cache,
                "limit": limit,
                "bootstrap_iters": bootstrap_iters,
                "gen_kwargs": gen_kwargs,
377
378
379
380
                "random_seed": random_seed,
                "numpy_seed": numpy_random_seed,
                "torch_seed": torch_random_seed,
                "fewshot_seed": fewshot_random_seed,
381
382
            }
        )
383
        results["git_hash"] = get_git_commit_hash()
384
        results["date"] = start_date
385
        add_env_info(results)  # additional environment info to results
achervyakov's avatar
achervyakov committed
386
        add_tokenizer_info(results, lm)  # additional info about tokenizer
387
388
389
        return results
    else:
        return None
390

Leo Gao's avatar
Leo Gao committed
391

392
@positional_deprecated
Fabrizio Milo's avatar
Fabrizio Milo committed
393
def evaluate(
394
    lm: "LM",
Fabrizio Milo's avatar
Fabrizio Milo committed
395
    task_dict,
Baber Abbasi's avatar
Baber Abbasi committed
396
    limit: Optional[int] = None,
397
    samples: Optional[dict] = None,
398
399
    cache_requests: bool = False,
    rewrite_requests_cache: bool = False,
Baber Abbasi's avatar
Baber Abbasi committed
400
    bootstrap_iters: Optional[int] = 100000,
Ethan Smith's avatar
Ethan Smith committed
401
402
    write_out: bool = False,
    log_samples: bool = True,
KonradSzafer's avatar
KonradSzafer committed
403
    system_instruction: Optional[str] = None,
404
    apply_chat_template: Union[bool, str] = False,
KonradSzafer's avatar
KonradSzafer committed
405
    fewshot_as_multiturn: bool = False,
406
    verbosity: str = "INFO",
Hojin Lee's avatar
Hojin Lee committed
407
    confirm_run_unsafe_code: bool = False,
Fabrizio Milo's avatar
Fabrizio Milo committed
408
):
409
410
411
412
413
    """Instantiate and evaluate a model on a list of tasks.

    :param lm: obj
        Language Model
    :param task_dict: dict[str, Task]
haileyschoelkopf's avatar
haileyschoelkopf committed
414
        Dictionary of tasks. Tasks will be taken to have name type(task).config.task .
415
416
    :param limit: int, optional
        Limit the number of examples per task (only use this for testing)
417
418
    :param samples: dictionary, optional
        Dictionary indicating which examples should be tested in each task, e.g., {"mmlu_astronomy":[0,3,6],"mmlu_anatomy":[1,4,7,10]}.
Hojin Lee's avatar
Hojin Lee committed
419
420
421
422
    :param cache_requests: bool, optional
        Speed up evaluation by caching the building of dataset requests.
    :param rewrite_requests_cache: bool, optional
        Rewrites all the request cache if set to `True`.
423
    :param bootstrap_iters:
424
        Number of iterations for bootstrap statistics, used when calculating stderr. Set to 0 for skipping all stderr calculations.
425
    :param write_out: bool
426
427
428
        If True, write out an example document and model input for checking task integrity
    :param log_samples: bool
        If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
KonradSzafer's avatar
KonradSzafer committed
429
430
    :param system_instruction: str
        System instruction to be applied to the prompt
431
432
433
434
435
    :param apply_chat_template: Union[bool, str]
        Specifies whether to apply a chat template to the prompt.
        - If set to True, the default chat template is applied.
        - If set to a string, applies the specified chat template by name.
        Defaults to False (no chat template applied).
KonradSzafer's avatar
KonradSzafer committed
436
437
    :param fewshot_as_multiturn: bool
        Whether to provide the fewshot examples as a multiturn conversation or a single user turn.
Hojin Lee's avatar
Hojin Lee committed
438
439
440
441
    :param verbosity: str
        Verbosity level for logging
    :param confirm_run_unsafe_code: bool
        Whether to confirm running tasks marked as unsafe.
442
443
444
    :return
        Dictionary of results
    """
445

446
447
448
449
450
451
    if limit is not None and samples is not None:
        raise ValueError(
            "Either 'limit' or 'samples' must be None, but both are not None."
        )
    if samples is not None:
        eval_logger.info(f"Evaluating examples for tasks {list(samples.keys())}")
452
453
454
455
    if apply_chat_template:
        eval_logger.warning(
            "Chat template formatting change affects loglikelihood and multiple-choice tasks. See docs/chat-template-readme.md for details."
        )
456
    # tracks all Instances/requests a model must generate output on.
457
    requests = defaultdict(list)
458
459
    # stores the amount to pad out reqs per req. type so that
    # number of fwd passes per distributed rank is equal
460
    padding_requests = defaultdict(int)
461

462
    # get lists of group hierarchy and each type of request
Lintang Sutawika's avatar
Lintang Sutawika committed
463
    eval_tasks = get_task_list(task_dict)
464
    if not log_samples:
465
        if not all(
466
467
            "bypass" not in getattr(task_output.task, "_metric_fn_list", {}).keys()
            for task_output in eval_tasks
468
469
        ):
            raise ValueError("log_samples must be True for 'bypass' metric-only tasks")
470

Hojin Lee's avatar
Hojin Lee committed
471
472
473
    # validation checks:
    # 1.are we running multimodal task <-> non-multimodal model class, or vice-versa.
    # 2.are we running code that is marked as unsafe.
474
    incompatible_tasks = []
475
476
    for task_output in eval_tasks:
        task: Task = task_output.task
477

478
        if getattr(task, "MULTIMODAL", False) and not getattr(lm, "MULTIMODAL", False):
479
            incompatible_tasks.append(task_output.task_name)
Hojin Lee's avatar
Hojin Lee committed
480
481
482
483
        elif getattr(task, "UNSAFE_CODE", False) and not confirm_run_unsafe_code:
            raise ValueError(
                f"Attempted to run task: {task_output.task_name} which is marked as unsafe. Set confirm_run_unsafe_code=True to run this task."
            )
484
485
486
487
488
489
490
491
492
    if len(incompatible_tasks) > 0:
        if not getattr(lm, "MULTIMODAL", False):
            raise ValueError(
                f"Attempted to run tasks: {incompatible_tasks} which require multimodal input, but the selected model type does not currently implement this. Multimodal support is currently restricted to the ['hf-multimodal', 'vllm-vlm'] model type."
            )
        else:
            raise ValueError(
                f"Attempted to run tasks: {incompatible_tasks} which are text-only, but used a model type which only currently supports multimodal tasks."
            )
Hojin Lee's avatar
Hojin Lee committed
493
    # end validation check
494

Chenjie Luo's avatar
Chenjie Luo committed
495
496
497
    # Cache the limit arg.
    limit_arg = limit
    limits = []
498
499
500
    for task_output in eval_tasks:
        task: Task = task_output.task

Chenjie Luo's avatar
Chenjie Luo committed
501
502
        limit = get_sample_size(task, limit_arg)
        limits.append(limit)
503
504
        task.build_all_requests(
            limit=limit,
505
506
507
            samples=samples.get(task_output.task_name, None)
            if samples is not None
            else samples,
508
509
510
511
            rank=lm.rank,
            world_size=lm.world_size,
            cache_requests=cache_requests,
            rewrite_requests_cache=rewrite_requests_cache,
KonradSzafer's avatar
KonradSzafer committed
512
            system_instruction=system_instruction,
513
            apply_chat_template=bool(apply_chat_template),
KonradSzafer's avatar
KonradSzafer committed
514
            fewshot_as_multiturn=fewshot_as_multiturn,
515
516
517
518
519
520
            chat_template=getattr(lm, "apply_chat_template")
            if apply_chat_template
            else None,
            tokenizer_name=getattr(lm, "tokenizer_name", "")
            if apply_chat_template
            else "",
521
        )
522
        eval_logger.debug(
523
            f"Task: {task_output.task_name}; number of requests on this rank: {len(task.instances)}"
haileyschoelkopf's avatar
haileyschoelkopf committed
524
525
        )
        if write_out:
526
            print_writeout(task)
527
        # aggregate Instances by LM method requested to get output.
lintangsutawika's avatar
lintangsutawika committed
528
529
530
        for instance in task.instances:
            reqtype = instance.request_type
            requests[reqtype].append(instance)
531
532

        if lm.world_size > 1:
533
534
535
536
            instances_rnk = torch.tensor(len(task._instances), device=lm.device)
            gathered_item = (
                lm.accelerator.gather(instances_rnk).cpu().detach().numpy().tolist()
            )
537
538
539
540
541
542
            # "multiple_choice" task types dispatch (several) "loglikelihood" request types
            reqtype = (
                "loglikelihood"
                if task.OUTPUT_TYPE == "multiple_choice"
                else task.OUTPUT_TYPE
            )
543
            # compute number of pseudo-batches to pad with (FSDP/DDP require even batches among ranks)
544
            numpad = max(gathered_item) - gathered_item[lm.rank]
545
546
            # todo: may not account for padding in cases like SquadV2 which has multiple req types
            padding_requests[reqtype] += numpad
547

548
    ### Run LM on inputs, get all outputs ###
Leo Gao's avatar
Leo Gao committed
549
550
    # execute each type of request
    for reqtype, reqs in requests.items():
551
        eval_logger.info(f"Running {reqtype} requests")
552
553
554
555
        # create `K` copies of each request `req` based off `K = req.repeats`
        cloned_reqs = []
        for req in reqs:
            cloned_reqs.extend([req] * req.repeats)
lintangsutawika's avatar
lintangsutawika committed
556

557
558
        if (lm.world_size > 1) and (padding_requests[reqtype] > 0):
            for _ in range(padding_requests[reqtype]):
559
560
                cloned_reqs.extend([req] * req.repeats)

561
562
563
564
565
566
567
        # run requests through model
        resps = getattr(lm, reqtype)(cloned_reqs)

        # put responses from model into a list of length K for each request.
        for x, req in zip(resps, cloned_reqs):
            req.resps.append(x)

568
569
        if lm.world_size > 1:
            lm.accelerator.wait_for_everyone()
570

571
572
    RANK = lm.rank
    WORLD_SIZE = lm.world_size
573
574
    ### Postprocess outputs ###
    # TODO: del model here, maybe (idea: allow user to specify device of e.g. reward model separately)
Chenjie Luo's avatar
Chenjie Luo committed
575
    for task_output, limit in zip(eval_tasks, limits):
576
        task = task_output.task
577
578
        task.apply_filters()

579
580
        ### Collect values of metrics on all datapoints ###
        # # unpack results and sort back in order and return control to Task
haileyschoelkopf's avatar
haileyschoelkopf committed
581
        # TODO: make it possible to use a different metric per filter
582
        # Pre-process task.instances to group by doc_id
583
        instances_by_doc_id = defaultdict(list)
584
585
586
587
588
        for instance in task.instances:
            instances_by_doc_id[instance.doc_id].append(instance)
        # Sort instances within each group
        for instances in instances_by_doc_id.values():
            instances.sort(key=lambda x: x.idx)
haileyschoelkopf's avatar
haileyschoelkopf committed
589
        # iterate over different filters used
590
        for filter_key in task.instances[0].filtered_resps.keys():
591
592
593
594
595
            indices = (
                samples.get(task_output.task_name, None)
                if samples is not None
                else None
            )
596
            doc_iterator = task.doc_iterator(
597
598
599
600
                rank=RANK,
                limit=limit,
                world_size=WORLD_SIZE,
                samples=indices,
601
            )
602
            for doc_id, doc in doc_iterator:
603
604
605
606
                if indices:
                    doc_id_true = indices[doc_id]
                else:
                    doc_id_true = doc_id
607
                requests = instances_by_doc_id[doc_id]
lintangsutawika's avatar
lintangsutawika committed
608
                metrics = task.process_results(
609
                    doc, [req.filtered_resps[filter_key] for req in requests]
lintangsutawika's avatar
lintangsutawika committed
610
                )
611
612
613
                if log_samples:
                    target = task.doc_to_target(doc)
                    example = {
614
                        "doc_id": doc_id_true,
615
616
617
618
                        "doc": doc,
                        "target": target,
                        "arguments": [req.args for req in requests],
                        "resps": [req.resps for req in requests],
619
620
621
                        "filtered_resps": [
                            req.filtered_resps[filter_key] for req in requests
                        ],
622
623
                        "filter": filter_key,
                        "metrics": list(metrics.keys()),
624
625
626
627
628
629
630
631
632
633
                        "doc_hash": hash_string(
                            json.dumps(
                                requests[0].doc,
                                indent=2,
                                default=handle_non_serializable,
                                ensure_ascii=False,
                            )
                        ),
                        "prompt_hash": hash_string(requests[0].arguments[0]),
                        "target_hash": hash_string(str(target)),
634
635
                    }
                    example.update(metrics)
636
                    task_output.logged_samples.append(example)
637
                for metric, value in metrics.items():
638
                    task_output.sample_metrics[(metric, filter_key)].append(value)
639

640
641
    if WORLD_SIZE > 1:
        # if multigpu, then gather data across all ranks to rank 0
642
        # first gather logged samples across all ranks
643
644
645
646
647
648
649
650
        for task_output in eval_tasks:
            if log_samples:
                # for task_name, task_samples in list(samples.items()):
                full_samples = [None] * WORLD_SIZE if RANK == 0 else None
                torch.distributed.gather_object(
                    obj=task_output.logged_samples,
                    object_gather_list=full_samples,
                    dst=0,
651
                )
652

653
654
655
656
                if RANK == 0:
                    task_output.logged_samples = list(
                        itertools.chain.from_iterable(full_samples)
                    )
657

658
659
660
661
662
663
664
            # then collect metrics across all ranks
            for metrics in task_output.sample_metrics:
                metric_list = [None] * WORLD_SIZE if RANK == 0 else None
                torch.distributed.gather_object(
                    obj=task_output.sample_metrics[metrics],
                    object_gather_list=metric_list,
                    dst=0,
665
                )
666
667
668
669
                if RANK == 0:
                    task_output.sample_metrics[metrics] = list(
                        itertools.chain.from_iterable(metric_list)
                    )
670

671
    if RANK == 0:
672
673
        ### Aggregate results over all datapoints ###
        # aggregate results ; run bootstrap CIs
674
675
        for task_output in eval_tasks:
            task_output.calculate_aggregate_metric(bootstrap_iters=bootstrap_iters)
676
677
678
679
680
681
682
683
        (
            results,
            samples,
            configs,
            versions,
            num_fewshot,
            higher_is_better,
        ) = consolidate_results(eval_tasks)
Fabrizio Milo's avatar
Fabrizio Milo committed
684

685
        ### Calculate group metrics ###
lintangsutawika's avatar
lintangsutawika committed
686
        if bool(results):
Lintang Sutawika's avatar
Lintang Sutawika committed
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
            results, versions, show_group_table, *_ = consolidate_group_results(
                results, versions, task_dict
            )

        results_agg, group_agg = prepare_print_tasks(task_dict, results)
        subtask_list = get_subtask_list(task_dict)

        # collect all higher_is_better values for metrics
        # in the group's subtasks.
        # TODO: clean this up ; unify with the below metric_list loop?
        _higher_is_better = {}
        for group, task_list in subtask_list.items():
            if (
                len(task_list) != 0
            ):  # subtask list will list "task_name": [] for solo tasks
702
703
704
705
                for task in task_list:
                    for m, h in higher_is_better[task].items():
                        if m not in _higher_is_better.keys():
                            _higher_is_better[m] = h
lintangsutawika's avatar
lintangsutawika committed
706

Lintang Sutawika's avatar
Lintang Sutawika committed
707
708
709
710
711
712
713
714
715
716
                        if (
                            m in _higher_is_better
                            and _higher_is_better[m] is not None
                            and _higher_is_better[m] != h
                        ):
                            eval_logger.warning(
                                f"Higher_is_better values for metric {m} in group {group} are not consistent. Defaulting to None."
                            )
                            _higher_is_better[m] = None
                higher_is_better[group] = _higher_is_better
717

718
        results_dict = {
719
            "results": dict(results_agg.items()),
Lintang Sutawika's avatar
Lintang Sutawika committed
720
721
722
723
724
725
            **(
                {"groups": dict(group_agg.items())}
                if (bool(group_agg) & show_group_table)
                else {}
            ),
            "group_subtasks": dict(reversed(subtask_list.items())),
726
727
            "configs": dict(sorted(configs.items())),
            "versions": dict(sorted(versions.items())),
728
            "n-shot": dict(sorted(num_fewshot.items())),
729
            "higher_is_better": dict(sorted(higher_is_better.items())),
730
731
732
            "n-samples": {
                task_output.task_name: {
                    "original": len(task_output.task.eval_docs),
KonradSzafer's avatar
KonradSzafer committed
733
734
735
736
                    "effective": min(
                        limit if limit else len(task_output.task.eval_docs),
                        len(task_output.task.eval_docs),
                    ),
737
                }
Chenjie Luo's avatar
Chenjie Luo committed
738
                for task_output, limit in zip(eval_tasks, limits)
739
            },
740
        }
741
742
743
744
        if log_samples:
            results_dict["samples"] = dict(samples)

        return results_dict
Fabrizio Milo's avatar
Fabrizio Milo committed
745

746
747
    else:
        return None
748
749
750
751


def request_caching_arg_to_dict(cache_requests: str) -> dict:
    request_caching_args = {
752
753
754
        "cache_requests": cache_requests in {"true", "refresh"},
        "rewrite_requests_cache": cache_requests == "refresh",
        "delete_requests_cache": cache_requests == "delete",
755
756
757
    }

    return request_caching_args