__main__.py 17.4 KB
Newer Older
1
2
3
import argparse
import json
import logging
lintangsutawika's avatar
lintangsutawika committed
4
import os
5
import sys
6
from functools import partial
haileyschoelkopf's avatar
haileyschoelkopf committed
7
from typing import Union
Leo Gao's avatar
Leo Gao committed
8

9
from lm_eval import evaluator, utils
10
from lm_eval.evaluator import request_caching_arg_to_dict
11
from lm_eval.loggers import EvaluationTracker, WandbLogger
12
from lm_eval.tasks import TaskManager
Baber's avatar
Baber committed
13
14
15
16
17
18
from lm_eval.utils import (
    handle_non_serializable,
    make_table,
    parse_keyed_list_string,
    simple_parse_args_string,
)
Fabrizio Milo's avatar
Fabrizio Milo committed
19

20

21
22
23
def _int_or_none_list_arg_type(
    min_len: int, max_len: int, defaults: str, value: str, split_char: str = ","
):
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
    def parse_value(item):
        item = item.strip().lower()
        if item == "none":
            return None
        try:
            return int(item)
        except ValueError:
            raise argparse.ArgumentTypeError(f"{item} is not an integer or None")

    items = [parse_value(v) for v in value.split(split_char)]
    num_items = len(items)

    if num_items == 1:
        # Makes downstream handling the same for single and multiple values
        items = items * max_len
39
    elif num_items < min_len or num_items > max_len:
40
41
42
        raise argparse.ArgumentTypeError(
            f"Argument requires {max_len} integers or None, separated by '{split_char}'"
        )
43
44
45
46
47
48
49
50
51
    elif num_items != max_len:
        logging.warning(
            f"Argument requires {max_len} integers or None, separated by '{split_char}'. "
            "Missing values will be filled with defaults."
        )
        default_items = [parse_value(v) for v in defaults.split(split_char)]
        items.extend(
            default_items[num_items:]
        )  # extend items list with missing defaults
52
53
54
55

    return items


56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
def check_argument_types(parser: argparse.ArgumentParser):
    """
    Check to make sure all CLI args are typed, raises error if not
    """
    for action in parser._actions:
        if action.dest != "help" and not action.const:
            if action.type is None:
                raise ValueError(
                    f"Argument '{action.dest}' doesn't have a type specified."
                )
            else:
                continue


def setup_parser() -> argparse.ArgumentParser:
lintangsutawika's avatar
lintangsutawika committed
71
    parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
72
73
74
    parser.add_argument(
        "--model", "-m", type=str, default="hf", help="Name of model e.g. `hf`"
    )
lintangsutawika's avatar
lintangsutawika committed
75
76
    parser.add_argument(
        "--tasks",
Baber Abbasi's avatar
Baber Abbasi committed
77
        "-t",
lintangsutawika's avatar
lintangsutawika committed
78
        default=None,
79
        type=str,
80
        metavar="task1,task2",
81
        help="Comma-separated list of task names or task groupings to evaluate on.\nTo get full list of tasks, use one of the commands `lm-eval --tasks {{list_groups,list_subtasks,list_tags,list}}` to list out all available names for task groupings; only (sub)tasks; tags; or all of the above",
lintangsutawika's avatar
lintangsutawika committed
82
    )
83
84
    parser.add_argument(
        "--model_args",
Baber Abbasi's avatar
Baber Abbasi committed
85
        "-a",
86
        default="",
87
        type=str,
88
        help="Comma separated string arguments for model, e.g. `pretrained=EleutherAI/pythia-160m,dtype=float32`",
89
    )
lintangsutawika's avatar
lintangsutawika committed
90
    parser.add_argument(
91
        "--num_fewshot",
Baber Abbasi's avatar
Baber Abbasi committed
92
        "-f",
93
        type=int,
94
        default=None,
95
        metavar="N",
96
97
        help="Number of examples in few-shot context",
    )
98
99
    parser.add_argument(
        "--batch_size",
Baber Abbasi's avatar
Baber Abbasi committed
100
        "-b",
101
102
103
104
105
        type=str,
        default=1,
        metavar="auto|auto:N|N",
        help="Acceptable values are 'auto', 'auto:N' or N, where N is an integer. Default 1.",
    )
lintangsutawika's avatar
lintangsutawika committed
106
107
108
109
    parser.add_argument(
        "--max_batch_size",
        type=int,
        default=None,
110
111
        metavar="N",
        help="Maximal batch size to try with --batch_size auto.",
lintangsutawika's avatar
lintangsutawika committed
112
    )
113
114
115
116
    parser.add_argument(
        "--device",
        type=str,
        default=None,
117
        help="Device to use (e.g. cuda, cuda:0, cpu).",
118
119
120
    )
    parser.add_argument(
        "--output_path",
Baber Abbasi's avatar
Baber Abbasi committed
121
        "-o",
122
123
        default=None,
        type=str,
124
        metavar="DIR|DIR/file.json",
125
        help="The path to the output file where the result metrics will be saved. If the path is a directory and log_samples is true, the results will be saved in the directory. Else the parent directory will be used.",
126
    )
lintangsutawika's avatar
lintangsutawika committed
127
128
    parser.add_argument(
        "--limit",
Baber Abbasi's avatar
Baber Abbasi committed
129
        "-L",
lintangsutawika's avatar
lintangsutawika committed
130
131
        type=float,
        default=None,
132
        metavar="N|0<N<1",
lintangsutawika's avatar
lintangsutawika committed
133
134
135
        help="Limit the number of examples per task. "
        "If <1, limit is a percentage of the total number of examples.",
    )
136
137
    parser.add_argument(
        "--use_cache",
Baber Abbasi's avatar
Baber Abbasi committed
138
        "-c",
139
140
        type=str,
        default=None,
141
        metavar="DIR",
142
143
        help="A path to a sqlite db file for caching model responses. `None` if not caching.",
    )
144
145
146
147
148
149
150
    parser.add_argument(
        "--cache_requests",
        type=str,
        default=None,
        choices=["true", "refresh", "delete"],
        help="Speed up evaluation by caching the building of dataset requests. `None` if not caching.",
    )
151
152
153
    parser.add_argument(
        "--check_integrity",
        action="store_true",
154
        help="Whether to run the relevant part of the test suite for the tasks.",
155
156
157
    )
    parser.add_argument(
        "--write_out",
Baber Abbasi's avatar
Baber Abbasi committed
158
        "-w",
159
160
        action="store_true",
        default=False,
161
        help="Prints the prompt for the first few documents.",
162
163
164
    )
    parser.add_argument(
        "--log_samples",
Baber Abbasi's avatar
Baber Abbasi committed
165
        "-s",
166
167
        action="store_true",
        default=False,
168
        help="If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis. Use with --output_path.",
169
    )
KonradSzafer's avatar
KonradSzafer committed
170
171
172
173
174
175
176
177
    parser.add_argument(
        "--system_instruction",
        type=str,
        default=None,
        help="System instruction to be used in the prompt",
    )
    parser.add_argument(
        "--apply_chat_template",
178
179
180
        type=str,
        nargs="?",
        const=True,
KonradSzafer's avatar
KonradSzafer committed
181
        default=False,
182
183
184
185
186
187
        help=(
            "If True, apply chat template to the prompt. "
            "Providing `--apply_chat_template` without an argument will apply the default chat template to the prompt. "
            "To apply a specific template from the available list of templates, provide the template name as an argument. "
            "E.g. `--apply_chat_template template_name`"
        ),
KonradSzafer's avatar
KonradSzafer committed
188
189
190
191
192
193
194
    )
    parser.add_argument(
        "--fewshot_as_multiturn",
        action="store_true",
        default=False,
        help="If True, uses the fewshot as a multi-turn conversation",
    )
195
196
197
198
199
200
    parser.add_argument(
        "--show_config",
        action="store_true",
        default=False,
        help="If True, shows the the full config of all tasks at the end of the evaluation.",
    )
201
202
203
204
    parser.add_argument(
        "--include_path",
        type=str,
        default=None,
205
        metavar="DIR",
206
207
        help="Additional path to include if there are external tasks to include.",
    )
208
209
    parser.add_argument(
        "--gen_kwargs",
210
        type=str,
211
        default=None,
USVSN Sai Prashanth's avatar
USVSN Sai Prashanth committed
212
213
        help=(
            "String arguments for model generation on greedy_until tasks,"
214
            " e.g. `temperature=0,top_k=0,top_p=0`."
lintangsutawika's avatar
lintangsutawika committed
215
216
217
        ),
    )
    parser.add_argument(
lintangsutawika's avatar
lintangsutawika committed
218
        "--verbosity",
Baber Abbasi's avatar
Baber Abbasi committed
219
220
        "-v",
        type=str.upper,
lintangsutawika's avatar
lintangsutawika committed
221
        default="INFO",
222
223
        metavar="CRITICAL|ERROR|WARNING|INFO|DEBUG",
        help="Controls the reported logging error level. Set to DEBUG when testing + adding new task configurations for comprehensive log output.",
224
    )
225
226
    parser.add_argument(
        "--wandb_args",
227
        type=str,
228
229
230
        default="",
        help="Comma separated string arguments passed to wandb.init, e.g. `project=lm-eval,job_type=eval",
    )
231
232
233
234
235
236
    parser.add_argument(
        "--hf_hub_log_args",
        type=str,
        default="",
        help="Comma separated string arguments passed to Hugging Face Hub's log function, e.g. `hub_results_org=EleutherAI,hub_repo_name=lm-eval-results`",
    )
Baber Abbasi's avatar
Baber Abbasi committed
237
238
239
240
241
242
243
    parser.add_argument(
        "--predict_only",
        "-x",
        action="store_true",
        default=False,
        help="Use with --log_samples. Only model outputs will be saved and metrics will not be evaluated.",
    )
244
    default_seed_string = "0,1234,1234,1234"
245
246
    parser.add_argument(
        "--seed",
247
248
        type=partial(_int_or_none_list_arg_type, 3, 4, default_seed_string),
        default=default_seed_string,  # for backward compatibility
249
        help=(
250
251
            "Set seed for python's random, numpy, torch, and fewshot sampling.\n"
            "Accepts a comma-separated list of 4 values for python's random, numpy, torch, and fewshot sampling seeds, "
Sadra Barikbin's avatar
Sadra Barikbin committed
252
            "respectively, or a single integer to set the same seed for all four.\n"
253
254
255
256
257
            f"The values are either an integer or 'None' to not set the seed. Default is `{default_seed_string}` "
            "(for backward compatibility).\n"
            "E.g. `--seed 0,None,8,52` sets `random.seed(0)`, `torch.manual_seed(8)`, and fewshot sampling seed to 52. "
            "Here numpy's seed is not set since the second value is `None`.\n"
            "E.g, `--seed 42` sets all four seeds to 42."
258
259
        ),
    )
260
261
    parser.add_argument(
        "--trust_remote_code",
262
        action="store_true",
263
264
        help="Sets trust_remote_code to True to execute code to create HF Datasets from the Hub",
    )
Hojin Lee's avatar
Hojin Lee committed
265
266
267
268
269
    parser.add_argument(
        "--confirm_run_unsafe_code",
        action="store_true",
        help="Confirm that you understand the risks of running unsafe code for tasks that require it",
    )
Baber's avatar
Baber committed
270
271
272
273
    parser.add_argument(
        "--metadata",
        type=str,
        default=None,
Baber's avatar
Baber committed
274
        help="Comma separated string argument metadata to pass to task configs, for example max_context_len=4096,8192. Will be parsed as a dictionary with all values as lists.",
Baber's avatar
Baber committed
275
    )
276
277
278
279
280
    return parser


def parse_eval_args(parser: argparse.ArgumentParser) -> argparse.Namespace:
    check_argument_types(parser)
Jason Phang's avatar
Jason Phang committed
281
282
    return parser.parse_args()

Fabrizio Milo's avatar
Fabrizio Milo committed
283

haileyschoelkopf's avatar
haileyschoelkopf committed
284
285
286
def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
    if not args:
        # we allow for args to be passed externally, else we parse them ourselves
287
288
        parser = setup_parser()
        args = parse_eval_args(parser)
haileyschoelkopf's avatar
haileyschoelkopf committed
289

290
    if args.wandb_args:
291
        wandb_logger = WandbLogger(**simple_parse_args_string(args.wandb_args))
292

293
    eval_logger = utils.eval_logger
lintangsutawika's avatar
lintangsutawika committed
294
    eval_logger.setLevel(getattr(logging, f"{args.verbosity}"))
295
    eval_logger.info(f"Verbosity set to {args.verbosity}")
haileyschoelkopf's avatar
haileyschoelkopf committed
296
    os.environ["TOKENIZERS_PARALLELISM"] = "false"
Fabrizio Milo's avatar
Fabrizio Milo committed
297

298
    # update the evaluation tracker args with the output path and the HF token
299
300
301
302
    if args.output_path:
        args.hf_hub_log_args += f",output_path={args.output_path}"
    if os.environ.get("HF_TOKEN", None):
        args.hf_hub_log_args += f",token={os.environ.get('HF_TOKEN')}"
303
304
305
    evaluation_tracker_args = simple_parse_args_string(args.hf_hub_log_args)
    evaluation_tracker = EvaluationTracker(**evaluation_tracker_args)

Baber Abbasi's avatar
Baber Abbasi committed
306
307
308
    if args.predict_only:
        args.log_samples = True
    if (args.log_samples or args.predict_only) and not args.output_path:
309
310
311
        raise ValueError(
            "Specify --output_path if providing --log_samples or --predict_only"
        )
Baber Abbasi's avatar
Baber Abbasi committed
312

KonradSzafer's avatar
KonradSzafer committed
313
314
    if args.fewshot_as_multiturn and args.apply_chat_template is False:
        raise ValueError(
315
            "When `fewshot_as_multiturn` is selected, `apply_chat_template` must be set (either to `True` or to the chosen template name)."
KonradSzafer's avatar
KonradSzafer committed
316
317
        )

318
319
    if args.include_path is not None:
        eval_logger.info(f"Including path: {args.include_path}")
320
    task_manager = TaskManager(args.verbosity, include_path=args.include_path)
Fabrizio Milo's avatar
Fabrizio Milo committed
321

KonradSzafer's avatar
KonradSzafer committed
322
    if "push_samples_to_hub" in evaluation_tracker_args and not args.log_samples:
323
324
325
326
        eval_logger.warning(
            "Pushing samples to the Hub requires --log_samples to be set. Samples will not be pushed to the Hub."
        )

Leo Gao's avatar
Leo Gao committed
327
    if args.limit:
lintangsutawika's avatar
lintangsutawika committed
328
329
330
        eval_logger.warning(
            " --limit SHOULD ONLY BE USED FOR TESTING."
            "REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT."
Fabrizio Milo's avatar
Fabrizio Milo committed
331
        )
lintangsutawika's avatar
lintangsutawika committed
332

333
    if args.tasks is None:
334
335
        eval_logger.error("Need to specify task to evaluate.")
        sys.exit()
336
    elif args.tasks == "list":
337
338
339
340
341
342
343
344
345
346
        print(task_manager.list_all_tasks())
        sys.exit()
    elif args.tasks == "list_groups":
        print(task_manager.list_all_tasks(list_subtasks=False, list_tags=False))
        sys.exit()
    elif args.tasks == "list_tags":
        print(task_manager.list_all_tasks(list_groups=False, list_subtasks=False))
        sys.exit()
    elif args.tasks == "list_subtasks":
        print(task_manager.list_all_tasks(list_groups=False, list_tags=False))
Lintang Sutawika's avatar
Lintang Sutawika committed
347
        sys.exit()
Jason Phang's avatar
Jason Phang committed
348
    else:
349
350
        if os.path.isdir(args.tasks):
            import glob
351
352

            task_names = []
353
354
            yaml_path = os.path.join(args.tasks, "*.yaml")
            for yaml_file in glob.glob(yaml_path):
lintangsutawika's avatar
lintangsutawika committed
355
                config = utils.load_yaml_config(yaml_file)
356
357
                task_names.append(config)
        else:
358
359
360
            task_list = args.tasks.split(",")
            task_names = task_manager.match_tasks(task_list)
            for task in [task for task in task_list if task not in task_names]:
361
                if os.path.isfile(task):
lintangsutawika's avatar
lintangsutawika committed
362
                    config = utils.load_yaml_config(task)
363
                    task_names.append(config)
364
            task_missing = [
365
                task for task in task_list if task not in task_names and "*" not in task
366
            ]  # we don't want errors if a wildcard ("*") task name was used
lintangsutawika's avatar
lintangsutawika committed
367

baberabb's avatar
baberabb committed
368
369
370
371
            if task_missing:
                missing = ", ".join(task_missing)
                eval_logger.error(
                    f"Tasks were not found: {missing}\n"
lintangsutawika's avatar
lintangsutawika committed
372
                    f"{utils.SPACING}Try `lm-eval --tasks list` for list of available tasks",
baberabb's avatar
baberabb committed
373
374
                )
                raise ValueError(
375
                    f"Tasks not found: {missing}. Try `lm-eval --tasks {{list_groups,list_subtasks,list_tags,list}}` to list out all available names for task groupings; only (sub)tasks; tags; or all of the above, or pass '--verbosity DEBUG' to troubleshoot task registration issues."
baberabb's avatar
baberabb committed
376
                )
lintangsutawika's avatar
lintangsutawika committed
377

378
379
    # Respect user's value passed in via CLI, otherwise default to True and add to comma-separated model args
    if args.trust_remote_code:
380
381
        eval_logger.info(
            "Passed `--trust_remote_code`, setting environment variable `HF_DATASETS_TRUST_REMOTE_CODE=true`"
382
        )
383
384
385
386
387
388
389
390
        # HACK: import datasets and override its HF_DATASETS_TRUST_REMOTE_CODE value internally,
        # because it's already been determined based on the prior env var before launching our
        # script--`datasets` gets imported by lm_eval internally before these lines can update the env.
        import datasets

        datasets.config.HF_DATASETS_TRUST_REMOTE_CODE = True

        args.model_args = args.model_args + ",trust_remote_code=True"
391

lintangsutawika's avatar
lintangsutawika committed
392
    eval_logger.info(f"Selected Tasks: {task_names}")
393

394
395
396
397
    request_caching_args = request_caching_arg_to_dict(
        cache_requests=args.cache_requests
    )

398
399
400
401
402
403
    results = evaluator.simple_evaluate(
        model=args.model,
        model_args=args.model_args,
        tasks=task_names,
        num_fewshot=args.num_fewshot,
        batch_size=args.batch_size,
404
        max_batch_size=args.max_batch_size,
405
        device=args.device,
haileyschoelkopf's avatar
haileyschoelkopf committed
406
        use_cache=args.use_cache,
407
408
        limit=args.limit,
        check_integrity=args.check_integrity,
409
        write_out=args.write_out,
410
        log_samples=args.log_samples,
KonradSzafer's avatar
KonradSzafer committed
411
412
413
414
        evaluation_tracker=evaluation_tracker,
        system_instruction=args.system_instruction,
        apply_chat_template=args.apply_chat_template,
        fewshot_as_multiturn=args.fewshot_as_multiturn,
lintangsutawika's avatar
lintangsutawika committed
415
        gen_kwargs=args.gen_kwargs,
416
        task_manager=task_manager,
417
        verbosity=args.verbosity,
Baber Abbasi's avatar
Baber Abbasi committed
418
        predict_only=args.predict_only,
419
420
421
        random_seed=args.seed[0],
        numpy_random_seed=args.seed[1],
        torch_random_seed=args.seed[2],
422
        fewshot_random_seed=args.seed[3],
Hojin Lee's avatar
Hojin Lee committed
423
        confirm_run_unsafe_code=args.confirm_run_unsafe_code,
Baber's avatar
Baber committed
424
        metadata=parse_keyed_list_string(args.metadata),
425
        **request_caching_args,
426
    )
427

428
    if results is not None:
429
430
        if args.log_samples:
            samples = results.pop("samples")
431
        dumped = json.dumps(
432
            results, indent=2, default=handle_non_serializable, ensure_ascii=False
433
        )
434
435
        if args.show_config:
            print(dumped)
436

437
438
        batch_sizes = ",".join(map(str, results["config"]["batch_sizes"]))

439
440
441
442
443
444
445
446
447
448
        # Add W&B logging
        if args.wandb_args:
            try:
                wandb_logger.post_init(results)
                wandb_logger.log_eval_result()
                if args.log_samples:
                    wandb_logger.log_eval_samples(samples)
            except Exception as e:
                eval_logger.info(f"Logging to Weights and Biases failed due to {e}")

KonradSzafer's avatar
KonradSzafer committed
449
450
451
        evaluation_tracker.save_results_aggregated(
            results=results, samples=samples if args.log_samples else None
        )
452
453
454
455
456
457

        if args.log_samples:
            for task_name, config in results["configs"].items():
                evaluation_tracker.save_results_samples(
                    task_name=task_name, samples=samples[task_name]
                )
lintangsutawika's avatar
lintangsutawika committed
458

459
460
461
462
463
464
        if (
            evaluation_tracker.push_results_to_hub
            or evaluation_tracker.push_samples_to_hub
        ):
            evaluation_tracker.recreate_metadata_card()

465
        print(
466
            f"{args.model} ({args.model_args}), gen_kwargs: ({args.gen_kwargs}), limit: {args.limit}, num_fewshot: {args.num_fewshot}, "
467
            f"batch_size: {args.batch_size}{f' ({batch_sizes})' if batch_sizes else ''}"
468
        )
469
        print(make_table(results))
lintangsutawika's avatar
lintangsutawika committed
470
        if "groups" in results:
471
            print(make_table(results, "groups"))
Jason Phang's avatar
lib  
Jason Phang committed
472

473
474
475
476
        if args.wandb_args:
            # Tear down wandb run once all the logging is done.
            wandb_logger.run.finish()

477

Jason Phang's avatar
Jason Phang committed
478
if __name__ == "__main__":
haileyschoelkopf's avatar
haileyschoelkopf committed
479
    cli_evaluate()