__main__.py 20.8 KB
Newer Older
1
2
3
import argparse
import json
import logging
lintangsutawika's avatar
lintangsutawika committed
4
import os
5
import sys
6
from functools import partial
7
from pathlib import Path
haileyschoelkopf's avatar
haileyschoelkopf committed
8
from typing import Union
Leo Gao's avatar
Leo Gao committed
9

10
from lm_eval import evaluator, utils
11
from lm_eval.evaluator import request_caching_arg_to_dict
12
from lm_eval.loggers import EvaluationTracker, WandbLogger
13
from lm_eval.tasks import TaskManager
Baber Abbasi's avatar
Baber Abbasi committed
14
from lm_eval.utils import (
15
16
    TrackExplicitAction,
    TrackExplicitStoreTrue,
Baber Abbasi's avatar
Baber Abbasi committed
17
    handle_non_serializable,
18
    load_yaml_config,
Baber Abbasi's avatar
Baber Abbasi committed
19
    make_table,
20
21
    non_default_update,
    parse_namespace,
Baber Abbasi's avatar
Baber Abbasi committed
22
23
24
25
26
27
28
29
30
31
32
33
34
35
)


def try_parse_json(value: str) -> Union[str, dict, None]:
    if value is None:
        return None
    try:
        return json.loads(value)
    except json.JSONDecodeError:
        if "{" in value:
            raise argparse.ArgumentTypeError(
                f"Invalid JSON: {value}. Hint: Use double quotes for JSON strings."
            )
        return value
Fabrizio Milo's avatar
Fabrizio Milo committed
36

37

38
39
40
def _int_or_none_list_arg_type(
    min_len: int, max_len: int, defaults: str, value: str, split_char: str = ","
):
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
    def parse_value(item):
        item = item.strip().lower()
        if item == "none":
            return None
        try:
            return int(item)
        except ValueError:
            raise argparse.ArgumentTypeError(f"{item} is not an integer or None")

    items = [parse_value(v) for v in value.split(split_char)]
    num_items = len(items)

    if num_items == 1:
        # Makes downstream handling the same for single and multiple values
        items = items * max_len
56
    elif num_items < min_len or num_items > max_len:
57
58
59
        raise argparse.ArgumentTypeError(
            f"Argument requires {max_len} integers or None, separated by '{split_char}'"
        )
60
61
62
63
64
65
66
67
68
    elif num_items != max_len:
        logging.warning(
            f"Argument requires {max_len} integers or None, separated by '{split_char}'. "
            "Missing values will be filled with defaults."
        )
        default_items = [parse_value(v) for v in defaults.split(split_char)]
        items.extend(
            default_items[num_items:]
        )  # extend items list with missing defaults
69
70
71
72

    return items


73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
def check_argument_types(parser: argparse.ArgumentParser):
    """
    Check to make sure all CLI args are typed, raises error if not
    """
    for action in parser._actions:
        if action.dest != "help" and not action.const:
            if action.type is None:
                raise ValueError(
                    f"Argument '{action.dest}' doesn't have a type specified."
                )
            else:
                continue


def setup_parser() -> argparse.ArgumentParser:
lintangsutawika's avatar
lintangsutawika committed
88
    parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
89
    parser.add_argument(
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
        "--config",
        "-C",
        default=None,
        type=str,
        metavar="DIR/file.yaml",
        action=TrackExplicitAction,
        help="Path to config with all arguments for `lm-eval`",
    )
    parser.add_argument(
        "--model",
        "-m",
        type=str,
        default="hf",
        action=TrackExplicitAction,
        help="Name of model e.g. `hf`",
105
    )
lintangsutawika's avatar
lintangsutawika committed
106
107
    parser.add_argument(
        "--tasks",
Baber Abbasi's avatar
Baber Abbasi committed
108
        "-t",
lintangsutawika's avatar
lintangsutawika committed
109
        default=None,
110
        type=str,
111
        action=TrackExplicitAction,
112
        metavar="task1,task2",
113
        help="Comma-separated list of task names or task groupings to evaluate on.\nTo get full list of tasks, use one of the commands `lm-eval --tasks {{list_groups,list_subtasks,list_tags,list}}` to list out all available names for task groupings; only (sub)tasks; tags; or all of the above",
lintangsutawika's avatar
lintangsutawika committed
114
    )
115
116
    parser.add_argument(
        "--model_args",
Baber Abbasi's avatar
Baber Abbasi committed
117
        "-a",
118
        default="",
119
        action=TrackExplicitAction,
Baber Abbasi's avatar
Baber Abbasi committed
120
121
        type=try_parse_json,
        help="""Comma separated string or JSON formatted arguments for model, e.g. `pretrained=EleutherAI/pythia-160m,dtype=float32` or '{"pretrained":"EleutherAI/pythia-160m","dtype":"float32"}'""",
122
    )
lintangsutawika's avatar
lintangsutawika committed
123
    parser.add_argument(
124
        "--num_fewshot",
Baber Abbasi's avatar
Baber Abbasi committed
125
        "-f",
126
        type=int,
127
        default=None,
128
        action=TrackExplicitAction,
129
        metavar="N",
130
131
        help="Number of examples in few-shot context",
    )
132
133
    parser.add_argument(
        "--batch_size",
Baber Abbasi's avatar
Baber Abbasi committed
134
        "-b",
135
        type=str,
136
        action=TrackExplicitAction,
137
138
139
140
        default=1,
        metavar="auto|auto:N|N",
        help="Acceptable values are 'auto', 'auto:N' or N, where N is an integer. Default 1.",
    )
lintangsutawika's avatar
lintangsutawika committed
141
142
143
144
    parser.add_argument(
        "--max_batch_size",
        type=int,
        default=None,
145
        action=TrackExplicitAction,
146
147
        metavar="N",
        help="Maximal batch size to try with --batch_size auto.",
lintangsutawika's avatar
lintangsutawika committed
148
    )
149
150
151
152
    parser.add_argument(
        "--device",
        type=str,
        default=None,
153
        action=TrackExplicitAction,
154
        help="Device to use (e.g. cuda, cuda:0, cpu).",
155
156
157
    )
    parser.add_argument(
        "--output_path",
Baber Abbasi's avatar
Baber Abbasi committed
158
        "-o",
159
160
        default=None,
        type=str,
161
        action=TrackExplicitAction,
162
        metavar="DIR|DIR/file.json",
163
        help="The path to the output file where the result metrics will be saved. If the path is a directory and log_samples is true, the results will be saved in the directory. Else the parent directory will be used.",
164
    )
lintangsutawika's avatar
lintangsutawika committed
165
166
    parser.add_argument(
        "--limit",
Baber Abbasi's avatar
Baber Abbasi committed
167
        "-L",
lintangsutawika's avatar
lintangsutawika committed
168
169
        type=float,
        default=None,
170
        action=TrackExplicitAction,
171
        metavar="N|0<N<1",
lintangsutawika's avatar
lintangsutawika committed
172
173
174
        help="Limit the number of examples per task. "
        "If <1, limit is a percentage of the total number of examples.",
    )
175
176
177
178
179
    parser.add_argument(
        "--samples",
        "-E",
        default=None,
        type=str,
180
        action=TrackExplicitAction,
181
182
183
        metavar="/path/to/json",
        help='JSON string or path to JSON file containing doc indices of selected examples to test. Format: {"task_name":[indices],...}',
    )
184
185
    parser.add_argument(
        "--use_cache",
Baber Abbasi's avatar
Baber Abbasi committed
186
        "-c",
187
        type=str,
188
        action=TrackExplicitAction,
189
        default=None,
190
        metavar="DIR",
191
192
        help="A path to a sqlite db file for caching model responses. `None` if not caching.",
    )
193
194
195
196
    parser.add_argument(
        "--cache_requests",
        type=str,
        default=None,
197
        action=TrackExplicitAction,
198
199
200
        choices=["true", "refresh", "delete"],
        help="Speed up evaluation by caching the building of dataset requests. `None` if not caching.",
    )
201
202
    parser.add_argument(
        "--check_integrity",
203
        action=TrackExplicitStoreTrue,
204
        help="Whether to run the relevant part of the test suite for the tasks.",
205
206
207
    )
    parser.add_argument(
        "--write_out",
Baber Abbasi's avatar
Baber Abbasi committed
208
        "-w",
209
        action=TrackExplicitStoreTrue,
210
        default=False,
211
        help="Prints the prompt for the first few documents.",
212
213
214
    )
    parser.add_argument(
        "--log_samples",
Baber Abbasi's avatar
Baber Abbasi committed
215
        "-s",
216
        action=TrackExplicitStoreTrue,
217
        default=False,
218
        help="If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis. Use with --output_path.",
219
    )
KonradSzafer's avatar
KonradSzafer committed
220
221
222
223
    parser.add_argument(
        "--system_instruction",
        type=str,
        default=None,
224
        action=TrackExplicitAction,
KonradSzafer's avatar
KonradSzafer committed
225
226
227
228
        help="System instruction to be used in the prompt",
    )
    parser.add_argument(
        "--apply_chat_template",
229
230
        type=str,
        nargs="?",
231
        action=TrackExplicitAction,
232
        const=True,
KonradSzafer's avatar
KonradSzafer committed
233
        default=False,
234
235
236
237
238
239
        help=(
            "If True, apply chat template to the prompt. "
            "Providing `--apply_chat_template` without an argument will apply the default chat template to the prompt. "
            "To apply a specific template from the available list of templates, provide the template name as an argument. "
            "E.g. `--apply_chat_template template_name`"
        ),
KonradSzafer's avatar
KonradSzafer committed
240
241
242
    )
    parser.add_argument(
        "--fewshot_as_multiturn",
243
        action=TrackExplicitStoreTrue,
KonradSzafer's avatar
KonradSzafer committed
244
245
246
        default=False,
        help="If True, uses the fewshot as a multi-turn conversation",
    )
247
248
    parser.add_argument(
        "--show_config",
249
        action=TrackExplicitStoreTrue,
250
251
252
        default=False,
        help="If True, shows the the full config of all tasks at the end of the evaluation.",
    )
253
254
255
256
    parser.add_argument(
        "--include_path",
        type=str,
        default=None,
257
        action=TrackExplicitAction,
258
        metavar="DIR",
259
260
        help="Additional path to include if there are external tasks to include.",
    )
261
262
    parser.add_argument(
        "--gen_kwargs",
Baber Abbasi's avatar
Baber Abbasi committed
263
        type=try_parse_json,
264
        default=None,
265
        action=TrackExplicitAction,
USVSN Sai Prashanth's avatar
USVSN Sai Prashanth committed
266
        help=(
Baber Abbasi's avatar
Baber Abbasi committed
267
268
            "Either comma delimited string or JSON formatted arguments for model generation on greedy_until tasks,"
            """ e.g. '{"temperature":0.7,"until":["hello"]}' or temperature=0,top_p=0.1."""
lintangsutawika's avatar
lintangsutawika committed
269
270
271
        ),
    )
    parser.add_argument(
lintangsutawika's avatar
lintangsutawika committed
272
        "--verbosity",
Baber Abbasi's avatar
Baber Abbasi committed
273
274
        "-v",
        type=str.upper,
Lintang Sutawika's avatar
Lintang Sutawika committed
275
        default=None,
276
        action=TrackExplicitAction,
277
        metavar="CRITICAL|ERROR|WARNING|INFO|DEBUG",
Lintang Sutawika's avatar
Lintang Sutawika committed
278
        help="(Deprecated) Controls logging verbosity level. Use the `LOGLEVEL` environment variable instead. Set to DEBUG for detailed output when testing or adding new task configurations.",
279
    )
280
281
    parser.add_argument(
        "--wandb_args",
282
        type=str,
283
        default="",
284
        action=TrackExplicitAction,
285
286
        help="Comma separated string arguments passed to wandb.init, e.g. `project=lm-eval,job_type=eval",
    )
287
288
289
290
    parser.add_argument(
        "--wandb_config_args",
        type=str,
        default="",
291
        action=TrackExplicitAction,
292
293
        help="Comma separated string arguments passed to wandb.config.update. Use this to trace parameters that aren't already traced by default. eg. `lr=0.01,repeats=3",
    )
294
295
296
297
    parser.add_argument(
        "--hf_hub_log_args",
        type=str,
        default="",
298
        action=TrackExplicitAction,
299
300
        help="Comma separated string arguments passed to Hugging Face Hub's log function, e.g. `hub_results_org=EleutherAI,hub_repo_name=lm-eval-results`",
    )
Baber Abbasi's avatar
Baber Abbasi committed
301
302
303
    parser.add_argument(
        "--predict_only",
        "-x",
304
        action=TrackExplicitStoreTrue,
Baber Abbasi's avatar
Baber Abbasi committed
305
306
307
        default=False,
        help="Use with --log_samples. Only model outputs will be saved and metrics will not be evaluated.",
    )
308
    default_seed_string = "0,1234,1234,1234"
309
310
    parser.add_argument(
        "--seed",
311
        type=partial(_int_or_none_list_arg_type, 3, 4, default_seed_string),
312
        action=TrackExplicitAction,
313
        default=default_seed_string,  # for backward compatibility
314
        help=(
315
316
            "Set seed for python's random, numpy, torch, and fewshot sampling.\n"
            "Accepts a comma-separated list of 4 values for python's random, numpy, torch, and fewshot sampling seeds, "
Sadra Barikbin's avatar
Sadra Barikbin committed
317
            "respectively, or a single integer to set the same seed for all four.\n"
318
319
320
321
322
            f"The values are either an integer or 'None' to not set the seed. Default is `{default_seed_string}` "
            "(for backward compatibility).\n"
            "E.g. `--seed 0,None,8,52` sets `random.seed(0)`, `torch.manual_seed(8)`, and fewshot sampling seed to 52. "
            "Here numpy's seed is not set since the second value is `None`.\n"
            "E.g, `--seed 42` sets all four seeds to 42."
323
324
        ),
    )
325
326
    parser.add_argument(
        "--trust_remote_code",
327
        action=TrackExplicitStoreTrue,
328
329
        help="Sets trust_remote_code to True to execute code to create HF Datasets from the Hub",
    )
Hojin Lee's avatar
Hojin Lee committed
330
331
    parser.add_argument(
        "--confirm_run_unsafe_code",
332
        action=TrackExplicitStoreTrue,
Hojin Lee's avatar
Hojin Lee committed
333
334
        help="Confirm that you understand the risks of running unsafe code for tasks that require it",
    )
Baber Abbasi's avatar
Baber Abbasi committed
335
336
337
338
    parser.add_argument(
        "--metadata",
        type=json.loads,
        default=None,
339
        action=TrackExplicitAction,
Baber Abbasi's avatar
Baber Abbasi committed
340
341
        help="""JSON string metadata to pass to task configs, for example '{"max_seq_lengths":[4096,8192]}'. Will be merged with model_args. Can also be set in task config.""",
    )
342
343
344
345
346
    return parser


def parse_eval_args(parser: argparse.ArgumentParser) -> argparse.Namespace:
    check_argument_types(parser)
Jason Phang's avatar
Jason Phang committed
347
348
    return parser.parse_args()

Fabrizio Milo's avatar
Fabrizio Milo committed
349

haileyschoelkopf's avatar
haileyschoelkopf committed
350
351
352
def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
    if not args:
        # we allow for args to be passed externally, else we parse them ourselves
353
354
        parser = setup_parser()
        args = parse_eval_args(parser)
haileyschoelkopf's avatar
haileyschoelkopf committed
355

356
357
358
359
360
361
362
363
    # get namespace from console, including config arg
    config, non_default_args = parse_namespace(args)

    # if config is passed, load it
    if config.get("config", False):
        local_config = load_yaml_config(yaml_path=config["config"])
        config = non_default_update(config, local_config, non_default_args)

364
    if args.wandb_args:
365
        wandb_logger = WandbLogger(config["wandb_args"], config["wandb_config_args"])
366

367
368
    utils.setup_logging(config["verbosity"])
    # utils.setup_logging(args.verbosity)
Lintang Sutawika's avatar
Lintang Sutawika committed
369
    eval_logger = logging.getLogger(__name__)
haileyschoelkopf's avatar
haileyschoelkopf committed
370
    os.environ["TOKENIZERS_PARALLELISM"] = "false"
Fabrizio Milo's avatar
Fabrizio Milo committed
371

372
    # update the evaluation tracker args with the output path and the HF token
373
374
375
    if config["output_path"]:
        config.setdefault("hf_hub_log_args", {})["output_path"] = config["output_path"]

376
    if os.environ.get("HF_TOKEN", None):
377
378
        config.setdefault("hf_hub_log_args", {})["token"] = os.environ.get("HF_TOKEN")
    evaluation_tracker_args = config["hf_hub_log_args"]
379
380
    evaluation_tracker = EvaluationTracker(**evaluation_tracker_args)

381
382
383
384
    if config["predict_only"]:
        config["log_samples"] = True

    if (config["log_samples"] or config["predict_only"]) and not config["output_path"]:
385
386
387
        raise ValueError(
            "Specify --output_path if providing --log_samples or --predict_only"
        )
Baber Abbasi's avatar
Baber Abbasi committed
388

389
    if config["fewshot_as_multiturn"] and config["apply_chat_template"] is False:
KonradSzafer's avatar
KonradSzafer committed
390
        raise ValueError(
391
            "When `fewshot_as_multiturn` is selected, `apply_chat_template` must be set (either to `True` or to the chosen template name)."
KonradSzafer's avatar
KonradSzafer committed
392
393
        )

394
395
396
397
    if config["include_path"] is not None:
        eval_logger.info(f"Including path: {config['include_path']}")

    metadata = (config["model_args"]) | (config["metadata"])
Baber Abbasi's avatar
Baber Abbasi committed
398

399
    task_manager = TaskManager(include_path=config["include_path"], metadata=metadata)
Fabrizio Milo's avatar
Fabrizio Milo committed
400

401
    if "push_samples_to_hub" in evaluation_tracker_args and not config["log_samples"]:
402
403
404
405
        eval_logger.warning(
            "Pushing samples to the Hub requires --log_samples to be set. Samples will not be pushed to the Hub."
        )

406
    if config["limit"]:
lintangsutawika's avatar
lintangsutawika committed
407
408
409
        eval_logger.warning(
            " --limit SHOULD ONLY BE USED FOR TESTING."
            "REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT."
Fabrizio Milo's avatar
Fabrizio Milo committed
410
        )
411
412
    if config["samples"]:
        assert config["limit"] is None, (
413
414
            "If --samples is not None, then --limit must be None."
        )
415
416
        if (samples := Path(config["samples"])).is_file():
            config["samples"] = json.loads(samples.read_text())
417
        else:
418
            config["samples"] = json.loads(config["samples"])
lintangsutawika's avatar
lintangsutawika committed
419

420
    if config["tasks"] is None:
421
422
        eval_logger.error("Need to specify task to evaluate.")
        sys.exit()
423
    elif config["tasks"] == "list":
424
425
        print(task_manager.list_all_tasks())
        sys.exit()
426
    elif config["tasks"] == "list_groups":
427
428
        print(task_manager.list_all_tasks(list_subtasks=False, list_tags=False))
        sys.exit()
429
    elif config["tasks"] == "list_tags":
430
431
        print(task_manager.list_all_tasks(list_groups=False, list_subtasks=False))
        sys.exit()
432
    elif config["tasks"] == "list_subtasks":
433
        print(task_manager.list_all_tasks(list_groups=False, list_tags=False))
Lintang Sutawika's avatar
Lintang Sutawika committed
434
        sys.exit()
Jason Phang's avatar
Jason Phang committed
435
    else:
436
        if os.path.isdir(config["tasks"]):
437
            import glob
438
439

            task_names = []
440
            yaml_path = os.path.join(config["tasks"], "*.yaml")
441
            for yaml_file in glob.glob(yaml_path):
lintangsutawika's avatar
lintangsutawika committed
442
                config = utils.load_yaml_config(yaml_file)
443
444
                task_names.append(config)
        else:
445
            task_list = config["tasks"].split(",")
446
447
            task_names = task_manager.match_tasks(task_list)
            for task in [task for task in task_list if task not in task_names]:
448
                if os.path.isfile(task):
lintangsutawika's avatar
lintangsutawika committed
449
                    config = utils.load_yaml_config(task)
450
                    task_names.append(config)
451
            task_missing = [
452
                task for task in task_list if task not in task_names and "*" not in task
453
            ]  # we don't want errors if a wildcard ("*") task name was used
lintangsutawika's avatar
lintangsutawika committed
454

baberabb's avatar
baberabb committed
455
456
457
458
            if task_missing:
                missing = ", ".join(task_missing)
                eval_logger.error(
                    f"Tasks were not found: {missing}\n"
lintangsutawika's avatar
lintangsutawika committed
459
                    f"{utils.SPACING}Try `lm-eval --tasks list` for list of available tasks",
baberabb's avatar
baberabb committed
460
461
                )
                raise ValueError(
462
                    f"Tasks not found: {missing}. Try `lm-eval --tasks {{list_groups,list_subtasks,list_tags,list}}` to list out all available names for task groupings; only (sub)tasks; tags; or all of the above, or pass '--verbosity DEBUG' to troubleshoot task registration issues."
baberabb's avatar
baberabb committed
463
                )
lintangsutawika's avatar
lintangsutawika committed
464

465
    # Respect user's value passed in via CLI, otherwise default to True and add to comma-separated model args
466
    if config["trust_remote_code"]:
467
468
        eval_logger.info(
            "Passed `--trust_remote_code`, setting environment variable `HF_DATASETS_TRUST_REMOTE_CODE=true`"
469
        )
470
471
472
473
474
475
476
        # HACK: import datasets and override its HF_DATASETS_TRUST_REMOTE_CODE value internally,
        # because it's already been determined based on the prior env var before launching our
        # script--`datasets` gets imported by lm_eval internally before these lines can update the env.
        import datasets

        datasets.config.HF_DATASETS_TRUST_REMOTE_CODE = True

477
        config.setdefault("model_args", {})["trust_remote_code"] = True
478
479
480
481
    (
        eval_logger.info(f"Selected Tasks: {task_names}")
        if eval_logger.getEffectiveLevel() >= logging.INFO
        else print(f"Selected Tasks: {task_names}")
Baber Abbasi's avatar
Baber Abbasi committed
482
    )
483

484
    request_caching_args = request_caching_arg_to_dict(
485
        cache_requests=config["cache_requests"]
486
487
    )

488
    results = evaluator.simple_evaluate(
489
490
        model=config["model"],
        model_args=config["model_args"],
491
        tasks=task_names,
492
493
494
495
496
497
498
499
500
501
        num_fewshot=config["num_fewshot"],
        batch_size=config["batch_size"],
        max_batch_size=config["max_batch_size"],
        device=config["device"],
        use_cache=config["use_cache"],
        limit=config["limit"],
        samples=config["samples"],
        check_integrity=config["check_integrity"],
        write_out=config["write_out"],
        log_samples=config["log_samples"],
KonradSzafer's avatar
KonradSzafer committed
502
        evaluation_tracker=evaluation_tracker,
503
504
505
506
        system_instruction=config["system_instruction"],
        apply_chat_template=config["apply_chat_template"],
        fewshot_as_multiturn=config["fewshot_as_multiturn"],
        gen_kwargs=config["gen_kwargs"],
507
        task_manager=task_manager,
508
509
510
511
512
513
        predict_only=config["predict_only"],
        random_seed=config["seed"][0],
        numpy_random_seed=config["seed"][1],
        torch_random_seed=config["seed"][2],
        fewshot_random_seed=config["seed"][3],
        confirm_run_unsafe_code=config["confirm_run_unsafe_code"],
Baber Abbasi's avatar
Baber Abbasi committed
514
        metadata=metadata,
515
        **request_caching_args,
516
    )
517

518
    if results is not None:
519
        if config["log_samples"]:
520
            samples = results.pop("samples")
521
        dumped = json.dumps(
522
            results, indent=2, default=handle_non_serializable, ensure_ascii=False
523
        )
524
        if config["show_config"]:
525
            print(dumped)
526

527
528
        batch_sizes = ",".join(map(str, results["config"]["batch_sizes"]))

529
        # Add W&B logging
530
        if config["wandb_args"]:
531
532
533
            try:
                wandb_logger.post_init(results)
                wandb_logger.log_eval_result()
534
                if config["log_samples"]:
535
536
537
538
                    wandb_logger.log_eval_samples(samples)
            except Exception as e:
                eval_logger.info(f"Logging to Weights and Biases failed due to {e}")

KonradSzafer's avatar
KonradSzafer committed
539
540
541
        evaluation_tracker.save_results_aggregated(
            results=results, samples=samples if args.log_samples else None
        )
542

543
544
        if config["log_samples"]:
            for task_name, _ in results["configs"].items():
545
546
547
                evaluation_tracker.save_results_samples(
                    task_name=task_name, samples=samples[task_name]
                )
lintangsutawika's avatar
lintangsutawika committed
548

549
550
551
552
553
554
        if (
            evaluation_tracker.push_results_to_hub
            or evaluation_tracker.push_samples_to_hub
        ):
            evaluation_tracker.recreate_metadata_card()

555
        print(
556
557
            f"{config['model']} ({config['model_args']}), gen_kwargs: ({config['gen_kwargs']}), limit: {config['limit']}, num_fewshot: {config['num_fewshot']}, "
            f"batch_size: {config['batch_size']}{f' ({batch_sizes})' if batch_sizes else ''}"
558
        )
559
        print(make_table(results))
lintangsutawika's avatar
lintangsutawika committed
560
        if "groups" in results:
561
            print(make_table(results, "groups"))
Jason Phang's avatar
lib  
Jason Phang committed
562

563
        if config["wandb_args"]:
564
565
566
            # Tear down wandb run once all the logging is done.
            wandb_logger.run.finish()

567

Jason Phang's avatar
Jason Phang committed
568
if __name__ == "__main__":
haileyschoelkopf's avatar
haileyschoelkopf committed
569
    cli_evaluate()