__main__.py 19.6 KB
Newer Older
1
2
3
import argparse
import json
import logging
lintangsutawika's avatar
lintangsutawika committed
4
import os
5
import sys
6
from functools import partial
7
from pathlib import Path
haileyschoelkopf's avatar
haileyschoelkopf committed
8
from typing import Union
Leo Gao's avatar
Leo Gao committed
9

10
from lm_eval import evaluator, utils
artemorloff's avatar
artemorloff committed
11
12
13
14
15
16
from lm_eval.api.eval_config import (
    EvaluationConfig,
    TrackExplicitAction,
    TrackExplicitStoreTrue,
)

artemorloff's avatar
artemorloff committed
17
# from lm_eval.evaluator import request_caching_arg_to_dict
18
from lm_eval.loggers import EvaluationTracker, WandbLogger
19
from lm_eval.tasks import TaskManager
Baber Abbasi's avatar
Baber Abbasi committed
20
21
22
from lm_eval.utils import (
    handle_non_serializable,
    make_table,
artemorloff's avatar
artemorloff committed
23
24
25
26
    request_caching_arg_to_dict,
    # non_default_update,
    # parse_namespace,
)
Baber Abbasi's avatar
Baber Abbasi committed
27
28
29
30
31
32
33
34
35
36
37
38
39


def try_parse_json(value: str) -> Union[str, dict, None]:
    if value is None:
        return None
    try:
        return json.loads(value)
    except json.JSONDecodeError:
        if "{" in value:
            raise argparse.ArgumentTypeError(
                f"Invalid JSON: {value}. Hint: Use double quotes for JSON strings."
            )
        return value
Fabrizio Milo's avatar
Fabrizio Milo committed
40

41

42
43
44
def _int_or_none_list_arg_type(
    min_len: int, max_len: int, defaults: str, value: str, split_char: str = ","
):
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
    def parse_value(item):
        item = item.strip().lower()
        if item == "none":
            return None
        try:
            return int(item)
        except ValueError:
            raise argparse.ArgumentTypeError(f"{item} is not an integer or None")

    items = [parse_value(v) for v in value.split(split_char)]
    num_items = len(items)

    if num_items == 1:
        # Makes downstream handling the same for single and multiple values
        items = items * max_len
60
    elif num_items < min_len or num_items > max_len:
61
62
63
        raise argparse.ArgumentTypeError(
            f"Argument requires {max_len} integers or None, separated by '{split_char}'"
        )
64
65
66
67
68
69
70
71
72
    elif num_items != max_len:
        logging.warning(
            f"Argument requires {max_len} integers or None, separated by '{split_char}'. "
            "Missing values will be filled with defaults."
        )
        default_items = [parse_value(v) for v in defaults.split(split_char)]
        items.extend(
            default_items[num_items:]
        )  # extend items list with missing defaults
73
74
75
76

    return items


77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
def check_argument_types(parser: argparse.ArgumentParser):
    """
    Check to make sure all CLI args are typed, raises error if not
    """
    for action in parser._actions:
        if action.dest != "help" and not action.const:
            if action.type is None:
                raise ValueError(
                    f"Argument '{action.dest}' doesn't have a type specified."
                )
            else:
                continue


def setup_parser() -> argparse.ArgumentParser:
lintangsutawika's avatar
lintangsutawika committed
92
    parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
93
    parser.add_argument(
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
        "--config",
        "-C",
        default=None,
        type=str,
        metavar="DIR/file.yaml",
        action=TrackExplicitAction,
        help="Path to config with all arguments for `lm-eval`",
    )
    parser.add_argument(
        "--model",
        "-m",
        type=str,
        default="hf",
        action=TrackExplicitAction,
        help="Name of model e.g. `hf`",
109
    )
lintangsutawika's avatar
lintangsutawika committed
110
111
    parser.add_argument(
        "--tasks",
Baber Abbasi's avatar
Baber Abbasi committed
112
        "-t",
lintangsutawika's avatar
lintangsutawika committed
113
        default=None,
114
        type=str,
115
        action=TrackExplicitAction,
116
        metavar="task1,task2",
117
        help="Comma-separated list of task names or task groupings to evaluate on.\nTo get full list of tasks, use one of the commands `lm-eval --tasks {{list_groups,list_subtasks,list_tags,list}}` to list out all available names for task groupings; only (sub)tasks; tags; or all of the above",
lintangsutawika's avatar
lintangsutawika committed
118
    )
119
120
    parser.add_argument(
        "--model_args",
Baber Abbasi's avatar
Baber Abbasi committed
121
        "-a",
122
        default="",
123
        action=TrackExplicitAction,
Baber Abbasi's avatar
Baber Abbasi committed
124
125
        type=try_parse_json,
        help="""Comma separated string or JSON formatted arguments for model, e.g. `pretrained=EleutherAI/pythia-160m,dtype=float32` or '{"pretrained":"EleutherAI/pythia-160m","dtype":"float32"}'""",
126
    )
lintangsutawika's avatar
lintangsutawika committed
127
    parser.add_argument(
128
        "--num_fewshot",
Baber Abbasi's avatar
Baber Abbasi committed
129
        "-f",
130
        type=int,
131
        default=None,
132
        action=TrackExplicitAction,
133
        metavar="N",
134
135
        help="Number of examples in few-shot context",
    )
136
137
    parser.add_argument(
        "--batch_size",
Baber Abbasi's avatar
Baber Abbasi committed
138
        "-b",
139
        type=str,
140
        action=TrackExplicitAction,
141
142
143
144
        default=1,
        metavar="auto|auto:N|N",
        help="Acceptable values are 'auto', 'auto:N' or N, where N is an integer. Default 1.",
    )
lintangsutawika's avatar
lintangsutawika committed
145
146
147
148
    parser.add_argument(
        "--max_batch_size",
        type=int,
        default=None,
149
        action=TrackExplicitAction,
150
151
        metavar="N",
        help="Maximal batch size to try with --batch_size auto.",
lintangsutawika's avatar
lintangsutawika committed
152
    )
153
154
155
156
    parser.add_argument(
        "--device",
        type=str,
        default=None,
157
        action=TrackExplicitAction,
158
        help="Device to use (e.g. cuda, cuda:0, cpu).",
159
160
161
    )
    parser.add_argument(
        "--output_path",
Baber Abbasi's avatar
Baber Abbasi committed
162
        "-o",
163
164
        default=None,
        type=str,
165
        action=TrackExplicitAction,
166
        metavar="DIR|DIR/file.json",
167
        help="The path to the output file where the result metrics will be saved. If the path is a directory and log_samples is true, the results will be saved in the directory. Else the parent directory will be used.",
168
    )
lintangsutawika's avatar
lintangsutawika committed
169
170
    parser.add_argument(
        "--limit",
Baber Abbasi's avatar
Baber Abbasi committed
171
        "-L",
lintangsutawika's avatar
lintangsutawika committed
172
173
        type=float,
        default=None,
174
        action=TrackExplicitAction,
175
        metavar="N|0<N<1",
lintangsutawika's avatar
lintangsutawika committed
176
177
178
        help="Limit the number of examples per task. "
        "If <1, limit is a percentage of the total number of examples.",
    )
179
180
181
182
183
    parser.add_argument(
        "--samples",
        "-E",
        default=None,
        type=str,
184
        action=TrackExplicitAction,
185
186
187
        metavar="/path/to/json",
        help='JSON string or path to JSON file containing doc indices of selected examples to test. Format: {"task_name":[indices],...}',
    )
188
189
    parser.add_argument(
        "--use_cache",
Baber Abbasi's avatar
Baber Abbasi committed
190
        "-c",
191
        type=str,
192
        action=TrackExplicitAction,
193
        default=None,
194
        metavar="DIR",
195
196
        help="A path to a sqlite db file for caching model responses. `None` if not caching.",
    )
197
198
199
200
    parser.add_argument(
        "--cache_requests",
        type=str,
        default=None,
201
        action=TrackExplicitAction,
202
203
204
        choices=["true", "refresh", "delete"],
        help="Speed up evaluation by caching the building of dataset requests. `None` if not caching.",
    )
205
206
    parser.add_argument(
        "--check_integrity",
207
        action=TrackExplicitStoreTrue,
208
        help="Whether to run the relevant part of the test suite for the tasks.",
209
210
211
    )
    parser.add_argument(
        "--write_out",
Baber Abbasi's avatar
Baber Abbasi committed
212
        "-w",
213
        action=TrackExplicitStoreTrue,
214
        default=False,
215
        help="Prints the prompt for the first few documents.",
216
217
218
    )
    parser.add_argument(
        "--log_samples",
Baber Abbasi's avatar
Baber Abbasi committed
219
        "-s",
220
        action=TrackExplicitStoreTrue,
221
        default=False,
222
        help="If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis. Use with --output_path.",
223
    )
KonradSzafer's avatar
KonradSzafer committed
224
225
226
227
    parser.add_argument(
        "--system_instruction",
        type=str,
        default=None,
228
        action=TrackExplicitAction,
KonradSzafer's avatar
KonradSzafer committed
229
230
231
232
        help="System instruction to be used in the prompt",
    )
    parser.add_argument(
        "--apply_chat_template",
233
234
        type=str,
        nargs="?",
235
        action=TrackExplicitAction,
236
        const=True,
KonradSzafer's avatar
KonradSzafer committed
237
        default=False,
238
239
240
241
242
243
        help=(
            "If True, apply chat template to the prompt. "
            "Providing `--apply_chat_template` without an argument will apply the default chat template to the prompt. "
            "To apply a specific template from the available list of templates, provide the template name as an argument. "
            "E.g. `--apply_chat_template template_name`"
        ),
KonradSzafer's avatar
KonradSzafer committed
244
245
246
    )
    parser.add_argument(
        "--fewshot_as_multiturn",
247
        action=TrackExplicitStoreTrue,
KonradSzafer's avatar
KonradSzafer committed
248
249
250
        default=False,
        help="If True, uses the fewshot as a multi-turn conversation",
    )
251
252
    parser.add_argument(
        "--show_config",
253
        action=TrackExplicitStoreTrue,
254
255
256
        default=False,
        help="If True, shows the the full config of all tasks at the end of the evaluation.",
    )
257
258
259
260
    parser.add_argument(
        "--include_path",
        type=str,
        default=None,
261
        action=TrackExplicitAction,
262
        metavar="DIR",
263
264
        help="Additional path to include if there are external tasks to include.",
    )
265
266
    parser.add_argument(
        "--gen_kwargs",
Baber Abbasi's avatar
Baber Abbasi committed
267
        type=try_parse_json,
268
        default=None,
269
        action=TrackExplicitAction,
USVSN Sai Prashanth's avatar
USVSN Sai Prashanth committed
270
        help=(
Baber Abbasi's avatar
Baber Abbasi committed
271
272
            "Either comma delimited string or JSON formatted arguments for model generation on greedy_until tasks,"
            """ e.g. '{"temperature":0.7,"until":["hello"]}' or temperature=0,top_p=0.1."""
lintangsutawika's avatar
lintangsutawika committed
273
274
275
        ),
    )
    parser.add_argument(
lintangsutawika's avatar
lintangsutawika committed
276
        "--verbosity",
Baber Abbasi's avatar
Baber Abbasi committed
277
278
        "-v",
        type=str.upper,
Lintang Sutawika's avatar
Lintang Sutawika committed
279
        default=None,
280
        action=TrackExplicitAction,
281
        metavar="CRITICAL|ERROR|WARNING|INFO|DEBUG",
Lintang Sutawika's avatar
Lintang Sutawika committed
282
        help="(Deprecated) Controls logging verbosity level. Use the `LOGLEVEL` environment variable instead. Set to DEBUG for detailed output when testing or adding new task configurations.",
283
    )
284
285
    parser.add_argument(
        "--wandb_args",
286
        type=str,
287
        default="",
288
        action=TrackExplicitAction,
289
290
        help="Comma separated string arguments passed to wandb.init, e.g. `project=lm-eval,job_type=eval",
    )
291
292
293
294
    parser.add_argument(
        "--wandb_config_args",
        type=str,
        default="",
295
        action=TrackExplicitAction,
296
297
        help="Comma separated string arguments passed to wandb.config.update. Use this to trace parameters that aren't already traced by default. eg. `lr=0.01,repeats=3",
    )
298
299
300
301
    parser.add_argument(
        "--hf_hub_log_args",
        type=str,
        default="",
302
        action=TrackExplicitAction,
303
304
        help="Comma separated string arguments passed to Hugging Face Hub's log function, e.g. `hub_results_org=EleutherAI,hub_repo_name=lm-eval-results`",
    )
Baber Abbasi's avatar
Baber Abbasi committed
305
306
307
    parser.add_argument(
        "--predict_only",
        "-x",
308
        action=TrackExplicitStoreTrue,
Baber Abbasi's avatar
Baber Abbasi committed
309
310
311
        default=False,
        help="Use with --log_samples. Only model outputs will be saved and metrics will not be evaluated.",
    )
312
    default_seed_string = "0,1234,1234,1234"
313
314
    parser.add_argument(
        "--seed",
315
        type=partial(_int_or_none_list_arg_type, 3, 4, default_seed_string),
316
        action=TrackExplicitAction,
317
        default=default_seed_string,  # for backward compatibility
318
        help=(
319
320
            "Set seed for python's random, numpy, torch, and fewshot sampling.\n"
            "Accepts a comma-separated list of 4 values for python's random, numpy, torch, and fewshot sampling seeds, "
Sadra Barikbin's avatar
Sadra Barikbin committed
321
            "respectively, or a single integer to set the same seed for all four.\n"
322
323
324
325
326
            f"The values are either an integer or 'None' to not set the seed. Default is `{default_seed_string}` "
            "(for backward compatibility).\n"
            "E.g. `--seed 0,None,8,52` sets `random.seed(0)`, `torch.manual_seed(8)`, and fewshot sampling seed to 52. "
            "Here numpy's seed is not set since the second value is `None`.\n"
            "E.g, `--seed 42` sets all four seeds to 42."
327
328
        ),
    )
329
330
    parser.add_argument(
        "--trust_remote_code",
331
        action=TrackExplicitStoreTrue,
332
333
        help="Sets trust_remote_code to True to execute code to create HF Datasets from the Hub",
    )
Hojin Lee's avatar
Hojin Lee committed
334
335
    parser.add_argument(
        "--confirm_run_unsafe_code",
336
        action=TrackExplicitStoreTrue,
Hojin Lee's avatar
Hojin Lee committed
337
338
        help="Confirm that you understand the risks of running unsafe code for tasks that require it",
    )
Baber Abbasi's avatar
Baber Abbasi committed
339
340
341
342
    parser.add_argument(
        "--metadata",
        type=json.loads,
        default=None,
343
        action=TrackExplicitAction,
Baber Abbasi's avatar
Baber Abbasi committed
344
345
        help="""JSON string metadata to pass to task configs, for example '{"max_seq_lengths":[4096,8192]}'. Will be merged with model_args. Can also be set in task config.""",
    )
346
347
348
349
350
    return parser


def parse_eval_args(parser: argparse.ArgumentParser) -> argparse.Namespace:
    check_argument_types(parser)
Jason Phang's avatar
Jason Phang committed
351
352
    return parser.parse_args()

Fabrizio Milo's avatar
Fabrizio Milo committed
353

haileyschoelkopf's avatar
haileyschoelkopf committed
354
355
356
def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
    if not args:
        # we allow for args to be passed externally, else we parse them ourselves
357
358
        parser = setup_parser()
        args = parse_eval_args(parser)
haileyschoelkopf's avatar
haileyschoelkopf committed
359

artemorloff's avatar
artemorloff committed
360
    config = EvaluationConfig.from_cli(args)
361

362
    if args.wandb_args:
artemorloff's avatar
artemorloff committed
363
        wandb_logger = WandbLogger(config.wandb_args, config.wandb_config_args)
364

artemorloff's avatar
artemorloff committed
365
    utils.setup_logging(config.verbosity)
Lintang Sutawika's avatar
Lintang Sutawika committed
366
    eval_logger = logging.getLogger(__name__)
haileyschoelkopf's avatar
haileyschoelkopf committed
367
    os.environ["TOKENIZERS_PARALLELISM"] = "false"
Fabrizio Milo's avatar
Fabrizio Milo committed
368

369
    # update the evaluation tracker args with the output path and the HF token
artemorloff's avatar
artemorloff committed
370
371
    if config.output_path:
        config.hf_hub_log_args["output_path"] = config.output_path
372

373
    if os.environ.get("HF_TOKEN", None):
artemorloff's avatar
artemorloff committed
374
375
376
        config.hf_hub_log_args["token"] = os.environ.get("HF_TOKEN")

    evaluation_tracker_args = config.hf_hub_log_args
377
378
    evaluation_tracker = EvaluationTracker(**evaluation_tracker_args)

artemorloff's avatar
artemorloff committed
379
380
    if config.predict_only:
        config.log_samples = True
381

artemorloff's avatar
artemorloff committed
382
    if (config.log_samples or config.predict_only) and not config.output_path:
383
384
385
        raise ValueError(
            "Specify --output_path if providing --log_samples or --predict_only"
        )
Baber Abbasi's avatar
Baber Abbasi committed
386

artemorloff's avatar
artemorloff committed
387
    if config.fewshot_as_multiturn and config.apply_chat_template is False:
KonradSzafer's avatar
KonradSzafer committed
388
        raise ValueError(
389
            "When `fewshot_as_multiturn` is selected, `apply_chat_template` must be set (either to `True` or to the chosen template name)."
KonradSzafer's avatar
KonradSzafer committed
390
391
        )

artemorloff's avatar
artemorloff committed
392
393
    if config.include_path is not None:
        eval_logger.info(f"Including path: {config.include_path}")
394

artemorloff's avatar
artemorloff committed
395
396
    metadata = (config.model_args) | (config.metadata)
    config.metadata = metadata
Baber Abbasi's avatar
Baber Abbasi committed
397

artemorloff's avatar
artemorloff committed
398
399
    # task_manager = TaskManager(include_path=config["include_path"], metadata=metadata)
    task_manager = TaskManager(include_path=config.include_path, metadata=metadata)
Fabrizio Milo's avatar
Fabrizio Milo committed
400

artemorloff's avatar
artemorloff committed
401
    if "push_samples_to_hub" in evaluation_tracker_args and not config.log_samples:
402
403
404
405
        eval_logger.warning(
            "Pushing samples to the Hub requires --log_samples to be set. Samples will not be pushed to the Hub."
        )

artemorloff's avatar
artemorloff committed
406
    if config.limit:
lintangsutawika's avatar
lintangsutawika committed
407
408
409
        eval_logger.warning(
            " --limit SHOULD ONLY BE USED FOR TESTING."
            "REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT."
Fabrizio Milo's avatar
Fabrizio Milo committed
410
        )
artemorloff's avatar
artemorloff committed
411
412
413

    if config.samples:
        assert config.limit is None, (
414
415
            "If --samples is not None, then --limit must be None."
        )
artemorloff's avatar
artemorloff committed
416
417
        if (samples := Path(config.samples)).is_file():
            config.samples = json.loads(samples.read_text())
418
        else:
artemorloff's avatar
artemorloff committed
419
            config.samples = json.loads(config.samples)
lintangsutawika's avatar
lintangsutawika committed
420

artemorloff's avatar
artemorloff committed
421
    if config.tasks is None:
422
423
        eval_logger.error("Need to specify task to evaluate.")
        sys.exit()
artemorloff's avatar
artemorloff committed
424
    elif config.tasks == "list":
425
426
        print(task_manager.list_all_tasks())
        sys.exit()
artemorloff's avatar
artemorloff committed
427
    elif config.tasks == "list_groups":
428
429
        print(task_manager.list_all_tasks(list_subtasks=False, list_tags=False))
        sys.exit()
artemorloff's avatar
artemorloff committed
430
    elif config.tasks == "list_tags":
431
432
        print(task_manager.list_all_tasks(list_groups=False, list_subtasks=False))
        sys.exit()
artemorloff's avatar
artemorloff committed
433
    elif config.tasks == "list_subtasks":
434
        print(task_manager.list_all_tasks(list_groups=False, list_tags=False))
Lintang Sutawika's avatar
Lintang Sutawika committed
435
        sys.exit()
Jason Phang's avatar
Jason Phang committed
436
    else:
artemorloff's avatar
artemorloff committed
437
        if os.path.isdir(config.tasks):
438
            import glob
439
440

            task_names = []
artemorloff's avatar
artemorloff committed
441
            yaml_path = os.path.join(config.tasks, "*.yaml")
442
            for yaml_file in glob.glob(yaml_path):
lintangsutawika's avatar
lintangsutawika committed
443
                config = utils.load_yaml_config(yaml_file)
444
445
                task_names.append(config)
        else:
artemorloff's avatar
artemorloff committed
446
            task_list = config.tasks.split(",")
447
448
            task_names = task_manager.match_tasks(task_list)
            for task in [task for task in task_list if task not in task_names]:
449
                if os.path.isfile(task):
lintangsutawika's avatar
lintangsutawika committed
450
                    config = utils.load_yaml_config(task)
451
                    task_names.append(config)
452
            task_missing = [
453
                task for task in task_list if task not in task_names and "*" not in task
454
            ]  # we don't want errors if a wildcard ("*") task name was used
lintangsutawika's avatar
lintangsutawika committed
455

baberabb's avatar
baberabb committed
456
457
458
459
            if task_missing:
                missing = ", ".join(task_missing)
                eval_logger.error(
                    f"Tasks were not found: {missing}\n"
lintangsutawika's avatar
lintangsutawika committed
460
                    f"{utils.SPACING}Try `lm-eval --tasks list` for list of available tasks",
baberabb's avatar
baberabb committed
461
462
                )
                raise ValueError(
463
                    f"Tasks not found: {missing}. Try `lm-eval --tasks {{list_groups,list_subtasks,list_tags,list}}` to list out all available names for task groupings; only (sub)tasks; tags; or all of the above, or pass '--verbosity DEBUG' to troubleshoot task registration issues."
baberabb's avatar
baberabb committed
464
                )
artemorloff's avatar
artemorloff committed
465
        config.tasks = task_names
lintangsutawika's avatar
lintangsutawika committed
466

467
    # Respect user's value passed in via CLI, otherwise default to True and add to comma-separated model args
artemorloff's avatar
artemorloff committed
468
    if config.trust_remote_code:
469
470
        eval_logger.info(
            "Passed `--trust_remote_code`, setting environment variable `HF_DATASETS_TRUST_REMOTE_CODE=true`"
471
        )
472
473
474
475
476
477
478
        # HACK: import datasets and override its HF_DATASETS_TRUST_REMOTE_CODE value internally,
        # because it's already been determined based on the prior env var before launching our
        # script--`datasets` gets imported by lm_eval internally before these lines can update the env.
        import datasets

        datasets.config.HF_DATASETS_TRUST_REMOTE_CODE = True

artemorloff's avatar
artemorloff committed
479
        config.model_args["trust_remote_code"] = True
480
481
482
483
    (
        eval_logger.info(f"Selected Tasks: {task_names}")
        if eval_logger.getEffectiveLevel() >= logging.INFO
        else print(f"Selected Tasks: {task_names}")
Baber Abbasi's avatar
Baber Abbasi committed
484
    )
485

486
    request_caching_args = request_caching_arg_to_dict(
artemorloff's avatar
artemorloff committed
487
        cache_requests=config.cache_requests
488
    )
artemorloff's avatar
artemorloff committed
489
490
491
    config.request_caching_args = request_caching_args

    print(f"CONFIG_AFTER: {config}")
492

493
    results = evaluator.simple_evaluate(
artemorloff's avatar
artemorloff committed
494
        config=config,
KonradSzafer's avatar
KonradSzafer committed
495
        evaluation_tracker=evaluation_tracker,
496
        task_manager=task_manager,
497
    )
498

499
    if results is not None:
artemorloff's avatar
artemorloff committed
500
        if config.log_samples:
501
            samples = results.pop("samples")
502
        dumped = json.dumps(
503
            results, indent=2, default=handle_non_serializable, ensure_ascii=False
504
        )
artemorloff's avatar
artemorloff committed
505
        if config.show_config:
506
            print(dumped)
507

508
509
        batch_sizes = ",".join(map(str, results["config"]["batch_sizes"]))

510
        # Add W&B logging
artemorloff's avatar
artemorloff committed
511
        if config.wandb_args:
512
513
514
            try:
                wandb_logger.post_init(results)
                wandb_logger.log_eval_result()
artemorloff's avatar
artemorloff committed
515
                if config.log_samples:
516
517
518
519
                    wandb_logger.log_eval_samples(samples)
            except Exception as e:
                eval_logger.info(f"Logging to Weights and Biases failed due to {e}")

KonradSzafer's avatar
KonradSzafer committed
520
521
522
        evaluation_tracker.save_results_aggregated(
            results=results, samples=samples if args.log_samples else None
        )
523

artemorloff's avatar
artemorloff committed
524
        if config.log_samples:
525
            for task_name, _ in results["configs"].items():
526
527
528
                evaluation_tracker.save_results_samples(
                    task_name=task_name, samples=samples[task_name]
                )
lintangsutawika's avatar
lintangsutawika committed
529

530
531
532
533
534
535
        if (
            evaluation_tracker.push_results_to_hub
            or evaluation_tracker.push_samples_to_hub
        ):
            evaluation_tracker.recreate_metadata_card()

536
        print(
artemorloff's avatar
artemorloff committed
537
538
            f"{config.model} ({config.model_args}), gen_kwargs: ({config.gen_kwargs}), limit: {config.limit}, num_fewshot: {config.num_fewshot}, "
            f"batch_size: {config.batch_size}{f' ({batch_sizes})' if batch_sizes else ''}"
539
        )
540
        print(make_table(results))
lintangsutawika's avatar
lintangsutawika committed
541
        if "groups" in results:
542
            print(make_table(results, "groups"))
Jason Phang's avatar
lib  
Jason Phang committed
543

artemorloff's avatar
artemorloff committed
544
        if config.wandb_args:
545
546
547
            # Tear down wandb run once all the logging is done.
            wandb_logger.run.finish()

548

Jason Phang's avatar
Jason Phang committed
549
if __name__ == "__main__":
haileyschoelkopf's avatar
haileyschoelkopf committed
550
    cli_evaluate()