run.py 16.6 KB
Newer Older
Baber's avatar
Baber committed
1
2
3
4
import argparse
import json
import logging
import os
Baber's avatar
cleanup  
Baber committed
5
import textwrap
Baber's avatar
Baber committed
6
7
from functools import partial

Baber's avatar
cleanup  
Baber committed
8
from lm_eval._cli.subcommand import SubCommand
Baber's avatar
Baber committed
9
10
from lm_eval._cli.utils import (
    _int_or_none_list_arg_type,
11
12
    key_val_to_dict,
    merge_dicts,
Baber's avatar
Baber committed
13
14
15
    request_caching_arg_to_dict,
    try_parse_json,
)
Baber's avatar
Baber committed
16
17


Baber's avatar
Baber committed
18
class Run(SubCommand):
Baber's avatar
Baber committed
19
20
21
22
    """Command for running language model evaluation."""

    def __init__(self, subparsers: argparse._SubParsersAction, *args, **kwargs):
        super().__init__(*args, **kwargs)
Baber's avatar
cleanup  
Baber committed
23
        self._parser = subparsers.add_parser(
Baber's avatar
Baber committed
24
            "run",
Baber's avatar
cleanup  
Baber committed
25
            help="Run the evaluation harness on specified tasks",
Baber's avatar
Baber committed
26
            description="Evaluate language models on various benchmarks and tasks.",
27
            usage="lm-eval run --model <model> --tasks <task> <task> --model_args <arg=value> <arg=value> [options]",
Baber's avatar
cleanup  
Baber committed
28
29
30
            epilog=textwrap.dedent("""
                examples:
                  # Basic evaluation with HuggingFace model
31
                  $ lm-eval run --model hf --model_args pretrained=gpt2 dtype=float32 --tasks hellaswag
Baber's avatar
Baber committed
32

Baber's avatar
cleanup  
Baber committed
33
                  # Evaluate on multiple tasks with few-shot examples
34
                  $ lm-eval run --model vllm --model_args pretrained=EleutherAI/gpt-j-6B --tasks arc_easy arc_challenge --num_fewshot 5
Baber's avatar
Baber committed
35

Baber's avatar
cleanup  
Baber committed
36
                  # Evaluation with custom generation parameters
37
                  $ lm-eval run --model hf --model_args pretrained=gpt2 --tasks lambada --gen_kwargs temperature=0.8 top_p=0.95 'stop=["\\n\\n"]'
Baber's avatar
Baber committed
38

Baber's avatar
cleanup  
Baber committed
39
40
                  # Use configuration file
                  $ lm-eval run --config my_config.yaml --tasks mmlu
Baber's avatar
Baber committed
41

Baber's avatar
cleanup  
Baber committed
42
43
                For more information, see: https://github.com/EleutherAI/lm-evaluation-harness
            """),
Baber's avatar
Baber committed
44
45
            formatter_class=argparse.RawDescriptionHelpFormatter,
        )
Baber's avatar
cleanup  
Baber committed
46
        self._add_args()
Baber's avatar
nit  
Baber committed
47
        self._parser.set_defaults(func=self._execute)
Baber's avatar
Baber committed
48

Baber's avatar
cleanup  
Baber committed
49
50
    def _add_args(self) -> None:
        self._parser = self._parser
Baber's avatar
Baber committed
51

Baber's avatar
Baber committed
52
        # Defaults are set in config/evaluate_config.py
Baber's avatar
Baber committed
53
54
        config_group = self._parser.add_argument_group("configuration")
        config_group.add_argument(
Baber's avatar
Baber committed
55
56
57
58
            "--config",
            "-C",
            default=None,
            type=str,
Baber's avatar
Baber committed
59
60
            metavar="YAML_PATH",
            help="Set initial arguments from YAML config",
Baber's avatar
Baber committed
61
        )
Baber's avatar
Baber committed
62
63
64
65

        # Model and Tasks
        model_group = self._parser.add_argument_group("model and tasks")
        model_group.add_argument(
Baber's avatar
Baber committed
66
67
68
            "--model",
            "-m",
            type=str,
Baber's avatar
Baber committed
69
            default=None,
Baber's avatar
Baber committed
70
71
            metavar="MODEL_NAME",
            help="Model name (default: hf)",
Baber's avatar
Baber committed
72
        )
Baber's avatar
Baber committed
73
        model_group.add_argument(
Baber's avatar
Baber committed
74
75
76
77
            "--tasks",
            "-t",
            default=None,
            type=str,
78
79
            nargs="*",
            metavar="TASK1 TASK2",
Baber's avatar
Baber committed
80
            help=textwrap.dedent("""
81
                Space or Comma-separated list of task names or groupings.
Baber's avatar
Baber committed
82
83
                Use 'lm-eval list tasks' to see all available tasks.
            """).strip(),
Baber's avatar
Baber committed
84
        )
Baber's avatar
Baber committed
85
        model_group.add_argument(
Baber's avatar
Baber committed
86
87
            "--model_args",
            "-a",
Baber's avatar
Baber committed
88
            default=None,
89
90
            nargs="*",
            type=key_val_to_dict,
Baber's avatar
Baber committed
91
            metavar="ARGS",
92
            help="Model arguments as 'key=val,key2=val2' or `key=val` `key2=val2`",
Baber's avatar
Baber committed
93
        )
Baber's avatar
Baber committed
94
95
96
97

        # Evaluation Settings
        eval_group = self._parser.add_argument_group("evaluation settings")
        eval_group.add_argument(
Baber's avatar
Baber committed
98
99
100
101
102
103
104
            "--num_fewshot",
            "-f",
            type=int,
            default=None,
            metavar="N",
            help="Number of examples in few-shot context",
        )
Baber's avatar
Baber committed
105
        eval_group.add_argument(
Baber's avatar
Baber committed
106
107
108
            "--batch_size",
            "-b",
            type=str,
Baber's avatar
Baber committed
109
            default=argparse.SUPPRESS,
Baber's avatar
Baber committed
110
            metavar="auto|auto:N|N",
Baber's avatar
Baber committed
111
112
113
            help=textwrap.dedent(
                "Batch size: 'auto', 'auto:N' (auto-tune N times), or integer (default: 1)"
            ),
Baber's avatar
Baber committed
114
        )
Baber's avatar
Baber committed
115
        eval_group.add_argument(
Baber's avatar
Baber committed
116
117
118
119
            "--max_batch_size",
            type=int,
            default=None,
            metavar="N",
Baber's avatar
Baber committed
120
            help="Maximum batch size when using --batch_size auto",
Baber's avatar
Baber committed
121
        )
Baber's avatar
Baber committed
122
        eval_group.add_argument(
Baber's avatar
Baber committed
123
124
125
            "--device",
            type=str,
            default=None,
Baber's avatar
Baber committed
126
127
            metavar="DEVICE",
            help="Device to use (e.g. cuda, cuda:0, cpu, mps)",
Baber's avatar
Baber committed
128
        )
Baber's avatar
Baber committed
129
130
        eval_group.add_argument(
            "--gen_kwargs",
131
            type=key_val_to_dict,
Baber's avatar
Baber committed
132
            default=None,
133
            nargs="*",
Baber's avatar
Baber committed
134
            metavar="KWARGS",
135
            help=textwrap.dedent(
136
137
                'Generation arguments as `temperature=0,stop=["stop"]` or `key=val` `key2=val2`.'
                "Values should be parsable with ast.literal_eval."
138
            ),
Baber's avatar
Baber committed
139
140
141
142
143
        )

        # Data and Output
        data_group = self._parser.add_argument_group("data and output")
        data_group.add_argument(
Baber's avatar
Baber committed
144
145
146
147
            "--output_path",
            "-o",
            default=None,
            type=str,
Baber's avatar
Baber committed
148
149
150
151
152
153
154
155
156
            metavar="OUTPUT_PATH",
            help="Output dir or json file for results (and samples)",
        )
        data_group.add_argument(
            "--log_samples",
            "-s",
            action="store_true",
            default=argparse.SUPPRESS,
            help="Save all model outputs and documents for post-hoc analysis",
Baber's avatar
Baber committed
157
        )
Baber's avatar
Baber committed
158
        data_group.add_argument(
Baber's avatar
Baber committed
159
160
161
162
            "--limit",
            "-L",
            type=float,
            default=None,
Baber's avatar
Baber committed
163
164
            metavar="N|0.0-1.0",
            help="Limit examples per task (integer count or fraction)",
Baber's avatar
Baber committed
165
        )
Baber's avatar
Baber committed
166
        data_group.add_argument(
Baber's avatar
Baber committed
167
168
169
            "--samples",
            "-E",
            default=None,
Baber's avatar
Baber committed
170
            type=try_parse_json,
171
            metavar='"task1": [1,2,3,4,...]"',
Baber's avatar
Baber committed
172
            help=textwrap.dedent(
173
174
                "`...` `...` Sample indices for inputs. Incompatible with --limit."
                " Values be parsable with ast.literal_eval."
Baber's avatar
Baber committed
175
            ),
Baber's avatar
Baber committed
176
        )
Baber's avatar
Baber committed
177
178
179
180

        # Caching and Performance
        cache_group = self._parser.add_argument_group("caching and performance")
        cache_group.add_argument(
Baber's avatar
Baber committed
181
182
183
184
            "--use_cache",
            "-c",
            type=str,
            default=None,
Baber's avatar
Baber committed
185
186
            metavar="CACHE_DIR",
            help="SQLite database path for caching model outputs.",
Baber's avatar
Baber committed
187
        )
Baber's avatar
Baber committed
188
        cache_group.add_argument(
Baber's avatar
Baber committed
189
            "--cache_requests",
Baber's avatar
Baber committed
190
            type=request_caching_arg_to_dict,
Baber's avatar
Baber committed
191
192
            default=None,
            choices=["true", "refresh", "delete"],
Baber's avatar
Baber committed
193
            help="Cache dataset request building (true|refresh|delete)",
Baber's avatar
Baber committed
194
        )
Baber's avatar
Baber committed
195
        cache_group.add_argument(
Baber's avatar
Baber committed
196
197
            "--check_integrity",
            action="store_true",
Baber's avatar
Baber committed
198
            default=argparse.SUPPRESS,
Baber's avatar
Baber committed
199
            help="Run task test suite validation",
Baber's avatar
Baber committed
200
        )
Baber's avatar
Baber committed
201
202
203
204

        # Prompt Formatting
        template_group = self._parser.add_argument_group("instruct formatting")
        template_group.add_argument(
Baber's avatar
Baber committed
205
206
207
            "--system_instruction",
            type=str,
            default=None,
Baber's avatar
Baber committed
208
209
            metavar="INSTRUCTION",
            help="Add custom system instruction.",
Baber's avatar
Baber committed
210
        )
Baber's avatar
Baber committed
211
        template_group.add_argument(
Baber's avatar
Baber committed
212
213
214
215
            "--apply_chat_template",
            type=str,
            nargs="?",
            const=True,
Baber's avatar
Baber committed
216
            default=argparse.SUPPRESS,
Baber's avatar
Baber committed
217
218
            metavar="TEMPLATE",
            help="Apply chat template to prompts (optional template name)",
Baber's avatar
Baber committed
219
        )
Baber's avatar
Baber committed
220
        template_group.add_argument(
Baber's avatar
Baber committed
221
222
            "--fewshot_as_multiturn",
            action="store_true",
Baber's avatar
Baber committed
223
            default=argparse.SUPPRESS,
Baber's avatar
Baber committed
224
            help="Use fewshot examples as multi-turn conversation",
Baber's avatar
Baber committed
225
        )
Baber's avatar
Baber committed
226
227
228
229

        # Task Management
        task_group = self._parser.add_argument_group("task management")
        task_group.add_argument(
Baber's avatar
Baber committed
230
231
232
            "--include_path",
            type=str,
            default=None,
Baber's avatar
Baber committed
233
234
            metavar="TASK_DIR",
            help="Additional directory for external tasks",
Baber's avatar
Baber committed
235
        )
Baber's avatar
Baber committed
236
237
238
239

        # Logging and Tracking
        logging_group = self._parser.add_argument_group("logging and tracking")
        logging_group.add_argument(
Baber's avatar
Baber committed
240
241
242
243
            "--verbosity",
            "-v",
            type=str.upper,
            default=None,
Baber's avatar
Baber committed
244
245
            metavar="LEVEL",
            help="(Deprecated) Log level. Use LOGLEVEL env var instead",
Baber's avatar
Baber committed
246
        )
Baber's avatar
Baber committed
247
248
249
250
251
252
253
254
255
256
257
258
259
260
        logging_group.add_argument(
            "--write_out",
            "-w",
            action="store_true",
            default=argparse.SUPPRESS,
            help="Print prompts for first few documents",
        )
        logging_group.add_argument(
            "--show_config",
            action="store_true",
            default=argparse.SUPPRESS,
            help="Display full task configuration after evaluation",
        )
        logging_group.add_argument(
Baber's avatar
Baber committed
261
            "--wandb_args",
262
            type=key_val_to_dict,
Baber's avatar
Baber committed
263
            default=argparse.SUPPRESS,
Baber's avatar
Baber committed
264
            metavar="ARGS",
265
            help="Weights & Biases init arguments key=val key2=val2",
Baber's avatar
Baber committed
266
        )
Baber's avatar
Baber committed
267
        logging_group.add_argument(
Baber's avatar
Baber committed
268
            "--wandb_config_args",
269
            type=key_val_to_dict,
Baber's avatar
Baber committed
270
            default=argparse.SUPPRESS,
Baber's avatar
Baber committed
271
            metavar="ARGS",
272
            help="Weights & Biases config arguments key=val key2=val2",
Baber's avatar
Baber committed
273
        )
Baber's avatar
Baber committed
274
        logging_group.add_argument(
Baber's avatar
Baber committed
275
            "--hf_hub_log_args",
276
            type=key_val_to_dict,
Baber's avatar
Baber committed
277
            default=argparse.SUPPRESS,
Baber's avatar
Baber committed
278
            metavar="ARGS",
279
            help="Hugging Face Hub logging arguments key=val key2=val2",
Baber's avatar
Baber committed
280
        )
Baber's avatar
Baber committed
281
282
283
284

        # Advanced Options
        advanced_group = self._parser.add_argument_group("advanced options")
        advanced_group.add_argument(
Baber's avatar
Baber committed
285
286
287
            "--predict_only",
            "-x",
            action="store_true",
Baber's avatar
Baber committed
288
            default=argparse.SUPPRESS,
Baber's avatar
Baber committed
289
            help="Save predictions only, skip metric computation",
Baber's avatar
Baber committed
290
291
        )
        default_seed_string = "0,1234,1234,1234"
Baber's avatar
Baber committed
292
        advanced_group.add_argument(
Baber's avatar
Baber committed
293
294
            "--seed",
            type=partial(_int_or_none_list_arg_type, 3, 4, default_seed_string),
Baber's avatar
Baber committed
295
            default=None,
Baber's avatar
Baber committed
296
297
298
299
300
301
302
303
            metavar="SEED|S1,S2,S3,S4",
            help=textwrap.dedent(f"""
                Random seeds for python,numpy,torch,fewshot (default: {default_seed_string}).
                Use single integer for all, or comma-separated list of 4 values.
                Use 'None' to skip setting a seed. Example: --seed 42 or --seed 0,None,8,52
            """).strip(),
        )
        advanced_group.add_argument(
Baber's avatar
Baber committed
304
305
            "--trust_remote_code",
            action="store_true",
Baber's avatar
Baber committed
306
            default=argparse.SUPPRESS,
Baber's avatar
Baber committed
307
            help="Allow executing remote code from Hugging Face Hub",
Baber's avatar
Baber committed
308
        )
Baber's avatar
Baber committed
309
        advanced_group.add_argument(
Baber's avatar
Baber committed
310
311
            "--confirm_run_unsafe_code",
            action="store_true",
Baber's avatar
Baber committed
312
            default=argparse.SUPPRESS,
Baber's avatar
Baber committed
313
            help="Confirm understanding of unsafe code execution risks",
Baber's avatar
Baber committed
314
        )
Baber's avatar
Baber committed
315
        advanced_group.add_argument(
Baber's avatar
Baber committed
316
317
318
            "--metadata",
            type=json.loads,
            default=None,
319
            metavar="`key=val` `key2=val2`",
Baber's avatar
Baber committed
320
            help=textwrap.dedent(
321
322
                """`key=val` `key2=val` args parsable by ast.literal_eval (merged with model_args),
                required for some tasks such as RULER"""
Baber's avatar
Baber committed
323
            ),
Baber's avatar
Baber committed
324
325
        )

326
327
    @staticmethod
    def _execute(args: argparse.Namespace) -> None:
Baber's avatar
cleanup  
Baber committed
328
        """Runs the evaluation harness with the provided arguments."""
Baber's avatar
nit  
Baber committed
329
        os.environ["TOKENIZERS_PARALLELISM"] = "false"
330
331
332
333
334
335
336
337
338
339
340
        MERGE_ARGS_DICTS = [
            "model_args",
            "gen_kwargs",
            "wandb_args",
            "wandb_config_args",
            "hf_hub_log_args",
        ]
        for arg_name in MERGE_ARGS_DICTS:
            if current_value := getattr(args, arg_name, None):
                setattr(args, arg_name, merge_dicts(*current_value))

Baber's avatar
Baber committed
341
342
        from lm_eval.config.evaluate_config import EvaluatorConfig

Baber's avatar
nit  
Baber committed
343
344
345
        eval_logger = logging.getLogger(__name__)

        # Create and validate config (most validation now occurs in EvaluationConfig)
Baber's avatar
Baber committed
346
        cfg = EvaluatorConfig.from_cli(args)
Baber's avatar
Baber committed
347

Baber's avatar
nit  
Baber committed
348
        from lm_eval import simple_evaluate
Baber's avatar
Baber committed
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
        from lm_eval.loggers import EvaluationTracker, WandbLogger
        from lm_eval.utils import handle_non_serializable, make_table

        # Set up logging
        if cfg.wandb_args:
            wandb_logger = WandbLogger(cfg.wandb_args, cfg.wandb_config_args)

        # Set up evaluation tracker
        if cfg.output_path:
            cfg.hf_hub_log_args["output_path"] = cfg.output_path

        if os.environ.get("HF_TOKEN", None):
            cfg.hf_hub_log_args["token"] = os.environ.get("HF_TOKEN")

        evaluation_tracker = EvaluationTracker(**cfg.hf_hub_log_args)

        # Create task manager (metadata already set up in config validation)
Baber's avatar
nit  
Baber committed
366
        task_manager = cfg.process_tasks(cfg.metadata)
Baber's avatar
Baber committed
367
368
369
370
371
372
373
374
375
376
377
378
379

        # Validation warnings (keep these in CLI as they're logging-specific)
        if "push_samples_to_hub" in cfg.hf_hub_log_args and not cfg.log_samples:
            eval_logger.warning(
                "Pushing samples to the Hub requires --log_samples to be set."
            )

        # Log task selection (tasks already processed in config)
        if cfg.include_path is not None:
            eval_logger.info(f"Including path: {cfg.include_path}")
        eval_logger.info(f"Selected Tasks: {cfg.tasks}")

        # Run evaluation
Baber's avatar
Baber committed
380
        results = simple_evaluate(
Baber's avatar
Baber committed
381
382
383
384
385
386
387
388
            model=cfg.model,
            model_args=cfg.model_args,
            tasks=cfg.tasks,
            num_fewshot=cfg.num_fewshot,
            batch_size=cfg.batch_size,
            max_batch_size=cfg.max_batch_size,
            device=cfg.device,
            use_cache=cfg.use_cache,
Baber's avatar
Baber committed
389
390
            cache_requests=cfg.cache_requests.get("cache_requests", False),
            rewrite_requests_cache=cfg.cache_requests.get(
Baber's avatar
Baber committed
391
392
                "rewrite_requests_cache", False
            ),
Baber's avatar
Baber committed
393
            delete_requests_cache=cfg.cache_requests.get(
Baber's avatar
Baber committed
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
                "delete_requests_cache", False
            ),
            limit=cfg.limit,
            samples=cfg.samples,
            check_integrity=cfg.check_integrity,
            write_out=cfg.write_out,
            log_samples=cfg.log_samples,
            evaluation_tracker=evaluation_tracker,
            system_instruction=cfg.system_instruction,
            apply_chat_template=cfg.apply_chat_template,
            fewshot_as_multiturn=cfg.fewshot_as_multiturn,
            gen_kwargs=cfg.gen_kwargs,
            task_manager=task_manager,
            verbosity=cfg.verbosity,
            predict_only=cfg.predict_only,
            random_seed=cfg.seed[0] if cfg.seed else None,
            numpy_random_seed=cfg.seed[1] if cfg.seed else None,
            torch_random_seed=cfg.seed[2] if cfg.seed else None,
            fewshot_random_seed=cfg.seed[3] if cfg.seed else None,
            confirm_run_unsafe_code=cfg.confirm_run_unsafe_code,
            metadata=cfg.metadata,
        )

        # Process results
        if results is not None:
            if cfg.log_samples:
                samples = results.pop("samples")

            dumped = json.dumps(
                results, indent=2, default=handle_non_serializable, ensure_ascii=False
            )
            if cfg.show_config:
                print(dumped)

            batch_sizes = ",".join(map(str, results["config"]["batch_sizes"]))

            # W&B logging
            if cfg.wandb_args:
                try:
                    wandb_logger.post_init(results)
                    wandb_logger.log_eval_result()
                    if cfg.log_samples:
                        wandb_logger.log_eval_samples(samples)
                except Exception as e:
                    eval_logger.info(f"Logging to W&B failed: {e}")

            # Save results
            evaluation_tracker.save_results_aggregated(
                results=results, samples=samples if cfg.log_samples else None
            )

            if cfg.log_samples:
                for task_name, _ in results["configs"].items():
                    evaluation_tracker.save_results_samples(
                        task_name=task_name, samples=samples[task_name]
                    )

            if (
                evaluation_tracker.push_results_to_hub
                or evaluation_tracker.push_samples_to_hub
            ):
                evaluation_tracker.recreate_metadata_card()

            # Print results
Baber's avatar
Baber committed
458
            cfg.model_args.pop("trust_remote_code", None)
Baber's avatar
Baber committed
459
460
461
462
463
464
465
466
467
468
469
            print(
                f"{cfg.model} ({cfg.model_args}), gen_kwargs: ({cfg.gen_kwargs}), "
                f"limit: {cfg.limit}, num_fewshot: {cfg.num_fewshot}, "
                f"batch_size: {cfg.batch_size}{f' ({batch_sizes})' if batch_sizes else ''}"
            )
            print(make_table(results))
            if "groups" in results:
                print(make_table(results, "groups"))

            if cfg.wandb_args:
                wandb_logger.run.finish()