"...multi_subject_dreambooth/requirements.txt" did not exist on "7674a36a34556c9f478f63c3bfb0670ad398f892"
run.py 15.8 KB
Newer Older
Baber's avatar
Baber committed
1
2
3
4
import argparse
import json
import logging
import os
Baber's avatar
cleanup  
Baber committed
5
import textwrap
Baber's avatar
Baber committed
6
7
from functools import partial

Baber's avatar
cleanup  
Baber committed
8
from lm_eval._cli.subcommand import SubCommand
Baber's avatar
Baber committed
9
10
11
12
13
from lm_eval._cli.utils import (
    _int_or_none_list_arg_type,
    request_caching_arg_to_dict,
    try_parse_json,
)
Baber's avatar
Baber committed
14
15


Baber's avatar
Baber committed
16
class Run(SubCommand):
Baber's avatar
Baber committed
17
18
19
20
    """Command for running language model evaluation."""

    def __init__(self, subparsers: argparse._SubParsersAction, *args, **kwargs):
        super().__init__(*args, **kwargs)
Baber's avatar
cleanup  
Baber committed
21
        self._parser = subparsers.add_parser(
Baber's avatar
Baber committed
22
            "run",
Baber's avatar
cleanup  
Baber committed
23
            help="Run the evaluation harness on specified tasks",
Baber's avatar
Baber committed
24
            description="Evaluate language models on various benchmarks and tasks.",
Baber's avatar
cleanup  
Baber committed
25
            usage="lm-eval run --model <model> --tasks <task1,task2,...> [options]",
Baber's avatar
cleanup  
Baber committed
26
27
28
29
            epilog=textwrap.dedent("""
                examples:
                  # Basic evaluation with HuggingFace model
                  $ lm-eval run --model hf --model_args pretrained=gpt2 --tasks hellaswag
Baber's avatar
Baber committed
30

Baber's avatar
cleanup  
Baber committed
31
32
                  # Evaluate on multiple tasks with few-shot examples
                  $ lm-eval run --model vllm --model_args pretrained=EleutherAI/gpt-j-6B --tasks arc_easy,arc_challenge --num_fewshot 5
Baber's avatar
Baber committed
33

Baber's avatar
cleanup  
Baber committed
34
35
                  # Evaluation with custom generation parameters
                  $ lm-eval run --model hf --model_args pretrained=gpt2 --tasks lambada --gen_kwargs "temperature=0.8,top_p=0.95"
Baber's avatar
Baber committed
36

Baber's avatar
cleanup  
Baber committed
37
38
                  # Use configuration file
                  $ lm-eval run --config my_config.yaml --tasks mmlu
Baber's avatar
Baber committed
39

Baber's avatar
cleanup  
Baber committed
40
41
                For more information, see: https://github.com/EleutherAI/lm-evaluation-harness
            """),
Baber's avatar
Baber committed
42
43
            formatter_class=argparse.RawDescriptionHelpFormatter,
        )
Baber's avatar
cleanup  
Baber committed
44
        self._add_args()
Baber's avatar
nit  
Baber committed
45
        self._parser.set_defaults(func=self._execute)
Baber's avatar
Baber committed
46

Baber's avatar
cleanup  
Baber committed
47
48
    def _add_args(self) -> None:
        self._parser = self._parser
Baber's avatar
Baber committed
49

Baber's avatar
Baber committed
50
        # Defaults are set in config/evaluate_config.py
Baber's avatar
Baber committed
51
52
        config_group = self._parser.add_argument_group("configuration")
        config_group.add_argument(
Baber's avatar
Baber committed
53
54
55
56
            "--config",
            "-C",
            default=None,
            type=str,
Baber's avatar
Baber committed
57
58
            metavar="YAML_PATH",
            help="Set initial arguments from YAML config",
Baber's avatar
Baber committed
59
        )
Baber's avatar
Baber committed
60
61
62
63

        # Model and Tasks
        model_group = self._parser.add_argument_group("model and tasks")
        model_group.add_argument(
Baber's avatar
Baber committed
64
65
66
            "--model",
            "-m",
            type=str,
Baber's avatar
Baber committed
67
            default=None,
Baber's avatar
Baber committed
68
69
            metavar="MODEL_NAME",
            help="Model name (default: hf)",
Baber's avatar
Baber committed
70
        )
Baber's avatar
Baber committed
71
        model_group.add_argument(
Baber's avatar
Baber committed
72
73
74
75
            "--tasks",
            "-t",
            default=None,
            type=str,
Baber's avatar
Baber committed
76
77
78
79
80
            metavar="TASK1,TASK2",
            help=textwrap.dedent("""
                Comma-separated list of task names or groupings.
                Use 'lm-eval list tasks' to see all available tasks.
            """).strip(),
Baber's avatar
Baber committed
81
        )
Baber's avatar
Baber committed
82
        model_group.add_argument(
Baber's avatar
Baber committed
83
84
            "--model_args",
            "-a",
Baber's avatar
Baber committed
85
            default=None,
Baber's avatar
Baber committed
86
            type=try_parse_json,
Baber's avatar
Baber committed
87
88
            metavar="ARGS",
            help="Model arguments as 'key=val,key2=val2' or JSON string",
Baber's avatar
Baber committed
89
        )
Baber's avatar
Baber committed
90
91
92
93

        # Evaluation Settings
        eval_group = self._parser.add_argument_group("evaluation settings")
        eval_group.add_argument(
Baber's avatar
Baber committed
94
95
96
97
98
99
100
            "--num_fewshot",
            "-f",
            type=int,
            default=None,
            metavar="N",
            help="Number of examples in few-shot context",
        )
Baber's avatar
Baber committed
101
        eval_group.add_argument(
Baber's avatar
Baber committed
102
103
104
            "--batch_size",
            "-b",
            type=str,
Baber's avatar
Baber committed
105
            default=argparse.SUPPRESS,
Baber's avatar
Baber committed
106
            metavar="auto|auto:N|N",
Baber's avatar
Baber committed
107
108
109
            help=textwrap.dedent(
                "Batch size: 'auto', 'auto:N' (auto-tune N times), or integer (default: 1)"
            ),
Baber's avatar
Baber committed
110
        )
Baber's avatar
Baber committed
111
        eval_group.add_argument(
Baber's avatar
Baber committed
112
113
114
115
            "--max_batch_size",
            type=int,
            default=None,
            metavar="N",
Baber's avatar
Baber committed
116
            help="Maximum batch size when using --batch_size auto",
Baber's avatar
Baber committed
117
        )
Baber's avatar
Baber committed
118
        eval_group.add_argument(
Baber's avatar
Baber committed
119
120
121
            "--device",
            type=str,
            default=None,
Baber's avatar
Baber committed
122
123
            metavar="DEVICE",
            help="Device to use (e.g. cuda, cuda:0, cpu, mps)",
Baber's avatar
Baber committed
124
        )
Baber's avatar
Baber committed
125
126
127
128
129
130
131
132
133
134
135
        eval_group.add_argument(
            "--gen_kwargs",
            type=try_parse_json,
            default=None,
            metavar="KWARGS",
            help="Generation arguments as 'key=val,key2=val2' or JSON string",
        )

        # Data and Output
        data_group = self._parser.add_argument_group("data and output")
        data_group.add_argument(
Baber's avatar
Baber committed
136
137
138
139
            "--output_path",
            "-o",
            default=None,
            type=str,
Baber's avatar
Baber committed
140
141
142
143
144
145
146
147
148
            metavar="OUTPUT_PATH",
            help="Output dir or json file for results (and samples)",
        )
        data_group.add_argument(
            "--log_samples",
            "-s",
            action="store_true",
            default=argparse.SUPPRESS,
            help="Save all model outputs and documents for post-hoc analysis",
Baber's avatar
Baber committed
149
        )
Baber's avatar
Baber committed
150
        data_group.add_argument(
Baber's avatar
Baber committed
151
152
153
154
            "--limit",
            "-L",
            type=float,
            default=None,
Baber's avatar
Baber committed
155
156
            metavar="N|0.0-1.0",
            help="Limit examples per task (integer count or fraction)",
Baber's avatar
Baber committed
157
        )
Baber's avatar
Baber committed
158
        data_group.add_argument(
Baber's avatar
Baber committed
159
160
161
            "--samples",
            "-E",
            default=None,
Baber's avatar
Baber committed
162
            type=try_parse_json,
Baber's avatar
Baber committed
163
164
165
166
            metavar="JSON_FILE",
            help=textwrap.dedent(
                'JSON file with specific sample indices for inputs: {"task_name":[indices],...}. Incompatible with --limit.'
            ),
Baber's avatar
Baber committed
167
        )
Baber's avatar
Baber committed
168
169
170
171

        # Caching and Performance
        cache_group = self._parser.add_argument_group("caching and performance")
        cache_group.add_argument(
Baber's avatar
Baber committed
172
173
174
175
            "--use_cache",
            "-c",
            type=str,
            default=None,
Baber's avatar
Baber committed
176
177
            metavar="CACHE_DIR",
            help="SQLite database path for caching model outputs.",
Baber's avatar
Baber committed
178
        )
Baber's avatar
Baber committed
179
        cache_group.add_argument(
Baber's avatar
Baber committed
180
            "--cache_requests",
Baber's avatar
Baber committed
181
            type=request_caching_arg_to_dict,
Baber's avatar
Baber committed
182
183
            default=None,
            choices=["true", "refresh", "delete"],
Baber's avatar
Baber committed
184
            help="Cache dataset request building (true|refresh|delete)",
Baber's avatar
Baber committed
185
        )
Baber's avatar
Baber committed
186
        cache_group.add_argument(
Baber's avatar
Baber committed
187
188
            "--check_integrity",
            action="store_true",
Baber's avatar
Baber committed
189
            default=argparse.SUPPRESS,
Baber's avatar
Baber committed
190
            help="Run task test suite validation",
Baber's avatar
Baber committed
191
        )
Baber's avatar
Baber committed
192
193
194
195

        # Prompt Formatting
        template_group = self._parser.add_argument_group("instruct formatting")
        template_group.add_argument(
Baber's avatar
Baber committed
196
197
198
            "--system_instruction",
            type=str,
            default=None,
Baber's avatar
Baber committed
199
200
            metavar="INSTRUCTION",
            help="Add custom system instruction.",
Baber's avatar
Baber committed
201
        )
Baber's avatar
Baber committed
202
        template_group.add_argument(
Baber's avatar
Baber committed
203
204
205
206
            "--apply_chat_template",
            type=str,
            nargs="?",
            const=True,
Baber's avatar
Baber committed
207
            default=argparse.SUPPRESS,
Baber's avatar
Baber committed
208
209
            metavar="TEMPLATE",
            help="Apply chat template to prompts (optional template name)",
Baber's avatar
Baber committed
210
        )
Baber's avatar
Baber committed
211
        template_group.add_argument(
Baber's avatar
Baber committed
212
213
            "--fewshot_as_multiturn",
            action="store_true",
Baber's avatar
Baber committed
214
            default=argparse.SUPPRESS,
Baber's avatar
Baber committed
215
            help="Use fewshot examples as multi-turn conversation",
Baber's avatar
Baber committed
216
        )
Baber's avatar
Baber committed
217
218
219
220

        # Task Management
        task_group = self._parser.add_argument_group("task management")
        task_group.add_argument(
Baber's avatar
Baber committed
221
222
223
            "--include_path",
            type=str,
            default=None,
Baber's avatar
Baber committed
224
225
            metavar="TASK_DIR",
            help="Additional directory for external tasks",
Baber's avatar
Baber committed
226
        )
Baber's avatar
Baber committed
227
228
229
230

        # Logging and Tracking
        logging_group = self._parser.add_argument_group("logging and tracking")
        logging_group.add_argument(
Baber's avatar
Baber committed
231
232
233
234
            "--verbosity",
            "-v",
            type=str.upper,
            default=None,
Baber's avatar
Baber committed
235
236
            metavar="LEVEL",
            help="(Deprecated) Log level. Use LOGLEVEL env var instead",
Baber's avatar
Baber committed
237
        )
Baber's avatar
Baber committed
238
239
240
241
242
243
244
245
246
247
248
249
250
251
        logging_group.add_argument(
            "--write_out",
            "-w",
            action="store_true",
            default=argparse.SUPPRESS,
            help="Print prompts for first few documents",
        )
        logging_group.add_argument(
            "--show_config",
            action="store_true",
            default=argparse.SUPPRESS,
            help="Display full task configuration after evaluation",
        )
        logging_group.add_argument(
Baber's avatar
Baber committed
252
253
            "--wandb_args",
            type=str,
Baber's avatar
Baber committed
254
            default=argparse.SUPPRESS,
Baber's avatar
Baber committed
255
256
            metavar="ARGS",
            help="Weights & Biases init arguments (key=val,key2=val2)",
Baber's avatar
Baber committed
257
        )
Baber's avatar
Baber committed
258
        logging_group.add_argument(
Baber's avatar
Baber committed
259
260
            "--wandb_config_args",
            type=str,
Baber's avatar
Baber committed
261
            default=argparse.SUPPRESS,
Baber's avatar
Baber committed
262
263
            metavar="ARGS",
            help="Weights & Biases config arguments (key=val,key2=val2)",
Baber's avatar
Baber committed
264
        )
Baber's avatar
Baber committed
265
        logging_group.add_argument(
Baber's avatar
Baber committed
266
267
            "--hf_hub_log_args",
            type=str,
Baber's avatar
Baber committed
268
            default=argparse.SUPPRESS,
Baber's avatar
Baber committed
269
270
            metavar="ARGS",
            help="Hugging Face Hub logging arguments (key=val,key2=val2)",
Baber's avatar
Baber committed
271
        )
Baber's avatar
Baber committed
272
273
274
275

        # Advanced Options
        advanced_group = self._parser.add_argument_group("advanced options")
        advanced_group.add_argument(
Baber's avatar
Baber committed
276
277
278
            "--predict_only",
            "-x",
            action="store_true",
Baber's avatar
Baber committed
279
            default=argparse.SUPPRESS,
Baber's avatar
Baber committed
280
            help="Save predictions only, skip metric computation",
Baber's avatar
Baber committed
281
282
        )
        default_seed_string = "0,1234,1234,1234"
Baber's avatar
Baber committed
283
        advanced_group.add_argument(
Baber's avatar
Baber committed
284
285
            "--seed",
            type=partial(_int_or_none_list_arg_type, 3, 4, default_seed_string),
Baber's avatar
Baber committed
286
            default=None,
Baber's avatar
Baber committed
287
288
289
290
291
292
293
294
            metavar="SEED|S1,S2,S3,S4",
            help=textwrap.dedent(f"""
                Random seeds for python,numpy,torch,fewshot (default: {default_seed_string}).
                Use single integer for all, or comma-separated list of 4 values.
                Use 'None' to skip setting a seed. Example: --seed 42 or --seed 0,None,8,52
            """).strip(),
        )
        advanced_group.add_argument(
Baber's avatar
Baber committed
295
296
            "--trust_remote_code",
            action="store_true",
Baber's avatar
Baber committed
297
            default=argparse.SUPPRESS,
Baber's avatar
Baber committed
298
            help="Allow executing remote code from Hugging Face Hub",
Baber's avatar
Baber committed
299
        )
Baber's avatar
Baber committed
300
        advanced_group.add_argument(
Baber's avatar
Baber committed
301
302
            "--confirm_run_unsafe_code",
            action="store_true",
Baber's avatar
Baber committed
303
            default=argparse.SUPPRESS,
Baber's avatar
Baber committed
304
            help="Confirm understanding of unsafe code execution risks",
Baber's avatar
Baber committed
305
        )
Baber's avatar
Baber committed
306
        advanced_group.add_argument(
Baber's avatar
Baber committed
307
308
309
            "--metadata",
            type=json.loads,
            default=None,
Baber's avatar
Baber committed
310
311
            metavar="JSON",
            help=textwrap.dedent(
Baber's avatar
Baber committed
312
                """JSON metadata for task configs (merged with model_args), required for some tasks such as RULER"""
Baber's avatar
Baber committed
313
            ),
Baber's avatar
Baber committed
314
315
        )

Baber's avatar
nit  
Baber committed
316
    def _execute(self, args: argparse.Namespace) -> None:
Baber's avatar
cleanup  
Baber committed
317
        """Runs the evaluation harness with the provided arguments."""
Baber's avatar
nit  
Baber committed
318
        os.environ["TOKENIZERS_PARALLELISM"] = "false"
Baber's avatar
Baber committed
319
320
        from lm_eval.config.evaluate_config import EvaluatorConfig

Baber's avatar
nit  
Baber committed
321
322
323
        eval_logger = logging.getLogger(__name__)

        # Create and validate config (most validation now occurs in EvaluationConfig)
Baber's avatar
Baber committed
324
        cfg = EvaluatorConfig.from_cli(args)
Baber's avatar
Baber committed
325

Baber's avatar
nit  
Baber committed
326
        from lm_eval import simple_evaluate
Baber's avatar
Baber committed
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
        from lm_eval.loggers import EvaluationTracker, WandbLogger
        from lm_eval.utils import handle_non_serializable, make_table

        # Set up logging
        if cfg.wandb_args:
            wandb_logger = WandbLogger(cfg.wandb_args, cfg.wandb_config_args)

        # Set up evaluation tracker
        if cfg.output_path:
            cfg.hf_hub_log_args["output_path"] = cfg.output_path

        if os.environ.get("HF_TOKEN", None):
            cfg.hf_hub_log_args["token"] = os.environ.get("HF_TOKEN")

        evaluation_tracker = EvaluationTracker(**cfg.hf_hub_log_args)

        # Create task manager (metadata already set up in config validation)
Baber's avatar
nit  
Baber committed
344
        task_manager = cfg.process_tasks(cfg.metadata)
Baber's avatar
Baber committed
345
346
347
348
349
350
351
352
353
354
355
356
357

        # Validation warnings (keep these in CLI as they're logging-specific)
        if "push_samples_to_hub" in cfg.hf_hub_log_args and not cfg.log_samples:
            eval_logger.warning(
                "Pushing samples to the Hub requires --log_samples to be set."
            )

        # Log task selection (tasks already processed in config)
        if cfg.include_path is not None:
            eval_logger.info(f"Including path: {cfg.include_path}")
        eval_logger.info(f"Selected Tasks: {cfg.tasks}")

        # Run evaluation
Baber's avatar
Baber committed
358
        results = simple_evaluate(
Baber's avatar
Baber committed
359
360
361
362
363
364
365
366
            model=cfg.model,
            model_args=cfg.model_args,
            tasks=cfg.tasks,
            num_fewshot=cfg.num_fewshot,
            batch_size=cfg.batch_size,
            max_batch_size=cfg.max_batch_size,
            device=cfg.device,
            use_cache=cfg.use_cache,
Baber's avatar
Baber committed
367
368
            cache_requests=cfg.cache_requests.get("cache_requests", False),
            rewrite_requests_cache=cfg.cache_requests.get(
Baber's avatar
Baber committed
369
370
                "rewrite_requests_cache", False
            ),
Baber's avatar
Baber committed
371
            delete_requests_cache=cfg.cache_requests.get(
Baber's avatar
Baber committed
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
                "delete_requests_cache", False
            ),
            limit=cfg.limit,
            samples=cfg.samples,
            check_integrity=cfg.check_integrity,
            write_out=cfg.write_out,
            log_samples=cfg.log_samples,
            evaluation_tracker=evaluation_tracker,
            system_instruction=cfg.system_instruction,
            apply_chat_template=cfg.apply_chat_template,
            fewshot_as_multiturn=cfg.fewshot_as_multiturn,
            gen_kwargs=cfg.gen_kwargs,
            task_manager=task_manager,
            verbosity=cfg.verbosity,
            predict_only=cfg.predict_only,
            random_seed=cfg.seed[0] if cfg.seed else None,
            numpy_random_seed=cfg.seed[1] if cfg.seed else None,
            torch_random_seed=cfg.seed[2] if cfg.seed else None,
            fewshot_random_seed=cfg.seed[3] if cfg.seed else None,
            confirm_run_unsafe_code=cfg.confirm_run_unsafe_code,
            metadata=cfg.metadata,
        )

        # Process results
        if results is not None:
            if cfg.log_samples:
                samples = results.pop("samples")

            dumped = json.dumps(
                results, indent=2, default=handle_non_serializable, ensure_ascii=False
            )
            if cfg.show_config:
                print(dumped)

            batch_sizes = ",".join(map(str, results["config"]["batch_sizes"]))

            # W&B logging
            if cfg.wandb_args:
                try:
                    wandb_logger.post_init(results)
                    wandb_logger.log_eval_result()
                    if cfg.log_samples:
                        wandb_logger.log_eval_samples(samples)
                except Exception as e:
                    eval_logger.info(f"Logging to W&B failed: {e}")

            # Save results
            evaluation_tracker.save_results_aggregated(
                results=results, samples=samples if cfg.log_samples else None
            )

            if cfg.log_samples:
                for task_name, _ in results["configs"].items():
                    evaluation_tracker.save_results_samples(
                        task_name=task_name, samples=samples[task_name]
                    )

            if (
                evaluation_tracker.push_results_to_hub
                or evaluation_tracker.push_samples_to_hub
            ):
                evaluation_tracker.recreate_metadata_card()

            # Print results
Baber's avatar
Baber committed
436
            cfg.model_args.pop("trust_remote_code", None)
Baber's avatar
Baber committed
437
438
439
440
441
442
443
444
445
446
447
            print(
                f"{cfg.model} ({cfg.model_args}), gen_kwargs: ({cfg.gen_kwargs}), "
                f"limit: {cfg.limit}, num_fewshot: {cfg.num_fewshot}, "
                f"batch_size: {cfg.batch_size}{f' ({batch_sizes})' if batch_sizes else ''}"
            )
            print(make_table(results))
            if "groups" in results:
                print(make_table(results, "groups"))

            if cfg.wandb_args:
                wandb_logger.run.finish()