import argparse import sys import textwrap from lm_eval._cli.listall import ListAll from lm_eval._cli.run import Run from lm_eval._cli.validate import Validate class Eval: """Main CLI parser that manages all subcommands.""" def __init__(self): self._parser = argparse.ArgumentParser( prog="lm-eval", description="Language Model Evaluation Harness", epilog=textwrap.dedent(""" quick start: # Basic evaluation lm-eval run --model hf --model_args pretrained=gpt2 --tasks hellaswag # List available tasks lm-eval list tasks # Validate task configurations lm-eval validate --tasks hellaswag,arc_easy legacy compatibility: The harness maintains backward compatibility with the original interface. If no command is specified, 'run' is automatically inserted: lm-eval --model hf --tasks hellaswag # Equivalent to 'lm-eval run --model hf --tasks hellaswag' For documentation, visit: https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/interface.md """), formatter_class=argparse.RawDescriptionHelpFormatter, ) self._parser.set_defaults(func=lambda args: self._parser.print_help()) self._subparsers = self._parser.add_subparsers( dest="command", help="Available commands", metavar="COMMAND" ) Run.create(self._subparsers) ListAll.create(self._subparsers) Validate.create(self._subparsers) def parse_args(self) -> argparse.Namespace: """Parse arguments using the main parser.""" if len(sys.argv) > 2 and sys.argv[1] not in self._subparsers.choices: # Backward compatibility: arguments provided but no valid subcommand - insert 'run' sys.argv.insert(1, "run") elif len(sys.argv) == 2 and "run" in sys.argv: # if only 'run' is specified, ensure it is treated as a subcommand self._subparsers.choices["run"].print_help() sys.exit(0) return self._parser.parse_args() def execute(self, args: argparse.Namespace) -> None: """Main execution method that handles subcommands and legacy support.""" args.func(args)