eval.py

import argparse
import sys
import textwrap

from lm_eval._cli.list import List
from lm_eval._cli.run import Run
from lm_eval._cli.validate import Validate


class Eval:
    """Main CLI parser that manages all subcommands."""

    def __init__(self):
        self._parser = argparse.ArgumentParser(
            prog="lm-eval",
            description="Language Model Evaluation Harness",
            epilog=textwrap.dedent("""
                quick start:
                  # Basic evaluation
                  lm-eval run --model hf --model_args pretrained=gpt2 --tasks hellaswag

                  # List available tasks
                  lm-eval list tasks

                  # Validate task configurations
                  lm-eval validate --tasks hellaswag,arc_easy

                legacy compatibility:
                  The harness maintains backward compatibility with the original interface.
                  If no command is specified, 'run' is automatically inserted:

                  lm-eval --model hf --tasks hellaswag  # Equivalent to 'lm-eval run --model hf --tasks hellaswag'

                For documentation, visit: https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/interface.md
            """),
            formatter_class=argparse.RawDescriptionHelpFormatter,
        )
        self._parser.set_defaults(func=lambda args: self._parser.print_help())
        self._subparsers = self._parser.add_subparsers(
            dest="command", help="Available commands", metavar="COMMAND"
        )
        Run.create(self._subparsers)
        List.create(self._subparsers)
        Validate.create(self._subparsers)

    def parse_args(self) -> argparse.Namespace:
        """Parse arguments using the main parser."""
        if len(sys.argv) > 2 and sys.argv[1] not in self._subparsers.choices:
            # Arguments provided but no valid subcommand - insert 'run'
            sys.argv.insert(1, "run")
        return self._parser.parse_args()

    def execute(self, args: argparse.Namespace) -> None:
        """Main execution method that handles subcommands and legacy support."""

        # Handle legacy task listing
        if hasattr(args, "tasks") and args.tasks in [
            "list",
            "list_groups",
            "list_subtasks",
            "list_tags",
        ]:
            print(
                f"'--tasks {args.tasks}' is no longer supported.\n"
                f"Use the 'list' command instead:\n",
                file=sys.stderr,
            )

            # Show list command help
            list_parser = self._subparsers.choices["list"]
            list_parser.print_help()
            sys.exit(1)

        args.func(args)