CLI - Add command sb [version,deploy,exec,run] (#10)

- Add CLI commands
  * sb version
  * sb deploy
  * sb exec
  * sb run
- Add interface with executor and runner
- Add cli test cases

MANIFEST.in

 include LICENSE README.md
+recursive-include superbench *.py
+recursive-include superbench *.yaml
+global-exclude *.pyc
+global-exclude __pycache__

setup.py

@@ -132,18 +132,23 @@ def run(self):
     keywords='benchmark, AI systems',
     packages=find_packages(exclude=['tests']),
     python_requires='>=3.6, <4',
-    install_requires=[],
+    install_requires=[
+        'hydra-colorlog>=1.0.0',
+        'hydra-core>=1.0.4',
+        'knack>=0.7.2',
+    ],
     extras_require={
         'dev': ['pre-commit>=2.10.0'],
         'test': [
-            'yapf>=0.30.0',
-            'mypy>=0.800',
-            'flake8>=3.8.4',
-            'flake8-quotes>=3.2.0',
             'flake8-docstrings>=1.5.0',
+            'flake8-quotes>=3.2.0',
+            'flake8>=3.8.4',
+            'mypy>=0.800',
             'pydocstyle>=5.1.1',
-            'pytest>=6.2.2',
             'pytest-cov>=2.11.1',
+            'pytest>=6.2.2',
+            'vcrpy>=4.1.1',
+            'yapf>=0.30.0',
         ],
         'torch': [
             'torch==1.7.0',
@@ -151,9 +156,13 @@ def run(self):
             'transformers==4.3.3',
         ],
     },
-    package_data={},
+    include_package_data=True,
     entry_points={
-        'console_scripts': [],
+        'console_scripts': [
+            'sb = superbench.cli.sb:main',
+            'sb-exec = superbench.cli.sb_exec:main',
+            'sb-run = superbench.cli.sb_run:main',
+        ],
     },
     cmdclass={
         'format': Formatter,

superbench/cli/__init__.py

+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""SuperBench cli module."""
+
+from .sb import SuperBenchCLI
+
+__all__ = ['SuperBenchCLI']

superbench/cli/_commands.py

+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""SuperBench CLI commands."""
+
+from knack.arguments import ArgumentsContext
+from knack.commands import CLICommandsLoader, CommandGroup
+
+
+class SuperBenchCommandsLoader(CLICommandsLoader):
+    """SuperBench CLI commands loader."""
+    def load_command_table(self, args):
+        """Load commands into the command table.
+
+        Args:
+            args (list): List of arguments from the command line.
+
+        Returns:
+            collections.OrderedDict: Load commands into the command table.
+        """
+        with CommandGroup(self, '', 'superbench.cli._handler#{}') as g:
+            g.command('version', 'version_command_handler')
+            g.command('deploy', 'deploy_command_handler')
+            g.command('exec', 'exec_command_handler')
+            g.command('run', 'run_command_handler')
+        return super().load_command_table(args)
+
+    def load_arguments(self, command):
+        """Load arguments for commands.
+
+        Args:
+            command: The command to load arguments for.
+        """
+        with ArgumentsContext(self, '') as ac:
+            ac.argument('docker_image', options_list=('--docker-image', '-i'), type=str, help='Docker image URI.')
+            ac.argument('docker_username', type=str, help='Docker registry username if authentication is needed.')
+            ac.argument('docker_password', type=str, help='Docker registry password if authentication is needed.')
+            ac.argument(
+                'host_file', options_list=('--host-file', '-f'), type=str, help='Path to Ansible inventory host file.'
+            )
+            ac.argument('host_list', options_list=('--host-list', '-l'), type=str, help='Comma separated host list.')
+            ac.argument('host_username', type=str, help='Host username if needed.')
+            ac.argument('host_password', type=str, help='Host password or key passphase if needed.')
+            ac.argument('private_key', type=str, help='Path to private key if needed.')
+            ac.argument(
+                'config_file', options_list=('--config-file', '-c'), type=str, help='Path to SuperBench config file.'
+            )
+            ac.argument(
+                'config_override',
+                options_list=('--config-override', '-C'),
+                type=str,
+                help='Extra arguments to override config_file, following Hydra syntax.'
+            )
+        super().load_arguments(command)

superbench/cli/_handler.py

+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""SuperBench CLI command handler."""
+
+import os
+import yaml
+from pathlib import Path
+
+from knack.util import CLIError
+
+import superbench
+from superbench.common.utils import get_sb_command, get_config, new_output_dir
+
+
+def check_argument_file(name, file):
+    """Check file path in CLI arguments.
+
+    Args:
+        name (str): argument name.
+        file (str): file path.
+
+    Raises:
+        CLIError: If file does not exist.
+    """
+    if file and not Path(file).exists():
+        raise CLIError('{} {} does not exist.'.format(name, file))
+
+
+def version_command_handler():
+    """Print the current SuperBench tool version.
+
+    Returns:
+        str: current SuperBench tool version.
+    """
+    return superbench.__version__
+
+
+def deploy_command_handler(
+    docker_image,
+    docker_username=None,
+    docker_password=None,
+    host_file=None,
+    host_list=None,
+    host_username=None,
+    host_password=None,
+    private_key=None
+):
+    """Deploy the SuperBench environments to all given nodes.
+
+    Deploy SuperBench environments on all nodes, including:
+    1. check drivers
+    2. install required system dependencies
+    3. install Docker and container runtime
+    4. pull Docker image
+
+    Args:
+        docker_image (str): Docker image URI.
+        docker_username (str, optional): Docker registry username if authentication is needed. Defaults to None.
+        docker_password (str, optional): Docker registry password if authentication is needed. Defaults to None.
+        host_file (str, optional): Path to Ansible inventory host file. Defaults to None.
+        host_list (str, optional): Comma separated host list. Defaults to None.
+        host_username (str, optional): Host username if needed. Defaults to None.
+        host_password (str, optional): Host password or key passphase if needed. Defaults to None.
+        private_key (str, optional): Path to private key if needed. Defaults to None.
+
+    Raises:
+        CLIError: If input arguments are invalid.
+    """
+    if not (host_file or host_list):
+        raise CLIError('Must specify one of host_file or host_list.')
+    check_argument_file('host_file', host_file)
+    check_argument_file('private_key', private_key)
+
+    raise NotImplementedError
+
+
+def exec_command_handler(
+    docker_image, docker_username=None, docker_password=None, config_file=None, config_override=None
+):
+    """Run the SuperBench benchmarks locally.
+
+    Args:
+        docker_image (str): Docker image URI.
+        docker_username (str, optional): Docker registry username if authentication is needed. Defaults to None.
+        docker_password (str, optional): Docker registry password if authentication is needed. Defaults to None.
+        config_file (str, optional): Path to SuperBench config file. Defaults to None.
+        config_override (str, optional): Extra arguments to override config_file,
+            following [Hydra syntax](https://hydra.cc/docs/advanced/override_grammar/basic). Defaults to None.
+
+    Raises:
+        CLIError: If input arguments are invalid.
+    """
+    if bool(docker_username) != bool(docker_password):
+        raise CLIError('Must specify both docker_username and docker_password if authentication is needed.')
+    check_argument_file('config_file', config_file)
+
+    # Dump configs into outputs/date/config.merge.yaml
+    config = get_config(config_file)
+    config['docker'] = {}
+    for key in ['image', 'username', 'password']:
+        config['docker'][key] = eval('docker_{}'.format(key))
+    output_dir = new_output_dir()
+    with (Path(output_dir) / 'config.merge.yaml').open(mode='w') as f:
+        yaml.safe_dump(config, f)
+    os.system(get_sb_command('sb-exec', output_dir, config_override or ''))
+
+
+def run_command_handler(
+    docker_image,
+    docker_username=None,
+    docker_password=None,
+    host_file=None,
+    host_list=None,
+    host_username=None,
+    host_password=None,
+    private_key=None,
+    config_file=None,
+    config_override=None
+):
+    """Run the SuperBench benchmarks distributedly.
+
+    Run all benchmarks on given nodes.
+
+    Args:
+        docker_image (str): Docker image URI.
+        docker_username (str, optional): Docker registry username if authentication is needed. Defaults to None.
+        docker_password (str, optional): Docker registry password if authentication is needed. Defaults to None.
+        host_file (str, optional): Path to Ansible inventory host file. Defaults to None.
+        host_list (str, optional): Comma separated host list. Defaults to None.
+        host_username (str, optional): Host username if needed. Defaults to None.
+        host_password (str, optional): Host password or key passphase if needed. Defaults to None.
+        private_key (str, optional): Path to private key if needed. Defaults to None.
+        config_file (str, optional): Path to SuperBench config file. Defaults to None.
+        config_override (str, optional): Extra arguments to override config_file,
+            following [Hydra syntax](https://hydra.cc/docs/advanced/override_grammar/basic). Defaults to None.
+
+    Raises:
+        CLIError: If input arguments are invalid.
+    """
+    if bool(docker_username) != bool(docker_password):
+        raise CLIError('Must specify both docker_username and docker_password if authentication is needed.')
+    if not (host_file or host_list):
+        raise CLIError('Must specify one of host_file or host_list.')
+    check_argument_file('host_file', host_file)
+    check_argument_file('private_key', private_key)
+    check_argument_file('config_file', config_file)
+
+    # Dump configs into outputs/date/config.merge.yaml
+    config = get_config(config_file)
+    config['docker'] = {}
+    for key in ['image', 'username', 'password']:
+        config['docker'][key] = eval('docker_{}'.format(key))
+    config['ansible'] = {}
+    for key in ['file', 'list', 'username', 'password']:
+        config['ansible']['host_{}'.format(key)] = eval('host_{}'.format(key))
+    output_dir = new_output_dir()
+    with (Path(output_dir) / 'config.merge.yaml').open(mode='w') as f:
+        yaml.safe_dump(config, f)
+    os.system(get_sb_command('sb-run', output_dir, config_override or ''))

superbench/cli/_help.py

+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""SuperBench CLI help."""
+
+from knack.help import CLIHelp
+from knack.help_files import helps
+
+CLI_NAME = 'sb'
+WELCOME_MESSAGE = r"""
+   _____                       ____                  _
+  / ____|                     |  _ \                | |
+ | (___  _   _ _ __   ___ _ __| |_) | ___ _ __   ___| |__
+  \___ \| | | | '_ \ / _ \ '__|  _ < / _ \ '_ \ / __| '_ \
+  ____) | |_| | |_) |  __/ |  | |_) |  __/ | | | (__| | | |
+ |_____/ \__,_| .__/ \___|_|  |____/ \___|_| |_|\___|_| |_|
+              | |
+              |_|
+
+Welcome to the SB CLI!
+"""
+
+helps['version'] = """
+    type: command
+    short-summary: Print the current SuperBench CLI version.
+    examples:
+        - name: print version
+          text: {cli_name} version
+""".format(cli_name=CLI_NAME)
+
+helps['deploy'] = """
+    type: command
+    short-summary: Deploy the SuperBench environments to all given nodes.
+    examples:
+        - name: deploy image "superbench/cuda:11.1" to all nodes in ./host.yaml
+          text: {cli_name} deploy --docker-image superbench/cuda:11.1 --host-file ./host.yaml
+        - name: deploy image "superbench/rocm:4.0" to node-0 and node-2, using key file id_rsa for ssh
+          text: {cli_name} deploy --docker-image superbench/rocm:4.0 --host-list node-0,node-2 --private-key id_rsa
+""".format(cli_name=CLI_NAME)
+
+helps['exec'] = """
+    type: command
+    short-summary: Execute the SuperBench benchmarks locally.
+    examples:
+        - name: execute all benchmarks using image "superbench/cuda:11.1" and default benchmarking configuration
+          text: {cli_name} exec --docker-image superbench/cuda:11.1
+        - name: execute all benchmarks using image "superbench/rocm:4.0" and custom config file ./config.yaml
+          text: {cli_name} exec --docker-image superbench/rocm:4.0 --config-file ./config.yaml
+""".format(cli_name=CLI_NAME)
+
+helps['run'] = """
+    type: command
+    short-summary: Run the SuperBench benchmarks distributedly.
+    examples:
+        - name: run all benchmarks on all nodes in ./host.yaml using image "superbench/cuda:11.1"
+            and default benchmarking configuration
+          text: {cli_name} run --docker-image superbench/cuda:11.1 --host-file ./host.yaml
+""".format(cli_name=CLI_NAME)
+
+
+class SuperBenchCLIHelp(CLIHelp):
+    """SuperBench CLI help loader."""
+    def __init__(self, cli_ctx=None):
+        """Init CLI help loader.
+
+        Args:
+            cli_ctx (knack.cli.CLI, optional): CLI Context. Defaults to None.
+        """
+        super().__init__(cli_ctx=cli_ctx, welcome_message=WELCOME_MESSAGE)

superbench/cli/sb.py

+#!/usr/bin/env python3
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""SuperBench command line interface."""
+
+import sys
+
+from knack import CLI
+
+import superbench
+from superbench.cli._help import CLI_NAME, SuperBenchCLIHelp
+from superbench.cli._commands import SuperBenchCommandsLoader
+
+
+class SuperBenchCLI(CLI):
+    """The main driver for SuperBench CLI."""
+    def get_cli_version(self):
+        """Get the CLI version.
+
+        Returns:
+            str: CLI semantic version.
+        """
+        return superbench.__version__
+
+    @classmethod
+    def get_cli(cls):
+        """Get CLI instance.
+
+        Returns:
+            SuperBenchCLI: An instance for SuperBench CLI.
+        """
+        return cls(
+            cli_name=CLI_NAME,
+            config_env_var_prefix=CLI_NAME,
+            commands_loader_cls=SuperBenchCommandsLoader,
+            help_cls=SuperBenchCLIHelp,
+        )
+
+
+def main():
+    """The main function for CLI."""
+    sb_cli = SuperBenchCLI.get_cli()
+    exit_code = sb_cli.invoke(sys.argv[1:])
+    sys.exit(exit_code)
+
+
+if __name__ == '__main__':
+    main()

superbench/cli/sb_exec.py

+#!/usr/bin/env python3
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""SuperBench sb exec command."""
+
+import hydra
+
+from superbench.common.utils import logger
+
+
+@hydra.main(config_path='../config', config_name='default')
+def main(config):
+    """The main entrypoint for sb-exec."""
+    logger.info(config)
+    # executor = SuperBenchExecutor(config)
+    # executor.exec()
+
+
+if __name__ == '__main__':
+    main()

superbench/cli/sb_run.py

+#!/usr/bin/env python3
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""SuperBench sb run command."""
+
+import hydra
+
+from superbench.common.utils import logger
+
+
+@hydra.main(config_path='../config', config_name='default')
+def main(config):
+    """The main entrypoint for sb-run."""
+    logger.info(config)
+    # runner = SuperBenchRunner(config)
+    # runner.run()
+
+
+if __name__ == '__main__':
+    main()

superbench/common/utils/__init__.py

@@ -4,5 +4,7 @@
 """Exposes the interface of SuperBench common utilities."""
 
 from .logging import logger
+from .file_handler import new_output_dir, get_config
+from .command import get_sb_command
 
-__all__ = ['logger']
+__all__ = ['logger', 'new_output_dir', 'get_config', 'get_sb_command']

superbench/common/utils/command.py

+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""Utilities for command."""
+
+
+def get_sb_command(cli, output_path, config_override):
+    """Get sb command for sb-run or sb-exec.
+
+    Args:
+        cli (str): CLI name.
+        output_path (str): Output directory path.
+        config_override (str): Extra arguments to override config.
+
+    Returns:
+        str: Command to run.
+    """
+    sb_cmd = '{cli} ' \
+        '--config-name=config.merge ' \
+        '--config-dir={path} ' \
+        'hydra.run.dir={path} ' \
+        'hydra.sweep.dir={path} ' \
+        '{args}'.format(cli=cli, path=output_path, args=config_override)
+    return sb_cmd.strip()

superbench/common/utils/file_handler.py

+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""Utilities for file."""
+
+import yaml
+from pathlib import Path
+from datetime import datetime
+
+
+def new_output_dir():
+    """Generate a new output directory.
+
+    Generate a new output directory name based on current time and create it on filesystem.
+
+    Returns:
+        str: Output directory name.
+    """
+    output_name = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
+    output_path = Path('.', 'outputs', output_name)
+    output_path.mkdir(mode=0o755, parents=True, exist_ok=True)
+    return str(output_path)
+
+
+def get_config(config_file):
+    """Read SuperBench config yaml.
+
+    Read config file, use default config if None is provided.
+
+    Args:
+        config_file (str): config file path.
+
+    Returns:
+        dict: Config object, None if file does not exist.
+    """
+    here = Path(__file__).parent.resolve()
+    p = Path(config_file) if config_file else here / '../../config/default.yaml'
+    if not p.is_file():
+        return None
+    with p.open() as f:
+        config = yaml.safe_load(f)
+    return config

superbench/config/__init__.py

+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""SuperBench config module."""

superbench/config/default.yaml

+# @package _global_
+
+# Hydra config
+hydra:
+  run:
+    dir: ./outputs/${now:%Y-%m-%d_%H-%M-%S}
+  sweep:
+    dir: ./outputs/${now:%Y-%m-%d_%H-%M-%S}
+  job_logging:
+    formatters:
+      colorlog:
+        format: >-
+          [%(cyan)s%(asctime)s %(hostname)s%(reset)s][%(blue)s%(filename)s:%(lineno)s%(reset)s][%(log_color)s%(levelname)s%(reset)s] %(message)s
+defaults:
+  - hydra/job_logging: colorlog
+  - hydra/hydra_logging: colorlog
+
+# SuperBench config
+superbench:
+  use: []
+  benchmarks:
+    pytorch_models:
+      enable: true
+      parameters:
+        batch_size: 32

tests/cli/test_sb.py

+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""SuperBench CLI command and scenario tests."""
+
+import io
+import contextlib
+from functools import wraps
+from knack.testsdk import ScenarioTest, StringCheck, NoneCheck
+
+import superbench
+from superbench.cli import SuperBenchCLI
+
+
+def capture_system_exit(func):
+    """Decorator to capture SystemExit in testing.
+
+    Args:
+        func (Callable): Decorated function.
+
+    Returns:
+        Callable: Decorator.
+    """
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        f = io.StringIO()
+        with self.assertRaises(SystemExit) as cm, contextlib.redirect_stderr(f):
+            func(self, *args, **kwargs)
+        self.assertEqual(cm.exception.code, 2)
+        self.stderr = f.getvalue()
+
+    return wrapper
+
+
+class SuperBenchCLIScenarioTest(ScenarioTest):
+    """A class whose instances are CLI single test cases.
+
+    Args:
+        ScenarioTest (knack.testsdk.ScenarioTest): Test class for knack.
+    """
+    def __init__(self, method_name):
+        """Override __init__ method for ScenarioTest.
+
+        Args:
+            method_name (str): ScenarioTest method_name.
+        """
+        sb_cli = SuperBenchCLI.get_cli()
+        super().__init__(sb_cli, method_name)
+
+    def test_sb_version(self):
+        """Test sb version."""
+        self.cmd('sb version', checks=[StringCheck(superbench.__version__)])
+
+    def test_sb_deploy(self):
+        """Test sb deploy."""
+        self.cmd('sb deploy --docker-image test:cuda11.1 --host-list localhost', expect_failure=True)
+
+    @capture_system_exit
+    def test_sb_deploy_no_docker_image(self):
+        """Test sb deploy, no --docker-image argument, should fail."""
+        self.cmd('sb deploy', expect_failure=True)
+        self.assertIn('sb deploy: error: the following arguments are required: --docker-image', self.stderr)
+
+    def test_sb_exec(self):
+        """Test sb exec."""
+        self.cmd('sb exec --docker-image test:cuda11.1', checks=[NoneCheck()])
+
+    @capture_system_exit
+    def test_sb_exec_no_docker_image(self):
+        """Test sb exec, no --docker-image argument, should fail."""
+        self.cmd('sb exec', expect_failure=True)
+        self.assertIn('sb exec: error: the following arguments are required: --docker-image', self.stderr)
+
+    def test_sb_run(self):
+        """Test sb run."""
+        self.cmd('sb run --docker-image test:cuda11.1 --host-list localhost', checks=[NoneCheck()])
+
+    @capture_system_exit
+    def test_sb_run_no_docker_image(self):
+        """Test sb run, no --docker-image argument, should fail."""
+        self.cmd('sb run', expect_failure=True)
+        self.assertIn('sb run: error: the following arguments are required: --docker-image', self.stderr)
+
+    def test_sb_run_no_host(self):
+        """Test sb run, no --host-file or --host-list, should fail."""
+        result = self.cmd('sb run --docker-image test:cuda11.1', expect_failure=True)
+        self.assertEqual(result.exit_code, 1)
+
+    def test_sb_run_nonexist_host_file(self):
+        """Test sb run, --host-file does not exist, should fail."""
+        result = self.cmd('sb run --docker-image test:cuda11.1 --host-file ./nonexist.yaml', expect_failure=True)
+        self.assertEqual(result.exit_code, 1)