CLI - Integration with Executor and Runner (#26)

* CLI integration with Executor and Runner

setup.py

@@ -160,8 +160,6 @@ def run(self):
     entry_points={
         'console_scripts': [
             'sb = superbench.cli.sb:main',
-            'sb-exec = superbench.cli.sb_exec:main',
-            'sb-run = superbench.cli.sb_run:main',
         ],
     },
     cmdclass={

superbench/cli/_handler.py

@@ -3,13 +3,14 @@
 
 """SuperBench CLI command handler."""
 
-import os
 from pathlib import Path
 
 from knack.util import CLIError
 from omegaconf import OmegaConf
 
 import superbench
+from superbench.runner import SuperBenchRunner
+from superbench.executor import SuperBenchExecutor
 from superbench.common.utils import create_output_dir, get_sb_config
 
 
@@ -78,12 +79,12 @@ def deploy_command_handler(
 
 
 def exec_command_handler(
-    docker_image, docker_username=None, docker_password=None, config_file=None, config_override=None
+    docker_image=None, docker_username=None, docker_password=None, config_file=None, config_override=None
 ):
     """Run the SuperBench benchmarks locally.
 
     Args:
-        docker_image (str): Docker image URI.
+        docker_image (str, optional): Docker image URI.
         docker_username (str, optional): Docker registry username if authentication is needed. Defaults to None.
         docker_password (str, optional): Docker registry password if authentication is needed. Defaults to None.
         config_file (str, optional): Path to SuperBench config file. Defaults to None.
@@ -110,7 +111,8 @@ def exec_command_handler(
     # Create output directory
     output_dir = create_output_dir()
 
-    os.system('sb-exec {}'.format(output_dir))
+    executor = SuperBenchExecutor(sb_config, docker_config, output_dir)
+    executor.exec()
 
 
 def run_command_handler(
@@ -170,4 +172,5 @@ def run_command_handler(
     # Create output directory
     output_dir = create_output_dir()
 
-    os.system('sb-run {}'.format(output_dir))
+    runner = SuperBenchRunner(sb_config, docker_config, ansible_config, output_dir)
+    runner.run()

superbench/cli/sb_exec.py

-#!/usr/bin/env python3
-
-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT License.
-
-"""SuperBench sb exec command."""
-
-
-def main():
-    """The main entrypoint for sb-exec."""
-    # executor = SuperBenchExecutor(config)
-    # executor.exec()
-
-
-if __name__ == '__main__':
-    main()

superbench/cli/sb_run.py

-#!/usr/bin/env python3
-
-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT License.
-
-"""SuperBench sb run command."""
-
-
-def main():
-    """The main entrypoint for sb-run."""
-    # runner = SuperBenchRunner(config)
-    # runner.run()
-
-
-if __name__ == '__main__':
-    main()

superbench/executor/executor.py

@@ -123,7 +123,7 @@ def exec(self):
                         context = BenchmarkRegistry.create_benchmark_context(
                             model,
                             platform=self.__get_platform(),
-                            framework=Framework(framework.lower()).name,
+                            framework=Framework(framework.lower()),
                             parameters=self.__get_arguments(benchmark_config.parameters)
                         )
                         self.__exec_benchmark(context, log_suffix)
@@ -133,7 +133,7 @@ def exec(self):
                     context = BenchmarkRegistry.create_benchmark_context(
                         benchmark_name,
                         platform=self.__get_platform(),
-                        framework=Framework(framework.lower()).name,
+                        framework=Framework(framework.lower()),
                         parameters=self.__get_arguments(benchmark_config.parameters)
                     )
                     self.__exec_benchmark(context, log_suffix)

tests/cli/test_sb.py

@@ -63,13 +63,7 @@ def test_sb_deploy_no_docker_image(self):
 
     def test_sb_exec(self):
         """Test sb exec."""
-        self.cmd('sb exec --docker-image test:cuda11.1', checks=[NoneCheck()])
-
-    @capture_system_exit
-    def test_sb_exec_no_docker_image(self):
-        """Test sb exec, no --docker-image argument, should fail."""
-        self.cmd('sb exec', expect_failure=True)
-        self.assertIn('sb exec: error: the following arguments are required: --docker-image', self.stderr)
+        self.cmd('sb exec --config-override superbench.enable=["none"]', checks=[NoneCheck()])
 
     def test_sb_run(self):
         """Test sb run."""