CLI - Add non-zero return code for `sb [deploy,run]` (#425)

Add non-zero return code for `sb deploy` and `sb run` command when
there're Ansible failures in control plane.
Return code is set to count of failure.

For failures caused by benchmarks, return code is still set per benchmark
in results json file.

superbench/cli/_handler.py

@@ -3,6 +3,7 @@
 
 """SuperBench CLI command handler."""
 
+import sys
 from pathlib import Path
 from importlib_metadata import version, PackageNotFoundError
 
@@ -252,6 +253,8 @@ def deploy_command_handler(
 
     runner = SuperBenchRunner(sb_config, docker_config, ansible_config, sb_output_dir)
     runner.deploy()
+    if runner.get_failure_count() != 0:
+        sys.exit(runner.get_failure_count())
 
 
 def run_command_handler(
@@ -307,3 +310,5 @@ def run_command_handler(
 
     runner = SuperBenchRunner(sb_config, docker_config, ansible_config, sb_output_dir)
     runner.run()
+    if runner.get_failure_count() != 0:
+        sys.exit(runner.get_failure_count())

superbench/runner/ansible.py

@@ -27,6 +27,7 @@ def __init__(self, config):
             'cmdline': '--forks 128',
         }
         self._head_host = None
+        self.failure_count = 0
         if config:
             inventory_file = getattr(config, 'host_file', None)
             inventory_list = getattr(config, 'host_list', None)
@@ -77,6 +78,7 @@ def run(self, ansible_config, sudo=False):    # pragma: no cover
         if r.rc == 0:
             logger.info('Run succeed, return code {}.'.format(r.rc))
         else:
+            self.failure_count += 1
             logger.warning('Run failed, return code {}.'.format(r.rc))
         return r.rc
 

superbench/runner/runner.py

@@ -160,6 +160,14 @@ def __get_mode_command(self, benchmark_name, mode, timeout=None):
             logger.warning('Unknown mode %s.', mode.name)
         return mode_command.strip()
 
+    def get_failure_count(self):
+        """Get failure count during Ansible run.
+
+        Return:
+            int: Failure count.
+        """
+        return self._ansible_client.failure_count
+
     def deploy(self):    # pragma: no cover
         """Deploy SuperBench environment."""
         logger.info('Preparing SuperBench environment.')

tests/cli/test_sb.py

@@ -8,6 +8,7 @@
 from functools import wraps
 from knack.testsdk import ScenarioTest, StringContainCheck, NoneCheck, JMESPathCheck
 from pathlib import Path
+from unittest import mock
 
 import superbench
 from superbench.cli import SuperBenchCLI
@@ -53,8 +54,10 @@ def test_sb_version(self):
         """Test sb version."""
         self.cmd('sb version', checks=[StringContainCheck(superbench.__version__)])
 
-    def test_sb_deploy(self):
+    @mock.patch('superbench.runner.SuperBenchRunner.get_failure_count')
+    def test_sb_deploy(self, mocked_failure_count):
         """Test sb deploy."""
+        mocked_failure_count.return_value = 0
         self.cmd('sb deploy --host-list localhost', checks=[NoneCheck()])
 
     def test_sb_deploy_no_host(self):
@@ -65,12 +68,16 @@ def test_sb_exec(self):
         """Test sb exec."""
         self.cmd('sb exec --config-override superbench.enable=["none"]', checks=[NoneCheck()])
 
-    def test_sb_run(self):
+    @mock.patch('superbench.runner.SuperBenchRunner.get_failure_count')
+    def test_sb_run(self, mocked_failure_count):
         """Test sb run."""
+        mocked_failure_count.return_value = 0
         self.cmd('sb run --host-list localhost --config-override superbench.enable=none', checks=[NoneCheck()])
 
-    def test_sb_run_skipdocker(self):
+    @mock.patch('superbench.runner.SuperBenchRunner.get_failure_count')
+    def test_sb_run_skipdocker(self, mocked_failure_count):
         """Test sb run without docker."""
+        mocked_failure_count.return_value = 0
         self.cmd('sb run -l localhost -C superbench.enable=none --no-docker', checks=[NoneCheck()])
 
     def test_sb_run_no_docker_auth(self):

tests/runner/test_runner.py

@@ -41,6 +41,10 @@ def test_set_logger(self):
         expected_log_file = Path(self.runner._sb_output_dir) / 'sb-run.log'
         self.assertTrue(expected_log_file.is_file())
 
+    def test_get_failure_count(self):
+        """Test get_failure_count."""
+        self.assertEqual(0, self.runner.get_failure_count())
+
     def test_get_mode_command(self):
         """Test __get_mode_command."""
         test_cases = [