Benchmarks: Code Revision - Revise MicroBenchmark class to be more flexible. (#66)

* Revise MicroBenchmark class to be more flexible.
* use command index not the command as the parameter.
* changes according to discussion.

superbench/benchmarks/micro_benchmarks/__init__.py

@@ -3,8 +3,8 @@
 
 """A module containing all the micro-benchmarks."""
 
-from superbench.benchmarks.micro_benchmarks.micro_base import MicroBenchmark
+from superbench.benchmarks.micro_benchmarks.micro_base import MicroBenchmark, MicroBenchmarkWithInvoke
 from superbench.benchmarks.micro_benchmarks.sharding_matmul import ShardingMatmul
 from superbench.benchmarks.micro_benchmarks.computation_communication_overlap import ComputationCommunicationOverlap
 
-__all__ = ['MicroBenchmark', 'ShardingMatmul', 'ComputationCommunicationOverlap']
+__all__ = ['MicroBenchmark', 'MicroBenchmarkWithInvoke', 'ShardingMatmul', 'ComputationCommunicationOverlap']

superbench/benchmarks/micro_benchmarks/micro_base.py

@@ -3,10 +3,13 @@
 
 """Module of the micro-benchmark base class."""
 
+import os
+import subprocess
+import shutil
 from abc import abstractmethod
 
 from superbench.common.utils import logger
-from superbench.benchmarks import BenchmarkType
+from superbench.benchmarks import BenchmarkType, ReturnCode
 from superbench.benchmarks.base import Benchmark
 
 
@@ -21,8 +24,6 @@ def __init__(self, name, parameters=''):
         """
         super().__init__(name, parameters)
         self._benchmark_type = BenchmarkType.MICRO
-        # Command lines to launch the micro-benchmarks.
-        self.__commands = list()
 
     '''
     # If need to add new arguments, super().add_parser_arguments() must be called.
@@ -70,21 +71,134 @@ def _process_numeric_result(self, metric, result):
         self._result.add_result(metric, sum(result) / len(result))
         return True
 
-    def _process_raw_result(self, raw_output):
+    def print_env_info(self):
+        """Print environments or dependencies information."""
+        # TODO: will implement it when add real benchmarks in the future.
+        pass
+
+
+class MicroBenchmarkWithInvoke(MicroBenchmark):
+    """The base class of micro-benchmarks that need to invoke subprocesses."""
+    def __init__(self, name, parameters=''):
+        """Constructor.
+
+        Args:
+            name (str): benchmark name.
+            parameters (str): benchmark parameters.
+        """
+        super().__init__(name, parameters)
+
+        # Command lines to launch the micro-benchmarks.
+        self._commands = list()
+
+        # Binary name of the current micro-benchmark.
+        self._bin_name = None
+
+    def add_parser_arguments(self):
+        """Add the specified arguments."""
+        super().add_parser_arguments()
+
+        self._parser.add_argument(
+            '--bin_dir',
+            type=str,
+            default=None,
+            required=False,
+            help='Specify the directory of the benchmark binary.',
+        )
+
+    def _set_binary_path(self):
+        """Search the binary from self._args.bin_dir or from system environment path and set the binary directory.
+
+        If self._args.bin_dir is specified, the binary is only searched inside it. Otherwise, the binary is searched
+        from system environment path.
+
+        Return:
+            True if the binary exists.
+        """
+        if self._bin_name is None:
+            self._result.set_return_code(ReturnCode.MICROBENCHMARK_BINARY_NAME_NOT_SET)
+            logger.error('The binary name is not set - benchmark: {}.'.format(self._name))
+            return False
+
+        self._args.bin_dir = shutil.which(self._bin_name, mode=os.X_OK, path=self._args.bin_dir)
+
+        if self._args.bin_dir is None:
+            self._result.set_return_code(ReturnCode.MICROBENCHMARK_BINARY_NOT_EXIST)
+            logger.error(
+                'The binary does not exist - benchmark: {}, binary name: {}, binary directory: {}.'.format(
+                    self._name, self._bin_name, self._args.bin_dir
+                )
+            )
+            return False
+
+        self._args.bin_dir = os.path.dirname(self._args.bin_dir)
+
+        return True
+
+    def _preprocess(self):
+        """Preprocess/preparation operations before the benchmarking.
+
+        Return:
+            True if _preprocess() succeed.
+        """
+        if not super()._preprocess():
+            return False
+
+        # Set the environment path.
+        if 'SB_MICRO_PATH' in os.environ:
+            os.environ['PATH'] = os.getenv('SB_MICRO_PATH', '') + os.pathsep + os.getenv('PATH', '')
+
+        if not self._set_binary_path():
+            return False
+
+        return True
+
+    def _benchmark(self):
+        """Implementation for benchmarking.
+
+        Return:
+            True if run benchmark successfully.
+        """
+        for cmd_idx in range(len(self._commands)):
+            logger.info(
+                'Execute command - round: {}, benchmark: {}, command: {}.'.format(
+                    self._curr_run_index, self._name, self._commands[cmd_idx]
+                )
+            )
+            output = subprocess.run(
+                self._commands[cmd_idx],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                shell=True,
+                check=False,
+                universal_newlines=True
+            )
+            if output.returncode != 0:
+                self._result.set_return_code(ReturnCode.MICROBENCHMARK_EXECUTION_FAILURE)
+                logger.error(
+                    'Microbenchmark execution failed - round: {}, benchmark: {}, error message: {}.'.format(
+                        self._curr_run_index, self._name, output.stdout
+                    )
+                )
+                return False
+            else:
+                if not self._process_raw_result(cmd_idx, output.stdout):
+                    self._result.set_return_code(ReturnCode.MICROBENCHMARK_RESULT_PARSING_FAILURE)
+                    return False
+
+        return True
+
+    @abstractmethod
+    def _process_raw_result(self, cmd_idx, raw_output):
         """Function to process raw results and save the summarized results.
 
           self._result.add_raw_data() and self._result.add_result() need to be called to save the results.
 
         Args:
+            cmd_idx (int): the index of command corresponding with the raw_output.
             raw_output (str): raw output string of the micro-benchmark.
 
         Return:
             True if the raw output string is valid and result can be extracted.
         """
-        # TODO: will implement it when add real benchmarks in the future.
-        return True
-
-    def print_env_info(self):
-        """Print environments or dependencies information."""
-        # TODO: will implement it when add real benchmarks in the future.
         pass

superbench/benchmarks/return_code.py

@@ -23,3 +23,8 @@ class ReturnCode(Enum):
     DATALOADER_INIT_FAILURE = 16
     OPTIMIZER_CREATION_FAILURE = 17
     MODEL_CREATION_FAILURE = 18
+    # Return codes related with micro benchmarks.
+    MICROBENCHMARK_BINARY_NAME_NOT_SET = 30
+    MICROBENCHMARK_BINARY_NOT_EXIST = 31
+    MICROBENCHMARK_EXECUTION_FAILURE = 32
+    MICROBENCHMARK_RESULT_PARSING_FAILURE = 33

tests/benchmarks/micro_benchmarks/test_micro_base.py

+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""Tests for MicroBenchmark and MicroBenchmarkWithInvoke modules."""
+
+import os
+import re
+import shutil
+
+from superbench.benchmarks import BenchmarkType, ReturnCode
+from superbench.benchmarks.micro_benchmarks import MicroBenchmark, MicroBenchmarkWithInvoke
+
+
+class FakeMicroBenchmark(MicroBenchmark):
+    """Fake benchmark inherit from MicroBenchmark."""
+    def __init__(self, name, parameters=''):
+        """Constructor.
+
+        Args:
+            name: benchmark name.
+            parameters: benchmark parameters.
+        """
+        super().__init__(name, parameters)
+
+    def _benchmark(self):
+        """Implementation for benchmarking.
+
+        Return:
+            True if run benchmark successfully.
+        """
+        return True
+
+
+class FakeMicroBenchmarkWithInvoke(MicroBenchmarkWithInvoke):
+    """Fake benchmark inherit from MicroBenchmarkWithInvoke."""
+    def __init__(self, name, parameters=''):
+        """Constructor.
+
+        Args:
+            name: benchmark name.
+            parameters: benchmark parameters.
+        """
+        super().__init__(name, parameters)
+
+    def _preprocess(self):
+        """Preprocess/preparation operations before the benchmarking.
+
+        Return:
+            True if _preprocess() succeed.
+        """
+        if not super()._preprocess():
+            return False
+
+        command = os.path.join(self._args.bin_dir, self._bin_name)
+        command += " -n 'cost1: 10.2, cost2: 20.2'"
+        self._commands.append(command)
+
+        return True
+
+    def _process_raw_result(self, cmd_idx, raw_output):
+        """Function to process raw results and save the summarized results.
+
+          self._result.add_raw_data() and self._result.add_result() need to be called to save the results.
+
+        Args:
+            cmd_idx (int): the index of command corresponding with the raw_output.
+            raw_output (str): raw output string of the micro-benchmark.
+
+        Return:
+            True if the raw output string is valid and result can be extracted.
+        """
+        self._result.add_raw_data('raw_output_' + str(cmd_idx), raw_output)
+        pattern = r'\d+\.\d+'
+        result = re.findall(pattern, raw_output)
+        if len(result) != 2:
+            return False
+
+        try:
+            result = [float(item) for item in result]
+        except BaseException:
+            return False
+
+        self._result.add_result('cost1', result[0])
+        self._result.add_result('cost2', result[1])
+
+        return True
+
+
+def test_micro_benchmark_base():
+    """Test MicroBenchmark."""
+    benchmark = FakeMicroBenchmark('fake')
+    assert (benchmark._benchmark_type == BenchmarkType.MICRO)
+    assert (benchmark.run())
+    assert (benchmark.return_code == ReturnCode.SUCCESS)
+    benchmark._process_numeric_result('metric1', [1, 2, 3, 4, 5, 6])
+    assert (benchmark.result['metric1'] == [3.5])
+    assert (benchmark.raw_data['metric1'] == [[1, 2, 3, 4, 5, 6]])
+
+
+def test_micro_benchmark_with_invoke_base():
+    """Test MicroBenchmarkWithInvoke."""
+    # Negative case - MICROBENCHMARK_BINARY_NAME_NOT_SET.
+    benchmark = FakeMicroBenchmarkWithInvoke('fake')
+    assert (benchmark._benchmark_type == BenchmarkType.MICRO)
+    assert (benchmark.run() is False)
+    assert (benchmark.return_code == ReturnCode.MICROBENCHMARK_BINARY_NAME_NOT_SET)
+
+    # Negative case - MICROBENCHMARK_BINARY_NOT_EXIST.
+    benchmark = FakeMicroBenchmarkWithInvoke('fake')
+    benchmark._bin_name = 'not_existed_binary'
+    assert (benchmark.run() is False)
+    assert (benchmark.return_code == ReturnCode.MICROBENCHMARK_BINARY_NOT_EXIST)
+
+    # Positive case.
+    benchmark = FakeMicroBenchmarkWithInvoke('fake')
+    benchmark._bin_name = 'echo'
+    assert (benchmark.run())
+    assert (benchmark.return_code == ReturnCode.SUCCESS)
+    assert (os.path.join(benchmark._args.bin_dir, benchmark._bin_name) == shutil.which(benchmark._bin_name))
+    assert (benchmark._commands[0] == (shutil.which(benchmark._bin_name) + " -n 'cost1: 10.2, cost2: 20.2'"))
+    assert (benchmark.raw_data['raw_output_0'] == ['cost1: 10.2, cost2: 20.2'])
+    assert (benchmark.result['cost1'] == [10.2])
+    assert (benchmark.result['cost2'] == [20.2])