# Copyright (c) Microsoft Corporation. # Licensed under the MIT license. """Module of the docker-benchmark base class.""" from abc import abstractmethod from superbench.common.utils import logger, run_command from superbench.benchmarks import BenchmarkType, ReturnCode from superbench.benchmarks.base import Benchmark class DockerBenchmark(Benchmark): """The base class of benchmarks packaged in docker container.""" def __init__(self, name, parameters=''): """Constructor. Args: name (str): benchmark name. parameters (str): benchmark parameters. """ super().__init__(name, parameters) self._benchmark_type = BenchmarkType.DOCKER # Command lines to launch the docker image and run the benchmarks inside docker. self._commands = list() # Image uri of the current docker-benchmark. self._image_uri = None # Container name of the current docker-benchmark. self._container_name = None # Default options for docker run. self._default_options = '-i --rm --privileged --net=host --ipc=host' # Platform-specific options for docker run. self._platform_options = None # Entrypoint option of the current docker-benchmark. self._entrypoint = None # CMD option of the current docker-benchmark. self._cmd = None ''' # If need to add new arguments, super().add_parser_arguments() must be called. def add_parser_arguments(self): """Add the specified arguments.""" super().add_parser_arguments() ''' def _preprocess(self): """Preprocess/preparation operations before the benchmarking. Return: True if _preprocess() succeed. """ if not super()._preprocess(): return False if self._image_uri is None: self._result.set_return_code(ReturnCode.DOCKERBENCHMARK_IMAGE_NOT_SET) logger.error('The image uri is not set - benchmark: {}.'.format(self._name)) return False if self._container_name is None: self._result.set_return_code(ReturnCode.DOCKERBENCHMARK_CONTAINER_NOT_SET) logger.error('The container name is not set - benchmark: {}.'.format(self._name)) return False output = run_command('docker pull --quiet {}'.format(self._image_uri)) if output.returncode != 0: self._result.set_return_code(ReturnCode.DOCKERBENCHMARK_IMAGE_PULL_FAILURE) logger.error( 'DockerBenchmark pull image failed - benchmark: {}, error message: {}.'.format( self._name, output.stdout ) ) return False command = 'docker run ' command += self._default_options command += ' --name={container_name} {platform_options} {entrypoint} {image} {cmd}' self._commands.append( command.format( container_name=self._container_name, platform_options=self._platform_options or '', entrypoint='' if self._entrypoint is None else '--entrypoint {}'.format(self._entrypoint), image=self._image_uri, cmd=self._cmd or '' ) ) return True def _postprocess(self): """Postprocess/cleanup operations after the benchmarking. Return: True if _postprocess() succeed. """ rm_containers = 'docker stop --time 20 {container} && docker rm {container}'.format( container=self._container_name ) run_command(rm_containers) rm_image = 'docker rmi {}'.format(self._image_uri) run_command(rm_image) return True def _benchmark(self): """Implementation for benchmarking. Return: True if run benchmark successfully. """ for cmd_idx in range(len(self._commands)): logger.info( 'Execute command - round: {}, benchmark: {}, command: {}.'.format( self._curr_run_index, self._name, self._commands[cmd_idx] ) ) output = run_command(self._commands[cmd_idx], flush_output=self._args.log_flushing) if output.returncode != 0: self._result.set_return_code(ReturnCode.DOCKERBENCHMARK_EXECUTION_FAILURE) logger.error( 'DockerBenchmark execution failed - round: {}, benchmark: {}, error message: {}.'.format( self._curr_run_index, self._name, output.stdout ) ) return False else: if not self._process_raw_result(cmd_idx, output.stdout): self._result.set_return_code(ReturnCode.DOCKERBENCHMARK_RESULT_PARSING_FAILURE) return False return True @abstractmethod def _process_raw_result(self, cmd_idx, raw_output): """Function to process raw results and save the summarized results. self._result.add_raw_data() and self._result.add_result() need to be called to save the results. Args: cmd_idx (int): the index of command corresponding with the raw_output. raw_output (str): raw output string of the docker-benchmark. Return: True if the raw output string is valid and result can be extracted. """ pass def print_env_info(self): """Print environments or dependencies information.""" # TODO: will implement it when add real benchmarks in the future. pass class CudaDockerBenchmark(DockerBenchmark): """The base class of benchmarks packaged in nvidia docker container.""" def __init__(self, name, parameters=''): """Constructor. Args: name (str): benchmark name. parameters (str): benchmark parameters. """ super().__init__(name, parameters) self._platform_options = '--gpus=all' class RocmDockerBenchmark(DockerBenchmark): """The base class of benchmarks packaged in amd docker container.""" def __init__(self, name, parameters=''): """Constructor. Args: name (str): benchmark name. parameters (str): benchmark parameters. """ super().__init__(name, parameters) self._platform_options = '--security-opt seccomp=unconfined --group-add video'