Benchmarks: Add Feature - Add 'ignore_invalid' option when register benchmarks. (#247)

**Description**
If `ignore_invalid` is True, and 'required' arguments are not set when register the benchmark, the arguments should be provided by user in config and skip the arguments checking.

superbench/benchmarks/base.py

@@ -60,7 +60,7 @@ def get_configurable_settings(self):
         """
         return self._parser.format_help().strip()
 
-    def parse_args(self):
+    def parse_args(self, ignore_invalid=False):
         """Parse the arguments.
 
         Return:
@@ -71,8 +71,12 @@ def parse_args(self):
         try:
             args, unknown = self._parser.parse_known_args(self._argv)
         except BaseException as e:
-            logger.error('Invalid argument - benchmark: {}, message: {}.'.format(self._name, str(e)))
-            return False, None, None
+            if ignore_invalid:
+                logger.info('Missing or invliad parameters, will ignore the error and skip the args checking.')
+                return True, None, []
+            else:
+                logger.error('Invalid argument - benchmark: {}, message: {}.'.format(self._name, str(e)))
+                return False, None, []
 
         ret = True
         if len(unknown) > 0:

superbench/benchmarks/registry.py

@@ -67,17 +67,37 @@ def register_benchmark(cls, name, class_def, parameters='', platform=None):
 
                 cls.benchmarks[name][p] = (class_def, parameters)
 
+        cls.__parse_and_check_args(name, class_def, parameters)
+
+    @classmethod
+    def __parse_and_check_args(cls, name, class_def, parameters):
+        """Parse and check the predefine parameters.
+
+        If ignore_invalid is True, and 'required' arguments are not set when register the benchmark,
+        the arguments should be provided by user in config and skip the arguments checking.
+
+        Args:
+            name (str): internal name of benchmark.
+            class_def (Benchmark): class object of benchmark.
+            parameters (str): predefined parameters of benchmark.
+        """
         benchmark = class_def(name, parameters)
         benchmark.add_parser_arguments()
-        ret, args, unknown = benchmark.parse_args()
+        ret, args, unknown = benchmark.parse_args(ignore_invalid=True)
         if not ret or len(unknown) >= 1:
             logger.log_and_raise(
                 TypeError,
                 'Registered benchmark has invalid arguments - benchmark: {}, parameters: {}'.format(name, parameters)
             )
-        else:
+        elif args is not None:
             cls.benchmarks[name]['predefine_param'] = vars(args)
             logger.debug('Benchmark registration - benchmark: {}, predefine_parameters: {}'.format(name, vars(args)))
+        else:
+            cls.benchmarks[name]['predefine_param'] = dict()
+            logger.info(
+                'Benchmark registration - benchmark: {}, missing required parameters or invalid parameters, '
+                'skip the arguments checking.'.format(name)
+            )
 
     @classmethod
     def is_benchmark_context_valid(cls, benchmark_context):

tests/benchmarks/test_registry.py

@@ -28,8 +28,7 @@ def add_parser_arguments(self):
         self._parser.add_argument(
             '--lower_bound',
             type=int,
-            default=0,
-            required=False,
+            required=True,
             help='The lower bound for accumulation.',
         )