Executor - Fix issues when executing benchmarks (#51)

* fix missing package in dockerfile
* update benchmark list and parameters
* catch runtime errors
* refine logging info

dockerfile/cuda11.1.1.dockerfile

@@ -21,6 +21,7 @@ RUN apt-get update && \
     openssh-client \
     openssh-server \
     pciutils \
+    libpci-dev \
     libaio-dev \
     libcap2 \
     libtinfo5

superbench/config/default.yaml

@@ -8,12 +8,6 @@ superbench:
         - pytorch
       parameters:
         num_steps: 2048
-    sharding_matmul:
-      enable: true
-      frameworks:
-        - pytorch
-      parameters:
-        num_steps: 2048
     bert_models:
       enable: true
       frameworks:
@@ -25,7 +19,7 @@ superbench:
         duration: 0
         num_warmup: 64
         num_steps: 2048
-        batch_size: 32
+        batch_size: 16
         precision:
           - float32
           - float16

superbench/executor/executor.py

@@ -97,17 +97,21 @@ def __exec_benchmark(self, context, log_suffix):
             context (BenchmarkContext): Benchmark context to launch.
             log_suffix (str): Log string suffix.
         """
+        try:
             benchmark = BenchmarkRegistry.launch_benchmark(context)
             if benchmark:
-            logger.debug(
-                'benchmark: %s, return code: %s, result: %s.', benchmark.name, benchmark.return_code, benchmark.result
+                logger.info(
+                    'benchmark: %s, return code: %s, result: %s.', benchmark.name, benchmark.return_code,
+                    benchmark.result
                 )
-            if benchmark.return_code == 0:
+                if benchmark.return_code.value == 0:
                     logger.info('Executor succeeded in %s.', log_suffix)
                 else:
                     logger.error('Executor failed in %s.', log_suffix)
             else:
                 logger.error('Executor failed in %s, invalid context.', log_suffix)
+        except Exception:
+            logger.error('Executor failed in %s.', log_suffix)
 
     def exec(self):
         """Run the SuperBench benchmarks locally."""
@@ -128,7 +132,7 @@ def exec(self):
                         )
                         self.__exec_benchmark(context, log_suffix)
                 else:
-                    log_suffix = 'micro-benchmark {}: {}'.format(benchmark_name, framework)
+                    log_suffix = 'micro-benchmark {}'.format(benchmark_name)
                     logger.info('Executor is going to execute %s.', log_suffix)
                     context = BenchmarkRegistry.create_benchmark_context(
                         benchmark_name,

tests/executor/test_executor.py

@@ -65,14 +65,8 @@ def test_get_arguments(self):
                 self.default_config.superbench.benchmarks.matmul.parameters
             ), expected_matmul_args
         )
-        expected_sharding_matmul_args = '--num_steps 2048'
-        self.assertEqual(
-            self.executor._SuperBenchExecutor__get_arguments(
-                self.default_config.superbench.benchmarks.sharding_matmul.parameters
-            ), expected_sharding_matmul_args
-        )
         expected_bert_models_args = \
-            '--duration 0 --num_warmup 64 --num_steps 2048 --batch_size 32 ' \
+            '--duration 0 --num_warmup 64 --num_steps 2048 --batch_size 16 ' \
             '--precision float32 float16 --model_action train inference'
         self.assertEqual(
             self.executor._SuperBenchExecutor__get_arguments(