Benchmarks: Fix Bug - Fix return code overwrite issue (#94)

* fix return code reset issue

superbench/benchmarks/model_benchmarks/model_base.py

@@ -237,6 +237,7 @@ def __train(self, precision):
         # The unit of step time should be millisecond.
         step_times = self._train_step(precision)
         if not self.__process_model_result(ModelAction.TRAIN, precision, step_times):
+            self._result.set_return_code(ReturnCode.INVALID_BENCHMARK_RESULT)
             return False
 
         logger.info(
@@ -261,6 +262,7 @@ def __inference(self, precision):
         # The unit of step time should be millisecond.
         step_times = self._inference_step(precision)
         if not self.__process_model_result(ModelAction.INFERENCE, precision, step_times):
+            self._result.set_return_code(ReturnCode.INVALID_BENCHMARK_RESULT)
             return False
 
         logger.info(
@@ -323,11 +325,9 @@ def _benchmark(self):
                 self._sub_benchmark_start_time = time.time()
                 if model_action == ModelAction.TRAIN:
                     if not self.__train(precision):
-                        self._result.set_return_code(ReturnCode.MODEL_TRAIN_FAILURE)
                         return False
                 elif model_action == ModelAction.INFERENCE:
                     if not self.__inference(precision):
-                        self._result.set_return_code(ReturnCode.MODEL_INFERENCE_FAILURE)
                         return False
                 else:
                     logger.warning(

superbench/benchmarks/return_code.py

@@ -15,8 +15,6 @@ class ReturnCode(Enum):
     INVALID_BENCHMARK_RESULT = 3
     # Return codes related with model benchmarks.
     NO_SUPPORTED_PRECISION = 10
-    MODEL_TRAIN_FAILURE = 11
-    MODEL_INFERENCE_FAILURE = 12
     DISTRIBUTED_SETTING_INIT_FAILURE = 13
     DISTRIBUTED_SETTING_DESTROY_FAILURE = 14
     DATASET_GENERATION_FAILURE = 15

tests/benchmarks/model_benchmarks/test_model_base.py

@@ -228,7 +228,7 @@ def test_train():
     # Step time list is empty (simulate training failure).
     benchmark = create_benchmark('--num_steps 0')
     expected_result = (
-        '{"name": "pytorch-fake-model", "type": "model", "run_count": 1, "return_code": 0, '
+        '{"name": "pytorch-fake-model", "type": "model", "run_count": 1, "return_code": 3, '
         '"start_time": null, "end_time": null, "raw_data": {}, "result": {}}'
     )
     assert (benchmark._preprocess())
@@ -252,7 +252,7 @@ def test_inference():
     # Step time list is empty (simulate inference failure).
     benchmark = create_benchmark('--num_steps 0')
     expected_result = (
-        '{"name": "pytorch-fake-model", "type": "model", "run_count": 1, "return_code": 0, '
+        '{"name": "pytorch-fake-model", "type": "model", "run_count": 1, "return_code": 3, '
         '"start_time": null, "end_time": null, "raw_data": {}, "result": {}}'
     )
     assert (benchmark._preprocess())
@@ -306,13 +306,13 @@ def test_benchmark():
     benchmark = create_benchmark('--num_steps 0')
     assert (benchmark._preprocess())
     assert (benchmark._benchmark() is False)
-    assert (benchmark.return_code == ReturnCode.MODEL_TRAIN_FAILURE)
+    assert (benchmark.return_code == ReturnCode.INVALID_BENCHMARK_RESULT)
 
     # Negative case for _benchmark() - model inference failure, step time list is empty.
     benchmark = create_benchmark('--model_action inference --num_steps 0')
     assert (benchmark._preprocess())
     assert (benchmark._benchmark() is False)
-    assert (benchmark.return_code == ReturnCode.MODEL_INFERENCE_FAILURE)
+    assert (benchmark.return_code == ReturnCode.INVALID_BENCHMARK_RESULT)
 
 
 def test_check_result_format():