Analyzer: Rename fields in json of data diagnosis to be more readable (#382)

**Description**
Rename field in data diagnosis to be more readable.

**Major Revision**
- rename fields according to diagnosis/metric format

**Minor Revision**
- change type of diagnosis/issue_num to be int

superbench/analyzer/data_diagnosis.py

@@ -232,20 +232,22 @@ def output_all_nodes_results(self, raw_data_df, data_not_accept_df):
             data_not_accept_df (DataFrame): defective nodes's detailed information
 
         Returns:
-            DataFrame: all nodes' detailed information inluding ['Accept','#Issues','Category','Issue_Details']
+            DataFrame: all nodes' detailed information inluding ['Accept','Number_Of_Issues','Category','Issue_Details']
         """
-        append_columns = ['Accept', '#Issues', 'Category', 'Issue_Details']
+        append_columns = ['Accept', 'Number_Of_Issues', 'Category', 'Issue_Details']
         all_data_df = (raw_data_df).astype('float64')
 
         if data_not_accept_df.shape[0] == 0:
             all_data_df['Accept'] = [True for i in range(len(all_data_df))]
-            all_data_df['#Issues'] = [0 for i in range(len(all_data_df))]
+            all_data_df['Number_Of_Issues'] = [0 for i in range(len(all_data_df))]
             all_data_df['Category'] = [None for i in range(len(all_data_df))]
             all_data_df['Issue_Details'] = [None for i in range(len(all_data_df))]
 
         elif data_not_accept_df.shape[0] > 0:
             data_not_accept_df['Accept'] = [False for i in range(len(data_not_accept_df))]
-            data_not_accept_df['#Issues'] = data_not_accept_df['Defective Details'].map(lambda x: len(x.split(',')))
+            data_not_accept_df['Number_Of_Issues'] = data_not_accept_df['Defective Details'].map(
+                lambda x: len(x.split(','))
+            )
             data_not_accept_df = data_not_accept_df.rename(columns={'Defective Details': 'Issue_Details'})
             for index in range(len(append_columns)):
                 if append_columns[index] not in data_not_accept_df:
@@ -260,7 +262,8 @@ def output_all_nodes_results(self, raw_data_df, data_not_accept_df):
                         data_not_accept_df[[append_columns[index]]], left_index=True, right_index=True, how='left'
                     )
             all_data_df['Accept'] = all_data_df['Accept'].replace(np.nan, True)
-            all_data_df['#Issues'] = all_data_df['#Issues'].replace(np.nan, 0)
+            all_data_df['Number_Of_Issues'] = all_data_df['Number_Of_Issues'].replace(np.nan, 0)
+            all_data_df['Number_Of_Issues'] = all_data_df['Number_Of_Issues'].astype(int)
 
         all_data_df = all_data_df.replace(np.nan, '')
 
@@ -321,6 +324,14 @@ def output_diagnosis_in_json(self, data_not_accept_df, output_path):
             output_path (str): the path of output jsonl file
         """
         data_not_accept_df['Index'] = data_not_accept_df.index
+        data_not_accept_df = data_not_accept_df.rename(
+            columns={
+                'Issue_Details': 'diagnosis/issue_details',
+                'Category': 'diagnosis/category',
+                'Number_Of_Issues': 'diagnosis/issue_num',
+                'Accept': 'diagnosis/accept'
+            }
+        )
         data_not_accept_json = data_not_accept_df.to_json(orient='records')
         data_not_accept = json.loads(data_not_accept_json)
         p = Path(output_path)

tests/data/diagnosis_summary.json

@@ -462,10 +462,10 @@
         "vgg_models/pytorch-vgg19/throughput_train_float32": 429.8092158311,
         "vgg_models/pytorch-vgg19/steptime_train_float16": 45.2033062465,
         "vgg_models/pytorch-vgg19/throughput_train_float16": 709.1127328377,
-        "Accept": false,
-        "#Issues": 1.0,
-        "Category": "KernelLaunch",
-        "Issue_Details": "kernel-launch/event_overhead:0(B/L: 0.0060 VAL: 0.1000 VAR: 1577.85% Rule:lambda x:x>0.05)",
+        "diagnosis/accept": false,
+        "diagnosis/issue_num": 1,
+        "diagnosis/category": "KernelLaunch",
+        "diagnosis/issue_details": "kernel-launch/event_overhead:0(B/L: 0.0060 VAL: 0.1000 VAR: 1577.85% Rule:lambda x:x>0.05)",
         "Index": "sb-validation-01"
     },
     {
@@ -931,10 +931,10 @@
         "vgg_models/pytorch-vgg19/throughput_train_float32": 429.8092158311,
         "vgg_models/pytorch-vgg19/steptime_train_float16": 45.2033062465,
         "vgg_models/pytorch-vgg19/throughput_train_float16": 709.1127328377,
-        "Accept": true,
-        "#Issues": 0.0,
-        "Category": "",
-        "Issue_Details": "",
+        "diagnosis/accept": true,
+        "diagnosis/issue_num": 0,
+        "diagnosis/category": "",
+        "diagnosis/issue_details": "",
         "Index": "sb-validation-02"
     },
     {
@@ -1400,10 +1400,10 @@
         "vgg_models/pytorch-vgg19/throughput_train_float32": 429.8092158311,
         "vgg_models/pytorch-vgg19/steptime_train_float16": 45.2033062465,
         "vgg_models/pytorch-vgg19/throughput_train_float16": 709.1127328377,
-        "Accept": false,
-        "#Issues": 17.0,
-        "Category": "FailedTest",
-        "Issue_Details": "mem-bw/D2H_Mem_BW:0_miss,mem-bw/D2H_Mem_BW:1_miss,mem-bw/D2H_Mem_BW:2_miss,mem-bw/D2H_Mem_BW:3_miss,mem-bw/D2H_Mem_BW:4_miss,mem-bw/D2H_Mem_BW:5_miss,mem-bw/D2H_Mem_BW:6_miss,mem-bw/D2H_Mem_BW:7_miss,mem-bw/H2D_Mem_BW:0_miss,mem-bw/H2D_Mem_BW:1_miss,mem-bw/H2D_Mem_BW:2_miss,mem-bw/H2D_Mem_BW:3_miss,mem-bw/H2D_Mem_BW:4_miss,mem-bw/H2D_Mem_BW:5_miss,mem-bw/H2D_Mem_BW:6_miss,mem-bw/H2D_Mem_BW:7_miss,mem-bw/return_code(VAL: 1.0000 Rule:lambda x:x>0)",
+        "diagnosis/accept": false,
+        "diagnosis/issue_num": 17,
+        "diagnosis/category": "FailedTest",
+        "diagnosis/issue_details": "mem-bw/D2H_Mem_BW:0_miss,mem-bw/D2H_Mem_BW:1_miss,mem-bw/D2H_Mem_BW:2_miss,mem-bw/D2H_Mem_BW:3_miss,mem-bw/D2H_Mem_BW:4_miss,mem-bw/D2H_Mem_BW:5_miss,mem-bw/D2H_Mem_BW:6_miss,mem-bw/D2H_Mem_BW:7_miss,mem-bw/H2D_Mem_BW:0_miss,mem-bw/H2D_Mem_BW:1_miss,mem-bw/H2D_Mem_BW:2_miss,mem-bw/H2D_Mem_BW:3_miss,mem-bw/H2D_Mem_BW:4_miss,mem-bw/H2D_Mem_BW:5_miss,mem-bw/H2D_Mem_BW:6_miss,mem-bw/H2D_Mem_BW:7_miss,mem-bw/return_code(VAL: 1.0000 Rule:lambda x:x>0)",
         "Index": "sb-validation-03"
     }
 ]
\ No newline at end of file