Skip to content

GitLab

Unverified Commit 55b0f9d2 authored by user4543's avatar user4543 Committed by GitHub
Browse files

Analyzer: Add Feature - Output results of all nodes in data diagnosis (#336) 

**Description**
Output results of all nodes in data diagnosis.
parent 56c9a711
Hide whitespace changes
Inline Side-by-side
Showing with 239 additions and 9 deletions
superbench/analyzer/data_diagnosis.py
View file @ 55b0f9d2
...@@ -7,6 +7,7 @@ ...@@ -7,6 +7,7 @@
import json import json
import pandas as pd import pandas as pd
import numpy as np
from superbench.common.utils import logger from superbench.common.utils import logger
from superbench.analyzer.diagnosis_rule_op import RuleOp, DiagnosisRuleType from superbench.analyzer.diagnosis_rule_op import RuleOp, DiagnosisRuleType
...@@ -209,6 +210,48 @@ def run_diagnosis_rules(self, rules, baseline): ...@@ -209,6 +210,48 @@ def run_diagnosis_rules(self, rules, baseline):
logger.error('DataDiagnosis: run diagnosis rules failed, message: {}'.format(str(e))) logger.error('DataDiagnosis: run diagnosis rules failed, message: {}'.format(str(e)))
return data_not_accept_df, label_df return data_not_accept_df, label_df
def output_all_nodes_results(self, raw_data_df, data_not_accept_df):
"""Output diagnosis results of all nodes.
Args:
raw_data_df (DataFrame): raw data
data_not_accept_df (DataFrame): defective nodes's detailed information
Returns:
DataFrame: all nodes' detailed information inluding ['Accept','#Issues','Category','Issue_Details']
"""
append_columns = ['Accept', '#Issues', 'Category', 'Issue_Details']
all_data_df = (raw_data_df[self._enable_metrics]).astype('float64')
if data_not_accept_df.shape[0] == 0:
all_data_df['Accept'] = [True for i in range(len(all_data_df))]
all_data_df['#Issues'] = [0 for i in range(len(all_data_df))]
all_data_df['Category'] = [None for i in range(len(all_data_df))]
all_data_df['Issue_Details'] = [None for i in range(len(all_data_df))]
elif data_not_accept_df.shape[0] > 0:
data_not_accept_df['Accept'] = [False for i in range(len(data_not_accept_df))]
data_not_accept_df['#Issues'] = data_not_accept_df['Defective Details'].map(lambda x: len(x.split(',')))
data_not_accept_df = data_not_accept_df.rename(columns={'Defective Details': 'Issue_Details'})
for index in range(len(append_columns)):
if append_columns[index] not in data_not_accept_df:
logger.warning(
'DataDiagnosis: output_all_nodes_results - column {} not found in data_not_accept_df.'.format(
append_columns[index]
)
)
all_data_df[append_columns[index]] = None
else:
all_data_df = all_data_df.merge(
data_not_accept_df[[append_columns[index]]], left_index=True, right_index=True, how='left'
)
all_data_df['Accept'] = all_data_df['Accept'].replace(np.nan, True)
all_data_df['#Issues'] = all_data_df['#Issues'].replace(np.nan, 0)
all_data_df = all_data_df.replace(np.nan, '')
return all_data_df
def output_diagnosis_in_excel(self, raw_data_df, data_not_accept_df, output_path, rules): def output_diagnosis_in_excel(self, raw_data_df, data_not_accept_df, output_path, rules):
"""Output the raw_data_df and data_not_accept_df results into excel file. """Output the raw_data_df and data_not_accept_df results into excel file.
...@@ -230,7 +273,7 @@ def output_diagnosis_in_excel(self, raw_data_df, data_not_accept_df, output_path ...@@ -230,7 +273,7 @@ def output_diagnosis_in_excel(self, raw_data_df, data_not_accept_df, output_path
except Exception as e: except Exception as e:
logger.error('DataDiagnosis: excel_data_output - {}'.format(str(e))) logger.error('DataDiagnosis: excel_data_output - {}'.format(str(e)))
def output_diagnosis_in_json(self, data_not_accept_df, output_path): def output_diagnosis_in_jsonl(self, data_not_accept_df, output_path):
"""Output data_not_accept_df into jsonl file. """Output data_not_accept_df into jsonl file.
Args: Args:
...@@ -256,6 +299,20 @@ def output_diagnosis_in_json(self, data_not_accept_df, output_path): ...@@ -256,6 +299,20 @@ def output_diagnosis_in_json(self, data_not_accept_df, output_path):
except Exception as e: except Exception as e:
logger.error('DataDiagnosis: output json data failed, msg: {}'.format(str(e))) logger.error('DataDiagnosis: output json data failed, msg: {}'.format(str(e)))
def output_diagnosis_in_json(self, data_not_accept_df, output_path):
"""Output data_not_accept_df into json file.
Args:
data_not_accept_df (DataFrame): the DataFrame to output
output_path (str): the path of output jsonl file
"""
data_not_accept_df['Index'] = data_not_accept_df.index
data_not_accept_json = data_not_accept_df.to_json(orient='records')
data_not_accept = json.loads(data_not_accept_json)
p = Path(output_path)
with p.open('w') as f:
json.dump(data_not_accept, f, indent=4)
def generate_md_lines(self, data_not_accept_df, rules, round): def generate_md_lines(self, data_not_accept_df, rules, round):
"""Convert DataFrame into markdown lines. """Convert DataFrame into markdown lines.
...@@ -293,7 +350,9 @@ def generate_md_lines(self, data_not_accept_df, rules, round): ...@@ -293,7 +350,9 @@ def generate_md_lines(self, data_not_accept_df, rules, round):
lines = file_handler.generate_md_table(data_not_accept_df, header) lines = file_handler.generate_md_table(data_not_accept_df, header)
return lines return lines
def run(self, raw_data_file, rule_file, baseline_file, output_dir, output_format='excel', round=2): def run(
self, raw_data_file, rule_file, baseline_file, output_dir, output_format='excel', output_all=False, round=2
):
"""Run the data diagnosis and output the results. """Run the data diagnosis and output the results.
Args: Args:
...@@ -301,6 +360,7 @@ def run(self, raw_data_file, rule_file, baseline_file, output_dir, output_format ...@@ -301,6 +360,7 @@ def run(self, raw_data_file, rule_file, baseline_file, output_dir, output_format
rule_file (str): The path of baseline yaml file rule_file (str): The path of baseline yaml file
baseline_file (str): The path of baseline json file baseline_file (str): The path of baseline json file
output_dir (str): the directory of output file output_dir (str): the directory of output file
output_all (bool): output diagnosis results for all nodes
output_format (str): the format of the output, 'excel' or 'json' output_format (str): the format of the output, 'excel' or 'json'
round (int): the number of decimal digits round (int): the number of decimal digits
""" """
...@@ -312,12 +372,21 @@ def run(self, raw_data_file, rule_file, baseline_file, output_dir, output_format ...@@ -312,12 +372,21 @@ def run(self, raw_data_file, rule_file, baseline_file, output_dir, output_format
data_not_accept_df, label_df = self.run_diagnosis_rules(rules, baseline) data_not_accept_df, label_df = self.run_diagnosis_rules(rules, baseline)
logger.info('DataDiagnosis: Processed finished') logger.info('DataDiagnosis: Processed finished')
output_path = '' output_path = ''
# generate all nodes' info
if output_all:
output_path = str(Path(output_dir) / 'diagnosis_summary.json')
data_not_accept_df = self.output_all_nodes_results(self._raw_data_df, data_not_accept_df)
# output according format
if output_format == 'excel': if output_format == 'excel':
output_path = str(Path(output_dir) / 'diagnosis_summary.xlsx') output_path = str(Path(output_dir) / 'diagnosis_summary.xlsx')
self.output_diagnosis_in_excel(self._raw_data_df, data_not_accept_df, output_path, self._sb_rules) self.output_diagnosis_in_excel(self._raw_data_df, data_not_accept_df, output_path, self._sb_rules)
elif output_format == 'json': elif output_format == 'json':
output_path = str(Path(output_dir) / 'diagnosis_summary.jsonl') if output_all:
self.output_diagnosis_in_json(data_not_accept_df, output_path) output_path = str(Path(output_dir) / 'diagnosis_summary.json')
self.output_diagnosis_in_json(data_not_accept_df, output_path)
else:
output_path = str(Path(output_dir) / 'diagnosis_summary.jsonl')
self.output_diagnosis_in_jsonl(data_not_accept_df, output_path)
elif output_format == 'md' or output_format == 'html': elif output_format == 'md' or output_format == 'html':
lines = self.generate_md_lines(data_not_accept_df, self._sb_rules, round) lines = self.generate_md_lines(data_not_accept_df, self._sb_rules, round)
if output_format == 'md': if output_format == 'md':
......
tests/analyzer/test_data_diagnosis.py
View file @ 55b0f9d2
...@@ -24,12 +24,13 @@ def setUp(self): ...@@ -24,12 +24,13 @@ def setUp(self):
self.output_json_file = str(self.parent_path / 'diagnosis_summary.jsonl') self.output_json_file = str(self.parent_path / 'diagnosis_summary.jsonl')
self.output_md_file = str(self.parent_path / 'diagnosis_summary.md') self.output_md_file = str(self.parent_path / 'diagnosis_summary.md')
self.output_html_file = str(self.parent_path / 'diagnosis_summary.html') self.output_html_file = str(self.parent_path / 'diagnosis_summary.html')
self.output_all_json_file = str(self.parent_path / 'diagnosis_summary.json')
def tearDown(self): def tearDown(self):
"""Method called after the test method has been called and the result recorded.""" """Method called after the test method has been called and the result recorded."""
for file in [ for file in [
self.output_excel_file, self.output_json_file, self.test_rule_file_fake, self.output_md_file, self.output_excel_file, self.output_json_file, self.test_rule_file_fake, self.output_md_file,
self.output_html_file self.output_html_file, self.output_all_json_file
]: ]:
p = Path(file) p = Path(file)
if p.is_file(): if p.is_file():
...@@ -185,8 +186,8 @@ def test_data_diagnosis(self): ...@@ -185,8 +186,8 @@ def test_data_diagnosis(self):
assert (len(data_not_accept_read_from_excel) == 2) assert (len(data_not_accept_read_from_excel) == 2)
assert ('Category' in data_not_accept_read_from_excel) assert ('Category' in data_not_accept_read_from_excel)
assert ('Defective Details' in data_not_accept_read_from_excel) assert ('Defective Details' in data_not_accept_read_from_excel)
# Test - output in json # Test - output in jsonl
diag1.output_diagnosis_in_json(data_not_accept_df, self.output_json_file) diag1.output_diagnosis_in_jsonl(data_not_accept_df, self.output_json_file)
assert (Path(self.output_json_file).is_file()) assert (Path(self.output_json_file).is_file())
with Path(self.output_json_file).open() as f: with Path(self.output_json_file).open() as f:
data_not_accept_read_from_json = f.readlines() data_not_accept_read_from_json = f.readlines()
...@@ -203,6 +204,30 @@ def test_data_diagnosis(self): ...@@ -203,6 +204,30 @@ def test_data_diagnosis(self):
with open(expected_md_file, 'r') as f: with open(expected_md_file, 'r') as f:
expect_result = f.readlines() expect_result = f.readlines()
assert (lines == expect_result) assert (lines == expect_result)
# Test - output_all_nodes_results
# case 1: 1 accept, 2 not accept
data_df = diag1.output_all_nodes_results(diag1._raw_data_df, data_not_accept_df)
assert (len(data_df) == 3)
assert (not data_df.loc['sb-validation-01']['Accept'])
assert (data_df.loc['sb-validation-02']['Accept'])
assert (not data_df.loc['sb-validation-03']['Accept'])
assert ('Category' in data_df)
assert ('Issue_Details' in data_df)
# case 1: 3 accept, 0 not accept
data_df_all_accept = diag1.output_all_nodes_results(diag1._raw_data_df, pd.DataFrame())
assert (len(data_df_all_accept) == 3)
assert (data_df_all_accept.loc['sb-validation-01']['Accept'])
assert (data_df_all_accept.loc['sb-validation-02']['Accept'])
assert (data_df_all_accept.loc['sb-validation-03']['Accept'])
# Test - output in json
diag1.output_diagnosis_in_json(data_df, self.output_all_json_file)
assert (Path(self.output_all_json_file).is_file())
expected_result_file = str(self.parent_path / '../data/diagnosis_summary.json')
with Path(self.output_all_json_file).open() as f:
data_not_accept_read_from_json = f.read()
with Path(expected_result_file).open() as f:
expect_result = f.read()
assert (data_not_accept_read_from_json == expect_result)
def test_data_diagnosis_run(self): def test_data_diagnosis_run(self):
"""Test for the run process of rule-based data diagnosis.""" """Test for the run process of rule-based data diagnosis."""
...@@ -228,7 +253,7 @@ def test_data_diagnosis_run(self): ...@@ -228,7 +253,7 @@ def test_data_diagnosis_run(self):
expect_result = f.read() expect_result = f.read()
assert (data_not_accept_read_from_json == expect_result) assert (data_not_accept_read_from_json == expect_result)
# Test - output in md # Test - output in md
DataDiagnosis().run(test_raw_data, test_rule_file, test_baseline_file, str(self.parent_path), 'md', 2) DataDiagnosis().run(test_raw_data, test_rule_file, test_baseline_file, str(self.parent_path), 'md', round=2)
assert (Path(self.output_md_file).is_file()) assert (Path(self.output_md_file).is_file())
expected_md_file = str(self.parent_path / '../data/diagnosis_summary.md') expected_md_file = str(self.parent_path / '../data/diagnosis_summary.md')
with open(expected_md_file, 'r') as f: with open(expected_md_file, 'r') as f:
...@@ -237,7 +262,7 @@ def test_data_diagnosis_run(self): ...@@ -237,7 +262,7 @@ def test_data_diagnosis_run(self):
summary = f.read() summary = f.read()
assert (summary == expect_result) assert (summary == expect_result)
# Test - output in html # Test - output in html
DataDiagnosis().run(test_raw_data, test_rule_file, test_baseline_file, str(self.parent_path), 'html', 2) DataDiagnosis().run(test_raw_data, test_rule_file, test_baseline_file, str(self.parent_path), 'html', round=2)
assert (Path(self.output_html_file).is_file()) assert (Path(self.output_html_file).is_file())
expected_html_file = str(self.parent_path / '../data/diagnosis_summary.html') expected_html_file = str(self.parent_path / '../data/diagnosis_summary.html')
with open(expected_html_file, 'r') as f: with open(expected_html_file, 'r') as f:
...@@ -245,6 +270,17 @@ def test_data_diagnosis_run(self): ...@@ -245,6 +270,17 @@ def test_data_diagnosis_run(self):
with open(self.output_html_file, 'r') as f: with open(self.output_html_file, 'r') as f:
summary = f.read() summary = f.read()
assert (summary == expect_result) assert (summary == expect_result)
# Test - output all nodes results
DataDiagnosis().run(
test_raw_data, test_rule_file, test_baseline_file, str(self.parent_path), 'json', output_all=True
)
assert (Path(self.output_all_json_file).is_file())
expected_result_file = str(self.parent_path / '../data/diagnosis_summary.json')
with Path(self.output_all_json_file).open() as f:
data_not_accept_read_from_json = f.read()
with Path(expected_result_file).open() as f:
expect_result = f.read()
assert (data_not_accept_read_from_json == expect_result)
def test_mutli_rules(self): def test_mutli_rules(self):
"""Test multi rules check feature.""" """Test multi rules check feature."""
......
tests/data/diagnosis_summary.json 0 → 100644
View file @ 55b0f9d2
[
{
"kernel-launch/event_overhead:0": 0.1,
"kernel-launch/event_overhead:1": 0.00595,
"kernel-launch/event_overhead:2": 0.00557,
"kernel-launch/event_overhead:3": 0.0055,
"kernel-launch/event_overhead:4": 0.00592,
"kernel-launch/event_overhead:5": 0.00589,
"kernel-launch/event_overhead:6": 0.00572,
"kernel-launch/event_overhead:7": 0.0059,
"kernel-launch/return_code": 0.0,
"kernel-launch/wall_overhead:0": 0.01026,
"kernel-launch/wall_overhead:1": 0.01026,
"kernel-launch/wall_overhead:2": 0.01046,
"kernel-launch/wall_overhead:3": 0.01049,
"kernel-launch/wall_overhead:4": 0.01063,
"kernel-launch/wall_overhead:5": 0.01006,
"kernel-launch/wall_overhead:6": 0.01045,
"kernel-launch/wall_overhead:7": 0.01071,
"mem-bw/D2H_Mem_BW:0": 24.3,
"mem-bw/D2H_Mem_BW:1": 24.6,
"mem-bw/D2H_Mem_BW:2": 24.5,
"mem-bw/D2H_Mem_BW:3": 24.6,
"mem-bw/D2H_Mem_BW:4": 24.3,
"mem-bw/D2H_Mem_BW:5": 24.3,
"mem-bw/D2H_Mem_BW:6": 23.9,
"mem-bw/D2H_Mem_BW:7": 24.6,
"mem-bw/H2D_Mem_BW:0": 25.6,
"mem-bw/H2D_Mem_BW:1": 25.8,
"mem-bw/H2D_Mem_BW:2": 26.0,
"mem-bw/H2D_Mem_BW:3": 26.1,
"mem-bw/H2D_Mem_BW:4": 26.2,
"mem-bw/H2D_Mem_BW:5": 25.8,
"mem-bw/H2D_Mem_BW:6": 25.3,
"mem-bw/H2D_Mem_BW:7": 26.1,
"mem-bw/return_code": 0.0,
"Accept": false,
"#Issues": 1.0,
"Category": "KernelLaunch",
"Issue_Details": "kernel-launch/event_overhead:0(B/L: 0.0060 VAL: 0.1000 VAR: 1577.85% Rule:lambda x:x>0.05)",
"Index": "sb-validation-01"
},
{
"kernel-launch/event_overhead:0": 0.00595,
"kernel-launch/event_overhead:1": 0.00595,
"kernel-launch/event_overhead:2": 0.00557,
"kernel-launch/event_overhead:3": 0.0055,
"kernel-launch/event_overhead:4": 0.00592,
"kernel-launch/event_overhead:5": 0.00589,
"kernel-launch/event_overhead:6": 0.00572,
"kernel-launch/event_overhead:7": 0.0059,
"kernel-launch/return_code": 0.0,
"kernel-launch/wall_overhead:0": 0.01026,
"kernel-launch/wall_overhead:1": 0.01026,
"kernel-launch/wall_overhead:2": 0.01046,
"kernel-launch/wall_overhead:3": 0.01049,
"kernel-launch/wall_overhead:4": 0.01063,
"kernel-launch/wall_overhead:5": 0.01006,
"kernel-launch/wall_overhead:6": 0.01045,
"kernel-launch/wall_overhead:7": 0.01071,
"mem-bw/D2H_Mem_BW:0": 24.3,
"mem-bw/D2H_Mem_BW:1": 24.6,
"mem-bw/D2H_Mem_BW:2": 24.5,
"mem-bw/D2H_Mem_BW:3": 24.6,
"mem-bw/D2H_Mem_BW:4": 24.3,
"mem-bw/D2H_Mem_BW:5": 24.3,
"mem-bw/D2H_Mem_BW:6": 23.9,
"mem-bw/D2H_Mem_BW:7": 24.6,
"mem-bw/H2D_Mem_BW:0": 25.6,
"mem-bw/H2D_Mem_BW:1": 25.8,
"mem-bw/H2D_Mem_BW:2": 26.0,
"mem-bw/H2D_Mem_BW:3": 26.1,
"mem-bw/H2D_Mem_BW:4": 26.2,
"mem-bw/H2D_Mem_BW:5": 25.8,
"mem-bw/H2D_Mem_BW:6": 25.3,
"mem-bw/H2D_Mem_BW:7": 26.1,
"mem-bw/return_code": 0.0,
"Accept": true,
"#Issues": 0.0,
"Category": "",
"Issue_Details": "",
"Index": "sb-validation-02"
},
{
"kernel-launch/event_overhead:0": 0.00596,
"kernel-launch/event_overhead:1": 0.00595,
"kernel-launch/event_overhead:2": 0.00557,
"kernel-launch/event_overhead:3": 0.0055,
"kernel-launch/event_overhead:4": 0.00592,
"kernel-launch/event_overhead:5": 0.00589,
"kernel-launch/event_overhead:6": 0.00572,
"kernel-launch/event_overhead:7": 0.0059,
"kernel-launch/return_code": 0.0,
"kernel-launch/wall_overhead:0": 0.01026,
"kernel-launch/wall_overhead:1": 0.01026,
"kernel-launch/wall_overhead:2": 0.01046,
"kernel-launch/wall_overhead:3": 0.01049,
"kernel-launch/wall_overhead:4": 0.01063,
"kernel-launch/wall_overhead:5": 0.01006,
"kernel-launch/wall_overhead:6": 0.01045,
"kernel-launch/wall_overhead:7": 0.01071,
"mem-bw/D2H_Mem_BW:0": "",
"mem-bw/D2H_Mem_BW:1": "",
"mem-bw/D2H_Mem_BW:2": "",
"mem-bw/D2H_Mem_BW:3": "",
"mem-bw/D2H_Mem_BW:4": "",
"mem-bw/D2H_Mem_BW:5": "",
"mem-bw/D2H_Mem_BW:6": "",
"mem-bw/D2H_Mem_BW:7": "",
"mem-bw/H2D_Mem_BW:0": "",
"mem-bw/H2D_Mem_BW:1": "",
"mem-bw/H2D_Mem_BW:2": "",
"mem-bw/H2D_Mem_BW:3": "",
"mem-bw/H2D_Mem_BW:4": "",
"mem-bw/H2D_Mem_BW:5": "",
"mem-bw/H2D_Mem_BW:6": "",
"mem-bw/H2D_Mem_BW:7": "",
"mem-bw/return_code": 1.0,
"Accept": false,
"#Issues": 17.0,
"Category": "FailedTest,Mem",
"Issue_Details": "mem-bw/D2H_Mem_BW:0_miss,mem-bw/D2H_Mem_BW:1_miss,mem-bw/D2H_Mem_BW:2_miss,mem-bw/D2H_Mem_BW:3_miss,mem-bw/D2H_Mem_BW:4_miss,mem-bw/D2H_Mem_BW:5_miss,mem-bw/D2H_Mem_BW:6_miss,mem-bw/D2H_Mem_BW:7_miss,mem-bw/H2D_Mem_BW:0_miss,mem-bw/H2D_Mem_BW:1_miss,mem-bw/H2D_Mem_BW:2_miss,mem-bw/H2D_Mem_BW:3_miss,mem-bw/H2D_Mem_BW:4_miss,mem-bw/H2D_Mem_BW:5_miss,mem-bw/H2D_Mem_BW:6_miss,mem-bw/H2D_Mem_BW:7_miss,mem-bw/return_code(VAL: 1.0000 Rule:lambda x:x>0)",
"Index": "sb-validation-03"
}
]
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment