Bug - Support no matching rules and unify the output name in result_summary (#345)

**Description**
Support no matching rules and unify the output name in result_summary

**Major Revision**
- Support rule with no matched metrics in result summary
- Unify output file name to 'results-summary'

docs/user-tutorial/result-summary.md

@@ -20,14 +20,12 @@ This tool is to generate a readable summary report based on the raw benchmark re
   sb result summary --data-file ./results-summary.jsonl --rule-file ./rule.yaml --output-file-format md --output-dir ${output-dir}
   ```
 
-4. Find the output result file named 'results_summary.md' under ${output_dir}.
+4. Find the output result file named 'results-summary.md' under ${output_dir}.
 
 ## Input
 
 The input includes 2 files:
 
-
-
 - **Raw Data**: jsonl file including multiple nodes' results automatically generated by SuperBench runner.
 
 :::tip Tips
@@ -122,3 +120,8 @@ The following illustrates all statistical functions:
 - `min`
 - `p${value}`: ${value} can be 1-99. For example, p50, p90, etc.
 - `std`
+
+## Output
+
+We support different output formats for result sumamry including markdown, html, etc.
+The output includes the metrics grouped by category and their values obtained by applying statistical methods to all raw results.

superbench/analyzer/result_summary.py

@@ -84,19 +84,23 @@ def _parse_rules(self, rules):
             logger.error('ResultSummary: parse rules failed - {}'.format(str(e)))
             return False
 
-    def _format_summary_of_rule(self, category, summary_df_of_rule):
+    def _format_summary_of_rule(self, category, summary_df_of_rule, statistics):
         """Format summary_df of a rule info list of lines.
 
         Args:
             category (str): category in the rule
             summary_df_of_rule ([type]): summary df of a rule, the columns are metrics, the index are statistics
+            statistics (list): statistics in the rule
         Returns:
             list: list of summary lines like [category, metric, statistic, value]
         """
         summary = []
         metrics = summary_df_of_rule.columns
+        if metrics.empty is True:
+            for statistic in statistics:
+                summary.append([category, '', statistic, ''])
         for metric in metrics:
-            for statistic in summary_df_of_rule.index:
+            for statistic in statistics:
                 summary.append([category, metric, statistic, summary_df_of_rule.loc[statistic, metric]])
         return summary
 
@@ -132,32 +136,35 @@ def _generate_summary(self, round):
             metrics = list(self._sb_rules[rule]['metrics'].keys())
             category = self._sb_rules[rule]['categories']
             data_df_of_rule = self._raw_data_df[metrics]
-            if self._sb_rules[rule]['aggregate']:
-                # if aggregate is True, aggregate in ranks
-                if self._sb_rules[rule]['aggregate'] is True:
-                    data_df_of_rule = data_analysis.aggregate(data_df_of_rule)
-                # if aggregate is not empty and is a pattern in regex, aggregate according to pattern
-                else:
-                    data_df_of_rule = data_analysis.aggregate(data_df_of_rule, self._sb_rules[rule]['aggregate'])
             statistics = self._sb_rules[rule]['statistics']
-            summary_df_of_rule = pd.DataFrame(columns=sorted(data_df_of_rule.columns))
-            for statistic_name in statistics:
-                # get SummaryOp and calculate statistics
-                # if statistic_name is 'p\d\d?', SummaryOp should be pencentile
-                if str.startswith(statistic_name, 'p'):
-                    rule_op = SummaryOp.get_summary_func(SummaryType('percentile'))
-                    val = int(statistic_name.strip('p'))
-                    summary_df_of_rule.loc[statistic_name] = rule_op(data_df_of_rule, val)
-                else:
-                    rule_op = SummaryOp.get_summary_func(SummaryType(statistic_name))
-                    summary_df_of_rule.loc[statistic_name] = rule_op(data_df_of_rule)
-            # format values to n significant decimal digits
-            if round and isinstance(round, int):
-                summary_df_of_rule = data_analysis.round_significant_decimal_places(
-                    summary_df_of_rule, round, list(summary_df_of_rule.columns)
-                )
+            summary_df_of_rule = pd.DataFrame()
+            # skip metrics aggregation and statistics calculation fot the rule with no matched metrics
+            if len(metrics) != 0:
+                if self._sb_rules[rule]['aggregate']:
+                    # if aggregate is True, aggregate in ranks
+                    if self._sb_rules[rule]['aggregate'] is True:
+                        data_df_of_rule = data_analysis.aggregate(data_df_of_rule)
+                    # if aggregate is not empty and is a pattern in regex, aggregate according to pattern
+                    else:
+                        data_df_of_rule = data_analysis.aggregate(data_df_of_rule, self._sb_rules[rule]['aggregate'])
+                summary_df_of_rule = pd.DataFrame(columns=sorted(data_df_of_rule.columns))
+                for statistic_name in statistics:
+                    # get SummaryOp and calculate statistics
+                    # if statistic_name is 'p\d\d?', SummaryOp should be pencentile
+                    if str.startswith(statistic_name, 'p'):
+                        rule_op = SummaryOp.get_summary_func(SummaryType('percentile'))
+                        val = int(statistic_name.strip('p'))
+                        summary_df_of_rule.loc[statistic_name] = rule_op(data_df_of_rule, val)
+                    else:
+                        rule_op = SummaryOp.get_summary_func(SummaryType(statistic_name))
+                        summary_df_of_rule.loc[statistic_name] = rule_op(data_df_of_rule)
+                # format values to n significant decimal digits
+                if round and isinstance(round, int):
+                    summary_df_of_rule = data_analysis.round_significant_decimal_places(
+                        summary_df_of_rule, round, list(summary_df_of_rule.columns)
+                    )
             # format summary_df of a rule to list of lines
-            summary_lines_of_rule = self._format_summary_of_rule(category, summary_df_of_rule)
+            summary_lines_of_rule = self._format_summary_of_rule(category, summary_df_of_rule, statistics)
             summary[category] = summary_lines_of_rule
 
         return summary
@@ -233,15 +240,15 @@ def run(self, raw_data_file, rule_file, output_dir, output_format, round=2):
             # output result summary to file
             output_path = ''
             if output_format == 'excel':
-                output_path = str(Path(output_dir) / 'results_summary.xlsx')
+                output_path = str(Path(output_dir) / 'results-summary.xlsx')
                 summary_df = self._merge_summary(summary)
                 self.output_summary_in_excel(self._raw_data_df, summary_df, output_path)
             elif output_format == 'md':
-                output_path = str(Path(output_dir) / 'results_summary.md')
+                output_path = str(Path(output_dir) / 'results-summary.md')
                 lines = self.generate_md_lines(summary)
                 file_handler.output_lines_in_md(lines, output_path)
             elif output_format == 'html':
-                output_path = str(Path(output_dir) / 'results_summary.html')
+                output_path = str(Path(output_dir) / 'results-summary.html')
                 lines = self.generate_md_lines(summary)
                 file_handler.output_lines_in_html(lines, output_path)
             else:

tests/analyzer/test_result_summary.py

@@ -18,9 +18,9 @@ class TestResultSummary(unittest.TestCase):
     def setUp(self):
         """Method called to prepare the test fixture."""
         self.parent_path = Path(__file__).parent
-        self.output_excel_file = str(self.parent_path / 'results_summary.xlsx')
-        self.output_md_file = str(self.parent_path / 'results_summary.md')
-        self.output_html_file = str(self.parent_path / 'results_summary.html')
+        self.output_excel_file = str(self.parent_path / 'results-summary.xlsx')
+        self.output_md_file = str(self.parent_path / 'results-summary.md')
+        self.output_html_file = str(self.parent_path / 'results-summary.html')
         self.test_rule_file_fake = str(self.parent_path / 'test_rules_fake.yaml')
         self.test_raw_data = str(self.parent_path / 'test_results.jsonl')
         self.test_rule_file = str(self.parent_path / 'test_summary_rules.yaml')
@@ -119,6 +119,29 @@ def test_result_summary(self):
         summary_merge_df = rs1._merge_summary(summary)
         pd.testing.assert_frame_equal(expected_summary_merge_df, summary_merge_df)
 
+    def test_no_matched_rule(self):
+        """Test for support no matching rules."""
+        # Positive case
+        rules = {
+            'superbench': {
+                'rules': {
+                    'fake': {
+                        'categories': 'FAKE',
+                        'statistics': ['mean', 'max'],
+                        'metrics': ['abb/fake:\\d+'],
+                        'aggregate': True
+                    }
+                }
+            }
+        }
+        rs1 = ResultSummary()
+        rs1._raw_data_df = file_handler.read_raw_data(self.test_raw_data)
+        rs1._benchmark_metrics_dict = rs1._get_metrics_by_benchmarks(list(rs1._raw_data_df))
+        assert (rs1._parse_rules(rules))
+        summary = rs1._generate_summary(round=2)
+        assert (len(summary) == 1)
+        assert (summary['FAKE'] == [['FAKE', '', 'mean', ''], ['FAKE', '', 'max', '']])
+
     def test_result_summary_run(self):
         """Test for the run process of result summary."""
         # Test - output in excel