CLI - Integrage result summary and update output format of data diagnosis (#335)

**Description**
Integrage result summary and update output format of data diagnosis.

**Major Revision**
- integrage result summary 
- add md and html format for data diagnosis

docs/cli.md

@@ -168,11 +168,13 @@ sb exec --config-override superbench.enable="['gpt2_models']"
 Filter the defective machines automatically from benchmarking results according to rules defined in rule file.
 
 ```bash title="SB CLI"
-sb result diagnosis [--baseline-file]
-       [--data-file]
+sb result diagnosis --baseline-file
+                    --data-file
+                    --rule-file
+                    [--decimal-place-value]
                     [--rule-file]
                     [--output-dir]
-       [--output-file-format]
+                    [--output-file-format {excel, json, md, html}]
 ```
 
 #### Required arguments
@@ -186,9 +188,10 @@ sb result diagnosis [--baseline-file]
 #### Optional arguments
 
 | Name                    | Default | Description                                                                 |
-|------------------------|---------|-----------------------------------------------------------------------------|
+|-------------------------|---------|-----------------------------------------------------------------------------|
+| `--decimal-place-value` | 2       | Number of valid decimal places to show in output. Default: 2.               |
 | `--output-dir`          | `None`  | Path to output directory, outputs/{datetime} will be used if not specified. |
-| `--output-file-format` | `excel` | Format of output file, excel or json. Default: excel.                       |
+| `--output-file-format`  | `excel` | Format of output file, 'excel', 'json', 'md' or 'html'. Default: excel.     |
 
 #### Global arguments
 
@@ -208,6 +211,56 @@ Run data diagnosis and output the results in jsonl format:
 sb result diagnosis --data-file outputs/results-summary.jsonl --rule-file rule.yaml --baseline-file baseline.json --output-file-format json
 ```
 
+Run data diagnosis and output the results in markdown format with 2 valid decimal places:
+```bash title="SB CLI"
+sb result diagnosis --data-file outputs/results-summary.jsonl --rule-file rule.yaml --baseline-file baseline.json --output-file-format md --decimal-place-value 2
+```
+
+### `sb result summary`
+
+Generate the readable summary report automatically from benchmarking results according to rules defined in rule file.
+
+```bash title="SB CLI"
+sb result summary --data-file
+                  --rule-file
+                  [--decimal-place-value]
+                  [--output-dir]
+                  [--output-file-format {md, excel, html}]
+```
+
+#### Required arguments
+
+| Name               | Description            |
+|--------------------|------------------------|
+| `--data-file` `-d` | Path to raw data file. |
+| `--rule-file` `-r` | Path to rule file.     |
+
+#### Optional arguments
+
+| Name                    | Default | Description                                                                 |
+|-------------------------|---------|-----------------------------------------------------------------------------|
+| `--decimal-place-value` | 2       | Number of valid decimal places to show in output. Default: 2.               |
+| `--output-dir`          | `None`  | Path to output directory, outputs/{datetime} will be used if not specified. |
+| `--output-file-format`  | `md`    | Format of output file, 'excel', 'md' or 'html'. Default: md.                |
+
+#### Global arguments
+
+| Name          | Default | Description        |
+|---------------|---------|--------------------|
+| `--help` `-h` | N/A     | Show help message. |
+
+#### Examples
+
+Run data summary and output the results in markdown format with 2 valid decimal places:
+```bash title="SB CLI"
+sb result summary --data-file outputs/results-summary.jsonl --rule-file rule.yaml --output-file-format md --decimal-place-value 2
+```
+
+Run data diagnosis and output the results in html format:
+```bash title="SB CLI"
+sb result summary --data-file outputs/results-summary.jsonl --rule-file rule.yaml --output-file-format html
+```
+
 ### `sb run`
 
 Run the SuperBench benchmarks distributedly.

superbench/cli/_commands.py

@@ -30,6 +30,7 @@ def load_command_table(self, args):
             g.command('info', 'info_command_handler')
         with CommandGroup(self, 'result', 'superbench.cli._result_handler#{}') as g:
             g.command('diagnosis', 'diagnosis_command_handler')
+            g.command('summary', 'summary_command_handler')
         return super().load_command_table(args)
 
     def load_arguments(self, command):
@@ -80,5 +81,6 @@ def load_arguments(self, command):
                 help='Path to output directory, outputs/{datetime} will be used if not specified.'
             )
             ac.argument('output_file_format', type=str, help='Format of output file, excel or json.')
+            ac.argument('decimal_place_value', type=int, help='Number of decimal places to show in output.')
 
         super().load_arguments(command)

superbench/cli/_help.py

@@ -122,6 +122,46 @@
             --rule-file rule.yaml
             --baseline-file baseline.json
             --output-file-format json
+        - name: run data diagnosis and output the results in markdown format
+          text: >
+            {cli_name} result diagnosis
+            --data-file outputs/results-summary.jsonl
+            --rule-file rule.yaml
+            --baseline-file baseline.json
+            --output-file-format md
+        - name: run data diagnosis and output the results in html format
+          text: >
+            {cli_name} result diagnosis
+            --data-file outputs/results-summary.jsonl
+            --rule-file rule.yaml
+            --baseline-file baseline.json
+            --output-file-format html
+""".format(cli_name=CLI_NAME)
+
+helps['result summary'] = """
+    type: command
+    short-summary: >
+        Generate the readable summary of benchmarking results
+        according to rules defined in rule file.
+    examples:
+        - name: run result summary and output the results in excel format
+          text: >
+            {cli_name} result summary
+            --data-file outputs/results-summary.jsonl
+            --rule-file rule.yaml
+            --output-file-format excel
+        - name: run result summary and output the results in markdown format
+          text: >
+            {cli_name} result summary
+            --data-file outputs/results-summary.jsonl
+            --rule-file rule.yaml
+            --output-file-format md
+        - name: run result summary and output the results in html format
+          text: >
+            {cli_name} result summary
+            --data-file outputs/results-summary.jsonl
+            --rule-file rule.yaml
+            --output-file-format html
 """.format(cli_name=CLI_NAME)
 
 

superbench/cli/_result_handler.py

@@ -6,11 +6,14 @@
 from knack.util import CLIError
 
 from superbench.analyzer import DataDiagnosis
+from superbench.analyzer import ResultSummary
 from superbench.common.utils import create_sb_output_dir
 from superbench.cli._handler import check_argument_file
 
 
-def diagnosis_command_handler(raw_data_file, rule_file, baseline_file, output_dir=None, output_file_format='excel'):
+def diagnosis_command_handler(
+    raw_data_file, rule_file, baseline_file, output_dir=None, output_file_format='excel', decimal_place_value=2
+):
     """Run data diagnosis.
 
     Args:
@@ -18,18 +21,47 @@ def diagnosis_command_handler(raw_data_file, rule_file, baseline_file, output_di
         rule_file (str): Path to baseline yaml file.
         baseline_file (str): Path to baseline json file.
         output_dir (str): Path to output directory.
-        output_file_format (str): Format of the output file, 'excel' or 'json'. Defaults to 'excel'.
+        output_file_format (str): Format of the output file, 'excel', 'json', 'md' or 'html'. Defaults to 'excel'.
+        decimal_place_value (int): Number of decimal places to show in output.
     """
     try:
         # Create output directory
         sb_output_dir = create_sb_output_dir(output_dir)
         # Check arguments
-        if output_file_format not in ['excel', 'json']:
-            raise CLIError('Output format must be excel or json.')
+        supported_output_format = ['excel', 'json', 'md', 'html']
+        if output_file_format not in supported_output_format:
+            raise CLIError('Output format must be in {}.'.format(str(supported_output_format)))
         check_argument_file('raw_data_file', raw_data_file)
         check_argument_file('rule_file', rule_file)
         check_argument_file('baseline_file', baseline_file)
         # Run data diagnosis
-        DataDiagnosis().run(raw_data_file, rule_file, baseline_file, sb_output_dir, output_file_format)
+        DataDiagnosis().run(
+            raw_data_file, rule_file, baseline_file, sb_output_dir, output_file_format, decimal_place_value
+        )
     except Exception as ex:
         raise RuntimeError('Failed to run diagnosis command.') from ex
+
+
+def summary_command_handler(raw_data_file, rule_file, output_dir=None, output_file_format='md', decimal_place_value=2):
+    """Run result summary.
+
+    Args:
+        raw_data_file (str): Path to raw data jsonl file.
+        rule_file (str): Path to baseline yaml file.
+        output_dir (str): Path to output directory.
+        output_file_format (str): Format of the output file, 'excel', 'md' or 'html'. Defaults to 'md'.
+        decimal_place_value (int): Number of decimal places to show in output.
+    """
+    try:
+        # Create output directory
+        sb_output_dir = create_sb_output_dir(output_dir)
+        # Check arguments
+        supported_output_format = ['excel', 'html', 'md']
+        if output_file_format not in supported_output_format:
+            raise CLIError('Output format must be in {}.'.format(str(supported_output_format)))
+        check_argument_file('raw_data_file', raw_data_file)
+        check_argument_file('rule_file', rule_file)
+        # Run result summary
+        ResultSummary().run(raw_data_file, rule_file, sb_output_dir, output_file_format, decimal_place_value)
+    except Exception as ex:
+        raise RuntimeError('Failed to run summary command.') from ex

tests/cli/test_sb.py

@@ -116,3 +116,22 @@ def test_sb_result_diagnosis(self):
             format(dir=test_analyzer_dir) + ' --output-dir outputs/test-diagnosis/ --output-file-format abb',
             expect_failure=True
         )
+
+    def test_sb_result_summary(self):
+        """Test sb result summary."""
+        test_analyzer_dir = str(Path(__file__).parent.resolve() / '../analyzer/')
+        # test positive case
+        self.cmd(
+            'sb result summary -d {dir}/test_results.jsonl -r {dir}/test_summary_rules.yaml'.
+            format(dir=test_analyzer_dir) + ' --output-dir /tmp/outputs/test-summary/'
+        )
+        self.cmd(
+            'sb result summary -d {dir}/test_results.jsonl -r {dir}/test_summary_rules.yaml'.
+            format(dir=test_analyzer_dir) + ' --output-dir /tmp/outputs/test-summary/ --decimal-place-value 4'
+        )
+        # test invalid output format
+        self.cmd(
+            'sb result summary -d {dir}/test_results.jsonl -r {dir}/test_rules.yaml'.format(dir=test_analyzer_dir) +
+            ' --output-dir /tmp/outputs/test-summary/ --output-file-format abb',
+            expect_failure=True
+        )