data_diagnosis.py 18.5 KB
Newer Older
1
2
3
4
5
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.

"""A module for baseline-based data diagnosis."""
from typing import Callable
6
from pathlib import Path
7
import json
8
9

import pandas as pd
10
import numpy as np
11
12
13

from superbench.common.utils import logger
from superbench.analyzer.diagnosis_rule_op import RuleOp, DiagnosisRuleType
14
from superbench.analyzer import file_handler
15
from superbench.analyzer import RuleBase
16
from superbench.analyzer import data_analysis
17
18


19
class DataDiagnosis(RuleBase):
20
21
22
    """The DataDiagnosis class to do the baseline-based data diagnosis."""
    def __init__(self):
        """Init function."""
23
        super().__init__()
24

25
    def _check_and_format_rules(self, rule, name):
26
27
28
29
30
31
32
33
34
35
        """Check the rule of the metric whether the formart is valid.

        Args:
            rule (dict): the rule
            name (str): the rule name

        Returns:
            dict: the rule for the metric
        """
        # check if rule is supported
36
        super()._check_and_format_rules(rule, name)
37
38
39
40
41
42
43
44
45
        if 'function' not in rule:
            logger.log_and_raise(exception=Exception, msg='{} lack of function'.format(name))
        if not isinstance(DiagnosisRuleType(rule['function']), DiagnosisRuleType):
            logger.log_and_raise(exception=Exception, msg='{} invalid function name'.format(name))
        # check rule format
        if 'criteria' not in rule:
            logger.log_and_raise(exception=Exception, msg='{} lack of criteria'.format(name))
        if not isinstance(eval(rule['criteria']), Callable):
            logger.log_and_raise(exception=Exception, msg='invalid criteria format')
46
47
48
49
50
        if rule['function'] != 'multi_rules':
            if 'metrics' not in rule:
                logger.log_and_raise(exception=Exception, msg='{} lack of metrics'.format(name))
        if 'store' in rule and not isinstance(rule['store'], bool):
            logger.log_and_raise(exception=Exception, msg='{} store must be bool type'.format(name))
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
        return rule

    def _get_baseline_of_metric(self, baseline, metric):
        """Get the baseline value of the metric.

        Args:
            baseline (dict): baseline defined in baseline file
            metric (str): the full name of the metric

        Returns:
            numeric: the baseline value of the metric
        """
        if metric in baseline:
            return baseline[metric]
        else:
            # exclude rank info
            short = metric.split(':')[0]
            if short in baseline:
                return baseline[short]
            # baseline not defined
            else:
                logger.warning('DataDiagnosis: get baseline - {} baseline not found'.format(metric))
                return -1

75
76
    def __get_metrics_and_baseline(self, rule, benchmark_rules, baseline):
        """Get metrics with baseline in the rule.
77

78
79
        Parse metric regex in the rule, and store the (baseline, metric) pair
        in _sb_rules[rule]['metrics'] and metric in _enable_metrics。
80
81

        Args:
82
83
84
85
            rule (str): the name of the rule
            benchmark_rules (dict): the dict of rules
            baseline (dict): the dict of baseline of metrics
        """
86
        if 'function' in self._sb_rules[rule] and self._sb_rules[rule]['function'] == 'multi_rules':
87
            return
88
89
90
        self._get_metrics(rule, benchmark_rules)
        for metric in self._sb_rules[rule]['metrics']:
            self._sb_rules[rule]['metrics'][metric] = self._get_baseline_of_metric(baseline, metric)
91
92
93
94
95
96
97

    def _parse_rules_and_baseline(self, rules, baseline):
        """Parse and merge rules and baseline read from file.

        Args:
            rules (dict): rules from rule yaml file
            baseline (dict): baseline of metrics from baseline json file
98
99
100
101
102

        Returns:
            bool: return True if successfully get the criteria for all rules, otherwise False.
        """
        try:
103
            if not rules:
104
105
106
                logger.error('DataDiagnosis: get criteria failed')
                return False
            self._sb_rules = {}
107
            self._enable_metrics = set()
108
109
            benchmark_rules = rules['superbench']['rules']
            for rule in benchmark_rules:
110
                benchmark_rules[rule] = self._check_and_format_rules(benchmark_rules[rule], rule)
111
                self._sb_rules[rule] = {}
112
                self._sb_rules[rule]['name'] = rule
113
                self._sb_rules[rule]['function'] = benchmark_rules[rule]['function']
114
115
                self._sb_rules[rule]['store'] = True if 'store' in benchmark_rules[
                    rule] and benchmark_rules[rule]['store'] is True else False
116
117
118
                self._sb_rules[rule]['criteria'] = benchmark_rules[rule]['criteria']
                self._sb_rules[rule]['categories'] = benchmark_rules[rule]['categories']
                self._sb_rules[rule]['metrics'] = {}
119
120
                self.__get_metrics_and_baseline(rule, benchmark_rules, baseline)
            self._enable_metrics = sorted(list(self._enable_metrics))
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
        except Exception as e:
            logger.error('DataDiagnosis: get criteria failed - {}'.format(str(e)))
            return False

        return True

    def _run_diagnosis_rules_for_single_node(self, node):
        """Use rules to diagnosis single node data.

        Use the rules defined in rule_file to diagnose the raw data of each node,
        if the node violate any rule, label as defective node and save
        the 'Category', 'Defective Details' and data summary of defective node.

        Args:
            node (str): the node to do the diagosis

        Returns:
            details_row (list): None if the node is not labeled as defective,
                otherwise details of ['Category', 'Defective Details']
            summary_data_row (dict): None if the node is not labeled as defective,
                otherwise data summary of the metrics
        """
        data_row = self._raw_data_df.loc[node]
        issue_label = False
        details = []
        categories = set()
147
        violation = {}
148
149
150
151
152
153
        summary_data_row = pd.Series(index=self._enable_metrics, name=node, dtype=float)
        # Check each rule
        for rule in self._sb_rules:
            # Get rule op function and run the rule
            function_name = self._sb_rules[rule]['function']
            rule_op = RuleOp.get_rule_func(DiagnosisRuleType(function_name))
154
155
156
157
158
            violated_num = 0
            if rule_op == RuleOp.multi_rules:
                violated_num = rule_op(self._sb_rules[rule], details, categories, violation)
            else:
                violated_num = rule_op(data_row, self._sb_rules[rule], summary_data_row, details, categories)
159
            # label the node as defective one
160
161
162
            if self._sb_rules[rule]['store']:
                violation[rule] = violated_num
            elif violated_num:
163
164
165
                issue_label = True
        if issue_label:
            # Add category information
166
167
            general_cat_str = ','.join(sorted(list(categories)))
            details_cat_str = ','.join(sorted((details)))
168
169
170
171
172
            details_row = [general_cat_str, details_cat_str]
            return details_row, summary_data_row

        return None, None

173
    def run_diagnosis_rules(self, rules, baseline):
174
175
        """Rule-based data diagnosis for multiple nodes' raw data.

176
        Use the rules defined in rules to diagnose the raw data of each node,
177
178
179
180
        if the node violate any rule, label as defective node and save
        the 'Category', 'Defective Details' and processed data of defective node.

        Args:
181
182
            rules (dict): rules from rule yaml file
            baseline (dict): baseline of metrics from baseline json file
183
184
185
186
187
188
189
190
191
192

        Returns:
            data_not_accept_df (DataFrame): defective nodes's detailed information
            label_df (DataFrame): labels for all nodes
        """
        try:
            summary_columns = ['Category', 'Defective Details']
            data_not_accept_df = pd.DataFrame(columns=summary_columns)
            summary_details_df = pd.DataFrame()
            label_df = pd.DataFrame(columns=['label'])
193
            if not self._parse_rules_and_baseline(rules, baseline):
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
                return data_not_accept_df, label_df
            # run diagnosis rules for each node
            for node in self._raw_data_df.index:
                details_row, summary_data_row = self._run_diagnosis_rules_for_single_node(node)
                if details_row:
                    data_not_accept_df.loc[node] = details_row
                    summary_details_df = summary_details_df.append(summary_data_row)
                    label_df.loc[node] = 1
                else:
                    label_df.loc[node] = 0
            # combine details for defective nodes
            if len(data_not_accept_df) != 0:
                data_not_accept_df = data_not_accept_df.join(summary_details_df)
                data_not_accept_df = data_not_accept_df.sort_values(by=summary_columns, ascending=False)

        except Exception as e:
            logger.error('DataDiagnosis: run diagnosis rules failed, message: {}'.format(str(e)))
        return data_not_accept_df, label_df

213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
    def output_all_nodes_results(self, raw_data_df, data_not_accept_df):
        """Output diagnosis results of all nodes.

        Args:
            raw_data_df (DataFrame): raw data
            data_not_accept_df (DataFrame): defective nodes's detailed information

        Returns:
            DataFrame: all nodes' detailed information inluding ['Accept','#Issues','Category','Issue_Details']
        """
        append_columns = ['Accept', '#Issues', 'Category', 'Issue_Details']
        all_data_df = (raw_data_df[self._enable_metrics]).astype('float64')

        if data_not_accept_df.shape[0] == 0:
            all_data_df['Accept'] = [True for i in range(len(all_data_df))]
            all_data_df['#Issues'] = [0 for i in range(len(all_data_df))]
            all_data_df['Category'] = [None for i in range(len(all_data_df))]
            all_data_df['Issue_Details'] = [None for i in range(len(all_data_df))]

        elif data_not_accept_df.shape[0] > 0:
            data_not_accept_df['Accept'] = [False for i in range(len(data_not_accept_df))]
            data_not_accept_df['#Issues'] = data_not_accept_df['Defective Details'].map(lambda x: len(x.split(',')))
            data_not_accept_df = data_not_accept_df.rename(columns={'Defective Details': 'Issue_Details'})
            for index in range(len(append_columns)):
                if append_columns[index] not in data_not_accept_df:
                    logger.warning(
                        'DataDiagnosis: output_all_nodes_results - column {} not found in data_not_accept_df.'.format(
                            append_columns[index]
                        )
                    )
                    all_data_df[append_columns[index]] = None
                else:
                    all_data_df = all_data_df.merge(
                        data_not_accept_df[[append_columns[index]]], left_index=True, right_index=True, how='left'
                    )
            all_data_df['Accept'] = all_data_df['Accept'].replace(np.nan, True)
            all_data_df['#Issues'] = all_data_df['#Issues'].replace(np.nan, 0)

        all_data_df = all_data_df.replace(np.nan, '')

        return all_data_df

255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
    def output_diagnosis_in_excel(self, raw_data_df, data_not_accept_df, output_path, rules):
        """Output the raw_data_df and data_not_accept_df results into excel file.

        Args:
            raw_data_df (DataFrame): raw data
            data_not_accept_df (DataFrame): defective nodes's detailed information
            output_path (str): the path of output excel file
            rules (dict): the rules of DataDiagnosis
        """
        try:
            writer = pd.ExcelWriter(output_path, engine='xlsxwriter')
            # Check whether writer is valiad
            if not isinstance(writer, pd.ExcelWriter):
                logger.error('DataDiagnosis: excel_data_output - invalid file path.')
                return
            file_handler.output_excel_raw_data(writer, raw_data_df, 'Raw Data')
            file_handler.output_excel_data_not_accept(writer, data_not_accept_df, rules)
            writer.save()
        except Exception as e:
            logger.error('DataDiagnosis: excel_data_output - {}'.format(str(e)))

276
    def output_diagnosis_in_jsonl(self, data_not_accept_df, output_path):
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
        """Output data_not_accept_df into jsonl file.

        Args:
            data_not_accept_df (DataFrame): the DataFrame to output
            output_path (str): the path of output jsonl file
        """
        p = Path(output_path)
        try:
            data_not_accept_json = data_not_accept_df.to_json(orient='index')
            data_not_accept = json.loads(data_not_accept_json)
            if not isinstance(data_not_accept_df, pd.DataFrame):
                logger.warning('DataDiagnosis: output json data - data_not_accept_df is not DataFrame.')
                return
            if data_not_accept_df.empty:
                logger.warning('DataDiagnosis: output json data - data_not_accept_df is empty.')
                return
            with p.open('w') as f:
                for node in data_not_accept:
                    line = data_not_accept[node]
                    line['Index'] = node
                    json_str = json.dumps(line)
                    f.write(json_str + '\n')
        except Exception as e:
            logger.error('DataDiagnosis: output json data failed, msg: {}'.format(str(e)))

302
303
304
305
306
307
308
309
310
311
312
313
314
315
    def output_diagnosis_in_json(self, data_not_accept_df, output_path):
        """Output data_not_accept_df into json file.

        Args:
            data_not_accept_df (DataFrame): the DataFrame to output
            output_path (str): the path of output jsonl file
        """
        data_not_accept_df['Index'] = data_not_accept_df.index
        data_not_accept_json = data_not_accept_df.to_json(orient='records')
        data_not_accept = json.loads(data_not_accept_json)
        p = Path(output_path)
        with p.open('w') as f:
            json.dump(data_not_accept, f, indent=4)

316
    def generate_md_lines(self, data_not_accept_df, rules, round):
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
        """Convert DataFrame into markdown lines.

        Args:
            data_not_accept_df (DataFrame): the DataFrame to output
            rules (dict): the rules of DataDiagnosis
            round (int): the number of decimal digits

        Returns:
            list: lines in markdown format
        """
        data_not_accept_df['machine'] = data_not_accept_df.index
        header = data_not_accept_df.columns.tolist()
        header = header[-1:] + header[:-1]
        data_not_accept_df = data_not_accept_df[header]
        # format precision of values to n decimal digits
        for rule in rules:
            for metric in rules[rule]['metrics']:
                if rules[rule]['function'] == 'variance':
                    if round and isinstance(round, int):
                        data_not_accept_df[metric] = data_not_accept_df[metric].map(
                            lambda x: x * 100, na_action='ignore'
                        )
                        data_not_accept_df = data_analysis.round_significant_decimal_places(
                            data_not_accept_df, round, [metric]
                        )
                    data_not_accept_df[metric] = data_not_accept_df[metric].map(
                        lambda x: '{}%'.format(x), na_action='ignore'
                    )
                elif rules[rule]['function'] == 'value':
                    if round and isinstance(round, int):
                        data_not_accept_df = data_analysis.round_significant_decimal_places(
                            data_not_accept_df, round, [metric]
                        )
350
        lines = file_handler.generate_md_table(data_not_accept_df, header)
351
352
        return lines

353
354
355
    def run(
        self, raw_data_file, rule_file, baseline_file, output_dir, output_format='excel', output_all=False, round=2
    ):
356
357
358
359
360
361
362
        """Run the data diagnosis and output the results.

        Args:
            raw_data_file (str): the path of raw data jsonl file.
            rule_file (str): The path of baseline yaml file
            baseline_file (str): The path of baseline json file
            output_dir (str): the directory of output file
363
            output_all (bool): output diagnosis results for all nodes
364
            output_format (str): the format of the output, 'excel' or 'json'
365
            round (int): the number of decimal digits
366
367
        """
        try:
368
369
370
            rules = self._preprocess(raw_data_file, rule_file)
            # read baseline
            baseline = file_handler.read_baseline(baseline_file)
371
            logger.info('DataDiagnosis: Begin to process {} nodes'.format(len(self._raw_data_df)))
372
            data_not_accept_df, label_df = self.run_diagnosis_rules(rules, baseline)
373
            logger.info('DataDiagnosis: Processed finished')
374
            output_path = ''
375
376
377
378
379
            # generate all nodes' info
            if output_all:
                output_path = str(Path(output_dir) / 'diagnosis_summary.json')
                data_not_accept_df = self.output_all_nodes_results(self._raw_data_df, data_not_accept_df)
            # output according format
380
            if output_format == 'excel':
381
                output_path = str(Path(output_dir) / 'diagnosis_summary.xlsx')
382
                self.output_diagnosis_in_excel(self._raw_data_df, data_not_accept_df, output_path, self._sb_rules)
383
            elif output_format == 'json':
384
385
386
387
388
389
                if output_all:
                    output_path = str(Path(output_dir) / 'diagnosis_summary.json')
                    self.output_diagnosis_in_json(data_not_accept_df, output_path)
                else:
                    output_path = str(Path(output_dir) / 'diagnosis_summary.jsonl')
                    self.output_diagnosis_in_jsonl(data_not_accept_df, output_path)
390
            elif output_format == 'md' or output_format == 'html':
391
                lines = self.generate_md_lines(data_not_accept_df, self._sb_rules, round)
392
393
394
395
396
397
                if output_format == 'md':
                    output_path = str(Path(output_dir) / 'diagnosis_summary.md')
                    file_handler.output_lines_in_md(lines, output_path)
                else:
                    output_path = str(Path(output_dir) / 'diagnosis_summary.html')
                    file_handler.output_lines_in_html(lines, output_path)
398
399
400
401
402
            else:
                logger.error('DataDiagnosis: output failed - unsupported output format')
            logger.info('DataDiagnosis: Output results to {}'.format(output_path))
        except Exception as e:
            logger.error('DataDiagnosis: run failed - {}'.format(str(e)))