# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from collections import defaultdict, OrderedDict import numpy as np class InstanceEvaluator(object): """ Refer to 'https://github.com/mcordts/cityscapesScripts/blob/master/cityscapesscripts/evaluation/evalInstanceLevelSemanticLabeling.py' Calculate the matching results of each image, each class, each IoU, and then get the final matching results of each class and each IoU of dataset. Base on the matching results, the AP and mAP can be calculated. we need two vectors for each class and for each overlap The first vector (y_true) is binary and is 1, where the ground truth says true, and is 0 otherwise. The second vector (y_score) is float [0...1] and represents the confidence of the prediction. We represent the following cases as: | y_true | y_score gt instance with matched prediction | 1 | confidence gt instance w/o matched prediction | 1 | 0.0 false positive prediction | 0 | confidence The current implementation makes only sense for an overlap threshold >= 0.5, since only then, a single prediction can either be ignored or matched, but never both. Further, it can never match to two gt instances. For matching, we vary the overlap and do the following steps: 1.) remove all predictions that satisfy the overlap criterion with an ignore region (either void or *group) 2.) remove matches that do not satisfy the overlap 3.) mark non-matched predictions as false positive In the processing, 0 represent the first class of 'thing'. So the label will less 1 than the dataset. Args: num_classes (int): The unique number of target classes. Exclude background class, labeled 0 usually. overlaps (float|list, optional): The threshold of IoU. Default: 0.5. thing_list (list|None, optional): Thing class, only calculate AP for the thing class. Default: None. """ def __init__(self, num_classes, overlaps=0.5, thing_list=None): super().__init__() self.num_classes = num_classes if isinstance(overlaps, float): overlaps = [overlaps] self.overlaps = overlaps self.y_true = [[np.empty(0) for _i in range(len(overlaps))] for _j in range(num_classes)] self.y_score = [[np.empty(0) for _i in range(len(overlaps))] for _j in range(num_classes)] self.hard_fns = [[0] * len(overlaps) for _ in range(num_classes)] if thing_list is None: self.thing_list = list(range(num_classes)) else: self.thing_list = thing_list def update(self, preds, gts, ignore_mask=None): """ compute y_true and y_score in this image. preds (list): tuple list [(label, confidence, mask), ...]. gts (list): tuple list [(label, mask), ...]. ignore_mask (np.ndarray): Mask to ignore. """ pred_instances, gt_instances = self.get_instances( preds, gts, ignore_mask=ignore_mask) for i in range(self.num_classes): if i not in self.thing_list: continue for oi, oth in enumerate(self.overlaps): cur_true = np.ones((len(gt_instances[i]))) cur_score = np.ones(len(gt_instances[i])) * (-float("inf")) cur_match = np.zeros(len(gt_instances[i]), dtype=np.bool) for gti, gt_instance in enumerate(gt_instances[i]): found_match = False for pred_instance in gt_instance['matched_pred']: overlap = float(pred_instance['intersection']) / ( gt_instance['pixel_count'] + pred_instance[ 'pixel_count'] - pred_instance['intersection']) if overlap > oth: confidence = pred_instance['confidence'] # if we already has a prediction for this groundtruth # the prediction with the lower score is automatically a false positive if cur_match[gti]: max_score = max(cur_score[gti], confidence) min_score = min(cur_score[gti], confidence) cur_score = max_score # append false positive cur_true = np.append(cur_true, 0) cur_score = np.append(cur_score, min_score) cur_match = np.append(cur_match, True) # otherwise set score else: found_match = True cur_match[gti] = True cur_score[gti] = confidence if not found_match: self.hard_fns[i][oi] += 1 # remove not-matched ground truth instances cur_true = cur_true[cur_match == True] cur_score = cur_score[cur_match == True] # collect not-matched predictions as false positive for pred_instance in pred_instances[i]: found_gt = False for gt_instance in pred_instance['matched_gt']: overlap = float(gt_instance['intersection']) / ( gt_instance['pixel_count'] + pred_instance[ 'pixel_count'] - gt_instance['intersection']) if overlap > oth: found_gt = True break if not found_gt: proportion_ignore = 0 if ignore_mask is not None: nb_ignore_pixels = pred_instance[ 'void_intersection'] proportion_ignore = float( nb_ignore_pixels) / pred_instance['pixel_count'] if proportion_ignore <= oth: cur_true = np.append(cur_true, 0) cur_score = np.append(cur_score, pred_instance['confidence']) self.y_true[i][oi] = np.append(self.y_true[i][oi], cur_true) self.y_score[i][oi] = np.append(self.y_score[i][oi], cur_score) def evaluate(self): ap = self.cal_ap() map = self.cal_map() res = {} res["AP"] = [{i: ap[i] * 100} for i in self.thing_list] res["mAP"] = 100 * map results = OrderedDict({"ins_seg": res}) return results def cal_ap(self): """ calculate ap for every classes """ self.ap = [0] * self.num_classes self.ap_overlap = [[0] * len(self.overlaps) for _ in range(self.num_classes)] for i in range(self.num_classes): if i not in self.thing_list: continue for j in range(len(self.overlaps)): y_true = self.y_true[i][j] y_score = self.y_score[i][j] if len(y_true) == 0: self.ap_overlap[i][j] = 0 continue score_argsort = np.argsort(y_score) y_score_sorted = y_score[score_argsort] y_true_sorted = y_true[score_argsort] y_true_sorted_cumsum = np.cumsum(y_true_sorted) # unique thresholds thresholds, unique_indices = np.unique( y_score_sorted, return_index=True) # since we need to add an artificial point to the precision-recall curve # increase its length by 1 nb_pr = len(unique_indices) + 1 # calculate precision and recall nb_examples = len(y_score_sorted) nb_true_exampels = y_true_sorted_cumsum[-1] precision = np.zeros(nb_pr) recall = np.zeros(nb_pr) # deal with the first point # only thing we need to do, is to append a zero to the cumsum at the end. # an index of -1 uses that zero then y_true_sorted_cumsum = np.append(y_true_sorted_cumsum, 0) # deal with remaining for idx_res, idx_scores in enumerate(unique_indices): cumsum = y_true_sorted_cumsum[idx_scores - 1] tp = nb_true_exampels - cumsum fp = nb_examples - idx_scores - tp fn = cumsum + self.hard_fns[i][j] p = float(tp) / (tp + fp) r = float(tp) / (tp + fn) precision[idx_res] = p recall[idx_res] = r # add first point in curve precision[-1] = 1. # In some calculation，make precision the max after this point in curve. #precision = [np.max(precision[:i+1]) for i in range(len(precision))] recall[-1] = 0. # compute average of precision-recall curve # integration is performed via zero order, or equivalently step-wise integration # first compute the widths of each step: # use a convolution with appropriate kernel, manually deal with the boundaries first recall_for_conv = np.copy(recall) recall_for_conv = np.append(recall_for_conv[0], recall_for_conv) recall_for_conv = np.append(recall_for_conv, 0.) step_widths = np.convolve(recall_for_conv, [-0.5, 0, 0.5], 'valid') # integrate is now simply a dot product ap_current = np.dot(precision, step_widths) self.ap_overlap[i][j] = ap_current ap = [np.average(i) for i in self.ap_overlap] self.ap = ap return ap def cal_map(self): """ calculate map for all classes """ self.cal_ap() valid_ap = [self.ap[i] for i in self.thing_list] map = np.mean(valid_ap) self.map = map return map def get_instances(self, preds, gts, ignore_mask=None): """ In this method, we create two dicts of list - pred_instances: contains all predictions and their associated gt - gtInstances: contains all gt instances and their associated predictions Args: preds (list): Prediction of image. gts (list): Ground truth of image. ignore_mask (np.ndarray, optional): Ignore mask. Default: None. Return: dict: pred_instances, the type is dict(list(dict))), e.g. {0: [{'pred_id':0, 'label':0', 'pixel_count':100, 'confidence': 0.9, 'void_intersection': 0, 'matched_gt': [gt_instance0, gt_instance1, ...]}, ], 1: } dict: gt_instances, the type is dict(list(dict))), e.g. {0: [{'inst_id':0, 'label':0', 'pixel_count':100, 'mask': np.ndarray, 'matched_pred': [pred_instance0, pred_instance1, ...]}, ], 1: } """ pred_instances = defaultdict(list) gt_instances = defaultdict(list) gt_inst_count = 0 for gt in gts: label, mask = gt gt_instance = defaultdict(list) gt_instance['inst_id'] = gt_inst_count gt_instance['label'] = label gt_instance['pixel_count'] = np.count_nonzero(mask) gt_instance['mask'] = mask gt_instances[label].append(gt_instance) gt_inst_count += 1 pred_inst_count = 0 for pred in preds: label, conf, mask = pred pred_instance = defaultdict(list) pred_instance['label'] = label pred_instance['pred_id'] = pred_inst_count pred_instance['pixel_count'] = np.count_nonzero(mask) pred_instance['confidence'] = conf if ignore_mask is not None: pred_instance['void_intersection'] = np.count_nonzero( np.logical_and(mask, ignore_mask)) # Loop through all ground truth instances with matching label matched_gt = [] for gt_num, gt_instance in enumerate(gt_instances[label]): # print(gt_instances) intersection = np.count_nonzero( np.logical_and(mask, gt_instances[label][gt_num]['mask'])) if intersection > 0: gt_copy = gt_instance.copy() pred_copy = pred_instance.copy() gt_copy['intersection'] = intersection pred_copy['intersection'] = intersection matched_gt.append(gt_copy) gt_instances[label][gt_num]['matched_pred'].append( pred_copy) pred_instance['matched_gt'] = matched_gt pred_inst_count += 1 pred_instances[label].append(pred_instance) return pred_instances, gt_instances @staticmethod def convert_gt_map(seg_map, ins_map): """ Convet the ground truth with format (h*w) to the format that satisfies the AP calculation. Args: seg_map (np.ndarray): the sementic segmentation map with shape H * W. Value is 0, 1, 2, ... ins_map (np.ndarray): the instance segmentation map with shape H * W. Value is 0, 1, 2, ... Returns: list: tuple list like: [(label, mask), ...] """ gts = [] instance_cnt = np.unique(ins_map) for i in instance_cnt: if i == 0: continue mask = ins_map == i label = seg_map[mask][0] gts.append((label, mask.astype('int32'))) return gts @staticmethod def convert_pred_map(seg_pred, pan_pred): """ Convet the predictions with format (h*w) to the format that satisfies the AP calculation. Args: seg_pred (np.ndarray): the sementic segmentation map with shape C * H * W. Value is probability. pan_pred (np.ndarray): panoptic predictions, void_label, stuff_id * label_divisor, thing_id * label_divisor + ins_id , ins_id >= 1. Returns: list: tuple list like: [(label, score, mask), ...] """ preds = [] instance_cnt = np.unique(pan_pred) for i in instance_cnt: if (i < 1000) or (i % 1000 == 0): continue mask = pan_pred == i label = i // 1000 score = np.mean(seg_pred[label][mask]) preds.append((label, score, mask.astype('int32'))) return preds