Merge branch 'pretty_ap' into 'master'

Pretty ap

See merge request open-mmlab/mmdet.3d!56

.isort.cfg

@@ -3,6 +3,6 @@ line_length = 79
 multi_line_output = 0
 known_standard_library = setuptools
 known_first_party = mmdet,mmdet3d
-known_third_party = cv2,mmcv,numba,numpy,nuscenes,pycocotools,pyquaternion,pytest,scipy,shapely,six,skimage,torch,torchvision
+known_third_party = cv2,mmcv,numba,numpy,nuscenes,pycocotools,pyquaternion,pytest,scipy,shapely,six,skimage,terminaltables,torch,torchvision
 no_lines_before = STDLIB,LOCALFOLDER
 default_section = THIRDPARTY

mmdet3d/core/evaluation/indoor_eval.py

 import numpy as np
 import torch
+from mmcv.utils import print_log
+from terminaltables import AsciiTable
 
 from mmdet3d.core.bbox.iou_calculators.iou3d_calculator import bbox_overlaps_3d
 
@@ -263,14 +265,14 @@ def eval_map_recall(det_infos, gt_infos, ovthresh=None):
                 recall[iou_idx][label], precision[iou_idx][label], ap[iou_idx][
                     label] = ret_values[i][iou_idx]
             else:
-                recall[iou_idx][label] = [0]
-                precision[iou_idx][label] = [0]
-                ap[iou_idx][label] = [0]
+                recall[iou_idx][label] = np.zeros(1)
+                precision[iou_idx][label] = np.zeros(1)
+                ap[iou_idx][label] = np.zeros(1)
 
     return recall, precision, ap
 
 
-def indoor_eval(gt_annos, dt_annos, metric, label2cat):
+def indoor_eval(gt_annos, dt_annos, metric, label2cat, logger=None):
     """Scannet Evaluation.
 
     Evaluate the result of the detection.
@@ -280,6 +282,8 @@ def indoor_eval(gt_annos, dt_annos, metric, label2cat):
         dt_annos (list[dict]): Detection annotations.
         metric (list[float]): AP IoU thresholds.
         label2cat (dict): {label: cat}.
+        logger (logging.Logger | str | None): The way to print the mAP
+            summary. See `mmdet.utils.print_log()` for details. Default: None.
 
     Return:
         dict: Dict of results.
@@ -301,20 +305,41 @@ def indoor_eval(gt_annos, dt_annos, metric, label2cat):
                     boxes_3d=np.array([], dtype=np.float32),
                     labels_3d=np.array([], dtype=np.int64)))
 
-    result_str = str()
-    result_str += 'mAP'
     rec, prec, ap = eval_map_recall(dt_annos, gt_infos, metric)
-    ret_dict = {}
+    ret_dict = dict()
+    header = ['classes']
+    table_columns = [[label2cat[label]
+                      for label in ap[0].keys()] + ['Overall']]
+
     for i, iou_thresh in enumerate(metric):
+        header.append(f'AP_{iou_thresh:.2f}')
+        header.append(f'AR_{iou_thresh:.2f}')
         rec_list = []
         for label in ap[i].keys():
             ret_dict[f'{label2cat[label]}_AP_{iou_thresh:.2f}'] = float(
                 ap[i][label][0])
         ret_dict[f'mAP_{iou_thresh:.2f}'] = float(
             np.mean(list(ap[i].values())))
+
+        table_columns.append(list(map(float, list(ap[i].values()))))
+        table_columns[-1] += [ret_dict[f'mAP_{iou_thresh:.2f}']]
+        table_columns[-1] = [f'{x:.4f}' for x in table_columns[-1]]
+
         for label in rec[i].keys():
             ret_dict[f'{label2cat[label]}_rec_{iou_thresh:.2f}'] = float(
                 rec[i][label][-1])
             rec_list.append(rec[i][label][-1])
         ret_dict[f'mAR_{iou_thresh:.2f}'] = float(np.mean(rec_list))
+
+        table_columns.append(list(map(float, rec_list)))
+        table_columns[-1] += [ret_dict[f'mAR_{iou_thresh:.2f}']]
+        table_columns[-1] = [f'{x:.4f}' for x in table_columns[-1]]
+
+    table_data = [header]
+    table_rows = list(zip(*table_columns))
+    table_data += table_rows
+    table = AsciiTable(table_data)
+    table.inner_footing_row_border = True
+    print_log('\n' + table.table, logger=logger)
+
     return ret_dict

mmdet3d/datasets/custom_3d.py

@@ -3,7 +3,6 @@ import tempfile
 
 import mmcv
 import numpy as np
-from mmcv.utils import print_log
 from torch.utils.data import Dataset
 
 from mmdet.datasets import DATASETS
@@ -139,14 +138,8 @@ class Custom3DDataset(Dataset):
         ), f'Expect elements in results to be dict, got {type(results[0])}.'
         gt_annos = [info['annos'] for info in self.data_infos]
         label2cat = {i: cat_id for i, cat_id in enumerate(self.CLASSES)}
-        ret_dict = indoor_eval(gt_annos, results, iou_thr, label2cat)
-
-        result_str = str()
-        for key, val in ret_dict.items():
-            result_str += f'{key} : {val} \n'
-        mAP_25, mAP_50 = ret_dict['mAP_0.25'], ret_dict['mAP_0.50']
-        result_str += f'mAP(0.25): {mAP_25}    mAP(0.50): {mAP_50}'
-        print_log('\n' + result_str, logger=logger)
+        ret_dict = indoor_eval(
+            gt_annos, results, iou_thr, label2cat, logger=logger)
 
         return ret_dict