[Feature] add kitti AP40 evaluation metric (v1.0.0.dev0) (#927)

* Add citation (#901)

* [Feature] Add python3.9 in CI (#900)

* Add python3.0 in CI

* Add python3.0 in CI

* Bump to v0.17.0 (#898)

* Update README.md

* Update README_zh-CN.md

* Update version.py

* Update getting_started.md

* Update getting_started.md

* Update changelog.md

* Remove "recent" in the news

* Remove "recent" in the news

* Fix comments

* [Docs] Fix the version of sphinx (#902)

* Fix sphinx version

* Fix sphinx version

* Fix sphinx version

* Fix sphinx version

* Fix sphinx version

* Fix sphinx version

* Fix sphinx version

* Fix sphinx version

* Fix sphinx version

* Fix sphinx version

* Fix sphinx version

* Fix sphinx version

* add AP40

* add unitest

* add unitest

* seperate AP11 and AP40

* fix some typos
Co-authored-by: dingchang <hudingchang.vendor@sensetime.com>
Co-authored-by: Tai-Wang <tab_wang@outlook.com>

mmdet3d/core/evaluation/kitti_utils/eval.py

@@ -569,13 +569,20 @@ def eval_class(gt_annos,
     return ret_dict
 
 
-def get_mAP(prec):
+def get_mAP11(prec):
     sums = 0
     for i in range(0, prec.shape[-1], 4):
         sums = sums + prec[..., i]
     return sums / 11 * 100
 
 
+def get_mAP40(prec):
+    sums = 0
+    for i in range(1, prec.shape[-1]):
+        sums = sums + prec[..., i]
+    return sums / 40 * 100
+
+
 def print_str(value, *arg, sstream=None):
     if sstream is None:
         sstream = sysio.StringIO()
@@ -592,8 +599,10 @@ def do_eval(gt_annos,
             eval_types=['bbox', 'bev', '3d']):
     # min_overlaps: [num_minoverlap, metric, num_class]
     difficultys = [0, 1, 2]
-    mAP_bbox = None
-    mAP_aos = None
+    mAP11_bbox = None
+    mAP11_aos = None
+    mAP40_bbox = None
+    mAP40_aos = None
     if 'bbox' in eval_types:
         ret = eval_class(
             gt_annos,
@@ -604,22 +613,29 @@ def do_eval(gt_annos,
             min_overlaps,
             compute_aos=('aos' in eval_types))
         # ret: [num_class, num_diff, num_minoverlap, num_sample_points]
-        mAP_bbox = get_mAP(ret['precision'])
+        mAP11_bbox = get_mAP11(ret['precision'])
+        mAP40_bbox = get_mAP40(ret['precision'])
         if 'aos' in eval_types:
-            mAP_aos = get_mAP(ret['orientation'])
+            mAP11_aos = get_mAP11(ret['orientation'])
+            mAP40_aos = get_mAP40(ret['orientation'])
 
-    mAP_bev = None
+    mAP11_bev = None
+    mAP40_bev = None
     if 'bev' in eval_types:
         ret = eval_class(gt_annos, dt_annos, current_classes, difficultys, 1,
                          min_overlaps)
-        mAP_bev = get_mAP(ret['precision'])
+        mAP11_bev = get_mAP11(ret['precision'])
+        mAP40_bev = get_mAP40(ret['precision'])
 
-    mAP_3d = None
+    mAP11_3d = None
+    mAP40_3d = None
     if '3d' in eval_types:
         ret = eval_class(gt_annos, dt_annos, current_classes, difficultys, 2,
                          min_overlaps)
-        mAP_3d = get_mAP(ret['precision'])
-    return mAP_bbox, mAP_bev, mAP_3d, mAP_aos
+        mAP11_3d = get_mAP11(ret['precision'])
+        mAP40_3d = get_mAP40(ret['precision'])
+    return (mAP11_bbox, mAP11_bev, mAP11_3d, mAP11_aos, mAP40_bbox, mAP40_bev,
+            mAP40_3d, mAP40_aos)
 
 
 def do_coco_style_eval(gt_annos, dt_annos, current_classes, overlap_ranges,
@@ -629,9 +645,10 @@ def do_coco_style_eval(gt_annos, dt_annos, current_classes, overlap_ranges,
     for i in range(overlap_ranges.shape[1]):
         for j in range(overlap_ranges.shape[2]):
             min_overlaps[:, i, j] = np.linspace(*overlap_ranges[:, i, j])
-    mAP_bbox, mAP_bev, mAP_3d, mAP_aos = do_eval(gt_annos, dt_annos,
-                                                 current_classes, min_overlaps,
-                                                 compute_aos)
+    mAP_bbox, mAP_bev, mAP_3d, mAP_aos, _, _, \
+        _, _ = do_eval(gt_annos, dt_annos,
+                       current_classes, min_overlaps,
+                       compute_aos)
     # ret: [num_class, num_diff, num_minoverlap]
     mAP_bbox = mAP_bbox.mean(-1)
     mAP_bev = mAP_bev.mean(-1)
@@ -703,33 +720,109 @@ def kitti_eval(gt_annos,
     if compute_aos:
         eval_types.append('aos')
 
-    mAPbbox, mAPbev, mAP3d, mAPaos = do_eval(gt_annos, dt_annos,
-                                             current_classes, min_overlaps,
-                                             eval_types)
+    mAP11_bbox, mAP11_bev, mAP11_3d, mAP11_aos, mAP40_bbox, mAP40_bev, \
+        mAP40_3d, mAP40_aos = do_eval(gt_annos, dt_annos,
+                                      current_classes, min_overlaps,
+                                      eval_types)
 
     ret_dict = {}
     difficulty = ['easy', 'moderate', 'hard']
+
+    # calculate AP11
+    result += '\n----------- AP11 Results ------------\n\n'
     for j, curcls in enumerate(current_classes):
         # mAP threshold array: [num_minoverlap, metric, class]
         # mAP result: [num_class, num_diff, num_minoverlap]
         curcls_name = class_to_name[curcls]
         for i in range(min_overlaps.shape[0]):
             # prepare results for print
-            result += ('{} AP@{:.2f}, {:.2f}, {:.2f}:\n'.format(
+            result += ('{} AP11@{:.2f}, {:.2f}, {:.2f}:\n'.format(
                 curcls_name, *min_overlaps[i, :, j]))
-            if mAPbbox is not None:
-                result += 'bbox AP:{:.4f}, {:.4f}, {:.4f}\n'.format(
-                    *mAPbbox[j, :, i])
-            if mAPbev is not None:
-                result += 'bev  AP:{:.4f}, {:.4f}, {:.4f}\n'.format(
-                    *mAPbev[j, :, i])
-            if mAP3d is not None:
-                result += '3d   AP:{:.4f}, {:.4f}, {:.4f}\n'.format(
-                    *mAP3d[j, :, i])
+            if mAP11_bbox is not None:
+                result += 'bbox AP11:{:.4f}, {:.4f}, {:.4f}\n'.format(
+                    *mAP11_bbox[j, :, i])
+            if mAP11_bev is not None:
+                result += 'bev  AP11:{:.4f}, {:.4f}, {:.4f}\n'.format(
+                    *mAP11_bev[j, :, i])
+            if mAP11_3d is not None:
+                result += '3d   AP11:{:.4f}, {:.4f}, {:.4f}\n'.format(
+                    *mAP11_3d[j, :, i])
+            if compute_aos:
+                result += 'aos  AP11:{:.2f}, {:.2f}, {:.2f}\n'.format(
+                    *mAP11_aos[j, :, i])
+
+            # prepare results for logger
+            for idx in range(3):
+                if i == 0:
+                    postfix = f'{difficulty[idx]}_strict'
+                else:
+                    postfix = f'{difficulty[idx]}_loose'
+                prefix = f'KITTI/{curcls_name}'
+                if mAP11_3d is not None:
+                    ret_dict[f'{prefix}_3D_AP11_{postfix}'] =\
+                        mAP11_3d[j, idx, i]
+                if mAP11_bev is not None:
+                    ret_dict[f'{prefix}_BEV_AP11_{postfix}'] =\
+                        mAP11_bev[j, idx, i]
+                if mAP11_bbox is not None:
+                    ret_dict[f'{prefix}_2D_AP11_{postfix}'] =\
+                        mAP11_bbox[j, idx, i]
+
+    # calculate mAP11 over all classes if there are multiple classes
+    if len(current_classes) > 1:
+        # prepare results for print
+        result += ('\nOverall AP11@{}, {}, {}:\n'.format(*difficulty))
+        if mAP11_bbox is not None:
+            mAP11_bbox = mAP11_bbox.mean(axis=0)
+            result += 'bbox AP11:{:.4f}, {:.4f}, {:.4f}\n'.format(
+                *mAP11_bbox[:, 0])
+        if mAP11_bev is not None:
+            mAP11_bev = mAP11_bev.mean(axis=0)
+            result += 'bev  AP11:{:.4f}, {:.4f}, {:.4f}\n'.format(
+                *mAP11_bev[:, 0])
+        if mAP11_3d is not None:
+            mAP11_3d = mAP11_3d.mean(axis=0)
+            result += '3d   AP11:{:.4f}, {:.4f}, {:.4f}\n'.format(*mAP11_3d[:,
+                                                                            0])
+        if compute_aos:
+            mAP11_aos = mAP11_aos.mean(axis=0)
+            result += 'aos  AP11:{:.2f}, {:.2f}, {:.2f}\n'.format(
+                *mAP11_aos[:, 0])
 
+        # prepare results for logger
+        for idx in range(3):
+            postfix = f'{difficulty[idx]}'
+            if mAP11_3d is not None:
+                ret_dict[f'KITTI/Overall_3D_AP11_{postfix}'] = mAP11_3d[idx, 0]
+            if mAP11_bev is not None:
+                ret_dict[f'KITTI/Overall_BEV_AP11_{postfix}'] =\
+                    mAP11_bev[idx, 0]
+            if mAP11_bbox is not None:
+                ret_dict[f'KITTI/Overall_2D_AP11_{postfix}'] =\
+                    mAP11_bbox[idx, 0]
+
+    # Calculate AP40
+    result += '\n----------- AP40 Results ------------\n\n'
+    for j, curcls in enumerate(current_classes):
+        # mAP threshold array: [num_minoverlap, metric, class]
+        # mAP result: [num_class, num_diff, num_minoverlap]
+        curcls_name = class_to_name[curcls]
+        for i in range(min_overlaps.shape[0]):
+            # prepare results for print
+            result += ('{} AP40@{:.2f}, {:.2f}, {:.2f}:\n'.format(
+                curcls_name, *min_overlaps[i, :, j]))
+            if mAP40_bbox is not None:
+                result += 'bbox AP40:{:.4f}, {:.4f}, {:.4f}\n'.format(
+                    *mAP40_bbox[j, :, i])
+            if mAP40_bev is not None:
+                result += 'bev  AP40:{:.4f}, {:.4f}, {:.4f}\n'.format(
+                    *mAP40_bev[j, :, i])
+            if mAP40_3d is not None:
+                result += '3d   AP40:{:.4f}, {:.4f}, {:.4f}\n'.format(
+                    *mAP40_3d[j, :, i])
             if compute_aos:
-                result += 'aos  AP:{:.2f}, {:.2f}, {:.2f}\n'.format(
-                    *mAPaos[j, :, i])
+                result += 'aos  AP40:{:.2f}, {:.2f}, {:.2f}\n'.format(
+                    *mAP40_aos[j, :, i])
 
             # prepare results for logger
             for idx in range(3):
@@ -738,39 +831,48 @@ def kitti_eval(gt_annos,
                 else:
                     postfix = f'{difficulty[idx]}_loose'
                 prefix = f'KITTI/{curcls_name}'
-                if mAP3d is not None:
-                    ret_dict[f'{prefix}_3D_{postfix}'] = mAP3d[j, idx, i]
-                if mAPbev is not None:
-                    ret_dict[f'{prefix}_BEV_{postfix}'] = mAPbev[j, idx, i]
-                if mAPbbox is not None:
-                    ret_dict[f'{prefix}_2D_{postfix}'] = mAPbbox[j, idx, i]
-
-    # calculate mAP over all classes if there are multiple classes
+                if mAP40_3d is not None:
+                    ret_dict[f'{prefix}_3D_AP40_{postfix}'] =\
+                        mAP40_3d[j, idx, i]
+                if mAP40_bev is not None:
+                    ret_dict[f'{prefix}_BEV_AP40_{postfix}'] =\
+                        mAP40_bev[j, idx, i]
+                if mAP40_bbox is not None:
+                    ret_dict[f'{prefix}_2D_AP40_{postfix}'] =\
+                        mAP40_bbox[j, idx, i]
+
+    # calculate mAP40 over all classes if there are multiple classes
     if len(current_classes) > 1:
         # prepare results for print
-        result += ('\nOverall AP@{}, {}, {}:\n'.format(*difficulty))
-        if mAPbbox is not None:
-            mAPbbox = mAPbbox.mean(axis=0)
-            result += 'bbox AP:{:.4f}, {:.4f}, {:.4f}\n'.format(*mAPbbox[:, 0])
-        if mAPbev is not None:
-            mAPbev = mAPbev.mean(axis=0)
-            result += 'bev  AP:{:.4f}, {:.4f}, {:.4f}\n'.format(*mAPbev[:, 0])
-        if mAP3d is not None:
-            mAP3d = mAP3d.mean(axis=0)
-            result += '3d   AP:{:.4f}, {:.4f}, {:.4f}\n'.format(*mAP3d[:, 0])
+        result += ('\nOverall AP40@{}, {}, {}:\n'.format(*difficulty))
+        if mAP40_bbox is not None:
+            mAP40_bbox = mAP40_bbox.mean(axis=0)
+            result += 'bbox AP40:{:.4f}, {:.4f}, {:.4f}\n'.format(
+                *mAP40_bbox[:, 0])
+        if mAP40_bev is not None:
+            mAP40_bev = mAP40_bev.mean(axis=0)
+            result += 'bev  AP40:{:.4f}, {:.4f}, {:.4f}\n'.format(
+                *mAP40_bev[:, 0])
+        if mAP40_3d is not None:
+            mAP40_3d = mAP40_3d.mean(axis=0)
+            result += '3d   AP40:{:.4f}, {:.4f}, {:.4f}\n'.format(*mAP40_3d[:,
+                                                                            0])
         if compute_aos:
-            mAPaos = mAPaos.mean(axis=0)
-            result += 'aos  AP:{:.2f}, {:.2f}, {:.2f}\n'.format(*mAPaos[:, 0])
+            mAP40_aos = mAP40_aos.mean(axis=0)
+            result += 'aos  AP40:{:.2f}, {:.2f}, {:.2f}\n'.format(
+                *mAP40_aos[:, 0])
 
         # prepare results for logger
         for idx in range(3):
             postfix = f'{difficulty[idx]}'
-            if mAP3d is not None:
-                ret_dict[f'KITTI/Overall_3D_{postfix}'] = mAP3d[idx, 0]
-            if mAPbev is not None:
-                ret_dict[f'KITTI/Overall_BEV_{postfix}'] = mAPbev[idx, 0]
-            if mAPbbox is not None:
-                ret_dict[f'KITTI/Overall_2D_{postfix}'] = mAPbbox[idx, 0]
+            if mAP40_3d is not None:
+                ret_dict[f'KITTI/Overall_3D_AP40_{postfix}'] = mAP40_3d[idx, 0]
+            if mAP40_bev is not None:
+                ret_dict[f'KITTI/Overall_BEV_AP40_{postfix}'] =\
+                    mAP40_bev[idx, 0]
+            if mAP40_bbox is not None:
+                ret_dict[f'KITTI/Overall_2D_AP40_{postfix}'] =\
+                    mAP40_bbox[idx, 0]
 
     return result, ret_dict
 

tests/test_metrics/test_kitti_eval.py

@@ -83,31 +83,49 @@ def test_do_eval():
                              [[0.5, 0.5, 0.7], [0.25, 0.25, 0.5],
                               [0.25, 0.25, 0.5]]])
     eval_types = ['bbox', 'bev', '3d', 'aos']
-    mAP_bbox, mAP_bev, mAP_3d, mAP_aos = do_eval([gt_anno], [dt_anno],
+    mAP11_bbox, mAP11_bev, mAP11_3d, mAP11_aos, mAP40_bbox,\
+        mAP40_bev, mAP40_3d, mAP40_aos = do_eval([gt_anno], [dt_anno],
                                                  current_classes, min_overlaps,
                                                  eval_types)
-    expected_mAP_bbox = np.array([[[0., 0.], [9.09090909, 9.09090909],
-                                   [9.09090909, 9.09090909]],
-                                  [[0., 0.], [9.09090909, 9.09090909],
-                                   [9.09090909, 9.09090909]],
-                                  [[0., 0.], [9.09090909, 9.09090909],
-                                   [9.09090909, 9.09090909]]])
-    expected_mAP_bev = np.array([[[0., 0.], [0., 0.], [0., 0.]],
-                                 [[0., 0.], [0., 0.], [0., 0.]],
-                                 [[0., 0.], [0., 0.], [0., 0.]]])
-    expected_mAP_3d = np.array([[[0., 0.], [0., 0.], [0., 0.]],
-                                [[0., 0.], [0., 0.], [0., 0.]],
-                                [[0., 0.], [0., 0.], [0., 0.]]])
-    expected_mAP_aos = np.array([[[0., 0.], [0.55020816, 0.55020816],
-                                  [0.55020816, 0.55020816]],
-                                 [[0., 0.], [8.36633862, 8.36633862],
-                                  [8.36633862, 8.36633862]],
-                                 [[0., 0.], [8.63476893, 8.63476893],
-                                  [8.63476893, 8.63476893]]])
-    assert np.allclose(mAP_bbox, expected_mAP_bbox)
-    assert np.allclose(mAP_bev, expected_mAP_bev)
-    assert np.allclose(mAP_3d, expected_mAP_3d)
-    assert np.allclose(mAP_aos, expected_mAP_aos)
+    expected_mAP11_bbox = np.array([[[0., 0.], [9.09090909, 9.09090909],
+                                     [9.09090909, 9.09090909]],
+                                    [[0., 0.], [9.09090909, 9.09090909],
+                                     [9.09090909, 9.09090909]],
+                                    [[0., 0.], [9.09090909, 9.09090909],
+                                     [9.09090909, 9.09090909]]])
+    expected_mAP40_bbox = np.array([[[0., 0.], [0., 0.], [0., 0.]],
+                                    [[0., 0.], [0., 0.], [0., 0.]],
+                                    [[0., 0.], [2.5, 2.5], [2.5, 2.5]]])
+    expected_mAP11_bev = np.array([[[0., 0.], [0., 0.], [0., 0.]],
+                                   [[0., 0.], [0., 0.], [0., 0.]],
+                                   [[0., 0.], [0., 0.], [0., 0.]]])
+    expected_mAP40_bev = np.array([[[0., 0.], [0., 0.], [0., 0.]],
+                                   [[0., 0.], [0., 0.], [0., 0.]],
+                                   [[0., 0.], [0., 0.], [0., 0.]]])
+    expected_mAP11_3d = np.array([[[0., 0.], [0., 0.], [0., 0.]],
+                                  [[0., 0.], [0., 0.], [0., 0.]],
+                                  [[0., 0.], [0., 0.], [0., 0.]]])
+    expected_mAP40_3d = np.array([[[0., 0.], [0., 0.], [0., 0.]],
+                                  [[0., 0.], [0., 0.], [0., 0.]],
+                                  [[0., 0.], [0., 0.], [0., 0.]]])
+    expected_mAP11_aos = np.array([[[0., 0.], [0.55020816, 0.55020816],
+                                    [0.55020816, 0.55020816]],
+                                   [[0., 0.], [8.36633862, 8.36633862],
+                                    [8.36633862, 8.36633862]],
+                                   [[0., 0.], [8.63476893, 8.63476893],
+                                    [8.63476893, 8.63476893]]])
+    expected_mAP40_aos = np.array([[[0., 0.], [0., 0.], [0., 0.]],
+                                   [[0., 0.], [0., 0.], [0., 0.]],
+                                   [[0., 0.], [1.58140643, 1.58140643],
+                                    [1.58140643, 1.58140643]]])
+    assert np.allclose(mAP11_bbox, expected_mAP11_bbox)
+    assert np.allclose(mAP11_bev, expected_mAP11_bev)
+    assert np.allclose(mAP11_3d, expected_mAP11_3d)
+    assert np.allclose(mAP11_aos, expected_mAP11_aos)
+    assert np.allclose(mAP40_bbox, expected_mAP40_bbox)
+    assert np.allclose(mAP40_bev, expected_mAP40_bev)
+    assert np.allclose(mAP40_3d, expected_mAP40_3d)
+    assert np.allclose(mAP40_aos, expected_mAP40_aos)
 
 
 def test_kitti_eval():
@@ -183,8 +201,14 @@ def test_kitti_eval():
 
     current_classes = [1, 2, 0]
     result, ret_dict = kitti_eval([gt_anno], [dt_anno], current_classes)
-    assert np.isclose(ret_dict['KITTI/Overall_2D_moderate'], 9.090909090909092)
-    assert np.isclose(ret_dict['KITTI/Overall_2D_hard'], 9.090909090909092)
+    assert np.isclose(ret_dict['KITTI/Overall_2D_AP11_moderate'],
+                      9.090909090909092)
+    assert np.isclose(ret_dict['KITTI/Overall_2D_AP11_hard'],
+                      9.090909090909092)
+    assert np.isclose(ret_dict['KITTI/Overall_2D_AP40_moderate'],
+                      0.8333333333333334)
+    assert np.isclose(ret_dict['KITTI/Overall_2D_AP40_hard'],
+                      0.8333333333333334)
 
 
 def test_eval_class():