Skip to content

GitLab

Unverified Commit 8bf2f5a4 authored by Xiang Xu's avatar Xiang Xu Committed by GitHub
Browse files

[Enhance]: Add format_only option for nuScenes and Waymo evaluation (#2151) 

* support submit test set on nuscenes and waymo dataset

* Fix typo
parent ed081770
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 341 additions and 295 deletions
configs/_base_/datasets/kitti-mono3d.py
View file @ 8bf2f5a4
......@@ -80,8 +80,7 @@ test_dataloader = val_dataloader
val_evaluator = dict(
type='KittiMetric',
ann_file=data_root + 'kitti_infos_val.pkl',
metric='bbox',
pred_box_type_3d='Camera')
metric='bbox')
test_evaluator = val_evaluator
......
mmdet3d/evaluation/metrics/kitti_metric.py
View file @ 8bf2f5a4
This diff is collapsed.
mmdet3d/evaluation/metrics/waymo_metric.py
View file @ 8bf2f5a4
# Copyright (c) OpenMMLab. All rights reserved.
import tempfile
from os import path as osp
from typing import Dict, List, Optional, Union
from typing import Dict, List, Optional, Tuple, Union
import mmengine
import numpy as np
......@@ -25,50 +25,56 @@ class WaymoMetric(KittiMetric):
ann_file (str): The path of the annotation file in kitti format.
waymo_bin_file (str): The path of the annotation file in waymo format.
data_root (str): Path of dataset root.
Used for storing waymo evaluation programs.
split (str): The split of the evaluation set.
metric (str | list[str]): Metrics to be evaluated.
Default to 'mAP'.
pcd_limit_range (list): The range of point cloud used to
Used for storing waymo evaluation programs.
split (str): The split of the evaluation set. Defaults to 'training'.
metric (str or List[str]): Metrics to be evaluated.
Defaults to 'mAP'.
pcd_limit_range (List[float]): The range of point cloud used to
filter invalid predicted boxes.
Default to [0, -40, -3, 70.4, 40, 0.0].
Defaults to [-85, -85, -5, 85, 85, 5].
convert_kitti_format (bool): Whether to convert the results to
kitti format. Now, in order to be compatible with camera-based
methods, defaults to True.
prefix (str, optional): The prefix that will be added in the metric
names to disambiguate homonymous metrics of different evaluators.
If prefix is not provided in the argument, self.default_prefix
will be used instead. Defaults to None.
convert_kitti_format (bool, optional): Whether convert the reuslts to
kitti format. Now, in order to be compatible with camera-based
methods, defaults to True.
format_only (bool): Format the output results without perform
evaluation. It is useful when you want to format the result
to a specific format and submit it to the test server.
Defaults to False.
pklfile_prefix (str, optional): The prefix of pkl files, including
the file path and the prefix of filename, e.g., "a/b/prefix".
If not specified, a temp file will be created. Default: None.
If not specified, a temp file will be created. Defaults to None.
submission_prefix (str, optional): The prefix of submission data.
If not specified, the submission data will not be generated.
Default: None.
load_type (str, optional): Type of loading mode during training.
Defaults to None.
load_type (str): Type of loading mode during training.
- 'frame_based': Load all of the instances in the frame.
- 'mv_image_based': Load all of the instances in the frame and need
to convert to the FOV-based data type to support image-based
detector.
- 'fov_image_base': Only load the instances inside the default cam,
and need to convert to the FOV-based data type to support
image-based detector.
default_cam_key (str, optional): The default camera for lidar to
camear conversion. By default, KITTI: CAM2, Waymo: CAM_FRONT
use_pred_sample_idx (bool, optional): In formating results, use the
sample index from the prediction or from the load annoataitons.
to convert to the FOV-based data type to support image-based
detector.
- 'fov_image_based': Only load the instances inside the default
cam, and need to convert to the FOV-based data type to support
image-based detector.
default_cam_key (str): The default camera for lidar to camera
conversion. By default, KITTI: 'CAM2', Waymo: 'CAM_FRONT'.
Defaults to 'CAM_FRONT'.
use_pred_sample_idx (bool): In formating results, use the
sample index from the prediction or from the load annotations.
By default, KITTI: True, Waymo: False, Waymo has a conversion
process, which needs to use the sample id from load annotation.
process, which needs to use the sample idx from load annotation.
Defaults to False.
collect_device (str): Device name used for collecting results
from different ranks during distributed training. Must be 'cpu' or
'gpu'. Defaults to 'cpu'.
file_client_args (dict): file client for reading gt in waymo format.
file_client_args (dict): File client for reading gt in waymo format.
Defaults to ``dict(backend='disk')``.
idx2metainfo (Optional[str], optional): The file path of the metainfo
in waymmo. It stores the mapping from sample_idx to metainfo.
The metainfo must contain the keys: 'idx2contextname' and
'idx2timestamp'. Defaults to None.
idx2metainfo (str, optional): The file path of the metainfo in waymo.
It stores the mapping from sample_idx to metainfo. The metainfo
must contain the keys: 'idx2contextname' and 'idx2timestamp'.
Defaults to None.
"""
num_cams = 5
......@@ -81,14 +87,15 @@ class WaymoMetric(KittiMetric):
pcd_limit_range: List[float] = [-85, -85, -5, 85, 85, 5],
convert_kitti_format: bool = True,
prefix: Optional[str] = None,
pklfile_prefix: str = None,
submission_prefix: str = None,
format_only: bool = False,
pklfile_prefix: Optional[str] = None,
submission_prefix: Optional[str] = None,
load_type: str = 'frame_based',
default_cam_key: str = 'CAM_FRONT',
use_pred_sample_idx: bool = False,
collect_device: str = 'cpu',
file_client_args: dict = dict(backend='disk'),
idx2metainfo: Optional[str] = None):
idx2metainfo: Optional[str] = None) -> None:
self.waymo_bin_file = waymo_bin_file
self.data_root = data_root
self.split = split
......@@ -101,7 +108,7 @@ class WaymoMetric(KittiMetric):
else:
self.idx2metainfo = None
super().__init__(
super(WaymoMetric, self).__init__(
ann_file=ann_file,
metric=metric,
pcd_limit_range=pcd_limit_range,
......@@ -111,13 +118,20 @@ class WaymoMetric(KittiMetric):
default_cam_key=default_cam_key,
collect_device=collect_device,
file_client_args=file_client_args)
self.format_only = format_only
if self.format_only:
assert pklfile_prefix is not None, 'pklfile_prefix must be '
'not None when format_only is True, otherwise the result files '
'will be saved to a temp directory which will be cleaned up at '
'the end.'
self.default_prefix = 'Waymo metric'
def compute_metrics(self, results: list) -> Dict[str, float]:
def compute_metrics(self, results: List[dict]) -> Dict[str, float]:
"""Compute the metrics from processed results.
Args:
results (list): The processed results of the whole dataset.
results (List[dict]): The processed results of the whole dataset.
Returns:
Dict[str, float]: The computed metrics. The keys are the names of
......@@ -155,7 +169,7 @@ class WaymoMetric(KittiMetric):
if 'image_sweeps' in info:
camera_info['image_sweeps'] = info['image_sweeps']
# TODO check if need to modify the sample id
# TODO check if need to modify the sample idx
# TODO check when will use it except for evaluation.
camera_info['sample_idx'] = info['sample_idx']
new_data_infos.append(camera_info)
......@@ -175,6 +189,12 @@ class WaymoMetric(KittiMetric):
classes=self.classes)
metric_dict = {}
if self.format_only:
logger.info('results are saved in '
f'{osp.dirname(self.pklfile_prefix)}')
return metric_dict
for metric in self.metrics:
ap_dict = self.waymo_evaluate(
pklfile_prefix, metric=metric, logger=logger)
......@@ -188,19 +208,19 @@ class WaymoMetric(KittiMetric):
def waymo_evaluate(self,
pklfile_prefix: str,
metric: str = None,
logger: MMLogger = None) -> dict:
metric: Optional[str] = None,
logger: Optional[MMLogger] = None) -> Dict[str, float]:
"""Evaluation in Waymo protocol.
Args:
pklfile_prefix (str): The location that stored the prediction
results.
metric (str): Metric to be evaluated. Defaults to None.
metric (str, optional): Metric to be evaluated. Defaults to None.
logger (MMLogger, optional): Logger used for printing
related information during evaluation. Default: None.
related information during evaluation. Defaults to None.
Returns:
dict[str, float]: Results of each evaluation metric.
Dict[str, float]: Results of each evaluation metric.
"""
import subprocess
......@@ -238,8 +258,6 @@ class WaymoMetric(KittiMetric):
}
mAP_splits = ret_texts.split('mAP ')
mAPH_splits = ret_texts.split('mAPH ')
mAP_splits = ret_texts.split('mAP ')
mAPH_splits = ret_texts.split('mAPH ')
for idx, key in enumerate(ap_dict.keys()):
split_idx = int(idx / 2) + 1
if idx % 2 == 0: # mAP
......@@ -307,31 +325,32 @@ class WaymoMetric(KittiMetric):
ap_dict['Cyclist mAPH']) / 3
return ap_dict
def format_results(self,
results: List[dict],
pklfile_prefix: str = None,
submission_prefix: str = None,
classes: List[str] = None):
def format_results(
self,
results: List[dict],
pklfile_prefix: Optional[str] = None,
submission_prefix: Optional[str] = None,
classes: Optional[List[str]] = None
) -> Tuple[dict, Union[tempfile.TemporaryDirectory, None]]:
"""Format the results to bin file.
Args:
results (list[dict]): Testing results of the
dataset.
results (List[dict]): Testing results of the dataset.
pklfile_prefix (str, optional): The prefix of pkl files. It
includes the file path and the prefix of filename, e.g.,
"a/b/prefix". If not specified, a temp file will be created.
Default: None.
Defaults to None.
submission_prefix (str, optional): The prefix of submitted files.
It includes the file path and the prefix of filename, e.g.,
"a/b/prefix". If not specified, a temp file will be created.
Default: None.
classes (list[String], optional): A list of class name. Defaults
to None.
Defaults to None.
classes (List[str], optional): A list of class name.
Defaults to None.
Returns:
tuple: (result_dict, tmp_dir), result_dict is a dict containing
the formatted result, tmp_dir is the temporal directory created
for saving json files when jsonfile_prefix is not specified.
the formatted result, tmp_dir is the temporal directory created
for saving json files when jsonfile_prefix is not specified.
"""
waymo_save_tmp_dir = tempfile.TemporaryDirectory()
waymo_results_save_dir = waymo_save_tmp_dir.name
......@@ -378,15 +397,16 @@ class WaymoMetric(KittiMetric):
return final_results, waymo_save_tmp_dir
def merge_multi_view_boxes(self, box_dict_per_frame: List[dict],
cam0_info: dict):
cam0_info: dict) -> dict:
"""Merge bounding boxes predicted from multi-view images.
Args:
box_dict_per_frame (list[dict]): The results of prediction
box_dict_per_frame (List[dict]): The results of prediction
for each camera.
cam2_info (dict): store the sample id for the given frame.
cam0_info (dict): Store the sample idx for the given frame.
Returns:
merged_box_dict (dict), store the merge results
dict: Merged results.
"""
box_dict = dict()
# convert list[dict] to dict[list]
......@@ -444,27 +464,28 @@ class WaymoMetric(KittiMetric):
)
return merged_box_dict
def bbox2result_kitti(self,
net_outputs: list,
sample_id_list: list,
class_names: list,
pklfile_prefix: str = None,
submission_prefix: str = None):
def bbox2result_kitti(
self,
net_outputs: List[dict],
sample_idx_list: List[int],
class_names: List[str],
pklfile_prefix: Optional[str] = None,
submission_prefix: Optional[str] = None) -> List[dict]:
"""Convert 3D detection results to kitti format for evaluation and test
submission.
Args:
net_outputs (list[dict]): List of array storing the
net_outputs (List[dict]): List of dict storing the
inferenced bounding boxes and scores.
sample_id_list (list[int]): List of input sample id.
class_names (list[String]): A list of class names.
sample_idx_list (List[int]): List of input sample idx.
class_names (List[str]): A list of class names.
pklfile_prefix (str, optional): The prefix of pkl file.
Defaults to None.
submission_prefix (str, optional): The prefix of submission file.
Defaults to None.
Returns:
list[dict]: A list of dictionaries with the kitti format.
List[dict]: A list of dictionaries with the kitti format.
"""
if submission_prefix is not None:
mmengine.mkdir_or_exist(submission_prefix)
......@@ -473,8 +494,7 @@ class WaymoMetric(KittiMetric):
print('\nConverting prediction to KITTI format')
for idx, pred_dicts in enumerate(
mmengine.track_iter_progress(net_outputs)):
annos = []
sample_idx = sample_id_list[idx]
sample_idx = sample_idx_list[idx]
info = self.data_infos[sample_idx]
if self.load_type == 'mv_image_based':
......@@ -536,7 +556,6 @@ class WaymoMetric(KittiMetric):
anno['score'].append(score)
anno = {k: np.stack(v) for k, v in anno.items()}
annos.append(anno)
else:
anno = {
'name': np.array([]),
......@@ -549,7 +568,6 @@ class WaymoMetric(KittiMetric):
'rotation_y': np.array([]),
'score': np.array([]),
}
annos.append(anno)
if submission_prefix is not None:
curr_file = f'{submission_prefix}/{sample_idx:06d}.txt'
......@@ -577,10 +595,10 @@ class WaymoMetric(KittiMetric):
# In waymo validation sample_idx in prediction is 000xxx
# but in info file it is 1000xxx
save_sample_idx = box_dict['sample_idx']
annos[-1]['sample_idx'] = np.array(
[save_sample_idx] * len(annos[-1]['score']), dtype=np.int64)
anno['sample_idx'] = np.array(
[save_sample_idx] * len(anno['score']), dtype=np.int64)
det_annos += annos
det_annos.append(anno)
if pklfile_prefix is not None:
if not pklfile_prefix.endswith(('.pkl', '.pickle')):
......@@ -592,16 +610,16 @@ class WaymoMetric(KittiMetric):
return det_annos
def convert_valid_bboxes(self, box_dict: dict, info: dict):
def convert_valid_bboxes(self, box_dict: dict, info: dict) -> dict:
"""Convert the predicted boxes into valid ones. Should handle the
load_model (frame_based, mv_image_based, fov_image_based), separately.
Args:
box_dict (dict): Box dictionaries to be converted.
- bboxes_3d (:obj:`LiDARInstance3DBoxes`): 3D bounding boxes.
- scores_3d (torch.Tensor): Scores of boxes.
- labels_3d (torch.Tensor): Class labels of boxes.
- bboxes_3d (:obj:`BaseInstance3DBoxes`): 3D bounding boxes.
- scores_3d (Tensor): Scores of boxes.
- labels_3d (Tensor): Class labels of boxes.
info (dict): Data info.
Returns:
......@@ -609,9 +627,9 @@ class WaymoMetric(KittiMetric):
- bbox (np.ndarray): 2D bounding boxes.
- box3d_camera (np.ndarray): 3D bounding boxes in
camera coordinate.
camera coordinate.
- box3d_lidar (np.ndarray): 3D bounding boxes in
LiDAR coordinate.
LiDAR coordinate.
- scores (np.ndarray): Scores of boxes.
- label_preds (np.ndarray): Class label predictions.
- sample_idx (int): Sample index.
......@@ -673,7 +691,7 @@ class WaymoMetric(KittiMetric):
valid_pcd_inds = ((box_preds_lidar.center > limit_range[:3]) &
(box_preds_lidar.center < limit_range[3:]))
valid_inds = valid_pcd_inds.all(-1)
if self.load_type in ['mv_image_based', 'fov_image_based']:
elif self.load_type in ['mv_image_based', 'fov_image_based']:
valid_inds = valid_cam_inds
if valid_inds.sum() > 0:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment