Unverified Commit 8bf2f5a4 authored by Xiang Xu's avatar Xiang Xu Committed by GitHub
Browse files

[Enhance]: Add format_only option for nuScenes and Waymo evaluation (#2151)

* support submit test set on nuscenes and waymo dataset

* Fix typo
parent ed081770
...@@ -80,8 +80,7 @@ test_dataloader = val_dataloader ...@@ -80,8 +80,7 @@ test_dataloader = val_dataloader
val_evaluator = dict( val_evaluator = dict(
type='KittiMetric', type='KittiMetric',
ann_file=data_root + 'kitti_infos_val.pkl', ann_file=data_root + 'kitti_infos_val.pkl',
metric='bbox', metric='bbox')
pred_box_type_3d='Camera')
test_evaluator = val_evaluator test_evaluator = val_evaluator
......
This diff is collapsed.
# Copyright (c) OpenMMLab. All rights reserved. # Copyright (c) OpenMMLab. All rights reserved.
import tempfile import tempfile
from os import path as osp from os import path as osp
from typing import Dict, List, Optional, Union from typing import Dict, List, Optional, Tuple, Union
import mmengine import mmengine
import numpy as np import numpy as np
...@@ -25,50 +25,56 @@ class WaymoMetric(KittiMetric): ...@@ -25,50 +25,56 @@ class WaymoMetric(KittiMetric):
ann_file (str): The path of the annotation file in kitti format. ann_file (str): The path of the annotation file in kitti format.
waymo_bin_file (str): The path of the annotation file in waymo format. waymo_bin_file (str): The path of the annotation file in waymo format.
data_root (str): Path of dataset root. data_root (str): Path of dataset root.
Used for storing waymo evaluation programs. Used for storing waymo evaluation programs.
split (str): The split of the evaluation set. split (str): The split of the evaluation set. Defaults to 'training'.
metric (str | list[str]): Metrics to be evaluated. metric (str or List[str]): Metrics to be evaluated.
Default to 'mAP'. Defaults to 'mAP'.
pcd_limit_range (list): The range of point cloud used to pcd_limit_range (List[float]): The range of point cloud used to
filter invalid predicted boxes. filter invalid predicted boxes.
Default to [0, -40, -3, 70.4, 40, 0.0]. Defaults to [-85, -85, -5, 85, 85, 5].
convert_kitti_format (bool): Whether to convert the results to
kitti format. Now, in order to be compatible with camera-based
methods, defaults to True.
prefix (str, optional): The prefix that will be added in the metric prefix (str, optional): The prefix that will be added in the metric
names to disambiguate homonymous metrics of different evaluators. names to disambiguate homonymous metrics of different evaluators.
If prefix is not provided in the argument, self.default_prefix If prefix is not provided in the argument, self.default_prefix
will be used instead. Defaults to None. will be used instead. Defaults to None.
convert_kitti_format (bool, optional): Whether convert the reuslts to format_only (bool): Format the output results without perform
kitti format. Now, in order to be compatible with camera-based evaluation. It is useful when you want to format the result
methods, defaults to True. to a specific format and submit it to the test server.
Defaults to False.
pklfile_prefix (str, optional): The prefix of pkl files, including pklfile_prefix (str, optional): The prefix of pkl files, including
the file path and the prefix of filename, e.g., "a/b/prefix". the file path and the prefix of filename, e.g., "a/b/prefix".
If not specified, a temp file will be created. Default: None. If not specified, a temp file will be created. Defaults to None.
submission_prefix (str, optional): The prefix of submission data. submission_prefix (str, optional): The prefix of submission data.
If not specified, the submission data will not be generated. If not specified, the submission data will not be generated.
Default: None. Defaults to None.
load_type (str, optional): Type of loading mode during training. load_type (str): Type of loading mode during training.
- 'frame_based': Load all of the instances in the frame. - 'frame_based': Load all of the instances in the frame.
- 'mv_image_based': Load all of the instances in the frame and need - 'mv_image_based': Load all of the instances in the frame and need
to convert to the FOV-based data type to support image-based to convert to the FOV-based data type to support image-based
detector. detector.
- 'fov_image_base': Only load the instances inside the default cam, - 'fov_image_based': Only load the instances inside the default
and need to convert to the FOV-based data type to support cam, and need to convert to the FOV-based data type to support
image-based detector. image-based detector.
default_cam_key (str, optional): The default camera for lidar to default_cam_key (str): The default camera for lidar to camera
camear conversion. By default, KITTI: CAM2, Waymo: CAM_FRONT conversion. By default, KITTI: 'CAM2', Waymo: 'CAM_FRONT'.
use_pred_sample_idx (bool, optional): In formating results, use the Defaults to 'CAM_FRONT'.
sample index from the prediction or from the load annoataitons. use_pred_sample_idx (bool): In formating results, use the
sample index from the prediction or from the load annotations.
By default, KITTI: True, Waymo: False, Waymo has a conversion By default, KITTI: True, Waymo: False, Waymo has a conversion
process, which needs to use the sample id from load annotation. process, which needs to use the sample idx from load annotation.
Defaults to False.
collect_device (str): Device name used for collecting results collect_device (str): Device name used for collecting results
from different ranks during distributed training. Must be 'cpu' or from different ranks during distributed training. Must be 'cpu' or
'gpu'. Defaults to 'cpu'. 'gpu'. Defaults to 'cpu'.
file_client_args (dict): file client for reading gt in waymo format. file_client_args (dict): File client for reading gt in waymo format.
Defaults to ``dict(backend='disk')``. Defaults to ``dict(backend='disk')``.
idx2metainfo (Optional[str], optional): The file path of the metainfo idx2metainfo (str, optional): The file path of the metainfo in waymo.
in waymmo. It stores the mapping from sample_idx to metainfo. It stores the mapping from sample_idx to metainfo. The metainfo
The metainfo must contain the keys: 'idx2contextname' and must contain the keys: 'idx2contextname' and 'idx2timestamp'.
'idx2timestamp'. Defaults to None. Defaults to None.
""" """
num_cams = 5 num_cams = 5
...@@ -81,14 +87,15 @@ class WaymoMetric(KittiMetric): ...@@ -81,14 +87,15 @@ class WaymoMetric(KittiMetric):
pcd_limit_range: List[float] = [-85, -85, -5, 85, 85, 5], pcd_limit_range: List[float] = [-85, -85, -5, 85, 85, 5],
convert_kitti_format: bool = True, convert_kitti_format: bool = True,
prefix: Optional[str] = None, prefix: Optional[str] = None,
pklfile_prefix: str = None, format_only: bool = False,
submission_prefix: str = None, pklfile_prefix: Optional[str] = None,
submission_prefix: Optional[str] = None,
load_type: str = 'frame_based', load_type: str = 'frame_based',
default_cam_key: str = 'CAM_FRONT', default_cam_key: str = 'CAM_FRONT',
use_pred_sample_idx: bool = False, use_pred_sample_idx: bool = False,
collect_device: str = 'cpu', collect_device: str = 'cpu',
file_client_args: dict = dict(backend='disk'), file_client_args: dict = dict(backend='disk'),
idx2metainfo: Optional[str] = None): idx2metainfo: Optional[str] = None) -> None:
self.waymo_bin_file = waymo_bin_file self.waymo_bin_file = waymo_bin_file
self.data_root = data_root self.data_root = data_root
self.split = split self.split = split
...@@ -101,7 +108,7 @@ class WaymoMetric(KittiMetric): ...@@ -101,7 +108,7 @@ class WaymoMetric(KittiMetric):
else: else:
self.idx2metainfo = None self.idx2metainfo = None
super().__init__( super(WaymoMetric, self).__init__(
ann_file=ann_file, ann_file=ann_file,
metric=metric, metric=metric,
pcd_limit_range=pcd_limit_range, pcd_limit_range=pcd_limit_range,
...@@ -111,13 +118,20 @@ class WaymoMetric(KittiMetric): ...@@ -111,13 +118,20 @@ class WaymoMetric(KittiMetric):
default_cam_key=default_cam_key, default_cam_key=default_cam_key,
collect_device=collect_device, collect_device=collect_device,
file_client_args=file_client_args) file_client_args=file_client_args)
self.format_only = format_only
if self.format_only:
assert pklfile_prefix is not None, 'pklfile_prefix must be '
'not None when format_only is True, otherwise the result files '
'will be saved to a temp directory which will be cleaned up at '
'the end.'
self.default_prefix = 'Waymo metric' self.default_prefix = 'Waymo metric'
def compute_metrics(self, results: list) -> Dict[str, float]: def compute_metrics(self, results: List[dict]) -> Dict[str, float]:
"""Compute the metrics from processed results. """Compute the metrics from processed results.
Args: Args:
results (list): The processed results of the whole dataset. results (List[dict]): The processed results of the whole dataset.
Returns: Returns:
Dict[str, float]: The computed metrics. The keys are the names of Dict[str, float]: The computed metrics. The keys are the names of
...@@ -155,7 +169,7 @@ class WaymoMetric(KittiMetric): ...@@ -155,7 +169,7 @@ class WaymoMetric(KittiMetric):
if 'image_sweeps' in info: if 'image_sweeps' in info:
camera_info['image_sweeps'] = info['image_sweeps'] camera_info['image_sweeps'] = info['image_sweeps']
# TODO check if need to modify the sample id # TODO check if need to modify the sample idx
# TODO check when will use it except for evaluation. # TODO check when will use it except for evaluation.
camera_info['sample_idx'] = info['sample_idx'] camera_info['sample_idx'] = info['sample_idx']
new_data_infos.append(camera_info) new_data_infos.append(camera_info)
...@@ -175,6 +189,12 @@ class WaymoMetric(KittiMetric): ...@@ -175,6 +189,12 @@ class WaymoMetric(KittiMetric):
classes=self.classes) classes=self.classes)
metric_dict = {} metric_dict = {}
if self.format_only:
logger.info('results are saved in '
f'{osp.dirname(self.pklfile_prefix)}')
return metric_dict
for metric in self.metrics: for metric in self.metrics:
ap_dict = self.waymo_evaluate( ap_dict = self.waymo_evaluate(
pklfile_prefix, metric=metric, logger=logger) pklfile_prefix, metric=metric, logger=logger)
...@@ -188,19 +208,19 @@ class WaymoMetric(KittiMetric): ...@@ -188,19 +208,19 @@ class WaymoMetric(KittiMetric):
def waymo_evaluate(self, def waymo_evaluate(self,
pklfile_prefix: str, pklfile_prefix: str,
metric: str = None, metric: Optional[str] = None,
logger: MMLogger = None) -> dict: logger: Optional[MMLogger] = None) -> Dict[str, float]:
"""Evaluation in Waymo protocol. """Evaluation in Waymo protocol.
Args: Args:
pklfile_prefix (str): The location that stored the prediction pklfile_prefix (str): The location that stored the prediction
results. results.
metric (str): Metric to be evaluated. Defaults to None. metric (str, optional): Metric to be evaluated. Defaults to None.
logger (MMLogger, optional): Logger used for printing logger (MMLogger, optional): Logger used for printing
related information during evaluation. Default: None. related information during evaluation. Defaults to None.
Returns: Returns:
dict[str, float]: Results of each evaluation metric. Dict[str, float]: Results of each evaluation metric.
""" """
import subprocess import subprocess
...@@ -238,8 +258,6 @@ class WaymoMetric(KittiMetric): ...@@ -238,8 +258,6 @@ class WaymoMetric(KittiMetric):
} }
mAP_splits = ret_texts.split('mAP ') mAP_splits = ret_texts.split('mAP ')
mAPH_splits = ret_texts.split('mAPH ') mAPH_splits = ret_texts.split('mAPH ')
mAP_splits = ret_texts.split('mAP ')
mAPH_splits = ret_texts.split('mAPH ')
for idx, key in enumerate(ap_dict.keys()): for idx, key in enumerate(ap_dict.keys()):
split_idx = int(idx / 2) + 1 split_idx = int(idx / 2) + 1
if idx % 2 == 0: # mAP if idx % 2 == 0: # mAP
...@@ -307,31 +325,32 @@ class WaymoMetric(KittiMetric): ...@@ -307,31 +325,32 @@ class WaymoMetric(KittiMetric):
ap_dict['Cyclist mAPH']) / 3 ap_dict['Cyclist mAPH']) / 3
return ap_dict return ap_dict
def format_results(self, def format_results(
results: List[dict], self,
pklfile_prefix: str = None, results: List[dict],
submission_prefix: str = None, pklfile_prefix: Optional[str] = None,
classes: List[str] = None): submission_prefix: Optional[str] = None,
classes: Optional[List[str]] = None
) -> Tuple[dict, Union[tempfile.TemporaryDirectory, None]]:
"""Format the results to bin file. """Format the results to bin file.
Args: Args:
results (list[dict]): Testing results of the results (List[dict]): Testing results of the dataset.
dataset.
pklfile_prefix (str, optional): The prefix of pkl files. It pklfile_prefix (str, optional): The prefix of pkl files. It
includes the file path and the prefix of filename, e.g., includes the file path and the prefix of filename, e.g.,
"a/b/prefix". If not specified, a temp file will be created. "a/b/prefix". If not specified, a temp file will be created.
Default: None. Defaults to None.
submission_prefix (str, optional): The prefix of submitted files. submission_prefix (str, optional): The prefix of submitted files.
It includes the file path and the prefix of filename, e.g., It includes the file path and the prefix of filename, e.g.,
"a/b/prefix". If not specified, a temp file will be created. "a/b/prefix". If not specified, a temp file will be created.
Default: None. Defaults to None.
classes (list[String], optional): A list of class name. Defaults classes (List[str], optional): A list of class name.
to None. Defaults to None.
Returns: Returns:
tuple: (result_dict, tmp_dir), result_dict is a dict containing tuple: (result_dict, tmp_dir), result_dict is a dict containing
the formatted result, tmp_dir is the temporal directory created the formatted result, tmp_dir is the temporal directory created
for saving json files when jsonfile_prefix is not specified. for saving json files when jsonfile_prefix is not specified.
""" """
waymo_save_tmp_dir = tempfile.TemporaryDirectory() waymo_save_tmp_dir = tempfile.TemporaryDirectory()
waymo_results_save_dir = waymo_save_tmp_dir.name waymo_results_save_dir = waymo_save_tmp_dir.name
...@@ -378,15 +397,16 @@ class WaymoMetric(KittiMetric): ...@@ -378,15 +397,16 @@ class WaymoMetric(KittiMetric):
return final_results, waymo_save_tmp_dir return final_results, waymo_save_tmp_dir
def merge_multi_view_boxes(self, box_dict_per_frame: List[dict], def merge_multi_view_boxes(self, box_dict_per_frame: List[dict],
cam0_info: dict): cam0_info: dict) -> dict:
"""Merge bounding boxes predicted from multi-view images. """Merge bounding boxes predicted from multi-view images.
Args: Args:
box_dict_per_frame (list[dict]): The results of prediction box_dict_per_frame (List[dict]): The results of prediction
for each camera. for each camera.
cam2_info (dict): store the sample id for the given frame. cam0_info (dict): Store the sample idx for the given frame.
Returns: Returns:
merged_box_dict (dict), store the merge results dict: Merged results.
""" """
box_dict = dict() box_dict = dict()
# convert list[dict] to dict[list] # convert list[dict] to dict[list]
...@@ -444,27 +464,28 @@ class WaymoMetric(KittiMetric): ...@@ -444,27 +464,28 @@ class WaymoMetric(KittiMetric):
) )
return merged_box_dict return merged_box_dict
def bbox2result_kitti(self, def bbox2result_kitti(
net_outputs: list, self,
sample_id_list: list, net_outputs: List[dict],
class_names: list, sample_idx_list: List[int],
pklfile_prefix: str = None, class_names: List[str],
submission_prefix: str = None): pklfile_prefix: Optional[str] = None,
submission_prefix: Optional[str] = None) -> List[dict]:
"""Convert 3D detection results to kitti format for evaluation and test """Convert 3D detection results to kitti format for evaluation and test
submission. submission.
Args: Args:
net_outputs (list[dict]): List of array storing the net_outputs (List[dict]): List of dict storing the
inferenced bounding boxes and scores. inferenced bounding boxes and scores.
sample_id_list (list[int]): List of input sample id. sample_idx_list (List[int]): List of input sample idx.
class_names (list[String]): A list of class names. class_names (List[str]): A list of class names.
pklfile_prefix (str, optional): The prefix of pkl file. pklfile_prefix (str, optional): The prefix of pkl file.
Defaults to None. Defaults to None.
submission_prefix (str, optional): The prefix of submission file. submission_prefix (str, optional): The prefix of submission file.
Defaults to None. Defaults to None.
Returns: Returns:
list[dict]: A list of dictionaries with the kitti format. List[dict]: A list of dictionaries with the kitti format.
""" """
if submission_prefix is not None: if submission_prefix is not None:
mmengine.mkdir_or_exist(submission_prefix) mmengine.mkdir_or_exist(submission_prefix)
...@@ -473,8 +494,7 @@ class WaymoMetric(KittiMetric): ...@@ -473,8 +494,7 @@ class WaymoMetric(KittiMetric):
print('\nConverting prediction to KITTI format') print('\nConverting prediction to KITTI format')
for idx, pred_dicts in enumerate( for idx, pred_dicts in enumerate(
mmengine.track_iter_progress(net_outputs)): mmengine.track_iter_progress(net_outputs)):
annos = [] sample_idx = sample_idx_list[idx]
sample_idx = sample_id_list[idx]
info = self.data_infos[sample_idx] info = self.data_infos[sample_idx]
if self.load_type == 'mv_image_based': if self.load_type == 'mv_image_based':
...@@ -536,7 +556,6 @@ class WaymoMetric(KittiMetric): ...@@ -536,7 +556,6 @@ class WaymoMetric(KittiMetric):
anno['score'].append(score) anno['score'].append(score)
anno = {k: np.stack(v) for k, v in anno.items()} anno = {k: np.stack(v) for k, v in anno.items()}
annos.append(anno)
else: else:
anno = { anno = {
'name': np.array([]), 'name': np.array([]),
...@@ -549,7 +568,6 @@ class WaymoMetric(KittiMetric): ...@@ -549,7 +568,6 @@ class WaymoMetric(KittiMetric):
'rotation_y': np.array([]), 'rotation_y': np.array([]),
'score': np.array([]), 'score': np.array([]),
} }
annos.append(anno)
if submission_prefix is not None: if submission_prefix is not None:
curr_file = f'{submission_prefix}/{sample_idx:06d}.txt' curr_file = f'{submission_prefix}/{sample_idx:06d}.txt'
...@@ -577,10 +595,10 @@ class WaymoMetric(KittiMetric): ...@@ -577,10 +595,10 @@ class WaymoMetric(KittiMetric):
# In waymo validation sample_idx in prediction is 000xxx # In waymo validation sample_idx in prediction is 000xxx
# but in info file it is 1000xxx # but in info file it is 1000xxx
save_sample_idx = box_dict['sample_idx'] save_sample_idx = box_dict['sample_idx']
annos[-1]['sample_idx'] = np.array( anno['sample_idx'] = np.array(
[save_sample_idx] * len(annos[-1]['score']), dtype=np.int64) [save_sample_idx] * len(anno['score']), dtype=np.int64)
det_annos += annos det_annos.append(anno)
if pklfile_prefix is not None: if pklfile_prefix is not None:
if not pklfile_prefix.endswith(('.pkl', '.pickle')): if not pklfile_prefix.endswith(('.pkl', '.pickle')):
...@@ -592,16 +610,16 @@ class WaymoMetric(KittiMetric): ...@@ -592,16 +610,16 @@ class WaymoMetric(KittiMetric):
return det_annos return det_annos
def convert_valid_bboxes(self, box_dict: dict, info: dict): def convert_valid_bboxes(self, box_dict: dict, info: dict) -> dict:
"""Convert the predicted boxes into valid ones. Should handle the """Convert the predicted boxes into valid ones. Should handle the
load_model (frame_based, mv_image_based, fov_image_based), separately. load_model (frame_based, mv_image_based, fov_image_based), separately.
Args: Args:
box_dict (dict): Box dictionaries to be converted. box_dict (dict): Box dictionaries to be converted.
- bboxes_3d (:obj:`LiDARInstance3DBoxes`): 3D bounding boxes. - bboxes_3d (:obj:`BaseInstance3DBoxes`): 3D bounding boxes.
- scores_3d (torch.Tensor): Scores of boxes. - scores_3d (Tensor): Scores of boxes.
- labels_3d (torch.Tensor): Class labels of boxes. - labels_3d (Tensor): Class labels of boxes.
info (dict): Data info. info (dict): Data info.
Returns: Returns:
...@@ -609,9 +627,9 @@ class WaymoMetric(KittiMetric): ...@@ -609,9 +627,9 @@ class WaymoMetric(KittiMetric):
- bbox (np.ndarray): 2D bounding boxes. - bbox (np.ndarray): 2D bounding boxes.
- box3d_camera (np.ndarray): 3D bounding boxes in - box3d_camera (np.ndarray): 3D bounding boxes in
camera coordinate. camera coordinate.
- box3d_lidar (np.ndarray): 3D bounding boxes in - box3d_lidar (np.ndarray): 3D bounding boxes in
LiDAR coordinate. LiDAR coordinate.
- scores (np.ndarray): Scores of boxes. - scores (np.ndarray): Scores of boxes.
- label_preds (np.ndarray): Class label predictions. - label_preds (np.ndarray): Class label predictions.
- sample_idx (int): Sample index. - sample_idx (int): Sample index.
...@@ -673,7 +691,7 @@ class WaymoMetric(KittiMetric): ...@@ -673,7 +691,7 @@ class WaymoMetric(KittiMetric):
valid_pcd_inds = ((box_preds_lidar.center > limit_range[:3]) & valid_pcd_inds = ((box_preds_lidar.center > limit_range[:3]) &
(box_preds_lidar.center < limit_range[3:])) (box_preds_lidar.center < limit_range[3:]))
valid_inds = valid_pcd_inds.all(-1) valid_inds = valid_pcd_inds.all(-1)
if self.load_type in ['mv_image_based', 'fov_image_based']: elif self.load_type in ['mv_image_based', 'fov_image_based']:
valid_inds = valid_cam_inds valid_inds = valid_cam_inds
if valid_inds.sum() > 0: if valid_inds.sum() > 0:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment