Skip to content

GitLab

Commit 19472568 authored by 雍大凯's avatar 雍大凯
Browse files

将子模块转换为普通目录

parent 51e55208
Hide whitespace changes
Inline Side-by-side
Showing with 7297 additions and 0 deletions
docker-hub/MapTRv2/MapTR/projects/mmdet3d_plugin/datasets/map_utils/mean_ap.py 0 → 100644
View file @ 19472568
# Copyright (c) OpenMMLab. All rights reserved.
from multiprocessing import Pool
from shapely.geometry import LineString, Polygon
import mmcv
import numpy as np
from mmcv.utils import print_log
from terminaltables import AsciiTable
import json
from os import path as osp
import os
from functools import partial
from .tpfp import custom_tpfp_gen
def average_precision(recalls, precisions, mode='area'):
"""Calculate average precision (for single or multiple scales).
Args:
recalls (ndarray): shape (num_scales, num_dets) or (num_dets, )
precisions (ndarray): shape (num_scales, num_dets) or (num_dets, )
mode (str): 'area' or '11points', 'area' means calculating the area
under precision-recall curve, '11points' means calculating
the average precision of recalls at [0, 0.1, ..., 1]
Returns:
float or ndarray: calculated average precision
"""
no_scale = False
if recalls.ndim == 1:
no_scale = True
recalls = recalls[np.newaxis, :]
precisions = precisions[np.newaxis, :]
assert recalls.shape == precisions.shape and recalls.ndim == 2
num_scales = recalls.shape[0]
ap = np.zeros(num_scales, dtype=np.float32)
if mode == 'area':
zeros = np.zeros((num_scales, 1), dtype=recalls.dtype)
ones = np.ones((num_scales, 1), dtype=recalls.dtype)
mrec = np.hstack((zeros, recalls, ones))
mpre = np.hstack((zeros, precisions, zeros))
for i in range(mpre.shape[1] - 1, 0, -1):
mpre[:, i - 1] = np.maximum(mpre[:, i - 1], mpre[:, i])
for i in range(num_scales):
ind = np.where(mrec[i, 1:] != mrec[i, :-1])[0]
ap[i] = np.sum(
(mrec[i, ind + 1] - mrec[i, ind]) * mpre[i, ind + 1])
elif mode == '11points':
for i in range(num_scales):
for thr in np.arange(0, 1 + 1e-3, 0.1):
precs = precisions[i, recalls[i, :] >= thr]
prec = precs.max() if precs.size > 0 else 0
ap[i] += prec
ap /= 11
else:
raise ValueError(
'Unrecognized mode, only "area" and "11points" are supported')
if no_scale:
ap = ap[0]
return ap
def get_cls_results(gen_results,
annotations,
num_sample=100,
num_pred_pts_per_instance=30,
eval_use_same_gt_sample_num_flag=False,
class_id=0,
fix_interval=False,
code_size=2):
"""Get det results and gt information of a certain class.
Args:
gen_results (list[list]): Same as `eval_map()`.
annotations (list[dict]): Same as `eval_map()`.
class_id (int): ID of a specific class.
Returns:
tuple[list[np.ndarray]]: detected bboxes, gt bboxes
"""
# if len(gen_results) == 0 or
cls_gens, cls_scores = [], []
for res in gen_results['vectors']:
if res['type'] == class_id:
if len(res['pts']) < 2:
continue
if not eval_use_same_gt_sample_num_flag:
sampled_points = np.array(res['pts'])
else:
line = res['pts']
line = LineString(line)
if fix_interval:
distances = list(np.arange(1., line.length, 1.))
distances = [0,] + distances + [line.length,]
sampled_points = np.array([list(line.interpolate(distance).coords)
for distance in distances]).reshape(-1, code_size)
else:
line_pts = np.array(line.coords)
if line_pts.shape[0] == num_sample:
sampled_points = line_pts
else:
distances = np.linspace(0, line.length, num_sample)
sampled_points = np.array([list(line.interpolate(distance).coords)
for distance in distances]).reshape(-1, code_size)
cls_gens.append(sampled_points)
cls_scores.append(res['confidence_level'])
num_res = len(cls_gens)
if num_res > 0:
cls_gens = np.stack(cls_gens).reshape(num_res,-1)
cls_scores = np.array(cls_scores)[:,np.newaxis]
cls_gens = np.concatenate([cls_gens,cls_scores],axis=-1)
# print(f'for class {i}, cls_gens has shape {cls_gens.shape}')
else:
if not eval_use_same_gt_sample_num_flag:
cls_gens = np.zeros((0,num_pred_pts_per_instance*code_size+1))
else:
cls_gens = np.zeros((0,num_sample*code_size+1))
# print(f'for class {i}, cls_gens has shape {cls_gens.shape}')
cls_gts = []
for ann in annotations['vectors']:
if ann['type'] == class_id:
# line = ann['pts'] + np.array((1,1)) # for hdmapnet
line = ann['pts']
# line = ann['pts'].cumsum(0)
line = LineString(line)
distances = np.linspace(0, line.length, num_sample)
sampled_points = np.array([list(line.interpolate(distance).coords)
for distance in distances]).reshape(-1, code_size)
cls_gts.append(sampled_points)
num_gts = len(cls_gts)
if num_gts > 0:
cls_gts = np.stack(cls_gts).reshape(num_gts,-1)
else:
cls_gts = np.zeros((0,num_sample*code_size))
return cls_gens, cls_gts
# ones = np.ones((num_gts,1))
# tmp_cls_gens = np.concatenate([cls_gts,ones],axis=-1)
# return tmp_cls_gens, cls_gts
def format_res_gt_by_classes(result_path,
gen_results,
annotations,
cls_names=None,
num_pred_pts_per_instance=30,
eval_use_same_gt_sample_num_flag=False,
pc_range=[-15.0, -30.0, -5.0, 15.0, 30.0, 3.0],
code_size=2,
nproc=24):
assert cls_names is not None
timer = mmcv.Timer()
num_fixed_sample_pts = 100
fix_interval = False
print('results path: {}'.format(result_path))
output_dir = osp.join(*osp.split(result_path)[:-1])
assert len(gen_results) == len(annotations)
pool = Pool(nproc)
cls_gens, cls_gts = {}, {}
print('Formatting ...')
formatting_file = 'cls_formatted.pkl'
formatting_file = osp.join(output_dir,formatting_file)
# for vis
if False:
from PIL import Image
import matplotlib.pyplot as plt
from matplotlib import transforms
from matplotlib.patches import Rectangle
show_dir = osp.join(output_dir,'vis_json')
mmcv.mkdir_or_exist(osp.abspath(show_dir))
# import pdb;pdb.set_trace()
car_img = Image.open('./figs/lidar_car.png')
colors_plt = ['r', 'b', 'g']
for i in range(20):
plt.figure(figsize=(2, 4))
plt.xlim(pc_range[0], pc_range[3])
plt.ylim(pc_range[1], pc_range[4])
plt.axis('off')
for line in gen_results[i]['vectors']:
l = np.array(line['pts'])
plt.plot(l[:,0],l[:,1],'-',
# color=colors[line['type']]
color = 'red',
)
for line in annotations[i]['vectors']:
# l = np.array(line['pts']) + np.array((1,1))
l = np.array(line['pts'])
# l = line['pts']
plt.plot(l[:,0],l[:,1],'-',
# color=colors[line['type']],
color = 'blue',
)
plt.imshow(car_img, extent=[-1.2, 1.2, -1.5, 1.5])
map_path = osp.join(show_dir, 'COMPARE_MAP_{}.jpg'.format(i))
plt.savefig(map_path, bbox_inches='tight', dpi=400)
plt.close()
for i, clsname in enumerate(cls_names):
gengts = pool.starmap(
partial(get_cls_results, num_sample=num_fixed_sample_pts,
num_pred_pts_per_instance=num_pred_pts_per_instance,
eval_use_same_gt_sample_num_flag=eval_use_same_gt_sample_num_flag,class_id=i,fix_interval=fix_interval,
code_size=code_size),
zip(gen_results, annotations))
# gengts = map(partial(get_cls_results, num_sample=num_fixed_sample_pts, class_id=i,fix_interval=fix_interval),
# zip(gen_results, annotations))
# import pdb;pdb.set_trace()
gens, gts = tuple(zip(*gengts))
cls_gens[clsname] = gens
cls_gts[clsname] = gts
mmcv.dump([cls_gens, cls_gts],formatting_file)
print('Cls data formatting done in {:2f}s!! with {}'.format(float(timer.since_start()),formatting_file))
pool.close()
return cls_gens, cls_gts
def eval_map(gen_results,
annotations,
cls_gens,
cls_gts,
threshold=0.5,
cls_names=None,
logger=None,
tpfp_fn=None,
pc_range=[-15.0, -30.0, -5.0, 15.0, 30.0, 3.0],
metric=None,
num_pred_pts_per_instance=30,
code_size=2,
nproc=24):
timer = mmcv.Timer()
pool = Pool(nproc)
eval_results = []
for i, clsname in enumerate(cls_names):
# get gt and det bboxes of this class
cls_gen = cls_gens[clsname]
cls_gt = cls_gts[clsname]
# choose proper function according to datasets to compute tp and fp
# XXX
# func_name = cls2func[clsname]
# tpfp_fn = tpfp_fn_dict[tpfp_fn_name]
tpfp_fn = custom_tpfp_gen
# Trick for serialized
# only top-level function can be serized
# somehow use partitial the return function is defined
# at the top level.
# tpfp = tpfp_fn(cls_gen[i], cls_gt[i],threshold=threshold, metric=metric)
# import pdb; pdb.set_trace()
# TODO this is a hack
tpfp_fn = partial(tpfp_fn, threshold=threshold, metric=metric, code_size=code_size)
args = []
# compute tp and fp for each image with multiple processes
tpfp = pool.starmap(
tpfp_fn,
zip(cls_gen, cls_gt, *args))
# import pdb;pdb.set_trace()
tp, fp = tuple(zip(*tpfp))
# map_results = map(
# tpfp_fn,
# cls_gen, cls_gt)
# tp, fp = tuple(map(list, zip(*map_results)))
# debug and testing
# for i in range(len(cls_gen)):
# # print(i)
# tpfp = tpfp_fn(cls_gen[i], cls_gt[i],threshold=threshold)
# print(i)
# tpfp = (tpfp,)
# print(tpfp)
# i = 0
# tpfp = tpfp_fn(cls_gen[i], cls_gt[i],threshold=threshold)
# import pdb; pdb.set_trace()
# XXX
num_gts = 0
for j, bbox in enumerate(cls_gt):
num_gts += bbox.shape[0]
# sort all det bboxes by score, also sort tp and fp
# import pdb;pdb.set_trace()
cls_gen = np.vstack(cls_gen)
num_dets = cls_gen.shape[0]
sort_inds = np.argsort(-cls_gen[:, -1]) #descending, high score front
tp = np.hstack(tp)[sort_inds]
fp = np.hstack(fp)[sort_inds]
# calculate recall and precision with tp and fp
# num_det*num_res
tp = np.cumsum(tp, axis=0)
fp = np.cumsum(fp, axis=0)
eps = np.finfo(np.float32).eps
recalls = tp / np.maximum(num_gts, eps)
precisions = tp / np.maximum((tp + fp), eps)
# calculate AP
# if dataset != 'voc07' else '11points'
mode = 'area'
ap = average_precision(recalls, precisions, mode)
eval_results.append({
'num_gts': num_gts,
'num_dets': num_dets,
'recall': recalls,
'precision': precisions,
'ap': ap
})
print('cls:{} done in {:2f}s!!'.format(clsname,float(timer.since_last_check())))
pool.close()
aps = []
for cls_result in eval_results:
if cls_result['num_gts'] > 0:
aps.append(cls_result['ap'])
mean_ap = np.array(aps).mean().item() if len(aps) else 0.0
print_map_summary(
mean_ap, eval_results, class_name=cls_names, logger=logger)
return mean_ap, eval_results
def print_map_summary(mean_ap,
results,
class_name=None,
scale_ranges=None,
logger=None):
"""Print mAP and results of each class.
A table will be printed to show the gts/dets/recall/AP of each class and
the mAP.
Args:
mean_ap (float): Calculated from `eval_map()`.
results (list[dict]): Calculated from `eval_map()`.
dataset (list[str] | str | None): Dataset name or dataset classes.
scale_ranges (list[tuple] | None): Range of scales to be evaluated.
logger (logging.Logger | str | None): The way to print the mAP
summary. See `mmcv.utils.print_log()` for details. Default: None.
"""
if logger == 'silent':
return
if isinstance(results[0]['ap'], np.ndarray):
num_scales = len(results[0]['ap'])
else:
num_scales = 1
if scale_ranges is not None:
assert len(scale_ranges) == num_scales
num_classes = len(results)
recalls = np.zeros((num_scales, num_classes), dtype=np.float32)
aps = np.zeros((num_scales, num_classes), dtype=np.float32)
num_gts = np.zeros((num_scales, num_classes), dtype=int)
for i, cls_result in enumerate(results):
if cls_result['recall'].size > 0:
recalls[:, i] = np.array(cls_result['recall'], ndmin=2)[:, -1]
aps[:, i] = cls_result['ap']
num_gts[:, i] = cls_result['num_gts']
label_names = class_name
if not isinstance(mean_ap, list):
mean_ap = [mean_ap]
header = ['class', 'gts', 'dets', 'recall', 'ap']
for i in range(num_scales):
if scale_ranges is not None:
print_log(f'Scale range {scale_ranges[i]}', logger=logger)
table_data = [header]
for j in range(num_classes):
row_data = [
label_names[j], num_gts[i, j], results[j]['num_dets'],
f'{recalls[i, j]:.3f}', f'{aps[i, j]:.3f}'
]
table_data.append(row_data)
table_data.append(['mAP', '', '', '', f'{mean_ap[i]:.3f}'])
table = AsciiTable(table_data)
table.inner_footing_row_border = True
print_log('\n' + table.table, logger=logger)
\ No newline at end of file
docker-hub/MapTRv2/MapTR/projects/mmdet3d_plugin/datasets/map_utils/tpfp.py 0 → 100644
View file @ 19472568
import mmcv
import numpy as np
from mmdet.core.evaluation.bbox_overlaps import bbox_overlaps
from .tpfp_chamfer import custom_polyline_score
from shapely.geometry import LineString, Polygon
def custom_tpfp_gen(gen_lines,
gt_lines,
threshold=0.5,
metric='chamfer',
code_size=2):
"""Check if detected bboxes are true positive or false positive.
Args:
det_bbox (ndarray): Detected bboxes of this image, of shape (m, 5).
gt_bboxes (ndarray): GT bboxes of this image, of shape (n, 4).
gt_bboxes_ignore (ndarray): Ignored gt bboxes of this image,
of shape (k, 4). Default: None
iou_thr (float): IoU threshold to be considered as matched.
Default: 0.5.
use_legacy_coordinate (bool): Whether to use coordinate system in
mmdet v1.x. which means width, height should be
calculated as 'x2 - x1 + 1` and 'y2 - y1 + 1' respectively.
Default: False.
Returns:
tuple[np.ndarray]: (tp, fp) whose elements are 0 and 1. The shape of
each array is (num_scales, m).
"""
if metric == 'chamfer':
if threshold >0:
threshold= -threshold
# else:
# raise NotImplementedError
# import pdb;pdb.set_trace()
num_gens = gen_lines.shape[0]
num_gts = gt_lines.shape[0]
# tp and fp
tp = np.zeros((num_gens), dtype=np.float32)
fp = np.zeros((num_gens), dtype=np.float32)
# if there is no gt bboxes in this image, then all det bboxes
# within area range are false positives
if num_gts == 0:
fp[...] = 1
return tp, fp
if num_gens == 0:
return tp, fp
gen_scores = gen_lines[:,-1] # n
# distance matrix: n x m
matrix = custom_polyline_score(
gen_lines[:,:-1].reshape(num_gens,-1,code_size),
gt_lines.reshape(num_gts,-1,code_size),linewidth=2.,metric=metric)
# for each det, the max iou with all gts
matrix_max = matrix.max(axis=1)
# for each det, which gt overlaps most with it
matrix_argmax = matrix.argmax(axis=1)
# sort all dets in descending order by scores
sort_inds = np.argsort(-gen_scores)
gt_covered = np.zeros(num_gts, dtype=bool)
# tp = 0 and fp = 0 means ignore this detected bbox,
for i in sort_inds:
if matrix_max[i] >= threshold:
matched_gt = matrix_argmax[i]
if not gt_covered[matched_gt]:
gt_covered[matched_gt] = True
tp[i] = 1
else:
fp[i] = 1
else:
fp[i] = 1
return tp, fp
docker-hub/MapTRv2/MapTR/projects/mmdet3d_plugin/datasets/map_utils/tpfp_chamfer.py 0 → 100644
View file @ 19472568
# from ..chamfer_dist import ChamferDistance
import numpy as np
from shapely.geometry import LineString, Polygon
from shapely.strtree import STRtree
from shapely.geometry import CAP_STYLE, JOIN_STYLE
from scipy.spatial import distance
def custom_polyline_score(pred_lines, gt_lines, linewidth=1., metric='chamfer'):
'''
each line with 1 meter width
pred_lines: num_preds, List [npts, 2]
gt_lines: num_gts, npts, 2
gt_mask: num_gts, npts, 2
'''
if metric == 'iou':
linewidth = 1.0
positive_threshold = 1.
num_preds = len(pred_lines)
num_gts = len(gt_lines)
line_length = pred_lines.shape[1]
# gt_lines = gt_lines + np.array((1.,1.))
pred_lines_shapely = \
[LineString(i).buffer(linewidth,
cap_style=CAP_STYLE.flat, join_style=JOIN_STYLE.mitre)
for i in pred_lines]
gt_lines_shapely =\
[LineString(i).buffer(linewidth,
cap_style=CAP_STYLE.flat, join_style=JOIN_STYLE.mitre)
for i in gt_lines]
# construct tree
tree = STRtree(pred_lines_shapely)
index_by_id = dict((id(pt), i) for i, pt in enumerate(pred_lines_shapely))
if metric=='chamfer':
iou_matrix = np.full((num_preds, num_gts), -100.)
elif metric=='iou':
iou_matrix = np.zeros((num_preds, num_gts),dtype=np.float64)
else:
raise NotImplementedError
for i, pline in enumerate(gt_lines_shapely):
for o in tree.query(pline):
if o.intersects(pline):
pred_id = index_by_id[id(o)]
if metric=='chamfer':
dist_mat = distance.cdist(
pred_lines[pred_id], gt_lines[i], 'euclidean')
# import pdb;pdb.set_trace()
valid_ab = dist_mat.min(-1).mean()
valid_ba = dist_mat.min(-2).mean()
iou_matrix[pred_id, i] = -(valid_ba+valid_ab)/2
elif metric=='iou':
inter = o.intersection(pline).area
union = o.union(pline).area
iou_matrix[pred_id, i] = inter / union
return iou_matrix
if __name__ == '__main__':
import torch
line1 = torch.tensor([
[1, 5,-1], [3, 5,-1], [5, 5,-1]
])
line0 = torch.tensor([
[3, 6,-1], [4, 8,-2], [5, 6,-1]
])
line2 = torch.tensor([
[1, 4,-1], [3, 4,-1], [5, 4,-1]
])
line3 = torch.tensor([
[4, 4,-1], [3, 3,-2], [5, 3,-3]
])
gt = torch.stack((line2, line3), dim=0).type(torch.float32)
pred = torch.stack((line0, line1), dim=0).type(torch.float32)
import ipdb; ipdb.set_trace()
# import mmcv
# with mmcv.Timer():
# gt = upsampler(gt, pts=10)
# pred = upsampler(pred, pts=10)
import matplotlib.pyplot as plt
from shapely.geometry import LineString
from descartes import PolygonPatch
iou_matrix = vec_iou(pred,gt)
print(iou_matrix)
# import pdb;pdb.set_trace()
score_matrix = custom_polyline_score(pred, gt, linewidth=1., metric='chamfer')
print(score_matrix)
fig, ax = plt.subplots()
for i in gt:
i = i.numpy()
plt.plot(i[:, 0], i[:, 1], 'o', color='red')
plt.plot(i[:, 0], i[:, 1], '-', color='red')
dilated = LineString(i).buffer(1, cap_style=CAP_STYLE.round, join_style=JOIN_STYLE.round)
patch1 = PolygonPatch(dilated, fc='red', ec='red', alpha=0.5, zorder=-1)
ax.add_patch(patch1)
for i in pred:
i = i.numpy()
plt.plot(i[:, 0], i[:, 1], 'o', color='blue')
plt.plot(i[:, 0], i[:, 1], '-', color='blue')
dilated = LineString(i).buffer(1, cap_style=CAP_STYLE.flat, join_style=JOIN_STYLE.mitre)
patch1 = PolygonPatch(dilated, fc='blue', ec='blue', alpha=0.5, zorder=-1)
ax.add_patch(patch1)
ax.axis('equal')
plt.savefig('test3.png')
\ No newline at end of file
docker-hub/MapTRv2/MapTR/projects/mmdet3d_plugin/datasets/nuscenes_dataset.py 0 → 100644
View file @ 19472568
import copy
import numpy as np
from mmdet.datasets import DATASETS
from mmdet3d.datasets import NuScenesDataset
import mmcv
from os import path as osp
from mmdet.datasets import DATASETS
import torch
import numpy as np
from nuscenes.eval.common.utils import quaternion_yaw, Quaternion
from .nuscnes_eval import NuScenesEval_custom
from projects.mmdet3d_plugin.models.utils.visual import save_tensor
from mmcv.parallel import DataContainer as DC
import random
@DATASETS.register_module()
class CustomNuScenesDataset(NuScenesDataset):
r"""NuScenes Dataset.
This datset only add camera intrinsics and extrinsics to the results.
"""
def __init__(self, queue_length=4, bev_size=(200, 200), overlap_test=False, *args, **kwargs):
super().__init__(*args, **kwargs)
self.queue_length = queue_length
self.overlap_test = overlap_test
self.bev_size = bev_size
def prepare_train_data(self, index):
"""
Training data preparation.
Args:
index (int): Index for accessing the target data.
Returns:
dict: Training data dict of the corresponding index.
"""
data_queue = []
# temporal aug
prev_indexs_list = list(range(index-self.queue_length, index))
random.shuffle(prev_indexs_list)
prev_indexs_list = sorted(prev_indexs_list[1:], reverse=True)
##
input_dict = self.get_data_info(index)
if input_dict is None:
return None
frame_idx = input_dict['frame_idx']
scene_token = input_dict['scene_token']
self.pre_pipeline(input_dict)
example = self.pipeline(input_dict)
if self.filter_empty_gt and \
(example is None or ~(example['gt_labels_3d']._data != -1).any()):
return None
data_queue.insert(0, example)
for i in prev_indexs_list:
i = max(0, i)
input_dict = self.get_data_info(i)
if input_dict is None:
return None
if input_dict['frame_idx'] < frame_idx and input_dict['scene_token'] == scene_token:
self.pre_pipeline(input_dict)
example = self.pipeline(input_dict)
if self.filter_empty_gt and \
(example is None or ~(example['gt_labels_3d']._data != -1).any()):
return None
frame_idx = input_dict['frame_idx']
data_queue.insert(0, copy.deepcopy(example))
return self.union2one(data_queue)
def union2one(self, queue):
"""
convert sample queue into one single sample.
"""
imgs_list = [each['img'].data for each in queue]
metas_map = {}
prev_pos = None
prev_angle = None
for i, each in enumerate(queue):
metas_map[i] = each['img_metas'].data
if i == 0:
metas_map[i]['prev_bev'] = False
prev_pos = copy.deepcopy(metas_map[i]['can_bus'][:3])
prev_angle = copy.deepcopy(metas_map[i]['can_bus'][-1])
metas_map[i]['can_bus'][:3] = 0
metas_map[i]['can_bus'][-1] = 0
else:
metas_map[i]['prev_bev'] = True
tmp_pos = copy.deepcopy(metas_map[i]['can_bus'][:3])
tmp_angle = copy.deepcopy(metas_map[i]['can_bus'][-1])
metas_map[i]['can_bus'][:3] -= prev_pos
metas_map[i]['can_bus'][-1] -= prev_angle
prev_pos = copy.deepcopy(tmp_pos)
prev_angle = copy.deepcopy(tmp_angle)
queue[-1]['img'] = DC(torch.stack(imgs_list),
cpu_only=False, stack=True)
queue[-1]['img_metas'] = DC(metas_map, cpu_only=True)
queue = queue[-1]
return queue
def get_data_info(self, index):
"""Get data info according to the given index.
Args:
index (int): Index of the sample data to get.
Returns:
dict: Data information that will be passed to the data \
preprocessing pipelines. It includes the following keys:
- sample_idx (str): Sample index.
- pts_filename (str): Filename of point clouds.
- sweeps (list[dict]): Infos of sweeps.
- timestamp (float): Sample timestamp.
- img_filename (str, optional): Image filename.
- lidar2img (list[np.ndarray], optional): Transformations \
from lidar to different cameras.
- ann_info (dict): Annotation info.
"""
info = self.data_infos[index]
# standard protocal modified from SECOND.Pytorch
input_dict = dict(
sample_idx=info['token'],
pts_filename=info['lidar_path'],
sweeps=info['sweeps'],
ego2global_translation=info['ego2global_translation'],
ego2global_rotation=info['ego2global_rotation'],
prev_idx=info['prev'],
next_idx=info['next'],
scene_token=info['scene_token'],
can_bus=info['can_bus'],
frame_idx=info['frame_idx'],
timestamp=info['timestamp'] / 1e6,
)
if self.modality['use_camera']:
image_paths = []
lidar2img_rts = []
lidar2cam_rts = []
cam_intrinsics = []
for cam_type, cam_info in info['cams'].items():
image_paths.append(cam_info['data_path'])
# obtain lidar to image transformation matrix
lidar2cam_r = np.linalg.inv(cam_info['sensor2lidar_rotation'])
lidar2cam_t = cam_info[
'sensor2lidar_translation'] @ lidar2cam_r.T
lidar2cam_rt = np.eye(4)
lidar2cam_rt[:3, :3] = lidar2cam_r.T
lidar2cam_rt[3, :3] = -lidar2cam_t
intrinsic = cam_info['cam_intrinsic']
viewpad = np.eye(4)
viewpad[:intrinsic.shape[0], :intrinsic.shape[1]] = intrinsic
lidar2img_rt = (viewpad @ lidar2cam_rt.T)
lidar2img_rts.append(lidar2img_rt)
cam_intrinsics.append(viewpad)
lidar2cam_rts.append(lidar2cam_rt.T)
input_dict.update(
dict(
img_filename=image_paths,
lidar2img=lidar2img_rts,
cam_intrinsic=cam_intrinsics,
lidar2cam=lidar2cam_rts,
))
if not self.test_mode:
annos = self.get_ann_info(index)
input_dict['ann_info'] = annos
rotation = Quaternion(input_dict['ego2global_rotation'])
translation = input_dict['ego2global_translation']
can_bus = input_dict['can_bus']
can_bus[:3] = translation
can_bus[3:7] = rotation
patch_angle = quaternion_yaw(rotation) / np.pi * 180
if patch_angle < 0:
patch_angle += 360
can_bus[-2] = patch_angle / 180 * np.pi
can_bus[-1] = patch_angle
return input_dict
def __getitem__(self, idx):
"""Get item from infos according to the given index.
Returns:
dict: Data dictionary of the corresponding index.
"""
if self.test_mode:
return self.prepare_test_data(idx)
while True:
data = self.prepare_train_data(idx)
if data is None:
idx = self._rand_another(idx)
continue
return data
def _evaluate_single(self,
result_path,
logger=None,
metric='bbox',
result_name='pts_bbox'):
"""Evaluation for a single model in nuScenes protocol.
Args:
result_path (str): Path of the result file.
logger (logging.Logger | str | None): Logger used for printing
related information during evaluation. Default: None.
metric (str): Metric name used for evaluation. Default: 'bbox'.
result_name (str): Result name in the metric prefix.
Default: 'pts_bbox'.
Returns:
dict: Dictionary of evaluation details.
"""
from nuscenes import NuScenes
self.nusc = NuScenes(version=self.version, dataroot=self.data_root,
verbose=True)
output_dir = osp.join(*osp.split(result_path)[:-1])
eval_set_map = {
'v1.0-mini': 'mini_val',
'v1.0-trainval': 'val',
}
self.nusc_eval = NuScenesEval_custom(
self.nusc,
config=self.eval_detection_configs,
result_path=result_path,
eval_set=eval_set_map[self.version],
output_dir=output_dir,
verbose=True,
overlap_test=self.overlap_test,
data_infos=self.data_infos
)
self.nusc_eval.main(plot_examples=0, render_curves=False)
# record metrics
metrics = mmcv.load(osp.join(output_dir, 'metrics_summary.json'))
detail = dict()
metric_prefix = f'{result_name}_NuScenes'
for name in self.CLASSES:
for k, v in metrics['label_aps'][name].items():
val = float('{:.4f}'.format(v))
detail['{}/{}_AP_dist_{}'.format(metric_prefix, name, k)] = val
for k, v in metrics['label_tp_errors'][name].items():
val = float('{:.4f}'.format(v))
detail['{}/{}_{}'.format(metric_prefix, name, k)] = val
for k, v in metrics['tp_errors'].items():
val = float('{:.4f}'.format(v))
detail['{}/{}'.format(metric_prefix,
self.ErrNameMapping[k])] = val
detail['{}/NDS'.format(metric_prefix)] = metrics['nd_score']
detail['{}/mAP'.format(metric_prefix)] = metrics['mean_ap']
return detail
docker-hub/MapTRv2/MapTR/projects/mmdet3d_plugin/datasets/nuscenes_map_dataset.py 0 → 100644
View file @ 19472568
import copy
import numpy as np
from mmdet.datasets import DATASETS
from mmdet3d.datasets import NuScenesDataset
import mmcv
import os
from os import path as osp
from mmdet.datasets import DATASETS
import torch
import numpy as np
from nuscenes.eval.common.utils import quaternion_yaw, Quaternion
from .nuscnes_eval import NuScenesEval_custom
from projects.mmdet3d_plugin.models.utils.visual import save_tensor
from mmcv.parallel import DataContainer as DC
import random
from .nuscenes_dataset import CustomNuScenesDataset
from nuscenes.map_expansion.map_api import NuScenesMap, NuScenesMapExplorer
from nuscenes.eval.common.utils import quaternion_yaw, Quaternion
from shapely import affinity, ops
from shapely.geometry import LineString, box, MultiPolygon, MultiLineString
from mmdet.datasets.pipelines import to_tensor
import json
def add_rotation_noise(extrinsics, std=0.01, mean=0.0):
#n = extrinsics.shape[0]
noise_angle = torch.normal(mean, std=std, size=(3,))
# extrinsics[:, 0:3, 0:3] *= (1 + noise)
sin_noise = torch.sin(noise_angle)
cos_noise = torch.cos(noise_angle)
rotation_matrix = torch.eye(4).view(4, 4)
# rotation_matrix[]
rotation_matrix_x = rotation_matrix.clone()
rotation_matrix_x[1, 1] = cos_noise[0]
rotation_matrix_x[1, 2] = sin_noise[0]
rotation_matrix_x[2, 1] = -sin_noise[0]
rotation_matrix_x[2, 2] = cos_noise[0]
rotation_matrix_y = rotation_matrix.clone()
rotation_matrix_y[0, 0] = cos_noise[1]
rotation_matrix_y[0, 2] = -sin_noise[1]
rotation_matrix_y[2, 0] = sin_noise[1]
rotation_matrix_y[2, 2] = cos_noise[1]
rotation_matrix_z = rotation_matrix.clone()
rotation_matrix_z[0, 0] = cos_noise[2]
rotation_matrix_z[0, 1] = sin_noise[2]
rotation_matrix_z[1, 0] = -sin_noise[2]
rotation_matrix_z[1, 1] = cos_noise[2]
rotation_matrix = rotation_matrix_x @ rotation_matrix_y @ rotation_matrix_z
rotation = torch.from_numpy(extrinsics.astype(np.float32))
rotation[:3, -1] = 0.0
# import pdb;pdb.set_trace()
rotation = rotation_matrix @ rotation
extrinsics[:3, :3] = rotation[:3, :3].numpy()
return extrinsics
def add_translation_noise(extrinsics, std=0.01, mean=0.0):
# n = extrinsics.shape[0]
noise = torch.normal(mean, std=std, size=(3,))
extrinsics[0:3, -1] += noise.numpy()
return extrinsics
class LiDARInstanceLines(object):
"""Line instance in LIDAR coordinates
"""
def __init__(self,
instance_line_list,
sample_dist=1,
num_samples=250,
padding=False,
fixed_num=-1,
padding_value=-10000,
patch_size=None):
assert isinstance(instance_line_list, list)
assert patch_size is not None
if len(instance_line_list) != 0:
assert isinstance(instance_line_list[0], LineString)
self.patch_size = patch_size
self.max_x = self.patch_size[1] / 2
self.max_y = self.patch_size[0] / 2
self.sample_dist = sample_dist
self.num_samples = num_samples
self.padding = padding
self.fixed_num = fixed_num
self.padding_value = padding_value
self.instance_list = instance_line_list
@property
def start_end_points(self):
"""
return torch.Tensor([N,4]), in xstart, ystart, xend, yend form
"""
assert len(self.instance_list) != 0
instance_se_points_list = []
for instance in self.instance_list:
se_points = []
se_points.extend(instance.coords[0])
se_points.extend(instance.coords[-1])
instance_se_points_list.append(se_points)
instance_se_points_array = np.array(instance_se_points_list)
instance_se_points_tensor = to_tensor(instance_se_points_array)
instance_se_points_tensor = instance_se_points_tensor.to(
dtype=torch.float32)
instance_se_points_tensor[:,0] = torch.clamp(instance_se_points_tensor[:,0], min=-self.max_x,max=self.max_x)
instance_se_points_tensor[:,1] = torch.clamp(instance_se_points_tensor[:,1], min=-self.max_y,max=self.max_y)
instance_se_points_tensor[:,2] = torch.clamp(instance_se_points_tensor[:,2], min=-self.max_x,max=self.max_x)
instance_se_points_tensor[:,3] = torch.clamp(instance_se_points_tensor[:,3], min=-self.max_y,max=self.max_y)
return instance_se_points_tensor
@property
def bbox(self):
"""
return torch.Tensor([N,4]), in xmin, ymin, xmax, ymax form
"""
assert len(self.instance_list) != 0
instance_bbox_list = []
for instance in self.instance_list:
# bounds is bbox: [xmin, ymin, xmax, ymax]
instance_bbox_list.append(instance.bounds)
instance_bbox_array = np.array(instance_bbox_list)
instance_bbox_tensor = to_tensor(instance_bbox_array)
instance_bbox_tensor = instance_bbox_tensor.to(
dtype=torch.float32)
instance_bbox_tensor[:,0] = torch.clamp(instance_bbox_tensor[:,0], min=-self.max_x,max=self.max_x)
instance_bbox_tensor[:,1] = torch.clamp(instance_bbox_tensor[:,1], min=-self.max_y,max=self.max_y)
instance_bbox_tensor[:,2] = torch.clamp(instance_bbox_tensor[:,2], min=-self.max_x,max=self.max_x)
instance_bbox_tensor[:,3] = torch.clamp(instance_bbox_tensor[:,3], min=-self.max_y,max=self.max_y)
return instance_bbox_tensor
@property
def fixed_num_sampled_points(self):
"""
return torch.Tensor([N,fixed_num,2]), in xmin, ymin, xmax, ymax form
N means the num of instances
"""
assert len(self.instance_list) != 0
instance_points_list = []
for instance in self.instance_list:
distances = np.linspace(0, instance.length, self.fixed_num)
sampled_points = np.array([list(instance.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
instance_points_list.append(sampled_points)
instance_points_array = np.array(instance_points_list)
instance_points_tensor = to_tensor(instance_points_array)
instance_points_tensor = instance_points_tensor.to(
dtype=torch.float32)
instance_points_tensor[:,:,0] = torch.clamp(instance_points_tensor[:,:,0], min=-self.max_x,max=self.max_x)
instance_points_tensor[:,:,1] = torch.clamp(instance_points_tensor[:,:,1], min=-self.max_y,max=self.max_y)
return instance_points_tensor
@property
def fixed_num_sampled_points_ambiguity(self):
"""
return torch.Tensor([N,fixed_num,2]), in xmin, ymin, xmax, ymax form
N means the num of instances
"""
assert len(self.instance_list) != 0
instance_points_list = []
for instance in self.instance_list:
distances = np.linspace(0, instance.length, self.fixed_num)
sampled_points = np.array([list(instance.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
instance_points_list.append(sampled_points)
instance_points_array = np.array(instance_points_list)
instance_points_tensor = to_tensor(instance_points_array)
instance_points_tensor = instance_points_tensor.to(
dtype=torch.float32)
instance_points_tensor[:,:,0] = torch.clamp(instance_points_tensor[:,:,0], min=-self.max_x,max=self.max_x)
instance_points_tensor[:,:,1] = torch.clamp(instance_points_tensor[:,:,1], min=-self.max_y,max=self.max_y)
instance_points_tensor = instance_points_tensor.unsqueeze(1)
return instance_points_tensor
@property
def fixed_num_sampled_points_torch(self):
"""
return torch.Tensor([N,fixed_num,2]), in xmin, ymin, xmax, ymax form
N means the num of instances
"""
assert len(self.instance_list) != 0
instance_points_list = []
for instance in self.instance_list:
# distances = np.linspace(0, instance.length, self.fixed_num)
# sampled_points = np.array([list(instance.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
poly_pts = to_tensor(np.array(list(instance.coords)))
poly_pts = poly_pts.unsqueeze(0).permute(0,2,1)
sampled_pts = torch.nn.functional.interpolate(poly_pts,size=(self.fixed_num),mode='linear',align_corners=True)
sampled_pts = sampled_pts.permute(0,2,1).squeeze(0)
instance_points_list.append(sampled_pts)
# instance_points_array = np.array(instance_points_list)
# instance_points_tensor = to_tensor(instance_points_array)
instance_points_tensor = torch.stack(instance_points_list,dim=0)
instance_points_tensor = instance_points_tensor.to(
dtype=torch.float32)
instance_points_tensor[:,:,0] = torch.clamp(instance_points_tensor[:,:,0], min=-self.max_x,max=self.max_x)
instance_points_tensor[:,:,1] = torch.clamp(instance_points_tensor[:,:,1], min=-self.max_y,max=self.max_y)
return instance_points_tensor
@property
def shift_fixed_num_sampled_points(self):
"""
return [instances_num, num_shifts, fixed_num, 2]
"""
fixed_num_sampled_points = self.fixed_num_sampled_points
instances_list = []
is_poly = False
# is_line = False
# import pdb;pdb.set_trace()
for fixed_num_pts in fixed_num_sampled_points:
# [fixed_num, 2]
is_poly = fixed_num_pts[0].equal(fixed_num_pts[-1])
fixed_num = fixed_num_pts.shape[0]
shift_pts_list = []
if is_poly:
# import pdb;pdb.set_trace()
for shift_right_i in range(fixed_num):
shift_pts_list.append(fixed_num_pts.roll(shift_right_i,0))
else:
shift_pts_list.append(fixed_num_pts)
shift_pts_list.append(fixed_num_pts.flip(0))
shift_pts = torch.stack(shift_pts_list,dim=0)
shift_pts[:,:,0] = torch.clamp(shift_pts[:,:,0], min=-self.max_x,max=self.max_x)
shift_pts[:,:,1] = torch.clamp(shift_pts[:,:,1], min=-self.max_y,max=self.max_y)
if not is_poly:
padding = torch.full([fixed_num-shift_pts.shape[0],fixed_num,2], self.padding_value)
shift_pts = torch.cat([shift_pts,padding],dim=0)
# padding = np.zeros((self.num_samples - len(sampled_points), 2))
# sampled_points = np.concatenate([sampled_points, padding], axis=0)
instances_list.append(shift_pts)
instances_tensor = torch.stack(instances_list, dim=0)
instances_tensor = instances_tensor.to(
dtype=torch.float32)
return instances_tensor
@property
def shift_fixed_num_sampled_points_v1(self):
"""
return [instances_num, num_shifts, fixed_num, 2]
"""
fixed_num_sampled_points = self.fixed_num_sampled_points
instances_list = []
is_poly = False
# is_line = False
# import pdb;pdb.set_trace()
for fixed_num_pts in fixed_num_sampled_points:
# [fixed_num, 2]
is_poly = fixed_num_pts[0].equal(fixed_num_pts[-1])
pts_num = fixed_num_pts.shape[0]
shift_num = pts_num - 1
if is_poly:
pts_to_shift = fixed_num_pts[:-1,:]
shift_pts_list = []
if is_poly:
for shift_right_i in range(shift_num):
shift_pts_list.append(pts_to_shift.roll(shift_right_i,0))
else:
shift_pts_list.append(fixed_num_pts)
shift_pts_list.append(fixed_num_pts.flip(0))
shift_pts = torch.stack(shift_pts_list,dim=0)
if is_poly:
_, _, num_coords = shift_pts.shape
tmp_shift_pts = shift_pts.new_zeros((shift_num, pts_num, num_coords))
tmp_shift_pts[:,:-1,:] = shift_pts
tmp_shift_pts[:,-1,:] = shift_pts[:,0,:]
shift_pts = tmp_shift_pts
shift_pts[:,:,0] = torch.clamp(shift_pts[:,:,0], min=-self.max_x,max=self.max_x)
shift_pts[:,:,1] = torch.clamp(shift_pts[:,:,1], min=-self.max_y,max=self.max_y)
if not is_poly:
padding = torch.full([shift_num-shift_pts.shape[0],pts_num,2], self.padding_value)
shift_pts = torch.cat([shift_pts,padding],dim=0)
# padding = np.zeros((self.num_samples - len(sampled_points), 2))
# sampled_points = np.concatenate([sampled_points, padding], axis=0)
instances_list.append(shift_pts)
instances_tensor = torch.stack(instances_list, dim=0)
instances_tensor = instances_tensor.to(
dtype=torch.float32)
return instances_tensor
@property
def shift_fixed_num_sampled_points_v2(self):
"""
return [instances_num, num_shifts, fixed_num, 2]
"""
assert len(self.instance_list) != 0
instances_list = []
for instance in self.instance_list:
distances = np.linspace(0, instance.length, self.fixed_num)
poly_pts = np.array(list(instance.coords))
start_pts = poly_pts[0]
end_pts = poly_pts[-1]
is_poly = np.equal(start_pts, end_pts)
is_poly = is_poly.all()
shift_pts_list = []
pts_num, coords_num = poly_pts.shape
shift_num = pts_num - 1
final_shift_num = self.fixed_num - 1
if is_poly:
pts_to_shift = poly_pts[:-1,:]
for shift_right_i in range(shift_num):
shift_pts = np.roll(pts_to_shift,shift_right_i,axis=0)
pts_to_concat = shift_pts[0]
pts_to_concat = np.expand_dims(pts_to_concat,axis=0)
shift_pts = np.concatenate((shift_pts,pts_to_concat),axis=0)
shift_instance = LineString(shift_pts)
shift_sampled_points = np.array([list(shift_instance.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
shift_pts_list.append(shift_sampled_points)
# import pdb;pdb.set_trace()
else:
sampled_points = np.array([list(instance.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
flip_sampled_points = np.flip(sampled_points, axis=0)
shift_pts_list.append(sampled_points)
shift_pts_list.append(flip_sampled_points)
multi_shifts_pts = np.stack(shift_pts_list,axis=0)
shifts_num,_,_ = multi_shifts_pts.shape
if shifts_num > final_shift_num:
index = np.random.choice(multi_shifts_pts.shape[0], final_shift_num, replace=False)
multi_shifts_pts = multi_shifts_pts[index]
multi_shifts_pts_tensor = to_tensor(multi_shifts_pts)
multi_shifts_pts_tensor = multi_shifts_pts_tensor.to(
dtype=torch.float32)
multi_shifts_pts_tensor[:,:,0] = torch.clamp(multi_shifts_pts_tensor[:,:,0], min=-self.max_x,max=self.max_x)
multi_shifts_pts_tensor[:,:,1] = torch.clamp(multi_shifts_pts_tensor[:,:,1], min=-self.max_y,max=self.max_y)
# if not is_poly:
if multi_shifts_pts_tensor.shape[0] < final_shift_num:
padding = torch.full([final_shift_num-multi_shifts_pts_tensor.shape[0],self.fixed_num,2], self.padding_value)
multi_shifts_pts_tensor = torch.cat([multi_shifts_pts_tensor,padding],dim=0)
instances_list.append(multi_shifts_pts_tensor)
instances_tensor = torch.stack(instances_list, dim=0)
instances_tensor = instances_tensor.to(
dtype=torch.float32)
return instances_tensor
@property
def shift_fixed_num_sampled_points_v3(self):
"""
return [instances_num, num_shifts, fixed_num, 2]
"""
assert len(self.instance_list) != 0
instances_list = []
for instance in self.instance_list:
distances = np.linspace(0, instance.length, self.fixed_num)
poly_pts = np.array(list(instance.coords))
start_pts = poly_pts[0]
end_pts = poly_pts[-1]
is_poly = np.equal(start_pts, end_pts)
is_poly = is_poly.all()
shift_pts_list = []
pts_num, coords_num = poly_pts.shape
shift_num = pts_num - 1
final_shift_num = self.fixed_num - 1
if is_poly:
pts_to_shift = poly_pts[:-1,:]
for shift_right_i in range(shift_num):
shift_pts = np.roll(pts_to_shift,shift_right_i,axis=0)
pts_to_concat = shift_pts[0]
pts_to_concat = np.expand_dims(pts_to_concat,axis=0)
shift_pts = np.concatenate((shift_pts,pts_to_concat),axis=0)
shift_instance = LineString(shift_pts)
shift_sampled_points = np.array([list(shift_instance.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
shift_pts_list.append(shift_sampled_points)
flip_pts_to_shift = np.flip(pts_to_shift, axis=0)
for shift_right_i in range(shift_num):
shift_pts = np.roll(flip_pts_to_shift,shift_right_i,axis=0)
pts_to_concat = shift_pts[0]
pts_to_concat = np.expand_dims(pts_to_concat,axis=0)
shift_pts = np.concatenate((shift_pts,pts_to_concat),axis=0)
shift_instance = LineString(shift_pts)
shift_sampled_points = np.array([list(shift_instance.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
shift_pts_list.append(shift_sampled_points)
else:
sampled_points = np.array([list(instance.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
flip_sampled_points = np.flip(sampled_points, axis=0)
shift_pts_list.append(sampled_points)
shift_pts_list.append(flip_sampled_points)
multi_shifts_pts = np.stack(shift_pts_list,axis=0)
shifts_num,_,_ = multi_shifts_pts.shape
if shifts_num > 2*final_shift_num:
index = np.random.choice(shift_num, final_shift_num, replace=False)
flip0_shifts_pts = multi_shifts_pts[index]
flip1_shifts_pts = multi_shifts_pts[index+shift_num]
multi_shifts_pts = np.concatenate((flip0_shifts_pts,flip1_shifts_pts),axis=0)
multi_shifts_pts_tensor = to_tensor(multi_shifts_pts)
multi_shifts_pts_tensor = multi_shifts_pts_tensor.to(
dtype=torch.float32)
multi_shifts_pts_tensor[:,:,0] = torch.clamp(multi_shifts_pts_tensor[:,:,0], min=-self.max_x,max=self.max_x)
multi_shifts_pts_tensor[:,:,1] = torch.clamp(multi_shifts_pts_tensor[:,:,1], min=-self.max_y,max=self.max_y)
if multi_shifts_pts_tensor.shape[0] < 2*final_shift_num:
padding = torch.full([final_shift_num*2-multi_shifts_pts_tensor.shape[0],self.fixed_num,2], self.padding_value)
multi_shifts_pts_tensor = torch.cat([multi_shifts_pts_tensor,padding],dim=0)
instances_list.append(multi_shifts_pts_tensor)
instances_tensor = torch.stack(instances_list, dim=0)
instances_tensor = instances_tensor.to(
dtype=torch.float32)
return instances_tensor
@property
def shift_fixed_num_sampled_points_v4(self):
"""
return [instances_num, num_shifts, fixed_num, 2]
"""
fixed_num_sampled_points = self.fixed_num_sampled_points
instances_list = []
is_poly = False
for fixed_num_pts in fixed_num_sampled_points:
is_poly = fixed_num_pts[0].equal(fixed_num_pts[-1])
pts_num = fixed_num_pts.shape[0]
shift_num = pts_num - 1
shift_pts_list = []
if is_poly:
pts_to_shift = fixed_num_pts[:-1,:]
for shift_right_i in range(shift_num):
shift_pts_list.append(pts_to_shift.roll(shift_right_i,0))
flip_pts_to_shift = pts_to_shift.flip(0)
for shift_right_i in range(shift_num):
shift_pts_list.append(flip_pts_to_shift.roll(shift_right_i,0))
else:
shift_pts_list.append(fixed_num_pts)
shift_pts_list.append(fixed_num_pts.flip(0))
shift_pts = torch.stack(shift_pts_list,dim=0)
if is_poly:
_, _, num_coords = shift_pts.shape
tmp_shift_pts = shift_pts.new_zeros((shift_num*2, pts_num, num_coords))
tmp_shift_pts[:,:-1,:] = shift_pts
tmp_shift_pts[:,-1,:] = shift_pts[:,0,:]
shift_pts = tmp_shift_pts
shift_pts[:,:,0] = torch.clamp(shift_pts[:,:,0], min=-self.max_x,max=self.max_x)
shift_pts[:,:,1] = torch.clamp(shift_pts[:,:,1], min=-self.max_y,max=self.max_y)
if not is_poly:
padding = torch.full([shift_num*2-shift_pts.shape[0],pts_num,2], self.padding_value)
shift_pts = torch.cat([shift_pts,padding],dim=0)
instances_list.append(shift_pts)
instances_tensor = torch.stack(instances_list, dim=0)
instances_tensor = instances_tensor.to(
dtype=torch.float32)
return instances_tensor
@property
def shift_fixed_num_sampled_points_torch(self):
"""
return [instances_num, num_shifts, fixed_num, 2]
"""
fixed_num_sampled_points = self.fixed_num_sampled_points_torch
instances_list = []
is_poly = False
for fixed_num_pts in fixed_num_sampled_points:
is_poly = fixed_num_pts[0].equal(fixed_num_pts[-1])
fixed_num = fixed_num_pts.shape[0]
shift_pts_list = []
if is_poly:
for shift_right_i in range(fixed_num):
shift_pts_list.append(fixed_num_pts.roll(shift_right_i,0))
else:
shift_pts_list.append(fixed_num_pts)
shift_pts_list.append(fixed_num_pts.flip(0))
shift_pts = torch.stack(shift_pts_list,dim=0)
shift_pts[:,:,0] = torch.clamp(shift_pts[:,:,0], min=-self.max_x,max=self.max_x)
shift_pts[:,:,1] = torch.clamp(shift_pts[:,:,1], min=-self.max_y,max=self.max_y)
if not is_poly:
padding = torch.full([fixed_num-shift_pts.shape[0],fixed_num,2], self.padding_value)
shift_pts = torch.cat([shift_pts,padding],dim=0)
instances_list.append(shift_pts)
instances_tensor = torch.stack(instances_list, dim=0)
instances_tensor = instances_tensor.to(
dtype=torch.float32)
return instances_tensor
class VectorizedLocalMap(object):
CLASS2LABEL = {
'road_divider': 0,
'lane_divider': 0,
'ped_crossing': 1,
'contours': 2,
'others': -1
}
def __init__(self,
dataroot,
patch_size,
map_classes=['divider','ped_crossing','boundary'],
line_classes=['road_divider', 'lane_divider'],
ped_crossing_classes=['ped_crossing'],
contour_classes=['road_segment', 'lane'],
sample_dist=1,
num_samples=250,
padding=False,
fixed_ptsnum_per_line=-1,
padding_value=-10000,):
'''
Args:
fixed_ptsnum_per_line = -1 : no fixed num
'''
super().__init__()
self.data_root = dataroot
self.MAPS = ['boston-seaport', 'singapore-hollandvillage',
'singapore-onenorth', 'singapore-queenstown']
self.vec_classes = map_classes
self.line_classes = line_classes
self.ped_crossing_classes = ped_crossing_classes
self.polygon_classes = contour_classes
self.nusc_maps = {}
self.map_explorer = {}
for loc in self.MAPS:
self.nusc_maps[loc] = NuScenesMap(dataroot=self.data_root, map_name=loc)
self.map_explorer[loc] = NuScenesMapExplorer(self.nusc_maps[loc])
self.patch_size = patch_size
self.sample_dist = sample_dist
self.num_samples = num_samples
self.padding = padding
self.fixed_num = fixed_ptsnum_per_line
self.padding_value = padding_value
def gen_vectorized_samples(self, location, lidar2global_translation, lidar2global_rotation):
'''
use lidar2global to get gt map layers
'''
map_pose = lidar2global_translation[:2]
rotation = Quaternion(lidar2global_rotation)
patch_box = (map_pose[0], map_pose[1], self.patch_size[0], self.patch_size[1])
patch_angle = quaternion_yaw(rotation) / np.pi * 180
vectors = []
for vec_class in self.vec_classes:
if vec_class == 'divider':
line_geom = self.get_map_geom(patch_box, patch_angle, self.line_classes, location)
line_instances_dict = self.line_geoms_to_instances(line_geom)
for line_type, instances in line_instances_dict.items():
for instance in instances:
vectors.append((instance, self.CLASS2LABEL.get(line_type, -1)))
elif vec_class == 'ped_crossing':
ped_geom = self.get_map_geom(patch_box, patch_angle, self.ped_crossing_classes, location)
ped_instance_list = self.ped_poly_geoms_to_instances(ped_geom)
for instance in ped_instance_list:
vectors.append((instance, self.CLASS2LABEL.get('ped_crossing', -1)))
elif vec_class == 'boundary':
polygon_geom = self.get_map_geom(patch_box, patch_angle, self.polygon_classes, location)
poly_bound_list = self.poly_geoms_to_instances(polygon_geom)
for contour in poly_bound_list:
vectors.append((contour, self.CLASS2LABEL.get('contours', -1)))
else:
raise ValueError(f'WRONG vec_class: {vec_class}')
filtered_vectors = []
gt_pts_loc_3d = []
gt_pts_num_3d = []
gt_labels = []
gt_instance = []
for instance, type in vectors:
if type != -1:
gt_instance.append(instance)
gt_labels.append(type)
gt_instance = LiDARInstanceLines(gt_instance,self.sample_dist,
self.num_samples, self.padding, self.fixed_num,self.padding_value, patch_size=self.patch_size)
anns_results = dict(
gt_vecs_pts_loc=gt_instance,
gt_vecs_label=gt_labels,
)
return anns_results
def get_map_geom(self, patch_box, patch_angle, layer_names, location):
map_geom = []
for layer_name in layer_names:
if layer_name in self.line_classes:
geoms = self.get_divider_line(patch_box, patch_angle, layer_name, location)
map_geom.append((layer_name, geoms))
elif layer_name in self.polygon_classes:
geoms = self.get_contour_line(patch_box, patch_angle, layer_name, location)
map_geom.append((layer_name, geoms))
elif layer_name in self.ped_crossing_classes:
geoms = self.get_ped_crossing_line(patch_box, patch_angle, location)
map_geom.append((layer_name, geoms))
return map_geom
def _one_type_line_geom_to_vectors(self, line_geom):
line_vectors = []
for line in line_geom:
if not line.is_empty:
if line.geom_type == 'MultiLineString':
for single_line in line.geoms:
line_vectors.append(self.sample_pts_from_line(single_line))
elif line.geom_type == 'LineString':
line_vectors.append(self.sample_pts_from_line(line))
else:
raise NotImplementedError
return line_vectors
def _one_type_line_geom_to_instances(self, line_geom):
line_instances = []
for line in line_geom:
if not line.is_empty:
if line.geom_type == 'MultiLineString':
for single_line in line.geoms:
line_instances.append(single_line)
elif line.geom_type == 'LineString':
line_instances.append(line)
else:
raise NotImplementedError
return line_instances
def poly_geoms_to_vectors(self, polygon_geom):
roads = polygon_geom[0][1]
lanes = polygon_geom[1][1]
union_roads = ops.unary_union(roads)
union_lanes = ops.unary_union(lanes)
union_segments = ops.unary_union([union_roads, union_lanes])
max_x = self.patch_size[1] / 2
max_y = self.patch_size[0] / 2
local_patch = box(-max_x + 0.2, -max_y + 0.2, max_x - 0.2, max_y - 0.2)
exteriors = []
interiors = []
if union_segments.geom_type != 'MultiPolygon':
union_segments = MultiPolygon([union_segments])
for poly in union_segments.geoms:
exteriors.append(poly.exterior)
for inter in poly.interiors:
interiors.append(inter)
results = []
for ext in exteriors:
if ext.is_ccw:
ext.coords = list(ext.coords)[::-1]
lines = ext.intersection(local_patch)
if isinstance(lines, MultiLineString):
lines = ops.linemerge(lines)
results.append(lines)
for inter in interiors:
if not inter.is_ccw:
inter.coords = list(inter.coords)[::-1]
lines = inter.intersection(local_patch)
if isinstance(lines, MultiLineString):
lines = ops.linemerge(lines)
results.append(lines)
return self._one_type_line_geom_to_vectors(results)
def ped_poly_geoms_to_instances(self, ped_geom):
ped = ped_geom[0][1]
union_segments = ops.unary_union(ped)
max_x = self.patch_size[1] / 2
max_y = self.patch_size[0] / 2
local_patch = box(-max_x - 0.2, -max_y - 0.2, max_x + 0.2, max_y + 0.2)
exteriors = []
interiors = []
if union_segments.geom_type != 'MultiPolygon':
union_segments = MultiPolygon([union_segments])
for poly in union_segments.geoms:
exteriors.append(poly.exterior)
for inter in poly.interiors:
interiors.append(inter)
results = []
for ext in exteriors:
if ext.is_ccw:
ext.coords = list(ext.coords)[::-1]
lines = ext.intersection(local_patch)
if isinstance(lines, MultiLineString):
lines = ops.linemerge(lines)
results.append(lines)
for inter in interiors:
if not inter.is_ccw:
inter.coords = list(inter.coords)[::-1]
lines = inter.intersection(local_patch)
if isinstance(lines, MultiLineString):
lines = ops.linemerge(lines)
results.append(lines)
return self._one_type_line_geom_to_instances(results)
def poly_geoms_to_instances(self, polygon_geom):
roads = polygon_geom[0][1]
lanes = polygon_geom[1][1]
union_roads = ops.unary_union(roads)
union_lanes = ops.unary_union(lanes)
union_segments = ops.unary_union([union_roads, union_lanes])
max_x = self.patch_size[1] / 2
max_y = self.patch_size[0] / 2
local_patch = box(-max_x + 0.2, -max_y + 0.2, max_x - 0.2, max_y - 0.2)
exteriors = []
interiors = []
if union_segments.geom_type != 'MultiPolygon':
union_segments = MultiPolygon([union_segments])
for poly in union_segments.geoms:
exteriors.append(poly.exterior)
for inter in poly.interiors:
interiors.append(inter)
results = []
for ext in exteriors:
if ext.is_ccw:
ext.coords = list(ext.coords)[::-1]
lines = ext.intersection(local_patch)
if isinstance(lines, MultiLineString):
lines = ops.linemerge(lines)
results.append(lines)
for inter in interiors:
if not inter.is_ccw:
inter.coords = list(inter.coords)[::-1]
lines = inter.intersection(local_patch)
if isinstance(lines, MultiLineString):
lines = ops.linemerge(lines)
results.append(lines)
return self._one_type_line_geom_to_instances(results)
def line_geoms_to_vectors(self, line_geom):
line_vectors_dict = dict()
for line_type, a_type_of_lines in line_geom:
one_type_vectors = self._one_type_line_geom_to_vectors(a_type_of_lines)
line_vectors_dict[line_type] = one_type_vectors
return line_vectors_dict
def line_geoms_to_instances(self, line_geom):
line_instances_dict = dict()
for line_type, a_type_of_lines in line_geom:
one_type_instances = self._one_type_line_geom_to_instances(a_type_of_lines)
line_instances_dict[line_type] = one_type_instances
return line_instances_dict
def ped_geoms_to_vectors(self, ped_geom):
ped_geom = ped_geom[0][1]
union_ped = ops.unary_union(ped_geom)
if union_ped.geom_type != 'MultiPolygon':
union_ped = MultiPolygon([union_ped])
max_x = self.patch_size[1] / 2
max_y = self.patch_size[0] / 2
local_patch = box(-max_x + 0.2, -max_y + 0.2, max_x - 0.2, max_y - 0.2)
results = []
for ped_poly in union_ped:
# rect = ped_poly.minimum_rotated_rectangle
ext = ped_poly.exterior
if not ext.is_ccw:
ext.coords = list(ext.coords)[::-1]
lines = ext.intersection(local_patch)
results.append(lines)
return self._one_type_line_geom_to_vectors(results)
def get_contour_line(self,patch_box,patch_angle,layer_name,location):
if layer_name not in self.map_explorer[location].map_api.non_geometric_polygon_layers:
raise ValueError('{} is not a polygonal layer'.format(layer_name))
patch_x = patch_box[0]
patch_y = patch_box[1]
patch = self.map_explorer[location].get_patch_coord(patch_box, patch_angle)
records = getattr(self.map_explorer[location].map_api, layer_name)
polygon_list = []
if layer_name == 'drivable_area':
for record in records:
polygons = [self.map_explorer[location].map_api.extract_polygon(polygon_token) for polygon_token in record['polygon_tokens']]
for polygon in polygons:
new_polygon = polygon.intersection(patch)
if not new_polygon.is_empty:
new_polygon = affinity.rotate(new_polygon, -patch_angle,
origin=(patch_x, patch_y), use_radians=False)
new_polygon = affinity.affine_transform(new_polygon,
[1.0, 0.0, 0.0, 1.0, -patch_x, -patch_y])
if new_polygon.geom_type is 'Polygon':
new_polygon = MultiPolygon([new_polygon])
polygon_list.append(new_polygon)
else:
for record in records:
polygon = self.map_explorer[location].map_api.extract_polygon(record['polygon_token'])
if polygon.is_valid:
new_polygon = polygon.intersection(patch)
if not new_polygon.is_empty:
new_polygon = affinity.rotate(new_polygon, -patch_angle,
origin=(patch_x, patch_y), use_radians=False)
new_polygon = affinity.affine_transform(new_polygon,
[1.0, 0.0, 0.0, 1.0, -patch_x, -patch_y])
if new_polygon.geom_type is 'Polygon':
new_polygon = MultiPolygon([new_polygon])
polygon_list.append(new_polygon)
return polygon_list
def get_divider_line(self,patch_box,patch_angle,layer_name,location):
if layer_name not in self.map_explorer[location].map_api.non_geometric_line_layers:
raise ValueError("{} is not a line layer".format(layer_name))
if layer_name is 'traffic_light':
return None
patch_x = patch_box[0]
patch_y = patch_box[1]
patch = self.map_explorer[location].get_patch_coord(patch_box, patch_angle)
line_list = []
records = getattr(self.map_explorer[location].map_api, layer_name)
for record in records:
line = self.map_explorer[location].map_api.extract_line(record['line_token'])
if line.is_empty: # Skip lines without nodes.
continue
new_line = line.intersection(patch)
if not new_line.is_empty:
new_line = affinity.rotate(new_line, -patch_angle, origin=(patch_x, patch_y), use_radians=False)
new_line = affinity.affine_transform(new_line,
[1.0, 0.0, 0.0, 1.0, -patch_x, -patch_y])
line_list.append(new_line)
return line_list
def get_ped_crossing_line(self, patch_box, patch_angle, location):
patch_x = patch_box[0]
patch_y = patch_box[1]
patch = self.map_explorer[location].get_patch_coord(patch_box, patch_angle)
polygon_list = []
records = getattr(self.map_explorer[location].map_api, 'ped_crossing')
# records = getattr(self.nusc_maps[location], 'ped_crossing')
for record in records:
polygon = self.map_explorer[location].map_api.extract_polygon(record['polygon_token'])
if polygon.is_valid:
new_polygon = polygon.intersection(patch)
if not new_polygon.is_empty:
new_polygon = affinity.rotate(new_polygon, -patch_angle,
origin=(patch_x, patch_y), use_radians=False)
new_polygon = affinity.affine_transform(new_polygon,
[1.0, 0.0, 0.0, 1.0, -patch_x, -patch_y])
if new_polygon.geom_type is 'Polygon':
new_polygon = MultiPolygon([new_polygon])
polygon_list.append(new_polygon)
return polygon_list
def sample_pts_from_line(self, line):
if self.fixed_num < 0:
distances = np.arange(0, line.length, self.sample_dist)
sampled_points = np.array([list(line.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
else:
# fixed number of points, so distance is line.length / self.fixed_num
distances = np.linspace(0, line.length, self.fixed_num)
sampled_points = np.array([list(line.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
num_valid = len(sampled_points)
if not self.padding or self.fixed_num > 0:
return sampled_points, num_valid
# fixed distance sampling need padding!
num_valid = len(sampled_points)
if self.fixed_num < 0:
if num_valid < self.num_samples:
padding = np.zeros((self.num_samples - len(sampled_points), 2))
sampled_points = np.concatenate([sampled_points, padding], axis=0)
else:
sampled_points = sampled_points[:self.num_samples, :]
num_valid = self.num_samples
return sampled_points, num_valid
@DATASETS.register_module()
class CustomNuScenesLocalMapDataset(CustomNuScenesDataset):
r"""NuScenes Dataset.
This datset add static map elements
"""
MAPCLASSES = ('divider',)
def __init__(self,
map_ann_file=None,
queue_length=4,
bev_size=(200, 200),
pc_range=[-51.2, -51.2, -5.0, 51.2, 51.2, 3.0],
overlap_test=False,
fixed_ptsnum_per_line=-1,
eval_use_same_gt_sample_num_flag=False,
padding_value=-10000,
map_classes=None,
noise='None',
noise_std=0,
*args,
**kwargs):
super().__init__(*args, **kwargs)
self.map_ann_file = map_ann_file
self.queue_length = queue_length
self.overlap_test = overlap_test
self.bev_size = bev_size
self.MAPCLASSES = self.get_map_classes(map_classes)
self.NUM_MAPCLASSES = len(self.MAPCLASSES)
self.pc_range = pc_range
patch_h = pc_range[4]-pc_range[1]
patch_w = pc_range[3]-pc_range[0]
self.patch_size = (patch_h, patch_w)
self.padding_value = padding_value
self.fixed_num = fixed_ptsnum_per_line
self.eval_use_same_gt_sample_num_flag = eval_use_same_gt_sample_num_flag
self.vector_map = VectorizedLocalMap(kwargs['data_root'],
patch_size=self.patch_size, map_classes=self.MAPCLASSES,
fixed_ptsnum_per_line=fixed_ptsnum_per_line,
padding_value=self.padding_value)
self.is_vis_on_test = False
self.noise = noise
self.noise_std = noise_std
@classmethod
def get_map_classes(cls, map_classes=None):
"""Get class names of current dataset.
Args:
classes (Sequence[str] | str | None): If classes is None, use
default CLASSES defined by builtin dataset. If classes is a
string, take it as a file name. The file contains the name of
classes where each line contains one class name. If classes is
a tuple or list, override the CLASSES defined by the dataset.
Return:
list[str]: A list of class names.
"""
if map_classes is None:
return cls.MAPCLASSES
if isinstance(map_classes, str):
# take it as a file path
class_names = mmcv.list_from_file(map_classes)
elif isinstance(map_classes, (tuple, list)):
class_names = map_classes
else:
raise ValueError(f'Unsupported type {type(map_classes)} of map classes.')
return class_names
def vectormap_pipeline(self, example, input_dict):
'''
`example` type: <class 'dict'>
keys: 'img_metas', 'gt_bboxes_3d', 'gt_labels_3d', 'img';
all keys type is 'DataContainer';
'img_metas' cpu_only=True, type is dict, others are false;
'gt_labels_3d' shape torch.size([num_samples]), stack=False,
padding_value=0, cpu_only=False
'gt_bboxes_3d': stack=False, cpu_only=True
'''
# import pdb;pdb.set_trace()
lidar2ego = np.eye(4)
lidar2ego[:3,:3] = Quaternion(input_dict['lidar2ego_rotation']).rotation_matrix
lidar2ego[:3, 3] = input_dict['lidar2ego_translation']
ego2global = np.eye(4)
ego2global[:3,:3] = Quaternion(input_dict['ego2global_rotation']).rotation_matrix
ego2global[:3, 3] = input_dict['ego2global_translation']
lidar2global = ego2global @ lidar2ego
lidar2global_translation = list(lidar2global[:3,3])
lidar2global_rotation = list(Quaternion(matrix=lidar2global).q)
location = input_dict['map_location']
ego2global_translation = input_dict['ego2global_translation']
ego2global_rotation = input_dict['ego2global_rotation']
anns_results = self.vector_map.gen_vectorized_samples(location, lidar2global_translation, lidar2global_rotation)
'''
anns_results, type: dict
'gt_vecs_pts_loc': list[num_vecs], vec with num_points*2 coordinates
'gt_vecs_pts_num': list[num_vecs], vec with num_points
'gt_vecs_label': list[num_vecs], vec with cls index
'''
gt_vecs_label = to_tensor(anns_results['gt_vecs_label'])
if isinstance(anns_results['gt_vecs_pts_loc'], LiDARInstanceLines):
gt_vecs_pts_loc = anns_results['gt_vecs_pts_loc']
else:
gt_vecs_pts_loc = to_tensor(anns_results['gt_vecs_pts_loc'])
try:
gt_vecs_pts_loc = gt_vecs_pts_loc.flatten(1).to(dtype=torch.float32)
except:
# empty tensor, will be passed in train,
# but we preserve it for test
gt_vecs_pts_loc = gt_vecs_pts_loc
example['gt_labels_3d'] = DC(gt_vecs_label, cpu_only=False)
example['gt_bboxes_3d'] = DC(gt_vecs_pts_loc, cpu_only=True)
return example
def prepare_train_data(self, index):
"""
Training data preparation.
Args:
index (int): Index for accessing the target data.
Returns:
dict: Training data dict of the corresponding index.
"""
data_queue = []
# temporal aug
prev_indexs_list = list(range(index-self.queue_length, index))
random.shuffle(prev_indexs_list)
prev_indexs_list = sorted(prev_indexs_list[1:], reverse=True)
##
input_dict = self.get_data_info(index)
if input_dict is None:
return None
frame_idx = input_dict['frame_idx']
scene_token = input_dict['scene_token']
self.pre_pipeline(input_dict)
example = self.pipeline(input_dict)
example = self.vectormap_pipeline(example,input_dict)
if self.filter_empty_gt and \
(example is None or ~(example['gt_labels_3d']._data != -1).any()):
return None
data_queue.insert(0, example)
for i in prev_indexs_list:
i = max(0, i)
input_dict = self.get_data_info(i)
if input_dict is None:
return None
if input_dict['frame_idx'] < frame_idx and input_dict['scene_token'] == scene_token:
self.pre_pipeline(input_dict)
example = self.pipeline(input_dict)
example = self.vectormap_pipeline(example,input_dict)
if self.filter_empty_gt and \
(example is None or ~(example['gt_labels_3d']._data != -1).any()):
return None
frame_idx = input_dict['frame_idx']
data_queue.insert(0, copy.deepcopy(example))
return self.union2one(data_queue)
def union2one(self, queue):
"""
convert sample queue into one single sample.
"""
imgs_list = [each['img'].data for each in queue]
metas_map = {}
prev_pos = None
prev_angle = None
for i, each in enumerate(queue):
metas_map[i] = each['img_metas'].data
if i == 0:
metas_map[i]['prev_bev'] = False
prev_pos = copy.deepcopy(metas_map[i]['can_bus'][:3])
prev_angle = copy.deepcopy(metas_map[i]['can_bus'][-1])
metas_map[i]['can_bus'][:3] = 0
metas_map[i]['can_bus'][-1] = 0
else:
metas_map[i]['prev_bev'] = True
tmp_pos = copy.deepcopy(metas_map[i]['can_bus'][:3])
tmp_angle = copy.deepcopy(metas_map[i]['can_bus'][-1])
metas_map[i]['can_bus'][:3] -= prev_pos
metas_map[i]['can_bus'][-1] -= prev_angle
prev_pos = copy.deepcopy(tmp_pos)
prev_angle = copy.deepcopy(tmp_angle)
queue[-1]['img'] = DC(torch.stack(imgs_list),
cpu_only=False, stack=True)
queue[-1]['img_metas'] = DC(metas_map, cpu_only=True)
queue = queue[-1]
return queue
def get_data_info(self, index):
"""Get data info according to the given index.
Args:
index (int): Index of the sample data to get.
Returns:
dict: Data information that will be passed to the data \
preprocessing pipelines. It includes the following keys:
- sample_idx (str): Sample index.
- pts_filename (str): Filename of point clouds.
- sweeps (list[dict]): Infos of sweeps.
- timestamp (float): Sample timestamp.
- img_filename (str, optional): Image filename.
- lidar2img (list[np.ndarray], optional): Transformations \
from lidar to different cameras.
- ann_info (dict): Annotation info.
"""
info = self.data_infos[index]
# standard protocal modified from SECOND.Pytorch
input_dict = dict(
sample_idx=info['token'],
pts_filename=info['lidar_path'],
lidar_path=info["lidar_path"],
sweeps=info['sweeps'],
ego2global_translation=info['ego2global_translation'],
ego2global_rotation=info['ego2global_rotation'],
lidar2ego_translation=info['lidar2ego_translation'],
lidar2ego_rotation=info['lidar2ego_rotation'],
prev_idx=info['prev'],
next_idx=info['next'],
scene_token=info['scene_token'],
can_bus=info['can_bus'],
frame_idx=info['frame_idx'],
timestamp=info['timestamp'],
map_location = info['map_location'],
)
# lidar to ego transform
lidar2ego = np.eye(4).astype(np.float32)
lidar2ego[:3, :3] = Quaternion(info["lidar2ego_rotation"]).rotation_matrix
lidar2ego[:3, 3] = info["lidar2ego_translation"]
input_dict["lidar2ego"] = lidar2ego
if self.modality['use_camera']:
image_paths = []
lidar2img_rts = []
lidar2cam_rts = []
cam_intrinsics = []
input_dict["camera2ego"] = []
input_dict["camera_intrinsics"] = []
for cam_type, cam_info in info['cams'].items():
image_paths.append(cam_info['data_path'])
# obtain lidar to image transformation matrix
lidar2cam_r = np.linalg.inv(cam_info['sensor2lidar_rotation'])
lidar2cam_t = cam_info[
'sensor2lidar_translation'] @ lidar2cam_r.T
lidar2cam_rt = np.eye(4)
lidar2cam_rt[:3, :3] = lidar2cam_r.T
lidar2cam_rt[3, :3] = -lidar2cam_t
lidar2cam_rt_t = lidar2cam_rt.T
if self.noise == 'rotation':
lidar2cam_rt_t = add_rotation_noise(lidar2cam_rt_t, std=self.noise_std)
elif self.noise == 'translation':
lidar2cam_rt_t = add_translation_noise(
lidar2cam_rt_t, std=self.noise_std)
intrinsic = cam_info['cam_intrinsic']
viewpad = np.eye(4)
viewpad[:intrinsic.shape[0], :intrinsic.shape[1]] = intrinsic
lidar2img_rt = (viewpad @ lidar2cam_rt_t)
lidar2img_rts.append(lidar2img_rt)
cam_intrinsics.append(viewpad)
lidar2cam_rts.append(lidar2cam_rt_t)
# camera to ego transform
camera2ego = np.eye(4).astype(np.float32)
camera2ego[:3, :3] = Quaternion(
cam_info["sensor2ego_rotation"]
).rotation_matrix
camera2ego[:3, 3] = cam_info["sensor2ego_translation"]
input_dict["camera2ego"].append(camera2ego)
# camera intrinsics
camera_intrinsics = np.eye(4).astype(np.float32)
camera_intrinsics[:3, :3] = cam_info["cam_intrinsic"]
input_dict["camera_intrinsics"].append(camera_intrinsics)
input_dict.update(
dict(
img_filename=image_paths,
lidar2img=lidar2img_rts,
cam_intrinsic=cam_intrinsics,
lidar2cam=lidar2cam_rts,
))
if not self.test_mode:
annos = self.get_ann_info(index)
input_dict['ann_info'] = annos
rotation = Quaternion(input_dict['ego2global_rotation'])
translation = input_dict['ego2global_translation']
can_bus = input_dict['can_bus']
can_bus[:3] = translation
can_bus[3:7] = rotation
patch_angle = quaternion_yaw(rotation) / np.pi * 180
if patch_angle < 0:
patch_angle += 360
can_bus[-2] = patch_angle / 180 * np.pi
can_bus[-1] = patch_angle
lidar2ego = np.eye(4)
lidar2ego[:3,:3] = Quaternion(input_dict['lidar2ego_rotation']).rotation_matrix
lidar2ego[:3, 3] = input_dict['lidar2ego_translation']
ego2global = np.eye(4)
ego2global[:3,:3] = Quaternion(input_dict['ego2global_rotation']).rotation_matrix
ego2global[:3, 3] = input_dict['ego2global_translation']
lidar2global = ego2global @ lidar2ego
input_dict['lidar2global'] = lidar2global
return input_dict
def prepare_test_data(self, index):
"""Prepare data for testing.
Args:
index (int): Index for accessing the target data.
Returns:
dict: Testing data dict of the corresponding index.
"""
input_dict = self.get_data_info(index)
self.pre_pipeline(input_dict)
example = self.pipeline(input_dict)
if self.is_vis_on_test:
example = self.vectormap_pipeline(example, input_dict)
return example
def __getitem__(self, idx):
"""Get item from infos according to the given index.
Returns:
dict: Data dictionary of the corresponding index.
"""
if self.test_mode:
return self.prepare_test_data(idx)
while True:
data = self.prepare_train_data(idx)
if data is None:
idx = self._rand_another(idx)
continue
return data
def _format_gt(self):
gt_annos = []
print('Start to convert gt map format...')
assert self.map_ann_file is not None
if (not os.path.exists(self.map_ann_file)) :
dataset_length = len(self)
prog_bar = mmcv.ProgressBar(dataset_length)
mapped_class_names = self.MAPCLASSES
for sample_id in range(dataset_length):
sample_token = self.data_infos[sample_id]['token']
gt_anno = {}
gt_anno['sample_token'] = sample_token
# gt_sample_annos = []
gt_sample_dict = {}
gt_sample_dict = self.vectormap_pipeline(gt_sample_dict, self.data_infos[sample_id])
gt_labels = gt_sample_dict['gt_labels_3d'].data.numpy()
gt_vecs = gt_sample_dict['gt_bboxes_3d'].data.instance_list
gt_vec_list = []
for i, (gt_label, gt_vec) in enumerate(zip(gt_labels, gt_vecs)):
name = mapped_class_names[gt_label]
anno = dict(
pts=np.array(list(gt_vec.coords)),
pts_num=len(list(gt_vec.coords)),
cls_name=name,
type=gt_label,
)
gt_vec_list.append(anno)
gt_anno['vectors']=gt_vec_list
gt_annos.append(gt_anno)
prog_bar.update()
nusc_submissions = {
'GTs': gt_annos
}
print('\n GT anns writes to', self.map_ann_file)
mmcv.dump(nusc_submissions, self.map_ann_file)
else:
print(f'{self.map_ann_file} exist, not update')
def _format_bbox(self, results, jsonfile_prefix=None):
"""Convert the results to the standard format.
Args:
results (list[dict]): Testing results of the dataset.
jsonfile_prefix (str): The prefix of the output jsonfile.
You can specify the output directory/filename by
modifying the jsonfile_prefix. Default: None.
Returns:
str: Path of the output json file.
"""
assert self.map_ann_file is not None
pred_annos = []
mapped_class_names = self.MAPCLASSES
# import pdb;pdb.set_trace()
print('Start to convert map detection format...')
for sample_id, det in enumerate(mmcv.track_iter_progress(results)):
pred_anno = {}
vecs = output_to_vecs(det)
sample_token = self.data_infos[sample_id]['token']
pred_anno['sample_token'] = sample_token
pred_vec_list=[]
for i, vec in enumerate(vecs):
name = mapped_class_names[vec['label']]
anno = dict(
pts=vec['pts'],
pts_num=len(vec['pts']),
cls_name=name,
type=vec['label'],
confidence_level=vec['score'])
pred_vec_list.append(anno)
pred_anno['vectors'] = pred_vec_list
pred_annos.append(pred_anno)
if not os.path.exists(self.map_ann_file):
self._format_gt()
else:
print(f'{self.map_ann_file} exist, not update')
nusc_submissions = {
'meta': self.modality,
'results': pred_annos,
}
mmcv.mkdir_or_exist(jsonfile_prefix)
res_path = osp.join(jsonfile_prefix, 'nuscmap_results.json')
print('Results writes to', res_path)
mmcv.dump(nusc_submissions, res_path)
return res_path
def to_gt_vectors(self,
gt_dict):
# import pdb;pdb.set_trace()
gt_labels = gt_dict['gt_labels_3d'].data
gt_instances = gt_dict['gt_bboxes_3d'].data.instance_list
gt_vectors = []
for gt_instance, gt_label in zip(gt_instances, gt_labels):
pts, pts_num = sample_pts_from_line(gt_instance, patch_size=self.patch_size)
gt_vectors.append({
'pts': pts,
'pts_num': pts_num,
'type': int(gt_label)
})
vector_num_list = {}
for i in range(self.NUM_MAPCLASSES):
vector_num_list[i] = []
for vec in gt_vectors:
if vector['pts_num'] >= 2:
vector_num_list[vector['type']].append((LineString(vector['pts'][:vector['pts_num']]), vector.get('confidence_level', 1)))
return gt_vectors
def _evaluate_single(self,
result_path,
logger=None,
metric='chamfer',
result_name='pts_bbox'):
"""Evaluation for a single model in nuScenes protocol.
Args:
result_path (str): Path of the result file.
logger (logging.Logger | str | None): Logger used for printing
related information during evaluation. Default: None.
metric (str): Metric name used for evaluation. Default: 'bbox'.
result_name (str): Result name in the metric prefix.
Default: 'pts_bbox'.
Returns:
dict: Dictionary of evaluation details.
"""
from projects.mmdet3d_plugin.datasets.map_utils.mean_ap import eval_map
from projects.mmdet3d_plugin.datasets.map_utils.mean_ap import format_res_gt_by_classes
result_path = osp.abspath(result_path)
detail = dict()
print('Formating results & gts by classes')
with open(result_path,'r') as f:
pred_results = json.load(f)
gen_results = pred_results['results']
with open(self.map_ann_file,'r') as ann_f:
gt_anns = json.load(ann_f)
annotations = gt_anns['GTs']
cls_gens, cls_gts = format_res_gt_by_classes(result_path,
gen_results,
annotations,
cls_names=self.MAPCLASSES,
num_pred_pts_per_instance=self.fixed_num,
eval_use_same_gt_sample_num_flag=self.eval_use_same_gt_sample_num_flag,
pc_range=self.pc_range)
metrics = metric if isinstance(metric, list) else [metric]
allowed_metrics = ['chamfer', 'iou']
for metric in metrics:
if metric not in allowed_metrics:
raise KeyError(f'metric {metric} is not supported')
for metric in metrics:
print('-*'*10+f'use metric:{metric}'+'-*'*10)
if metric == 'chamfer':
thresholds = [0.5,1.0,1.5]
elif metric == 'iou':
thresholds= np.linspace(.5, 0.95, int(np.round((0.95 - .5) / .05)) + 1, endpoint=True)
cls_aps = np.zeros((len(thresholds),self.NUM_MAPCLASSES))
for i, thr in enumerate(thresholds):
print('-*'*10+f'threshhold:{thr}'+'-*'*10)
mAP, cls_ap = eval_map(
gen_results,
annotations,
cls_gens,
cls_gts,
threshold=thr,
cls_names=self.MAPCLASSES,
logger=logger,
num_pred_pts_per_instance=self.fixed_num,
pc_range=self.pc_range,
metric=metric)
for j in range(self.NUM_MAPCLASSES):
cls_aps[i, j] = cls_ap[j]['ap']
for i, name in enumerate(self.MAPCLASSES):
print('{}: {}'.format(name, cls_aps.mean(0)[i]))
detail['NuscMap_{}/{}_AP'.format(metric,name)] = cls_aps.mean(0)[i]
print('map: {}'.format(cls_aps.mean(0).mean()))
detail['NuscMap_{}/mAP'.format(metric)] = cls_aps.mean(0).mean()
for i, name in enumerate(self.MAPCLASSES):
for j, thr in enumerate(thresholds):
if metric == 'chamfer':
detail['NuscMap_{}/{}_AP_thr_{}'.format(metric,name,thr)]=cls_aps[j][i]
elif metric == 'iou':
if thr == 0.5 or thr == 0.75:
detail['NuscMap_{}/{}_AP_thr_{}'.format(metric,name,thr)]=cls_aps[j][i]
return detail
def evaluate(self,
results,
metric='bbox',
logger=None,
jsonfile_prefix=None,
result_names=['pts_bbox'],
show=False,
out_dir=None,
pipeline=None):
"""Evaluation in nuScenes protocol.
Args:
results (list[dict]): Testing results of the dataset.
metric (str | list[str]): Metrics to be evaluated.
logger (logging.Logger | str | None): Logger used for printing
related information during evaluation. Default: None.
jsonfile_prefix (str | None): The prefix of json files. It includes
the file path and the prefix of filename, e.g., "a/b/prefix".
If not specified, a temp file will be created. Default: None.
show (bool): Whether to visualize.
Default: False.
out_dir (str): Path to save the visualization results.
Default: None.
pipeline (list[dict], optional): raw data loading for showing.
Default: None.
Returns:
dict[str, float]: Results of each evaluation metric.
"""
result_files, tmp_dir = self.format_results(results, jsonfile_prefix)
if isinstance(result_files, dict):
results_dict = dict()
for name in result_names:
print('Evaluating bboxes of {}'.format(name))
ret_dict = self._evaluate_single(result_files[name], metric=metric)
results_dict.update(ret_dict)
elif isinstance(result_files, str):
results_dict = self._evaluate_single(result_files, metric=metric)
if tmp_dir is not None:
tmp_dir.cleanup()
if show:
self.show(results, out_dir, pipeline=pipeline)
return results_dict
def output_to_vecs(detection):
box3d = detection['boxes_3d'].numpy()
scores = detection['scores_3d'].numpy()
labels = detection['labels_3d'].numpy()
pts = detection['pts_3d'].numpy()
vec_list = []
for i in range(box3d.shape[0]):
vec = dict(
bbox = box3d[i], # xyxy
label=labels[i],
score=scores[i],
pts=pts[i],
)
vec_list.append(vec)
return vec_list
def sample_pts_from_line(line,
fixed_num=-1,
sample_dist=1,
normalize=False,
patch_size=None,
padding=False,
num_samples=250,):
if fixed_num < 0:
distances = np.arange(0, line.length, sample_dist)
sampled_points = np.array([list(line.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
else:
# fixed number of points, so distance is line.length / fixed_num
distances = np.linspace(0, line.length, fixed_num)
sampled_points = np.array([list(line.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
if normalize:
sampled_points = sampled_points / np.array([patch_size[1], patch_size[0]])
num_valid = len(sampled_points)
if not padding or fixed_num > 0:
# fixed num sample can return now!
return sampled_points, num_valid
# fixed distance sampling need padding!
num_valid = len(sampled_points)
if fixed_num < 0:
if num_valid < num_samples:
padding = np.zeros((num_samples - len(sampled_points), 2))
sampled_points = np.concatenate([sampled_points, padding], axis=0)
else:
sampled_points = sampled_points[:num_samples, :]
num_valid = num_samples
if normalize:
sampled_points = sampled_points / np.array([patch_size[1], patch_size[0]])
num_valid = len(sampled_points)
return sampled_points, num_valid
docker-hub/MapTRv2/MapTR/projects/mmdet3d_plugin/datasets/nuscenes_mono_dataset.py 0 → 100644
View file @ 19472568
# Copyright (c) OpenMMLab. All rights reserved.
import copy
import mmcv
import numpy as np
import pyquaternion
import tempfile
import torch
import warnings
from nuscenes.utils.data_classes import Box as NuScenesBox
from os import path as osp
from mmdet3d.core import bbox3d2result, box3d_multiclass_nms, xywhr2xyxyr
from mmdet.datasets import DATASETS, CocoDataset
from mmdet3d.core import show_multi_modality_result
from mmdet3d.core.bbox import CameraInstance3DBoxes, get_box_type
from mmdet3d.datasets.pipelines import Compose
from mmdet3d.datasets.utils import extract_result_dict, get_loading_pipeline
@DATASETS.register_module()
class CustomNuScenesMonoDataset(CocoDataset):
r"""Monocular 3D detection on NuScenes Dataset.
This class serves as the API for experiments on the NuScenes Dataset.
Please refer to `NuScenes Dataset <https://www.nuscenes.org/download>`_
for data downloading.
Args:
ann_file (str): Path of annotation file.
data_root (str): Path of dataset root.
load_interval (int, optional): Interval of loading the dataset. It is
used to uniformly sample the dataset. Defaults to 1.
with_velocity (bool, optional): Whether include velocity prediction
into the experiments. Defaults to True.
modality (dict, optional): Modality to specify the sensor data used
as input. Defaults to None.
box_type_3d (str, optional): Type of 3D box of this dataset.
Based on the `box_type_3d`, the dataset will encapsulate the box
to its original format then converted them to `box_type_3d`.
Defaults to 'Camera' in this class. Available options includes.
- 'LiDAR': Box in LiDAR coordinates.
- 'Depth': Box in depth coordinates, usually for indoor dataset.
- 'Camera': Box in camera coordinates.
eval_version (str, optional): Configuration version of evaluation.
Defaults to 'detection_cvpr_2019'.
use_valid_flag (bool): Whether to use `use_valid_flag` key in the info
file as mask to filter gt_boxes and gt_names. Defaults to False.
version (str, optional): Dataset version. Defaults to 'v1.0-trainval'.
"""
CLASSES = ('car', 'truck', 'trailer', 'bus', 'construction_vehicle',
'bicycle', 'motorcycle', 'pedestrian', 'traffic_cone',
'barrier')
DefaultAttribute = {
'car': 'vehicle.parked',
'pedestrian': 'pedestrian.moving',
'trailer': 'vehicle.parked',
'truck': 'vehicle.parked',
'bus': 'vehicle.moving',
'motorcycle': 'cycle.without_rider',
'construction_vehicle': 'vehicle.parked',
'bicycle': 'cycle.without_rider',
'barrier': '',
'traffic_cone': '',
}
# https://github.com/nutonomy/nuscenes-devkit/blob/57889ff20678577025326cfc24e57424a829be0a/python-sdk/nuscenes/eval/detection/evaluate.py#L222 # noqa
ErrNameMapping = {
'trans_err': 'mATE',
'scale_err': 'mASE',
'orient_err': 'mAOE',
'vel_err': 'mAVE',
'attr_err': 'mAAE'
}
def __init__(self,
data_root,
load_interval=1,
with_velocity=True,
modality=None,
box_type_3d='Camera',
eval_version='detection_cvpr_2019',
use_valid_flag=False,
overlap_test=False,
version='v1.0-trainval',
**kwargs):
super().__init__(**kwargs)
# overlap_test = True
self.data_root = data_root
self.overlap_test = overlap_test
self.load_interval = load_interval
self.with_velocity = with_velocity
self.modality = modality
self.box_type_3d, self.box_mode_3d = get_box_type(box_type_3d)
self.eval_version = eval_version
self.use_valid_flag = use_valid_flag
self.bbox_code_size = 9
self.version = version
if self.eval_version is not None:
from nuscenes.eval.detection.config import config_factory
self.eval_detection_configs = config_factory(self.eval_version)
if self.modality is None:
self.modality = dict(
use_camera=True,
use_lidar=False,
use_radar=False,
use_map=False,
use_external=False)
def pre_pipeline(self, results):
"""Initialization before data preparation.
Args:
results (dict): Dict before data preprocessing.
- img_fields (list): Image fields.
- bbox3d_fields (list): 3D bounding boxes fields.
- pts_mask_fields (list): Mask fields of points.
- pts_seg_fields (list): Mask fields of point segments.
- bbox_fields (list): Fields of bounding boxes.
- mask_fields (list): Fields of masks.
- seg_fields (list): Segment fields.
- box_type_3d (str): 3D box type.
- box_mode_3d (str): 3D box mode.
"""
results['img_prefix'] = '' # self.img_prefix
# print('img_prefix', self.img_prefix)
results['seg_prefix'] = self.seg_prefix
results['proposal_file'] = self.proposal_file
results['img_fields'] = []
results['bbox3d_fields'] = []
results['pts_mask_fields'] = []
results['pts_seg_fields'] = []
results['bbox_fields'] = []
results['mask_fields'] = []
results['seg_fields'] = []
results['box_type_3d'] = self.box_type_3d
results['box_mode_3d'] = self.box_mode_3d
def _parse_ann_info(self, img_info, ann_info):
"""Parse bbox annotation.
Args:
img_info (list[dict]): Image info.
ann_info (list[dict]): Annotation info of an image.
Returns:
dict: A dict containing the following keys: bboxes, labels, \
gt_bboxes_3d, gt_labels_3d, attr_labels, centers2d, \
depths, bboxes_ignore, masks, seg_map
"""
gt_bboxes = []
gt_labels = []
attr_labels = []
gt_bboxes_ignore = []
gt_masks_ann = []
gt_bboxes_cam3d = []
centers2d = []
depths = []
for i, ann in enumerate(ann_info):
if ann.get('ignore', False):
continue
x1, y1, w, h = ann['bbox']
inter_w = max(0, min(x1 + w, img_info['width']) - max(x1, 0))
inter_h = max(0, min(y1 + h, img_info['height']) - max(y1, 0))
if inter_w * inter_h == 0:
continue
if ann['area'] <= 0 or w < 1 or h < 1:
continue
if ann['category_id'] not in self.cat_ids:
continue
bbox = [x1, y1, x1 + w, y1 + h]
if ann.get('iscrowd', False):
gt_bboxes_ignore.append(bbox)
else:
gt_bboxes.append(bbox)
gt_labels.append(self.cat2label[ann['category_id']])
attr_labels.append(ann['attribute_id'])
gt_masks_ann.append(ann.get('segmentation', None))
# 3D annotations in camera coordinates
bbox_cam3d = np.array(ann['bbox_cam3d']).reshape(1, -1)
velo_cam3d = np.array(ann['velo_cam3d']).reshape(1, 2)
nan_mask = np.isnan(velo_cam3d[:, 0])
velo_cam3d[nan_mask] = [0.0, 0.0]
bbox_cam3d = np.concatenate([bbox_cam3d, velo_cam3d], axis=-1)
gt_bboxes_cam3d.append(bbox_cam3d.squeeze())
# 2.5D annotations in camera coordinates
center2d = ann['center2d'][:2]
depth = ann['center2d'][2]
centers2d.append(center2d)
depths.append(depth)
if gt_bboxes:
gt_bboxes = np.array(gt_bboxes, dtype=np.float32)
gt_labels = np.array(gt_labels, dtype=np.int64)
attr_labels = np.array(attr_labels, dtype=np.int64)
else:
gt_bboxes = np.zeros((0, 4), dtype=np.float32)
gt_labels = np.array([], dtype=np.int64)
attr_labels = np.array([], dtype=np.int64)
if gt_bboxes_cam3d:
gt_bboxes_cam3d = np.array(gt_bboxes_cam3d, dtype=np.float32)
centers2d = np.array(centers2d, dtype=np.float32)
depths = np.array(depths, dtype=np.float32)
else:
gt_bboxes_cam3d = np.zeros((0, self.bbox_code_size),
dtype=np.float32)
centers2d = np.zeros((0, 2), dtype=np.float32)
depths = np.zeros((0), dtype=np.float32)
gt_bboxes_cam3d = CameraInstance3DBoxes(
gt_bboxes_cam3d,
box_dim=gt_bboxes_cam3d.shape[-1],
origin=(0.5, 0.5, 0.5))
gt_labels_3d = copy.deepcopy(gt_labels)
if gt_bboxes_ignore:
gt_bboxes_ignore = np.array(gt_bboxes_ignore, dtype=np.float32)
else:
gt_bboxes_ignore = np.zeros((0, 4), dtype=np.float32)
seg_map = img_info['filename'].replace('jpg', 'png')
ann = dict(
bboxes=gt_bboxes,
labels=gt_labels,
gt_bboxes_3d=gt_bboxes_cam3d,
gt_labels_3d=gt_labels_3d,
attr_labels=attr_labels,
centers2d=centers2d,
depths=depths,
bboxes_ignore=gt_bboxes_ignore,
masks=gt_masks_ann,
seg_map=seg_map)
return ann
def get_attr_name(self, attr_idx, label_name):
"""Get attribute from predicted index.
This is a workaround to predict attribute when the predicted velocity
is not reliable. We map the predicted attribute index to the one
in the attribute set. If it is consistent with the category, we will
keep it. Otherwise, we will use the default attribute.
Args:
attr_idx (int): Attribute index.
label_name (str): Predicted category name.
Returns:
str: Predicted attribute name.
"""
# TODO: Simplify the variable name
AttrMapping_rev2 = [
'cycle.with_rider', 'cycle.without_rider', 'pedestrian.moving',
'pedestrian.standing', 'pedestrian.sitting_lying_down',
'vehicle.moving', 'vehicle.parked', 'vehicle.stopped', 'None'
]
if label_name == 'car' or label_name == 'bus' \
or label_name == 'truck' or label_name == 'trailer' \
or label_name == 'construction_vehicle':
if AttrMapping_rev2[attr_idx] == 'vehicle.moving' or \
AttrMapping_rev2[attr_idx] == 'vehicle.parked' or \
AttrMapping_rev2[attr_idx] == 'vehicle.stopped':
return AttrMapping_rev2[attr_idx]
else:
return CustomNuScenesMonoDataset.DefaultAttribute[label_name]
elif label_name == 'pedestrian':
if AttrMapping_rev2[attr_idx] == 'pedestrian.moving' or \
AttrMapping_rev2[attr_idx] == 'pedestrian.standing' or \
AttrMapping_rev2[attr_idx] == \
'pedestrian.sitting_lying_down':
return AttrMapping_rev2[attr_idx]
else:
return CustomNuScenesMonoDataset.DefaultAttribute[label_name]
elif label_name == 'bicycle' or label_name == 'motorcycle':
if AttrMapping_rev2[attr_idx] == 'cycle.with_rider' or \
AttrMapping_rev2[attr_idx] == 'cycle.without_rider':
return AttrMapping_rev2[attr_idx]
else:
return CustomNuScenesMonoDataset.DefaultAttribute[label_name]
else:
return CustomNuScenesMonoDataset.DefaultAttribute[label_name]
def _format_bbox(self, results, jsonfile_prefix=None):
"""Convert the results to the standard format.
Args:
results (list[dict]): Testing results of the dataset.
jsonfile_prefix (str): The prefix of the output jsonfile.
You can specify the output directory/filename by
modifying the jsonfile_prefix. Default: None.
Returns:
str: Path of the output json file.
"""
nusc_annos = {}
mapped_class_names = self.CLASSES
print('Start to convert detection format...')
CAM_NUM = 6
for sample_id, det in enumerate(mmcv.track_iter_progress(results)):
if sample_id % CAM_NUM == 0:
boxes_per_frame = []
attrs_per_frame = []
# need to merge results from images of the same sample
annos = []
boxes, attrs = output_to_nusc_box(det)
sample_token = self.data_infos[sample_id]['token']
boxes, attrs = cam_nusc_box_to_global(self.data_infos[sample_id],
boxes, attrs,
mapped_class_names,
self.eval_detection_configs,
self.eval_version)
boxes_per_frame.extend(boxes)
attrs_per_frame.extend(attrs)
# Remove redundant predictions caused by overlap of images
if (sample_id + 1) % CAM_NUM != 0:
continue
boxes = global_nusc_box_to_cam(
self.data_infos[sample_id + 1 - CAM_NUM], boxes_per_frame,
mapped_class_names, self.eval_detection_configs,
self.eval_version)
cam_boxes3d, scores, labels = nusc_box_to_cam_box3d(boxes)
# box nms 3d over 6 images in a frame
# TODO: move this global setting into config
nms_cfg = dict(
use_rotate_nms=True,
nms_across_levels=False,
nms_pre=4096,
nms_thr=0.05,
score_thr=0.01,
min_bbox_size=0,
max_per_frame=500)
from mmcv import Config
nms_cfg = Config(nms_cfg)
cam_boxes3d_for_nms = xywhr2xyxyr(cam_boxes3d.bev)
boxes3d = cam_boxes3d.tensor
# generate attr scores from attr labels
attrs = labels.new_tensor([attr for attr in attrs_per_frame])
boxes3d, scores, labels, attrs = box3d_multiclass_nms(
boxes3d,
cam_boxes3d_for_nms,
scores,
nms_cfg.score_thr,
nms_cfg.max_per_frame,
nms_cfg,
mlvl_attr_scores=attrs)
cam_boxes3d = CameraInstance3DBoxes(boxes3d, box_dim=9)
det = bbox3d2result(cam_boxes3d, scores, labels, attrs)
boxes, attrs = output_to_nusc_box(det)
boxes, attrs = cam_nusc_box_to_global(
self.data_infos[sample_id + 1 - CAM_NUM], boxes, attrs,
mapped_class_names, self.eval_detection_configs,
self.eval_version)
for i, box in enumerate(boxes):
name = mapped_class_names[box.label]
attr = self.get_attr_name(attrs[i], name)
nusc_anno = dict(
sample_token=sample_token,
translation=box.center.tolist(),
size=box.wlh.tolist(),
rotation=box.orientation.elements.tolist(),
velocity=box.velocity[:2].tolist(),
detection_name=name,
detection_score=box.score,
attribute_name=attr)
annos.append(nusc_anno)
# other views results of the same frame should be concatenated
if sample_token in nusc_annos:
nusc_annos[sample_token].extend(annos)
else:
nusc_annos[sample_token] = annos
nusc_submissions = {
'meta': self.modality,
'results': nusc_annos,
}
mmcv.mkdir_or_exist(jsonfile_prefix)
res_path = osp.join(jsonfile_prefix, 'results_nusc.json')
print('Results writes to', res_path)
mmcv.dump(nusc_submissions, res_path)
return res_path
def _evaluate_single(self,
result_path,
logger=None,
metric='bbox',
result_name='img_bbox'):
"""Evaluation for a single model in nuScenes protocol.
Args:
result_path (str): Path of the result file.
logger (logging.Logger | str | None): Logger used for printing
related information during evaluation. Default: None.
metric (str): Metric name used for evaluation. Default: 'bbox'.
result_name (str): Result name in the metric prefix.
Default: 'img_bbox'.
Returns:
dict: Dictionary of evaluation details.
"""
from nuscenes import NuScenes
#from nuscenes.eval.detection.evaluate import NuScenesEval
from .nuscnes_eval import NuScenesEval_custom
output_dir = osp.join(*osp.split(result_path)[:-1])
self.nusc = NuScenes(
version=self.version, dataroot=self.data_root, verbose=False)
eval_set_map = {
'v1.0-mini': 'mini_val',
'v1.0-trainval': 'val',
}
# nusc_eval = NuScenesEval(
# nusc,
# config=self.eval_detection_configs,
# result_path=result_path,
# eval_set=eval_set_map[self.version],
# output_dir=output_dir,
# verbose=False)
self.nusc_eval = NuScenesEval_custom(
self.nusc,
config=self.eval_detection_configs,
result_path=result_path,
eval_set=eval_set_map[self.version],
output_dir=output_dir,
verbose=True,
overlap_test=self.overlap_test,
data_infos=self.data_infos
)
self.nusc_eval.main(render_curves=True)
# record metrics
metrics = mmcv.load(osp.join(output_dir, 'metrics_summary.json'))
detail = dict()
metric_prefix = f'{result_name}_NuScenes'
for name in self.CLASSES:
for k, v in metrics['label_aps'][name].items():
val = float('{:.4f}'.format(v))
detail['{}/{}_AP_dist_{}'.format(metric_prefix, name, k)] = val
for k, v in metrics['label_tp_errors'][name].items():
val = float('{:.4f}'.format(v))
detail['{}/{}_{}'.format(metric_prefix, name, k)] = val
for k, v in metrics['tp_errors'].items():
val = float('{:.4f}'.format(v))
detail['{}/{}'.format(metric_prefix,
self.ErrNameMapping[k])] = val
detail['{}/NDS'.format(metric_prefix)] = metrics['nd_score']
detail['{}/mAP'.format(metric_prefix)] = metrics['mean_ap']
return detail
def format_results(self, results, jsonfile_prefix=None, **kwargs):
"""Format the results to json (standard format for COCO evaluation).
Args:
results (list[tuple | numpy.ndarray]): Testing results of the
dataset.
jsonfile_prefix (str | None): The prefix of json files. It includes
the file path and the prefix of filename, e.g., "a/b/prefix".
If not specified, a temp file will be created. Default: None.
Returns:
tuple: (result_files, tmp_dir), result_files is a dict containing \
the json filepaths, tmp_dir is the temporal directory created \
for saving json files when jsonfile_prefix is not specified.
"""
assert isinstance(results, list), 'results must be a list'
assert len(results) == len(self), (
'The length of results is not equal to the dataset len: {} != {}'.
format(len(results), len(self)))
if jsonfile_prefix is None:
tmp_dir = tempfile.TemporaryDirectory()
jsonfile_prefix = osp.join(tmp_dir.name, 'results')
else:
tmp_dir = None
# currently the output prediction results could be in two formats
# 1. list of dict('boxes_3d': ..., 'scores_3d': ..., 'labels_3d': ...)
# 2. list of dict('pts_bbox' or 'img_bbox':
# dict('boxes_3d': ..., 'scores_3d': ..., 'labels_3d': ...))
# this is a workaround to enable evaluation of both formats on nuScenes
# refer to https://github.com/open-mmlab/mmdetection3d/issues/449
if not ('pts_bbox' in results[0] or 'img_bbox' in results[0]):
result_files = self._format_bbox(results, jsonfile_prefix)
else:
# should take the inner dict out of 'pts_bbox' or 'img_bbox' dict
result_files = dict()
for name in results[0]:
# not evaluate 2D predictions on nuScenes
if '2d' in name:
continue
print(f'\nFormating bboxes of {name}')
results_ = [out[name] for out in results]
tmp_file_ = osp.join(jsonfile_prefix, name)
result_files.update(
{name: self._format_bbox(results_, tmp_file_)})
return result_files, tmp_dir
def evaluate(self,
results,
metric='bbox',
logger=None,
jsonfile_prefix=None,
result_names=['img_bbox'],
show=False,
out_dir=None,
pipeline=None):
"""Evaluation in nuScenes protocol.
Args:
results (list[dict]): Testing results of the dataset.
metric (str | list[str]): Metrics to be evaluated.
logger (logging.Logger | str | None): Logger used for printing
related information during evaluation. Default: None.
jsonfile_prefix (str | None): The prefix of json files. It includes
the file path and the prefix of filename, e.g., "a/b/prefix".
If not specified, a temp file will be created. Default: None.
show (bool): Whether to visualize.
Default: False.
out_dir (str): Path to save the visualization results.
Default: None.
pipeline (list[dict], optional): raw data loading for showing.
Default: None.
Returns:
dict[str, float]: Results of each evaluation metric.
"""
result_files, tmp_dir = self.format_results(results, jsonfile_prefix)
if isinstance(result_files, dict):
results_dict = dict()
for name in result_names:
print('Evaluating bboxes of {}'.format(name))
ret_dict = self._evaluate_single(result_files[name])
results_dict.update(ret_dict)
elif isinstance(result_files, str):
results_dict = self._evaluate_single(result_files)
if tmp_dir is not None:
tmp_dir.cleanup()
if show:
self.show(results, out_dir, pipeline=pipeline)
return results_dict
def _extract_data(self, index, pipeline, key, load_annos=False):
"""Load data using input pipeline and extract data according to key.
Args:
index (int): Index for accessing the target data.
pipeline (:obj:`Compose`): Composed data loading pipeline.
key (str | list[str]): One single or a list of data key.
load_annos (bool): Whether to load data annotations.
If True, need to set self.test_mode as False before loading.
Returns:
np.ndarray | torch.Tensor | list[np.ndarray | torch.Tensor]:
A single or a list of loaded data.
"""
assert pipeline is not None, 'data loading pipeline is not provided'
img_info = self.data_infos[index]
input_dict = dict(img_info=img_info)
if load_annos:
ann_info = self.get_ann_info(index)
input_dict.update(dict(ann_info=ann_info))
self.pre_pipeline(input_dict)
example = pipeline(input_dict)
# extract data items according to keys
if isinstance(key, str):
data = extract_result_dict(example, key)
else:
data = [extract_result_dict(example, k) for k in key]
return data
def _get_pipeline(self, pipeline):
"""Get data loading pipeline in self.show/evaluate function.
Args:
pipeline (list[dict] | None): Input pipeline. If None is given, \
get from self.pipeline.
"""
if pipeline is None:
if not hasattr(self, 'pipeline') or self.pipeline is None:
warnings.warn(
'Use default pipeline for data loading, this may cause '
'errors when data is on ceph')
return self._build_default_pipeline()
loading_pipeline = get_loading_pipeline(self.pipeline.transforms)
return Compose(loading_pipeline)
return Compose(pipeline)
def _build_default_pipeline(self):
"""Build the default pipeline for this dataset."""
pipeline = [
dict(type='LoadImageFromFileMono3D'),
dict(
type='DefaultFormatBundle3D',
class_names=self.CLASSES,
with_label=False),
dict(type='Collect3D', keys=['img'])
]
return Compose(pipeline)
def show(self, results, out_dir, show=True, pipeline=None):
"""Results visualization.
Args:
results (list[dict]): List of bounding boxes results.
out_dir (str): Output directory of visualization result.
show (bool): Visualize the results online.
pipeline (list[dict], optional): raw data loading for showing.
Default: None.
"""
assert out_dir is not None, 'Expect out_dir, got none.'
pipeline = self._get_pipeline(pipeline)
for i, result in enumerate(results):
if 'img_bbox' in result.keys():
result = result['img_bbox']
data_info = self.data_infos[i]
img_path = data_info['file_name']
file_name = osp.split(img_path)[-1].split('.')[0]
img, img_metas = self._extract_data(i, pipeline,
['img', 'img_metas'])
# need to transpose channel to first dim
img = img.numpy().transpose(1, 2, 0)
gt_bboxes = self.get_ann_info(i)['gt_bboxes_3d']
pred_bboxes = result['boxes_3d']
show_multi_modality_result(
img,
gt_bboxes,
pred_bboxes,
img_metas['cam2img'],
out_dir,
file_name,
box_mode='camera',
show=show)
def output_to_nusc_box(detection):
"""Convert the output to the box class in the nuScenes.
Args:
detection (dict): Detection results.
- boxes_3d (:obj:`BaseInstance3DBoxes`): Detection bbox.
- scores_3d (torch.Tensor): Detection scores.
- labels_3d (torch.Tensor): Predicted box labels.
- attrs_3d (torch.Tensor, optional): Predicted attributes.
Returns:
list[:obj:`NuScenesBox`]: List of standard NuScenesBoxes.
"""
box3d = detection['boxes_3d']
scores = detection['scores_3d'].numpy()
labels = detection['labels_3d'].numpy()
attrs = None
if 'attrs_3d' in detection:
attrs = detection['attrs_3d'].numpy()
box_gravity_center = box3d.gravity_center.numpy()
box_dims = box3d.dims.numpy()
box_yaw = box3d.yaw.numpy()
# convert the dim/rot to nuscbox convention
box_dims[:, [0, 1, 2]] = box_dims[:, [2, 0, 1]]
box_yaw = -box_yaw
box_list = []
for i in range(len(box3d)):
q1 = pyquaternion.Quaternion(axis=[0, 0, 1], radians=box_yaw[i])
q2 = pyquaternion.Quaternion(axis=[1, 0, 0], radians=np.pi / 2)
quat = q2 * q1
velocity = (box3d.tensor[i, 7], 0.0, box3d.tensor[i, 8])
box = NuScenesBox(
box_gravity_center[i],
box_dims[i],
quat,
label=labels[i],
score=scores[i],
velocity=velocity)
box_list.append(box)
return box_list, attrs
def cam_nusc_box_to_global(info,
boxes,
attrs,
classes,
eval_configs,
eval_version='detection_cvpr_2019'):
"""Convert the box from camera to global coordinate.
Args:
info (dict): Info for a specific sample data, including the
calibration information.
boxes (list[:obj:`NuScenesBox`]): List of predicted NuScenesBoxes.
classes (list[str]): Mapped classes in the evaluation.
eval_configs (object): Evaluation configuration object.
eval_version (str): Evaluation version.
Default: 'detection_cvpr_2019'
Returns:
list: List of standard NuScenesBoxes in the global
coordinate.
"""
box_list = []
attr_list = []
for (box, attr) in zip(boxes, attrs):
# Move box to ego vehicle coord system
box.rotate(pyquaternion.Quaternion(info['cam2ego_rotation']))
box.translate(np.array(info['cam2ego_translation']))
# filter det in ego.
cls_range_map = eval_configs.class_range
radius = np.linalg.norm(box.center[:2], 2)
det_range = cls_range_map[classes[box.label]]
if radius > det_range:
continue
# Move box to global coord system
box.rotate(pyquaternion.Quaternion(info['ego2global_rotation']))
box.translate(np.array(info['ego2global_translation']))
box_list.append(box)
attr_list.append(attr)
return box_list, attr_list
def global_nusc_box_to_cam(info,
boxes,
classes,
eval_configs,
eval_version='detection_cvpr_2019'):
"""Convert the box from global to camera coordinate.
Args:
info (dict): Info for a specific sample data, including the
calibration information.
boxes (list[:obj:`NuScenesBox`]): List of predicted NuScenesBoxes.
classes (list[str]): Mapped classes in the evaluation.
eval_configs (object): Evaluation configuration object.
eval_version (str): Evaluation version.
Default: 'detection_cvpr_2019'
Returns:
list: List of standard NuScenesBoxes in the global
coordinate.
"""
box_list = []
for box in boxes:
# Move box to ego vehicle coord system
box.translate(-np.array(info['ego2global_translation']))
box.rotate(
pyquaternion.Quaternion(info['ego2global_rotation']).inverse)
# filter det in ego.
cls_range_map = eval_configs.class_range
radius = np.linalg.norm(box.center[:2], 2)
det_range = cls_range_map[classes[box.label]]
if radius > det_range:
continue
# Move box to camera coord system
box.translate(-np.array(info['cam2ego_translation']))
box.rotate(pyquaternion.Quaternion(info['cam2ego_rotation']).inverse)
box_list.append(box)
return box_list
def nusc_box_to_cam_box3d(boxes):
"""Convert boxes from :obj:`NuScenesBox` to :obj:`CameraInstance3DBoxes`.
Args:
boxes (list[:obj:`NuScenesBox`]): List of predicted NuScenesBoxes.
Returns:
tuple (:obj:`CameraInstance3DBoxes` | torch.Tensor | torch.Tensor): \
Converted 3D bounding boxes, scores and labels.
"""
locs = torch.Tensor([b.center for b in boxes]).view(-1, 3)
dims = torch.Tensor([b.wlh for b in boxes]).view(-1, 3)
rots = torch.Tensor([b.orientation.yaw_pitch_roll[0]
for b in boxes]).view(-1, 1)
velocity = torch.Tensor([b.velocity[:2] for b in boxes]).view(-1, 2)
# convert nusbox to cambox convention
dims[:, [0, 1, 2]] = dims[:, [1, 2, 0]]
rots = -rots
boxes_3d = torch.cat([locs, dims, rots, velocity], dim=1).cuda()
cam_boxes3d = CameraInstance3DBoxes(
boxes_3d, box_dim=9, origin=(0.5, 0.5, 0.5))
scores = torch.Tensor([b.score for b in boxes]).cuda()
labels = torch.LongTensor([b.label for b in boxes]).cuda()
nms_scores = scores.new_zeros(scores.shape[0], 10 + 1)
indices = labels.new_tensor(list(range(scores.shape[0])))
nms_scores[indices, labels] = scores
return cam_boxes3d, nms_scores, labels
\ No newline at end of file
docker-hub/MapTRv2/MapTR/projects/mmdet3d_plugin/datasets/nuscenes_offlinemap_dataset.py 0 → 100644
View file @ 19472568
import copy
import numpy as np
from mmdet.datasets import DATASETS
from mmdet3d.datasets import NuScenesDataset
import mmcv
import os
from os import path as osp
from mmdet.datasets import DATASETS
import torch
import numpy as np
from nuscenes.eval.common.utils import quaternion_yaw, Quaternion
from .nuscnes_eval import NuScenesEval_custom
from projects.mmdet3d_plugin.models.utils.visual import save_tensor
from mmcv.parallel import DataContainer as DC
import random
from .nuscenes_dataset import CustomNuScenesDataset
from nuscenes.map_expansion.map_api import NuScenesMap, NuScenesMapExplorer
from nuscenes.eval.common.utils import quaternion_yaw, Quaternion
from shapely import affinity, ops
from shapely.geometry import LineString, box, MultiPolygon, MultiLineString
from mmdet.datasets.pipelines import to_tensor
import json
import cv2
def add_rotation_noise(extrinsics, std=0.01, mean=0.0):
#n = extrinsics.shape[0]
noise_angle = torch.normal(mean, std=std, size=(3,))
# extrinsics[:, 0:3, 0:3] *= (1 + noise)
sin_noise = torch.sin(noise_angle)
cos_noise = torch.cos(noise_angle)
rotation_matrix = torch.eye(4).view(4, 4)
# rotation_matrix[]
rotation_matrix_x = rotation_matrix.clone()
rotation_matrix_x[1, 1] = cos_noise[0]
rotation_matrix_x[1, 2] = sin_noise[0]
rotation_matrix_x[2, 1] = -sin_noise[0]
rotation_matrix_x[2, 2] = cos_noise[0]
rotation_matrix_y = rotation_matrix.clone()
rotation_matrix_y[0, 0] = cos_noise[1]
rotation_matrix_y[0, 2] = -sin_noise[1]
rotation_matrix_y[2, 0] = sin_noise[1]
rotation_matrix_y[2, 2] = cos_noise[1]
rotation_matrix_z = rotation_matrix.clone()
rotation_matrix_z[0, 0] = cos_noise[2]
rotation_matrix_z[0, 1] = sin_noise[2]
rotation_matrix_z[1, 0] = -sin_noise[2]
rotation_matrix_z[1, 1] = cos_noise[2]
rotation_matrix = rotation_matrix_x @ rotation_matrix_y @ rotation_matrix_z
rotation = torch.from_numpy(extrinsics.astype(np.float32))
rotation[:3, -1] = 0.0
# import pdb;pdb.set_trace()
rotation = rotation_matrix @ rotation
extrinsics[:3, :3] = rotation[:3, :3].numpy()
return extrinsics
def add_translation_noise(extrinsics, std=0.01, mean=0.0):
# n = extrinsics.shape[0]
noise = torch.normal(mean, std=std, size=(3,))
extrinsics[0:3, -1] += noise.numpy()
return extrinsics
def perspective(cam_coords, proj_mat):
pix_coords = proj_mat @ cam_coords
valid_idx = pix_coords[2, :] > 0
pix_coords = pix_coords[:, valid_idx]
pix_coords = pix_coords[:2, :] / (pix_coords[2, :] + 1e-7)
pix_coords = pix_coords.transpose(1, 0)
return pix_coords
class LiDARInstanceLines(object):
"""Line instance in LIDAR coordinates
"""
def __init__(self,
instance_line_list,
instance_labels,
sample_dist=1,
num_samples=250,
padding=False,
fixed_num=-1,
padding_value=-10000,
patch_size=None):
assert isinstance(instance_line_list, list)
assert patch_size is not None
if len(instance_line_list) != 0:
assert isinstance(instance_line_list[0], LineString)
self.patch_size = patch_size
self.max_x = self.patch_size[1] / 2
self.max_y = self.patch_size[0] / 2
self.sample_dist = sample_dist
self.num_samples = num_samples
self.padding = padding
self.fixed_num = fixed_num
self.padding_value = padding_value
self.instance_list = instance_line_list
self.instance_labels = instance_labels
@property
def start_end_points(self):
"""
return torch.Tensor([N,4]), in xstart, ystart, xend, yend form
"""
assert len(self.instance_list) != 0
instance_se_points_list = []
for instance in self.instance_list:
se_points = []
se_points.extend(instance.coords[0])
se_points.extend(instance.coords[-1])
instance_se_points_list.append(se_points)
instance_se_points_array = np.array(instance_se_points_list)
instance_se_points_tensor = to_tensor(instance_se_points_array)
instance_se_points_tensor = instance_se_points_tensor.to(
dtype=torch.float32)
instance_se_points_tensor[:,0] = torch.clamp(instance_se_points_tensor[:,0], min=-self.max_x,max=self.max_x)
instance_se_points_tensor[:,1] = torch.clamp(instance_se_points_tensor[:,1], min=-self.max_y,max=self.max_y)
instance_se_points_tensor[:,2] = torch.clamp(instance_se_points_tensor[:,2], min=-self.max_x,max=self.max_x)
instance_se_points_tensor[:,3] = torch.clamp(instance_se_points_tensor[:,3], min=-self.max_y,max=self.max_y)
return instance_se_points_tensor
@property
def bbox(self):
"""
return torch.Tensor([N,4]), in xmin, ymin, xmax, ymax form
"""
assert len(self.instance_list) != 0
instance_bbox_list = []
for instance in self.instance_list:
# bounds is bbox: [xmin, ymin, xmax, ymax]
instance_bbox_list.append(instance.bounds)
instance_bbox_array = np.array(instance_bbox_list)
instance_bbox_tensor = to_tensor(instance_bbox_array)
instance_bbox_tensor = instance_bbox_tensor.to(
dtype=torch.float32)
instance_bbox_tensor[:,0] = torch.clamp(instance_bbox_tensor[:,0], min=-self.max_x,max=self.max_x)
instance_bbox_tensor[:,1] = torch.clamp(instance_bbox_tensor[:,1], min=-self.max_y,max=self.max_y)
instance_bbox_tensor[:,2] = torch.clamp(instance_bbox_tensor[:,2], min=-self.max_x,max=self.max_x)
instance_bbox_tensor[:,3] = torch.clamp(instance_bbox_tensor[:,3], min=-self.max_y,max=self.max_y)
# print("instance_bbox_tensor:", instance_bbox_tensor.shape)
return instance_bbox_tensor
@property
def fixed_num_sampled_points(self):
"""
return torch.Tensor([N,fixed_num,2]), in xmin, ymin, xmax, ymax form
N means the num of instances
"""
assert len(self.instance_list) != 0
instance_points_list = []
for instance in self.instance_list:
distances = np.linspace(0, instance.length, self.fixed_num)
sampled_points = np.array([list(instance.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
instance_points_list.append(sampled_points)
instance_points_array = np.array(instance_points_list)
instance_points_tensor = to_tensor(instance_points_array)
instance_points_tensor = instance_points_tensor.to(
dtype=torch.float32)
instance_points_tensor[:,:,0] = torch.clamp(instance_points_tensor[:,:,0], min=-self.max_x,max=self.max_x)
instance_points_tensor[:,:,1] = torch.clamp(instance_points_tensor[:,:,1], min=-self.max_y,max=self.max_y)
return instance_points_tensor
@property
def fixed_num_sampled_points_ambiguity(self):
"""
return torch.Tensor([N,fixed_num,2]), in xmin, ymin, xmax, ymax form
N means the num of instances
"""
assert len(self.instance_list) != 0
instance_points_list = []
for instance in self.instance_list:
distances = np.linspace(0, instance.length, self.fixed_num)
sampled_points = np.array([list(instance.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
instance_points_list.append(sampled_points)
instance_points_array = np.array(instance_points_list)
instance_points_tensor = to_tensor(instance_points_array)
instance_points_tensor = instance_points_tensor.to(
dtype=torch.float32)
instance_points_tensor[:,:,0] = torch.clamp(instance_points_tensor[:,:,0], min=-self.max_x,max=self.max_x)
instance_points_tensor[:,:,1] = torch.clamp(instance_points_tensor[:,:,1], min=-self.max_y,max=self.max_y)
instance_points_tensor = instance_points_tensor.unsqueeze(1)
return instance_points_tensor
@property
def fixed_num_sampled_points_torch(self):
"""
return torch.Tensor([N,fixed_num,2]), in xmin, ymin, xmax, ymax form
N means the num of instances
"""
assert len(self.instance_list) != 0
instance_points_list = []
for instance in self.instance_list:
# distances = np.linspace(0, instance.length, self.fixed_num)
# sampled_points = np.array([list(instance.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
poly_pts = to_tensor(np.array(list(instance.coords)))
poly_pts = poly_pts.unsqueeze(0).permute(0,2,1)
sampled_pts = torch.nn.functional.interpolate(poly_pts,size=(self.fixed_num),mode='linear',align_corners=True)
sampled_pts = sampled_pts.permute(0,2,1).squeeze(0)
instance_points_list.append(sampled_pts)
# instance_points_array = np.array(instance_points_list)
# instance_points_tensor = to_tensor(instance_points_array)
instance_points_tensor = torch.stack(instance_points_list,dim=0)
instance_points_tensor = instance_points_tensor.to(
dtype=torch.float32)
instance_points_tensor[:,:,0] = torch.clamp(instance_points_tensor[:,:,0], min=-self.max_x,max=self.max_x)
instance_points_tensor[:,:,1] = torch.clamp(instance_points_tensor[:,:,1], min=-self.max_y,max=self.max_y)
return instance_points_tensor
@property
def shift_fixed_num_sampled_points(self):
"""
return [instances_num, num_shifts, fixed_num, 2]
"""
assert len(self.instance_list) != 0
instances_list = []
for instance in self.instance_list:
distances = np.linspace(0, instance.length, self.fixed_num)
poly_pts = np.array(list(instance.coords))
start_pts = poly_pts[0]
end_pts = poly_pts[-1]
is_poly = np.equal(start_pts, end_pts)
is_poly = is_poly.all()
shift_pts_list = []
pts_num, coords_num = poly_pts.shape
shift_num = pts_num - 1
final_shift_num = self.fixed_num - 1
sampled_points = np.array([list(instance.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
shift_pts_list.append(sampled_points)
# if is_poly:
# pts_to_shift = poly_pts[:-1,:]
# for shift_right_i in range(shift_num):
# shift_pts = np.roll(pts_to_shift,shift_right_i,axis=0)
# pts_to_concat = shift_pts[0]
# pts_to_concat = np.expand_dims(pts_to_concat,axis=0)
# shift_pts = np.concatenate((shift_pts,pts_to_concat),axis=0)
# shift_instance = LineString(shift_pts)
# shift_sampled_points = np.array([list(shift_instance.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
# shift_pts_list.append(shift_sampled_points)
# # import pdb;pdb.set_trace()
# else:
# sampled_points = np.array([list(instance.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
# flip_sampled_points = np.flip(sampled_points, axis=0)
# shift_pts_list.append(sampled_points)
# shift_pts_list.append(flip_sampled_points)
multi_shifts_pts = np.stack(shift_pts_list,axis=0)
shifts_num,_,_ = multi_shifts_pts.shape
if shifts_num > final_shift_num:
index = np.random.choice(multi_shifts_pts.shape[0], final_shift_num, replace=False)
multi_shifts_pts = multi_shifts_pts[index]
multi_shifts_pts_tensor = to_tensor(multi_shifts_pts)
multi_shifts_pts_tensor = multi_shifts_pts_tensor.to(
dtype=torch.float32)
multi_shifts_pts_tensor[:,:,0] = torch.clamp(multi_shifts_pts_tensor[:,:,0], min=-self.max_x,max=self.max_x)
multi_shifts_pts_tensor[:,:,1] = torch.clamp(multi_shifts_pts_tensor[:,:,1], min=-self.max_y,max=self.max_y)
# if not is_poly:
if multi_shifts_pts_tensor.shape[0] < final_shift_num:
padding = torch.full([final_shift_num-multi_shifts_pts_tensor.shape[0],self.fixed_num,2], self.padding_value)
multi_shifts_pts_tensor = torch.cat([multi_shifts_pts_tensor,padding],dim=0)
instances_list.append(multi_shifts_pts_tensor)
instances_tensor = torch.stack(instances_list, dim=0)
instances_tensor = instances_tensor.to(
dtype=torch.float32)
return instances_tensor
@property
def shift_fixed_num_sampled_points_v1(self):
"""
return [instances_num, num_shifts, fixed_num, 2]
"""
fixed_num_sampled_points = self.fixed_num_sampled_points
instances_list = []
is_poly = False
# is_line = False
# import pdb;pdb.set_trace()
for fixed_num_pts in fixed_num_sampled_points:
# [fixed_num, 2]
is_poly = fixed_num_pts[0].equal(fixed_num_pts[-1])
pts_num = fixed_num_pts.shape[0]
shift_num = pts_num - 1
if is_poly:
pts_to_shift = fixed_num_pts[:-1,:]
shift_pts_list = []
if is_poly:
for shift_right_i in range(shift_num):
shift_pts_list.append(pts_to_shift.roll(shift_right_i,0))
else:
shift_pts_list.append(fixed_num_pts)
shift_pts_list.append(fixed_num_pts.flip(0))
shift_pts = torch.stack(shift_pts_list,dim=0)
if is_poly:
_, _, num_coords = shift_pts.shape
tmp_shift_pts = shift_pts.new_zeros((shift_num, pts_num, num_coords))
tmp_shift_pts[:,:-1,:] = shift_pts
tmp_shift_pts[:,-1,:] = shift_pts[:,0,:]
shift_pts = tmp_shift_pts
shift_pts[:,:,0] = torch.clamp(shift_pts[:,:,0], min=-self.max_x,max=self.max_x)
shift_pts[:,:,1] = torch.clamp(shift_pts[:,:,1], min=-self.max_y,max=self.max_y)
if not is_poly:
padding = torch.full([shift_num-shift_pts.shape[0],pts_num,2], self.padding_value)
shift_pts = torch.cat([shift_pts,padding],dim=0)
# padding = np.zeros((self.num_samples - len(sampled_points), 2))
# sampled_points = np.concatenate([sampled_points, padding], axis=0)
instances_list.append(shift_pts)
instances_tensor = torch.stack(instances_list, dim=0)
instances_tensor = instances_tensor.to(
dtype=torch.float32)
return instances_tensor
@property
def shift_fixed_num_sampled_points_v2(self):
"""
return [instances_num, num_shifts, fixed_num, 2]
"""
assert len(self.instance_list) != 0
instances_list = []
for idx, instance in enumerate(self.instance_list):
# import ipdb;ipdb.set_trace()
instance_label = self.instance_labels[idx]
distances = np.linspace(0, instance.length, self.fixed_num)
poly_pts = np.array(list(instance.coords))
start_pts = poly_pts[0]
end_pts = poly_pts[-1]
is_poly = np.equal(start_pts, end_pts)
is_poly = is_poly.all()
shift_pts_list = []
pts_num, coords_num = poly_pts.shape
shift_num = pts_num - 1
final_shift_num = self.fixed_num - 1
if instance_label == 3:
# import ipdb;ipdb.set_trace()
sampled_points = np.array([list(instance.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
shift_pts_list.append(sampled_points)
else:
if is_poly:
pts_to_shift = poly_pts[:-1,:]
for shift_right_i in range(shift_num):
shift_pts = np.roll(pts_to_shift,shift_right_i,axis=0)
pts_to_concat = shift_pts[0]
pts_to_concat = np.expand_dims(pts_to_concat,axis=0)
shift_pts = np.concatenate((shift_pts,pts_to_concat),axis=0)
shift_instance = LineString(shift_pts)
shift_sampled_points = np.array([list(shift_instance.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
shift_pts_list.append(shift_sampled_points)
# import pdb;pdb.set_trace()
else:
sampled_points = np.array([list(instance.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
flip_sampled_points = np.flip(sampled_points, axis=0)
shift_pts_list.append(sampled_points)
shift_pts_list.append(flip_sampled_points)
multi_shifts_pts = np.stack(shift_pts_list,axis=0)
shifts_num,_,_ = multi_shifts_pts.shape
if shifts_num > final_shift_num:
index = np.random.choice(multi_shifts_pts.shape[0], final_shift_num, replace=False)
multi_shifts_pts = multi_shifts_pts[index]
multi_shifts_pts_tensor = to_tensor(multi_shifts_pts)
multi_shifts_pts_tensor = multi_shifts_pts_tensor.to(
dtype=torch.float32)
multi_shifts_pts_tensor[:,:,0] = torch.clamp(multi_shifts_pts_tensor[:,:,0], min=-self.max_x,max=self.max_x)
multi_shifts_pts_tensor[:,:,1] = torch.clamp(multi_shifts_pts_tensor[:,:,1], min=-self.max_y,max=self.max_y)
# if not is_poly:
if multi_shifts_pts_tensor.shape[0] < final_shift_num:
padding = torch.full([final_shift_num-multi_shifts_pts_tensor.shape[0],self.fixed_num,2], self.padding_value)
multi_shifts_pts_tensor = torch.cat([multi_shifts_pts_tensor,padding],dim=0)
instances_list.append(multi_shifts_pts_tensor)
instances_tensor = torch.stack(instances_list, dim=0)
instances_tensor = instances_tensor.to(
dtype=torch.float32)
# print("instances_tensor:", instances_tensor.shape)
return instances_tensor
@property
def shift_fixed_num_sampled_points_v3(self):
"""
return [instances_num, num_shifts, fixed_num, 2]
"""
assert len(self.instance_list) != 0
instances_list = []
for instance in self.instance_list:
distances = np.linspace(0, instance.length, self.fixed_num)
poly_pts = np.array(list(instance.coords))
start_pts = poly_pts[0]
end_pts = poly_pts[-1]
is_poly = np.equal(start_pts, end_pts)
is_poly = is_poly.all()
shift_pts_list = []
pts_num, coords_num = poly_pts.shape
shift_num = pts_num - 1
final_shift_num = self.fixed_num - 1
if is_poly:
pts_to_shift = poly_pts[:-1,:]
for shift_right_i in range(shift_num):
shift_pts = np.roll(pts_to_shift,shift_right_i,axis=0)
pts_to_concat = shift_pts[0]
pts_to_concat = np.expand_dims(pts_to_concat,axis=0)
shift_pts = np.concatenate((shift_pts,pts_to_concat),axis=0)
shift_instance = LineString(shift_pts)
shift_sampled_points = np.array([list(shift_instance.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
shift_pts_list.append(shift_sampled_points)
flip_pts_to_shift = np.flip(pts_to_shift, axis=0)
for shift_right_i in range(shift_num):
shift_pts = np.roll(flip_pts_to_shift,shift_right_i,axis=0)
pts_to_concat = shift_pts[0]
pts_to_concat = np.expand_dims(pts_to_concat,axis=0)
shift_pts = np.concatenate((shift_pts,pts_to_concat),axis=0)
shift_instance = LineString(shift_pts)
shift_sampled_points = np.array([list(shift_instance.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
shift_pts_list.append(shift_sampled_points)
else:
sampled_points = np.array([list(instance.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
flip_sampled_points = np.flip(sampled_points, axis=0)
shift_pts_list.append(sampled_points)
shift_pts_list.append(flip_sampled_points)
multi_shifts_pts = np.stack(shift_pts_list,axis=0)
shifts_num,_,_ = multi_shifts_pts.shape
if shifts_num > 2*final_shift_num:
index = np.random.choice(shift_num, final_shift_num, replace=False)
flip0_shifts_pts = multi_shifts_pts[index]
flip1_shifts_pts = multi_shifts_pts[index+shift_num]
multi_shifts_pts = np.concatenate((flip0_shifts_pts,flip1_shifts_pts),axis=0)
multi_shifts_pts_tensor = to_tensor(multi_shifts_pts)
multi_shifts_pts_tensor = multi_shifts_pts_tensor.to(
dtype=torch.float32)
multi_shifts_pts_tensor[:,:,0] = torch.clamp(multi_shifts_pts_tensor[:,:,0], min=-self.max_x,max=self.max_x)
multi_shifts_pts_tensor[:,:,1] = torch.clamp(multi_shifts_pts_tensor[:,:,1], min=-self.max_y,max=self.max_y)
if multi_shifts_pts_tensor.shape[0] < 2*final_shift_num:
padding = torch.full([final_shift_num*2-multi_shifts_pts_tensor.shape[0],self.fixed_num,2], self.padding_value)
multi_shifts_pts_tensor = torch.cat([multi_shifts_pts_tensor,padding],dim=0)
instances_list.append(multi_shifts_pts_tensor)
instances_tensor = torch.stack(instances_list, dim=0)
instances_tensor = instances_tensor.to(
dtype=torch.float32)
return instances_tensor
@property
def shift_fixed_num_sampled_points_v4(self):
"""
return [instances_num, num_shifts, fixed_num, 2]
"""
fixed_num_sampled_points = self.fixed_num_sampled_points
instances_list = []
is_poly = False
for fixed_num_pts in fixed_num_sampled_points:
is_poly = fixed_num_pts[0].equal(fixed_num_pts[-1])
pts_num = fixed_num_pts.shape[0]
shift_num = pts_num - 1
shift_pts_list = []
if is_poly:
pts_to_shift = fixed_num_pts[:-1,:]
for shift_right_i in range(shift_num):
shift_pts_list.append(pts_to_shift.roll(shift_right_i,0))
flip_pts_to_shift = pts_to_shift.flip(0)
for shift_right_i in range(shift_num):
shift_pts_list.append(flip_pts_to_shift.roll(shift_right_i,0))
else:
shift_pts_list.append(fixed_num_pts)
shift_pts_list.append(fixed_num_pts.flip(0))
shift_pts = torch.stack(shift_pts_list,dim=0)
if is_poly:
_, _, num_coords = shift_pts.shape
tmp_shift_pts = shift_pts.new_zeros((shift_num*2, pts_num, num_coords))
tmp_shift_pts[:,:-1,:] = shift_pts
tmp_shift_pts[:,-1,:] = shift_pts[:,0,:]
shift_pts = tmp_shift_pts
shift_pts[:,:,0] = torch.clamp(shift_pts[:,:,0], min=-self.max_x,max=self.max_x)
shift_pts[:,:,1] = torch.clamp(shift_pts[:,:,1], min=-self.max_y,max=self.max_y)
if not is_poly:
padding = torch.full([shift_num*2-shift_pts.shape[0],pts_num,2], self.padding_value)
shift_pts = torch.cat([shift_pts,padding],dim=0)
instances_list.append(shift_pts)
instances_tensor = torch.stack(instances_list, dim=0)
instances_tensor = instances_tensor.to(
dtype=torch.float32)
return instances_tensor
@property
def shift_fixed_num_sampled_points_torch(self):
"""
return [instances_num, num_shifts, fixed_num, 2]
"""
fixed_num_sampled_points = self.fixed_num_sampled_points_torch
instances_list = []
is_poly = False
for fixed_num_pts in fixed_num_sampled_points:
is_poly = fixed_num_pts[0].equal(fixed_num_pts[-1])
fixed_num = fixed_num_pts.shape[0]
shift_pts_list = []
if is_poly:
for shift_right_i in range(fixed_num):
shift_pts_list.append(fixed_num_pts.roll(shift_right_i,0))
else:
shift_pts_list.append(fixed_num_pts)
shift_pts_list.append(fixed_num_pts.flip(0))
shift_pts = torch.stack(shift_pts_list,dim=0)
shift_pts[:,:,0] = torch.clamp(shift_pts[:,:,0], min=-self.max_x,max=self.max_x)
shift_pts[:,:,1] = torch.clamp(shift_pts[:,:,1], min=-self.max_y,max=self.max_y)
if not is_poly:
padding = torch.full([fixed_num-shift_pts.shape[0],fixed_num,2], self.padding_value)
shift_pts = torch.cat([shift_pts,padding],dim=0)
instances_list.append(shift_pts)
instances_tensor = torch.stack(instances_list, dim=0)
instances_tensor = instances_tensor.to(
dtype=torch.float32)
return instances_tensor
class VectorizedLocalMap(object):
CLASS2LABEL = {
'divider': 0,
'ped_crossing': 1,
'boundary': 2,
'centerline': 3,
'others': -1
}
def __init__(self,
canvas_size,
patch_size,
map_classes=['divider','ped_crossing','boundary'],
sample_dist=1,
num_samples=250,
padding=False,
fixed_ptsnum_per_line=-1,
padding_value=-10000,
thickness=3,
aux_seg = dict(
use_aux_seg=False,
bev_seg=False,
pv_seg=False,
seg_classes=1,
feat_down_sample=32)):
'''
Args:
fixed_ptsnum_per_line = -1 : no fixed num
'''
super().__init__()
self.vec_classes = map_classes
self.sample_dist = sample_dist
self.num_samples = num_samples
self.padding = padding
self.fixed_num = fixed_ptsnum_per_line
self.padding_value = padding_value
# for semantic mask
self.patch_size = patch_size
self.canvas_size = canvas_size
self.thickness = thickness
self.scale_x = self.canvas_size[1] / self.patch_size[1]
self.scale_y = self.canvas_size[0] / self.patch_size[0]
# self.auxseg_use_sem = auxseg_use_sem
self.aux_seg = aux_seg
def gen_vectorized_samples(self, map_annotation, example=None, feat_down_sample=32):
'''
use lidar2global to get gt map layers
'''
vectors = []
for vec_class in self.vec_classes:
instance_list = map_annotation[vec_class]
for instance in instance_list:
# vectors.append((LineString(np.array(instance)), self.CLASS2LABEL.get(vec_class, -1)))
vectors.append((LineString(instance), self.CLASS2LABEL.get(vec_class, -1)))
filtered_vectors = []
gt_pts_loc_3d = []
gt_pts_num_3d = []
gt_labels = []
gt_instance = []
if self.aux_seg['use_aux_seg']:
if self.aux_seg['seg_classes'] == 1:
if self.aux_seg['bev_seg']:
gt_semantic_mask = np.zeros((1, self.canvas_size[0], self.canvas_size[1]), dtype=np.uint8)
else:
gt_semantic_mask = None
if self.aux_seg['pv_seg']:
num_cam = len(example['img_metas'].data['pad_shape'])
img_shape = example['img_metas'].data['pad_shape'][0]
gt_pv_semantic_mask = np.zeros((num_cam, 1, img_shape[0] // feat_down_sample, img_shape[1] // feat_down_sample), dtype=np.uint8)
lidar2img = example['img_metas'].data['lidar2img']
scale_factor = np.eye(4)
scale_factor[0, 0] *= 1/32
scale_factor[1, 1] *= 1/32
lidar2feat = [scale_factor @ l2i for l2i in lidar2img]
else:
gt_pv_semantic_mask = None
for instance, instance_type in vectors:
if instance_type != -1:
gt_instance.append(instance)
gt_labels.append(instance_type)
if instance.geom_type == 'LineString':
if self.aux_seg['bev_seg']:
self.line_ego_to_mask(instance, gt_semantic_mask[0], color=1, thickness=self.thickness)
if self.aux_seg['pv_seg']:
for cam_index in range(num_cam):
self.line_ego_to_pvmask(instance, gt_pv_semantic_mask[cam_index][0], lidar2feat[cam_index], color=1, thickness=self.aux_seg['pv_thickness'])
else:
print(instance.geom_type)
else:
if self.aux_seg['bev_seg']:
gt_semantic_mask = np.zeros((len(self.vec_classes), self.canvas_size[0], self.canvas_size[1]), dtype=np.uint8)
else:
gt_semantic_mask = None
if self.aux_seg['pv_seg']:
num_cam = len(example['img_metas'].data['pad_shape'])
gt_pv_semantic_mask = np.zeros((num_cam, len(self.vec_classes), img_shape[0] // feat_down_sample, img_shape[1] // feat_down_sample), dtype=np.uint8)
lidar2img = example['img_metas'].data['lidar2img']
scale_factor = np.eye(4)
scale_factor[0, 0] *= 1/32
scale_factor[1, 1] *= 1/32
lidar2feat = [scale_factor @ l2i for l2i in lidar2img]
else:
gt_pv_semantic_mask = None
for instance, instance_type in vectors:
if instance_type != -1:
gt_instance.append(instance)
gt_labels.append(instance_type)
if instance.geom_type == 'LineString':
if self.aux_seg['bev_seg']:
self.line_ego_to_mask(instance, gt_semantic_mask[instance_type], color=1, thickness=self.thickness)
if self.aux_seg['pv_seg']:
for cam_index in range(num_cam):
self.line_ego_to_pvmask(instance, gt_pv_semantic_mask[cam_index][instance_type], lidar2feat[cam_index],color=1, thickness=self.aux_seg['pv_thickness'])
else:
print(instance.geom_type)
else:
for instance, instance_type in vectors:
if instance_type != -1:
gt_instance.append(instance)
gt_labels.append(instance_type)
gt_semantic_mask=None
gt_pv_semantic_mask=None
gt_instance = LiDARInstanceLines(gt_instance, gt_labels, self.sample_dist,
self.num_samples, self.padding, self.fixed_num,self.padding_value, patch_size=self.patch_size)
anns_results = dict(
gt_vecs_pts_loc=gt_instance,
gt_vecs_label=gt_labels,
gt_semantic_mask=gt_semantic_mask,
gt_pv_semantic_mask=gt_pv_semantic_mask,
)
return anns_results
def sample_line(self, line_ego, n_points=200):
x, y = np.array(line_ego.xy)
seg_len = np.sqrt(np.diff(x)**2 + np.diff(y)**2)
cum_len = np.concatenate([[0], np.cumsum(seg_len)])
total_len = cum_len[-1]
distances = np.linspace(0, total_len, n_points)
xs = np.interp(distances, cum_len, x)
ys = np.interp(distances, cum_len, y)
coords = np.stack([xs, ys], axis=1) # (N,2)
return coords
def project_points(self, coords, proj_mat, z=0.0):
pts_num = coords.shape[0]
lidar_coords = np.vstack([
coords[:,0],
coords[:,1],
np.full(pts_num, z, dtype=coords.dtype),
np.ones(pts_num, dtype=coords.dtype)
])
pix_coords = proj_mat @ lidar_coords
valid = pix_coords[2,:] > 0
pix_coords = pix_coords[:, valid]
pix_coords[:2,:] /= (pix_coords[2:3,:] + 1e-7)
return pix_coords[:2,:].T
def line_ego_to_pvmask(self,
line_ego,
mask,
lidar2feat,
color=1,
thickness=1,
z=-1.6):
# distances = np.linspace(0, line_ego.length, 200)
# coords = np.array([list(line_ego.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
# pts_num = coords.shape[0]
# zeros = np.zeros((pts_num,1))
# zeros[:] = z
# ones = np.ones((pts_num,1))
# lidar_coords = np.concatenate([coords,zeros,ones], axis=1).transpose(1,0)
# pix_coords = perspective(lidar_coords, lidar2feat)
coords = self.sample_line(line_ego)
pix_coords = self.project_points(coords, lidar2feat, z=z)
cv2.polylines(mask, np.int32([pix_coords]), False, color=color, thickness=thickness)
def scale_translate_geom(self, geom, scale_x, scale_y, trans_x, trans_y, origin=(0, 0)):
x0, y0 = origin
xoff = trans_x + x0 - scale_x * x0
yoff = trans_y + y0 - scale_y * y0
matrix = [scale_x, 0.0, 0.0, scale_y, xoff, yoff]
return affinity.affine_transform(geom, matrix)
def line_ego_to_mask(self,
line_ego,
mask,
color=1,
thickness=3):
''' Rasterize a single line to mask.
Args:
line_ego (LineString): line
mask (array): semantic mask to paint on
color (int): positive label, default: 1
thickness (int): thickness of rasterized lines, default: 3
'''
trans_x = self.canvas_size[1] / 2
trans_y = self.canvas_size[0] / 2
# line_ego = affinity.scale(line_ego, self.scale_x, self.scale_y, origin=(0, 0))
# line_ego = affinity.affine_transform(line_ego, [1.0, 0.0, 0.0, 1.0, trans_x, trans_y])
# coords = np.array(list(line_ego.coords), dtype=np.int32)[:, :2]
# coords = coords.reshape((-1, 2))
# assert len(coords) >= 2
# cv2.polylines(mask, np.int32([coords]), False, color=color, thickness=thickness)
line_ego = self.scale_translate_geom(line_ego, self.scale_x, self.scale_y, trans_x, trans_y)
coords = np.asarray(line_ego.coords, dtype=np.int32)
assert len(coords) >= 2
cv2.polylines(mask, [coords], False, color=color, thickness=thickness)
def get_map_geom(self, patch_box, patch_angle, layer_names, location):
map_geom = []
for layer_name in layer_names:
if layer_name in self.line_classes:
geoms = self.get_divider_line(patch_box, patch_angle, layer_name, location)
map_geom.append((layer_name, geoms))
elif layer_name in self.polygon_classes:
geoms = self.get_contour_line(patch_box, patch_angle, layer_name, location)
map_geom.append((layer_name, geoms))
elif layer_name in self.ped_crossing_classes:
geoms = self.get_ped_crossing_line(patch_box, patch_angle, location)
map_geom.append((layer_name, geoms))
return map_geom
def _one_type_line_geom_to_vectors(self, line_geom):
line_vectors = []
for line in line_geom:
if not line.is_empty:
if line.geom_type == 'MultiLineString':
for single_line in line.geoms:
line_vectors.append(self.sample_pts_from_line(single_line))
elif line.geom_type == 'LineString':
line_vectors.append(self.sample_pts_from_line(line))
else:
raise NotImplementedError
return line_vectors
def _one_type_line_geom_to_instances(self, line_geom):
line_instances = []
for line in line_geom:
if not line.is_empty:
if line.geom_type == 'MultiLineString':
for single_line in line.geoms:
line_instances.append(single_line)
elif line.geom_type == 'LineString':
line_instances.append(line)
else:
raise NotImplementedError
return line_instances
def poly_geoms_to_vectors(self, polygon_geom):
roads = polygon_geom[0][1]
lanes = polygon_geom[1][1]
union_roads = ops.unary_union(roads)
union_lanes = ops.unary_union(lanes)
union_segments = ops.unary_union([union_roads, union_lanes])
max_x = self.patch_size[1] / 2
max_y = self.patch_size[0] / 2
local_patch = box(-max_x + 0.2, -max_y + 0.2, max_x - 0.2, max_y - 0.2)
exteriors = []
interiors = []
if union_segments.geom_type != 'MultiPolygon':
union_segments = MultiPolygon([union_segments])
for poly in union_segments.geoms:
exteriors.append(poly.exterior)
for inter in poly.interiors:
interiors.append(inter)
results = []
for ext in exteriors:
if ext.is_ccw:
ext.coords = list(ext.coords)[::-1]
lines = ext.intersection(local_patch)
if isinstance(lines, MultiLineString):
lines = ops.linemerge(lines)
results.append(lines)
for inter in interiors:
if not inter.is_ccw:
inter.coords = list(inter.coords)[::-1]
lines = inter.intersection(local_patch)
if isinstance(lines, MultiLineString):
lines = ops.linemerge(lines)
results.append(lines)
return self._one_type_line_geom_to_vectors(results)
def ped_poly_geoms_to_instances(self, ped_geom):
ped = ped_geom[0][1]
union_segments = ops.unary_union(ped)
max_x = self.patch_size[1] / 2
max_y = self.patch_size[0] / 2
local_patch = box(-max_x - 0.2, -max_y - 0.2, max_x + 0.2, max_y + 0.2)
exteriors = []
interiors = []
if union_segments.geom_type != 'MultiPolygon':
union_segments = MultiPolygon([union_segments])
for poly in union_segments.geoms:
exteriors.append(poly.exterior)
for inter in poly.interiors:
interiors.append(inter)
results = []
for ext in exteriors:
if ext.is_ccw:
ext.coords = list(ext.coords)[::-1]
lines = ext.intersection(local_patch)
if isinstance(lines, MultiLineString):
lines = ops.linemerge(lines)
results.append(lines)
for inter in interiors:
if not inter.is_ccw:
inter.coords = list(inter.coords)[::-1]
lines = inter.intersection(local_patch)
if isinstance(lines, MultiLineString):
lines = ops.linemerge(lines)
results.append(lines)
return self._one_type_line_geom_to_instances(results)
def poly_geoms_to_instances(self, polygon_geom):
roads = polygon_geom[0][1]
lanes = polygon_geom[1][1]
union_roads = ops.unary_union(roads)
union_lanes = ops.unary_union(lanes)
union_segments = ops.unary_union([union_roads, union_lanes])
max_x = self.patch_size[1] / 2
max_y = self.patch_size[0] / 2
local_patch = box(-max_x + 0.2, -max_y + 0.2, max_x - 0.2, max_y - 0.2)
exteriors = []
interiors = []
if union_segments.geom_type != 'MultiPolygon':
union_segments = MultiPolygon([union_segments])
for poly in union_segments.geoms:
exteriors.append(poly.exterior)
for inter in poly.interiors:
interiors.append(inter)
results = []
for ext in exteriors:
if ext.is_ccw:
ext.coords = list(ext.coords)[::-1]
lines = ext.intersection(local_patch)
if isinstance(lines, MultiLineString):
lines = ops.linemerge(lines)
results.append(lines)
for inter in interiors:
if not inter.is_ccw:
inter.coords = list(inter.coords)[::-1]
lines = inter.intersection(local_patch)
if isinstance(lines, MultiLineString):
lines = ops.linemerge(lines)
results.append(lines)
return self._one_type_line_geom_to_instances(results)
def line_geoms_to_vectors(self, line_geom):
line_vectors_dict = dict()
for line_type, a_type_of_lines in line_geom:
one_type_vectors = self._one_type_line_geom_to_vectors(a_type_of_lines)
line_vectors_dict[line_type] = one_type_vectors
return line_vectors_dict
def line_geoms_to_instances(self, line_geom):
line_instances_dict = dict()
for line_type, a_type_of_lines in line_geom:
one_type_instances = self._one_type_line_geom_to_instances(a_type_of_lines)
line_instances_dict[line_type] = one_type_instances
return line_instances_dict
def ped_geoms_to_vectors(self, ped_geom):
ped_geom = ped_geom[0][1]
union_ped = ops.unary_union(ped_geom)
if union_ped.geom_type != 'MultiPolygon':
union_ped = MultiPolygon([union_ped])
max_x = self.patch_size[1] / 2
max_y = self.patch_size[0] / 2
local_patch = box(-max_x + 0.2, -max_y + 0.2, max_x - 0.2, max_y - 0.2)
results = []
for ped_poly in union_ped:
# rect = ped_poly.minimum_rotated_rectangle
ext = ped_poly.exterior
if not ext.is_ccw:
ext.coords = list(ext.coords)[::-1]
lines = ext.intersection(local_patch)
results.append(lines)
return self._one_type_line_geom_to_vectors(results)
def get_contour_line(self,patch_box,patch_angle,layer_name,location):
if layer_name not in self.map_explorer[location].map_api.non_geometric_polygon_layers:
raise ValueError('{} is not a polygonal layer'.format(layer_name))
patch_x = patch_box[0]
patch_y = patch_box[1]
patch = self.map_explorer[location].get_patch_coord(patch_box, patch_angle)
records = getattr(self.map_explorer[location].map_api, layer_name)
polygon_list = []
if layer_name == 'drivable_area':
for record in records:
polygons = [self.map_explorer[location].map_api.extract_polygon(polygon_token) for polygon_token in record['polygon_tokens']]
for polygon in polygons:
new_polygon = polygon.intersection(patch)
if not new_polygon.is_empty:
new_polygon = affinity.rotate(new_polygon, -patch_angle,
origin=(patch_x, patch_y), use_radians=False)
new_polygon = affinity.affine_transform(new_polygon,
[1.0, 0.0, 0.0, 1.0, -patch_x, -patch_y])
if new_polygon.geom_type == 'Polygon':
new_polygon = MultiPolygon([new_polygon])
polygon_list.append(new_polygon)
else:
for record in records:
polygon = self.map_explorer[location].map_api.extract_polygon(record['polygon_token'])
if polygon.is_valid:
new_polygon = polygon.intersection(patch)
if not new_polygon.is_empty:
new_polygon = affinity.rotate(new_polygon, -patch_angle,
origin=(patch_x, patch_y), use_radians=False)
new_polygon = affinity.affine_transform(new_polygon,
[1.0, 0.0, 0.0, 1.0, -patch_x, -patch_y])
if new_polygon.geom_type == 'Polygon':
new_polygon = MultiPolygon([new_polygon])
polygon_list.append(new_polygon)
return polygon_list
def get_divider_line(self,patch_box,patch_angle,layer_name,location):
if layer_name not in self.map_explorer[location].map_api.non_geometric_line_layers:
raise ValueError("{} is not a line layer".format(layer_name))
if layer_name == 'traffic_light':
return None
patch_x = patch_box[0]
patch_y = patch_box[1]
patch = self.map_explorer[location].get_patch_coord(patch_box, patch_angle)
line_list = []
records = getattr(self.map_explorer[location].map_api, layer_name)
for record in records:
line = self.map_explorer[location].map_api.extract_line(record['line_token'])
if line.is_empty: # Skip lines without nodes.
continue
new_line = line.intersection(patch)
if not new_line.is_empty:
new_line = affinity.rotate(new_line, -patch_angle, origin=(patch_x, patch_y), use_radians=False)
new_line = affinity.affine_transform(new_line,
[1.0, 0.0, 0.0, 1.0, -patch_x, -patch_y])
line_list.append(new_line)
return line_list
def get_ped_crossing_line(self, patch_box, patch_angle, location):
patch_x = patch_box[0]
patch_y = patch_box[1]
patch = self.map_explorer[location].get_patch_coord(patch_box, patch_angle)
polygon_list = []
records = getattr(self.map_explorer[location].map_api, 'ped_crossing')
# records = getattr(self.nusc_maps[location], 'ped_crossing')
for record in records:
polygon = self.map_explorer[location].map_api.extract_polygon(record['polygon_token'])
if polygon.is_valid:
new_polygon = polygon.intersection(patch)
if not new_polygon.is_empty:
new_polygon = affinity.rotate(new_polygon, -patch_angle,
origin=(patch_x, patch_y), use_radians=False)
new_polygon = affinity.affine_transform(new_polygon,
[1.0, 0.0, 0.0, 1.0, -patch_x, -patch_y])
if new_polygon.geom_type == 'Polygon':
new_polygon = MultiPolygon([new_polygon])
polygon_list.append(new_polygon)
return polygon_list
def sample_pts_from_line(self, line):
if self.fixed_num < 0:
distances = np.arange(0, line.length, self.sample_dist)
sampled_points = np.array([list(line.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
else:
# fixed number of points, so distance is line.length / self.fixed_num
distances = np.linspace(0, line.length, self.fixed_num)
sampled_points = np.array([list(line.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
num_valid = len(sampled_points)
if not self.padding or self.fixed_num > 0:
return sampled_points, num_valid
# fixed distance sampling need padding!
num_valid = len(sampled_points)
if self.fixed_num < 0:
if num_valid < self.num_samples:
padding = np.zeros((self.num_samples - len(sampled_points), 2))
sampled_points = np.concatenate([sampled_points, padding], axis=0)
else:
sampled_points = sampled_points[:self.num_samples, :]
num_valid = self.num_samples
return sampled_points, num_valid
@DATASETS.register_module()
class CustomNuScenesOfflineLocalMapDataset(CustomNuScenesDataset):
r"""NuScenes Dataset.
This datset add static map elements
"""
MAPCLASSES = ('divider',)
def __init__(self,
map_ann_file=None,
queue_length=4,
bev_size=(200, 200),
pc_range=[-51.2, -51.2, -5.0, 51.2, 51.2, 3.0],
overlap_test=False,
fixed_ptsnum_per_line=-1,
eval_use_same_gt_sample_num_flag=False,
padding_value=-10000,
map_classes=None,
noise='None',
noise_std=0,
aux_seg = dict(
use_aux_seg=False,
bev_seg=False,
pv_seg=False,
seg_classes=1,
feat_down_sample=32,
),
*args,
**kwargs):
super().__init__(*args, **kwargs)
self.map_ann_file = map_ann_file
self.queue_length = queue_length
self.overlap_test = overlap_test
self.bev_size = bev_size
self.MAPCLASSES = self.get_map_classes(map_classes)
self.NUM_MAPCLASSES = len(self.MAPCLASSES)
self.pc_range = pc_range
patch_h = pc_range[4]-pc_range[1]
patch_w = pc_range[3]-pc_range[0]
self.patch_size = (patch_h, patch_w)
self.padding_value = padding_value
self.fixed_num = fixed_ptsnum_per_line
self.eval_use_same_gt_sample_num_flag = eval_use_same_gt_sample_num_flag
self.aux_seg = aux_seg
self.vector_map = VectorizedLocalMap(canvas_size=bev_size,
patch_size=self.patch_size,
map_classes=self.MAPCLASSES,
fixed_ptsnum_per_line=fixed_ptsnum_per_line,
padding_value=self.padding_value,
aux_seg=aux_seg)
self.is_vis_on_test = False
self.noise = noise
self.noise_std = noise_std
@classmethod
def get_map_classes(cls, map_classes=None):
"""Get class names of current dataset.
Args:
classes (Sequence[str] | str | None): If classes is None, use
default CLASSES defined by builtin dataset. If classes is a
string, take it as a file name. The file contains the name of
classes where each line contains one class name. If classes is
a tuple or list, override the CLASSES defined by the dataset.
Return:
list[str]: A list of class names.
"""
if map_classes is None:
return cls.MAPCLASSES
if isinstance(map_classes, str):
# take it as a file path
class_names = mmcv.list_from_file(map_classes)
elif isinstance(map_classes, (tuple, list)):
class_names = map_classes
else:
raise ValueError(f'Unsupported type {type(map_classes)} of map classes.')
return class_names
def vectormap_pipeline(self, example, input_dict):
'''
`example` type: <class 'dict'>
keys: 'img_metas', 'gt_bboxes_3d', 'gt_labels_3d', 'img';
all keys type is 'DataContainer';
'img_metas' cpu_only=True, type is dict, others are false;
'gt_labels_3d' shape torch.size([num_samples]), stack=False,
padding_value=0, cpu_only=False
'gt_bboxes_3d': stack=False, cpu_only=True
'''
# import ipdb;ipdb.set_trace()
anns_results = self.vector_map.gen_vectorized_samples(input_dict['annotation'] if 'annotation' in input_dict.keys() else input_dict['ann_info'],
example=example, feat_down_sample=self.aux_seg['feat_down_sample'])
'''
anns_results, type: dict
'gt_vecs_pts_loc': list[num_vecs], vec with num_points*2 coordinates
'gt_vecs_pts_num': list[num_vecs], vec with num_points
'gt_vecs_label': list[num_vecs], vec with cls index
'''
gt_vecs_label = to_tensor(anns_results['gt_vecs_label'])
if isinstance(anns_results['gt_vecs_pts_loc'], LiDARInstanceLines):
gt_vecs_pts_loc = anns_results['gt_vecs_pts_loc']
else:
gt_vecs_pts_loc = to_tensor(anns_results['gt_vecs_pts_loc'])
try:
gt_vecs_pts_loc = gt_vecs_pts_loc.flatten(1).to(dtype=torch.float32)
except:
# empty tensor, will be passed in train,
# but we preserve it for test
gt_vecs_pts_loc = gt_vecs_pts_loc
example['gt_labels_3d'] = DC(gt_vecs_label, cpu_only=False)
example['gt_bboxes_3d'] = DC(gt_vecs_pts_loc, cpu_only=True)
# gt_seg_mask = to_tensor(anns_results['gt_semantic_mask'])
# gt_pv_seg_mask = to_tensor(anns_results['gt_pv_semantic_mask'])
if anns_results['gt_semantic_mask'] is not None:
example['gt_seg_mask'] = DC(to_tensor(anns_results['gt_semantic_mask']), cpu_only=False)
if anns_results['gt_pv_semantic_mask'] is not None:
example['gt_pv_seg_mask'] = DC(to_tensor(anns_results['gt_pv_semantic_mask']), cpu_only=False)
return example
def prepare_train_data(self, index):
"""
Training data preparation.
Args:
index (int): Index for accessing the target data.
Returns:
dict: Training data dict of the corresponding index.
"""
data_queue = []
# temporal aug
prev_indexs_list = list(range(index-self.queue_length, index))
random.shuffle(prev_indexs_list)
prev_indexs_list = sorted(prev_indexs_list[1:], reverse=True)
##
input_dict = self.get_data_info(index)
if input_dict is None:
return None
frame_idx = input_dict['frame_idx']
scene_token = input_dict['scene_token']
self.pre_pipeline(input_dict)
# import pdb;pdb.set_trace()
example = self.pipeline(input_dict)
example = self.vectormap_pipeline(example,input_dict)
if self.filter_empty_gt and \
(example is None or ~(example['gt_labels_3d']._data != -1).any()):
return None
data_queue.insert(0, example)
for i in prev_indexs_list:
i = max(0, i)
input_dict = self.get_data_info(i)
if input_dict is None:
return None
if input_dict['frame_idx'] < frame_idx and input_dict['scene_token'] == scene_token:
self.pre_pipeline(input_dict)
example = self.pipeline(input_dict)
example = self.vectormap_pipeline(example,input_dict)
if self.filter_empty_gt and \
(example is None or ~(example['gt_labels_3d']._data != -1).any()):
return None
frame_idx = input_dict['frame_idx']
data_queue.insert(0, copy.deepcopy(example))
return self.union2one(data_queue)
def union2one(self, queue):
"""
convert sample queue into one single sample.
"""
# import ipdb;ipdb.set_trace()
imgs_list = [each['img'].data for each in queue]
metas_map = {}
prev_pos = None
prev_angle = None
for i, each in enumerate(queue):
metas_map[i] = each['img_metas'].data
if i == 0:
metas_map[i]['prev_bev'] = False
prev_lidar2global = metas_map[i]['lidar2global']
prev_pos = copy.deepcopy(metas_map[i]['can_bus'][:3])
prev_angle = copy.deepcopy(metas_map[i]['can_bus'][-1])
metas_map[i]['can_bus'][:3] = 0
metas_map[i]['can_bus'][-1] = 0
tmp_lidar2prev_lidar = np.eye(4)
metas_map[i]['tmp_lidar2prev_lidar'] = tmp_lidar2prev_lidar
tmp_lidar2prev_lidar_translation = tmp_lidar2prev_lidar[:3,3]
tmp_lidar2prev_lidar_angle = quaternion_yaw(Quaternion(
matrix=tmp_lidar2prev_lidar)) / np.pi * 180
metas_map[i]['tmp_lidar2prev_lidar_translation'] = tmp_lidar2prev_lidar_translation
metas_map[i]['tmp_lidar2prev_lidar_angle'] = tmp_lidar2prev_lidar_angle
else:
metas_map[i]['prev_bev'] = True
tmp_lidar2global = metas_map[i]['lidar2global']
tmp_lidar2prev_lidar = np.linalg.inv(prev_lidar2global)@tmp_lidar2global
tmp_lidar2prev_lidar_translation = tmp_lidar2prev_lidar[:3,3]
tmp_lidar2prev_lidar_angle = quaternion_yaw(Quaternion(
matrix=tmp_lidar2prev_lidar)) / np.pi * 180
tmp_pos = copy.deepcopy(metas_map[i]['can_bus'][:3])
tmp_angle = copy.deepcopy(metas_map[i]['can_bus'][-1])
metas_map[i]['can_bus'][:3] -= prev_pos
metas_map[i]['can_bus'][-1] -= prev_angle
metas_map[i]['tmp_lidar2prev_lidar'] = tmp_lidar2prev_lidar
metas_map[i]['tmp_lidar2prev_lidar_translation'] = tmp_lidar2prev_lidar_translation
metas_map[i]['tmp_lidar2prev_lidar_angle'] = tmp_lidar2prev_lidar_angle
prev_pos = copy.deepcopy(tmp_pos)
prev_angle = copy.deepcopy(tmp_angle)
prev_lidar2global = copy.deepcopy(tmp_lidar2global)
queue[-1]['img'] = DC(torch.stack(imgs_list),
cpu_only=False, stack=True)
queue[-1]['img_metas'] = DC(metas_map, cpu_only=True)
queue = queue[-1]
return queue
def get_data_info(self, index):
"""Get data info according to the given index.
Args:
index (int): Index of the sample data to get.
Returns:
dict: Data information that will be passed to the data \
preprocessing pipelines. It includes the following keys:
- sample_idx (str): Sample index.
- pts_filename (str): Filename of point clouds.
- sweeps (list[dict]): Infos of sweeps.
- timestamp (float): Sample timestamp.
- img_filename (str, optional): Image filename.
- lidar2img (list[np.ndarray], optional): Transformations \
from lidar to different cameras.
- ann_info (dict): Annotation info.
"""
info = self.data_infos[index]
# standard protocal modified from SECOND.Pytorch
input_dict = dict(
sample_idx=info['token'],
pts_filename=info['lidar_path'],
lidar_path=info["lidar_path"],
sweeps=info['sweeps'],
ego2global_translation=info['ego2global_translation'],
ego2global_rotation=info['ego2global_rotation'],
lidar2ego_translation=info['lidar2ego_translation'],
lidar2ego_rotation=info['lidar2ego_rotation'],
prev_idx=info['prev'],
next_idx=info['next'],
scene_token=info['scene_token'],
can_bus=info['can_bus'],
frame_idx=info['frame_idx'],
timestamp=info['timestamp'],
map_location = info['map_location'],
)
# lidar to ego transform
lidar2ego = np.eye(4).astype(np.float32)
lidar2ego[:3, :3] = Quaternion(info["lidar2ego_rotation"]).rotation_matrix
lidar2ego[:3, 3] = info["lidar2ego_translation"]
input_dict["lidar2ego"] = lidar2ego
if self.modality['use_camera']:
image_paths = []
lidar2img_rts = []
lidar2cam_rts = []
cam_intrinsics = []
input_dict["camera2ego"] = []
input_dict["camera_intrinsics"] = []
input_dict["camego2global"] = []
for cam_type, cam_info in info['cams'].items():
image_paths.append(cam_info['data_path'])
# obtain lidar to image transformation matrix
lidar2cam_r = np.linalg.inv(cam_info['sensor2lidar_rotation'])
lidar2cam_t = cam_info[
'sensor2lidar_translation'] @ lidar2cam_r.T
lidar2cam_rt = np.eye(4)
lidar2cam_rt[:3, :3] = lidar2cam_r.T
lidar2cam_rt[3, :3] = -lidar2cam_t
lidar2cam_rt_t = lidar2cam_rt.T
if self.noise == 'rotation':
lidar2cam_rt_t = add_rotation_noise(lidar2cam_rt_t, std=self.noise_std)
elif self.noise == 'translation':
lidar2cam_rt_t = add_translation_noise(
lidar2cam_rt_t, std=self.noise_std)
intrinsic = cam_info['cam_intrinsic']
viewpad = np.eye(4)
viewpad[:intrinsic.shape[0], :intrinsic.shape[1]] = intrinsic
lidar2img_rt = (viewpad @ lidar2cam_rt_t)
lidar2img_rts.append(lidar2img_rt)
cam_intrinsics.append(viewpad)
lidar2cam_rts.append(lidar2cam_rt_t)
# camera to ego transform
camera2ego = np.eye(4).astype(np.float32)
camera2ego[:3, :3] = Quaternion(
cam_info["sensor2ego_rotation"]
).rotation_matrix
camera2ego[:3, 3] = cam_info["sensor2ego_translation"]
input_dict["camera2ego"].append(camera2ego)
# camego to global transform
camego2global = np.eye(4, dtype=np.float32)
camego2global[:3, :3] = Quaternion(
cam_info['ego2global_rotation']).rotation_matrix
camego2global[:3, 3] = cam_info['ego2global_translation']
camego2global = torch.from_numpy(camego2global)
input_dict["camego2global"].append(camego2global)
# camera intrinsics
camera_intrinsics = np.eye(4).astype(np.float32)
camera_intrinsics[:3, :3] = cam_info["cam_intrinsic"]
input_dict["camera_intrinsics"].append(camera_intrinsics)
input_dict.update(
dict(
img_filename=image_paths,
lidar2img=lidar2img_rts,
cam_intrinsic=cam_intrinsics,
lidar2cam=lidar2cam_rts,
))
# if not self.test_mode:
# # annos = self.get_ann_info(index)
input_dict['ann_info'] = info['annotation']
rotation = Quaternion(input_dict['ego2global_rotation'])
translation = input_dict['ego2global_translation']
can_bus = input_dict['can_bus']
can_bus[:3] = translation
can_bus[3:7] = rotation
patch_angle = quaternion_yaw(rotation) / np.pi * 180
if patch_angle < 0:
patch_angle += 360
can_bus[-2] = patch_angle / 180 * np.pi
can_bus[-1] = patch_angle
lidar2ego = np.eye(4)
lidar2ego[:3,:3] = Quaternion(input_dict['lidar2ego_rotation']).rotation_matrix
lidar2ego[:3, 3] = input_dict['lidar2ego_translation']
ego2global = np.eye(4)
ego2global[:3,:3] = Quaternion(input_dict['ego2global_rotation']).rotation_matrix
ego2global[:3, 3] = input_dict['ego2global_translation']
lidar2global = ego2global @ lidar2ego
input_dict['lidar2global'] = lidar2global
return input_dict
def prepare_test_data(self, index):
"""Prepare data for testing.
Args:
index (int): Index for accessing the target data.
Returns:
dict: Testing data dict of the corresponding index.
"""
input_dict = self.get_data_info(index)
self.pre_pipeline(input_dict)
example = self.pipeline(input_dict)
if self.is_vis_on_test:
example = self.vectormap_pipeline(example, input_dict)
return example
def __getitem__(self, idx):
"""Get item from infos according to the given index.
Returns:
dict: Data dictionary of the corresponding index.
"""
if self.test_mode:
return self.prepare_test_data(idx)
while True:
data = self.prepare_train_data(idx)
if data is None:
idx = self._rand_another(idx)
continue
return data
def _format_gt(self):
gt_annos = []
print('Start to convert gt map format...')
assert self.map_ann_file is not None
if (not os.path.exists(self.map_ann_file)) :
dataset_length = len(self)
prog_bar = mmcv.ProgressBar(dataset_length)
mapped_class_names = self.MAPCLASSES
for sample_id in range(dataset_length):
sample_token = self.data_infos[sample_id]['token']
gt_anno = {}
gt_anno['sample_token'] = sample_token
# gt_sample_annos = []
gt_sample_dict = {}
gt_sample_dict = self.vectormap_pipeline(gt_sample_dict, self.data_infos[sample_id])
gt_labels = gt_sample_dict['gt_labels_3d'].data.numpy()
gt_vecs = gt_sample_dict['gt_bboxes_3d'].data.instance_list
gt_vec_list = []
for i, (gt_label, gt_vec) in enumerate(zip(gt_labels, gt_vecs)):
name = mapped_class_names[gt_label]
anno = dict(
pts=np.array(list(gt_vec.coords)),
pts_num=len(list(gt_vec.coords)),
cls_name=name,
type=gt_label,
)
gt_vec_list.append(anno)
gt_anno['vectors']=gt_vec_list
gt_annos.append(gt_anno)
prog_bar.update()
nusc_submissions = {
'GTs': gt_annos
}
print('\n GT anns writes to', self.map_ann_file)
mmcv.dump(nusc_submissions, self.map_ann_file)
else:
print(f'{self.map_ann_file} exist, not update')
def _format_bbox(self, results, jsonfile_prefix=None):
"""Convert the results to the standard format.
Args:
results (list[dict]): Testing results of the dataset.
jsonfile_prefix (str): The prefix of the output jsonfile.
You can specify the output directory/filename by
modifying the jsonfile_prefix. Default: None.
Returns:
str: Path of the output json file.
"""
assert self.map_ann_file is not None
pred_annos = []
mapped_class_names = self.MAPCLASSES
# import pdb;pdb.set_trace()
print('Start to convert map detection format...')
for sample_id, det in enumerate(mmcv.track_iter_progress(results)):
pred_anno = {}
vecs = output_to_vecs(det)
sample_token = self.data_infos[sample_id]['token']
pred_anno['sample_token'] = sample_token
pred_vec_list=[]
for i, vec in enumerate(vecs):
name = mapped_class_names[vec['label']]
anno = dict(
pts=vec['pts'],
pts_num=len(vec['pts']),
cls_name=name,
type=vec['label'],
confidence_level=vec['score'])
pred_vec_list.append(anno)
pred_anno['vectors'] = pred_vec_list
pred_annos.append(pred_anno)
if not os.path.exists(self.map_ann_file):
self._format_gt()
else:
print(f'{self.map_ann_file} exist, not update')
nusc_submissions = {
'meta': self.modality,
'results': pred_annos,
}
mmcv.mkdir_or_exist(jsonfile_prefix)
res_path = osp.join(jsonfile_prefix, 'nuscmap_results.json')
print('Results writes to', res_path)
mmcv.dump(nusc_submissions, res_path)
return res_path
def to_gt_vectors(self,
gt_dict):
# import pdb;pdb.set_trace()
gt_labels = gt_dict['gt_labels_3d'].data
gt_instances = gt_dict['gt_bboxes_3d'].data.instance_list
gt_vectors = []
for gt_instance, gt_label in zip(gt_instances, gt_labels):
pts, pts_num = sample_pts_from_line(gt_instance, patch_size=self.patch_size)
gt_vectors.append({
'pts': pts,
'pts_num': pts_num,
'type': int(gt_label)
})
vector_num_list = {}
for i in range(self.NUM_MAPCLASSES):
vector_num_list[i] = []
for vec in gt_vectors:
if vector['pts_num'] >= 2:
vector_num_list[vector['type']].append((LineString(vector['pts'][:vector['pts_num']]), vector.get('confidence_level', 1)))
return gt_vectors
def _evaluate_single(self,
result_path,
logger=None,
metric='chamfer',
result_name='pts_bbox'):
"""Evaluation for a single model in nuScenes protocol.
Args:
result_path (str): Path of the result file.
logger (logging.Logger | str | None): Logger used for printing
related information during evaluation. Default: None.
metric (str): Metric name used for evaluation. Default: 'bbox'.
result_name (str): Result name in the metric prefix.
Default: 'pts_bbox'.
Returns:
dict: Dictionary of evaluation details.
"""
from projects.mmdet3d_plugin.datasets.map_utils.mean_ap import eval_map
from projects.mmdet3d_plugin.datasets.map_utils.mean_ap import format_res_gt_by_classes
result_path = osp.abspath(result_path)
detail = dict()
print('Formating results & gts by classes')
with open(result_path,'r') as f:
pred_results = json.load(f)
gen_results = pred_results['results']
with open(self.map_ann_file,'r') as ann_f:
gt_anns = json.load(ann_f)
annotations = gt_anns['GTs']
cls_gens, cls_gts = format_res_gt_by_classes(result_path,
gen_results,
annotations,
cls_names=self.MAPCLASSES,
num_pred_pts_per_instance=self.fixed_num,
eval_use_same_gt_sample_num_flag=self.eval_use_same_gt_sample_num_flag,
pc_range=self.pc_range)
metrics = metric if isinstance(metric, list) else [metric]
allowed_metrics = ['chamfer', 'iou']
for metric in metrics:
if metric not in allowed_metrics:
raise KeyError(f'metric {metric} is not supported')
for metric in metrics:
print('-*'*10+f'use metric:{metric}'+'-*'*10)
if metric == 'chamfer':
thresholds = [0.5,1.0,1.5]
elif metric == 'iou':
thresholds= np.linspace(.5, 0.95, int(np.round((0.95 - .5) / .05)) + 1, endpoint=True)
cls_aps = np.zeros((len(thresholds),self.NUM_MAPCLASSES))
for i, thr in enumerate(thresholds):
print('-*'*10+f'threshhold:{thr}'+'-*'*10)
mAP, cls_ap = eval_map(
gen_results,
annotations,
cls_gens,
cls_gts,
threshold=thr,
cls_names=self.MAPCLASSES,
logger=logger,
num_pred_pts_per_instance=self.fixed_num,
pc_range=self.pc_range,
metric=metric)
for j in range(self.NUM_MAPCLASSES):
cls_aps[i, j] = cls_ap[j]['ap']
for i, name in enumerate(self.MAPCLASSES):
print('{}: {}'.format(name, cls_aps.mean(0)[i]))
detail['NuscMap_{}/{}_AP'.format(metric,name)] = cls_aps.mean(0)[i]
print('map: {}'.format(cls_aps.mean(0).mean()))
detail['NuscMap_{}/mAP'.format(metric)] = cls_aps.mean(0).mean()
for i, name in enumerate(self.MAPCLASSES):
for j, thr in enumerate(thresholds):
if metric == 'chamfer':
detail['NuscMap_{}/{}_AP_thr_{}'.format(metric,name,thr)]=cls_aps[j][i]
elif metric == 'iou':
if thr == 0.5 or thr == 0.75:
detail['NuscMap_{}/{}_AP_thr_{}'.format(metric,name,thr)]=cls_aps[j][i]
return detail
def evaluate(self,
results,
metric='bbox',
logger=None,
jsonfile_prefix=None,
result_names=['pts_bbox'],
show=False,
out_dir=None,
pipeline=None):
"""Evaluation in nuScenes protocol.
Args:
results (list[dict]): Testing results of the dataset.
metric (str | list[str]): Metrics to be evaluated.
logger (logging.Logger | str | None): Logger used for printing
related information during evaluation. Default: None.
jsonfile_prefix (str | None): The prefix of json files. It includes
the file path and the prefix of filename, e.g., "a/b/prefix".
If not specified, a temp file will be created. Default: None.
show (bool): Whether to visualize.
Default: False.
out_dir (str): Path to save the visualization results.
Default: None.
pipeline (list[dict], optional): raw data loading for showing.
Default: None.
Returns:
dict[str, float]: Results of each evaluation metric.
"""
result_files, tmp_dir = self.format_results(results, jsonfile_prefix)
if isinstance(result_files, dict):
results_dict = dict()
for name in result_names:
print('Evaluating bboxes of {}'.format(name))
ret_dict = self._evaluate_single(result_files[name], metric=metric)
results_dict.update(ret_dict)
elif isinstance(result_files, str):
results_dict = self._evaluate_single(result_files, metric=metric)
if tmp_dir is not None:
tmp_dir.cleanup()
if show:
self.show(results, out_dir, pipeline=pipeline)
return results_dict
def output_to_vecs(detection):
box3d = detection['boxes_3d'].numpy()
scores = detection['scores_3d'].numpy()
labels = detection['labels_3d'].numpy()
pts = detection['pts_3d'].numpy()
vec_list = []
for i in range(box3d.shape[0]):
vec = dict(
bbox = box3d[i], # xyxy
label=labels[i],
score=scores[i],
pts=pts[i],
)
vec_list.append(vec)
return vec_list
def sample_pts_from_line(line,
fixed_num=-1,
sample_dist=1,
normalize=False,
patch_size=None,
padding=False,
num_samples=250,):
if fixed_num < 0:
distances = np.arange(0, line.length, sample_dist)
sampled_points = np.array([list(line.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
else:
# fixed number of points, so distance is line.length / fixed_num
distances = np.linspace(0, line.length, fixed_num)
sampled_points = np.array([list(line.interpolate(distance).coords) for distance in distances]).reshape(-1, 2)
if normalize:
sampled_points = sampled_points / np.array([patch_size[1], patch_size[0]])
num_valid = len(sampled_points)
if not padding or fixed_num > 0:
# fixed num sample can return now!
return sampled_points, num_valid
# fixed distance sampling need padding!
num_valid = len(sampled_points)
if fixed_num < 0:
if num_valid < num_samples:
padding = np.zeros((num_samples - len(sampled_points), 2))
sampled_points = np.concatenate([sampled_points, padding], axis=0)
else:
sampled_points = sampled_points[:num_samples, :]
num_valid = num_samples
if normalize:
sampled_points = sampled_points / np.array([patch_size[1], patch_size[0]])
num_valid = len(sampled_points)
return sampled_points, num_valid
docker-hub/MapTRv2/MapTR/projects/mmdet3d_plugin/datasets/nuscnes_eval.py 0 → 100644
View file @ 19472568
import argparse
import copy
import json
import os
import time
from typing import Tuple, Dict, Any
import torch
import numpy as np
from nuscenes import NuScenes
from nuscenes.eval.common.config import config_factory
from nuscenes.eval.common.data_classes import EvalBoxes
from nuscenes.eval.detection.data_classes import DetectionConfig
from nuscenes.eval.detection.evaluate import NuScenesEval
from pyquaternion import Quaternion
from nuscenes import NuScenes
from nuscenes.eval.common.data_classes import EvalBoxes
from nuscenes.eval.detection.data_classes import DetectionBox
from nuscenes.eval.detection.utils import category_to_detection_name
from nuscenes.eval.tracking.data_classes import TrackingBox
from nuscenes.utils.data_classes import Box
from nuscenes.utils.geometry_utils import points_in_box
from nuscenes.utils.splits import create_splits_scenes
from nuscenes.eval.common.loaders import load_prediction, add_center_dist, filter_eval_boxes
import tqdm
from nuscenes.utils.geometry_utils import view_points, box_in_image, BoxVisibility, transform_matrix
from torchvision.transforms.functional import rotate
import pycocotools.mask as mask_util
# from projects.mmdet3d_plugin.models.utils.visual import save_tensor
from torchvision.transforms.functional import rotate
import cv2
import argparse
import json
import os
import random
import time
from typing import Tuple, Dict, Any
import numpy as np
from nuscenes import NuScenes
from nuscenes.eval.common.config import config_factory
from nuscenes.eval.common.data_classes import EvalBoxes
from nuscenes.eval.common.loaders import load_prediction, load_gt, add_center_dist, filter_eval_boxes
from nuscenes.eval.detection.algo import accumulate, calc_ap, calc_tp
from nuscenes.eval.detection.constants import TP_METRICS
from nuscenes.eval.detection.data_classes import DetectionConfig, DetectionMetrics, DetectionBox, \
DetectionMetricDataList
from nuscenes.eval.detection.render import summary_plot, class_pr_curve, dist_pr_curve, visualize_sample
from nuscenes.eval.common.utils import quaternion_yaw, Quaternion
from mmdet3d.core.bbox.iou_calculators import BboxOverlaps3D
from IPython import embed
import json
from typing import Any
import numpy as np
from matplotlib import pyplot as plt
from nuscenes import NuScenes
from nuscenes.eval.common.data_classes import EvalBoxes
from nuscenes.eval.common.render import setup_axis
from nuscenes.eval.common.utils import boxes_to_sensor
from nuscenes.eval.detection.constants import TP_METRICS, DETECTION_NAMES, DETECTION_COLORS, TP_METRICS_UNITS, \
PRETTY_DETECTION_NAMES, PRETTY_TP_METRICS
from nuscenes.eval.detection.data_classes import DetectionMetrics, DetectionMetricData, DetectionMetricDataList
from nuscenes.utils.data_classes import LidarPointCloud
from nuscenes.utils.geometry_utils import view_points
Axis = Any
def class_tp_curve(md_list: DetectionMetricDataList,
metrics: DetectionMetrics,
detection_name: str,
min_recall: float,
dist_th_tp: float,
savepath: str = None,
ax: Axis = None) -> None:
"""
Plot the true positive curve for the specified class.
:param md_list: DetectionMetricDataList instance.
:param metrics: DetectionMetrics instance.
:param detection_name:
:param min_recall: Minimum recall value.
:param dist_th_tp: The distance threshold used to determine matches.
:param savepath: If given, saves the the rendering here instead of displaying.
:param ax: Axes onto which to render.
"""
# Get metric data for given detection class with tp distance threshold.
md = md_list[(detection_name, dist_th_tp)]
min_recall_ind = round(100 * min_recall)
if min_recall_ind <= md.max_recall_ind:
# For traffic_cone and barrier only a subset of the metrics are plotted.
rel_metrics = [m for m in TP_METRICS if not np.isnan(metrics.get_label_tp(detection_name, m))]
ylimit = max([max(getattr(md, metric)[min_recall_ind:md.max_recall_ind + 1]) for metric in rel_metrics]) * 1.1
else:
ylimit = 1.0
# Prepare axis.
if ax is None:
ax = setup_axis(title=PRETTY_DETECTION_NAMES[detection_name], xlabel='Recall', ylabel='Error', xlim=1,
min_recall=min_recall)
ax.set_ylim(0, ylimit)
# Plot the recall vs. error curve for each tp metric.
for metric in TP_METRICS:
tp = metrics.get_label_tp(detection_name, metric)
# Plot only if we have valid data.
if tp is not np.nan and min_recall_ind <= md.max_recall_ind:
recall, error = md.recall[:md.max_recall_ind + 1], getattr(md, metric)[:md.max_recall_ind + 1]
else:
recall, error = [], []
# Change legend based on tp value
if tp is np.nan:
label = '{}: n/a'.format(PRETTY_TP_METRICS[metric])
elif min_recall_ind > md.max_recall_ind:
label = '{}: nan'.format(PRETTY_TP_METRICS[metric])
else:
label = '{}: {:.2f} ({})'.format(PRETTY_TP_METRICS[metric], tp, TP_METRICS_UNITS[metric])
if metric == 'trans_err':
label += f' ({md.max_recall_ind})' # add recall
print(f'Recall: {detection_name}: {md.max_recall_ind/100}')
ax.plot(recall, error, label=label)
ax.axvline(x=md.max_recall, linestyle='-.', color=(0, 0, 0, 0.3))
ax.legend(loc='best')
if savepath is not None:
plt.savefig(savepath)
plt.close()
class DetectionBox_modified(DetectionBox):
def __init__(self, *args, token=None, visibility=None, index=None, **kwargs):
'''
add annotation token
'''
super().__init__(*args, **kwargs)
self.token = token
self.visibility = visibility
self.index = index
def serialize(self) -> dict:
""" Serialize instance into json-friendly format. """
return {
'token': self.token,
'sample_token': self.sample_token,
'translation': self.translation,
'size': self.size,
'rotation': self.rotation,
'velocity': self.velocity,
'ego_translation': self.ego_translation,
'num_pts': self.num_pts,
'detection_name': self.detection_name,
'detection_score': self.detection_score,
'attribute_name': self.attribute_name,
'visibility': self.visibility,
'index': self.index
}
@classmethod
def deserialize(cls, content: dict):
""" Initialize from serialized content. """
return cls(
token=content['token'],
sample_token=content['sample_token'],
translation=tuple(content['translation']),
size=tuple(content['size']),
rotation=tuple(content['rotation']),
velocity=tuple(content['velocity']),
ego_translation=(0.0, 0.0, 0.0) if 'ego_translation' not in content
else tuple(content['ego_translation']),
num_pts=-1 if 'num_pts' not in content else int(content['num_pts']),
detection_name=content['detection_name'],
detection_score=-1.0 if 'detection_score' not in content else float(content['detection_score']),
attribute_name=content['attribute_name'],
visibility=content['visibility'],
index=content['index'],
)
def center_in_image(box, intrinsic: np.ndarray, imsize: Tuple[int, int], vis_level: int = BoxVisibility.ANY) -> bool:
"""
Check if a box is visible inside an image without accounting for occlusions.
:param box: The box to be checked.
:param intrinsic: <float: 3, 3>. Intrinsic camera matrix.
:param imsize: (width, height).
:param vis_level: One of the enumerations of <BoxVisibility>.
:return True if visibility condition is satisfied.
"""
center_3d = box.center.reshape(3, 1)
center_img = view_points(center_3d, intrinsic, normalize=True)[:2, :]
visible = np.logical_and(center_img[0, :] > 0, center_img[0, :] < imsize[0])
visible = np.logical_and(visible, center_img[1, :] < imsize[1])
visible = np.logical_and(visible, center_img[1, :] > 0)
visible = np.logical_and(visible, center_3d[2, :] > 1)
in_front = center_3d[2, :] > 0.1 # True if a corner is at least 0.1 meter in front of the camera.
if vis_level == BoxVisibility.ALL:
return all(visible) and all(in_front)
elif vis_level == BoxVisibility.ANY:
return any(visible) and all(in_front)
elif vis_level == BoxVisibility.NONE:
return True
else:
raise ValueError("vis_level: {} not valid".format(vis_level))
def exist_corners_in_image_but_not_all(box, intrinsic: np.ndarray, imsize: Tuple[int, int],
vis_level: int = BoxVisibility.ANY) -> bool:
"""
Check if a box is visible in images but not all corners in image .
:param box: The box to be checked.
:param intrinsic: <float: 3, 3>. Intrinsic camera matrix.
:param imsize: (width, height).
:param vis_level: One of the enumerations of <BoxVisibility>.
:return True if visibility condition is satisfied.
"""
corners_3d = box.corners()
corners_img = view_points(corners_3d, intrinsic, normalize=True)[:2, :]
visible = np.logical_and(corners_img[0, :] > 0, corners_img[0, :] < imsize[0])
visible = np.logical_and(visible, corners_img[1, :] < imsize[1])
visible = np.logical_and(visible, corners_img[1, :] > 0)
visible = np.logical_and(visible, corners_3d[2, :] > 1)
in_front = corners_3d[2, :] > 0.1 # True if a corner is at least 0.1 meter in front of the camera.
if any(visible) and not all(visible) and all(in_front):
return True
else:
return False
def load_gt(nusc: NuScenes, eval_split: str, box_cls, verbose: bool = False):
"""
Loads ground truth boxes from DB.
:param nusc: A NuScenes instance.
:param eval_split: The evaluation split for which we load GT boxes.
:param box_cls: Type of box to load, e.g. DetectionBox or TrackingBox.
:param verbose: Whether to print messages to stdout.
:return: The GT boxes.
"""
# Init.
if box_cls == DetectionBox_modified:
attribute_map = {a['token']: a['name'] for a in nusc.attribute}
if verbose:
print('Loading annotations for {} split from nuScenes version: {}'.format(eval_split, nusc.version))
# Read out all sample_tokens in DB.
sample_tokens_all = [s['token'] for s in nusc.sample]
assert len(sample_tokens_all) > 0, "Error: Database has no samples!"
# Only keep samples from this split.
splits = create_splits_scenes()
# Check compatibility of split with nusc_version.
version = nusc.version
if eval_split in {'train', 'val', 'train_detect', 'train_track'}:
assert version.endswith('trainval'), \
'Error: Requested split {} which is not compatible with NuScenes version {}'.format(eval_split, version)
elif eval_split in {'mini_train', 'mini_val'}:
assert version.endswith('mini'), \
'Error: Requested split {} which is not compatible with NuScenes version {}'.format(eval_split, version)
elif eval_split == 'test':
assert version.endswith('test'), \
'Error: Requested split {} which is not compatible with NuScenes version {}'.format(eval_split, version)
else:
raise ValueError('Error: Requested split {} which this function cannot map to the correct NuScenes version.'
.format(eval_split))
if eval_split == 'test':
# Check that you aren't trying to cheat :).
assert len(nusc.sample_annotation) > 0, \
'Error: You are trying to evaluate on the test set but you do not have the annotations!'
index_map = {}
for scene in nusc.scene:
first_sample_token = scene['first_sample_token']
sample = nusc.get('sample', first_sample_token)
index_map[first_sample_token] = 1
index = 2
while sample['next'] != '':
sample = nusc.get('sample', sample['next'])
index_map[sample['token']] = index
index += 1
sample_tokens = []
for sample_token in sample_tokens_all:
scene_token = nusc.get('sample', sample_token)['scene_token']
scene_record = nusc.get('scene', scene_token)
if scene_record['name'] in splits[eval_split]:
sample_tokens.append(sample_token)
all_annotations = EvalBoxes()
# Load annotations and filter predictions and annotations.
tracking_id_set = set()
for sample_token in tqdm.tqdm(sample_tokens, leave=verbose):
sample = nusc.get('sample', sample_token)
sample_annotation_tokens = sample['anns']
sample_boxes = []
for sample_annotation_token in sample_annotation_tokens:
sample_annotation = nusc.get('sample_annotation', sample_annotation_token)
if box_cls == DetectionBox_modified:
# Get label name in detection task and filter unused labels.
detection_name = category_to_detection_name(sample_annotation['category_name'])
if detection_name is None:
continue
# Get attribute_name.
attr_tokens = sample_annotation['attribute_tokens']
attr_count = len(attr_tokens)
if attr_count == 0:
attribute_name = ''
elif attr_count == 1:
attribute_name = attribute_map[attr_tokens[0]]
else:
raise Exception('Error: GT annotations must not have more than one attribute!')
sample_boxes.append(
box_cls(
token=sample_annotation_token,
sample_token=sample_token,
translation=sample_annotation['translation'],
size=sample_annotation['size'],
rotation=sample_annotation['rotation'],
velocity=nusc.box_velocity(sample_annotation['token'])[:2],
num_pts=sample_annotation['num_lidar_pts'] + sample_annotation['num_radar_pts'],
detection_name=detection_name,
detection_score=-1.0, # GT samples do not have a score.
attribute_name=attribute_name,
visibility=sample_annotation['visibility_token'],
index=index_map[sample_token]
)
)
elif box_cls == TrackingBox:
assert False
else:
raise NotImplementedError('Error: Invalid box_cls %s!' % box_cls)
all_annotations.add_boxes(sample_token, sample_boxes)
if verbose:
print("Loaded ground truth annotations for {} samples.".format(len(all_annotations.sample_tokens)))
return all_annotations
def filter_eval_boxes_by_id(nusc: NuScenes,
eval_boxes: EvalBoxes,
id=None,
verbose: bool = False) -> EvalBoxes:
"""
Applies filtering to boxes. Distance, bike-racks and points per box.
:param nusc: An instance of the NuScenes class.
:param eval_boxes: An instance of the EvalBoxes class.
:param is: the anns token set that used to keep bboxes.
:param verbose: Whether to print to stdout.
"""
# Accumulators for number of filtered boxes.
total, anns_filter = 0, 0
for ind, sample_token in enumerate(eval_boxes.sample_tokens):
# Filter on anns
total += len(eval_boxes[sample_token])
filtered_boxes = []
for box in eval_boxes[sample_token]:
if box.token in id:
filtered_boxes.append(box)
anns_filter += len(filtered_boxes)
eval_boxes.boxes[sample_token] = filtered_boxes
if verbose:
print("=> Original number of boxes: %d" % total)
print("=> After anns based filtering: %d" % anns_filter)
return eval_boxes
def filter_eval_boxes_by_visibility(
ori_eval_boxes: EvalBoxes,
visibility=None,
verbose: bool = False) -> EvalBoxes:
"""
Applies filtering to boxes. Distance, bike-racks and points per box.
:param nusc: An instance of the NuScenes class.
:param eval_boxes: An instance of the EvalBoxes class.
:param is: the anns token set that used to keep bboxes.
:param verbose: Whether to print to stdout.
"""
# Accumulators for number of filtered boxes.
eval_boxes = copy.deepcopy(ori_eval_boxes)
total, anns_filter = 0, 0
for ind, sample_token in enumerate(eval_boxes.sample_tokens):
# Filter on anns
total += len(eval_boxes[sample_token])
filtered_boxes = []
for box in eval_boxes[sample_token]:
if box.visibility == visibility:
filtered_boxes.append(box)
anns_filter += len(filtered_boxes)
eval_boxes.boxes[sample_token] = filtered_boxes
if verbose:
print("=> Original number of boxes: %d" % total)
print("=> After visibility based filtering: %d" % anns_filter)
return eval_boxes
def filter_by_sample_token(ori_eval_boxes, valid_sample_tokens=[], verbose=False):
eval_boxes = copy.deepcopy(ori_eval_boxes)
for sample_token in eval_boxes.sample_tokens:
if sample_token not in valid_sample_tokens:
eval_boxes.boxes.pop(sample_token)
return eval_boxes
def filter_eval_boxes_by_overlap(nusc: NuScenes,
eval_boxes: EvalBoxes,
verbose: bool = False) -> EvalBoxes:
"""
Applies filtering to boxes. basedon overlap .
:param nusc: An instance of the NuScenes class.
:param eval_boxes: An instance of the EvalBoxes class.
:param verbose: Whether to print to stdout.
"""
# Accumulators for number of filtered boxes.
cams = ['CAM_FRONT',
'CAM_FRONT_RIGHT',
'CAM_BACK_RIGHT',
'CAM_BACK',
'CAM_BACK_LEFT',
'CAM_FRONT_LEFT']
total, anns_filter = 0, 0
for ind, sample_token in enumerate(eval_boxes.sample_tokens):
# Filter on anns
total += len(eval_boxes[sample_token])
sample_record = nusc.get('sample', sample_token)
filtered_boxes = []
for box in eval_boxes[sample_token]:
count = 0
for cam in cams:
'''
copy-paste form nuscens
'''
sample_data_token = sample_record['data'][cam]
sd_record = nusc.get('sample_data', sample_data_token)
cs_record = nusc.get('calibrated_sensor', sd_record['calibrated_sensor_token'])
sensor_record = nusc.get('sensor', cs_record['sensor_token'])
pose_record = nusc.get('ego_pose', sd_record['ego_pose_token'])
cam_intrinsic = np.array(cs_record['camera_intrinsic'])
imsize = (sd_record['width'], sd_record['height'])
new_box = Box(box.translation, box.size, Quaternion(box.rotation),
name=box.detection_name, token='')
# Move box to ego vehicle coord system.
new_box.translate(-np.array(pose_record['translation']))
new_box.rotate(Quaternion(pose_record['rotation']).inverse)
# Move box to sensor coord system.
new_box.translate(-np.array(cs_record['translation']))
new_box.rotate(Quaternion(cs_record['rotation']).inverse)
if center_in_image(new_box, cam_intrinsic, imsize, vis_level=BoxVisibility.ANY):
count += 1
# if exist_corners_in_image_but_not_all(new_box, cam_intrinsic, imsize, vis_level=BoxVisibility.ANY):
# count += 1
if count > 1:
with open('center_overlap.txt', 'a') as f:
try:
f.write(box.token + '\n')
except:
pass
filtered_boxes.append(box)
anns_filter += len(filtered_boxes)
eval_boxes.boxes[sample_token] = filtered_boxes
verbose = True
if verbose:
print("=> Original number of boxes: %d" % total)
print("=> After anns based filtering: %d" % anns_filter)
return eval_boxes
class NuScenesEval_custom(NuScenesEval):
"""
Dummy class for backward-compatibility. Same as DetectionEval.
"""
def __init__(self,
nusc: NuScenes,
config: DetectionConfig,
result_path: str,
eval_set: str,
output_dir: str = None,
verbose: bool = True,
overlap_test=False,
eval_mask=False,
data_infos=None
):
"""
Initialize a DetectionEval object.
:param nusc: A NuScenes object.
:param config: A DetectionConfig object.
:param result_path: Path of the nuScenes JSON result file.
:param eval_set: The dataset split to evaluate on, e.g. train, val or test.
:param output_dir: Folder to save plots and results to.
:param verbose: Whether to print to stdout.
"""
self.nusc = nusc
self.result_path = result_path
self.eval_set = eval_set
self.output_dir = output_dir
self.verbose = verbose
self.cfg = config
self.overlap_test = overlap_test
self.eval_mask = eval_mask
self.data_infos = data_infos
# Check result file exists.
assert os.path.exists(result_path), 'Error: The result file does not exist!'
# Make dirs.
self.plot_dir = os.path.join(self.output_dir, 'plots')
if not os.path.isdir(self.output_dir):
os.makedirs(self.output_dir)
if not os.path.isdir(self.plot_dir):
os.makedirs(self.plot_dir)
# Load data.
if verbose:
print('Initializing nuScenes detection evaluation')
self.pred_boxes, self.meta = load_prediction(self.result_path, self.cfg.max_boxes_per_sample, DetectionBox,
verbose=verbose)
self.gt_boxes = load_gt(self.nusc, self.eval_set, DetectionBox_modified, verbose=verbose)
assert set(self.pred_boxes.sample_tokens) == set(self.gt_boxes.sample_tokens), \
"Samples in split doesn't match samples in predictions."
# Add center distances.
self.pred_boxes = add_center_dist(nusc, self.pred_boxes)
self.gt_boxes = add_center_dist(nusc, self.gt_boxes)
# Filter boxes (distance, points per box, etc.).
if verbose:
print('Filtering predictions')
self.pred_boxes = filter_eval_boxes(nusc, self.pred_boxes, self.cfg.class_range, verbose=verbose)
if verbose:
print('Filtering ground truth annotations')
self.gt_boxes = filter_eval_boxes(nusc, self.gt_boxes, self.cfg.class_range, verbose=verbose)
if self.overlap_test:
self.pred_boxes = filter_eval_boxes_by_overlap(self.nusc, self.pred_boxes)
self.gt_boxes = filter_eval_boxes_by_overlap(self.nusc, self.gt_boxes, verbose=True)
self.all_gt = copy.deepcopy(self.gt_boxes)
self.all_preds = copy.deepcopy(self.pred_boxes)
self.sample_tokens = self.gt_boxes.sample_tokens
self.index_map = {}
for scene in nusc.scene:
first_sample_token = scene['first_sample_token']
sample = nusc.get('sample', first_sample_token)
self.index_map[first_sample_token] = 1
index = 2
while sample['next'] != '':
sample = nusc.get('sample', sample['next'])
self.index_map[sample['token']] = index
index += 1
def update_gt(self, type_='vis', visibility='1', index=1):
if type_ == 'vis':
self.visibility_test = True
if self.visibility_test:
'''[{'description': 'visibility of whole object is between 0 and 40%',
'token': '1',
'level': 'v0-40'},
{'description': 'visibility of whole object is between 40 and 60%',
'token': '2',
'level': 'v40-60'},
{'description': 'visibility of whole object is between 60 and 80%',
'token': '3',
'level': 'v60-80'},
{'description': 'visibility of whole object is between 80 and 100%',
'token': '4',
'level': 'v80-100'}]'''
self.gt_boxes = filter_eval_boxes_by_visibility(self.all_gt, visibility, verbose=True)
elif type_ == 'ord':
valid_tokens = [key for (key, value) in self.index_map.items() if value == index]
# from IPython import embed
# embed()
self.gt_boxes = filter_by_sample_token(self.all_gt, valid_tokens)
self.pred_boxes = filter_by_sample_token(self.all_preds, valid_tokens)
self.sample_tokens = self.gt_boxes.sample_tokens
def evaluate(self) -> Tuple[DetectionMetrics, DetectionMetricDataList]:
"""
Performs the actual evaluation.
:return: A tuple of high-level and the raw metric data.
"""
start_time = time.time()
# -----------------------------------
# Step 1: Accumulate metric data for all classes and distance thresholds.
# -----------------------------------
if self.verbose:
print('Accumulating metric data...')
metric_data_list = DetectionMetricDataList()
# print(self.cfg.dist_fcn_callable, self.cfg.dist_ths)
# self.cfg.dist_ths = [0.3]
# self.cfg.dist_fcn_callable
for class_name in self.cfg.class_names:
for dist_th in self.cfg.dist_ths:
md = accumulate(self.gt_boxes, self.pred_boxes, class_name, self.cfg.dist_fcn_callable, dist_th)
metric_data_list.set(class_name, dist_th, md)
# -----------------------------------
# Step 2: Calculate metrics from the data.
# -----------------------------------
if self.verbose:
print('Calculating metrics...')
metrics = DetectionMetrics(self.cfg)
for class_name in self.cfg.class_names:
# Compute APs.
for dist_th in self.cfg.dist_ths:
metric_data = metric_data_list[(class_name, dist_th)]
ap = calc_ap(metric_data, self.cfg.min_recall, self.cfg.min_precision)
metrics.add_label_ap(class_name, dist_th, ap)
# Compute TP metrics.
for metric_name in TP_METRICS:
metric_data = metric_data_list[(class_name, self.cfg.dist_th_tp)]
if class_name in ['traffic_cone'] and metric_name in ['attr_err', 'vel_err', 'orient_err']:
tp = np.nan
elif class_name in ['barrier'] and metric_name in ['attr_err', 'vel_err']:
tp = np.nan
else:
tp = calc_tp(metric_data, self.cfg.min_recall, metric_name)
metrics.add_label_tp(class_name, metric_name, tp)
# Compute evaluation time.
metrics.add_runtime(time.time() - start_time)
return metrics, metric_data_list
def render(self, metrics: DetectionMetrics, md_list: DetectionMetricDataList) -> None:
"""
Renders various PR and TP curves.
:param metrics: DetectionMetrics instance.
:param md_list: DetectionMetricDataList instance.
"""
if self.verbose:
print('Rendering PR and TP curves')
def savepath(name):
return os.path.join(self.plot_dir, name + '.pdf')
summary_plot(md_list, metrics, min_precision=self.cfg.min_precision, min_recall=self.cfg.min_recall,
dist_th_tp=self.cfg.dist_th_tp, savepath=savepath('summary'))
for detection_name in self.cfg.class_names:
class_pr_curve(md_list, metrics, detection_name, self.cfg.min_precision, self.cfg.min_recall,
savepath=savepath(detection_name + '_pr'))
class_tp_curve(md_list, metrics, detection_name, self.cfg.min_recall, self.cfg.dist_th_tp,
savepath=savepath(detection_name + '_tp'))
for dist_th in self.cfg.dist_ths:
dist_pr_curve(md_list, metrics, dist_th, self.cfg.min_precision, self.cfg.min_recall,
savepath=savepath('dist_pr_' + str(dist_th)))
if __name__ == "__main__":
# Settings.
parser = argparse.ArgumentParser(description='Evaluate nuScenes detection results.',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('result_path', type=str, help='The submission as a JSON file.')
parser.add_argument('--output_dir', type=str, default='~/nuscenes-metrics',
help='Folder to store result metrics, graphs and example visualizations.')
parser.add_argument('--eval_set', type=str, default='val',
help='Which dataset split to evaluate on, train, val or test.')
parser.add_argument('--dataroot', type=str, default='data/nuscenes',
help='Default nuScenes data directory.')
parser.add_argument('--version', type=str, default='v1.0-trainval',
help='Which version of the nuScenes dataset to evaluate on, e.g. v1.0-trainval.')
parser.add_argument('--config_path', type=str, default='',
help='Path to the configuration file.'
'If no path given, the CVPR 2019 configuration will be used.')
parser.add_argument('--plot_examples', type=int, default=0,
help='How many example visualizations to write to disk.')
parser.add_argument('--render_curves', type=int, default=1,
help='Whether to render PR and TP curves to disk.')
parser.add_argument('--verbose', type=int, default=1,
help='Whether to print to stdout.')
args = parser.parse_args()
result_path_ = os.path.expanduser(args.result_path)
output_dir_ = os.path.expanduser(args.output_dir)
eval_set_ = args.eval_set
dataroot_ = args.dataroot
version_ = args.version
config_path = args.config_path
plot_examples_ = args.plot_examples
render_curves_ = bool(args.render_curves)
verbose_ = bool(args.verbose)
if config_path == '':
cfg_ = config_factory('detection_cvpr_2019')
else:
with open(config_path, 'r') as _f:
cfg_ = DetectionConfig.deserialize(json.load(_f))
nusc_ = NuScenes(version=version_, verbose=verbose_, dataroot=dataroot_)
nusc_eval = NuScenesEval_custom(nusc_, config=cfg_, result_path=result_path_, eval_set=eval_set_,
output_dir=output_dir_, verbose=verbose_)
for vis in ['1', '2', '3', '4']:
nusc_eval.update_gt(type_='vis', visibility=vis)
print(f'================ {vis} ===============')
nusc_eval.main(plot_examples=plot_examples_, render_curves=render_curves_)
#for index in range(1, 41):
# nusc_eval.update_gt(type_='ord', index=index)
#
docker-hub/MapTRv2/MapTR/projects/mmdet3d_plugin/datasets/pipelines/__init__.py 0 → 100644
View file @ 19472568
from .transform_3d import (
PadMultiViewImage, PadMultiViewImageDepth, NormalizeMultiviewImage,
PhotoMetricDistortionMultiViewImage, CustomCollect3D, RandomScaleImageMultiViewImage, CustomPointsRangeFilter)
from .formating import CustomDefaultFormatBundle3D
from .loading import CustomLoadPointsFromFile, CustomLoadPointsFromMultiSweeps, CustomLoadMultiViewImageFromFiles, CustomPointToMultiViewDepth
__all__ = [
'PadMultiViewImage', 'NormalizeMultiviewImage',
'PhotoMetricDistortionMultiViewImage', 'CustomDefaultFormatBundle3D', 'CustomCollect3D', 'RandomScaleImageMultiViewImage'
]
\ No newline at end of file
docker-hub/MapTRv2/MapTR/projects/mmdet3d_plugin/datasets/pipelines/formating.py 0 → 100644
View file @ 19472568
# Copyright (c) OpenMMLab. All rights reserved.
import numpy as np
from mmcv.parallel import DataContainer as DC
from mmdet3d.core.bbox import BaseInstance3DBoxes
from mmdet3d.core.points import BasePoints
from mmdet.datasets.builder import PIPELINES
from mmdet.datasets.pipelines import to_tensor
from mmdet3d.datasets.pipelines import DefaultFormatBundle3D
@PIPELINES.register_module()
class CustomDefaultFormatBundle3D(DefaultFormatBundle3D):
"""Default formatting bundle.
It simplifies the pipeline of formatting common fields for voxels,
including "proposals", "gt_bboxes", "gt_labels", "gt_masks" and
"gt_semantic_seg".
These fields are formatted as follows.
- img: (1)transpose, (2)to tensor, (3)to DataContainer (stack=True)
- proposals: (1)to tensor, (2)to DataContainer
- gt_bboxes: (1)to tensor, (2)to DataContainer
- gt_bboxes_ignore: (1)to tensor, (2)to DataContainer
- gt_labels: (1)to tensor, (2)to DataContainer
"""
def __call__(self, results):
"""Call function to transform and format common fields in results.
Args:
results (dict): Result dict contains the data to convert.
Returns:
dict: The result dict contains the data that is formatted with
default bundle.
"""
# Format 3D data
results = super(CustomDefaultFormatBundle3D, self).__call__(results)
results['gt_map_masks'] = DC(
to_tensor(results['gt_map_masks']), stack=True)
return results
\ No newline at end of file
docker-hub/MapTRv2/MapTR/projects/mmdet3d_plugin/datasets/pipelines/loading.py 0 → 100644
View file @ 19472568
import os
from typing import Any, Dict, Tuple
import mmcv
import numpy as np
from nuscenes.map_expansion.map_api import NuScenesMap
from nuscenes.map_expansion.map_api import locations as LOCATIONS
from PIL import Image
from mmdet3d.core.points import BasePoints, get_points_type
from mmdet.datasets.builder import PIPELINES
from mmdet.datasets.pipelines import LoadAnnotations
from .loading_utils import load_augmented_point_cloud, reduce_LiDAR_beams
import torch
from pyquaternion import Quaternion
@PIPELINES.register_module()
class CustomLoadMultiViewImageFromFiles(object):
"""Load multi channel images from a list of separate channel files.
Expects results['img_filename'] to be a list of filenames.
Args:
to_float32 (bool): Whether to convert the img to float32.
Defaults to False.
color_type (str): Color type of the file. Defaults to 'unchanged'.
"""
def __init__(self, to_float32=False, padding=True,pad_val=128, color_type='unchanged'):
self.to_float32 = to_float32
self.color_type = color_type
self.padding = padding
self.pad_val = pad_val
def __call__(self, results):
"""Call function to load multi-view image from files.
Args:
results (dict): Result dict containing multi-view image filenames.
Returns:
dict: The result dict containing the multi-view image data. \
Added keys and values are described below.
- filename (str): Multi-view image filenames.
- img (np.ndarray): Multi-view image arrays.
- img_shape (tuple[int]): Shape of multi-view image arrays.
- ori_shape (tuple[int]): Shape of original image arrays.
- pad_shape (tuple[int]): Shape of padded image arrays.
- scale_factor (float): Scale factor.
- img_norm_cfg (dict): Normalization configuration of images.
"""
filename = results['img_filename']
# img is of shape (h, w, c, num_views)
# img = np.stack(
# [mmcv.imread(name, self.color_type) for name in filename], axis=-1)
img_list = [mmcv.imread(name, self.color_type) for name in filename]
img_shape_list = [img.shape for img in img_list]
max_h = max([shape[0] for shape in img_shape_list])
max_w = max([shape[1] for shape in img_shape_list])
size = (max_h, max_w)
# import pdb;pdb.set_trace()
img_list = [mmcv.impad(
img, shape=size, pad_val=self.pad_val) for img in img_list]
img = np.stack(img_list,axis=-1)
if self.to_float32:
img = img.astype(np.float32)
results['filename'] = filename
# unravel to list, see `DefaultFormatBundle` in formating.py
# which will transpose each image separately and then stack into array
results['img'] = [img[..., i] for i in range(img.shape[-1])]
results['img_shape'] = img.shape
results['ori_shape'] = img.shape
# Set initial values for default meta_keys
results['pad_shape'] = img.shape
results['scale_factor'] = 1.0
num_channels = 1 if len(img.shape) < 3 else img.shape[2]
results['img_norm_cfg'] = dict(
mean=np.zeros(num_channels, dtype=np.float32),
std=np.ones(num_channels, dtype=np.float32),
to_rgb=False)
return results
def __repr__(self):
"""str: Return a string that describes the module."""
repr_str = self.__class__.__name__
repr_str += f'(to_float32={self.to_float32}, '
repr_str += f"color_type='{self.color_type}')"
return repr_str
@PIPELINES.register_module()
class CustomLoadPointsFromMultiSweeps:
"""Load points from multiple sweeps.
This is usually used for nuScenes dataset to utilize previous sweeps.
Args:
sweeps_num (int): Number of sweeps. Defaults to 10.
load_dim (int): Dimension number of the loaded points. Defaults to 5.
use_dim (list[int]): Which dimension to use. Defaults to [0, 1, 2, 4].
pad_empty_sweeps (bool): Whether to repeat keyframe when
sweeps is empty. Defaults to False.
remove_close (bool): Whether to remove close points.
Defaults to False.
test_mode (bool): If test_model=True used for testing, it will not
randomly sample sweeps but select the nearest N frames.
Defaults to False.
"""
def __init__(
self,
sweeps_num=10,
load_dim=5,
use_dim=[0, 1, 2, 4],
pad_empty_sweeps=False,
remove_close=False,
test_mode=False,
load_augmented=None,
reduce_beams=None,
):
self.load_dim = load_dim
self.sweeps_num = sweeps_num
if isinstance(use_dim, int):
use_dim = list(range(use_dim))
self.use_dim = use_dim
self.pad_empty_sweeps = pad_empty_sweeps
self.remove_close = remove_close
self.test_mode = test_mode
self.load_augmented = load_augmented
self.reduce_beams = reduce_beams
def _load_points(self, lidar_path):
"""Private function to load point clouds data.
Args:
lidar_path (str): Filename of point clouds data.
Returns:
np.ndarray: An array containing point clouds data.
"""
mmcv.check_file_exist(lidar_path)
if self.load_augmented:
assert self.load_augmented in ["pointpainting", "mvp"]
virtual = self.load_augmented == "mvp"
points = load_augmented_point_cloud(
lidar_path, virtual=virtual, reduce_beams=self.reduce_beams
)
elif lidar_path.endswith(".npy"):
points = np.load(lidar_path)
else:
points = np.fromfile(lidar_path, dtype=np.float32)
return points
def _remove_close(self, points, radius=1.0):
"""Removes point too close within a certain radius from origin.
Args:
points (np.ndarray | :obj:`BasePoints`): Sweep points.
radius (float): Radius below which points are removed.
Defaults to 1.0.
Returns:
np.ndarray: Points after removing.
"""
if isinstance(points, np.ndarray):
points_numpy = points
elif isinstance(points, BasePoints):
points_numpy = points.tensor.numpy()
else:
raise NotImplementedError
x_filt = np.abs(points_numpy[:, 0]) < radius
y_filt = np.abs(points_numpy[:, 1]) < radius
not_close = np.logical_not(np.logical_and(x_filt, y_filt))
return points[not_close]
def __call__(self, results):
"""Call function to load multi-sweep point clouds from files.
Args:
results (dict): Result dict containing multi-sweep point cloud \
filenames.
Returns:
dict: The result dict containing the multi-sweep points data. \
Added key and value are described below.
- points (np.ndarray | :obj:`BasePoints`): Multi-sweep point \
cloud arrays.
"""
points = results["points"]
points.tensor[:, 4] = 0
sweep_points_list = [points]
ts = results["timestamp"] / 1e6
if self.pad_empty_sweeps and len(results["sweeps"]) == 0:
for i in range(self.sweeps_num):
if self.remove_close:
sweep_points_list.append(self._remove_close(points))
else:
sweep_points_list.append(points)
else:
if len(results["sweeps"]) <= self.sweeps_num:
choices = np.arange(len(results["sweeps"]))
elif self.test_mode:
choices = np.arange(self.sweeps_num)
else:
# NOTE: seems possible to load frame -11?
if not self.load_augmented:
choices = np.random.choice(
len(results["sweeps"]), self.sweeps_num, replace=False
)
else:
# don't allow to sample the earliest frame, match with Tianwei's implementation.
choices = np.random.choice(
len(results["sweeps"]) - 1, self.sweeps_num, replace=False
)
for idx in choices:
sweep = results["sweeps"][idx]
points_sweep = self._load_points(sweep["data_path"])
points_sweep = np.copy(points_sweep).reshape(-1, self.load_dim)
# TODO: make it more general
if self.reduce_beams and self.reduce_beams < 32:
points_sweep = reduce_LiDAR_beams(points_sweep, self.reduce_beams)
if self.remove_close:
points_sweep = self._remove_close(points_sweep)
sweep_ts = sweep["timestamp"] / 1e6
points_sweep[:, :3] = (
points_sweep[:, :3] @ sweep["sensor2lidar_rotation"].T
)
points_sweep[:, :3] += sweep["sensor2lidar_translation"]
points_sweep[:, 4] = ts - sweep_ts
points_sweep = points.new_point(points_sweep)
sweep_points_list.append(points_sweep)
points = points.cat(sweep_points_list)
points = points[:, self.use_dim]
results["points"] = points
return results
def __repr__(self):
"""str: Return a string that describes the module."""
return f"{self.__class__.__name__}(sweeps_num={self.sweeps_num})"
@PIPELINES.register_module()
class CustomLoadPointsFromFile:
"""Load Points From File.
Load sunrgbd and scannet points from file.
Args:
coord_type (str): The type of coordinates of points cloud.
Available options includes:
- 'LIDAR': Points in LiDAR coordinates.
- 'DEPTH': Points in depth coordinates, usually for indoor dataset.
- 'CAMERA': Points in camera coordinates.
load_dim (int): The dimension of the loaded points.
Defaults to 6.
use_dim (list[int]): Which dimensions of the points to be used.
Defaults to [0, 1, 2]. For KITTI dataset, set use_dim=4
or use_dim=[0, 1, 2, 3] to use the intensity dimension.
shift_height (bool): Whether to use shifted height. Defaults to False.
use_color (bool): Whether to use color features. Defaults to False.
"""
def __init__(
self,
coord_type,
load_dim=6,
use_dim=[0, 1, 2],
shift_height=False,
use_color=False,
load_augmented=None,
reduce_beams=None,
):
self.shift_height = shift_height
self.use_color = use_color
if isinstance(use_dim, int):
use_dim = list(range(use_dim))
assert (
max(use_dim) < load_dim
), f"Expect all used dimensions < {load_dim}, got {use_dim}"
assert coord_type in ["CAMERA", "LIDAR", "DEPTH"]
self.coord_type = coord_type
self.load_dim = load_dim
self.use_dim = use_dim
self.load_augmented = load_augmented
self.reduce_beams = reduce_beams
def _load_points(self, lidar_path):
"""Private function to load point clouds data.
Args:
lidar_path (str): Filename of point clouds data.
Returns:
np.ndarray: An array containing point clouds data.
"""
mmcv.check_file_exist(lidar_path)
if self.load_augmented:
assert self.load_augmented in ["pointpainting", "mvp"]
virtual = self.load_augmented == "mvp"
points = load_augmented_point_cloud(
lidar_path, virtual=virtual, reduce_beams=self.reduce_beams
)
elif lidar_path.endswith(".npy"):
points = np.load(lidar_path)
else:
points = np.fromfile(lidar_path, dtype=np.float32)
return points
def __call__(self, results):
"""Call function to load points data from file.
Args:
results (dict): Result dict containing point clouds data.
Returns:
dict: The result dict containing the point clouds data. \
Added key and value are described below.
- points (:obj:`BasePoints`): Point clouds data.
"""
lidar_path = results["lidar_path"]
points = self._load_points(lidar_path)
points = points.reshape(-1, self.load_dim)
# TODO: make it more general
if self.reduce_beams and self.reduce_beams < 32:
points = reduce_LiDAR_beams(points, self.reduce_beams)
points = points[:, self.use_dim]
attribute_dims = None
if self.shift_height:
floor_height = np.percentile(points[:, 2], 0.99)
height = points[:, 2] - floor_height
points = np.concatenate(
[points[:, :3], np.expand_dims(height, 1), points[:, 3:]], 1
)
attribute_dims = dict(height=3)
if self.use_color:
assert len(self.use_dim) >= 6
if attribute_dims is None:
attribute_dims = dict()
attribute_dims.update(
dict(
color=[
points.shape[1] - 3,
points.shape[1] - 2,
points.shape[1] - 1,
]
)
)
points_class = get_points_type(self.coord_type)
points = points_class(
points, points_dim=points.shape[-1], attribute_dims=attribute_dims
)
results["points"] = points
return results
@PIPELINES.register_module()
class CustomPointToMultiViewDepth(object):
def __init__(self, grid_config, downsample=1):
self.downsample = downsample
self.grid_config = grid_config
def points2depthmap(self, points, height, width):
height, width = height // self.downsample, width // self.downsample
depth_map = torch.zeros((height, width), dtype=torch.float32)
coor = torch.round(points[:, :2] / self.downsample)
depth = points[:, 2]
kept1 = (coor[:, 0] >= 0) & (coor[:, 0] < width) & (
coor[:, 1] >= 0) & (coor[:, 1] < height) & (
depth < self.grid_config['depth'][1]) & (
depth >= self.grid_config['depth'][0])
coor, depth = coor[kept1], depth[kept1]
ranks = coor[:, 0] + coor[:, 1] * width
sort = (ranks + depth / 100.).argsort()
coor, depth, ranks = coor[sort], depth[sort], ranks[sort]
kept2 = torch.ones(coor.shape[0], device=coor.device, dtype=torch.bool)
kept2[1:] = (ranks[1:] != ranks[:-1])
coor, depth = coor[kept2], depth[kept2]
coor = coor.to(torch.long)
depth_map[coor[:, 1], coor[:, 0]] = depth
return depth_map
def __call__(self, results):
points_lidar = results['points']
imgs = np.stack(results['img'])
img_aug_matrix = results['img_aug_matrix']
post_rots = [torch.tensor(single_aug_matrix[:3, :3]).to(torch.float) for single_aug_matrix in img_aug_matrix]
post_trans = torch.stack([torch.tensor(single_aug_matrix[:3, 3]).to(torch.float) for single_aug_matrix in img_aug_matrix])
# import pdb;pdb.set_trace()
intrins = results['camera_intrinsics']
depth_map_list = []
for cid in range(len(imgs)):
# import pdb;pdb.set_trace()
lidar2lidarego = torch.tensor(results['lidar2ego']).to(torch.float32)
lidarego2global = np.eye(4, dtype=np.float32)
lidarego2global[:3, :3] = Quaternion(results['ego2global_rotation']).rotation_matrix
lidarego2global[:3, 3] = results['ego2global_translation']
lidarego2global = torch.from_numpy(lidarego2global)
cam2camego = torch.tensor(results['camera2ego'][cid])
camego2global = results['camego2global'][cid]
cam2img = torch.tensor(intrins[cid]).to(torch.float32)
lidar2cam = torch.inverse(camego2global.matmul(cam2camego)).matmul(
lidarego2global.matmul(lidar2lidarego))
lidar2img = cam2img.matmul(lidar2cam)
points_img = points_lidar.tensor[:, :3].matmul(
lidar2img[:3, :3].T.to(torch.float)) + lidar2img[:3, 3].to(torch.float).unsqueeze(0)
points_img = torch.cat(
[points_img[:, :2] / points_img[:, 2:3], points_img[:, 2:3]],
1)
points_img = points_img.matmul(
post_rots[cid].T) + post_trans[cid:cid + 1, :]
depth_map = self.points2depthmap(points_img, imgs.shape[1],
imgs.shape[2])
depth_map_list.append(depth_map)
depth_map = torch.stack(depth_map_list)
##################################################################
# global i
# import cv2
# for image_id in range(imgs.shape[0]):
# i+=1
# image = imgs[image_id]
# gt_depth_image = depth_map[image_id].numpy()
# gt_depth_image = np.expand_dims(gt_depth_image,2).repeat(3,2)
# #apply colormap on deoth image(image must be converted to 8-bit per pixel first)
# im_color=cv2.applyColorMap(cv2.convertScaleAbs(gt_depth_image,alpha=15),cv2.COLORMAP_JET)
# #convert to mat png
# image[gt_depth_image>0] = im_color[gt_depth_image>0]
# im=Image.fromarray(np.uint8(image))
# #save image
# im.save('visualize_1/visualize_{}.png'.format(i))
#################################################################
results['gt_depth'] = depth_map
return results
docker-hub/MapTRv2/MapTR/projects/mmdet3d_plugin/datasets/pipelines/loading_utils.py 0 → 100644
View file @ 19472568
import os
import numpy as np
import torch
__all__ = ["load_augmented_point_cloud", "reduce_LiDAR_beams"]
def load_augmented_point_cloud(path, virtual=False, reduce_beams=32):
# NOTE: following Tianwei's implementation, it is hard coded for nuScenes
points = np.fromfile(path, dtype=np.float32).reshape(-1, 5)
# NOTE: path definition different from Tianwei's implementation.
tokens = path.split("/")
vp_dir = "_VIRTUAL" if reduce_beams == 32 else f"_VIRTUAL_{reduce_beams}BEAMS"
seg_path = os.path.join(
*tokens[:-3],
"virtual_points",
tokens[-3],
tokens[-2] + vp_dir,
tokens[-1] + ".pkl.npy",
)
assert os.path.exists(seg_path)
data_dict = np.load(seg_path, allow_pickle=True).item()
virtual_points1 = data_dict["real_points"]
# NOTE: add zero reflectance to virtual points instead of removing them from real points
virtual_points2 = np.concatenate(
[
data_dict["virtual_points"][:, :3],
np.zeros([data_dict["virtual_points"].shape[0], 1]),
data_dict["virtual_points"][:, 3:],
],
axis=-1,
)
points = np.concatenate(
[
points,
np.ones([points.shape[0], virtual_points1.shape[1] - points.shape[1] + 1]),
],
axis=1,
)
virtual_points1 = np.concatenate(
[virtual_points1, np.zeros([virtual_points1.shape[0], 1])], axis=1
)
# note: this part is different from Tianwei's implementation, we don't have duplicate foreground real points.
if len(data_dict["real_points_indice"]) > 0:
points[data_dict["real_points_indice"]] = virtual_points1
if virtual:
virtual_points2 = np.concatenate(
[virtual_points2, -1 * np.ones([virtual_points2.shape[0], 1])], axis=1
)
points = np.concatenate([points, virtual_points2], axis=0).astype(np.float32)
return points
def reduce_LiDAR_beams(pts, reduce_beams_to=32):
# print(pts.size())
if isinstance(pts, np.ndarray):
pts = torch.from_numpy(pts)
radius = torch.sqrt(pts[:, 0].pow(2) + pts[:, 1].pow(2) + pts[:, 2].pow(2))
sine_theta = pts[:, 2] / radius
# [-pi/2, pi/2]
theta = torch.asin(sine_theta)
phi = torch.atan2(pts[:, 1], pts[:, 0])
top_ang = 0.1862
down_ang = -0.5353
beam_range = torch.zeros(32)
beam_range[0] = top_ang
beam_range[31] = down_ang
for i in range(1, 31):
beam_range[i] = beam_range[i - 1] - 0.023275
# beam_range = [1, 0.18, 0.15, 0.13, 0.11, 0.085, 0.065, 0.03, 0.01, -0.01, -0.03, -0.055, -0.08, -0.105, -0.13, -0.155, -0.18, -0.205, -0.228, -0.251, -0.275,
# -0.295, -0.32, -0.34, -0.36, -0.38, -0.40, -0.425, -0.45, -0.47, -0.49, -0.52, -0.54]
num_pts, _ = pts.size()
mask = torch.zeros(num_pts)
if reduce_beams_to == 16:
for id in [1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]:
beam_mask = (theta < (beam_range[id - 1] - 0.012)) * (
theta > (beam_range[id] - 0.012)
)
mask = mask + beam_mask
mask = mask.bool()
elif reduce_beams_to == 4:
for id in [7, 9, 11, 13]:
beam_mask = (theta < (beam_range[id - 1] - 0.012)) * (
theta > (beam_range[id] - 0.012)
)
mask = mask + beam_mask
mask = mask.bool()
# [?] pick the 14th beam
elif reduce_beams_to == 1:
chosen_beam_id = 9
mask = (theta < (beam_range[chosen_beam_id - 1] - 0.012)) * (
theta > (beam_range[chosen_beam_id] - 0.012)
)
else:
raise NotImplementedError
# points = copy.copy(pts)
points = pts[mask]
# print(points.size())
return points.numpy()
docker-hub/MapTRv2/MapTR/projects/mmdet3d_plugin/datasets/pipelines/transform_3d.py 0 → 100644
View file @ 19472568
import numpy as np
from numpy import random
import mmcv
from mmdet.datasets.builder import PIPELINES
from mmcv.parallel import DataContainer as DC
import torch
# dk_test
'''
from mmcv.parallel import DataContainer
@PIPELINES.register_module()
class TransposeImage:
def __init__(self, enforce_format='nhwc'):
self.enforce_format = enforce_format.lower()
assert self.enforce_format in ['nhwc', 'nchw']
def _convert_format(self, img):
if img.dim() == 3:
current_format = 'nchw' if img.size(0) in [1, 3] else 'nhwc'
else:
current_format = 'nchw' if img.size(1) in [1, 3] else 'nhwc'
if current_format != self.enforce_format:
if self.enforce_format == 'nhwc':
img = img.permute(0, 2, 3, 1) if img.dim() == 4 else img.permute(1, 2, 0)
else:
img = img.permute(0, 3, 1, 2) if img.dim() == 4 else img.permute(2, 0, 1)
return img.contiguous()
def __call__(self, results):
if 'img' not in results:
return results
img = results['img']
if isinstance(img, DataContainer):
# 创建新DataContainer避免修改原对象
results['img'] = DataContainer(
self._convert_format(img.data),
stack=img.stack,
pad_dims=img.pad_dims,
cpu_only=img.cpu_only
)
elif isinstance(img, list):
results['img'] = [self._convert_format(i) for i in img]
else:
results['img'] = self._convert_format(img)
return results
'''
from mmcv.parallel import DataContainer
@PIPELINES.register_module()
class TransposeImage:
def __call__(self, results):
if 'img' not in results:
return results
def convert(x):
if isinstance(x, torch.Tensor):
return x.contiguous(memory_format=torch.channels_last)
return x
img = results['img']
if isinstance(img, DataContainer):
results['img'] = DataContainer(
convert(img.data),
stack=img.stack,
pad_dims=img.pad_dims,
cpu_only=img.cpu_only
)
elif isinstance(img, list):
results['img'] = [convert(i) for i in img]
else:
results['img'] = convert(img)
return results
@PIPELINES.register_module()
class PadMultiViewImage(object):
"""Pad the multi-view image.
There are two padding modes: (1) pad to a fixed size and (2) pad to the
minimum size that is divisible by some number.
Added keys are "pad_shape", "pad_fixed_size", "pad_size_divisor",
Args:
size (tuple, optional): Fixed padding size.
size_divisor (int, optional): The divisor of padded size.
pad_val (float, optional): Padding value, 0 by default.
"""
def __init__(self, size=None, size_divisor=None, pad_val=0):
self.size = size
self.size_divisor = size_divisor
self.pad_val = pad_val
# only one of size and size_divisor should be valid
assert size is not None or size_divisor is not None
assert size is None or size_divisor is None
def _pad_img(self, results):
"""Pad images according to ``self.size``."""
if self.size is not None:
padded_img = [mmcv.impad(
img, shape=self.size, pad_val=self.pad_val) for img in results['img']]
elif self.size_divisor is not None:
padded_img = [mmcv.impad_to_multiple(
img, self.size_divisor, pad_val=self.pad_val) for img in results['img']]
results['ori_shape'] = [img.shape for img in results['img']]
results['img'] = padded_img
results['img_shape'] = [img.shape for img in padded_img]
results['pad_shape'] = [img.shape for img in padded_img]
results['pad_fixed_size'] = self.size
results['pad_size_divisor'] = self.size_divisor
def __call__(self, results):
"""Call function to pad images, masks, semantic segmentation maps.
Args:
results (dict): Result dict from loading pipeline.
Returns:
dict: Updated result dict.
"""
self._pad_img(results)
return results
def __repr__(self):
repr_str = self.__class__.__name__
repr_str += f'(size={self.size}, '
repr_str += f'size_divisor={self.size_divisor}, '
repr_str += f'pad_val={self.pad_val})'
return repr_str
@PIPELINES.register_module()
class PadMultiViewImageDepth(object):
"""Pad the multi-view image.
There are two padding modes: (1) pad to a fixed size and (2) pad to the
minimum size that is divisible by some number.
Added keys are "pad_shape", "pad_fixed_size", "pad_size_divisor",
Args:
size (tuple, optional): Fixed padding size.
size_divisor (int, optional): The divisor of padded size.
pad_val (float, optional): Padding value, 0 by default.
"""
def __init__(self, size=None, size_divisor=None, pad_val=0):
self.size = size
self.size_divisor = size_divisor
self.pad_val = pad_val
# only one of size and size_divisor should be valid
assert size is not None or size_divisor is not None
assert size is None or size_divisor is None
def _pad_img(self, results):
"""Pad images according to ``self.size``."""
if self.size is not None:
padded_img = [mmcv.impad(
img, shape=self.size, pad_val=self.pad_val) for img in results['img']]
padded_gt_depth = [mmcv.impad(
img, shape=self.size, pad_val=self.pad_val) for img in results['gt_depth']]
elif self.size_divisor is not None:
padded_img = [mmcv.impad_to_multiple(
img, self.size_divisor, pad_val=self.pad_val) for img in results['img']]
padded_gt_depth = [mmcv.impad_to_multiple(
img.numpy(), self.size_divisor, pad_val=self.pad_val) for img in results['gt_depth']]
results['ori_shape'] = [img.shape for img in results['img']]
results['img'] = padded_img
results['gt_depth'] = np.stack(padded_gt_depth)
results['img_shape'] = [img.shape for img in padded_img]
results['pad_shape'] = [img.shape for img in padded_img]
results['pad_fixed_size'] = self.size
results['pad_size_divisor'] = self.size_divisor
def __call__(self, results):
"""Call function to pad images, masks, semantic segmentation maps.
Args:
results (dict): Result dict from loading pipeline.
Returns:
dict: Updated result dict.
"""
self._pad_img(results)
return results
def __repr__(self):
repr_str = self.__class__.__name__
repr_str += f'(size={self.size}, '
repr_str += f'size_divisor={self.size_divisor}, '
repr_str += f'pad_val={self.pad_val})'
return repr_str
@PIPELINES.register_module()
class NormalizeMultiviewImage(object):
"""Normalize the image.
Added key is "img_norm_cfg".
Args:
mean (sequence): Mean values of 3 channels.
std (sequence): Std values of 3 channels.
to_rgb (bool): Whether to convert the image from BGR to RGB,
default is true.
"""
def __init__(self, mean, std, to_rgb=True):
self.mean = np.array(mean, dtype=np.float32)
self.std = np.array(std, dtype=np.float32)
self.to_rgb = to_rgb
def __call__(self, results):
"""Call function to normalize images.
Args:
results (dict): Result dict from loading pipeline.
Returns:
dict: Normalized results, 'img_norm_cfg' key is added into
result dict.
"""
results['img'] = [mmcv.imnormalize(img, self.mean, self.std, self.to_rgb) for img in results['img']]
results['img_norm_cfg'] = dict(
mean=self.mean, std=self.std, to_rgb=self.to_rgb)
return results
def __repr__(self):
repr_str = self.__class__.__name__
repr_str += f'(mean={self.mean}, std={self.std}, to_rgb={self.to_rgb})'
return repr_str
@PIPELINES.register_module()
class PhotoMetricDistortionMultiViewImage:
"""Apply photometric distortion to image sequentially, every transformation
is applied with a probability of 0.5. The position of random contrast is in
second or second to last.
1. random brightness
2. random contrast (mode 0)
3. convert color from BGR to HSV
4. random saturation
5. random hue
6. convert color from HSV to BGR
7. random contrast (mode 1)
8. randomly swap channels
Args:
brightness_delta (int): delta of brightness.
contrast_range (tuple): range of contrast.
saturation_range (tuple): range of saturation.
hue_delta (int): delta of hue.
"""
def __init__(self,
brightness_delta=32,
contrast_range=(0.5, 1.5),
saturation_range=(0.5, 1.5),
hue_delta=18):
self.brightness_delta = brightness_delta
self.contrast_lower, self.contrast_upper = contrast_range
self.saturation_lower, self.saturation_upper = saturation_range
self.hue_delta = hue_delta
def __call__(self, results):
"""Call function to perform photometric distortion on images.
Args:
results (dict): Result dict from loading pipeline.
Returns:
dict: Result dict with images distorted.
"""
imgs = results['img']
new_imgs = []
for img in imgs:
assert img.dtype == np.float32, \
'PhotoMetricDistortion needs the input image of dtype np.float32,'\
' please set "to_float32=True" in "LoadImageFromFile" pipeline'
# random brightness
if random.randint(2):
delta = random.uniform(-self.brightness_delta,
self.brightness_delta)
img += delta
# mode == 0 --> do random contrast first
# mode == 1 --> do random contrast last
mode = random.randint(2)
if mode == 1:
if random.randint(2):
alpha = random.uniform(self.contrast_lower,
self.contrast_upper)
img *= alpha
# convert color from BGR to HSV
img = mmcv.bgr2hsv(img)
# random saturation
if random.randint(2):
img[..., 1] *= random.uniform(self.saturation_lower,
self.saturation_upper)
# random hue
if random.randint(2):
img[..., 0] += random.uniform(-self.hue_delta, self.hue_delta)
img[..., 0][img[..., 0] > 360] -= 360
img[..., 0][img[..., 0] < 0] += 360
# convert color from HSV to BGR
img = mmcv.hsv2bgr(img)
# random contrast
if mode == 0:
if random.randint(2):
alpha = random.uniform(self.contrast_lower,
self.contrast_upper)
img *= alpha
# randomly swap channels
if random.randint(2):
img = img[..., random.permutation(3)]
new_imgs.append(img)
results['img'] = new_imgs
return results
def __repr__(self):
repr_str = self.__class__.__name__
repr_str += f'(\nbrightness_delta={self.brightness_delta},\n'
repr_str += 'contrast_range='
repr_str += f'{(self.contrast_lower, self.contrast_upper)},\n'
repr_str += 'saturation_range='
repr_str += f'{(self.saturation_lower, self.saturation_upper)},\n'
repr_str += f'hue_delta={self.hue_delta})'
return repr_str
@PIPELINES.register_module()
class CustomCollect3D(object):
"""Collect data from the loader relevant to the specific task.
This is usually the last stage of the data loader pipeline. Typically keys
is set to some subset of "img", "proposals", "gt_bboxes",
"gt_bboxes_ignore", "gt_labels", and/or "gt_masks".
The "img_meta" item is always populated. The contents of the "img_meta"
dictionary depends on "meta_keys". By default this includes:
- 'img_shape': shape of the image input to the network as a tuple \
(h, w, c). Note that images may be zero padded on the \
bottom/right if the batch tensor is larger than this shape.
- 'scale_factor': a float indicating the preprocessing scale
- 'flip': a boolean indicating if image flip transform was used
- 'filename': path to the image file
- 'ori_shape': original shape of the image as a tuple (h, w, c)
- 'pad_shape': image shape after padding
- 'lidar2img': transform from lidar to image
- 'depth2img': transform from depth to image
- 'cam2img': transform from camera to image
- 'pcd_horizontal_flip': a boolean indicating if point cloud is \
flipped horizontally
- 'pcd_vertical_flip': a boolean indicating if point cloud is \
flipped vertically
- 'box_mode_3d': 3D box mode
- 'box_type_3d': 3D box type
- 'img_norm_cfg': a dict of normalization information:
- mean: per channel mean subtraction
- std: per channel std divisor
- to_rgb: bool indicating if bgr was converted to rgb
- 'pcd_trans': point cloud transformations
- 'sample_idx': sample index
- 'pcd_scale_factor': point cloud scale factor
- 'pcd_rotation': rotation applied to point cloud
- 'pts_filename': path to point cloud file.
Args:
keys (Sequence[str]): Keys of results to be collected in ``data``.
meta_keys (Sequence[str], optional): Meta keys to be converted to
``mmcv.DataContainer`` and collected in ``data[img_metas]``.
Default: ('filename', 'ori_shape', 'img_shape', 'lidar2img',
'depth2img', 'cam2img', 'pad_shape', 'scale_factor', 'flip',
'pcd_horizontal_flip', 'pcd_vertical_flip', 'box_mode_3d',
'box_type_3d', 'img_norm_cfg', 'pcd_trans',
'sample_idx', 'pcd_scale_factor', 'pcd_rotation', 'pts_filename')
"""
def __init__(self,
keys,
meta_keys=('filename', 'ori_shape', 'img_shape', 'lidar2img',
'depth2img', 'cam2img', 'pad_shape',
'scale_factor', 'flip', 'pcd_horizontal_flip',
'pcd_vertical_flip', 'box_mode_3d', 'box_type_3d',
'img_norm_cfg', 'pcd_trans', 'sample_idx', 'prev_idx', 'next_idx',
'pcd_scale_factor', 'pcd_rotation', 'pts_filename',
'transformation_3d_flow', 'scene_token','camera_intrinsics',
'can_bus','lidar2global','cam2lidar','lidar2cam',
'camera2ego','cam_intrinsic','img_aug_matrix','lidar2ego', 'lidar_aug_matrix',
'timestamp','img_inputs', 'gt_bboxes_3d', 'gt_labels_3d','gt_depth'
)):
self.keys = keys
self.meta_keys = meta_keys
def __call__(self, results):
"""Call function to collect keys in results. The keys in ``meta_keys``
will be converted to :obj:`mmcv.DataContainer`.
Args:
results (dict): Result dict contains the data to collect.
Returns:
dict: The result dict contains the following keys
- keys in ``self.keys``
- ``img_metas``
"""
data = {}
img_metas = {}
# import pdb;pdb.set_trace()
for key in self.meta_keys:
if key in results:
img_metas[key] = results[key]
data['img_metas'] = DC(img_metas, cpu_only=True)
for key in self.keys:
data[key] = results[key]
return data
def __repr__(self):
"""str: Return a string that describes the module."""
return self.__class__.__name__ + \
f'(keys={self.keys}, meta_keys={self.meta_keys})'
@PIPELINES.register_module()
class RandomScaleImageMultiViewImage(object):
"""Random scale the image
Args:
scales
"""
def __init__(self, scales=[]):
self.scales = scales
assert len(self.scales)==1
def __call__(self, results):
"""Call function to pad images, masks, semantic segmentation maps.
Args:
results (dict): Result dict from loading pipeline.
Returns:
dict: Updated result dict.
"""
rand_ind = np.random.permutation(range(len(self.scales)))[0]
rand_scale = self.scales[rand_ind]
y_size = [int(img.shape[0] * rand_scale) for img in results['img']]
x_size = [int(img.shape[1] * rand_scale) for img in results['img']]
scale_factor = np.eye(4)
scale_factor[0, 0] *= rand_scale
scale_factor[1, 1] *= rand_scale
results['img'] = [mmcv.imresize(img, (x_size[idx], y_size[idx]), return_scale=False) for idx, img in
enumerate(results['img'])]
lidar2img = [scale_factor @ l2i for l2i in results['lidar2img']]
img_aug_matrix = [scale_factor for _ in results['lidar2img']]
results['lidar2img'] = lidar2img
results['img_aug_matrix'] = img_aug_matrix
results['img_shape'] = [img.shape for img in results['img']]
results['ori_shape'] = [img.shape for img in results['img']]
return results
def __repr__(self):
repr_str = self.__class__.__name__
repr_str += f'(size={self.scales}, '
return repr_str
@PIPELINES.register_module()
class CustomPointsRangeFilter:
"""Filter points by the range.
Args:
point_cloud_range (list[float]): Point cloud range.
"""
def __init__(self, point_cloud_range):
self.pcd_range = np.array(point_cloud_range, dtype=np.float32)
def __call__(self, data):
"""Call function to filter points by the range.
Args:
data (dict): Result dict from loading pipeline.
Returns:
dict: Results after filtering, 'points', 'pts_instance_mask' \
and 'pts_semantic_mask' keys are updated in the result dict.
"""
points = data["points"]
points_mask = points.in_range_3d(self.pcd_range)
clean_points = points[points_mask]
data["points"] = clean_points
return data
docker-hub/MapTRv2/MapTR/projects/mmdet3d_plugin/datasets/samplers/__init__.py 0 → 100644
View file @ 19472568
from .group_sampler import DistributedGroupSampler
from .distributed_sampler import DistributedSampler
from .sampler import SAMPLER, build_sampler
docker-hub/MapTRv2/MapTR/projects/mmdet3d_plugin/datasets/samplers/distributed_sampler.py 0 → 100644
View file @ 19472568
import math
import torch
from torch.utils.data import DistributedSampler as _DistributedSampler
from .sampler import SAMPLER
@SAMPLER.register_module()
class DistributedSampler(_DistributedSampler):
def __init__(self,
dataset=None,
num_replicas=None,
rank=None,
shuffle=True,
seed=0):
super().__init__(
dataset, num_replicas=num_replicas, rank=rank, shuffle=shuffle)
# for the compatibility from PyTorch 1.3+
self.seed = seed if seed is not None else 0
def __iter__(self):
# deterministically shuffle based on epoch
if self.shuffle:
assert False
else:
indices = torch.arange(len(self.dataset)).tolist()
# add extra samples to make it evenly divisible
# in case that indices is shorter than half of total_size
indices = (indices *
math.ceil(self.total_size / len(indices)))[:self.total_size]
assert len(indices) == self.total_size
# subsample
per_replicas = self.total_size//self.num_replicas
# indices = indices[self.rank:self.total_size:self.num_replicas]
indices = indices[self.rank*per_replicas:(self.rank+1)*per_replicas]
assert len(indices) == self.num_samples
return iter(indices)
docker-hub/MapTRv2/MapTR/projects/mmdet3d_plugin/datasets/samplers/group_sampler.py 0 → 100644
View file @ 19472568
# Copyright (c) OpenMMLab. All rights reserved.
import math
import numpy as np
import torch
from mmcv.runner import get_dist_info
from torch.utils.data import Sampler
from .sampler import SAMPLER
import random
from IPython import embed
@SAMPLER.register_module()
class DistributedGroupSampler(Sampler):
"""Sampler that restricts data loading to a subset of the dataset.
It is especially useful in conjunction with
:class:`torch.nn.parallel.DistributedDataParallel`. In such case, each
process can pass a DistributedSampler instance as a DataLoader sampler,
and load a subset of the original dataset that is exclusive to it.
.. note::
Dataset is assumed to be of constant size.
Arguments:
dataset: Dataset used for sampling.
num_replicas (optional): Number of processes participating in
distributed training.
rank (optional): Rank of the current process within num_replicas.
seed (int, optional): random seed used to shuffle the sampler if
``shuffle=True``. This number should be identical across all
processes in the distributed group. Default: 0.
"""
def __init__(self,
dataset,
samples_per_gpu=1,
num_replicas=None,
rank=None,
seed=0):
_rank, _num_replicas = get_dist_info()
if num_replicas is None:
num_replicas = _num_replicas
if rank is None:
rank = _rank
self.dataset = dataset
self.samples_per_gpu = samples_per_gpu
self.num_replicas = num_replicas
self.rank = rank
self.epoch = 0
self.seed = seed if seed is not None else 0
assert hasattr(self.dataset, 'flag')
self.flag = self.dataset.flag
self.group_sizes = np.bincount(self.flag)
self.num_samples = 0
for i, j in enumerate(self.group_sizes):
self.num_samples += int(
math.ceil(self.group_sizes[i] * 1.0 / self.samples_per_gpu /
self.num_replicas)) * self.samples_per_gpu
self.total_size = self.num_samples * self.num_replicas
def __iter__(self):
# deterministically shuffle based on epoch
g = torch.Generator()
g.manual_seed(self.epoch + self.seed)
indices = []
for i, size in enumerate(self.group_sizes):
if size > 0:
indice = np.where(self.flag == i)[0]
assert len(indice) == size
# add .numpy() to avoid bug when selecting indice in parrots.
# TODO: check whether torch.randperm() can be replaced by
# numpy.random.permutation().
indice = indice[list(
torch.randperm(int(size), generator=g).numpy())].tolist()
extra = int(
math.ceil(
size * 1.0 / self.samples_per_gpu / self.num_replicas)
) * self.samples_per_gpu * self.num_replicas - len(indice)
# pad indice
tmp = indice.copy()
for _ in range(extra // size):
indice.extend(tmp)
indice.extend(tmp[:extra % size])
indices.extend(indice)
assert len(indices) == self.total_size
indices = [
indices[j] for i in list(
torch.randperm(
len(indices) // self.samples_per_gpu, generator=g))
for j in range(i * self.samples_per_gpu, (i + 1) *
self.samples_per_gpu)
]
# subsample
offset = self.num_samples * self.rank
indices = indices[offset:offset + self.num_samples]
assert len(indices) == self.num_samples
return iter(indices)
def __len__(self):
return self.num_samples
def set_epoch(self, epoch):
self.epoch = epoch
docker-hub/MapTRv2/MapTR/projects/mmdet3d_plugin/datasets/samplers/sampler.py 0 → 100644
View file @ 19472568
from mmcv.utils.registry import Registry, build_from_cfg
SAMPLER = Registry('sampler')
def build_sampler(cfg, default_args):
return build_from_cfg(cfg, SAMPLER, default_args)
docker-hub/MapTRv2/MapTR/projects/mmdet3d_plugin/maptr/__init__.py 0 → 100644
View file @ 19472568
from .assigners import *
from .dense_heads import *
from .detectors import *
from .modules import *
from .losses import *
docker-hub/MapTRv2/MapTR/projects/mmdet3d_plugin/maptr/assigners/__init__.py 0 → 100644
View file @ 19472568
from .maptr_assigner import MapTRAssigner
\ No newline at end of file
docker-hub/MapTRv2/MapTR/projects/mmdet3d_plugin/maptr/assigners/maptr_assigner.py 0 → 100644
View file @ 19472568
import torch
from mmdet.core.bbox.builder import BBOX_ASSIGNERS
from mmdet.core.bbox.assigners import AssignResult
from mmdet.core.bbox.assigners import BaseAssigner
from mmdet.core.bbox.match_costs import build_match_cost
import torch.nn.functional as F
from mmdet.core.bbox.transforms import bbox_xyxy_to_cxcywh, bbox_cxcywh_to_xyxy
try:
from scipy.optimize import linear_sum_assignment
except ImportError:
linear_sum_assignment = None
def denormalize_3d_pts(pts, pc_range):
new_pts = pts.clone()
new_pts[...,0:1] = (pts[..., 0:1]*(pc_range[3] -
pc_range[0]) + pc_range[0])
new_pts[...,1:2] = (pts[...,1:2]*(pc_range[4] -
pc_range[1]) + pc_range[1])
new_pts[...,2:3] = (pts[...,2:3]*(pc_range[5] -
pc_range[2]) + pc_range[2])
return new_pts
def normalize_3d_pts(pts, pc_range):
patch_h = pc_range[4]-pc_range[1]
patch_w = pc_range[3]-pc_range[0]
patch_z = pc_range[5]-pc_range[2]
new_pts = pts.clone()
new_pts[...,0:1] = pts[..., 0:1] - pc_range[0]
new_pts[...,1:2] = pts[...,1:2] - pc_range[1]
new_pts[...,2:3] = pts[...,2:3] - pc_range[2]
factor = pts.new_tensor([patch_w, patch_h,patch_z])
normalized_pts = new_pts / factor
return normalized_pts
def normalize_2d_bbox(bboxes, pc_range):
patch_h = pc_range[4]-pc_range[1]
patch_w = pc_range[3]-pc_range[0]
cxcywh_bboxes = bbox_xyxy_to_cxcywh(bboxes)
cxcywh_bboxes[...,0:1] = cxcywh_bboxes[..., 0:1] - pc_range[0]
cxcywh_bboxes[...,1:2] = cxcywh_bboxes[...,1:2] - pc_range[1]
factor = bboxes.new_tensor([patch_w, patch_h,patch_w,patch_h])
normalized_bboxes = cxcywh_bboxes / factor
return normalized_bboxes
def normalize_2d_pts(pts, pc_range):
patch_h = pc_range[4]-pc_range[1]
patch_w = pc_range[3]-pc_range[0]
new_pts = pts.clone()
new_pts[...,0:1] = pts[..., 0:1] - pc_range[0]
new_pts[...,1:2] = pts[...,1:2] - pc_range[1]
factor = pts.new_tensor([patch_w, patch_h])
normalized_pts = new_pts / factor
return normalized_pts
def denormalize_2d_bbox(bboxes, pc_range):
bboxes = bbox_cxcywh_to_xyxy(bboxes)
bboxes[..., 0::2] = (bboxes[..., 0::2]*(pc_range[3] -
pc_range[0]) + pc_range[0])
bboxes[..., 1::2] = (bboxes[..., 1::2]*(pc_range[4] -
pc_range[1]) + pc_range[1])
return bboxes
def denormalize_2d_pts(pts, pc_range):
new_pts = pts.clone()
new_pts[...,0:1] = (pts[..., 0:1]*(pc_range[3] -
pc_range[0]) + pc_range[0])
new_pts[...,1:2] = (pts[...,1:2]*(pc_range[4] -
pc_range[1]) + pc_range[1])
return new_pts
@BBOX_ASSIGNERS.register_module()
class MapTRAssigner(BaseAssigner):
"""Computes one-to-one matching between predictions and ground truth.
This class computes an assignment between the targets and the predictions
based on the costs. The costs are weighted sum of three components:
classification cost, regression L1 cost and regression iou cost. The
targets don't include the no_object, so generally there are more
predictions than targets. After the one-to-one matching, the un-matched
are treated as backgrounds. Thus each query prediction will be assigned
with `0` or a positive integer indicating the ground truth index:
- 0: negative sample, no assigned gt
- positive integer: positive sample, index (1-based) of assigned gt
Args:
cls_weight (int | float, optional): The scale factor for classification
cost. Default 1.0.
bbox_weight (int | float, optional): The scale factor for regression
L1 cost. Default 1.0.
iou_weight (int | float, optional): The scale factor for regression
iou cost. Default 1.0.
iou_calculator (dict | optional): The config for the iou calculation.
Default type `BboxOverlaps2D`.
iou_mode (str | optional): "iou" (intersection over union), "iof"
(intersection over foreground), or "giou" (generalized
intersection over union). Default "giou".
"""
def __init__(self,
z_cfg = dict(
pred_z_flag=False,
gt_z_flag=False,
),
cls_cost=dict(type='ClassificationCost', weight=1.),
reg_cost=dict(type='BBoxL1Cost', weight=1.0),
iou_cost=dict(type='IoUCost', weight=0.0),
pts_cost=dict(type='ChamferDistance',loss_src_weight=1.0,loss_dst_weight=1.0),
pc_range=None):
self.z_cfg = z_cfg
self.cls_cost = build_match_cost(cls_cost)
self.reg_cost = build_match_cost(reg_cost)
self.iou_cost = build_match_cost(iou_cost)
self.pts_cost = build_match_cost(pts_cost)
self.pc_range = pc_range
@torch._dynamo.disable
def hungarian_match(self, cost, gt_labels, assigned_gt_inds, assigned_labels, num_gts, device):
cost = cost.detach().cpu()
if linear_sum_assignment is None:
raise ImportError('Please run "pip install scipy" '
'to install scipy first.')
matched_row_inds, matched_col_inds = linear_sum_assignment(cost)
matched_row_inds = torch.as_tensor(matched_row_inds, device=device)
matched_col_inds = torch.as_tensor(matched_col_inds, device=device)
assigned_gt_inds[:] = 0
assigned_gt_inds[matched_row_inds] = matched_col_inds + 1
assigned_labels[matched_row_inds] = gt_labels[0][matched_col_inds]
assign_result = AssignResult(num_gts, assigned_gt_inds, None, labels=assigned_labels)
return assign_result
#@torch.compile(options={"triton.cudagraphs": True, "triton.cudagraph_trees": False})
def assign(self,
bbox_pred,
cls_pred,
pts_pred,
gt_bboxes,
gt_labels,
gt_pts,
gt_bboxes_ignore=None,
eps=1e-7):
"""Computes one-to-one matching based on the weighted costs.
This method assign each query prediction to a ground truth or
background. The `assigned_gt_inds` with -1 means don't care,
0 means negative sample, and positive number is the index (1-based)
of assigned gt.
The assignment is done in the following steps, the order matters.
1. assign every prediction to -1
2. compute the weighted costs
3. do Hungarian matching on CPU based on the costs
4. assign all to 0 (background) first, then for each matched pair
between predictions and gts, treat this prediction as foreground
and assign the corresponding gt index (plus 1) to it.
Args:
bbox_pred (Tensor): Predicted boxes with normalized coordinates
(cx, cy, w, h), which are all in range [0, 1]. Shape
[num_query, 4].
cls_pred (Tensor): Predicted classification logits, shape
[num_query, num_class].
gt_bboxes (Tensor): Ground truth boxes with unnormalized
coordinates (x1, y1, x2, y2). Shape [num_gt, 4].
gt_labels (Tensor): Label of `gt_bboxes`, shape (num_gt,).
gt_bboxes_ignore (Tensor, optional): Ground truth bboxes that are
labelled as `ignored`. Default None.
eps (int | float, optional): A value added to the denominator for
numerical stability. Default 1e-7.
Returns:
:obj:`AssignResult`: The assigned result.
"""
assert gt_bboxes_ignore is None, \
'Only case when gt_bboxes_ignore is None is supported.'
assert bbox_pred.shape[-1] == 4, \
'Only support bbox pred shape is 4 dims'
num_bboxes = bbox_pred.size(0)
# 1. assign -1 by default
assigned_gt_inds = bbox_pred.new_full((num_bboxes, ),
-1,
dtype=torch.long)
assigned_labels = bbox_pred.new_full((num_bboxes, ),
-1,
dtype=torch.long)
# if num_gts == 0 or num_bboxes == 0:
# # No ground truth or boxes, return empty assignment
# if num_gts == 0:
# # No ground truth, assign all to background
# assigned_gt_inds[:] = 0
# return AssignResult(
# num_gts, assigned_gt_inds, None, labels=assigned_labels), None
# 2. compute the weighted costs
# classification and bboxcost.
cls_cost = self.cls_cost(cls_pred, gt_labels[0])
# regression L1 cost
normalized_gt_bboxes = normalize_2d_bbox(gt_bboxes[0], self.pc_range)
# normalized_gt_bboxes = gt_bboxes
_, num_orders, num_pts_per_gtline, num_coords = gt_pts[0].shape
normalized_gt_pts = normalize_2d_pts(gt_pts[0], self.pc_range) if not self.z_cfg['gt_z_flag'] \
else normalize_3d_pts(gt_pts[0], self.pc_range)
num_pts_per_predline = pts_pred.size(1)
if num_pts_per_predline != num_pts_per_gtline:
pts_pred_interpolated = F.interpolate(pts_pred.permute(0,2,1), size=(num_pts_per_gtline), mode='linear', align_corners=True)
pts_pred_interpolated = pts_pred_interpolated.permute(0,2,1).contiguous()
else:
pts_pred_interpolated = pts_pred
bboxes = denormalize_2d_bbox(bbox_pred, self.pc_range)
# num_q, num_pts, 2 <-> num_gt, num_pts, 2
pts_cost_ordered = self.pts_cost(pts_pred_interpolated, normalized_gt_pts)
pts_cost_ordered = pts_cost_ordered.view(num_bboxes, gt_bboxes[0].size(0), num_orders)
pts_cost, order_index = torch.min(pts_cost_ordered, 2)
reg_cost = self.reg_cost(bbox_pred[:, :4], normalized_gt_bboxes[:, :4])
iou_cost = self.iou_cost(bboxes, gt_bboxes[0])
cost = cls_cost + reg_cost + iou_cost + pts_cost
# 3. do Hungarian matching on CPU using linear_sum_assignment
assign_result = self.hungarian_match(cost[:, gt_bboxes[1]], gt_labels, assigned_gt_inds, assigned_labels, gt_bboxes[2], bbox_pred.device)
# 4. assign backgrounds and foregrounds
# assign all indices to backgrounds first
return assign_result, order_index
# def assign(self,
# bbox_pred,
# cls_pred,
# pts_pred,
# gt_bboxes,
# gt_labels,
# gt_pts,
# gt_bboxes_ignore=None,
# eps=1e-7):
# """Computes one-to-one matching based on the weighted costs.
# This method assign each query prediction to a ground truth or
# background. The `assigned_gt_inds` with -1 means don't care,
# 0 means negative sample, and positive number is the index (1-based)
# of assigned gt.
# The assignment is done in the following steps, the order matters.
# 1. assign every prediction to -1
# 2. compute the weighted costs
# 3. do Hungarian matching on CPU based on the costs
# 4. assign all to 0 (background) first, then for each matched pair
# between predictions and gts, treat this prediction as foreground
# and assign the corresponding gt index (plus 1) to it.
# Args:
# bbox_pred (Tensor): Predicted boxes with normalized coordinates
# (cx, cy, w, h), which are all in range [0, 1]. Shape
# [num_query, 4].
# cls_pred (Tensor): Predicted classification logits, shape
# [num_query, num_class].
# gt_bboxes (Tensor): Ground truth boxes with unnormalized
# coordinates (x1, y1, x2, y2). Shape [num_gt, 4].
# gt_labels (Tensor): Label of `gt_bboxes`, shape (num_gt,).
# gt_bboxes_ignore (Tensor, optional): Ground truth bboxes that are
# labelled as `ignored`. Default None.
# eps (int | float, optional): A value added to the denominator for
# numerical stability. Default 1e-7.
# Returns:
# :obj:`AssignResult`: The assigned result.
# """
# assert gt_bboxes_ignore is None, \
# 'Only case when gt_bboxes_ignore is None is supported.'
# assert bbox_pred.shape[-1] == 4, \
# 'Only support bbox pred shape is 4 dims'
# num_bboxes = bbox_pred.size(0)
# # 1. assign -1 by default
# assigned_gt_inds = bbox_pred.new_full((num_bboxes, ),
# -1,
# dtype=torch.long)
# assigned_labels = bbox_pred.new_full((num_bboxes, ),
# -1,
# dtype=torch.long)
# # if num_gts == 0 or num_bboxes == 0:
# # # No ground truth or boxes, return empty assignment
# # if num_gts == 0:
# # # No ground truth, assign all to background
# # assigned_gt_inds[:] = 0
# # return AssignResult(
# # num_gts, assigned_gt_inds, None, labels=assigned_labels), None
# # 2. compute the weighted costs
# # classification and bboxcost.
# cls_cost = self.cls_cost(cls_pred, gt_labels)
# # regression L1 cost
# normalized_gt_bboxes = normalize_2d_bbox(gt_bboxes, self.pc_range)
# # normalized_gt_bboxes = gt_bboxes
# _, num_orders, num_pts_per_gtline, num_coords = gt_pts.shape
# normalized_gt_pts = normalize_2d_pts(gt_pts, self.pc_range) if not self.z_cfg['gt_z_flag'] \
# else normalize_3d_pts(gt_pts, self.pc_range)
# num_pts_per_predline = pts_pred.size(1)
# if num_pts_per_predline != num_pts_per_gtline:
# pts_pred_interpolated = F.interpolate(pts_pred.permute(0,2,1), size=(num_pts_per_gtline), mode='linear', align_corners=True)
# pts_pred_interpolated = pts_pred_interpolated.permute(0,2,1).contiguous()
# else:
# pts_pred_interpolated = pts_pred
# bboxes = denormalize_2d_bbox(bbox_pred, self.pc_range)
# # num_q, num_pts, 2 <-> num_gt, num_pts, 2
# pts_cost_ordered = self.pts_cost(pts_pred_interpolated, normalized_gt_pts)
# pts_cost_ordered = pts_cost_ordered.view(num_bboxes, gt_bboxes.size(0), num_orders)
# pts_cost, order_index = torch.min(pts_cost_ordered, 2)
# reg_cost = self.reg_cost(bbox_pred[:, :4], normalized_gt_bboxes[:, :4])
# iou_cost = self.iou_cost(bboxes, gt_bboxes)
# cost = cls_cost + reg_cost + iou_cost + pts_cost
# # 3. do Hungarian matching on CPU using linear_sum_assignment
# assign_result = self.hungarian_match(cost, gt_labels, assigned_gt_inds, assigned_labels, gt_bboxes.size(0), bbox_pred.device)
# # 4. assign backgrounds and foregrounds
# # assign all indices to backgrounds first
# return assign_result, order_index
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment