support to train WOD with speed, support to update infos with speed

pcdet/datasets/augmentor/database_sampler.py

@@ -469,7 +469,7 @@ class DataBaseSampler(object):
                 valid_sampled_dict = [sampled_dict[x] for x in valid_mask]
                 valid_sampled_boxes = sampled_boxes[valid_mask]
 
-                existed_boxes = np.concatenate((existed_boxes, valid_sampled_boxes), axis=0)
+                existed_boxes = np.concatenate((existed_boxes, valid_sampled_boxes[:, :existed_boxes.shape[-1]]), axis=0)
                 total_valid_sampled_dict.extend(valid_sampled_dict)
 
         sampled_gt_boxes = existed_boxes[gt_boxes.shape[0]:, :]

pcdet/datasets/waymo/waymo_dataset.py

@@ -268,6 +268,11 @@ class WaymoDataset(DatasetTemplate):
             else:
                 gt_boxes_lidar = annos['gt_boxes_lidar']
 
+            if self.dataset_cfg.get('TRAIN_WITH_SPEED', False):
+                assert gt_boxes_lidar.shape[-1] == 9
+            else:
+                gt_boxes_lidar = gt_boxes_lidar[:, 0:7]
+            
             if self.training and self.dataset_cfg.get('FILTER_EMPTY_BOXES_FOR_TRAIN', False):
                 mask = (annos['num_points_in_gt'] > 0)  # filter empty boxes
                 annos['name'] = annos['name'][mask]

pcdet/datasets/waymo/waymo_utils.py

@@ -20,7 +20,7 @@ except:
 WAYMO_CLASSES = ['unknown', 'Vehicle', 'Pedestrian', 'Sign', 'Cyclist']
 
 
-def generate_labels(frame):
+def generate_labels(frame, pose):
     obj_name, difficulty, dimensions, locations, heading_angles = [], [], [], [], []
     tracking_difficulty, speeds, accelerations, obj_ids = [], [], [], []
     num_points_in_gt = []
@@ -51,13 +51,17 @@ def generate_labels(frame):
     annotations['obj_ids'] = np.array(obj_ids)
     annotations['tracking_difficulty'] = np.array(tracking_difficulty)
     annotations['num_points_in_gt'] = np.array(num_points_in_gt)
-    annotations['speed'] = np.array(speeds)
-    annotations['accel'] = np.array(accelerations)
+    annotations['speed_global'] = np.array(speeds)
+    annotations['accel_global'] = np.array(accelerations)
 
     annotations = common_utils.drop_info_with_name(annotations, name='unknown')
     if annotations['name'].__len__() > 0:
+        global_speed = np.pad(annotations['speed_global'], ((0, 0), (0, 1)), mode='constant', constant_values=0)  # (N, 3)
+        speed = np.dot(global_speed, np.linalg.inv(pose[:3, :3].T))
+        speed = speed[:, :2]
+        
         gt_boxes_lidar = np.concatenate([
-            annotations['location'], annotations['dimensions'], annotations['heading_angles'][..., np.newaxis]],
+            annotations['location'], annotations['dimensions'], annotations['heading_angles'][..., np.newaxis], speed],
             axis=1
         )
     else:
@@ -241,7 +245,7 @@ def process_single_sequence(sequence_file, save_path, sampled_interval, has_labe
         info['pose'] = pose
 
         if has_label:
-            annotations = generate_labels(frame)
+            annotations = generate_labels(frame, pose=pose)
             info['annos'] = annotations
 
         if update_info_only:

tools/cfgs/dataset_configs/waymo_dataset_multiframe.yaml

@@ -24,6 +24,8 @@ SEQUENCE_CONFIG:
     ENABLED: True
     SAMPLE_OFFSET: [-3, 0]
     
+TRAIN_WITH_SPEED: True 
+
 
 DATA_AUGMENTOR:
     DISABLE_AUG_LIST: ['placeholder']

tools/cfgs/waymo_models/centerpoint_4frames.yaml

+CLASS_NAMES: ['Vehicle', 'Pedestrian', 'Cyclist']
+
+DATA_CONFIG:
+    _BASE_CONFIG_: cfgs/dataset_configs/waymo_dataset_multiframe.yaml
+
+MODEL:
+    NAME: CenterPoint
+
+    VFE:
+        NAME: MeanVFE
+
+    BACKBONE_3D:
+        NAME: VoxelResBackBone8x
+
+    MAP_TO_BEV:
+        NAME: HeightCompression
+        NUM_BEV_FEATURES: 256
+
+    BACKBONE_2D:
+        NAME: BaseBEVBackbone
+
+        LAYER_NUMS: [5, 5]
+        LAYER_STRIDES: [1, 2]
+        NUM_FILTERS: [128, 256]
+        UPSAMPLE_STRIDES: [1, 2]
+        NUM_UPSAMPLE_FILTERS: [256, 256]
+
+    DENSE_HEAD:
+        NAME: CenterHead
+        CLASS_AGNOSTIC: False
+
+        CLASS_NAMES_EACH_HEAD: [
+            ['Vehicle', 'Pedestrian', 'Cyclist']
+        ]
+
+        SHARED_CONV_CHANNEL: 64
+        USE_BIAS_BEFORE_NORM: True
+        NUM_HM_CONV: 2
+        SEPARATE_HEAD_CFG:
+            HEAD_ORDER: ['center', 'center_z', 'dim', 'rot', 'vel']
+            HEAD_DICT: {
+                'center': {'out_channels': 2, 'num_conv': 2},
+                'center_z': {'out_channels': 1, 'num_conv': 2},
+                'dim': {'out_channels': 3, 'num_conv': 2},
+                'rot': {'out_channels': 2, 'num_conv': 2},
+                'vel': {'out_channels': 2, 'num_conv': 2},
+            }
+
+        TARGET_ASSIGNER_CONFIG:
+            FEATURE_MAP_STRIDE: 8
+            NUM_MAX_OBJS: 500
+            GAUSSIAN_OVERLAP: 0.1
+            MIN_RADIUS: 2
+
+        LOSS_CONFIG:
+            LOSS_WEIGHTS: {
+                'cls_weight': 1.0,
+                'loc_weight': 2.0,
+                'code_weights': [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.2, 0.2]
+            }
+
+        POST_PROCESSING:
+            SCORE_THRESH: 0.1
+            POST_CENTER_LIMIT_RANGE: [-75.2, -75.2, -2, 75.2, 75.2, 4]
+            MAX_OBJ_PER_SAMPLE: 500
+            NMS_CONFIG:
+                NMS_TYPE: nms_gpu
+                NMS_THRESH: 0.7
+                NMS_PRE_MAXSIZE: 4096
+                NMS_POST_MAXSIZE: 500
+
+    POST_PROCESSING:
+        RECALL_THRESH_LIST: [0.3, 0.5, 0.7]
+
+        EVAL_METRIC: waymo
+
+
+OPTIMIZATION:
+    BATCH_SIZE_PER_GPU: 4
+    NUM_EPOCHS: 30
+
+    OPTIMIZER: adam_onecycle
+    LR: 0.003
+    WEIGHT_DECAY: 0.01
+    MOMENTUM: 0.9
+
+    MOMS: [0.95, 0.85]
+    PCT_START: 0.4
+    DIV_FACTOR: 10
+    DECAY_STEP_LIST: [35, 45]
+    LR_DECAY: 0.1
+    LR_CLIP: 0.0000001
+
+    LR_WARMUP: False
+    WARMUP_EPOCH: 1
+
+    GRAD_NORM_CLIP: 10