add sunrgbd

.isort.cfg

@@ -3,6 +3,6 @@ line_length = 79
 multi_line_output = 0
 known_standard_library = setuptools
 known_first_party = mmdet,mmdet3d
-known_third_party = cv2,load_scannet_data,mmcv,numba,numpy,nuscenes,plyfile,pycocotools,pyquaternion,pytest,scannet_utils,scipy,shapely,six,skimage,torch,torchvision
+known_third_party = cv2,load_scannet_data,mmcv,numba,numpy,nuscenes,plyfile,pycocotools,pyquaternion,pytest,scannet_utils,scipy,shapely,six,skimage,sunrgbd_utils,torch,torchvision
 no_lines_before = STDLIB,LOCALFOLDER
 default_section = THIRDPARTY

data/scannet/batch_load_scannet_data.py

@@ -88,27 +88,20 @@ def main():
     parser = argparse.ArgumentParser()
     parser.add_argument(
         '--max_num_point',
-        required=True,
         default=50000,
         help='The maximum number of the points.')
     parser.add_argument(
         '--output_folder',
-        required=True,
         default='./scannet_train_instance_data',
         help='output folder of the result.')
     parser.add_argument(
-        '--scannet_dir',
-        required=True,
-        default='scans',
-        help='scannet data directory.')
+        '--scannet_dir', default='scans', help='scannet data directory.')
     parser.add_argument(
         '--label_map_file',
-        required=True,
         default='meta_data/scannetv2-labels.combined.tsv',
         help='The path of label map file.')
     parser.add_argument(
         '--train_scan_names_file',
-        required=True,
         default='meta_data/scannet_train.txt',
         help='The path of the file that stores the scan names.')
     args = parser.parse_args()

data/sunrgbd/OFFICIAL_SUNRGBD/.gitignore

+*
+!.gitignore

data/sunrgbd/README.md

+### Prepare SUN RGB-D Data
+
+1. Download SUNRGBD v2 data [HERE](http://rgbd.cs.princeton.edu/data/) (SUNRGBD.zip, SUNRGBDMeta2DBB_v2.mat, SUNRGBDMeta3DBB_v2.mat) and the toolkits (SUNRGBDtoolbox.zip). Move all the downloaded files under OFFICIAL_SUNRGBD. Unzip the zip files.
+
+2. Extract point clouds and annotations (class, v2 2D -- xmin,ymin,xmax,ymax, and 3D bounding boxes -- centroids, size, 2D heading) by running `extract_split.m`, `extract_rgbd_data_v2.m` and `extract_rgbd_data_v1.m` under the `matlab` folder.
+
+3. Prepare data by running `python sunrgbd_data.py --gen_v1_data`
+
+You can also examine and visualize the data with `python sunrgbd_data.py --viz` and use MeshLab to view the generated PLY files at `data_viz_dump`.
+
+NOTE: SUNRGBDtoolbox.zip should have MD5 hash `18d22e1761d36352f37232cba102f91f` (you can check the hash with `md5 SUNRGBDtoolbox.zip` on Mac OS or `md5sum SUNRGBDtoolbox.zip` on Linux)

data/sunrgbd/matlab/extract_rgbd_data_v1.m

+% Copyright (c) Facebook, Inc. and its affiliates.
+%
+% This source code is licensed under the MIT license found in the
+% LICENSE file in the root directory of this source tree.
+
+%% Dump SUNRGBD data to our format
+% for each sample, we have RGB image, 2d boxes.
+% point cloud (in camera coordinate), calibration and 3d boxes.
+%
+% Extract using V1 labels.
+%
+% Author: Charles R. Qi
+%
+clear; close all; clc;
+addpath(genpath('.'))
+addpath('../OFFICIAL_SUNRGBD/SUNRGBDtoolbox')
+%% V1 2D&3D BB and Seg masks
+load('../OFFICIAL_SUNRGBD/SUNRGBDtoolbox/Metadata/SUNRGBDMeta.mat')
+% load('./Metadata/SUNRGBD2Dseg.mat')
+
+%% Create folders
+det_label_folder = '../sunrgbd_trainval/label_v1/';
+mkdir(det_label_folder);
+%% Read
+for imageId = 1:10335
+    imageId
+try
+data = SUNRGBDMeta(imageId);
+data.depthpath(1:16) = '';
+data.depthpath = strcat('../OFFICIAL_SUNRGBD/SUNRGBD', data.depthpath);
+data.rgbpath(1:16) = '';
+data.rgbpath = strcat('../OFFICIAL_SUNRGBD/SUNRGBD', data.rgbpath);
+
+% MAT files are 3x smaller than TXT files. In Python we can use
+% scipy.io.loadmat('xxx.mat')['points3d_rgb'] to load the data.
+mat_filename = strcat(num2str(imageId,'%06d'), '.mat');
+txt_filename = strcat(num2str(imageId,'%06d'), '.txt');
+
+% Write 2D and 3D box label
+data2d = data;
+fid = fopen(strcat(det_label_folder, txt_filename), 'w');
+for j = 1:length(data.groundtruth3DBB)
+    centroid = data.groundtruth3DBB(j).centroid;
+    classname = data.groundtruth3DBB(j).classname;
+    orientation = data.groundtruth3DBB(j).orientation;
+    coeffs = abs(data.groundtruth3DBB(j).coeffs);
+    box2d = data2d.groundtruth2DBB(j).gtBb2D;
+    fprintf(fid, '%s %d %d %d %d %f %f %f %f %f %f %f %f\n', classname, box2d(1), box2d(2), box2d(3), box2d(4), centroid(1), centroid(2), centroid(3), coeffs(1), coeffs(2), coeffs(3), orientation(1), orientation(2));
+end
+fclose(fid);
+
+catch
+end
+
+end

data/sunrgbd/matlab/extract_rgbd_data_v2.m

+% Copyright (c) Facebook, Inc. and its affiliates.
+%
+% This source code is licensed under the MIT license found in the
+% LICENSE file in the root directory of this source tree.
+
+%% Dump SUNRGBD data to our format
+% for each sample, we have RGB image, 2d boxes.
+% point cloud (in camera coordinate), calibration and 3d boxes.
+%
+% Compared to extract_rgbd_data.m in frustum_pointents, use v2 2D and 3D
+% bboxes.
+%
+% Author: Charles R. Qi
+%
+clear; close all; clc;
+addpath(genpath('.'))
+addpath('../OFFICIAL_SUNRGBD/SUNRGBDtoolbox/readData')
+%% V1 2D&3D BB and Seg masks
+% load('./Metadata/SUNRGBDMeta.mat')
+% load('./Metadata/SUNRGBD2Dseg.mat')
+
+%% V2 3DBB annotations (overwrites SUNRGBDMeta)
+load('../OFFICIAL_SUNRGBD/SUNRGBDMeta3DBB_v2.mat');
+load('../OFFICIAL_SUNRGBD/SUNRGBDMeta2DBB_v2.mat');
+%% Create folders
+depth_folder = '../sunrgbd_trainval/depth/';
+image_folder = '../sunrgbd_trainval/image/';
+calib_folder = '../sunrgbd_trainval/calib/';
+det_label_folder = '../sunrgbd_trainval/label/';
+seg_label_folder = '../sunrgbd_trainval/seg_label/';
+mkdir(depth_folder);
+mkdir(image_folder);
+mkdir(calib_folder);
+mkdir(det_label_folder);
+mkdir(seg_label_folder);
+%% Read
+parfor imageId = 1:10335
+    imageId
+try
+data = SUNRGBDMeta(imageId);
+data.depthpath(1:16) = '';
+data.depthpath = strcat('../OFFICIAL_SUNRGBD', data.depthpath);
+data.rgbpath(1:16) = '';
+data.rgbpath = strcat('../OFFICIAL_SUNRGBD', data.rgbpath);
+
+% Write point cloud in depth map
+[rgb,points3d,depthInpaint,imsize]=read3dPoints(data);
+rgb(isnan(points3d(:,1)),:) = [];
+points3d(isnan(points3d(:,1)),:) = [];
+points3d_rgb = [points3d, rgb];
+
+% MAT files are 3x smaller than TXT files. In Python we can use
+% scipy.io.loadmat('xxx.mat')['points3d_rgb'] to load the data.
+mat_filename = strcat(num2str(imageId,'%06d'), '.mat');
+txt_filename = strcat(num2str(imageId,'%06d'), '.txt');
+parsave(strcat(depth_folder, mat_filename), points3d_rgb);
+
+% Write images
+copyfile(data.rgbpath, sprintf('%s/%06d.jpg', image_folder, imageId));
+
+% Write calibration
+dlmwrite(strcat(calib_folder, txt_filename), data.Rtilt(:)', 'delimiter', ' ');
+dlmwrite(strcat(calib_folder, txt_filename), data.K(:)', 'delimiter', ' ', '-append');
+
+% Write 2D and 3D box label
+data2d = SUNRGBDMeta2DBB(imageId);
+fid = fopen(strcat(det_label_folder, txt_filename), 'w');
+for j = 1:length(data.groundtruth3DBB)
+    centroid = data.groundtruth3DBB(j).centroid;
+    classname = data.groundtruth3DBB(j).classname;
+    orientation = data.groundtruth3DBB(j).orientation;
+    coeffs = abs(data.groundtruth3DBB(j).coeffs);
+    box2d = data2d.groundtruth2DBB(j).gtBb2D;
+    assert(strcmp(data2d.groundtruth2DBB(j).classname, classname));
+    fprintf(fid, '%s %d %d %d %d %f %f %f %f %f %f %f %f\n', classname, box2d(1), box2d(2), box2d(3), box2d(4), centroid(1), centroid(2), centroid(3), coeffs(1), coeffs(2), coeffs(3), orientation(1), orientation(2));
+end
+fclose(fid);
+
+catch
+end
+
+end
+
+function parsave(filename, instance)
+save(filename, 'instance');
+end

data/sunrgbd/matlab/extract_split.m

+% Copyright (c) Facebook, Inc. and its affiliates.
+%
+% This source code is licensed under the MIT license found in the
+% LICENSE file in the root directory of this source tree.
+
+%% Dump train/val split.
+% Author: Charles R. Qi
+
+addpath('../OFFICIAL_SUNRGBD/SUNRGBDtoolbox')
+
+%% Construct Hash Map
+hash_train = java.util.Hashtable;
+hash_val = java.util.Hashtable;
+
+split = load('../OFFICIAL_SUNRGBD/SUNRGBDtoolbox/traintestSUNRGBD/allsplit.mat');
+
+N_train = length(split.alltrain);
+N_val = length(split.alltest);
+
+for i = 1:N_train
+    folder_path = split.alltrain{i};
+    folder_path(1:16) = '';
+    hash_train.put(folder_path,0);
+end
+for i = 1:N_val
+    folder_path = split.alltest{i};
+    folder_path(1:16) = '';
+    hash_val.put(folder_path,0);
+end
+
+%% Map data to train or val set.
+load('../OFFICIAL_SUNRGBD/SUNRGBDMeta3DBB_v2.mat');
+
+fid_train = fopen('../sunrgbd_trainval/train_data_idx.txt', 'w');
+fid_val = fopen('../sunrgbd_trainval/val_data_idx.txt', 'w');
+
+for imageId = 1:10335
+    data = SUNRGBDMeta(imageId);
+    depthpath = data.depthpath;
+    depthpath(1:16) = '';
+    [filepath,name,ext] = fileparts(depthpath);
+    [filepath,name,ext] = fileparts(filepath);
+    if hash_train.containsKey(filepath)
+        fprintf(fid_train, '%d\n', imageId);
+    elseif hash_val.containsKey(filepath)
+        fprintf(fid_val, '%d\n', imageId);
+    else
+        a = 1;
+    end
+end
+fclose(fid_train);
+fclose(fid_val);

data/sunrgbd/sunrgbd_data.py

+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+''' Helper class and functions for loading SUN RGB-D objects
+
+Author: Charles R. Qi
+Date: December, 2018
+
+Note: removed unused code for frustum preparation.
+Changed a way for data visualization (removed depdency on mayavi).
+Load depth with scipy.io
+'''
+import argparse
+import os
+import sys
+
+import numpy as np
+import sunrgbd_utils
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(BASE_DIR)
+sys.path.append(os.path.join(BASE_DIR, '../utils/'))
+
+DEFAULT_TYPE_WHITELIST = [
+    'bed', 'table', 'sofa', 'chair', 'toilet', 'desk', 'dresser',
+    'night_stand', 'bookshelf', 'bathtub'
+]
+
+
+def random_sampling(pc, num_sample, replace=None, return_choices=False):
+    """ Input is NxC, output is num_samplexC
+    """
+    if replace is None:
+        replace = (pc.shape[0] < num_sample)
+    choices = np.random.choice(pc.shape[0], num_sample, replace=replace)
+    if return_choices:
+        return pc[choices], choices
+    else:
+        return pc[choices]
+
+
+class sunrgbd_object(object):
+    ''' Load and parse object data '''
+
+    def __init__(self, root_dir, split='training', use_v1=False):
+        self.root_dir = root_dir
+        self.split = split
+        assert (self.split == 'training')
+        self.split_dir = os.path.join(root_dir)
+
+        if split == 'training':
+            self.num_samples = 10335
+        elif split == 'testing':
+            self.num_samples = 2860
+        else:
+            print('Unknown split: %s' % (split))
+            exit(-1)
+
+        self.image_dir = os.path.join(self.split_dir, 'image')
+        self.calib_dir = os.path.join(self.split_dir, 'calib')
+        self.depth_dir = os.path.join(self.split_dir, 'depth')
+        if use_v1:
+            self.label_dir = os.path.join(self.split_dir, 'label_v1')
+        else:
+            self.label_dir = os.path.join(self.split_dir, 'label')
+
+    def __len__(self):
+        return self.num_samples
+
+    def get_image(self, idx):
+        img_filename = os.path.join(self.image_dir, '%06d.jpg' % (idx))
+        return sunrgbd_utils.load_image(img_filename)
+
+    def get_depth(self, idx):
+        depth_filename = os.path.join(self.depth_dir, '%06d.mat' % (idx))
+        return sunrgbd_utils.load_depth_points_mat(depth_filename)
+
+    def get_calibration(self, idx):
+        calib_filename = os.path.join(self.calib_dir, '%06d.txt' % (idx))
+        return sunrgbd_utils.SUNRGBD_Calibration(calib_filename)
+
+    def get_label_objects(self, idx):
+        label_filename = os.path.join(self.label_dir, '%06d.txt' % (idx))
+        return sunrgbd_utils.read_sunrgbd_label(label_filename)
+
+
+def extract_sunrgbd_data(idx_filename,
+                         split,
+                         output_folder,
+                         num_point=20000,
+                         type_whitelist=DEFAULT_TYPE_WHITELIST,
+                         save_votes=False,
+                         use_v1=False,
+                         skip_empty_scene=True):
+    """ Extract scene point clouds and
+    bounding boxes (centroids, box sizes, heading angles, semantic classes).
+    Dumped point clouds and boxes are in upright depth coord.
+
+    Args:
+        idx_filename: a TXT file where each line is an int number (index)
+        split: training or testing
+        save_votes: whether to compute and save Ground truth votes.
+        use_v1: use the SUN RGB-D V1 data
+        skip_empty_scene: if True, skip scenes that contain no object
+        (no objet in whitelist)
+
+    Dumps:
+        <id>_pc.npz of (N,6) where N is for number of subsampled points
+            and 6 is for XYZ and RGB (in 0~1) in upright depth coord
+        <id>_bbox.npy of (K,8) where K is the number of objects, 8 is for
+            centroids (cx,cy,cz), dimension (l,w,h), heanding_angle and
+            semantic_class
+        <id>_votes.npz of (N,10) with 0/1 indicating whether the point
+            belongs to an object, then three sets of GT votes for up to
+            three objects. If the point is only in one object's OBB, then
+            the three GT votes are the same.
+    """
+    dataset = sunrgbd_object('./sunrgbd_trainval', split, use_v1=use_v1)
+    data_idx_list = [int(line.rstrip()) for line in open(idx_filename)]
+
+    if not os.path.exists(output_folder):
+        os.mkdir(output_folder)
+
+    for data_idx in data_idx_list:
+        print('------------- ', data_idx)
+        objects = dataset.get_label_objects(data_idx)
+
+        if skip_empty_scene and (len(objects) == 0 or len([
+                obj for obj in objects if obj.classname in type_whitelist
+        ]) == 0):  # noqa:
+            continue
+
+        object_list = []
+        for obj in objects:
+            if obj.classname not in type_whitelist:
+                continue
+            obb = np.zeros((8))
+            obb[0:3] = obj.centroid
+            obb[3:6] = np.array([obj.l, obj.w, obj.h])
+            obb[6] = obj.heading_angle
+            obb[7] = sunrgbd_utils.type2class[obj.classname]
+            object_list.append(obb)
+        if len(object_list) == 0:
+            obbs = np.zeros((0, 8))
+        else:
+            obbs = np.vstack(object_list)  # (K,8)
+
+        pc_upright_depth = dataset.get_depth(data_idx)
+        pc_upright_depth_subsampled = random_sampling(pc_upright_depth,
+                                                      num_point)
+
+        np.savez_compressed(
+            os.path.join(output_folder, '%06d_pc.npz' % (data_idx)),
+            pc=pc_upright_depth_subsampled)
+        np.save(
+            os.path.join(output_folder, '%06d_bbox.npy' % (data_idx)), obbs)
+
+        if save_votes:
+            N = pc_upright_depth_subsampled.shape[0]
+            point_votes = np.zeros((N, 10))  # 3 votes and 1 vote mask
+            point_vote_idx = np.zeros(
+                (N)).astype(np.int32)  # in the range of [0,2]
+            indices = np.arange(N)
+            for obj in objects:
+                if obj.classname not in type_whitelist:
+                    continue
+                try:
+                    # Find all points in this object's OBB
+                    box3d_pts_3d = sunrgbd_utils.my_compute_box_3d(
+                        obj.centroid, np.array([obj.l, obj.w, obj.h]),
+                        obj.heading_angle)
+                    pc_in_box3d, inds = sunrgbd_utils.extract_pc_in_box3d(
+                        pc_upright_depth_subsampled, box3d_pts_3d)
+                    point_votes[inds, 0] = 1
+                    votes = np.expand_dims(obj.centroid, 0) - pc_in_box3d[:,
+                                                                          0:3]
+                    sparse_inds = indices[inds]
+                    for i in range(len(sparse_inds)):
+                        j = sparse_inds[i]
+                        point_votes[j,
+                                    int(point_vote_idx[j] * 3 +
+                                        1):int((point_vote_idx[j] + 1) * 3 +
+                                               1)] = votes[i, :]
+                        # Populate votes with the fisrt vote
+                        if point_vote_idx[j] == 0:
+                            point_votes[j, 4:7] = votes[i, :]
+                            point_votes[j, 7:10] = votes[i, :]
+                    point_vote_idx[inds] = np.minimum(2,
+                                                      point_vote_idx[inds] + 1)
+                except Exception:
+                    print('ERROR ----', data_idx, obj.classname)
+            np.savez_compressed(
+                os.path.join(output_folder, '%06d_votes.npz' % (data_idx)),
+                point_votes=point_votes)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--compute_median_size',
+        action='store_true',
+        help='Compute median 3D bounding box sizes for each class.')
+    parser.add_argument(
+        '--gen_v1_data', action='store_true', help='Generate V1 dataset.')
+    parser.add_argument(
+        '--gen_v2_data', action='store_true', help='Generate V2 dataset.')
+    args = parser.parse_args()
+
+    if args.gen_v1_data:
+        extract_sunrgbd_data(
+            os.path.join(BASE_DIR, 'sunrgbd_trainval/train_data_idx.txt'),
+            split='training',
+            output_folder=os.path.join(BASE_DIR,
+                                       'sunrgbd_pc_bbox_votes_50k_v1_train'),
+            save_votes=True,
+            num_point=50000,
+            use_v1=True,
+            skip_empty_scene=False)
+        extract_sunrgbd_data(
+            os.path.join(BASE_DIR, 'sunrgbd_trainval/val_data_idx.txt'),
+            split='training',
+            output_folder=os.path.join(BASE_DIR,
+                                       'sunrgbd_pc_bbox_votes_50k_v1_val'),
+            save_votes=True,
+            num_point=50000,
+            use_v1=True,
+            skip_empty_scene=False)
+
+    if args.gen_v2_data:
+        extract_sunrgbd_data(
+            os.path.join(BASE_DIR, 'sunrgbd_trainval/train_data_idx.txt'),
+            split='training',
+            output_folder=os.path.join(BASE_DIR,
+                                       'sunrgbd_pc_bbox_votes_50k_v2_train'),
+            save_votes=True,
+            num_point=50000,
+            use_v1=False,
+            skip_empty_scene=False)
+        extract_sunrgbd_data(
+            os.path.join(BASE_DIR, 'sunrgbd_trainval/val_data_idx.txt'),
+            split='training',
+            output_folder=os.path.join(BASE_DIR,
+                                       'sunrgbd_pc_bbox_votes_50k_v2_val'),
+            save_votes=True,
+            num_point=50000,
+            use_v1=False,
+            skip_empty_scene=False)

data/sunrgbd/sunrgbd_trainval/.gitignore

+*
+!.gitignore

data/sunrgbd/sunrgbd_utils.py

+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+''' Provides Python helper function to read My SUNRGBD dataset.
+
+Author: Charles R. Qi
+Date: October, 2017
+
+Updated by Charles R. Qi
+Date: December, 2018
+Note: removed basis loading.
+'''
+import gzip
+import pickle
+
+import cv2
+import numpy as np
+import scipy.io as sio
+
+type2class = {
+    'bed': 0,
+    'table': 1,
+    'sofa': 2,
+    'chair': 3,
+    'toilet': 4,
+    'desk': 5,
+    'dresser': 6,
+    'night_stand': 7,
+    'bookshelf': 8,
+    'bathtub': 9
+}
+class2type = {type2class[t]: t for t in type2class}
+
+
+def flip_axis_to_camera(pc):
+    ''' Flip X-right,Y-forward,Z-up to X-right,Y-down,Z-forward
+        Input and output are both (N,3) array
+    '''
+    pc2 = np.copy(pc)
+    pc2[:, [0, 1, 2]] = pc2[:, [0, 2, 1]]  # cam X,Y,Z = depth X,-Z,Y
+    pc2[:, 1] *= -1
+    return pc2
+
+
+def flip_axis_to_depth(pc):
+    pc2 = np.copy(pc)
+    pc2[:, [0, 1, 2]] = pc2[:, [0, 2, 1]]  # depth X,Y,Z = cam X,Z,-Y
+    pc2[:, 2] *= -1
+    return pc2
+
+
+class SUNObject3d(object):
+
+    def __init__(self, line):
+        data = line.split(' ')
+        data[1:] = [float(x) for x in data[1:]]
+        self.classname = data[0]
+        self.xmin = data[1]
+        self.ymin = data[2]
+        self.xmax = data[1] + data[3]
+        self.ymax = data[2] + data[4]
+        self.box2d = np.array([self.xmin, self.ymin, self.xmax, self.ymax])
+        self.centroid = np.array([data[5], data[6], data[7]])
+        self.unused_dimension = np.array([data[8], data[9], data[10]])
+        self.width = data[8]
+        self.length = data[9]
+        self.height = data[10]
+        self.orientation = np.zeros((3, ))
+        self.orientation[0] = data[11]
+        self.orientation[1] = data[12]
+        self.heading_angle = -1 * np.arctan2(self.orientation[1],
+                                             self.orientation[0])
+
+
+class SUNRGBD_Calibration(object):
+    ''' Calibration matrices and utils
+        We define five coordinate system in SUN RGBD dataset
+
+        camera coodinate:
+            Z is forward, Y is downward, X is rightward
+
+        depth coordinate:
+            Just change axis order and flip up-down axis from camera coord
+
+        upright depth coordinate: tilted depth coordinate by Rtilt such that
+            Z is gravity direction, Z is up-axis, Y is forward, X is right-ward
+
+        upright camera coordinate:
+            Just change axis order and flip up-down axis from upright
+                depth coordinate
+
+        image coordinate:
+            ----> x-axis (u)
+           |
+           v
+            y-axis (v)
+
+        depth points are stored in upright depth coordinate.
+        labels for 3d box (basis, centroid, size) are in upright
+            depth coordinate.
+        2d boxes are in image coordinate
+
+        We generate frustum point cloud and 3d box in upright camera coordinate
+    '''
+
+    def __init__(self, calib_filepath):
+        lines = [line.rstrip() for line in open(calib_filepath)]
+        Rtilt = np.array([float(x) for x in lines[0].split(' ')])
+        self.Rtilt = np.reshape(Rtilt, (3, 3), order='F')
+        K = np.array([float(x) for x in lines[1].split(' ')])
+        self.K = np.reshape(K, (3, 3), order='F')
+        self.f_u = self.K[0, 0]
+        self.f_v = self.K[1, 1]
+        self.c_u = self.K[0, 2]
+        self.c_v = self.K[1, 2]
+
+    def project_upright_depth_to_camera(self, pc):
+        ''' project point cloud from depth coord to camera coordinate
+            Input: (N,3) Output: (N,3)
+        '''
+        # Project upright depth to depth coordinate
+        pc2 = np.dot(np.transpose(self.Rtilt), np.transpose(pc[:,
+                                                               0:3]))  # (3,n)
+        return flip_axis_to_camera(np.transpose(pc2))
+
+    def project_upright_depth_to_image(self, pc):
+        ''' Input: (N,3) Output: (N,2) UV and (N,) depth '''
+        pc2 = self.project_upright_depth_to_camera(pc)
+        uv = np.dot(pc2, np.transpose(self.K))  # (n,3)
+        uv[:, 0] /= uv[:, 2]
+        uv[:, 1] /= uv[:, 2]
+        return uv[:, 0:2], pc2[:, 2]
+
+    def project_upright_depth_to_upright_camera(self, pc):
+        return flip_axis_to_camera(pc)
+
+    def project_upright_camera_to_upright_depth(self, pc):
+        return flip_axis_to_depth(pc)
+
+    def project_image_to_camera(self, uv_depth):
+        n = uv_depth.shape[0]
+        x = ((uv_depth[:, 0] - self.c_u) * uv_depth[:, 2]) / self.f_u
+        y = ((uv_depth[:, 1] - self.c_v) * uv_depth[:, 2]) / self.f_v
+        pts_3d_camera = np.zeros((n, 3))
+        pts_3d_camera[:, 0] = x
+        pts_3d_camera[:, 1] = y
+        pts_3d_camera[:, 2] = uv_depth[:, 2]
+        return pts_3d_camera
+
+    def project_image_to_upright_camerea(self, uv_depth):
+        pts_3d_camera = self.project_image_to_camera(uv_depth)
+        pts_3d_depth = flip_axis_to_depth(pts_3d_camera)
+        pts_3d_upright_depth = np.transpose(
+            np.dot(self.Rtilt, np.transpose(pts_3d_depth)))
+        return self.project_upright_depth_to_upright_camera(
+            pts_3d_upright_depth)
+
+
+def rotx(t):
+    """Rotation about the x-axis."""
+    c = np.cos(t)
+    s = np.sin(t)
+    return np.array([[1, 0, 0], [0, c, -s], [0, s, c]])
+
+
+def roty(t):
+    """Rotation about the y-axis."""
+    c = np.cos(t)
+    s = np.sin(t)
+    return np.array([[c, 0, s], [0, 1, 0], [-s, 0, c]])
+
+
+def rotz(t):
+    """Rotation about the z-axis."""
+    c = np.cos(t)
+    s = np.sin(t)
+    return np.array([[c, -s, 0], [s, c, 0], [0, 0, 1]])
+
+
+def transform_from_rot_trans(R, t):
+    """Transforation matrix from rotation matrix and translation vector."""
+    R = R.reshape(3, 3)
+    t = t.reshape(3, 1)
+    return np.vstack((np.hstack([R, t]), [0, 0, 0, 1]))
+
+
+def inverse_rigid_trans(Tr):
+    """Inverse a rigid body transform matrix (3x4 as [R|t])
+        [R'|-R't; 0|1]
+    """
+    inv_Tr = np.zeros_like(Tr)  # 3x4
+    inv_Tr[0:3, 0:3] = np.transpose(Tr[0:3, 0:3])
+    inv_Tr[0:3, 3] = np.dot(-np.transpose(Tr[0:3, 0:3]), Tr[0:3, 3])
+    return inv_Tr
+
+
+def read_sunrgbd_label(label_filename):
+    lines = [line.rstrip() for line in open(label_filename)]
+    objects = [SUNObject3d(line) for line in lines]
+    return objects
+
+
+def load_image(img_filename):
+    return cv2.imread(img_filename)
+
+
+def load_depth_points(depth_filename):
+    depth = np.loadtxt(depth_filename)
+    return depth
+
+
+def load_depth_points_mat(depth_filename):
+    depth = sio.loadmat(depth_filename)['instance']
+    return depth
+
+
+def random_shift_box2d(box2d, shift_ratio=0.1):
+    ''' Randomly shift box center, randomly scale width and height
+    '''
+    r = shift_ratio
+    xmin, ymin, xmax, ymax = box2d
+    h = ymax - ymin
+    w = xmax - xmin
+    cx = (xmin + xmax) / 2.0
+    cy = (ymin + ymax) / 2.0
+    cx2 = cx + w * r * (np.random.random() * 2 - 1)
+    cy2 = cy + h * r * (np.random.random() * 2 - 1)
+    h2 = h * (1 + np.random.random() * 2 * r - r)  # 0.9 to 1.1
+    w2 = w * (1 + np.random.random() * 2 * r - r)  # 0.9 to 1.1
+    return np.array(
+        [cx2 - w2 / 2.0, cy2 - h2 / 2.0, cx2 + w2 / 2.0, cy2 + h2 / 2.0])
+
+
+def in_hull(p, hull):
+    from scipy.spatial import Delaunay
+    if not isinstance(hull, Delaunay):
+        hull = Delaunay(hull)
+    return hull.find_simplex(p) >= 0
+
+
+def extract_pc_in_box3d(pc, box3d):
+    ''' pc: (N,3), box3d: (8,3) '''
+    box3d_roi_inds = in_hull(pc[:, 0:3], box3d)
+    return pc[box3d_roi_inds, :], box3d_roi_inds
+
+
+def my_compute_box_3d(center, size, heading_angle):
+    R = rotz(-1 * heading_angle)
+    l, w, h = size
+    x_corners = [-l, l, l, -l, -l, l, l, -l]
+    y_corners = [w, w, -w, -w, w, w, -w, -w]
+    z_corners = [h, h, h, h, -h, -h, -h, -h]
+    corners_3d = np.dot(R, np.vstack([x_corners, y_corners, z_corners]))
+    corners_3d[0, :] += center[0]
+    corners_3d[1, :] += center[1]
+    corners_3d[2, :] += center[2]
+    return np.transpose(corners_3d)
+
+
+def compute_box_3d(obj, calib):
+    ''' Takes an object and a projection matrix (P) and projects the 3d
+        bounding box into the image plane.
+        Returns:
+            corners_2d: (8,2) array in image coord.
+            corners_3d: (8,3) array in in upright depth coord.
+    '''
+    center = obj.centroid
+
+    # compute rotational matrix around yaw axis
+    R = rotz(-1 * obj.heading_angle)
+
+    # 3d bounding box dimensions
+    length = obj.length  # along heading arrow
+    width = obj.width  # perpendicular to heading arrow
+    height = obj.height
+
+    # rotate and translate 3d bounding box
+    x_corners = [
+        -length, length, length, -length, -length, length, length, -length
+    ]
+    y_corners = [width, width, -width, -width, width, width, -width, -width]
+    z_corners = [
+        height, height, height, height, -height, -height, -height, -height
+    ]
+    corners_3d = np.dot(R, np.vstack([x_corners, y_corners, z_corners]))
+    corners_3d[0, :] += center[0]
+    corners_3d[1, :] += center[1]
+    corners_3d[2, :] += center[2]
+
+    # project the 3d bounding box into the image plane
+    corners_2d, _ = calib.project_upright_depth_to_image(
+        np.transpose(corners_3d))
+    return corners_2d, np.transpose(corners_3d)
+
+
+def compute_orientation_3d(obj, calib):
+    ''' Takes an object and a projection matrix (P) and projects the 3d
+        object orientation vector into the image plane.
+        Returns:
+            orientation_2d: (2,2) array in image coord.
+            orientation_3d: (2,3) array in depth coord.
+    '''
+
+    # orientation in object coordinate system
+    ori = obj.orientation
+    orientation_3d = np.array([[0, ori[0]], [0, ori[1]], [0, 0]])
+    center = obj.centroid
+    orientation_3d[0, :] = orientation_3d[0, :] + center[0]
+    orientation_3d[1, :] = orientation_3d[1, :] + center[1]
+    orientation_3d[2, :] = orientation_3d[2, :] + center[2]
+
+    # project orientation into the image plane
+    orientation_2d, _ = calib.project_upright_depth_to_image(
+        np.transpose(orientation_3d))
+    return orientation_2d, np.transpose(orientation_3d)
+
+
+def draw_projected_box3d(image, qs, color=(255, 255, 255), thickness=2):
+    ''' Draw 3d bounding box in image
+        qs: (8,2) array of vertices for the 3d box in following order:
+            1 -------- 0
+           /|         /|
+          2 -------- 3 .
+          | |        | |
+          . 5 -------- 4
+          |/         |/
+          6 -------- 7
+    '''
+    qs = qs.astype(np.int32)
+    for k in range(0, 4):
+        # http://docs.enthought.com/mayavi/mayavi/auto/mlab_helper_functions.html
+        i, j = k, (k + 1) % 4
+        cv2.line(image, (qs[i, 0], qs[i, 1]), (qs[j, 0], qs[j, 1]), color,
+                 thickness, cv2.CV_AA)  # use LINE_AA for opencv3
+
+        i, j = k + 4, (k + 1) % 4 + 4
+        cv2.line(image, (qs[i, 0], qs[i, 1]), (qs[j, 0], qs[j, 1]), color,
+                 thickness, cv2.CV_AA)
+
+        i, j = k, k + 4
+        cv2.line(image, (qs[i, 0], qs[i, 1]), (qs[j, 0], qs[j, 1]), color,
+                 thickness, cv2.CV_AA)
+    return image
+
+
+def save_zipped_pickle(obj, filename, protocol=-1):
+    with gzip.open(filename, 'wb') as f:
+        pickle.dump(obj, f, protocol)
+
+
+def load_zipped_pickle(filename):
+    with gzip.open(filename, 'rb') as f:
+        loaded_object = pickle.load(f)
+        return loaded_object