evtexture

others/event_utils/lib/data_loaders/hdf5_dataset.py

+import h5py
+from ..util.event_util import binary_search_h5_dset
+from .base_dataset import BaseVoxelDataset
+import matplotlib.pyplot as plt
+
+class DynamicH5Dataset(BaseVoxelDataset):
+    """
+    Dataloader for events saved in the Monash University HDF5 events format
+    (see https://github.com/TimoStoff/event_utils for code to convert datasets)
+    """
+
+    def get_frame(self, index):
+        return self.h5_file['images']['image{:09d}'.format(index)][:]
+
+    def get_flow(self, index):
+        return self.h5_file['flow']['flow{:09d}'.format(index)][:]
+
+    def get_events(self, idx0, idx1):
+        xs = self.h5_file['events/xs'][idx0:idx1]
+        ys = self.h5_file['events/ys'][idx0:idx1]
+        ts = self.h5_file['events/ts'][idx0:idx1]
+        ps = self.h5_file['events/ps'][idx0:idx1] * 2.0 - 1.0
+        return xs, ys, ts, ps
+
+    def load_data(self, data_path):
+        self.data_sources = ('esim', 'ijrr', 'mvsec', 'eccd', 'hqfd', 'unknown')
+        try:
+            self.h5_file = h5py.File(data_path, 'r')
+        except OSError as err:
+            print("Couldn't open {}: {}".format(data_path, err))
+
+        if self.sensor_resolution is None:
+            self.sensor_resolution = self.h5_file.attrs['sensor_resolution'][0:2]
+        else:
+            self.sensor_resolution = self.sensor_resolution[0:2]
+        print("sensor resolution = {}".format(self.sensor_resolution))
+        self.has_flow = 'flow' in self.h5_file.keys() and len(self.h5_file['flow']) > 0
+        self.t0 = self.h5_file['events/ts'][0]
+        self.tk = self.h5_file['events/ts'][-1]
+        self.num_events = self.h5_file.attrs["num_events"]
+        self.num_frames = self.h5_file.attrs["num_imgs"]
+
+        self.frame_ts = []
+        for img_name in self.h5_file['images']:
+            self.frame_ts.append(self.h5_file['images/{}'.format(img_name)].attrs['timestamp'])
+
+        data_source = self.h5_file.attrs.get('source', 'unknown')
+        try:
+            self.data_source_idx = self.data_sources.index(data_source)
+        except ValueError:
+            self.data_source_idx = -1
+
+    def find_ts_index(self, timestamp):
+        idx = binary_search_h5_dset(self.h5_file['events/ts'], timestamp)
+        return idx
+
+    def ts(self, index):
+        return self.h5_file['events/ts'][index]
+
+    def compute_frame_indices(self):
+        frame_indices = []
+        start_idx = 0
+        for img_name in self.h5_file['images']:
+            end_idx = self.h5_file['images/{}'.format(img_name)].attrs['event_idx']
+            frame_indices.append([start_idx, end_idx])
+            start_idx = end_idx
+        return frame_indices
+
+if __name__ == "__main__":
+    """
+    Tool to add events to a set of events.
+    """
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("path", help="Path to event file")
+    args = parser.parse_args()
+
+    dloader = DynamicH5Dataset(args.path)
+    for item in dloader:
+        print(item['events'].shape)

others/event_utils/lib/data_loaders/memmap_dataset.py

+import numpy as np
+import os
+from .base_dataset import BaseVoxelDataset
+
+class MemMapDataset(BaseVoxelDataset):
+    """
+    Dataloader for events saved in the MemMap events format used at RPG.
+    (see https://github.com/TimoStoff/event_utils for code to convert datasets)
+    """
+
+    def get_frame(self, index):
+        frame = self.filehandle['images'][index][:, :, 0]
+        return frame
+
+    def get_flow(self, index):
+        flow = self.filehandle['optic_flow'][index]
+        return flow
+
+    def get_events(self, idx0, idx1):
+        xy = self.filehandle["xy"][idx0:idx1]
+        xs = xy[:, 0].astype(np.float32)
+        ys = xy[:, 1].astype(np.float32)
+        ts = self.filehandle["t"][idx0:idx1]
+        ps = self.filehandle["p"][idx0:idx1] * 2.0 - 1.0
+        return xs, ys, ts, ps
+
+    def load_data(self, data_path, timestamp_fname="timestamps.npy", image_fname="images.npy",
+                  optic_flow_fname="optic_flow.npy", optic_flow_stamps_fname="optic_flow_stamps.npy",
+                  t_fname="t.npy", xy_fname="xy.npy", p_fname="p.npy"):
+
+        assert os.path.isdir(data_path), '%s is not a valid data_path' % data_path
+
+        data = {}
+        self.has_flow = False
+        for subroot, _, fnames in sorted(os.walk(data_path)):
+            for fname in sorted(fnames):
+                path = os.path.join(subroot, fname)
+                if fname.endswith(".npy"):
+                    if fname.endswith(timestamp_fname):
+                        frame_stamps = np.load(path)
+                        data["frame_stamps"] = frame_stamps
+                    elif fname.endswith(image_fname):
+                        data["images"] = np.load(path, mmap_mode="r")
+                    elif fname.endswith(optic_flow_fname):
+                        data["optic_flow"] = np.load(path, mmap_mode="r")
+                        self.has_flow = True
+                    elif fname.endswith(optic_flow_stamps_fname):
+                        optic_flow_stamps = np.load(path)
+                        data["optic_flow_stamps"] = optic_flow_stamps
+
+                    try:
+                        handle = np.load(path, mmap_mode="r")
+                    except Exception as err:
+                        print("Couldn't load {}:".format(path))
+                        raise err
+                    if fname.endswith(t_fname):  # timestamps
+                        data["t"] = handle.squeeze()
+                    elif fname.endswith(xy_fname):  # coordinates
+                        data["xy"] = handle.squeeze()
+                    elif fname.endswith(p_fname):  # polarity
+                        data["p"] = handle.squeeze()
+            if len(data) > 0:
+                data['path'] = subroot
+                if "t" not in data:
+                    print("Ignoring root {} since no events".format(subroot))
+                    continue
+                assert (len(data['p']) == len(data['xy']) and len(data['p']) == len(data['t']))
+
+                self.t0, self.tk = data['t'][0], data['t'][-1]
+                self.num_events = len(data['p'])
+                self.num_frames = len(data['images'])
+
+                self.frame_ts = []
+                for ts in data["frame_stamps"]:
+                    self.frame_ts.append(ts)
+                data["index"] = self.frame_ts
+
+        self.filehandle = data
+        self.find_config(data_path)
+
+    def find_ts_index(self, timestamp):
+        index = np.searchsorted(self.filehandle["t"], timestamp)
+        return index
+
+    def ts(self, index):
+        return self.filehandle["t"][index]
+
+    def infer_resolution(self):
+        if len(self.filehandle["images"]) > 0:
+            sr = self.filehandle["images"][0].shape[0:2]
+        else:
+            sr = [np.max(self.filehandle["xy"][:, 1]) + 1, np.max(self.filehandle["xy"][:, 0]) + 1]
+            print("Inferred sensor resolution: {}".format(self.sensor_resolution))
+        return sr
+
+    def find_config(self, data_path):
+        if self.sensor_resolution is None:
+            config = os.path.join(data_path, "dataset_config.json")
+            if os.path.exists(config):
+                self.config = read_json(config)
+                self.data_source = self.config['data_source']
+                self.sensor_resolution = self.config["sensor_resolution"]
+            else:
+                data_source = 'unknown'
+                self.sensor_resolution = self.infer_resolution()

others/event_utils/lib/data_loaders/npy_dataset.py

+from .base_dataset import BaseVoxelDataset
+import numpy as np
+
+class NpyDataset(BaseVoxelDataset):
+    """
+    Dataloader for events saved in the Monash University HDF5 events format
+    (see https://github.com/TimoStoff/event_utils for code to convert datasets)
+    """
+
+    def get_frame(self, index):
+        return None
+
+    def get_flow(self, index):
+        return None
+
+    def get_events(self, idx0, idx1):
+        xs = self.xs[idx0:idx1]
+        ys = self.ys[idx0:idx1]
+        ts = self.ts[idx0:idx1]
+        ps = self.ps[idx0:idx1]
+        return xs, ys, ts, ps
+
+    def load_data(self, data_path):
+        try:
+            self.data = np.load(data_path)
+            self.xs, self.ys, self.ps, self.ts = self.data[:, 0], self.data[:, 1], self.data[:, 2]*2-1, self.data[:, 3]*1e-6
+        except OSError as err:
+            print("Couldn't open {}: {}".format(data_path, err))
+        print(self.ps)
+
+        if self.sensor_resolution is None:
+            self.sensor_resolution = [np.max(self.xs), np.max(self.ys)]
+            print("Inferred resolution as {}".format(self.sensor_resolution))
+        else:
+            self.sensor_resolution = self.sensor_resolution[0:2]
+        print("sensor resolution = {}".format(self.sensor_resolution))
+        self.has_flow = False
+        self.has_frames = False
+        self.t0 = self.ts[0]
+        self.tk = self.ts[-1]
+        self.num_events = len(self.xs)
+        self.num_frames = 0
+        self.frame_ts = []
+
+    def find_ts_index(self, timestamp):
+        idx = np.searchsorted(self.ts, timestamp)
+        return idx
+
+    def ts(self, index):
+        return ts[index]
+
+    def compute_frame_indices(self):
+        return None
+
+if __name__ == "__main__":
+    """
+    Tool to add events to a set of events.
+    """
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("path", help="Path to event file")
+    args = parser.parse_args()
+
+    dloader = NpyDataset(args.path)
+    for item in dloader:
+        print(item['events'].shape)

others/event_utils/lib/representations/image.py

+import numpy as np
+from scipy.stats import rankdata
+import torch
+
+def events_to_image(xs, ys, ps, sensor_size=(180, 240), interpolation=None, padding=False, meanval=False, default=0):
+    """
+    Place events into an image using numpy
+    @param xs x coords of events
+    @param ys y coords of events
+    @param ps Event polarities/weights
+    @param sensor_size The size of the event camera sensor
+    @param interpolation Whether to add the events to the pixels by interpolation (values: None, 'bilinear')
+    @param padding If true, pad the output image to include events otherwise warped off sensor
+    @param meanval If true, divide the sum of the values by the number of events at that location
+    @returns Event image from the input events
+    """
+    img_size = (sensor_size[0]+1, sensor_size[1]+1)
+    if interpolation == 'bilinear' and xs.dtype is not torch.long and xs.dtype is not torch.long:
+        xt, yt, pt = torch.from_numpy(xs), torch.from_numpy(ys), torch.from_numpy(ps)
+        xt, yt, pt = xt.float(), yt.float(), pt.float()
+        img = events_to_image_torch(xt, yt, pt, clip_out_of_range=True, interpolation='bilinear', padding=padding)
+        img[img==0] = default
+        img = img.numpy()
+        if meanval:
+            event_count_image = events_to_image_torch(xt, yt, torch.ones_like(xt),
+                    clip_out_of_range=True, padding=padding)
+            event_count_image = event_count_image.numpy()
+    else:
+        coords = np.stack((ys, xs))
+        try:
+            abs_coords = np.ravel_multi_index(coords, img_size)
+        except ValueError:
+            print("Issue with input arrays! minx={}, maxx={}, miny={}, maxy={}, coords.shape={}, \
+                    sum(coords)={}, sensor_size={}".format(np.min(xs), np.max(xs), np.min(ys), np.max(ys),
+                        coords.shape, np.sum(coords), img_size))
+            raise ValueError
+        img = np.bincount(abs_coords, weights=ps, minlength=img_size[0]*img_size[1])
+        img = img.reshape(img_size)
+        if meanval:
+            event_count_image = np.bincount(abs_coords, weights=np.ones_like(xs), minlength=img_size[0]*img_size[1])
+            event_count_image = event_count_image.reshape(img_size)
+    if meanval:
+        img = np.divide(img, event_count_image, out=np.ones_like(img)*default, where=event_count_image!=0)
+    return img[0:sensor_size[0], 0:sensor_size[1]]
+
+def events_to_image_torch(xs, ys, ps,
+        device=None, sensor_size=(180, 240), clip_out_of_range=True,
+        interpolation=None, padding=True, default=0):
+    """
+    Method to turn event tensor to image. Allows for bilinear interpolation.
+    @param xs Tensor of x coords of events
+    @param ys Tensor of y coords of events
+    @param ps Tensor of event polarities/weights
+    @param device The device on which the image is. If none, set to events device
+    @param sensor_size The size of the image sensor/output image
+    @param clip_out_of_range If the events go beyond the desired image size,
+       clip the events to fit into the image
+    @param interpolation Which interpolation to use. Options=None,'bilinear'
+    @param padding If bilinear interpolation, allow padding the image by 1 to allow events to fit:
+    @returns Event image from the events
+    """
+    if device is None:
+        device = xs.device
+    if interpolation == 'bilinear' and padding:
+        img_size = (sensor_size[0]+1, sensor_size[1]+1)
+    else:
+        img_size = list(sensor_size)
+
+    mask = torch.ones(xs.size(), device=device)
+    if clip_out_of_range:
+        zero_v = torch.tensor([0.], device=device)
+        ones_v = torch.tensor([1.], device=device)
+        clipx = img_size[1] if interpolation is None and padding==False else img_size[1]-1
+        clipy = img_size[0] if interpolation is None and padding==False else img_size[0]-1
+        mask = torch.where(xs>=clipx, zero_v, ones_v)*torch.where(ys>=clipy, zero_v, ones_v)
+
+    img = (torch.ones(img_size)*default).to(device)
+    if interpolation == 'bilinear' and xs.dtype is not torch.long and xs.dtype is not torch.long:
+        pxs = (xs.floor()).float()
+        pys = (ys.floor()).float()
+        dxs = (xs-pxs).float()
+        dys = (ys-pys).float()
+        pxs = (pxs*mask).long()
+        pys = (pys*mask).long()
+        masked_ps = ps.squeeze()*mask
+        interpolate_to_image(pxs, pys, dxs, dys, masked_ps, img)
+    else:
+        if xs.dtype is not torch.long:
+            xs = xs.long().to(device)
+        if ys.dtype is not torch.long:
+            ys = ys.long().to(device)
+        try:
+            mask = mask.long().to(device)
+            xs, ys = xs*mask, ys*mask
+            img.index_put_((ys, xs), ps, accumulate=True)
+        except Exception as e:
+            print("Unable to put tensor {} positions ({}, {}) into {}. Range = {},{}".format(
+                ps.shape, ys.shape, xs.shape, img.shape,  torch.max(ys), torch.max(xs)))
+            raise e
+    return img
+
+def interpolate_to_image(pxs, pys, dxs, dys, weights, img):
+    """
+    Accumulate x and y coords to an image using bilinear interpolation
+    @param pxs Numpy array of integer typecast x coords of events
+    @param pys Numpy array of integer typecast y coords of events
+    @param dxs Numpy array of residual difference between x coord and int(x coord)
+    @param dys Numpy array of residual difference between y coord and int(y coord)
+    @returns Image
+    """
+    img.index_put_((pys,   pxs  ), weights*(1.0-dxs)*(1.0-dys), accumulate=True)
+    img.index_put_((pys,   pxs+1), weights*dxs*(1.0-dys), accumulate=True)
+    img.index_put_((pys+1, pxs  ), weights*(1.0-dxs)*dys, accumulate=True)
+    img.index_put_((pys+1, pxs+1), weights*dxs*dys, accumulate=True)
+    return img
+
+def interpolate_to_derivative_img(pxs, pys, dxs, dys, d_img, w1, w2):
+    """
+    Accumulate x and y coords to an image using double weighted bilinear interpolation.
+    This allows for computing gradient images, since in the gradient image the interpolation
+    is weighted by the values of the Jacobian.
+    @param pxs Numpy array of integer typecast x coords of events
+    @param pys Numpy array of integer typecast y coords of events
+    @param dxs Numpy array of residual difference between x coord and int(x coord)
+    @param dys Numpy array of residual difference between y coord and int(y coord)
+    @param dimg Derivative image (needs to be of appropriate dimensions)
+    @param w1 Weight for x component of bilinear interpolation
+    @param w2 Weight for y component of bilinear interpolation
+    @returns Image
+    """
+    for i in range(d_img.shape[0]):
+        d_img[i].index_put_((pys,   pxs  ), w1[i] * (-(1.0-dys)) + w2[i] * (-(1.0-dxs)), accumulate=True)
+        d_img[i].index_put_((pys,   pxs+1), w1[i] * (1.0-dys)    + w2[i] * (-dxs), accumulate=True)
+        d_img[i].index_put_((pys+1, pxs  ), w1[i] * (-dys)       + w2[i] * (1.0-dxs), accumulate=True)
+        d_img[i].index_put_((pys+1, pxs+1), w1[i] * dys          + w2[i] *  dxs, accumulate=True)
+    return d_img
+
+def image_to_event_weights(xs, ys, img):
+    """
+    Given an image and a set of event coordinates, get the pixel value
+    of the image for each event using reverse bilinear interpolation
+    @param xs x coords of events
+    @param ys y coords of events
+    @param img The image from which to draw the weights
+    @return List containing the value in the image for each event
+    """
+    clipx, clipy  = img.shape[1]-1, img.shape[0]-1
+    mask = np.where(xs>=clipx, 0, 1)*np.where(ys>=clipy, 0, 1)
+
+    pxs = np.floor(xs*mask).astype(int)
+    pys = np.floor(ys*mask).astype(int)
+    dxs = xs-pxs
+    dys = ys-pys
+    wxs, wys = 1.0-dxs, 1.0-dys
+
+    weights =  img[pys, pxs]      *wxs*wys
+    weights += img[pys, pxs+1]    *dxs*wys
+    weights += img[pys+1, pxs]    *wxs*dys
+    weights += img[pys+1, pxs+1]  *dxs*dys
+    return weights*mask
+
+def events_to_image_drv(xn, yn, pn, jacobian_xn, jacobian_yn,
+        device=None, sensor_size=(180, 240), clip_out_of_range=True,
+        interpolation='bilinear', padding=True, compute_gradient=False):
+    """
+    Method to turn event tensor to image and derivative image (given event Jacobians).
+    Allows for bilinear interpolation.
+    @param xs Tensor of x coords of events
+    @param ys Tensor of y coords of events
+    @param ps Tensor of event polarities/weights
+    @param device The device on which the image is. If none, set to events device
+    @param sensor_size The size of the image sensor/output image
+    @param clip_out_of_range If the events go beyond the desired image size,
+       clip the events to fit into the image
+    @param interpolation Which interpolation to use. Options=None,'bilinear'
+    @param padding If bilinear interpolation, allow padding the image by 1 to allow events to fit:
+    @param compute_gradient If True, compute the image gradient
+    """
+    xt, yt, pt = torch.from_numpy(xn), torch.from_numpy(yn), torch.from_numpy(pn)
+    xs, ys, ps, = xt.float(), yt.float(), pt.float()
+    if compute_gradient:
+        jacobian_x, jacobian_y = torch.from_numpy(jacobian_xn), torch.from_numpy(jacobian_yn)
+        jacobian_x, jacobian_y = jacobian_x.float(), jacobian_y.float()
+    if device is None:
+        device = xs.device
+    if padding:
+        img_size = (sensor_size[0]+1, sensor_size[1]+1)
+    else:
+        img_size = sensor_size
+
+    mask = torch.ones(xs.size())
+    if clip_out_of_range:
+        zero_v = torch.tensor([0.])
+        ones_v = torch.tensor([1.])
+        clipx = img_size[1] if interpolation is None and padding==False else img_size[1]-1
+        clipy = img_size[0] if interpolation is None and padding==False else img_size[0]-1
+        mask = torch.where(xs>=clipx, zero_v, ones_v)*torch.where(ys>=clipy, zero_v, ones_v)
+
+    pxs = xs.floor()
+    pys = ys.floor()
+    dxs = xs-pxs
+    dys = ys-pys
+    pxs = (pxs*mask).long()
+    pys = (pys*mask).long()
+    masked_ps = ps*mask
+    img = torch.zeros(img_size).to(device)
+    interpolate_to_image(pxs, pys, dxs, dys, masked_ps, img)
+
+    if compute_gradient:
+        d_img = torch.zeros((2, *img_size)).to(device)
+        w1 = jacobian_x*masked_ps
+        w2 = jacobian_y*masked_ps
+        interpolate_to_derivative_img(pxs, pys, dxs, dys, d_img, w1, w2)
+        d_img = d_img.numpy()
+    else:
+        d_img = None
+    return img.numpy(), d_img
+
+def events_to_timestamp_image(xn, yn, ts, pn,
+        device=None, sensor_size=(180, 240), clip_out_of_range=True,
+        interpolation='bilinear', padding=True, normalize_timestamps=True):
+    """
+    Method to generate the average timestamp images from 'Zhu19, Unsupervised Event-based Learning
+    of Optical Flow, Depth, and Egomotion'. This method does not have known derivative.
+    @param xs List of event x coordinates
+    @param ys List of event y coordinates
+    @param ts List of event timestamps
+    @param ps List of event polarities
+    @param device The device that the events are on
+    @param sensor_size The size of the event sensor/output voxels
+    @param clip_out_of_range If the events go beyond the desired image size,
+        clip the events to fit into the image
+    @param interpolation Which interpolation to use. Options=None,'bilinear'
+    @param padding If bilinear interpolation, allow padding the image by 1 to allow events to fit
+    @returns Timestamp images of the positive and negative events: ti_pos, ti_neg
+    """
+
+    t0 = ts[0]
+    xt, yt, ts, pt = torch.from_numpy(xn), torch.from_numpy(yn), torch.from_numpy(ts-t0), torch.from_numpy(pn)
+    xs, ys, ts, ps = xt.float(), yt.float(), ts.float(), pt.float()
+    zero_v = torch.tensor([0.])
+    ones_v = torch.tensor([1.])
+    if device is None:
+        device = xs.device
+    if padding:
+        img_size = (sensor_size[0]+1, sensor_size[1]+1)
+    else:
+        img_size = sensor_size
+
+    mask = torch.ones(xs.size())
+    if clip_out_of_range:
+        clipx = img_size[1] if interpolation is None and padding==False else img_size[1]-1
+        clipy = img_size[0] if interpolation is None and padding==False else img_size[0]-1
+        mask = torch.where(xs>=clipx, zero_v, ones_v)*torch.where(ys>=clipy, zero_v, ones_v)
+
+    pos_events_mask = torch.where(ps>0, ones_v, zero_v)
+    neg_events_mask = torch.where(ps<=0, ones_v, zero_v)
+    normalized_ts = (ts-ts[0])/(ts[-1]+1e-6) if normalize_timestamps else ts
+    pxs = xs.floor()
+    pys = ys.floor()
+    dxs = xs-pxs
+    dys = ys-pys
+    pxs = (pxs*mask).long()
+    pys = (pys*mask).long()
+    masked_ps = ps*mask
+
+    pos_weights = normalized_ts*pos_events_mask
+    neg_weights = normalized_ts*neg_events_mask
+    img_pos = torch.zeros(img_size).to(device)
+    img_pos_cnt = torch.ones(img_size).to(device)
+    img_neg = torch.zeros(img_size).to(device)
+    img_neg_cnt = torch.ones(img_size).to(device)
+
+    interpolate_to_image(pxs, pys, dxs, dys, pos_weights, img_pos)
+    interpolate_to_image(pxs, pys, dxs, dys, pos_events_mask, img_pos_cnt)
+    interpolate_to_image(pxs, pys, dxs, dys, neg_weights, img_neg)
+    interpolate_to_image(pxs, pys, dxs, dys, neg_events_mask, img_neg_cnt)
+
+    img_pos, img_pos_cnt = img_pos.numpy(), img_pos_cnt.numpy()
+    img_pos_cnt[img_pos_cnt==0] = 1
+    img_neg, img_neg_cnt = img_neg.numpy(), img_neg_cnt.numpy()
+    img_neg_cnt[img_neg_cnt==0] = 1
+    img_pos, img_neg = img_pos/img_pos_cnt, img_neg/img_neg_cnt
+    return img_pos, img_neg
+
+def events_to_timestamp_image_torch(xs, ys, ts, ps,
+        device=None, sensor_size=(180, 240), clip_out_of_range=True,
+        interpolation='bilinear', padding=True, timestamp_reverse=False):
+    """
+    Method to generate the average timestamp images from 'Zhu19, Unsupervised Event-based Learning
+    of Optical Flow, Depth, and Egomotion'. This method does not have known derivative.
+    @param xs List of event x coordinates
+    @param ys List of event y coordinates
+    @param ts List of event timestamps
+    @param ps List of event polarities
+    @param device The device that the events are on
+    @param sensor_size The size of the event sensor/output voxels
+    @param clip_out_of_range If the events go beyond the desired image size,
+        clip the events to fit into the image
+    @param interpolation Which interpolation to use. Options=None,'bilinear'
+    @param padding If bilinear interpolation, allow padding the image by 1 to allow events to fit
+    @param timestamp_reverse Reverse the timestamps of the events, for backward warping
+    @returns Timestamp images of the positive and negative events: ti_pos, ti_neg
+    """
+    if device is None:
+        device = xs.device
+    xs, ys, ps, ts = xs.squeeze(), ys.squeeze(), ps.squeeze(), ts.squeeze()
+    if padding:
+        img_size = (sensor_size[0]+1, sensor_size[1]+1)
+    else:
+        img_size = sensor_size
+    zero_v = torch.tensor([0.], device=device)
+    ones_v = torch.tensor([1.], device=device)
+
+    mask = torch.ones(xs.size(), device=device)
+    if clip_out_of_range:
+        clipx = img_size[1] if interpolation is None and padding==False else img_size[1]-1
+        clipy = img_size[0] if interpolation is None and padding==False else img_size[0]-1
+        mask = torch.where(xs>=clipx, zero_v, ones_v)*torch.where(ys>=clipy, zero_v, ones_v)
+
+    pos_events_mask = torch.where(ps>0, ones_v, zero_v)
+    neg_events_mask = torch.where(ps<=0, ones_v, zero_v)
+    epsilon = 1e-6
+    if timestamp_reverse:
+        normalized_ts = ((-ts+ts[-1])/(ts[-1]-ts[0]+epsilon)).squeeze()
+    else:
+        normalized_ts = ((ts-ts[0])/(ts[-1]-ts[0]+epsilon)).squeeze()
+    pxs = xs.floor().float()
+    pys = ys.floor().float()
+    dxs = (xs-pxs).float()
+    dys = (ys-pys).float()
+    pxs = (pxs*mask).long()
+    pys = (pys*mask).long()
+    masked_ps = ps*mask
+
+    pos_weights = (normalized_ts*pos_events_mask).float()
+    neg_weights = (normalized_ts*neg_events_mask).float()
+    img_pos = torch.zeros(img_size).to(device)
+    img_pos_cnt = torch.ones(img_size).to(device)
+    img_neg = torch.zeros(img_size).to(device)
+    img_neg_cnt = torch.ones(img_size).to(device)
+
+    interpolate_to_image(pxs, pys, dxs, dys, pos_weights, img_pos)
+    interpolate_to_image(pxs, pys, dxs, dys, pos_events_mask, img_pos_cnt)
+    interpolate_to_image(pxs, pys, dxs, dys, neg_weights, img_neg)
+    interpolate_to_image(pxs, pys, dxs, dys, neg_events_mask, img_neg_cnt)
+
+    # Avoid division by 0
+    img_pos_cnt[img_pos_cnt==0] = 1
+    img_neg_cnt[img_neg_cnt==0] = 1
+    img_pos = img_pos.div(img_pos_cnt)
+    img_neg = img_neg.div(img_neg_cnt)
+    return img_pos, img_neg #/img_pos_cnt, img_neg/img_neg_cnt
+
+class TimestampImage:
+
+    def __init__(self, sensor_size):
+        self.sensor_size = sensor_size
+        self.num_pixels = sensor_size[0]*sensor_size[1]
+        self.image = np.ones(sensor_size)
+
+    def set_init(self, value):
+        self.image = np.ones_like(self.image)*value
+
+    def add_event(self, x, y, t, p):
+        self.image[int(y), int(x)] = t
+
+    def add_events(self, xs, ys, ts, ps):
+        for x, y, t in zip(xs, ys, ts):
+            self.add_event(x, y, t, 0)
+
+    def get_image(self):
+        sort_args = rankdata(self.image, method='dense')
+        sort_args = sort_args-1
+        sort_args = sort_args.reshape(self.sensor_size)
+        sort_args = sort_args/np.max(sort_args)
+        return sort_args
+
+class EventImage:
+
+    def __init__(self, sensor_size):
+        self.sensor_size = sensor_size
+        self.num_pixels = sensor_size[0]*sensor_size[1]
+        self.image = np.ones(sensor_size)
+
+    def add_event(self, x, y, t, p):
+        self.image[int(y), int(x)] += p
+
+    def add_events(self, xs, ys, ts, ps):
+        for x, y, t in zip(xs, ys, ts):
+            self.add_event(x, y, t, 0)
+
+    def get_image(self):
+        mn, mx = np.min(self.image), np.max(self.image)
+        norm_img = (self.image-mn)/(mx-mn)
+        return norm_img

others/event_utils/lib/representations/voxel_grid.py

+import argparse
+import numpy as np
+import matplotlib.pyplot as plt
+import cv2 as cv
+import torch
+from ..util.event_util import events_bounds_mask
+from .image import events_to_image, events_to_image_torch
+
+def get_voxel_grid_as_image(voxelgrid):
+    """
+    Debug function. Returns a voxelgrid as a series of images,
+    one for each bin for display.
+    @param voxelgrid Input voxel grid
+    @returns Image of N bins placed side by side
+    """
+    images = []
+    splitter = np.ones((voxelgrid.shape[1], 2))*np.max(voxelgrid)
+    for image in voxelgrid:
+        images.append(image)
+        images.append(splitter)
+    images.pop()
+    sidebyside = np.hstack(images)
+    sidebyside = cv.normalize(sidebyside, None, 0, 255, cv.NORM_MINMAX)
+    return sidebyside
+
+def plot_voxel_grid(voxelgrid, cmap='gray'):
+    """
+    Debug function. Given a voxel grid, display it as an image.
+    @param voxelgrid The input voxel grid
+    @param cmap The color map to use
+    @returns None
+    """
+    sidebyside = get_voxel_grid_as_image(voxelgrid)
+    plt.imshow(sidebyside, cmap=cmap)
+    plt.show()
+
+def voxel_grids_fixed_n_torch(xs, ys, ts, ps, B, n, sensor_size=(180, 240), temporal_bilinear=True):
+    """
+    Given a set of events, return the voxel grid formed with a fixed number of events.
+    @param xs List of event x coordinates (torch tensor)
+    @param ys List of event y coordinates (torch tensor)
+    @param ts List of event timestamps (torch tensor)
+    @param ps List of event polarities (torch tensor)
+    @param B Number of bins in output voxel grids (int)
+    @param n The number of events per voxel
+    @param sensor_size The size of the event sensor/output voxels
+    @param temporal_bilinear Whether the events should be naively
+        accumulated to the voxels (faster), or properly
+        temporally distributed
+    @returns List of output voxel grids
+    """
+    voxels = []
+    for idx in range(0, len(xs)-n, n):
+        voxels.append(events_to_voxel_torch(xs[idx:idx+n], ys[idx:idx+n],
+            ts[idx:idx+n], ps[idx:idx+n], B, sensor_size=sensor_size,
+            temporal_bilinear=temporal_bilinear))
+    return voxels
+
+def voxel_grids_fixed_t_torch(xs, ys, ts, ps, B, t, sensor_size=(180, 240), temporal_bilinear=True):
+    """
+    Given a set of events, return a voxel grid with a fixed temporal width.
+    @param xs List of event x coordinates (torch tensor)
+    @param ys List of event y coordinates (torch tensor)
+    @param ts List of event timestamps (torch tensor)
+    @param ps List of event polarities (torch tensor)
+    @param B Number of bins in output voxel grids (int)
+    @param t The time width of the voxel grids
+    @param sensor_size The size of the event sensor/output voxels
+    @param temporal_bilinear Whether the events should be naively
+        accumulated to the voxels (faster), or properly
+        temporally distributed
+    @returns List of output voxel grids
+    """
+    device = xs.device
+    voxels = []
+    np_ts = ts.cpu().numpy()
+    for t_start in np.arange(ts[0].item(), ts[-1].item()-t, t):
+        voxels.append(events_to_voxel_timesync_torch(xs, ys, ts, ps, B, t_start, t_start+t, np_ts=np_ts,
+            sensor_size=sensor_size, temporal_bilinear=temporal_bilinear))
+    return voxels
+
+def events_to_voxel_timesync_torch(xs, ys, ts, ps, B, t0, t1, device=None, np_ts=None,
+        sensor_size=(180, 240), temporal_bilinear=True):
+    """
+    Given a set of events, return a voxel grid of the events between t0 and t1
+    @param xs List of event x coordinates (torch tensor)
+    @param ys List of event y coordinates (torch tensor)
+    @param ts List of event timestamps (torch tensor)
+    @param ps List of event polarities (torch tensor)
+    @param B Number of bins in output voxel grids (int)
+    @param t0 The start time of the voxel grid
+    @param t1 The end time of the voxel grid
+    @param device Device to put voxel grid. If left empty, same device as events
+    @param np_ts A numpy copy of ts (optional). If not given, will be created in situ
+    @param sensor_size The size of the event sensor/output voxels
+    @param temporal_bilinear Whether the events should be naively
+        accumulated to the voxels (faster), or properly
+        temporally distributed
+    @returns Voxel of the events between t0 and t1
+    """
+    assert(t1>t0)
+    if np_ts is None:
+        np_ts = ts.cpu().numpy()
+    if device is None:
+        device = xs.device
+    start_idx = np.searchsorted(np_ts, t0)
+    end_idx = np.searchsorted(np_ts, t1)
+    assert(start_idx < end_idx)
+    voxel = events_to_voxel_torch(xs[start_idx:end_idx], ys[start_idx:end_idx],
+        ts[start_idx:end_idx], ps[start_idx:end_idx], B, device, sensor_size=sensor_size,
+        temporal_bilinear=temporal_bilinear)
+    return voxel
+
+def events_to_voxel_torch(xs, ys, ts, ps, B, device=None, sensor_size=(180, 240), temporal_bilinear=True):
+    """
+    Turn set of events to a voxel grid tensor, using temporal bilinear interpolation
+    @param xs List of event x coordinates (torch tensor)
+    @param ys List of event y coordinates (torch tensor)
+    @param ts List of event timestamps (torch tensor)
+    @param ps List of event polarities (torch tensor)
+    @param B Number of bins in output voxel grids (int)
+    @param device Device to put voxel grid. If left empty, same device as events
+    @param sensor_size The size of the event sensor/output voxels
+    @param temporal_bilinear Whether the events should be naively
+        accumulated to the voxels (faster), or properly
+        temporally distributed
+    @returns Voxel of the events between t0 and t1
+    """
+    if device is None:
+        device = xs.device
+    assert(len(xs)==len(ys) and len(ys)==len(ts) and len(ts)==len(ps))
+    bins = []
+    dt = ts[-1]-ts[0]
+    t_norm = (ts-ts[0])/dt*(B-1)
+    zeros = torch.zeros(t_norm.size())
+    for bi in range(B):
+        if temporal_bilinear:
+            bilinear_weights = torch.max(zeros, 1.0-torch.abs(t_norm-bi))
+            weights = ps*bilinear_weights
+            vb = events_to_image_torch(xs, ys,
+                    weights, device, sensor_size=sensor_size,
+                    clip_out_of_range=False)
+        else:
+            tstart = t[0] + dt*bi
+            tend = tstart + dt
+            beg = binary_search_torch_tensor(t, 0, len(ts)-1, tstart)
+            end = binary_search_torch_tensor(t, 0, len(ts)-1, tend)
+            vb = events_to_image_torch(xs[beg:end], ys[beg:end],
+                    ps[beg:end], device, sensor_size=sensor_size,
+                    clip_out_of_range=False)
+        bins.append(vb)
+    bins = torch.stack(bins)
+    return bins
+
+def events_to_neg_pos_voxel_torch(xs, ys, ts, ps, B, device=None,
+        sensor_size=(180, 240), temporal_bilinear=True):
+    """
+    Turn set of events to a voxel grid tensor, using temporal bilinear interpolation.
+    Positive and negative events are put into separate voxel grids
+    @param xs List of event x coordinates (torch tensor)
+    @param ys List of event y coordinates (torch tensor)
+    @param ts List of event timestamps (torch tensor)
+    @param ps List of event polarities (torch tensor)
+    @param B Number of bins in output voxel grids (int)
+    @param device Device to put voxel grid. If left empty, same device as events
+    @param sensor_size The size of the event sensor/output voxels
+    @param temporal_bilinear Whether the events should be naively
+        accumulated to the voxels (faster), or properly
+        temporally distributed
+    @returns Two voxel grids, one for positive one for negative events
+    """
+    zero_v = torch.tensor([0.])
+    ones_v = torch.tensor([1.])
+    pos_weights = torch.where(ps>0, ones_v, zero_v)
+    neg_weights = torch.where(ps<=0, ones_v, zero_v)
+
+    voxel_pos = events_to_voxel_torch(xs, ys, ts, pos_weights, B, device=device,
+            sensor_size=sensor_size, temporal_bilinear=temporal_bilinear)
+    voxel_neg = events_to_voxel_torch(xs, ys, ts, neg_weights, B, device=device,
+            sensor_size=sensor_size, temporal_bilinear=temporal_bilinear)
+
+    return voxel_pos, voxel_neg
+
+def events_to_voxel(xs, ys, ts, ps, B, sensor_size=(180, 240), temporal_bilinear=True):
+    """
+    Turn set of events to a voxel grid tensor, using temporal bilinear interpolation
+    @param xs List of event x coordinates (torch tensor)
+    @param ys List of event y coordinates (torch tensor)
+    @param ts List of event timestamps (torch tensor)
+    @param ps List of event polarities (torch tensor)
+    @param B Number of bins in output voxel grids (int)
+    @param sensor_size The size of the event sensor/output voxels
+    @param temporal_bilinear Whether the events should be naively
+        accumulated to the voxels (faster), or properly
+        temporally distributed
+    @returns Voxel of the events between t0 and t1
+    """
+    assert(len(xs)==len(ys) and len(ys)==len(ts) and len(ts)==len(ps))
+    num_events_per_bin = len(xs)//B
+    bins = []
+    dt = ts[-1]-ts[0]
+    t_norm = (ts-ts[0])/dt*(B-1)
+    zeros = (np.expand_dims(np.zeros(t_norm.shape[0]), axis=0).transpose()).squeeze()
+    for bi in range(B):
+        if temporal_bilinear:
+            bilinear_weights = np.maximum(zeros, 1.0-np.abs(t_norm-bi))
+            weights = ps*bilinear_weights
+            vb = events_to_image(xs.squeeze(), ys.squeeze(), weights.squeeze(),
+                    sensor_size=sensor_size, interpolation=None)
+        else:
+            beg = bi*num_events_per_bin
+            end = beg + num_events_per_bin
+            vb = events_to_image(xs[beg:end], ys[beg:end],
+                    weights[beg:end], sensor_size=sensor_size)
+        bins.append(vb)
+    bins = np.stack(bins)
+    return bins
+
+def events_to_neg_pos_voxel(xs, ys, ts, ps, B,
+        sensor_size=(180, 240), temporal_bilinear=True):
+    """
+    Turn set of events to a voxel grid tensor, using temporal bilinear interpolation.
+    Positive and negative events are put into separate voxel grids
+    @param xs List of event x coordinates (torch tensor)
+    @param ys List of event y coordinates (torch tensor)
+    @param ts List of event timestamps (torch tensor)
+    @param ps List of event polarities (torch tensor)
+    @param B Number of bins in output voxel grids (int)
+    @param sensor_size The size of the event sensor/output voxels
+    @param temporal_bilinear Whether the events should be naively
+        accumulated to the voxels (faster), or properly
+        temporally distributed
+    @returns Two voxel grids, one for positive one for negative events
+    """
+    pos_weights = np.where(ps, 1, 0)
+    neg_weights = np.where(ps, 0, 1)
+
+    voxel_pos = events_to_voxel(xs, ys, ts, pos_weights, B,
+            sensor_size=sensor_size, temporal_bilinear=temporal_bilinear)
+    voxel_neg = events_to_voxel(xs, ys, ts, neg_weights, B,
+            sensor_size=sensor_size, temporal_bilinear=temporal_bilinear)
+
+    return voxel_pos, voxel_neg

others/event_utils/lib/transforms/optic_flow.py

+import numpy as np
+import torch
+import torch.nn.functional as F
+
+def warp_events_flow_torch(xt, yt, tt, pt, flow_field, t0=None,
+        batched=False, batch_indices=None):
+    """
+    Given events and a flow field, warp the events by the flow
+    Parameters
+    ----------
+    xs : list of event x coordinates 
+    ys : list of event y coordinates 
+    ts : list of event timestamps 
+    ps : list of event polarities 
+    flow_field : 2D tensor containing the flow at each x,y position
+    t0 : the reference time to warp events to. If empty, will use the
+        timestamp of the last event
+    Returns
+    -------
+    warped_xt: x coords of warped events
+    warped_yt: y coords of warped events
+    """
+    if len(xt.shape) > 1:
+        xt, yt, tt, pt = xt.squeeze(), yt.squeeze(), tt.squeeze(), pt.squeeze()
+    if t0 is None:
+        t0 = tt[-1]
+    while len(flow_field.size()) < 4:
+        flow_field = flow_field.unsqueeze(0)
+    if len(xt.size()) == 1:
+        event_indices = torch.transpose(torch.stack((xt, yt), dim=0), 0, 1)
+    else:
+        event_indices = torch.transpose(torch.cat((xt, yt), dim=1), 0, 1)
+    #event_indices.requires_grad_ = False
+    event_indices = torch.reshape(event_indices, [1, 1, len(xt), 2])
+
+    # Event indices need to be between -1 and 1 for F.gridsample
+    event_indices[:,:,:,0] = event_indices[:,:,:,0]/(flow_field.shape[-1]-1)*2.0-1.0
+    event_indices[:,:,:,1] = event_indices[:,:,:,1]/(flow_field.shape[-2]-1)*2.0-1.0
+
+    flow_at_event = F.grid_sample(flow_field, event_indices, align_corners=True)
+    dt = (tt-t0).squeeze()
+
+    warped_xt = xt+flow_at_event[:,0,:,:].squeeze()*dt
+    warped_yt = yt+flow_at_event[:,1,:,:].squeeze()*dt
+
+    return warped_xt, warped_yt
+

others/event_utils/lib/util/__init__.py

+# __init__.py
+from .event_util import *
+from .util import *

others/event_utils/lib/util/event_util.py

+import numpy as np
+import h5py
+from ..representations.image import events_to_image
+
+def infer_resolution(xs, ys):
+    """
+    Given events, guess the resolution by looking at the max and min values
+    @param xs Event x coords
+    @param ys Event y coords
+    @returns Inferred resolution
+    """
+    sr = [np.max(ys) + 1, np.max(xs) + 1]
+    return sr
+
+def events_bounds_mask(xs, ys, x_min, x_max, y_min, y_max):
+    """
+    Get a mask of the events that are within the given bounds
+    @param xs Event x coords
+    @param ys Event y coords
+    @param x_min Lower bound of x axis
+    @param x_max Upper bound of x axis
+    @param y_min Lower bound of y axis
+    @param y_max Upper bound of y axis
+    @returns mask
+    """
+    mask = np.where(np.logical_or(xs<=x_min, xs>x_max), 0.0, 1.0)
+    mask *= np.where(np.logical_or(ys<=y_min, ys>y_max), 0.0, 1.0)
+    return mask
+
+def cut_events_to_lifespan(xs, ys, ts, ps, params,
+        pixel_crossings, minimum_events=100, side='back'):
+    """
+    Given motion model parameters, compute the speed and thus
+    the lifespan, given a desired number of pixel crossings
+    @param xs Event x coords
+    @param ys Event y coords
+    @param ts Event timestamps
+    @param ps Event polarities
+    @param params Motion model parameters
+    @param pixel_crossings Number of pixel crossings
+    @param minimum_events The minimum number of events to cut down to
+    @param side Cut events from 'back' or 'front'
+    @returns Cut events
+    """
+    magnitude = np.linalg.norm(params)
+    dt = pixel_crossings/magnitude
+    if side == 'back':
+        s_idx = np.searchsorted(ts, ts[-1]-dt)
+        num_events = len(xs)-s_idx
+        s_idx = len(xs)-minimum_events if num_events < minimum_events else s_idx
+        return xs[s_idx:-1], ys[s_idx:-1], ts[s_idx:-1], ps[s_idx:-1]
+    elif side == 'front':
+        s_idx = np.searchsorted(ts, dt+ts[0])
+        num_events = s_idx
+        s_idx = minimum_events if num_events < minimum_events else s_idx
+        return xs[0:s_idx], ys[0:s_idx], ts[0:s_idx], ps[0:s_idx]
+    else:
+        raise Exception("Invalid side given: {}. To cut events, must provide an \
+                appropriate side to cut from, either 'front' or 'back'".format(side))
+
+def clip_events_to_bounds(xs, ys, ts, ps, bounds, set_zero=False):
+    """
+    Clip events to the given bounds.
+    @param xs x coords of events
+    @param ys y coords of events
+    @param ts Timestamps of events (may be None)
+    @param ps Polarities of events (may be None)
+    @param bounds the bounds of the events. Must be list of
+       length 2 (in which case the lower bound is assumed to be 0,0)
+       or length 4, in format [min_y, max_y, min_x, max_x]
+    @param: set_zero if True, simply multiplies the out of bounds events with 0 mask.
+        Otherwise, removes the events.
+    @returns Clipped events
+    """
+    if len(bounds) == 2:
+        bounds = [0, bounds[0], 0, bounds[1]]
+    elif len(bounds) != 4:
+        raise Exception("Bounds must be of length 2 or 4 (not {})".format(len(bounds)))
+    miny, maxy, minx, maxx = bounds
+    if set_zero:
+        mask = events_bounds_mask(xs, ys, minx, maxx, miny, maxy)
+        ts_mask = None if ts is None else ts*mask
+        ps_mask = None if ps is None else ps*mask
+        return xs*mask, ys*mask, ts_mask, ps_mask
+    else:
+        x_clip_idc = np.argwhere((xs >= minx) & (xs < maxx))[:, 0]
+        y_subset = ys[x_clip_idc]
+        y_clip_idc = np.argwhere((y_subset >= miny) & (y_subset < maxy))[:, 0]
+
+        xs_clip = xs[x_clip_idc][y_clip_idc]
+        ys_clip = ys[x_clip_idc][y_clip_idc]
+        ts_clip = None if ts is None else ts[x_clip_idc][y_clip_idc]
+        ps_clip = None if ps is None else ps[x_clip_idc][y_clip_idc]
+        return xs_clip, ys_clip, ts_clip, ps_clip
+
+def get_events_from_mask(mask, xs, ys):
+    """
+    Given an image mask, return the indices of all events at each location in the mask
+    @params mask The image mask
+    @param xs x components of events as list
+    @param ys y components of events as list
+    @returns Indices of events that lie on the mask
+    """
+    xs = xs.astype(int)
+    ys = ys.astype(int)
+    idx = np.stack((ys, xs))
+    event_vals = mask[tuple(idx)]
+    event_indices = np.argwhere(event_vals >= 0.01).squeeze()
+    return event_indices
+
+def binary_search_h5_dset(dset, x, l=None, r=None, side='left'):
+    """
+    Binary search for a timestamp in an HDF5 event file, without
+    loading the entire file into RAM
+    @param dset The HDF5 dataset
+    @param x The timestamp being searched for
+    @param l Starting guess for the left side (0 if None is chosen)
+    @param r Starting guess for the right side (-1 if None is chosen)
+    @param side Which side to take final result for if exact match is not found
+    @returns Index of nearest event to 'x'
+    """
+    l = 0 if l is None else l
+    r = len(dset)-1 if r is None else r
+    while l <= r:
+        mid = l + (r - l)//2;
+        midval = dset[mid]
+        if midval == x:
+            return mid
+        elif midval < x:
+            l = mid + 1
+        else:
+            r = mid - 1
+    if side == 'left':
+        return l
+    return r
+
+def binary_search_h5_timestamp(hdf_path, l, r, x, side='left'):
+    f = h5py.File(hdf_path, 'r')
+    return binary_search_h5_dset(f['events/ts'], x, l=l, r=r, side=side)
+
+def binary_search_torch_tensor(t, l, r, x, side='left'):
+    """
+    Binary search implemented for pytorch tensors (no native implementation exists)
+    @param t The tensor
+    @param x The value being searched for
+    @param l Starting lower bound (0 if None is chosen)
+    @param r Starting upper bound (-1 if None is chosen)
+    @param side Which side to take final result for if exact match is not found
+    @returns Index of nearest event to 'x'
+    """
+    if r is None:
+        r = len(t)-1
+    while l <= r:
+        mid = l + (r - l)//2;
+        midval = t[mid]
+        if midval == x:
+            return mid
+        elif midval < x:
+            l = mid + 1
+        else:
+            r = mid - 1
+    if side == 'left':
+        return l
+    return r
+
+def remove_hot_pixels(xs, ys, ts, ps, sensor_size=(180, 240), num_hot=50):
+    """
+    Given a set of events, removes the 'hot' pixel events.
+    Accumulates all of the events into an event image and removes
+    the 'num_hot' highest value pixels.
+    @param xs Event x coords
+    @param ys Event y coords
+    @param ts Event timestamps
+    @param ps Event polarities
+    @param sensor_size The size of the event camera sensor
+    @param num_hot The number of hot pixels to remove
+    """
+    img = events_to_image(xs, ys, ps, sensor_size=sensor_size)
+    hot = np.array([])
+    for i in range(num_hot):
+        maxc = np.unravel_index(np.argmax(img), sensor_size)
+        #print("{} = {}".format(maxc, img[maxc]))
+        img[maxc] = 0
+        h = np.where((xs == maxc[1]) & (ys == maxc[0]))
+        hot = np.concatenate((hot, h[0]))
+    xs, ys, ts, ps = np.delete(xs, hot), np.delete(ys, hot), np.delete(ts, hot), np.delete(ps, hot)
+    return xs, ys, ts, ps

others/event_utils/lib/util/util.py

+import json
+import numpy as np
+import cv2 as cv
+import pandas as pd
+from pathlib import Path
+from itertools import repeat
+from collections import OrderedDict
+from math import fabs, ceil, floor
+from torch.nn import ZeroPad2d
+import matplotlib.pyplot as plt
+import matplotlib.patches as patches
+import cv2 as cv
+
+
+def ensure_dir(dirname):
+    """
+    Ensure a directory exists, if not create it
+    @param dirname Directory name
+    @returns None
+    """
+    dirname = Path(dirname)
+    if not dirname.is_dir():
+        dirname.mkdir(parents=True, exist_ok=False)
+
+
+def read_json(fname):
+    fname = Path(fname)
+    with fname.open('rt') as handle:
+        return json.load(handle, object_hook=OrderedDict)
+
+
+def write_json(content, fname):
+    fname = Path(fname)
+    with fname.open('wt') as handle:
+        json.dump(content, handle, indent=4, sort_keys=False)
+
+
+def inf_loop(data_loader):
+    ''' wrapper function for endless data loader. '''
+    for loader in repeat(data_loader):
+        yield from loader
+
+
+def optimal_crop_size(max_size, max_subsample_factor, safety_margin=0):
+    """ Find the optimal crop size for a given max_size and subsample_factor.
+        The optimal crop size is the smallest integer which is greater or equal than max_size,
+        while being divisible by 2^max_subsample_factor.
+    """
+    crop_size = int(pow(2, max_subsample_factor) * ceil(max_size / pow(2, max_subsample_factor)))
+    crop_size += safety_margin * pow(2, max_subsample_factor)
+    return crop_size
+
+
+class CropParameters:
+    """ Helper class to compute and store useful parameters for pre-processing and post-processing
+        of images in and out of E2VID.
+        Pre-processing: finding the best image size for the network, and padding the input image with zeros
+        Post-processing: Crop the output image back to the original image size
+    """
+
+    def __init__(self, width, height, num_encoders, safety_margin=0):
+
+        self.height = height
+        self.width = width
+        self.num_encoders = num_encoders
+        self.width_crop_size = optimal_crop_size(self.width, num_encoders, safety_margin)
+        self.height_crop_size = optimal_crop_size(self.height, num_encoders, safety_margin)
+
+        self.padding_top = ceil(0.5 * (self.height_crop_size - self.height))
+        self.padding_bottom = floor(0.5 * (self.height_crop_size - self.height))
+        self.padding_left = ceil(0.5 * (self.width_crop_size - self.width))
+        self.padding_right = floor(0.5 * (self.width_crop_size - self.width))
+        self.pad = ZeroPad2d((self.padding_left, self.padding_right, self.padding_top, self.padding_bottom))
+
+        self.cx = floor(self.width_crop_size / 2)
+        self.cy = floor(self.height_crop_size / 2)
+
+        self.ix0 = self.cx - floor(self.width / 2)
+        self.ix1 = self.cx + ceil(self.width / 2)
+        self.iy0 = self.cy - floor(self.height / 2)
+        self.iy1 = self.cy + ceil(self.height / 2)
+
+    def crop(self, img):
+        return img[..., self.iy0:self.iy1, self.ix0:self.ix1]
+
+
+def format_power(size):
+    power = 1e3
+    n = 0
+    power_labels = {0: '', 1: 'K', 2: 'M', 3: 'G', 4: 'T'}
+    while size > power:
+        size /= power
+        n += 1
+    return size, power_labels[n]
+
+def plot_image(image, lognorm=False, cmap='gray', bbox=None, ticks=False, norm=True, savename=None, colorbar=False):
+    """
+    Plot an image
+    :param image: The image to plot, as np array
+    :param lognorm: If true, apply log transform the normalize image
+    :param cmap: Colormap (defaul gray)
+    :param bbox: Optional bounding box to draw on image, as array with [[top corner x,y,w,h]]
+    :param ticks: Whether or not to draw axis ticks
+    :param norm: Normalize image?
+    :param savename: Optional save path
+    :param colorbar: Display color bar if true
+    """
+    fig, ax = plt.subplots(1)
+    if lognorm:
+        image = np.log10(image)
+        cmap='viridis'
+    if norm:
+        image = cv.normalize(image, None, 0, 1.0, cv.NORM_MINMAX)
+    ims = ax.imshow(image, cmap=cmap)
+    if bbox is not None:
+        w,h = bbox[2], bbox[3]
+        rect = patches.Rectangle((bbox[0:2]), w, h, linewidth=1, edgecolor='r', facecolor='none')
+        ax.add_patch(rect)
+    if colorbar:
+        fig.colorbar(ims)
+    if not ticks:
+        plt.axis('off')
+    if savename is not None:
+        plt.savefig(savename)
+    plt.show()
+
+def plot_image_grid(images, grid_shape=None, lognorm=False,
+        cmap='gray', bbox=None, norm=True, savename=None,
+        colorbar=False):
+    """
+    Given a list of images, stitches them into a grid and displays/saves the grid
+    @param images List of images
+    @param grid_shape Shape of the grid
+    @param lognorm Logarithmic normalise the image
+    @param cmap Color map to use
+    @param bbox Draw a bounding box on the image
+    @param norm If True, normalise the image
+    @param savename If set, save the image to that path
+    @param colorbar If true, plot the colorbar
+    """
+    if grid_shape is None:
+        grid_shape = [1, len(images)]
+
+    col = []
+    img_idx = 0
+    for xc in range(grid_shape[0]):
+        row = []
+        for yc in range(grid_shape[1]):
+            image = images[img_idx]
+            if lognorm:
+                image = np.log10(image)
+                cmap='viridis'
+            if norm:
+                image = cv.normalize(image, None, 0, 1.0, cv.NORM_MINMAX)
+            row.append(image)
+            img_idx += 1
+        col.append(np.concatenate(row, axis=1))
+    comp_img = np.concatenate(col, axis=0)
+    if savename is None:
+        plot_image(comp_img, norm=False, colorbar=colorbar, cmap=cmap)
+    else:
+        save_image(comp_img, fname=savename, colorbar=colorbar, cmap=cmap)
+
+def save_image(image, fname=None, lognorm=False, cmap='gray', bbox=None, colorbar=False):
+    fname = "/tmp/img.png" if fname is None else fname
+    fig, ax = plt.subplots(1)
+    if lognorm:
+        image = np.log10(image)
+        cmap='viridis'
+    image = cv.normalize(image, None, 0, 1.0, cv.NORM_MINMAX)
+    ims = ax.imshow(image, cmap=cmap)
+    if bbox is not None:
+        w = bbox[1][0]-bbox[0][0]
+        h = bbox[1][1]-bbox[0][1]
+        rect = patches.Rectangle((bbox[0]), w, h, linewidth=1, edgecolor='r', facecolor='none')
+        ax.add_patch(rect)
+    if colorbar:
+        fig.colorbar(ims)
+    plt.savefig(fname, dpi=150)
+    plt.close()
+
+def flow2bgr_np(disp_x, disp_y, max_magnitude=None):
+    """
+    Convert an optic flow tensor to an RGB color map for visualization
+    Code adapted from: https://github.com/ClementPinard/FlowNetPytorch/blob/master/main.py#L339
+    @param disp_x A [H x W] NumPy array containing the X displacement
+    @param disp_y A [H x W] NumPy array containing the Y displacement
+    @returns A [H x W x 3] NumPy array containing a color-coded representation of the flow [0, 255]
+    """
+    assert(disp_x.shape == disp_y.shape)
+    H, W = disp_x.shape
+
+    # X, Y = np.meshgrid(np.linspace(-1, 1, H), np.linspace(-1, 1, W))
+
+    # flow_x = (X - disp_x) * float(W) / 2
+    # flow_y = (Y - disp_y) * float(H) / 2
+    # magnitude, angle = cv.cartToPolar(flow_x, flow_y)
+    # magnitude, angle = cv.cartToPolar(disp_x, disp_y)
+
+    # follow alex zhu color convention https://github.com/daniilidis-group/EV-FlowNet
+
+    flows = np.stack((disp_x, disp_y), axis=2)
+    magnitude = np.linalg.norm(flows, axis=2)
+
+    angle = np.arctan2(disp_y, disp_x)
+    angle += np.pi
+    angle *= 180. / np.pi / 2.
+    angle = angle.astype(np.uint8)
+
+    if max_magnitude is None:
+        v = np.zeros(magnitude.shape, dtype=np.uint8)
+        cv.normalize(src=magnitude, dst=v, alpha=0, beta=255, norm_type=cv.NORM_MINMAX, dtype=cv.CV_8U)
+    else:
+        v = np.clip(255.0 * magnitude / max_magnitude, 0, 255)
+        v = v.astype(np.uint8)
+
+    hsv = np.zeros((H, W, 3), dtype=np.uint8)
+    hsv[..., 1] = 255
+    hsv[..., 0] = angle
+    hsv[..., 2] = v
+    bgr = cv.cvtColor(hsv, cv.COLOR_HSV2BGR)
+
+    return bgr

others/event_utils/lib/visualization/__init__.py

+# __init__.py
+from . import draw_event_stream

others/event_utils/lib/visualization/draw_event_stream.py

+import numpy as np
+import numpy.lib.recfunctions as nlr
+import cv2 as cv
+from skimage.measure import block_reduce
+import os
+import matplotlib.pyplot as plt
+from mpl_toolkits.mplot3d import Axes3D
+
+from ..representations.image import events_to_image
+from ..representations.voxel_grid import events_to_voxel
+from ..util.event_util import clip_events_to_bounds
+from .visualization_utils import *
+from tqdm import tqdm
+
+def plot_events_sliding(xs, ys, ts, ps, args, frames=[], frame_ts=[]):
+    """
+    Plot the given events in a sliding window fashion to generate a video
+    @param xs x component of events
+    @param ys y component of events
+    @param ts t component of events
+    @param ps p component of events
+    @param args Arguments for the rendering (see args list
+        for 'plot_events' function)
+    @param frames List of image frames
+    @param frame_ts List of the image timestamps
+    @returns None
+    """
+    dt, sdt = args.w_width, args.sw_width
+    if dt is None:
+        dt = (ts[-1]-ts[0])/10
+        sdt = dt/10
+        print("Using dt={}, sdt={}".format(dt, sdt))
+
+    if len(frames) > 0:
+        has_frames = True
+        sensor_size = frames[0].shape
+        frame_ts = frame_ts[:,1] if len(frame_ts.shape) == 2 else frame_ts
+    else:
+        has_frames = False
+        sensor_size = [max(ys), max(xs)]
+
+    n_frames = len(np.arange(ts[0], ts[-1]-dt, sdt))
+    for i, t0 in enumerate(tqdm(np.arange(ts[0], ts[-1]-dt, sdt))):
+        te = t0+dt
+        eidx0 = np.searchsorted(ts, t0)
+        eidx1 = np.searchsorted(ts, te)
+        wxs, wys, wts, wps = xs[eidx0:eidx1], ys[eidx0:eidx1], ts[eidx0:eidx1], ps[eidx0:eidx1],
+
+        wframes, wframe_ts = [], []
+        if has_frames:
+            fidx0 = np.searchsorted(frame_ts, t0)
+            fidx1 = np.searchsorted(frame_ts, te)
+            wframes = [frames[fidx0]]
+            wframe_ts = [wts[0]]
+
+        save_path = os.path.join(args.output_path, "frame_{:010d}.jpg".format(i))
+
+        perc = i/n_frames
+        min_p, max_p = 0.2, 0.7
+        elev, azim = args.elev, args.azim
+        max_elev, max_azim = 10, 45
+        if perc > min_p and perc < max_p:
+            p_way = (perc-min_p)/(max_p-min_p)
+            elev = elev + (max_elev*p_way)
+            azim = azim - (max_azim*p_way)
+        elif perc >= max_p:
+            elev, azim = max_elev, max_azim
+
+        plot_events(wxs, wys, wts, wps, save_path=save_path, num_show=args.num_show, event_size=args.event_size,
+                imgs=wframes, img_ts=wframe_ts, show_events=not args.hide_events, azim=azim,
+                elev=elev, show_frames=not args.hide_frames, crop=args.crop, compress_front=args.compress_front,
+                invert=args.invert, num_compress=args.num_compress, show_plot=args.show_plot, img_size=sensor_size,
+                show_axes=args.show_axes, stride=args.stride)
+
+def plot_voxel_grid(xs, ys, ts, ps, bins=5, frames=[], frame_ts=[],
+        sensor_size=None, crop=None, elev=0, azim=45, show_axes=False):
+    """
+    @param xs x component of events
+    @param ys y component of events
+    @param ts t component of events
+    @param ps p component of events
+    @param bins The number of bins to have in the voxel grid
+    @param frames The list of image frames
+    @param frame_ts The list of image timestamps
+    @param sensor_size The size of the event sensor resolution
+    @param crop Cropping parameters for the voxel grid (no crop if None)
+    @param elev The elevation of the plot
+    @param azim The azimuth of the plot
+    @param show_axes Show the axes of the plot
+    @returns None
+    """
+    if sensor_size is None:
+        sensor_size = [np.max(ys)+1, np.max(xs)+1] if len(frames)==0 else frames[0].shape
+    if crop is not None:
+        xs, ys, ts, ps = clip_events_to_bounds(xs, ys, ts, ps, crop)
+        sensor_size = crop_to_size(crop)
+        xs, ys = xs-crop[2], ys-crop[0]
+    num = 10000
+    xs, ys, ts, ps = xs[0:num], ys[0:num], ts[0:num], ps[0:num]
+    if len(xs) == 0:
+        return
+    voxels = events_to_voxel(xs, ys, ts, ps, bins, sensor_size=sensor_size)
+    voxels = block_reduce(voxels, block_size=(1,10,10), func=np.mean, cval=0)
+    dimdiff = voxels.shape[1]-voxels.shape[0]
+    filler = np.zeros((dimdiff, *voxels.shape[1:]))
+    voxels = np.concatenate((filler, voxels), axis=0)
+    voxels = voxels.transpose(0,2,1)
+
+    pltvoxels = voxels != 0
+    pvp, nvp = voxels > 0, voxels < 0
+    pvox, nvox = voxels*np.where(voxels > 0, 1, 0), voxels*np.where(voxels < 0, 1, 0)
+    pvox, nvox = (pvox/np.max(pvox))*0.5+0.5, (np.abs(nvox)/np.max(np.abs(nvox)))*0.5+0.5
+    zeros = np.zeros_like(voxels)
+
+    colors = np.empty(voxels.shape, dtype=object)
+
+    redvals = np.stack((pvox, zeros, pvox-0.5), axis=3)
+    redvals = nlr.unstructured_to_structured(redvals).astype('O')
+
+    bluvals = np.stack((nvox-0.5, zeros, nvox), axis=3)
+    bluvals = nlr.unstructured_to_structured(bluvals).astype('O')
+
+    colors[pvp] = redvals[pvp]
+    colors[nvp] = bluvals[nvp]
+
+    fig = plt.figure()
+    ax = fig.gca(projection='3d')
+    ax.voxels(pltvoxels, facecolors=colors, edgecolor='k')
+    ax.view_init(elev=elev, azim=azim)
+
+    ax.grid(False)
+    # Hide panes
+    ax.xaxis.pane.fill = False
+    ax.yaxis.pane.fill = False
+    ax.zaxis.pane.fill = False
+    if not show_axes:
+        # Hide spines
+        ax.w_xaxis.line.set_color((1.0, 1.0, 1.0, 0.0))
+        ax.w_yaxis.line.set_color((1.0, 1.0, 1.0, 0.0))
+        ax.w_zaxis.line.set_color((1.0, 1.0, 1.0, 0.0))
+        ax.set_frame_on(False)
+    # Hide xy axes
+    ax.set_xticks([])
+    ax.set_yticks([])
+    ax.set_zticks([])
+
+    ax.xaxis.set_visible(False)
+    ax.axes.get_yaxis().set_visible(False)
+
+    plt.show()
+
+def plot_events(xs, ys, ts, ps, save_path=None, num_compress='auto', num_show=1000,
+        event_size=2, elev=0, azim=45, imgs=[], img_ts=[], show_events=True,
+        show_frames=True, show_plot=False, crop=None, compress_front=False,
+        marker='.', stride = 1, invert=False, img_size=None, show_axes=False):
+    """
+    Given events, plot these in a spatiotemporal volume.
+    @param xs x coords of events
+    @param ys y coords of events
+    @param ts t coords of events
+    @param ps p coords of events
+    @param save_path If set, will save plot to here
+    @param num_compress Takes num_compress events from the beginning of the
+        sequence and draws them in the plot at time $t=0$ in black
+    @param compress_front If True, display the compressed events in black at the
+        front of the spatiotemporal volume rather than the back
+    @param num_show Sets the number of events to plot. If set to -1
+        will plot all of the events (can be potentially expensive)
+    @param event_size Sets the size of the plotted events
+    @param elev Sets the elevation of the plot
+    @param azim Sets the azimuth of the plot
+    @param imgs A list of images to draw into the spatiotemporal volume
+    @param img_ts A list of the position on the temporal axis where each
+        image from 'imgs' is to be placed (the timestamp of the images, usually)
+    @param show_events If False, will not plot the events (only images)
+    @param show_plot If True, display the plot in a matplotlib window as
+        well as saving to disk
+    @param crop A list of length 4 that sets the crop of the plot (must
+        be in the format [top_left_y, top_left_x, height, width]
+    @param marker Which marker should be used to display the events (default
+        is '.', which results in points, but circles 'o' or crosses 'x' are
+        among many other possible options)
+    @param stride Determines the pixel stride of the image rendering
+        (1=full resolution, but can be quite resource intensive)
+    @param invert Inverts the color scheme for black backgrounds
+    @param img_size The size of the sensor resolution. Inferred if empty.
+    @param show_axes If True, draw axes onto the plot.
+    @returns None
+    """
+    #Crop events
+    if img_size is None:
+        img_size = [max(ys), max(xs)] if len(imgs)==0 else imgs[0].shape[0:2]
+        print("Inferred image size = {}".format(img_size))
+    crop = [0, img_size[0], 0, img_size[1]] if crop is None else crop
+    xs, ys, ts, ps = clip_events_to_bounds(xs, ys, ts, ps, crop, set_zero=False)
+    xs, ys = xs-crop[2], ys-crop[0]
+
+    #Defaults and range checks
+    num_show = len(xs) if num_show == -1 else num_show
+    skip = max(len(xs)//num_show, 1)
+    num_compress = len(xs) if num_compress == -1 else num_compress
+    num_compress = min(img_size[0]*img_size[1]*0.5, len(xs)) if num_compress=='auto' else num_compress
+    xs, ys, ts, ps = xs[::skip], ys[::skip], ts[::skip], ps[::skip]
+
+    #Prepare the plot, set colors
+    fig = plt.figure()
+    ax = fig.add_subplot(111, projection='3d', proj_type = 'ortho')
+    colors = ['r' if p>0 else ('#00DAFF' if invert else 'b') for p in ps]
+
+    #Plot images
+    if len(imgs)>0 and show_frames:
+        for imgidx, (img, img_ts) in enumerate(zip(imgs, img_ts)):
+            img = img[crop[0]:crop[1], crop[2]:crop[3]]
+            if len(img.shape)==2:
+                img = np.stack((img, img, img), axis=2)
+            if num_compress > 0:
+                events_img = events_to_image(xs[0:num_compress], ys[0:num_compress],
+                        np.ones(num_compress), sensor_size=img.shape[0:2])
+                events_img[events_img>0] = 1
+                img[:,:,1]+=events_img[:,:]
+                img = np.clip(img, 0, 1)
+            x, y = np.ogrid[0:img.shape[0], 0:img.shape[1]]
+            event_idx = np.searchsorted(ts, img_ts)
+
+            ax.scatter(xs[0:event_idx], ts[0:event_idx], ys[0:event_idx], zdir='z',
+                    c=colors[0:event_idx], facecolors=colors[0:event_idx],
+                    s=np.ones(xs.shape)*event_size, marker=marker, linewidths=0, alpha=1.0 if show_events else 0)
+
+            ax.plot_surface(y, img_ts, x, rstride=stride, cstride=stride, facecolors=img, alpha=1)
+
+            ax.scatter(xs[event_idx:-1], ts[event_idx:-1], ys[event_idx:-1], zdir='z',
+                    c=colors[event_idx:-1], facecolors=colors[event_idx:-1],
+                    s=np.ones(xs.shape)*event_size, marker=marker, linewidths=0, alpha=1.0 if show_events else 0)
+
+    elif num_compress > 0:
+        # Plot events
+        ax.scatter(xs[::skip], ts[::skip], ys[::skip], zdir='z', c=colors[::skip], facecolors=colors[::skip],
+                s=np.ones(xs.shape)*event_size, marker=marker, linewidths=0, alpha=1.0 if show_events else 0)
+        num_compress = min(num_compress, len(xs))
+        if not compress_front:
+            ax.scatter(xs[0:num_compress], np.ones(num_compress)*ts[0], ys[0:num_compress],
+                    marker=marker, zdir='z', c='w' if invert else 'k', s=np.ones(num_compress)*event_size)
+        else:
+            ax.scatter(xs[-num_compress-1:-1], np.ones(num_compress)*ts[-1], ys[-num_compress-1:-1],
+                    marker=marker, zdir='z', c='w' if invert else 'k', s=np.ones(num_compress)*event_size)
+    else:
+        # Plot events
+        ax.scatter(xs, ts, ys,zdir='z', c=colors, facecolors=colors, s=np.ones(xs.shape)*event_size, marker=marker, linewidths=0, alpha=1.0 if show_events else 0)
+
+    ax.view_init(elev=elev, azim=azim)
+    ax.grid(False)
+    # Hide panes
+    ax.xaxis.pane.fill = False
+    ax.yaxis.pane.fill = False
+    ax.zaxis.pane.fill = False
+    if not show_axes:
+        # Hide spines
+        ax.w_xaxis.line.set_color((1.0, 1.0, 1.0, 0.0))
+        ax.w_yaxis.line.set_color((1.0, 1.0, 1.0, 0.0))
+        ax.w_zaxis.line.set_color((1.0, 1.0, 1.0, 0.0))
+        ax.set_frame_on(False)
+    # Hide xy axes
+    ax.set_xticks([])
+    ax.set_yticks([])
+    ax.set_zticks([])
+    # Flush axes
+    ax.set_xlim3d(0, img_size[1])
+    ax.set_ylim3d(ts[0], ts[-1])
+    ax.set_zlim3d(0,img_size[0])
+
+    if show_plot:
+        plt.show()
+    if save_path is not None:
+        ensure_dir(save_path)
+        plt.savefig(save_path, transparent=True, dpi=600, bbox_inches = 'tight')
+    plt.close()
+
+def plot_between_frames(xs, ys, ts, ps, frames, frame_event_idx, args, plttype='voxel'):
+    """
+    Plot events between frames for an entire sequence to form a video
+    @param xs x component of events
+    @param ys y component of events
+    @param ts t component of events
+    @param ps p component of events
+    @param frames List of the frames
+    @param frame_event_idx The event index for each frame
+    @param args Arguments for the rendering function 'plot_events'
+    @param plttype Whether to plot 'voxel' or 'events'
+    @return None
+    """
+    args.crop = None if args.crop is None else parse_crop(args.crop)
+    prev_idx = 0
+    for i in range(0, len(frames), args.skip_frames):
+        if args.hide_skipped:
+            frame = [frames[i]]
+            frame_indices = frame_event_idx[i][np.newaxis, ...]
+        else:
+            frame = frames[i:i+args.skip_frames]
+            frame_indices = frame_event_idx[i:i+args.skip_frames]
+        print("Processing frame {}".format(i))
+        s, e = frame_indices[0,1], frame_indices[-1,0]
+        img_ts = []
+        for f_idx in frame_indices:
+            img_ts.append(ts[f_idx[1]])
+        fname = os.path.join(args.output_path, "events_{:09d}.png".format(i))
+        if plttype == 'voxel':
+            plot_voxel_grid(xs[s:e], ys[s:e], ts[s:e], ps[s:e], bins=args.num_bins, crop=args.crop,
+                    frames=frame, frame_ts=img_ts, elev=args.elev, azim=args.azim)
+        elif plttype == 'events':
+            plot_events(xs[s:e], ys[s:e], ts[s:e], ps[s:e], save_path=fname,
+                    num_show=args.num_show, event_size=args.event_size, imgs=frame,
+                    img_ts=img_ts, show_events=not args.hide_events, azim=args.azim,
+                    elev=args.elev, show_frames=not args.hide_frames, crop=args.crop,
+                    compress_front=args.compress_front, invert=args.invert,
+                    num_compress=args.num_compress, show_plot=args.show_plot, stride=args.stride)
+

others/event_utils/lib/visualization/draw_event_stream_mayavi.py

+from mayavi import mlab
+from mayavi.api import Engine
+import numpy as np
+import numpy.lib.recfunctions as nlr
+import cv2 as cv
+from skimage.measure import block_reduce
+import os
+#import matplotlib.pyplot as plt
+#from mpl_toolkits.mplot3d import Axes3D
+
+from ..representations.image import events_to_image
+from ..representations.voxel_grid import events_to_voxel
+from ..util.event_util import clip_events_to_bounds
+from ..visualization.visualization_utils import *
+from tqdm import tqdm
+
+def plot_events_sliding(xs, ys, ts, ps, args, dt=None, sdt=None, frames=None, frame_ts=None, padding=True):
+
+    skip = max(len(xs)//args.num_show, 1)
+    xs, ys, ts, ps = xs[::skip], ys[::skip], ts[::skip], ps[::skip]
+    t0 = ts[0]
+    sx,sy, st, sp = [], [], [], []
+    if padding:
+        for i in np.arange(ts[0]-dt, ts[0], sdt):
+            sx.append(0)
+            sy.append(0)
+            st.append(i)
+            sp.append(0)
+        print(len(sx))
+        print(st)
+        print(ts)
+        xs = np.concatenate((np.array(sx), xs))
+        ys = np.concatenate((np.array(sy), ys))
+        ts = np.concatenate((np.array(st), ts))
+        ps = np.concatenate((np.array(sp), ps))
+        print(ts)
+
+        ts += -st[0]
+        frame_ts += -st[0]
+        t0 += -st[0]
+        print(ts)
+
+    f = mlab.figure(bgcolor=(1,1,1), size=(1080, 720))
+    engine = mlab.get_engine()
+    scene = engine.scenes[0]
+    scene.scene.camera.position = [373.1207907160101, 5353.96218497846, 7350.065665045519]
+    scene.scene.camera.focal_point = [228.0033999234376, 37.75424682790012, 3421.439332472788]
+    scene.scene.camera.view_angle = 30.0
+    scene.scene.camera.view_up = [0.9997493712140433, -0.02027499237784438, -0.009493125997461629]
+    scene.scene.camera.clipping_range = [2400.251302762254, 11907.415293888362]
+    scene.scene.camera.compute_view_plane_normal()
+
+    print("ts from {} to {}, imgs from {} to {}".format(ts[0], ts[-1], frame_ts[0], frame_ts[-1]))
+    frame_ts = np.array([t0]+list(frame_ts[0:-1]))
+    if dt is None:
+        dt = (ts[-1]-ts[0])/10
+        sdt = dt/10
+        print("Using dt={}, sdt={}".format(dt, sdt))
+    if frames is not None:
+        sensor_size = frames[0].shape
+    else:
+        sensor_size = [max(ys), max(xs)]
+
+    if len(frame_ts.shape) == 2:
+        frame_ts = frame_ts[:,1]
+    for i, t0 in enumerate(tqdm(np.arange(ts[0], ts[-1]-dt, sdt))):
+        te = t0+dt
+        eidx0 = np.searchsorted(ts, t0)
+        eidx1 = np.searchsorted(ts, te)
+        fidx0 = np.searchsorted(frame_ts, t0)
+        fidx1 = np.searchsorted(frame_ts, te)
+        #print("{}:{} = {}".format(frame_ts[fidx0], ts[eidx0], fidx0))
+
+        wxs, wys, wts, wps = xs[eidx0:eidx1], ys[eidx0:eidx1], ts[eidx0:eidx1], ps[eidx0:eidx1],
+        if fidx0 == fidx1:
+            wframes=[]
+            wframe_ts=[]
+        else:
+            wframes = frames[fidx0:fidx1]
+            wframe_ts = frame_ts[fidx0:fidx1]
+
+        save_path = os.path.join(args.output_path, "frame_{:010d}.jpg".format(i))
+        plot_events(wxs, wys, wts, wps, save_path=save_path, num_show=-1, event_size=args.event_size,
+                imgs=wframes, img_ts=wframe_ts, show_events=not args.hide_events, azim=args.azim,
+                elev=args.elev, show_frames=not args.hide_frames, crop=args.crop, compress_front=args.compress_front,
+                invert=args.invert, num_compress=args.num_compress, show_plot=args.show_plot, img_size=sensor_size,
+                show_axes=args.show_axes, ts_scale=args.ts_scale)
+
+        if save_path is not None:
+            ensure_dir(save_path)
+            #mlab.savefig(save_path, figure=f, magnification=10)
+            #GUI().process_events()
+            #img = mlab.screenshot(figure=f, mode='rgba', antialiased=True)
+            #print(img.shape)
+            mlab.savefig(save_path, figure=f, magnification=8)
+
+        mlab.clf()
+
+def plot_voxel_grid(xs, ys, ts, ps, bins=5, frames=[], frame_ts=[],
+        sensor_size=None, crop=None, elev=0, azim=45, show_axes=False):
+    if sensor_size is None:
+        sensor_size = [np.max(ys)+1, np.max(xs)+1] if len(frames)==0 else frames[0].shape
+    if crop is not None:
+        xs, ys, ts, ps = clip_events_to_bounds(xs, ys, ts, ps, crop)
+        sensor_size = crop_to_size(crop)
+        xs, ys = xs-crop[2], ys-crop[0]
+    num = 10000
+    xs, ys, ts, ps = xs[0:num], ys[0:num], ts[0:num], ps[0:num]
+    if len(xs) == 0:
+        return
+    voxels = events_to_voxel(xs, ys, ts, ps, bins, sensor_size=sensor_size)
+    voxels = block_reduce(voxels, block_size=(1,10,10), func=np.mean, cval=0)
+    dimdiff = voxels.shape[1]-voxels.shape[0]
+    filler = np.zeros((dimdiff, *voxels.shape[1:]))
+    voxels = np.concatenate((filler, voxels), axis=0)
+    voxels = voxels.transpose(0,2,1)
+
+    pltvoxels = voxels != 0
+    pvp, nvp = voxels > 0, voxels < 0
+    pvox, nvox = voxels*np.where(voxels > 0, 1, 0), voxels*np.where(voxels < 0, 1, 0)
+    pvox, nvox = (pvox/np.max(pvox))*0.5+0.5, (np.abs(nvox)/np.max(np.abs(nvox)))*0.5+0.5
+    zeros = np.zeros_like(voxels)
+
+    colors = np.empty(voxels.shape, dtype=object)
+
+    redvals = np.stack((pvox, zeros, pvox-0.5), axis=3)
+    redvals = nlr.unstructured_to_structured(redvals).astype('O')
+
+    bluvals = np.stack((nvox-0.5, zeros, nvox), axis=3)
+    bluvals = nlr.unstructured_to_structured(bluvals).astype('O')
+
+    colors[pvp] = redvals[pvp]
+    colors[nvp] = bluvals[nvp]
+
+    fig = plt.figure()
+    ax = fig.gca(projection='3d')
+    ax.voxels(pltvoxels, facecolors=colors, edgecolor='k')
+    ax.view_init(elev=elev, azim=azim)
+
+    ax.grid(False)
+    # Hide panes
+    ax.xaxis.pane.fill = False
+    ax.yaxis.pane.fill = False
+    ax.zaxis.pane.fill = False
+    if not show_axes:
+        # Hide spines
+        ax.w_xaxis.line.set_color((1.0, 1.0, 1.0, 0.0))
+        ax.w_yaxis.line.set_color((1.0, 1.0, 1.0, 0.0))
+        ax.w_zaxis.line.set_color((1.0, 1.0, 1.0, 0.0))
+        ax.set_frame_on(False)
+    # Hide xy axes
+    ax.set_xticks([])
+    ax.set_yticks([])
+    ax.set_zticks([])
+
+    ax.xaxis.set_visible(False)
+    ax.axes.get_yaxis().set_visible(False)
+
+    plt.show()
+
+def plot_events(xs, ys, ts, ps, save_path=None, num_compress='auto', num_show=1000,
+        event_size=2, elev=0, azim=45, imgs=[], img_ts=[], show_events=True,
+        show_frames=True, show_plot=False, crop=None, compress_front=False,
+        marker='.', stride = 1, invert=False, img_size=None, show_axes=False,
+        ts_scale = 100000):
+    """
+    Given events, plot these in a spatiotemporal volume.
+    :param: xs x coords of events
+    :param: ys y coords of events
+    :param: ts t coords of events
+    :param: ps p coords of events
+    :param: save_path if set, will save plot to here
+    :param: num_compress will take this number of events from the end
+        and create an event image from these. This event image will
+        be displayed at the end of the spatiotemporal volume
+    :param: num_show sets the number of events to plot. If set to -1
+        will plot all of the events (can be potentially expensive)
+    :param: event_size sets the size of the plotted events
+    :param: elev sets the elevation of the plot
+    :param: azim sets the azimuth of the plot
+    :param: imgs a list of images to draw into the spatiotemporal volume
+    :param: img_ts a list of the position on the temporal axis where each
+        image from 'imgs' is to be placed (the timestamp of the images, usually)
+    :param: show_events if False, will not plot the events (only images)
+    :param: crop a list of length 4 that sets the crop of the plot (must
+        be in the format [top_left_y, top_left_x, height, width]
+    """
+    print("plot all")
+    #Crop events
+    if img_size is None:
+        img_size = [max(ys), max(ps)] if len(imgs)==0 else imgs[0].shape[0:2]
+    crop = [0, img_size[0], 0, img_size[1]] if crop is None else crop
+    xs, ys, ts, ps = clip_events_to_bounds(xs, ys, ts, ps, crop, set_zero=False)
+    xs, ys = xs-crop[2], ys-crop[0]
+
+    #Defaults and range checks
+    num_show = len(xs) if num_show == -1 else num_show
+    skip = max(len(xs)//num_show, 1)
+    print("Has {} events, show only {}, skip = {}".format(len(xs), num_show, skip))
+    num_compress = len(xs) if num_compress == -1 else num_compress
+    num_compress = min(img_size[0]*img_size[1]*0.5, len(xs)) if num_compress=='auto' else num_compress
+    xs, ys, ts, ps = xs[::skip], ys[::skip], ts[::skip], ps[::skip]
+
+    t0 = ts[0]
+    ts = ts-t0
+
+    #mlab.options.offscreen = True
+
+    #Plot images
+    if len(imgs)>0 and show_frames:
+        for imgidx, (img, img_t) in enumerate(zip(imgs, img_ts)):
+            img = img[crop[0]:crop[1], crop[2]:crop[3]]
+
+            mlab.imshow(img, colormap='gray', extent=[0, img.shape[0], 0, img.shape[1], (img_t-t0)*ts_scale, (img_t-t0)*ts_scale+0.01], opacity=1.0, transparent=False)
+
+    colors = [0 if p>0 else 240 for p in ps]
+    ones = np.array([0 if p==0 else 1 for p in ps])
+    p3d = mlab.quiver3d(ys, xs, ts*ts_scale, ones, ones, ones, scalars=colors, mode='sphere', scale_factor=event_size)
+    p3d.glyph.color_mode = 'color_by_scalar'
+    p3d.module_manager.scalar_lut_manager.lut.table = colors
+    #mlab.draw()
+
+    #mlab.view(84.5, 54, 5400, np.array([ 187,  175, 2276]), roll=95)
+
+    if show_plot:
+        mlab.show()
+    #if save_path is not None:
+    #    ensure_dir(save_path)
+    #    print("Saving to {}".format(save_path))
+    #    imgmap = mlab.screenshot(mode='rgba', antialiased=True)
+    #    print(imgmap.shape)
+    #    cv.imwrite(save_path, imgmap)
+
+def plot_between_frames(xs, ys, ts, ps, frames, frame_event_idx, args, plttype='voxel'):
+    args.crop = None if args.crop is None else parse_crop(args.crop)
+    prev_idx = 0
+    for i in range(0, len(frames), args.skip_frames):
+        if i != 3:
+            continue
+        if args.hide_skipped:
+            frame = [frames[i]]
+            frame_indices = frame_event_idx[i][np.newaxis, ...]
+        else:
+            frame = frames[i:i+args.skip_frames]
+            frame_indices = frame_event_idx[i:i+args.skip_frames]
+        print("Processing frame {}".format(i))
+        s, e = frame_indices[0,1], frame_indices[-1,0]
+        img_ts = []
+        for f_idx in frame_indices:
+            img_ts.append(ts[f_idx[1]])
+        fname = os.path.join(args.output_path, "events_{:09d}.png".format(i))
+        if plttype == 'voxel':
+            plot_voxel_grid(xs[s:e], ys[s:e], ts[s:e], ps[s:e], bins=args.num_bins, crop=args.crop,
+                    frames=frame, frame_ts=img_ts, elev=args.elev, azim=args.azim)
+        elif plttype == 'events':
+            print("plot events")
+            plot_events(xs[s:e], ys[s:e], ts[s:e], ps[s:e], save_path=fname,
+                    num_show=args.num_show, event_size=args.event_size, imgs=frame,
+                    img_ts=img_ts, show_events=not args.hide_events, azim=args.azim,
+                    elev=args.elev, show_frames=not args.hide_frames, crop=args.crop,
+                    compress_front=args.compress_front, invert=args.invert,
+                    num_compress=args.num_compress, show_plot=args.show_plot, stride=args.stride)

others/event_utils/lib/visualization/draw_flow.py

+import numpy as np
+import torch
+import cv2 as cv
+import os
+from tqdm import tqdm
+import matplotlib.pyplot as plt
+from mpl_toolkits.mplot3d import Axes3D
+
+from ..util.event_util import clip_events_to_bounds
+from ..util.util import flow2bgr_np
+from ..transforms.optic_flow import warp_events_flow_torch
+from ..representations.image import events_to_image_torch
+from .visualization_utils import *
+
+def motion_compensate(xs, ys, ts, ps, flow, fname="/tmp/img.png", crop=None):
+    xs, ys, ts, ps, flow = torch.from_numpy(xs).type(torch.float32), torch.from_numpy(ys).type(torch.float32),\
+        torch.from_numpy(ts).type(torch.float32), torch.from_numpy(ps).type(torch.float32), torch.from_numpy(flow).type(torch.float32)
+    xw, yw = warp_events_flow_torch(xs, ys, ts, ps, flow)
+    img_size = list(flow.shape)
+    img_size.remove(2)
+    img = events_to_image_torch(xw, yw, ps, sensor_size=img_size, interpolation='bilinear')
+    img = np.flip(np.flip(img.numpy(), axis=0), axis=1)
+    img = cv.normalize(img, None, 0, 255, cv.NORM_MINMAX)
+    if crop is not None:
+        img = img[crop[0]:crop[1], crop[2]: crop[3]]
+    cv.imwrite(fname, img)
+
+def plot_flow_and_events(xs, ys, ts, ps, flow, save_path=None,
+        num_show=1000, event_size=2, elev=0, azim=45, show_events=True,
+        show_frames=True, show_plot=False, crop=None,
+        marker='.', stride = 20, img_size=None, show_axes=False,
+        invert=False):
+
+    print(event_size)
+    #Crop events
+    if img_size is None:
+        img_size = [max(ys), max(xs)] if len(flow)==0 else flow[0].shape[1:3]
+    crop = [0, img_size[0], 0, img_size[1]] if crop is None else crop
+    xs, ys = img_size[1]-xs, img_size[0]-ys
+    xs, ys, ts, ps = clip_events_to_bounds(xs, ys, ts, ps, crop, set_zero=False)
+    xs -= crop[2]
+    ys -= crop[0]
+    img_size = [crop[1]-crop[0], crop[3]-crop[2]]
+    xs, ys = img_size[1]-xs, img_size[0]-ys
+    #flow[0] = flow[0][:, crop[0]:crop[1], crop[2]:crop[3]]
+    flow = flow[0][:, crop[0]:crop[1], crop[2]:crop[3]]
+    flow = np.flip(np.flip(flow, axis=1), axis=2)
+
+    #Defaults and range checks
+    num_show = len(xs) if num_show == -1 else num_show
+    skip = max(len(xs)//num_show, 1)
+    xs, ys, ts, ps = xs[::skip], ys[::skip], ts[::skip], ps[::skip]
+
+    #Prepare the plot, set colors
+    fig = plt.figure()
+    ax = fig.add_subplot(111, projection='3d', proj_type = 'ortho')
+    colors = ['r' if p>0 else ('#00DAFF' if invert else 'b') for p in ps]
+
+    # Plot quivers
+    f_reshape = flow.transpose(1,2,0)
+    print(f_reshape.shape)
+    t_w = ts[-1]-ts[0]
+    coords, flow_vals, magnitudes = [], [], []
+    s = 20
+    offset = 0
+    thresh = 0
+    print(img_size)
+    for x in np.linspace(offset, img_size[1]-1-offset, s):
+        for y in np.linspace(offset, img_size[0]-1-offset, s):
+            ix, iy = int(x), int(y)
+            flow_v = np.array([f_reshape[iy,ix,0]*t_w, f_reshape[iy,ix,1]*t_w, t_w])
+            mag = np.linalg.norm(flow_v)
+            if mag >= thresh:
+                flow_vals.append(flow_v)
+                magnitudes.append(mag)
+                coords.append([x,y])
+    magnitudes = np.array(magnitudes)
+    max_flow = np.percentile(magnitudes, 99)
+
+    x,y,z,u,v,w = [],[],[],[],[],[]
+    idx = 0
+    for coord, flow_vec, mag in zip(coords, flow_vals, magnitudes):
+        #q_start = [coord[0], ts[0], coord[1]]
+        rel_len = mag/max_flow
+        flow_vec = flow_vec*rel_len
+        x.append(coord[0])
+        y.append(0.065)
+        z.append(coord[1])
+        u.append(max(1, flow_vec[0]))
+        v.append(flow_vec[2])
+        w.append(max(1, flow_vec[1]))
+    ax.quiver(x,y,z,u,v,w,color='c', arrow_length_ratio=0, alpha=0.8)
+
+    img = flow2bgr_np(flow[0, :], flow[1, :])
+    img = img/255
+
+    x, y = np.ogrid[0:img.shape[0], 0:img.shape[1]]
+    ax.plot_surface(y, ts[0], x, rstride=stride, cstride=stride, facecolors=img, alpha=1)
+
+    ax.scatter(xs, ts, ys, zdir='z', c=colors, facecolors=colors,
+            s=np.ones(xs.shape)*event_size, marker=marker, linewidths=0, alpha=1.0 if show_events else 0)
+
+    ax.view_init(elev=elev, azim=azim)
+
+    ax.grid(False)
+    # Hide panes
+    ax.xaxis.pane.fill = False
+    ax.yaxis.pane.fill = False
+    ax.zaxis.pane.fill = False
+    if not show_axes:
+        # Hide spines
+        ax.w_xaxis.line.set_color((1.0, 1.0, 1.0, 0.0))
+        ax.w_yaxis.line.set_color((1.0, 1.0, 1.0, 0.0))
+        ax.w_zaxis.line.set_color((1.0, 1.0, 1.0, 0.0))
+        ax.set_frame_on(False)
+    # Hide xy axes
+    ax.set_xticks([])
+    ax.set_yticks([])
+    ax.set_zticks([])
+
+    ax.xaxis.set_visible(False)
+    ax.axes.get_yaxis().set_visible(False)
+
+    plt.show()
+
+
+
+def plot_between_frames(xs, ys, ts, ps, flows, flow_imgs, flow_ts, args, plttype='voxel'):
+    args.crop = None if args.crop is None else parse_crop(args.crop)
+
+    flow_event_idx = get_frame_indices(ts, flow_ts)
+    if len(flow_ts.shape) == 1:
+        flow_ts = frame_stamps_to_start_end(flow_ts)
+        flow_event_idx = frame_stamps_to_start_end(flow_event_idx)
+    prev_idx = 0
+    for i in range(0, len(flows), args.skip_frames):
+        if i != 12:
+            continue
+        flow = flows[i:i+args.skip_frames]
+        flow_indices = flow_event_idx[i:i+args.skip_frames]
+        s, e = flow_indices[-1,0], flow_indices[0,1]
+
+        motion_compensate(xs[s:e], ys[s:e], ts[s:e], ps[s:e], -np.flip(np.flip(flow[0], axis=1), axis=2).copy(), fname="/tmp/comp.png", crop=args.crop)
+        motion_compensate(xs[s:e], ys[s:e], ts[s:e], ps[s:e], np.zeros_like(flow[0]), fname="/tmp/zero.png", crop=args.crop)
+        e = np.searchsorted(ts, ts[s]+0.02)
+        flow_ts = []
+        for f_idx in flow_indices:
+            flow_ts.append(ts[f_idx[1]])
+        fname = os.path.join(args.output_path, "events_{:09d}.png".format(i))
+
+        print("se: {}, {}".format(s, e))
+        plot_flow_and_events(xs[s:e], ys[s:e], ts[s:e], ps[s:e], flow,
+        num_show=args.num_show, event_size=args.event_size, elev=args.elev,
+        azim=args.azim, show_events=not args.hide_events,
+        show_frames=not args.hide_frames, show_plot=args.show_plot, crop=args.crop,
+        stride=args.stride, show_axes=args.show_axes, invert=args.invert)

others/event_utils/lib/visualization/utils/draw_plane.py

+import numpy as np
+import matplotlib
+import matplotlib.pyplot as plt
+from mpl_toolkits.mplot3d import axes3d, Axes3D #<-- Note the capitalization!
+
+# z = ax + by + d
+
+x_min = 0
+x_max = 100
+y_min = 0
+y_max = 100
+
+a = 0
+b = 10
+d = 10
+
+num_points = 5000
+point_size = 10
+
+points = np.random.rand(num_points, 3)
+points[:, 0] = points[:, 0]*(x_max-x_min) + x_min
+points[:, 1] = points[:, 1]*(y_max-y_min) + y_min
+points[:, 2] = points[:, 0]*a + points[:, 1]*b + d
+
+mean = 0
+stdev = 10
+noise = np.random.normal(mean, stdev, num_points)
+points[:, 2] = points[:, 2] + noise
+
+print(points)
+new_points = points[np.where(points[:, 1] < 50)]
+print(new_points)
+
+for x in range(y_min, y_max, 1):
+
+    fig = plt.figure()
+    ax = Axes3D(fig)
+    ax.set_xlabel('x')
+    ax.set_ylabel('y')
+    ax.set_zlabel('time')
+    ax.set_ylim([0, 100])
+
+    new_points = points[np.where(points[:, 1] < x)]
+    ax.scatter(new_points[:, 0], new_points[:, 1], new_points[:, 2], s=point_size, c=(new_points[:, 2]),
+               edgecolors='none', cmap='plasma')
+    ax.scatter(points[:, 0], points[:, 1], points[:, 2], s=0, c=(points[:, 2]),
+               edgecolors='none', cmap='plasma')
+
+    point = np.array([0, 1, 0])
+    normal = np.array([0, 0, 1])
+
+    # a plane is a*x+b*y+c*z+d=0
+    # [a,b,c] is the normal. Thus, we have to calculate
+    # d and we're set
+    d = -point.dot(normal)
+
+    # create x,y
+    xx, yy = np.meshgrid(range(100), range(10))
+    yy = yy + x - 10
+
+    # calculate corresponding z
+    z = (-normal[0] * xx - normal[1] * yy - d) * 1. / normal[2]
+    # plot the surface
+    # plt3d = plt.figure().gca(projection='3d')
+    ax.plot_surface(xx, yy, z, alpha=1)
+
+    save_name = ("frame_" + str(x) + ".png")
+    fig.tight_layout()
+    fig.savefig(save_name, dpi=300, transparent=True)
+
+    # plt.show()
+    plt.close()
\ No newline at end of file

others/event_utils/lib/visualization/utils/draw_plane_simple.py

+import numpy as np
+import matplotlib
+import matplotlib.pyplot as plt
+from mpl_toolkits.mplot3d import axes3d, Axes3D #<-- Note the capitalization!
+
+fig = plt.figure()
+ax = Axes3D(fig)
+ax.set_xlabel('x')
+ax.set_ylabel('y')
+ax.set_zlabel('time')
+
+# z = ax + by + d
+x_min = 0
+x_max = 10
+y_min = 0
+y_max = 10
+
+a = 0
+b = 10
+d = 10
+
+num_points = 50
+point_size = 20
+
+points = np.random.rand(num_points, 3)
+points[:, 0] = points[:, 0]*(x_max-x_min) + x_min
+points[:, 1] = points[:, 1]*(y_max-y_min) + y_min
+points[:, 2] = points[:, 0]*a + points[:, 1]*b + d
+
+mean = 0
+stdev = 10
+noise = np.random.normal(mean, stdev, num_points)
+points[:, 2] = points[:, 2] + noise
+
+ax.scatter(points[:, 0], points[:, 1], points[:, 2], s=point_size, c=(points[:, 2]),
+               edgecolors='none', cmap='plasma')
+
+
+# create x,y
+xx, yy = np.meshgrid(range(10), range(10))
+yy = yy
+
+# calculate corresponding z
+# z = (-normal[0] * xx - normal[1] * yy - d) * 1. / normal[2]
+z = xx*a + yy*b + d
+# plot the surface
+# plt3d = plt.figure().gca(projection='3d')
+ax.plot_surface(xx, yy, z, alpha=0.2)
+
+save_name = ("plane.png")
+fig.tight_layout()
+fig.savefig(save_name, dpi=600, transparent=True)
+plt.close()
\ No newline at end of file

others/event_utils/lib/visualization/visualization_utils.py

+import numpy as np
+import os
+
+def frame_stamps_to_start_end(stamps):
+    ends = list(stamps[1:])
+    ends.append(ends[-1])
+    se_stamps = np.stack((stamps, np.array(ends)), axis=1)
+    return se_stamps
+
+def get_frame_indices(ts, frame_ts):
+    indices = [np.searchsorted(ts, fts) for fts in frame_ts]
+    return np.array(indices)
+
+def crop_to_size(crop):
+    return [crop[0]-crop[1], crop[2]-crop[3]]
+
+def parse_crop(cropstr):
+    """
+    Crop is provided as string, same as imagemagick:
+        size_x, size_y, offset_x, offset_y, eg 10x10+30+30 would cut a 10x10 square at 30,30
+    Output is the indices as would be used in a numpy array. In the example,
+    [30,40,30,40] (ie [miny, maxy, minx, maxx])
+
+    """
+    split = cropstr.split("x")
+    xsize = int(split[0])
+    split = split[1].split("+")
+    ysize = int(split[0])
+    xoff = int(split[1])
+    yoff = int(split[2])
+    crop = [yoff, yoff+ysize, xoff, xoff+xsize]
+    return crop
+
+def ensure_dir(file_path):
+    directory = os.path.dirname(file_path)
+    if not os.path.exists(directory):
+        print(f"Creating {directory}")
+        os.makedirs(directory)
+

others/event_utils/lib/visualization/visualizers.py

+import numpy as np
+import numpy.lib.recfunctions as nlr
+import cv2 as cv
+import colorsys
+from skimage.measure import block_reduce
+import os
+import matplotlib.pyplot as plt
+from mpl_toolkits.mplot3d import Axes3D
+
+from ..representations.image import events_to_image, TimestampImage
+from ..representations.voxel_grid import events_to_voxel
+from ..util.event_util import clip_events_to_bounds
+from .visualization_utils import *
+from tqdm import tqdm
+
+class Visualizer():
+
+    def __init__(self):
+        raise NotImplementedError
+
+    def plot_events(self, data, save_path, **kwargs):
+        raise NotImplementedError
+
+    @staticmethod
+    def unpackage_events(events):
+        return events[:,0].astype(int), events[:,1].astype(int), events[:,2], events[:,3]
+
+class TimeStampImageVisualizer(Visualizer):
+
+    def __init__(self, sensor_size):
+        self.ts_img = TimestampImage(sensor_size)
+        self.sensor_size = sensor_size
+
+    def plot_events(self, data, save_path, **kwargs):
+        xs, ys, ts, ps = self.unpackage_events(data['events'])
+        self.ts_img.set_init(ts[0])
+        self.ts_img.add_events(xs, ys, ts, ps)
+        timestamp_image = self.ts_img.get_image()
+        fig = plt.figure()
+        plt.imshow(timestamp_image, cmap='viridis')
+        ensure_dir(save_path)
+        plt.savefig(save_path, transparent=True, dpi=600, bbox_inches = 'tight')
+        #plt.show()
+
+class EventImageVisualizer(Visualizer):
+
+    def __init__(self, sensor_size):
+        self.sensor_size = sensor_size
+
+    def plot_events(self, data, save_path, **kwargs):
+        xs, ys, ts, ps = self.unpackage_events(data['events'])
+        img = events_to_image(xs.astype(int), ys.astype(int), ps, self.sensor_size, interpolation=None, padding=False)
+        mn, mx = np.min(img), np.max(img)
+        img = (img-mn)/(mx-mn)
+
+        fig = plt.figure()
+        plt.imshow(img, cmap='gray')
+        ensure_dir(save_path)
+        plt.savefig(save_path, transparent=True, dpi=600, bbox_inches = 'tight')
+        #plt.show()
+
+
+class EventsVisualizer(Visualizer):
+
+    def __init__(self, sensor_size):
+        self.sensor_size = sensor_size
+
+    def plot_events(self, data, save_path,
+            num_compress='auto', num_show=1000,
+            event_size=2, elev=0, azim=45, show_events=True,
+            show_frames=True, show_plot=False, crop=None, compress_front=False,
+            marker='.', stride = 1, invert=False, show_axes=False, flip_x=False):
+        """
+        Given events, plot these in a spatiotemporal volume.
+        :param: xs x coords of events
+        :param: ys y coords of events
+        :param: ts t coords of events
+        :param: ps p coords of events
+        :param: save_path if set, will save plot to here
+        :param: num_compress will take this number of events from the end
+            and create an event image from these. This event image will
+            be displayed at the end of the spatiotemporal volume
+        :param: num_show sets the number of events to plot. If set to -1
+            will plot all of the events (can be potentially expensive)
+        :param: event_size sets the size of the plotted events
+        :param: elev sets the elevation of the plot
+        :param: azim sets the azimuth of the plot
+        :param: imgs a list of images to draw into the spatiotemporal volume
+        :param: img_ts a list of the position on the temporal axis where each
+            image from 'imgs' is to be placed (the timestamp of the images, usually)
+        :param: show_events if False, will not plot the events (only images)
+        :param: crop a list of length 4 that sets the crop of the plot (must
+            be in the format [top_left_y, top_left_x, height, width]
+        """
+        xs, ys, ts, ps = self.unpackage_events(data['events'])
+        imgs, img_ts = data['frame'], data['frame_ts']
+        if not (isinstance(imgs, list) or isinstance(imgs, tuple)):
+            imgs, img_ts = [imgs], [img_ts]
+
+        ys = self.sensor_size[0]-ys
+        xs = self.sensor_size[1]-xs if flip_x else xs
+        #Crop events
+        img_size = self.sensor_size
+        if img_size is None:
+            img_size = [max(ys), max(ps)] if len(imgs)==0 else imgs[0].shape[0:2]
+        crop = [0, img_size[0], 0, img_size[1]] if crop is None else crop
+        xs, ys, ts, ps = clip_events_to_bounds(xs, ys, ts, ps, crop, set_zero=False)
+        xs, ys = xs-crop[2], ys-crop[0]
+
+        if len(xs) < 2:
+            xs = np.array([0,0])
+            ys = np.array([0,0])
+            if img_ts is None:
+                ts = np.array([0,0])
+            else:
+                ts = np.array([img_ts[0], img_ts[0]+0.000001])
+            ps = np.array([0.,0.])
+
+        #Defaults and range checks
+        num_show = len(xs) if num_show == -1 else num_show
+        skip = max(len(xs)//num_show, 1)
+        num_compress = len(xs) if num_compress == 'all' else num_compress
+        num_compress = min(int(img_size[0]*img_size[1]*0.5), len(xs)) if num_compress=='auto' else 0
+        xs, ys, ts, ps = xs[::skip], ys[::skip], ts[::skip], ps[::skip]
+
+        #Prepare the plot, set colors
+        fig = plt.figure()
+        ax = fig.add_subplot(111, projection='3d', proj_type = 'ortho')
+        colors = ['r' if p>0 else ('#00DAFF' if invert else 'b') for p in ps]
+
+        #Plot images
+        if len(imgs)>0 and show_frames:
+            for imgidx, (img, img_ts) in enumerate(zip(imgs, img_ts)):
+                img = img[crop[0]:crop[1], crop[2]:crop[3]].astype(float)
+                img = np.flip(img, axis=0)
+                img = np.flip(img, axis=1) if flip_x else img
+                if len(img.shape)==2:
+                    img = np.stack((img, img, img), axis=2)
+                if num_compress > 0:
+                    events_img = events_to_image(xs[0:num_compress], ys[0:num_compress],
+                            np.ones(min(num_compress, len(xs))), sensor_size=img.shape[0:2])
+                    events_img[events_img>0] = 1
+                    img[:,:,1] += events_img[:,:]
+                    img = np.clip(img, 0, 1)
+                x, y = np.ogrid[0:img.shape[0], 0:img.shape[1]]
+                event_idx = np.searchsorted(ts, img_ts)
+
+                ax.scatter(xs[0:event_idx], ts[0:event_idx], ys[0:event_idx], zdir='z',
+                        c=colors[0:event_idx], facecolors=colors[0:event_idx],
+                        s=event_size, marker=marker, linewidths=0, alpha=1.0 if show_events else 0)
+
+                img /= 255.0
+                #img = cv.normalize(img, None, 0, 1, cv.NORM_MINMAX)
+                ax.plot_surface(y, img_ts, x, rstride=stride, cstride=stride, facecolors=img, alpha=1)
+
+                ax.scatter(xs[event_idx:-1], ts[event_idx:-1], ys[event_idx:-1], zdir='z',
+                        c=colors[event_idx:-1], facecolors=colors[event_idx:-1],
+                        s=event_size, marker=marker, linewidths=0, alpha=1.0 if show_events else 0)
+    
+        elif num_compress > 0:
+            # Plot events
+            ax.scatter(xs[::skip], ts[::skip], ys[::skip], zdir='z', c=colors[::skip], facecolors=colors[::skip],
+                    s=np.ones(xs.shape)*event_size, marker=marker, linewidths=0, alpha=1.0 if show_events else 0)
+            num_compress = min(num_compress, len(xs))
+            if not compress_front:
+                ax.scatter(xs[0:num_compress], np.ones(num_compress)*ts[0], ys[0:num_compress],
+                        marker=marker, zdir='z', c='w' if invert else 'k', s=np.ones(num_compress)*event_size)
+            else:
+                ax.scatter(xs[-num_compress-1:-1], np.ones(num_compress)*ts[-1], ys[-num_compress-1:-1],
+                        marker=marker, zdir='z', c='w' if invert else 'k', s=np.ones(num_compress)*event_size)
+        else:
+            # Plot events
+            ax.scatter(xs, ts, ys,zdir='z', c=colors, facecolors=colors, s=np.ones(xs.shape)*event_size, marker=marker, linewidths=0, alpha=1.0 if show_events else 0)
+    
+        ax.view_init(elev=elev, azim=azim)
+        ax.grid(False)
+        # Hide panes
+        ax.xaxis.pane.fill = False
+        ax.yaxis.pane.fill = False
+        ax.zaxis.pane.fill = False
+        if not show_axes:
+            # Hide spines
+            ax.w_xaxis.line.set_color((1.0, 1.0, 1.0, 0.0))
+            ax.w_yaxis.line.set_color((1.0, 1.0, 1.0, 0.0))
+            ax.w_zaxis.line.set_color((1.0, 1.0, 1.0, 0.0))
+            ax.set_frame_on(False)
+        # Hide xy axes
+        ax.set_xticks([])
+        ax.set_yticks([])
+        ax.set_zticks([])
+        # Flush axes
+        ax.set_xlim3d(0, img_size[1])
+        ax.set_ylim3d(ts[0], ts[-1])
+        ax.set_zlim3d(0,img_size[0])
+        #ax.xaxis.set_visible(False)
+        #ax.axes.get_yaxis().set_visible(False)
+
+        if show_plot:
+            plt.show()
+        if save_path is not None:
+            ensure_dir(save_path)
+            print("Saving to {}".format(save_path))
+            plt.savefig(save_path, transparent=True, dpi=600, bbox_inches = 'tight')
+        plt.close()
+
+class VoxelVisualizer(Visualizer):
+
+    def __init__(self, sensor_size):
+        self.sensor_size = sensor_size
+
+    @staticmethod
+    def increase_brightness(rgb, increase=0.5):
+        rgb = (rgb*255).astype('uint8')
+        channels = rgb.shape[1]
+        hsv = (np.stack([cv.cvtColor(rgb[:,x,:,:], cv.COLOR_RGB2HSV) for x in range(channels)])).astype(float)
+        hsv[:,:,:,2] = np.clip(hsv[:,:,:,2] + increase*255, 0, 255)
+        hsv = hsv.astype('uint8')
+        rgb_new = np.stack([cv.cvtColor(hsv[x,:,:,:], cv.COLOR_HSV2RGB) for x in range(channels)])
+        rgb_new = (rgb_new.transpose(1,0,2,3)).astype(float)
+        return rgb_new/255.0
+
+    def plot_events(self, data, save_path, bins=5, crop=None, elev=0, azim=45, show_axes=False,
+            show_plot=False, flip_x=False, size_reduction=10):
+
+        xs, ys, ts, ps = self.unpackage_events(data['events'])
+        if len(xs) < 2:
+            return
+        ys = self.sensor_size[0]-ys
+        xs = self.sensor_size[1]-xs if flip_x else xs
+
+        frames, frame_ts = data['frame'], data['frame_ts']
+        if not isinstance(frames, list):
+            frames, frame_ts = [frames], [frame_ts]
+
+        if self.sensor_size is None:
+            self.sensor_size = [np.max(ys)+1, np.max(xs)+1] if len(frames)==0 else frames[0].shape
+        if crop is not None:
+            xs, ys, ts, ps = clip_events_to_bounds(xs, ys, ts, ps, crop)
+            self.sensor_size = crop_to_size(crop)
+            xs, ys = xs-crop[2], ys-crop[0]
+        num = 10000
+        xs, ys, ts, ps = xs[0:num], ys[0:num], ts[0:num], ps[0:num]
+        if len(xs) == 0:
+            return
+        voxels = events_to_voxel(xs, ys, ts, ps, bins, sensor_size=self.sensor_size)
+        voxels = block_reduce(voxels, block_size=(1,size_reduction,size_reduction), func=np.mean, cval=0)
+        dimdiff = voxels.shape[1]-voxels.shape[0]
+        filler = np.zeros((dimdiff, *voxels.shape[1:]))
+        voxels = np.concatenate((filler, voxels), axis=0)
+        voxels = voxels.transpose(0,2,1)
+
+        pltvoxels = voxels != 0
+        pvp, nvp = voxels > 0, voxels < 0
+        rng = 0.2
+        min_r, min_b, max_g = 80/255.0, 80/255.0, 0/255.0
+
+        vox_cols = voxels/(max(np.abs(np.max(voxels)), np.abs(np.min(voxels))))
+        pvox, nvox = vox_cols*np.where(vox_cols > 0, 1, 0), np.abs(vox_cols)*np.where(vox_cols < 0, 1, 0)
+        pvox, nvox = pvox*(1-min_r)+min_r, nvox*(1-min_b)+min_b
+        zeros = np.zeros_like(voxels)
+
+        colors = np.empty(voxels.shape, dtype=object)
+
+        increase = 0.5
+        redvals = np.stack((pvox, (1.0-pvox)*max_g, pvox-min_r), axis=3)
+        redvals = self.increase_brightness(redvals, increase=increase)
+        redvals = nlr.unstructured_to_structured(redvals).astype('O')
+
+        bluvals = np.stack((nvox-min_b, (1.0-nvox)*max_g, nvox), axis=3)
+        bluvals = self.increase_brightness(bluvals, increase=increase)
+        bluvals = nlr.unstructured_to_structured(bluvals).astype('O')
+
+        colors[pvp] = redvals[pvp]
+        colors[nvp] = bluvals[nvp]
+
+        fig = plt.figure()
+        ax = fig.gca(projection='3d')
+        ax.voxels(pltvoxels, facecolors=colors)
+        ax.view_init(elev=elev, azim=azim)
+
+        ax.grid(False)
+        # Hide panes
+        ax.xaxis.pane.fill = False
+        ax.yaxis.pane.fill = False
+        ax.zaxis.pane.fill = False
+        if not show_axes:
+            # Hide spines
+            ax.w_xaxis.line.set_color((1.0, 1.0, 1.0, 0.0))
+            ax.w_yaxis.line.set_color((1.0, 1.0, 1.0, 0.0))
+            ax.w_zaxis.line.set_color((1.0, 1.0, 1.0, 0.0))
+            ax.set_frame_on(False)
+        # Hide xy axes
+        ax.set_xticks([])
+        ax.set_yticks([])
+        ax.set_zticks([])
+
+        ax.xaxis.set_visible(False)
+        ax.axes.get_yaxis().set_visible(False)
+
+        if show_plot:
+            plt.show()
+        if save_path is not None:
+            ensure_dir(save_path)
+            print("Saving to {}".format(save_path))
+            plt.savefig(save_path, transparent=True, dpi=600, bbox_inches = 'tight')
+        plt.close()

others/event_utils/visualize.py

+import argparse
+import os
+from tqdm import tqdm
+import numpy as np
+from lib.data_formats.read_events import read_memmap_events, read_h5_events_dict
+from lib.data_loaders import MemMapDataset, DynamicH5Dataset, NpyDataset
+from lib.visualization.visualizers import TimeStampImageVisualizer, EventImageVisualizer, \
+        EventsVisualizer, VoxelVisualizer
+
+if __name__ == "__main__":
+    """
+    Quick demo
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument("path", help="memmap events path")
+    parser.add_argument("--output_path", type=str, default="/tmp/visualization", help="Where to save image outputs")
+    parser.add_argument("--filetype", type=str, default="png", help="Which filetype to save as", choices=["png", "jpg", "pdf"])
+
+    parser.add_argument('--plot_method', default='between_frames', type=str,
+                        help='which method should be used to visualize',
+                        choices=['between_frames', 'k_events', 't_seconds', 'fixed_frames'])
+    parser.add_argument('--w_width', type=float, default=0.01,
+                        help='new plot is formed every t seconds/k events (required if voxel_method is t_seconds)')
+    parser.add_argument('--sw_width', type=float,
+                        help='sliding_window size in seconds/events (required if voxel_method is t_seconds)')
+    parser.add_argument('--num_frames', type=int, default=100, help='if fixed_frames chosen as voxel method, sets the number of frames')
+
+    parser.add_argument('--visualization', type=str, default='events', choices=['events', 'voxels', 'event_image', 'ts_image'])
+
+    parser.add_argument("--num_bins", type=int, default=6, help="How many bins voxels should have.")
+
+    parser.add_argument('--show_plot', action='store_true', help='If true, will also display the plot in an interactive window.\
+            Useful for selecting the desired orientation.')
+
+    parser.add_argument("--num_show", type=int, default=-1, help="How many events to show per plot. If -1, show all events.")
+    parser.add_argument("--event_size", type=float, default=2, help="Marker size of the plotted events")
+    parser.add_argument("--ts_scale", type=int, default=10000, help="Scales the time axis. Only applicable for mayavi rendering.")
+    parser.add_argument("--elev", type=float, default=0, help="Elevation of plot")
+    parser.add_argument("--azim", type=float, default=45, help="Azimuth of plot")
+    parser.add_argument("--stride", type=int, default=1, help="Downsample stride for plotted images.")
+    parser.add_argument("--skip_frames", type=int, default=1, help="Amount of frames to place per plot.")
+    parser.add_argument("--start_frame", type=int, default=0, help="On which frame to start.")
+    parser.add_argument('--hide_skipped', action='store_true', help='Do not draw skipped frames into plot.')
+    parser.add_argument('--hide_events', action='store_true', help='Do not draw events')
+    parser.add_argument('--hide_frames', action='store_true', help='Do not draw frames')
+    parser.add_argument('--show_axes', action='store_true', help='Draw axes')
+    parser.add_argument('--flip_x', action='store_true', help='Flip in the x axis')
+    parser.add_argument("--num_compress", type=str, default='auto', help="How many events to draw compressed. If 'auto'\
+            will automatically determine.", choices=['auto', 'none', 'all'])
+    parser.add_argument('--compress_front', action='store_true', help='If set, will put the compressed events at the _start_\
+            of the event volume, rather than the back.')
+    parser.add_argument('--invert', action='store_true', help='If the figure is for a black background, you can invert the \
+            colors for better visibility.')
+    parser.add_argument("--crop", type=str, default=None, help="Set a crop of both images and events. Uses 'imagemagick' \
+            syntax, eg for a crop of 10x20 starting from point 30,40 use: 10x20+30+40.")
+    parser.add_argument("--renderer", type=str, default="matplotlib", help="Which renderer to use (mayavi is faster)", choices=["matplotlib", "mayavi"])
+    args = parser.parse_args()
+    if not os.path.exists(args.output_path):
+        os.makedirs(args.output_path)
+
+    if os.path.isdir(args.path):
+        loader_type = MemMapDataset
+    elif os.path.splitext(args.path)[1] == ".npy":
+        loader_type = NpyDataset
+    else:
+        loader_type = DynamicH5Dataset
+    dataloader = loader_type(args.path, voxel_method={'method':args.plot_method, 't':args.w_width,
+        'k':args.w_width, 'sliding_window_t':args.sw_width, 'sliding_window_w':args.sw_width, 'num_frames':args.num_frames},
+            return_events=True, return_voxelgrid=False, return_frame=True, return_flow=True, return_format='numpy')
+    sensor_size = dataloader.size()
+
+    if args.visualization == 'events':
+        kwargs = {'num_compress':args.num_compress, 'num_show':args.num_show, 'event_size':args.event_size,
+                'elev':args.elev, 'azim':args.azim, 'show_events':not args.hide_events,
+                'show_frames':not args.hide_frames, 'show_plot':args.show_plot, 'crop':args.crop,
+                'compress_front':args.compress_front, 'marker':'.', 'stride':args.stride,
+                'invert':args.invert, 'show_axes':args.show_axes, 'flip_x':args.flip_x}
+        visualizer = EventsVisualizer(sensor_size)
+    elif args.visualization == 'voxels':
+        kwargs = {'bins':args.num_bins, 'crop':args.crop, 'elev':args.elev, 'azim':args.azim,
+                'show_axes':args.show_axes, 'show_plot':args.show_plot, 'flip_x':args.flip_x}
+        visualizer = VoxelVisualizer(sensor_size)
+    elif args.visualization == 'event_image':
+        kwargs = {}
+        visualizer = EventImageVisualizer(sensor_size)
+    elif args.visualization == 'ts_image':
+        kwargs = {}
+        visualizer = TimeStampImageVisualizer(sensor_size)
+    else:
+        raise Exception("Unknown visualization chosen: {}".format(args.visualization))
+
+    plot_data = {'events':np.ones((0, 4)), 'frame':[], 'frame_ts':[]}
+    print("{} frames in sequence".format(len(dataloader)))
+    for i, data in enumerate(tqdm(dataloader)):
+        plot_data['events'] = np.concatenate((plot_data['events'], data['events']))
+        if args.plot_method == 'between_frames':
+            plot_data['frame'].append(data['frame'])
+            plot_data['frame_ts'].append(data['frame_ts'])
+        else:
+            plot_data['frame'] = data['frame']
+            plot_data['frame_ts'] = data['frame_ts']
+
+        output_path = os.path.join(args.output_path, "frame_{:010d}.{}".format(i, args.filetype))
+        if i%args.skip_frames == 0:
+            visualizer.plot_events(plot_data, output_path, **kwargs)
+            plot_data = {'events':np.ones((0, 4)), 'frame':[], 'frame_ts':[]}
+
+    #if args.plot_method == 'between_frames':
+    #    if args.renderer == "mayavi":
+    #        from lib.visualization.draw_event_stream_mayavi import plot_between_frames
+    #        plot_between_frames(xs, ys, ts, ps, frames, frame_idx, args, plttype='events')
+    #    elif args.renderer == "matplotlib":
+    #        from lib.visualization.draw_event_stream import plot_between_frames
+    #        plot_between_frames(xs, ys, ts, ps, frames, frame_idx, args, plttype='events')
+    #elif args.plot_method == 'k_events':
+    #    print(args.renderer)
+    #    pass
+    #elif args.plot_method == 't_seconds':
+    #    if args.renderer == "mayavi":
+    #        from lib.visualization.draw_event_stream_mayavi import plot_events_sliding
+    #        plot_events_sliding(xs, ys, ts, ps, args, dt=args.w_width, sdt=args.sw_width, frames=frames, frame_ts=frame_ts)
+    #    elif args.renderer == "matplotlib":
+    #        from lib.visualization.draw_event_stream import plot_events_sliding
+    #        plot_events_sliding(xs, ys, ts, ps, args, frames=frames, frame_ts=frame_ts)

others/event_utils/visualize_events.py

+import argparse
+import os
+import numpy as np
+from lib.data_formats.read_events import read_memmap_events, read_h5_events_dict
+
+if __name__ == "__main__":
+    """
+    Quick demo
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument("path", help="memmap events path")
+    parser.add_argument("--output_path", type=str, default="/tmp/visualization", help="Where to save image outputs")
+
+    parser.add_argument('--plot_method', default='between_frames', type=str,
+                        help='which method should be used to visualize',
+                        choices=['between_frames', 'k_events', 't_seconds'])
+    parser.add_argument('--w_width', type=float, default=0.01,
+                        help='new plot is formed every t seconds (required if voxel_method is t_seconds)')
+    parser.add_argument('--sw_width', type=float,
+                        help='sliding_window size in seconds (required if voxel_method is t_seconds)')
+
+    parser.add_argument("--num_bins", type=int, default=6, help="How many bins voxels should have.")
+
+    parser.add_argument('--show_plot', action='store_true', help='If true, will also display the plot in an interactive window.\
+            Useful for selecting the desired orientation.')
+
+    parser.add_argument("--num_show", type=int, default=-1, help="How many events to show per plot. If -1, show all events.")
+    parser.add_argument("--event_size", type=float, default=2, help="Marker size of the plotted events")
+    parser.add_argument("--ts_scale", type=int, default=10000, help="Scales the time axis. Only applicable for mayavi rendering.")
+    parser.add_argument("--elev", type=float, default=20, help="Elevation of plot")
+    parser.add_argument("--azim", type=float, default=45, help="Azimuth of plot")
+    parser.add_argument("--stride", type=int, default=1, help="Downsample stride for plotted images.")
+    parser.add_argument("--skip_frames", type=int, default=1, help="Amount of frames to place per plot.")
+    parser.add_argument("--start_frame", type=int, default=0, help="On which frame to start.")
+    parser.add_argument('--hide_skipped', action='store_true', help='Do not draw skipped frames into plot.')
+    parser.add_argument('--hide_events', action='store_true', help='Do not draw events')
+    parser.add_argument('--hide_frames', action='store_true', help='Do not draw frames')
+    parser.add_argument('--show_axes', action='store_true', help='Draw axes')
+    parser.add_argument("--num_compress", type=int, default=0, help="How many events to draw compressed. If 'auto'\
+            will automatically determine.", choices=['value', 'auto'])
+    parser.add_argument('--compress_front', action='store_true', help='If set, will put the compressed events at the _start_\
+            of the event volume, rather than the back.')
+    parser.add_argument('--invert', action='store_true', help='If the figure is for a black background, you can invert the \
+            colors for better visibility.')
+    parser.add_argument("--crop", type=str, default=None, help="Set a crop of both images and events. Uses 'imagemagick' \
+            syntax, eg for a crop of 10x20 starting from point 30,40 use: 10x20+30+40.")
+    parser.add_argument("--renderer", type=str, default="matplotlib", help="Which renderer to use (mayavi is faster)", choices=["matplotlib", "mayavi"])
+    args = parser.parse_args()
+
+    if os.path.isdir(args.path):
+        events = read_memmap_events(args.path)
+
+        ts = events['t'][:].squeeze()
+        t0 = ts[0]
+        ts = ts-t0
+        frames = (events['images'][args.start_frame+1::])/255
+        frame_idx = events['index'][args.start_frame::]
+        frame_ts = events['frame_stamps'][args.start_frame+1::]-t0
+
+        start_idx = np.searchsorted(ts, frame_ts[0])
+        print("Starting from frame {}, event {}".format(args.start_frame, start_idx))
+
+        xs = events['xy'][:,0]
+        ys = events['xy'][:,1]
+        ts = ts[:]
+        ps = events['p'][:]
+
+        print("Have {} frames".format(frames.shape))
+    else:
+        events = read_h5_events_dict(args.path)
+        xs = events['xs']
+        ys = events['ys']
+        ts = events['ts']
+        ps = events['ps']
+        t0 = ts[0]
+        ts = ts-t0
+        frames = [np.flip(np.flip(x/255., axis=0), axis=1) for x in events['frames']]
+        frame_ts = events['frame_timestamps'][1:]-t0
+        frame_end = events['frame_event_indices'][1:]
+        frame_start = np.concatenate((np.array([0]), frame_end))
+        frame_idx = np.stack((frame_end, frame_start[0:-1]), axis=1)
+        ys = frames[0].shape[0]-ys
+        xs = frames[0].shape[1]-xs
+
+    if args.plot_method == 'between_frames':
+        if args.renderer == "mayavi":
+            from lib.visualization.draw_event_stream_mayavi import plot_between_frames
+            plot_between_frames(xs, ys, ts, ps, frames, frame_idx, args, plttype='events')
+        elif args.renderer == "matplotlib":
+            from lib.visualization.draw_event_stream import plot_between_frames
+            plot_between_frames(xs, ys, ts, ps, frames, frame_idx, args, plttype='events')
+    elif args.plot_method == 'k_events':
+        print(args.renderer)
+        pass
+    elif args.plot_method == 't_seconds':
+        if args.renderer == "mayavi":
+            from lib.visualization.draw_event_stream_mayavi import plot_events_sliding
+            plot_events_sliding(xs, ys, ts, ps, args, dt=args.w_width, sdt=args.sw_width, frames=frames, frame_ts=frame_ts)
+        elif args.renderer == "matplotlib":
+            from lib.visualization.draw_event_stream import plot_events_sliding
+            plot_events_sliding(xs, ys, ts, ps, args, frames=frames, frame_ts=frame_ts)