viz.py

import os
import imageio
import glob
import torch
import numpy as np
import util
import subprocess
from config import config_parser
from trainer import BaseTrainer
import colorsys
from matplotlib import cm
import cv2


color_map = cm.get_cmap("jet")


def vis_trail(scene_dir, kpts_foreground, kpts_background, save_path):
    """
    This function calculates the median motion of the background, which is subsequently
    subtracted from the foreground motion. This subtraction process "stabilizes" the camera and
    improves the interpretability of the foreground motion trails.
    """
    img_dir = os.path.join(scene_dir, "color")
    img_files = sorted(list(glob.glob(os.path.join(img_dir, "*"))))
    images = np.array([imageio.imread(img_file) for img_file in img_files])

    kpts_foreground = kpts_foreground[:, ::1]  # can adjust kpts sampling rate here

    num_imgs, num_pts = kpts_foreground.shape[:2]

    frames = []

    for i in range(num_imgs):

        kpts = kpts_foreground - np.median(kpts_background - kpts_background[i], axis=1, keepdims=True)

        img_curr = images[i]

        for t in range(i):

            img1 = img_curr.copy()
            # changing opacity
            alpha = max(1 - 0.9 * ((i - t) / ((i + 1) * .99)), 0.1)

            for j in range(num_pts):

                color = np.array(color_map(j/max(1, float(num_pts - 1)))[:3]) * 255

                color_alpha = 1

                hsv = colorsys.rgb_to_hsv(color[0], color[1], color[2])
                color = colorsys.hsv_to_rgb(hsv[0], hsv[1]*color_alpha, hsv[2])

                pt1 = kpts[t, j]
                pt2 = kpts[t+1, j]
                p1 = (int(round(pt1[0])), int(round(pt1[1])))
                p2 = (int(round(pt2[0])), int(round(pt2[1])))

                cv2.line(img1, p1, p2, color, thickness=1, lineType=16)

            img_curr = cv2.addWeighted(img1, alpha, img_curr, 1 - alpha, 0)

        for j in range(num_pts):
            color = np.array(color_map(j/max(1, float(num_pts - 1)))[:3]) * 255
            pt1 = kpts[i, j]
            p1 = (int(round(pt1[0])), int(round(pt1[1])))
            cv2.circle(img_curr, p1, 2, color, -1, lineType=16)

        frames.append(img_curr)

    imageio.mimwrite(save_path, frames, quality=8, fps=10)


if __name__ == '__main__':
    args = config_parser()
    seq_name = os.path.basename(args.data_dir.rstrip('/'))

    trainer = BaseTrainer(args)
    num_imgs = trainer.num_imgs
    vis_dir = os.path.join(args.save_dir, '{}_{}'.format(args.expname, seq_name), 'vis')
    print('output will be saved in {}'.format(vis_dir))

    os.makedirs(vis_dir, exist_ok=True)
    query_id = args.query_frame_id
    radius = 3  # the point radius for point correspondence visualization

    mask = None
    if os.path.exists(args.foreground_mask_path):
        h, w = trainer.h, trainer.w
        mask = imageio.imread(args.foreground_mask_path)[..., -1]  # rgba image, take the alpha channel
        mask = cv2.resize(mask, dsize=(w, h)) == 255

    # for DAVIS video sequences which come with segmentation masks
    # or when a foreground mask for the query frame is provided
    if trainer.with_mask or mask is not None:
        # foreground
        frames, kpts_forground = trainer.eval_video_correspondences(query_id, use_mask=True,
                                                                    mask=mask,
                                                                    vis_occlusion=args.vis_occlusion,
                                                                    occlusion_th=args.occlusion_th,
                                                                    use_max_loc=args.use_max_loc,
                                                                    radius=radius,
                                                                    return_kpts=True)
        imageio.mimwrite(os.path.join(vis_dir, '{}_{:06d}_foreground_{}.mp4'.format(seq_name, trainer.step, query_id)),
                         frames, quality=8, fps=10)
        kpts_forground = kpts_forground.cpu().numpy()

        # background
        frames, kpts_background = trainer.eval_video_correspondences(query_id, use_mask=True,
                                                                     reverse_mask=True,
                                                                     mask=mask,
                                                                     vis_occlusion=args.vis_occlusion,
                                                                     occlusion_th=args.occlusion_th,
                                                                     use_max_loc=args.use_max_loc,
                                                                     radius=radius,
                                                                     return_kpts=True)
        kpts_background = kpts_background.cpu().numpy()
        imageio.mimwrite(os.path.join(vis_dir, '{}_{:06d}_background_{}.mp4'.format(seq_name, trainer.step, query_id)),
                         frames, quality=8, fps=10)
        # visualize trails
        vis_trail(args.data_dir, kpts_forground, kpts_background,
                  os.path.join(vis_dir, '{}_{:06d}_{}_trails.mp4'.format(seq_name, trainer.step, query_id)))

    else:
        frames = trainer.eval_video_correspondences(query_id,
                                                    vis_occlusion=args.vis_occlusion,
                                                    occlusion_th=args.occlusion_th,
                                                    use_max_loc=args.use_max_loc,
                                                    radius=radius)
        imageio.mimwrite(os.path.join(vis_dir, '{}_{:06d}_{}.mp4'.format(seq_name, trainer.step, query_id)),
                         frames, quality=8, fps=10)