Up vector for plotly

Summary: We previously did not send an `up` vector in to plotly when visualization_cameras is supplied. This meant the image would have the default orientation instead of the correct one. Now we use the library function `camera_to_eye_at_up` to calculate the plotly position, which includes the `up` vector.

Reviewed By: nikhilaravi

Differential Revision: D25981183

fbshipit-source-id: abec72b349f3a5519209e0e6c0518133c3750807

pytorch3d/vis/plotly_vis.py

@@ -8,6 +8,7 @@ import plotly.graph_objects as go
 import torch
 from plotly.subplots import make_subplots
 from pytorch3d.renderer import TexturesVertex
+from pytorch3d.renderer.camera_utils import camera_to_eye_at_up
 from pytorch3d.renderer.cameras import CamerasBase
 from pytorch3d.structures import Meshes, Pointclouds, join_meshes_as_scene
 
@@ -224,23 +225,14 @@ def plot_scene(
     }
     viewpoints_eye_at_up_world = None
     if viewpoint_cameras:
-        if len(viewpoint_cameras) == len(subplots) or len(viewpoint_cameras) == 1:
+        n_viewpoint_cameras = len(viewpoint_cameras)
+        if n_viewpoint_cameras == len(subplots) or n_viewpoint_cameras == 1:
             # Calculate the vectors eye, at, up in world space
             # to initialize the position of the camera in
             # the plotly figure
-            # TODO(T77879494): correct the up vector calculation in the world space.
-            eye_at_up_view = (
-                torch.tensor([[0, 0, 0], [0, 0, 1], [0, 1, 0]])
-                .float()
-                .to(viewpoint_cameras.device)
+            viewpoints_eye_at_up_world = camera_to_eye_at_up(
+                viewpoint_cameras.get_world_to_view_transform().cpu()
             )
-            viewpoints_eye_at_up_world = (
-                viewpoint_cameras.get_world_to_view_transform()
-                .inverse()
-                .transform_points(eye_at_up_view)
-            )
-            if len(viewpoints_eye_at_up_world.shape) < 3:
-                viewpoints_eye_at_up_world = viewpoints_eye_at_up_world.unsqueeze(0)
         else:
             msg = "Invalid number {} of viewpoint cameras were provided. Either 1 \
             or {} cameras are required".format(
@@ -290,29 +282,33 @@ def plot_scene(
         # update camera viewpoint if provided
         if viewpoints_eye_at_up_world is not None:
             # Use camera params for batch index or the first camera if only one provided.
-            viewpoint_idx = min(len(viewpoints_eye_at_up_world) - 1, subplot_idx)
-
-            eye, at, _up = viewpoints_eye_at_up_world[viewpoint_idx]
+            viewpoint_idx = min(n_viewpoint_cameras - 1, subplot_idx)
 
+            eye, at, up = (i[viewpoint_idx] for i in viewpoints_eye_at_up_world)
             eye_x, eye_y, eye_z = eye.tolist()
-
             at_x, at_y, at_z = at.tolist()
+            up_x, up_y, up_z = up.tolist()
 
             # scale camera eye to plotly [-1, 1] ranges
             x_range = xaxis["range"]
             y_range = yaxis["range"]
             z_range = zaxis["range"]
 
-            eye_x = _scale_camera_to_bounds(eye_x, x_range)
-            eye_y = _scale_camera_to_bounds(eye_y, y_range)
-            eye_z = _scale_camera_to_bounds(eye_z, z_range)
+            eye_x = _scale_camera_to_bounds(eye_x, x_range, True)
+            eye_y = _scale_camera_to_bounds(eye_y, y_range, True)
+            eye_z = _scale_camera_to_bounds(eye_z, z_range, True)
 
-            at_x = _scale_camera_to_bounds(at_x, x_range)
-            at_y = _scale_camera_to_bounds(at_y, y_range)
-            at_z = _scale_camera_to_bounds(at_z, z_range)
+            at_x = _scale_camera_to_bounds(at_x, x_range, True)
+            at_y = _scale_camera_to_bounds(at_y, y_range, True)
+            at_z = _scale_camera_to_bounds(at_z, z_range, True)
+
+            up_x = _scale_camera_to_bounds(up_x, x_range, False)
+            up_y = _scale_camera_to_bounds(up_y, y_range, False)
+            up_z = _scale_camera_to_bounds(up_z, z_range, False)
 
             camera["eye"] = {"x": eye_x, "y": eye_y, "z": eye_z}
             camera["center"] = {"x": at_x, "y": at_y, "z": at_z}
+            camera["up"] = {"x": up_x, "y": up_y, "z": up_z}
 
         current_layout.update(
             {
@@ -751,7 +747,9 @@ def _update_axes_bounds(
     current_layout.update({"xaxis": xaxis, "yaxis": yaxis, "zaxis": zaxis})
 
 
-def _scale_camera_to_bounds(coordinate: float, axis_bounds: Tuple[float, float]):
+def _scale_camera_to_bounds(
+    coordinate: float, axis_bounds: Tuple[float, float], is_position: bool
+):
     """
     We set our plotly plot's axes' bounding box to [-1,1]x[-1,1]x[-1,1]. As such,
     the plotly camera location has to be scaled accordingly to have its world coordinates
@@ -762,7 +760,12 @@ def _scale_camera_to_bounds(coordinate: float, axis_bounds: Tuple[float, float])
         coordinate: the float value to be transformed
         axis_bounds: the bounds of the plotly plot for the axis which
             the coordinate argument refers to
+        is_position: If true, the float value is the coordinate of a position, and so must
+            be moved in to [-1,1]. Otherwise it is a component of a direction, and so needs only
+            to be scaled.
     """
     scale = (axis_bounds[1] - axis_bounds[0]) / 2
+    if not is_position:
+        return coordinate / scale
     offset = (axis_bounds[1] / scale) - 1
     return coordinate / scale - offset