Skip to content

GitLab

"scripts/vscode:/vscode.git/clone" did not exist on "317a82e274c49dafb8a2de6722026f41e5640fe7"
Commit 8006842f authored by David Novotny's avatar David Novotny Committed by Facebook GitHub Bot
Browse files

Conversion from OpenCV cameras 

Summary: Implements a conversion function between OpenCV and PyTorch3D cameras.

Reviewed By: patricklabatut

Differential Revision: D28992470

fbshipit-source-id: dbcc9f213ec293c2f6938261c704aea09aad3c90
parent b2ac2655
Show whitespace changes
Inline Side-by-side
Showing with 1450 additions and 0 deletions
pytorch3d/utils/__init__.py
View file @ 8006842f
# Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.
from .camera_conversions import cameras_from_opencv_projection
from .ico_sphere import ico_sphere
from .torus import torus
......
pytorch3d/utils/camera_conversions.py 0 → 100644
View file @ 8006842f
import torch
from ..renderer import PerspectiveCameras
from ..transforms import so3_exponential_map
def cameras_from_opencv_projection(
rvec: torch.Tensor,
tvec: torch.Tensor,
camera_matrix: torch.Tensor,
image_size: torch.Tensor,
) -> PerspectiveCameras:
"""
Converts a batch of OpenCV-conventioned cameras parametrized with the
axis-angle rotation vectors `rvec`, translation vectors `tvec`, and the camera
calibration matrices `camera_matrix` to `PerspectiveCameras` in PyTorch3D
convention.
More specifically, the conversion is carried out such that a projection
of a 3D shape to the OpenCV-conventioned screen of size `image_size` results
in the same image as a projection with the corresponding PyTorch3D camera
to the NDC screen convention of PyTorch3D.
More specifically, the OpenCV convention projects points to the OpenCV screen
space as follows:
```
x_screen_opencv = camera_matrix @ (exp(rvec) @ x_world + tvec)
```
followed by the homogenization of `x_screen_opencv`.
Note:
The parameters `rvec, tvec, camera_matrix` correspond e.g. to the inputs
of `cv2.projectPoints`, or to the ouputs of `cv2.calibrateCamera`.
Args:
rvec: A batch of axis-angle rotation vectors of shape `(N, 3)`.
tvec: A batch of translation vectors of shape `(N, 3)`.
camera_matrix: A batch of camera calibration matrices of shape `(N, 3, 3)`.
image_size: A tensor of shape `(N, 2)` containing the sizes of the images
(height, width) attached to each camera.
Returns:
cameras_pytorch3d: A batch of `N` cameras in the PyTorch3D convention.
"""
R = so3_exponential_map(rvec)
focal_length = torch.stack([camera_matrix[:, 0, 0], camera_matrix[:, 1, 1]], dim=-1)
principal_point = camera_matrix[:, :2, 2]
# Retype the image_size correctly and flip to width, height.
image_size_wh = image_size.to(R).flip(dims=(1,))
# Get the PyTorch3D focal length and principal point.
focal_pytorch3d = focal_length / (0.5 * image_size_wh)
p0_pytorch3d = -(principal_point / (0.5 * image_size_wh) - 1)
# For R, T we flip x, y axes (opencv screen space has an opposite
# orientation of screen axes).
# We also transpose R (opencv multiplies points from the opposite=left side).
R_pytorch3d = R.permute(0, 2, 1)
T_pytorch3d = tvec.clone()
R_pytorch3d[:, :, :2] *= -1
T_pytorch3d[:, :2] *= -1
return PerspectiveCameras(
R=R_pytorch3d,
T=T_pytorch3d,
focal_length=focal_pytorch3d,
principal_point=p0_pytorch3d,
)
tests/data/cv_project_points_precomputed.json 0 → 100644
View file @ 8006842f
[
{
"rvec": [
-1.6336234226511284,
-1.009804818052615,
0.4160736184401035
],
"tvec": [
1.626905,
-1.5187958,
-4.6009063
],
"camera_matrix": [
[
-1.0406556,
0.0,
0.6160261
],
[
-0.0,
-4.6370989,
-2.2859802
],
[
-0.0,
-0.0,
1.0
]
],
"image_size": [
19,
16
],
"pts": [
[
0.7610377,
0.121675,
0.4438632
],
[
0.3336743,
1.4940791,
-0.2051583
],
[
0.3130677,
-0.8540957,
-2.5529898
],
[
0.6536186,
0.8644362,
-0.742165
],
[
2.2697546,
-1.4543657,
0.0457585
],
[
-0.1871839,
1.5327792,
1.4693588
],
[
0.1549474,
0.3781625,
-0.8877857
],
[
-1.9807965,
-0.3479121,
0.156349
],
[
1.2302907,
1.2023798,
-0.3873268
],
[
-0.3023028,
-1.048553,
-1.4200179
],
[
-1.7062702,
1.9507754,
-0.5096522
],
[
-0.4380743,
-1.2527954,
0.7774904
],
[
-1.6138978,
-0.2127403,
-0.8954666
],
[
0.3869025,
-0.5108051,
-1.1806322
],
[
-0.0281822,
0.4283319,
0.0665172
],
[
0.3024719,
-0.6343221,
-0.3627412
],
[
-0.6724604,
-0.3595532,
-0.8131463
],
[
-1.7262826,
0.1774261,
-0.4017809
],
[
-1.6301983,
0.4627823,
-0.9072984
],
[
0.0519454,
0.7290906,
0.1289829
],
[
1.1394007,
-1.2348258,
0.4023416
],
[
-0.6848101,
-0.8707971,
-0.5788497
],
[
-0.3115525,
0.0561653,
-1.1651498
],
[
0.9008265,
0.4656624,
-1.5362437
],
[
1.4882522,
1.8958892,
1.1787796
],
[
-0.1799248,
-1.0707526,
1.0544517
],
[
-0.4031769,
1.2224451,
0.208275
],
[
0.976639,
0.3563664,
0.7065732
],
[
0.0105,
1.7858705,
0.1269121
],
[
0.4019894,
1.8831507,
-1.3477591
]
],
"pts_proj": [
[
1.0145014,
-2.9284953
],
[
1.0724146,
-3.4080993
],
[
1.7917063,
-6.7793572
],
[
1.2140849,
-3.6559075
],
[
1.4910298,
-1.7436687
],
[
0.7868983,
-2.8461123
],
[
1.1790002,
-4.2421372
],
[
0.6688383,
-5.165954
],
[
1.2222519,
-2.9970978
],
[
1.2750471,
-6.013773
],
[
0.8970727,
-4.5353657
],
[
0.7041445,
-3.9451632
],
[
0.9085118,
-5.7326503
],
[
1.3246946,
-4.7337656
],
[
0.9750362,
-3.678297
],
[
1.1074375,
-4.0924549
],
[
1.0367661,
-5.1176712
],
[
0.8222725,
-5.1682077
],
[
0.9272976,
-5.3440215
],
[
0.9797946,
-3.5213536
],
[
1.0847165,
-2.7743226
],
[
0.9762082,
-5.2337584
],
[
1.1681587,
-4.9342569
],
[
1.4412056,
-4.1796204
],
[
1.0048257,
-2.0358393
],
[
0.7052604,
-3.4338048
],
[
0.9221008,
-3.6497452
],
[
1.0001084,
-2.588002
],
[
0.9887072,
-3.3682799
],
[
1.2562716,
-3.9393683
]
]
},
{
"rvec": [
-1.624698671197051,
1.2717319528620892,
0.026455585547038436
],
"tvec": [
-0.7416676,
-0.9513922,
1.2411273
],
"camera_matrix": [
[
-0.6674566,
0.0,
0.0366092
],
[
-0.0,
-0.4250565,
-0.1661673
],
[
-0.0,
-0.0,
1.0
]
],
"image_size": [
11,
19
],
"pts": [
[
0.9182028,
-0.1588005,
-0.9640634
],
[
-1.9907788,
0.0897307,
0.1148539
],
[
-0.5858152,
0.298772,
0.2222599
],
[
0.435183,
-0.0457481,
0.0498984
],
[
-0.9355305,
0.2873877,
0.3604273
],
[
0.4081481,
-1.9407157,
1.4448357
],
[
0.1928609,
-0.4208648,
1.7402535
],
[
-0.3640868,
1.3439544,
-0.818221
],
[
0.0827099,
-1.2910585,
-0.6611042
],
[
-1.180191,
0.1976426,
0.4139
],
[
1.197322,
1.8833539,
0.7142238
],
[
2.2843334,
1.5641026,
0.6111037
],
[
-0.8773633,
-1.6210875,
-0.581673
],
[
-0.537834,
-1.5560237,
-0.0544648
],
[
-1.8112788,
-0.6311752,
-0.9281592
],
[
1.4907219,
0.1954993,
-0.4716043
],
[
1.8123547,
-2.2941375,
0.6512093
],
[
-1.1304964,
-0.7773467,
1.1159385
],
[
1.339453,
-1.7674337,
0.4244125
],
[
1.089309,
-0.3841857,
0.6322014
],
[
-0.5496559,
0.5211257,
0.1083495
],
[
0.2616685,
-0.9147553,
0.8582378
],
[
0.0943343,
-1.4859039,
-1.9005843
],
[
-1.1375792,
-1.7620389,
-0.2886232
],
[
1.0479822,
0.2499575,
0.0469045
],
[
-1.032243,
0.4031857,
-0.6840593
],
[
1.2623222,
-2.0055566,
-0.3320304
],
[
-0.2961004,
-2.2183608,
-0.1835029
],
[
0.3923081,
0.2416348,
0.1039359
],
[
-0.8295712,
0.4927594,
0.0901128
]
],
"pts_proj": [
[
0.4141589,
0.5870938
],
[
0.5208963,
-0.2684058
],
[
0.6192571,
-0.0457759
],
[
0.3620623,
0.3519971
],
[
0.5966208,
-0.1618695
],
[
-0.6130961,
-0.0666413
],
[
-0.5973283,
-0.240145
],
[
1.7229771,
0.3713118
],
[
0.069312,
0.1164213
],
[
0.5411626,
-0.21708
],
[
-0.7298492,
-0.6226879
],
[
-0.2271261,
-0.7756765
],
[
0.0919778,
-0.0452863
],
[
0.005477,
-0.0454978
],
[
0.3725968,
-0.1135225
],
[
0.6738571,
1.806536
],
[
-0.8906672,
0.3956792
],
[
0.0670651,
-0.294005
],
[
-0.5670261,
0.3293613
],
[
-0.3558128,
0.7635874
],
[
0.7998255,
-0.0010898
],
[
-0.2129184,
0.031003
],
[
0.1711748,
0.1747995
],
[
0.0606019,
-0.0990852
],
[
0.6655512,
1.3313645
],
[
0.6986332,
-0.0128461
],
[
-0.3178348,
0.297116
],
[
-0.1058796,
-0.0156453
],
[
0.6059715,
0.4297142
],
[
0.750323,
-0.082273
]
]
},
{
"rvec": [
0.9529480300972241,
-0.8534383895555052,
-2.0235766239679127
],
"tvec": [
-0.3351616,
-1.6076185,
0.0039651
],
"camera_matrix": [
[
0.6332697,
0.0,
-0.0084798
],
[
0.0,
0.5719922,
0.5328057
],
[
0.0,
0.0,
1.0
]
],
"image_size": [
10,
19
],
"pts": [
[
-0.3824862,
0.6916619,
0.353885
],
[
1.0475853,
-0.4238962,
-3.5147681
],
[
-1.3431567,
1.4255061,
0.228582
],
[
-0.2576638,
0.0503707,
-1.3802109
],
[
-0.2616721,
-0.1793797,
-0.6927706
],
[
1.1378269,
-0.1691573,
-0.7639137
],
[
-0.4980731,
-0.3628911,
0.2639603
],
[
-0.6296419,
-0.4722584,
-1.513361
],
[
1.1076247,
0.1762388,
-0.9403535
],
[
0.9295943,
-1.0627949,
-0.8864063
],
[
1.921347,
-0.4597805,
-1.0890344
],
[
0.9841173,
-1.1592063,
-0.4365371
],
[
1.0092445,
0.7133896,
-0.7280577
],
[
0.8395165,
1.239021,
-1.7848039
],
[
-0.7961858,
-1.4005413,
-0.1843506
],
[
-1.3911931,
0.0362597,
-0.8144056
],
[
0.6973728,
-1.7374292,
0.1158557
],
[
0.3656514,
-0.0739235,
-0.4935176
],
[
3.1015306,
0.8587542,
-1.1547755
],
[
0.9418343,
-0.2821351,
-0.9756547
],
[
0.0981867,
0.90549,
1.0187414
],
[
-0.1148989,
1.7430387,
-0.3218792
],
[
0.8295711,
-0.207318,
1.1179986
],
[
1.0642497,
1.1513298,
-0.7724577
],
[
-1.2936343,
0.6770268,
0.4240552
],
[
-0.4856762,
-0.0516972,
0.5670564
],
[
1.0678336,
0.2715957,
0.6193018
],
[
-0.058626,
1.2565714,
0.2967472
],
[
0.3985857,
-1.0531744,
-0.6394763
],
[
-0.1485269,
-1.5745821,
-0.4956882
]
],
"pts_proj": [
[
-0.1735197,
-0.5026447
],
[
-0.5086927,
1.2602939
],
[
0.1995019,
0.1377832
],
[
-1.0602558,
2.3228582
],
[
-0.4811967,
2.6476893
],
[
0.1891248,
2.2078843
],
[
-14.5705317,
-27.5060102
],
[
-0.7295134,
1.2970094
],
[
-0.0004579,
2.8277094
],
[
0.1412656,
1.2635503
],
[
0.1810516,
1.7102113
],
[
0.3390042,
1.2833538
],
[
-0.1753715,
12.5495766
],
[
-3.6611631,
10.1461827
],
[
0.1812461,
0.7066171
],
[
6.5346752,
-3.5950883
],
[
0.5305561,
0.9966792
],
[
0.1684846,
3.1942163
],
[
0.3376347,
3.4565985
],
[
0.024885,
1.9060357
],
[
-0.4946605,
-0.3688437
],
[
0.2704817,
-0.5945693
],
[
-8.9040477,
-8.4144665
],
[
0.7361192,
-10.8591518
],
[
0.0607204,
0.1565487
],
[
-0.9204385,
-0.9006548
],
[
-4.7045052,
-7.8600902
],
[
-0.0843625,
-0.5081105
],
[
0.1565888,
1.2224583
],
[
0.1561078,
0.8506761
]
]
},
{
"rvec": [
-0.9637575928124262,
1.6344477621933204,
1.1377406680922277
],
"tvec": [
-5.0736742,
-15.0264648,
3.7663565
],
"camera_matrix": [
[
0.0592345,
0.0,
1.0948929
],
[
0.0,
2.0469542,
-1.6769676
],
[
0.0,
0.0,
1.0
]
],
"image_size": [
19,
12
],
"pts": [
[
0.0815831,
1.2381695,
2.1837783
],
[
0.1905641,
-0.4981669,
0.0475124
],
[
1.4483291,
0.4282211,
-0.1707534
],
[
-2.3547773,
-0.4678065,
-0.1294017
],
[
0.9000515,
-0.5156528,
0.5203996
],
[
1.1410499,
-1.4447244,
-1.0176718
],
[
-0.9902687,
1.5241503,
0.636245
],
[
0.1996184,
-0.0077118,
1.1754896
],
[
1.2556589,
-0.2809262,
-0.3669177
],
[
1.6537248,
-0.0856097,
0.2933573
],
[
1.4872575,
-0.6384799,
0.7087688
],
[
-0.0825277,
-0.6873837,
-1.4113116
],
[
0.6213837,
0.3085112,
-0.3309394
],
[
-0.4852161,
-0.0602118,
-1.5596469
],
[
-0.2914053,
0.8040719,
0.7358267
],
[
-0.2026395,
0.2902467,
-0.8548176
],
[
-0.3261322,
-0.952252,
1.1869633
],
[
-1.1571738,
0.0193,
1.4500207
],
[
0.6521225,
-0.3442692,
1.2908895
],
[
-0.8021245,
-1.3890878,
0.2576423
],
[
1.1274729,
2.7849331,
-0.0913946
],
[
-0.0290724,
1.030574,
1.0248824
],
[
1.5831991,
1.0291609,
1.8343708
],
[
-0.4910075,
-0.8094693,
-0.2462478
],
[
-0.4517529,
-0.6581127,
0.3922879
],
[
-1.3964618,
0.3368952,
0.6266755
],
[
0.5469885,
-0.3078414,
0.0734122
],
[
-0.2766632,
0.1395219,
-0.050913
],
[
0.740091,
-0.3233616,
2.4009981
],
[
2.8127453,
0.6191253,
-2.3287021
]
],
"pts_proj": [
[
1.0011881,
-8.7962669
],
[
1.0148904,
-10.6624766
],
[
0.9559355,
-13.9861497
],
[
1.0550145,
-6.889409
],
[
0.9903897,
-12.9240756
],
[
0.9922857,
-15.2615745
],
[
1.0231235,
-7.3797553
],
[
1.0095861,
-10.1396779
],
[
0.9732178,
-14.1103632
],
[
0.9437488,
-16.0891922
],
[
0.9584208,
-16.2312512
],
[
1.0231435,
-10.332697
],
[
0.9944343,
-11.1411459
],
[
1.0257093,
-9.1991957
],
[
1.0149771,
-8.6661286
],
[
1.0172749,
-9.379192
],
[
1.0324498,
-9.6570503
],
[
1.0382854,
-7.6709289
],
[
0.9976967,
-11.7654687
],
[
1.0447702,
-9.1607223
],
[
0.9566311,
-10.3084282
],
[
1.0066492,
-8.9367541
],
[
0.9320123,
-13.8820569
],
[
1.0331906,
-9.4447618
],
[
1.0314307,
-9.3262563
],
[
1.0384939,
-7.4069907
],
[
1.0022615,
-11.4736859
],
[
1.0202613,
-9.217644
],
[
0.9922728,
-11.9963374
],
[
0.8448706,
-23.0521993
]
]
}
]
tests/test_camera_conversions.py 0 → 100644
View file @ 8006842f
# Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.
import json
import unittest
import numpy as np
import torch
from common_testing import TestCaseMixin, get_tests_dir
from pytorch3d.ops import eyes
from pytorch3d.transforms import so3_exponential_map, so3_log_map
from pytorch3d.utils import (
cameras_from_opencv_projection,
)
DATA_DIR = get_tests_dir() / "data"
def _coords_opencv_screen_to_pytorch3d_ndc(xy_opencv, image_size):
"""
Converts the OpenCV screen coordinates `xy_opencv` to PyTorch3D NDC coordinates.
"""
xy_pytorch3d = -(2.0 * xy_opencv / image_size.flip(dims=(1,))[:, None] - 1.0)
return xy_pytorch3d
def cv2_project_points(pts, rvec, tvec, camera_matrix):
"""
Reproduces the `cv2.projectPoints` function from OpenCV using PyTorch.
"""
R = so3_exponential_map(rvec)
pts_proj_3d = (
camera_matrix.bmm(R.bmm(pts.permute(0, 2, 1)) + tvec[:, :, None])
).permute(0, 2, 1)
depth = pts_proj_3d[..., 2:]
pts_proj_2d = pts_proj_3d[..., :2] / depth
return pts_proj_2d
class TestCameraConversions(TestCaseMixin, unittest.TestCase):
def setUp(self) -> None:
super().setUp()
torch.manual_seed(42)
np.random.seed(42)
def test_cv2_project_points(self):
"""
Tests that the local implementation of cv2_project_points gives the same
restults OpenCV's `cv2.projectPoints`. The check is done against a set
of precomputed results `cv_project_points_precomputed`.
"""
with open(DATA_DIR / "cv_project_points_precomputed.json", "r") as f:
cv_project_points_precomputed = json.load(f)
for test_case in cv_project_points_precomputed:
_pts_proj = cv2_project_points(
**{
k: torch.tensor(test_case[k])[None]
for k in ("pts", "rvec", "tvec", "camera_matrix")
}
)
pts_proj = torch.tensor(test_case["pts_proj"])[None]
self.assertClose(_pts_proj, pts_proj, atol=1e-4)
def test_opencv_conversion(self):
"""
Tests that the cameras converted from opencv to pytorch3d convention
return correct projections of random 3D points. The check is done
against a set of results precomuted using `cv2.projectPoints` function.
"""
image_size = [[480, 640]] * 4
R = [
[
[1.0, 0.0, 0.0],
[0.0, 1.0, 0.0],
[0.0, 0.0, 1.0],
],
[
[1.0, 0.0, 0.0],
[0.0, 0.0, -1.0],
[0.0, 1.0, 0.0],
],
[
[0.0, 0.0, 1.0],
[1.0, 0.0, 0.0],
[0.0, 1.0, 0.0],
],
[
[0.0, 0.0, 1.0],
[1.0, 0.0, 0.0],
[0.0, 1.0, 0.0],
],
]
tvec = [
[0.0, 0.0, 3.0],
[0.3, -0.3, 3.0],
[-0.15, 0.1, 4.0],
[0.0, 0.0, 4.0],
]
focal_length = [
[100.0, 100.0],
[115.0, 115.0],
[105.0, 105.0],
[120.0, 120.0],
]
principal_point = [
[240, 320],
[240.5, 320.3],
[241, 318],
[242, 322],
]
principal_point, focal_length, R, tvec, image_size = [
torch.FloatTensor(x)
for x in (principal_point, focal_length, R, tvec, image_size)
]
camera_matrix = eyes(dim=3, N=4)
camera_matrix[:, 0, 0], camera_matrix[:, 1, 1] = (
focal_length[:, 0],
focal_length[:, 1],
)
camera_matrix[:, :2, 2] = principal_point
rvec = so3_log_map(R)
pts = torch.nn.functional.normalize(torch.randn(4, 1000, 3), dim=-1)
# project the 3D points with the opencv projection function
pts_proj_opencv = cv2_project_points(pts, rvec, tvec, camera_matrix)
# make the pytorch3d cameras
cameras_opencv_to_pytorch3d = cameras_from_opencv_projection(
rvec, tvec, camera_matrix, image_size
)
# project the 3D points with converted cameras
pts_proj_pytorch3d = cameras_opencv_to_pytorch3d.transform_points(pts)[..., :2]
# convert the opencv-projected points to pytorch3d screen coords
pts_proj_opencv_in_pytorch3d_screen = _coords_opencv_screen_to_pytorch3d_ndc(
pts_proj_opencv, image_size
)
# compare to the cached projected points
self.assertClose(
pts_proj_opencv_in_pytorch3d_screen, pts_proj_pytorch3d, atol=1e-5
)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment