Skip to content

GitLab

Commit c49ebad2 authored by Jeremy Reizenstein's avatar Jeremy Reizenstein Committed by Facebook GitHub Bot
Browse files

tutorial fix: add visdom 

Summary: need to pip install visdom in new volumes tutorial.

Reviewed By: kjchalup

Differential Revision: D38501905

fbshipit-source-id: 534bf097e41f05b3389e9420e6dd2b61a4517861
parent 1cd0cbff
Hide whitespace changes
Inline Side-by-side
Showing with 309 additions and 315 deletions
docs/tutorials/implicitron_volumes.ipynb
View file @ c49ebad2
{ {
"metadata": {
"dataExplorerConfig": {},
"bento_stylesheets": {
"bento/extensions/flow/main.css": true,
"bento/extensions/kernel_selector/main.css": true,
"bento/extensions/kernel_ui/main.css": true,
"bento/extensions/new_kernel/main.css": true,
"bento/extensions/system_usage/main.css": true,
"bento/extensions/theme/main.css": true
},
"kernelspec": {
"display_name": "pytorch3d",
"language": "python",
"name": "bento_kernel_pytorch3d",
"metadata": {
"kernel_name": "bento_kernel_pytorch3d",
"nightly_builds": true,
"fbpkg_supported": true,
"cinder_runtime": false,
"is_prebuilt": true
}
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3"
},
"last_server_session_id": "2944b203-9ea8-4c0e-9634-645dfea5f26b",
"last_kernel_id": "bb33cd83-7924-489a-8bd8-2d9d62eb0126",
"last_base_url": "https://9177.od.fbinfra.net:443/",
"last_msg_id": "99f7088e-d22b355b859660479ef0574e_5743",
"captumWidgetMessage": {},
"outputWidgetContext": {}
},
"nbformat": 4,
"nbformat_minor": 2,
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"originalKey": "d38652e8-200a-413c-a36a-f4d349b78a9d",
"showInput": true,
"customInput": null,
"collapsed": false, "collapsed": false,
"requestMsgId": "641de8aa-0e42-4446-9304-c160a2d226bf", "customInput": null,
"customOutput": null, "customOutput": null,
"executionStartTime": 1659619824914, "executionStartTime": 1659619824914,
"executionStopTime": 1659619825485 "executionStopTime": 1659619825485,
"originalKey": "d38652e8-200a-413c-a36a-f4d349b78a9d",
"requestMsgId": "641de8aa-0e42-4446-9304-c160a2d226bf",
"showInput": true
}, },
"outputs": [],
"source": [ "source": [
"# Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved." "# Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved."
], ]
"execution_count": null,
"outputs": []
}, },
{ {
"attachments": {},
"cell_type": "markdown", "cell_type": "markdown",
"metadata": { "metadata": {
"customInput": null,
"originalKey": "a48a9dcf-e80f-474b-a0c4-2c9a765b15c5", "originalKey": "a48a9dcf-e80f-474b-a0c4-2c9a765b15c5",
"showInput": false, "showInput": false
"customInput": null
}, },
"source": [ "source": [
"# A simple model using Implicitron\n", "# A simple model using Implicitron\n",
...@@ -73,36 +32,36 @@ ...@@ -73,36 +32,36 @@
"In this demo, we use the VolumeRenderer from PyTorch3D as a custom implicit function in Implicitron. We will see\n", "In this demo, we use the VolumeRenderer from PyTorch3D as a custom implicit function in Implicitron. We will see\n",
"* some of the main objects in Implicitron\n", "* some of the main objects in Implicitron\n",
"* how to plug in a custom part of a model" "* how to plug in a custom part of a model"
], ]
"attachments": {}
}, },
{ {
"attachments": {},
"cell_type": "markdown", "cell_type": "markdown",
"metadata": { "metadata": {
"customInput": null,
"originalKey": "51337c0e-ad27-4b75-ad6a-737dca5d7b95", "originalKey": "51337c0e-ad27-4b75-ad6a-737dca5d7b95",
"showInput": false, "showInput": false
"customInput": null
}, },
"source": [ "source": [
"## 0. Install and import modules\n", "## 0. Install and import modules\n",
"\n", "\n",
"Ensure `torch` and `torchvision` are installed. If `pytorch3d` is not installed, install it using the following cell:\n", "Ensure `torch` and `torchvision` are installed. If `pytorch3d` is not installed, install it using the following cell:\n"
"" ]
],
"attachments": {}
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"originalKey": "76f1ecd4-6b73-4214-81b0-118ef8d86872",
"showInput": true,
"customInput": null,
"collapsed": false, "collapsed": false,
"requestMsgId": "deb6a860-6923-4227-abef-d31388b5142d", "customInput": null,
"customOutput": null, "customOutput": null,
"executionStartTime": 1659619898147, "executionStartTime": 1659619898147,
"executionStopTime": 1659619898274 "executionStopTime": 1659619898274,
"originalKey": "76f1ecd4-6b73-4214-81b0-118ef8d86872",
"requestMsgId": "deb6a860-6923-4227-abef-d31388b5142d",
"showInput": true
}, },
"outputs": [],
"source": [ "source": [
"import os\n", "import os\n",
"import sys\n", "import sys\n",
...@@ -129,48 +88,48 @@ ...@@ -129,48 +88,48 @@
" !tar xzf 1.10.0.tar.gz\n", " !tar xzf 1.10.0.tar.gz\n",
" os.environ[\"CUB_HOME\"] = os.getcwd() + \"/cub-1.10.0\"\n", " os.environ[\"CUB_HOME\"] = os.getcwd() + \"/cub-1.10.0\"\n",
" !pip install 'git+https://github.com/facebookresearch/pytorch3d.git@stable'" " !pip install 'git+https://github.com/facebookresearch/pytorch3d.git@stable'"
], ]
"execution_count": null,
"outputs": []
}, },
{ {
"attachments": {},
"cell_type": "markdown", "cell_type": "markdown",
"metadata": { "metadata": {
"customInput": null,
"originalKey": "2c1020e6-eb4a-4644-9719-9147500d8e4f", "originalKey": "2c1020e6-eb4a-4644-9719-9147500d8e4f",
"showInput": false, "showInput": false
"customInput": null
}, },
"source": [ "source": [
"Ensure omegaconf is installed. If not, run this cell. (It should not be necessary to restart the runtime.)" "Ensure omegaconf and visdom are installed. If not, run this cell. (It should not be necessary to restart the runtime.)"
], ]
"attachments": {}
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"originalKey": "9e751931-a38d-44c9-9ff1-ac2f7d3a3f99",
"showInput": true,
"customInput": null, "customInput": null,
"customOutput": null "customOutput": null,
"originalKey": "9e751931-a38d-44c9-9ff1-ac2f7d3a3f99",
"showInput": true
}, },
"outputs": [],
"source": [ "source": [
"!pip install omegaconf" "!pip install omegaconf visdom"
], ]
"execution_count": null,
"outputs": []
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"code_folding": [],
"collapsed": false, "collapsed": false,
"originalKey": "86807e4a-1675-4520-a033-c7af85b233ec",
"requestMsgId": "880a7e20-4a90-4b37-a5eb-bccc0b23cac6",
"customOutput": null, "customOutput": null,
"executionStartTime": 1659612480556, "executionStartTime": 1659612480556,
"executionStopTime": 1659612480644, "executionStopTime": 1659612480644,
"code_folding": [], "hidden_ranges": [],
"hidden_ranges": [] "originalKey": "86807e4a-1675-4520-a033-c7af85b233ec",
"requestMsgId": "880a7e20-4a90-4b37-a5eb-bccc0b23cac6"
}, },
"outputs": [],
"source": [ "source": [
"import logging\n", "import logging\n",
"from typing import Tuple\n", "from typing import Tuple\n",
...@@ -193,56 +152,55 @@ ...@@ -193,56 +152,55 @@
"from pytorch3d.renderer.implicit.renderer import VolumeSampler\n", "from pytorch3d.renderer.implicit.renderer import VolumeSampler\n",
"from pytorch3d.structures import Volumes\n", "from pytorch3d.structures import Volumes\n",
"from pytorch3d.vis.plotly_vis import plot_batch_individually, plot_scene" "from pytorch3d.vis.plotly_vis import plot_batch_individually, plot_scene"
], ]
"execution_count": null,
"outputs": []
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"originalKey": "b2d9f5bd-a9d4-4f78-b21e-92f2658e0fe9", "code_folding": [],
"showInput": true,
"customInput": null,
"collapsed": false, "collapsed": false,
"requestMsgId": "7e43e623-4030-438b-af4e-b96170c9a052", "customInput": null,
"customOutput": null, "customOutput": null,
"executionStartTime": 1659610929375, "executionStartTime": 1659610929375,
"executionStopTime": 1659610929383, "executionStopTime": 1659610929383,
"code_folding": [], "hidden_ranges": [],
"hidden_ranges": [] "originalKey": "b2d9f5bd-a9d4-4f78-b21e-92f2658e0fe9",
"requestMsgId": "7e43e623-4030-438b-af4e-b96170c9a052",
"showInput": true
}, },
"outputs": [],
"source": [ "source": [
"output_resolution = 80" "output_resolution = 80"
], ]
"execution_count": null,
"outputs": []
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"originalKey": "0b0c2087-4c86-4c57-b0ee-6f48a70a9c78", "code_folding": [],
"showInput": true,
"customInput": null,
"collapsed": false, "collapsed": false,
"requestMsgId": "46883aad-f00b-4fd4-ac17-eec0b2ac272a", "customInput": null,
"customOutput": null, "customOutput": null,
"executionStartTime": 1659610930042, "executionStartTime": 1659610930042,
"executionStopTime": 1659610930050, "executionStopTime": 1659610930050,
"code_folding": [], "hidden_ranges": [],
"hidden_ranges": [] "originalKey": "0b0c2087-4c86-4c57-b0ee-6f48a70a9c78",
"requestMsgId": "46883aad-f00b-4fd4-ac17-eec0b2ac272a",
"showInput": true
}, },
"outputs": [],
"source": [ "source": [
"torch.set_printoptions(sci_mode=False)" "torch.set_printoptions(sci_mode=False)"
], ]
"execution_count": null,
"outputs": []
}, },
{ {
"attachments": {},
"cell_type": "markdown", "cell_type": "markdown",
"metadata": { "metadata": {
"customInput": null,
"originalKey": "37809d0d-b02e-42df-85b6-cdd038373653", "originalKey": "37809d0d-b02e-42df-85b6-cdd038373653",
"showInput": false, "showInput": false
"customInput": null
}, },
"source": [ "source": [
"## 1. Load renders of a mesh (the cow mesh) as a dataset\n", "## 1. Load renders of a mesh (the cow mesh) as a dataset\n",
...@@ -250,50 +208,49 @@ ...@@ -250,50 +208,49 @@
"A dataset's train, val and test parts in Implicitron are represented as a `dataset_map`, and provided by an implementation of `DatasetMapProvider`. \n", "A dataset's train, val and test parts in Implicitron are represented as a `dataset_map`, and provided by an implementation of `DatasetMapProvider`. \n",
"`RenderedMeshDatasetMapProvider` is one which generates a single-scene dataset with only a train component by taking a mesh and rendering it.\n", "`RenderedMeshDatasetMapProvider` is one which generates a single-scene dataset with only a train component by taking a mesh and rendering it.\n",
"We use it with the cow mesh." "We use it with the cow mesh."
], ]
"attachments": {}
}, },
{ {
"attachments": {},
"cell_type": "markdown", "cell_type": "markdown",
"metadata": { "metadata": {
"originalKey": "cc68cb9c-b8bf-4e9e-bef1-2cfafdf6caa2",
"showInput": false,
"customInput": null,
"collapsed": false, "collapsed": false,
"requestMsgId": "398cfcae-5d43-4b6f-9c75-db3d297364d4", "customInput": null,
"customOutput": null, "customOutput": null,
"executionStartTime": 1659620739780, "executionStartTime": 1659620739780,
"executionStopTime": 1659620739914 "executionStopTime": 1659620739914,
"originalKey": "cc68cb9c-b8bf-4e9e-bef1-2cfafdf6caa2",
"requestMsgId": "398cfcae-5d43-4b6f-9c75-db3d297364d4",
"showInput": false
}, },
"source": [ "source": [
"If running this notebook using **Google Colab**, run the following cell to fetch the mesh obj and texture files and save it at the path data/cow_mesh.\n", "If running this notebook using **Google Colab**, run the following cell to fetch the mesh obj and texture files and save it at the path data/cow_mesh.\n",
"If running locally, the data is already available at the correct path." "If running locally, the data is already available at the correct path."
], ]
"attachments": {}
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"originalKey": "2c55e002-a885-4169-8fdc-af9078b05968",
"showInput": true,
"customInput": null, "customInput": null,
"customOutput": null "customOutput": null,
"originalKey": "2c55e002-a885-4169-8fdc-af9078b05968",
"showInput": true
}, },
"outputs": [],
"source": [ "source": [
"!mkdir -p data/cow_mesh\n", "!mkdir -p data/cow_mesh\n",
"!wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow.obj\n", "!wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow.obj\n",
"!wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow.mtl\n", "!wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow.mtl\n",
"!wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow_texture.png" "!wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow_texture.png"
], ]
"execution_count": null,
"outputs": []
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
"metadata": { "metadata": {
"customInput": null,
"originalKey": "2a976be8-01bf-4a1c-a6e7-61d5d08c3dbd", "originalKey": "2a976be8-01bf-4a1c-a6e7-61d5d08c3dbd",
"showInput": false, "showInput": false
"customInput": null
}, },
"source": [ "source": [
"If we want to instantiate one of Implicitron's configurable objects, such as `RenderedMeshDatasetMapProvider`, without using the OmegaConf initialisation (get_default_args), we need to call `expand_args_fields` on the class first." "If we want to instantiate one of Implicitron's configurable objects, such as `RenderedMeshDatasetMapProvider`, without using the OmegaConf initialisation (get_default_args), we need to call `expand_args_fields` on the class first."
...@@ -301,17 +258,19 @@ ...@@ -301,17 +258,19 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"originalKey": "eb77aaec-048c-40bd-bd69-0e66b6ab60b1", "code_folding": [],
"showInput": true,
"collapsed": false, "collapsed": false,
"requestMsgId": "09b9975c-ff86-41c9-b4a9-975d23afc562",
"customOutput": null, "customOutput": null,
"executionStartTime": 1659621652237, "executionStartTime": 1659621652237,
"executionStopTime": 1659621652903, "executionStopTime": 1659621652903,
"code_folding": [], "hidden_ranges": [],
"hidden_ranges": [] "originalKey": "eb77aaec-048c-40bd-bd69-0e66b6ab60b1",
"requestMsgId": "09b9975c-ff86-41c9-b4a9-975d23afc562",
"showInput": true
}, },
"outputs": [],
"source": [ "source": [
"expand_args_fields(RenderedMeshDatasetMapProvider)\n", "expand_args_fields(RenderedMeshDatasetMapProvider)\n",
"cow_provider = RenderedMeshDatasetMapProvider(\n", "cow_provider = RenderedMeshDatasetMapProvider(\n",
...@@ -319,83 +278,82 @@ ...@@ -319,83 +278,82 @@
" use_point_light=False,\n", " use_point_light=False,\n",
" resolution=output_resolution,\n", " resolution=output_resolution,\n",
")" ")"
], ]
"execution_count": null,
"outputs": []
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"originalKey": "8210e15b-da48-4306-a49a-41c4e7e7d42f", "code_folding": [],
"showInput": true,
"customInput": null,
"collapsed": false, "collapsed": false,
"requestMsgId": "c243edd2-a106-4fba-8471-dfa4f99a2088", "customInput": null,
"customOutput": null, "customOutput": null,
"executionStartTime": 1659610966145, "executionStartTime": 1659610966145,
"executionStopTime": 1659610966255, "executionStopTime": 1659610966255,
"code_folding": [], "hidden_ranges": [],
"hidden_ranges": [] "originalKey": "8210e15b-da48-4306-a49a-41c4e7e7d42f",
"requestMsgId": "c243edd2-a106-4fba-8471-dfa4f99a2088",
"showInput": true
}, },
"outputs": [],
"source": [ "source": [
"dataset_map = cow_provider.get_dataset_map()\n", "dataset_map = cow_provider.get_dataset_map()\n",
"tr_cameras = [training_frame.camera for training_frame in dataset_map.train]" "tr_cameras = [training_frame.camera for training_frame in dataset_map.train]"
], ]
"execution_count": null,
"outputs": []
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"originalKey": "458d72ad-d9a7-4f13-b5b7-90d2aec61c16", "code_folding": [],
"showInput": true,
"customInput": null,
"collapsed": false, "collapsed": false,
"requestMsgId": "7f9431f3-8717-4d89-a7fe-1420dd0e00c4", "customInput": null,
"customOutput": null, "customOutput": null,
"executionStartTime": 1659610967703, "executionStartTime": 1659610967703,
"executionStopTime": 1659610967848, "executionStopTime": 1659610967848,
"code_folding": [], "hidden_ranges": [],
"hidden_ranges": [] "originalKey": "458d72ad-d9a7-4f13-b5b7-90d2aec61c16",
"requestMsgId": "7f9431f3-8717-4d89-a7fe-1420dd0e00c4",
"showInput": true
}, },
"outputs": [],
"source": [ "source": [
"# The cameras are all in the XZ plane, in a circle about 2.7 from the origin\n", "# The cameras are all in the XZ plane, in a circle about 2.7 from the origin\n",
"centers = torch.cat([i.get_camera_center() for i in tr_cameras])\n", "centers = torch.cat([i.get_camera_center() for i in tr_cameras])\n",
"print(centers.min(0).values)\n", "print(centers.min(0).values)\n",
"print(centers.max(0).values)" "print(centers.max(0).values)"
], ]
"execution_count": null,
"outputs": []
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"originalKey": "931e712b-b141-437a-97fb-dc2a07ce3458", "code_folding": [],
"showInput": true,
"customInput": null,
"collapsed": false, "collapsed": false,
"requestMsgId": "931e712b-b141-437a-97fb-dc2a07ce3458", "customInput": null,
"customOutput": null, "customOutput": null,
"executionStartTime": 1659552920194, "executionStartTime": 1659552920194,
"executionStopTime": 1659552923122, "executionStopTime": 1659552923122,
"code_folding": [], "hidden_ranges": [],
"hidden_ranges": [] "originalKey": "931e712b-b141-437a-97fb-dc2a07ce3458",
"requestMsgId": "931e712b-b141-437a-97fb-dc2a07ce3458",
"showInput": true
}, },
"outputs": [],
"source": [ "source": [
"# visualization of the cameras\n", "# visualization of the cameras\n",
"plot = plot_scene({\"k\": {i: camera for i, camera in enumerate(tr_cameras)}}, camera_scale=0.25)\n", "plot = plot_scene({\"k\": {i: camera for i, camera in enumerate(tr_cameras)}}, camera_scale=0.25)\n",
"plot.layout.scene.aspectmode = \"data\"\n", "plot.layout.scene.aspectmode = \"data\"\n",
"plot" "plot"
], ]
"execution_count": null,
"outputs": []
}, },
{ {
"attachments": {},
"cell_type": "markdown", "cell_type": "markdown",
"metadata": { "metadata": {
"customInput": null,
"originalKey": "afa9c02d-f76b-4f68-83e9-9733c615406b", "originalKey": "afa9c02d-f76b-4f68-83e9-9733c615406b",
"showInput": false, "showInput": false
"customInput": null
}, },
"source": [ "source": [
"## 2. Custom implicit function 🧊\n", "## 2. Custom implicit function 🧊\n",
...@@ -409,23 +367,24 @@ ...@@ -409,23 +367,24 @@
"We do this by subclassing `ImplicitFunctionBase`.\n", "We do this by subclassing `ImplicitFunctionBase`.\n",
"We need to register our subclass with a special decorator.\n", "We need to register our subclass with a special decorator.\n",
"We use Python's dataclass annotations for configuring the module." "We use Python's dataclass annotations for configuring the module."
], ]
"attachments": {}
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"originalKey": "61b55043-dc52-4de7-992e-e2195edd2123", "code_folding": [],
"showInput": true,
"customInput": null,
"collapsed": false, "collapsed": false,
"requestMsgId": "dfaace3c-098c-4ffe-9240-6a7ae0ff271e", "customInput": null,
"customOutput": null, "customOutput": null,
"executionStartTime": 1659613575850, "executionStartTime": 1659613575850,
"executionStopTime": 1659613575940, "executionStopTime": 1659613575940,
"code_folding": [], "hidden_ranges": [],
"hidden_ranges": [] "originalKey": "61b55043-dc52-4de7-992e-e2195edd2123",
"requestMsgId": "dfaace3c-098c-4ffe-9240-6a7ae0ff271e",
"showInput": true
}, },
"outputs": [],
"source": [ "source": [
"@registry.register\n", "@registry.register\n",
"class MyVolumes(ImplicitFunctionBase, torch.nn.Module):\n", "class MyVolumes(ImplicitFunctionBase, torch.nn.Module):\n",
...@@ -463,39 +422,38 @@ ...@@ -463,39 +422,38 @@
"\n", "\n",
" # When an implicit function is used for raymarching, i.e. for MultiPassEmissionAbsorptionRenderer,\n", " # When an implicit function is used for raymarching, i.e. for MultiPassEmissionAbsorptionRenderer,\n",
" # it must return (densities, features, an auxiliary tuple)\n", " # it must return (densities, features, an auxiliary tuple)\n",
" return densities, features, {}\n", " return densities, features, {}\n"
"" ]
],
"execution_count": null,
"outputs": []
}, },
{ {
"attachments": {},
"cell_type": "markdown", "cell_type": "markdown",
"metadata": { "metadata": {
"customInput": null,
"originalKey": "abaf2cd6-1b68-400e-a142-8fb9f49953f3", "originalKey": "abaf2cd6-1b68-400e-a142-8fb9f49953f3",
"showInput": false, "showInput": false
"customInput": null
}, },
"source": [ "source": [
"## 3. Construct the model object.\n", "## 3. Construct the model object.\n",
"\n", "\n",
"The main model object in PyTorch3D is `GenericModel`, which has pluggable components for the major steps, including the renderer and the implicit function(s).\n", "The main model object in PyTorch3D is `GenericModel`, which has pluggable components for the major steps, including the renderer and the implicit function(s).\n",
"There are two ways to construct it which are equivalent here." "There are two ways to construct it which are equivalent here."
], ]
"attachments": {}
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"originalKey": "f26c3dce-fbae-4592-bd0e-e4a8abc57c2c",
"showInput": true,
"customInput": null,
"collapsed": false, "collapsed": false,
"requestMsgId": "9213687e-1caf-46a8-a4e5-a9c531530092", "customInput": null,
"customOutput": null, "customOutput": null,
"executionStartTime": 1659621267561, "executionStartTime": 1659621267561,
"executionStopTime": 1659621267938 "executionStopTime": 1659621267938,
"originalKey": "f26c3dce-fbae-4592-bd0e-e4a8abc57c2c",
"requestMsgId": "9213687e-1caf-46a8-a4e5-a9c531530092",
"showInput": true
}, },
"outputs": [],
"source": [ "source": [
"CONSTRUCT_MODEL_FROM_CONFIG = True\n", "CONSTRUCT_MODEL_FROM_CONFIG = True\n",
"if CONSTRUCT_MODEL_FROM_CONFIG:\n", "if CONSTRUCT_MODEL_FROM_CONFIG:\n",
...@@ -521,82 +479,79 @@ ...@@ -521,82 +479,79 @@
" )\n", " )\n",
"\n", "\n",
" # In this case we can get the equivalent DictConfig cfg object to the way gm is configured as follows\n", " # In this case we can get the equivalent DictConfig cfg object to the way gm is configured as follows\n",
" cfg = OmegaConf.structured(gm)\n", " cfg = OmegaConf.structured(gm)\n"
"" ]
],
"execution_count": null,
"outputs": []
}, },
{ {
"attachments": {},
"cell_type": "markdown", "cell_type": "markdown",
"metadata": { "metadata": {
"originalKey": "4e659f7d-ce66-4999-83de-005eb09d7705", "code_folding": [],
"showInput": false,
"customInput": null,
"collapsed": false, "collapsed": false,
"requestMsgId": "7b815b2b-cf19-44d0-ae89-76fde6df35ec", "customInput": null,
"customOutput": null, "customOutput": null,
"executionStartTime": 1659611214689, "executionStartTime": 1659611214689,
"executionStopTime": 1659611214748, "executionStopTime": 1659611214748,
"code_folding": [], "hidden_ranges": [],
"hidden_ranges": [] "originalKey": "4e659f7d-ce66-4999-83de-005eb09d7705",
"requestMsgId": "7b815b2b-cf19-44d0-ae89-76fde6df35ec",
"showInput": false
}, },
"source": [ "source": [
" The default renderer is an emission-absorbtion raymarcher. We keep that default." " The default renderer is an emission-absorbtion raymarcher. We keep that default."
], ]
"attachments": {}
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"originalKey": "d37ae488-c57c-44d3-9def-825dc1a6495b", "code_folding": [],
"showInput": true,
"customInput": null,
"collapsed": false, "collapsed": false,
"requestMsgId": "71143ec1-730f-4876-8a14-e46eea9d6dd1", "customInput": null,
"customOutput": null, "customOutput": null,
"executionStartTime": 1659621268007, "executionStartTime": 1659621268007,
"executionStopTime": 1659621268190, "executionStopTime": 1659621268190,
"code_folding": [], "hidden_ranges": [],
"hidden_ranges": [] "originalKey": "d37ae488-c57c-44d3-9def-825dc1a6495b",
"requestMsgId": "71143ec1-730f-4876-8a14-e46eea9d6dd1",
"showInput": true
}, },
"outputs": [],
"source": [ "source": [
"# We can display the configuration in use as follows.\n", "# We can display the configuration in use as follows.\n",
"remove_unused_components(cfg)\n", "remove_unused_components(cfg)\n",
"yaml = OmegaConf.to_yaml(cfg, sort_keys=False)\n", "yaml = OmegaConf.to_yaml(cfg, sort_keys=False)\n",
"%page -r yaml" "%page -r yaml"
], ]
"execution_count": null,
"outputs": []
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"originalKey": "52e53179-3c6e-4c1f-a38a-3a6d803687bb", "code_folding": [],
"showInput": true,
"customInput": null,
"collapsed": false, "collapsed": false,
"requestMsgId": "05de9bc3-3f74-4a6f-851c-9ec919b59506", "customInput": null,
"customOutput": null, "customOutput": null,
"executionStartTime": 1659621268727, "executionStartTime": 1659621268727,
"executionStopTime": 1659621268776, "executionStopTime": 1659621268776,
"code_folding": [], "hidden_ranges": [],
"hidden_ranges": [] "originalKey": "52e53179-3c6e-4c1f-a38a-3a6d803687bb",
"requestMsgId": "05de9bc3-3f74-4a6f-851c-9ec919b59506",
"showInput": true
}, },
"outputs": [],
"source": [ "source": [
"device = torch.device(\"cuda:0\")\n", "device = torch.device(\"cuda:0\")\n",
"gm.to(device)\n", "gm.to(device)\n",
"assert next(gm.parameters()).is_cuda" "assert next(gm.parameters()).is_cuda"
], ]
"execution_count": null,
"outputs": []
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
"metadata": { "metadata": {
"customInput": null,
"originalKey": "528a7d53-c645-49c2-9021-09adbb18cd23", "originalKey": "528a7d53-c645-49c2-9021-09adbb18cd23",
"showInput": false, "showInput": false
"customInput": null
}, },
"source": [ "source": [
"## 4. train the model " "## 4. train the model "
...@@ -604,58 +559,60 @@ ...@@ -604,58 +559,60 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"originalKey": "953280bd-3161-42ba-8dcb-0c8ef2d5cc25", "code_folding": [],
"showInput": true,
"customInput": null,
"collapsed": false, "collapsed": false,
"requestMsgId": "9bba424b-7bfd-4e5a-9d79-ae316e20bab0", "customInput": null,
"customOutput": null, "customOutput": null,
"executionStartTime": 1659621270236, "executionStartTime": 1659621270236,
"executionStopTime": 1659621270446, "executionStopTime": 1659621270446,
"code_folding": [], "hidden_ranges": [],
"hidden_ranges": [] "originalKey": "953280bd-3161-42ba-8dcb-0c8ef2d5cc25",
"requestMsgId": "9bba424b-7bfd-4e5a-9d79-ae316e20bab0",
"showInput": true
}, },
"outputs": [],
"source": [ "source": [
"train_data_collated = [FrameData.collate([frame.to(device)]) for frame in dataset_map.train]" "train_data_collated = [FrameData.collate([frame.to(device)]) for frame in dataset_map.train]"
], ]
"execution_count": null,
"outputs": []
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"originalKey": "2fcf07f0-0c28-49c7-8c76-1c9a9d810167", "code_folding": [],
"showInput": true,
"customInput": null,
"collapsed": false, "collapsed": false,
"requestMsgId": "821deb43-6084-4ece-83c3-dee214562c47", "customInput": null,
"customOutput": null, "customOutput": null,
"executionStartTime": 1659621270815, "executionStartTime": 1659621270815,
"executionStopTime": 1659621270948, "executionStopTime": 1659621270948,
"code_folding": [], "hidden_ranges": [],
"hidden_ranges": [] "originalKey": "2fcf07f0-0c28-49c7-8c76-1c9a9d810167",
"requestMsgId": "821deb43-6084-4ece-83c3-dee214562c47",
"showInput": true
}, },
"outputs": [],
"source": [ "source": [
"gm.train()\n", "gm.train()\n",
"optimizer = torch.optim.Adam(gm.parameters(), lr=0.1)" "optimizer = torch.optim.Adam(gm.parameters(), lr=0.1)"
], ]
"execution_count": null,
"outputs": []
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"originalKey": "105099f7-ed0c-4e7f-a976-61a93fd0a8fe", "code_folding": [],
"showInput": true,
"collapsed": false, "collapsed": false,
"requestMsgId": "0c87c108-83e3-4129-ad02-85e0140f1368",
"customOutput": null, "customOutput": null,
"executionStartTime": 1659621271875, "executionStartTime": 1659621271875,
"executionStopTime": 1659621298146, "executionStopTime": 1659621298146,
"code_folding": [], "hidden_ranges": [],
"hidden_ranges": [] "originalKey": "105099f7-ed0c-4e7f-a976-61a93fd0a8fe",
"requestMsgId": "0c87c108-83e3-4129-ad02-85e0140f1368",
"showInput": true
}, },
"outputs": [],
"source": [ "source": [
"iterator = tqdm.tqdm(range(2000))\n", "iterator = tqdm.tqdm(range(2000))\n",
"for n_batch in iterator:\n", "for n_batch in iterator:\n",
...@@ -667,43 +624,43 @@ ...@@ -667,43 +624,43 @@
" if n_batch % 100 == 0:\n", " if n_batch % 100 == 0:\n",
" iterator.set_postfix_str(f\"loss: {float(out['objective']):.5f}\")\n", " iterator.set_postfix_str(f\"loss: {float(out['objective']):.5f}\")\n",
" optimizer.step()" " optimizer.step()"
], ]
"execution_count": null,
"outputs": []
}, },
{ {
"attachments": {},
"cell_type": "markdown", "cell_type": "markdown",
"metadata": { "metadata": {
"originalKey": "e3cd494a-536b-48bc-8290-c048118c82eb",
"showInput": false,
"customInput": null,
"collapsed": false, "collapsed": false,
"requestMsgId": "e3cd494a-536b-48bc-8290-c048118c82eb", "customInput": null,
"customOutput": null, "customOutput": null,
"executionStartTime": 1659535024768, "executionStartTime": 1659535024768,
"executionStopTime": 1659535024906 "executionStopTime": 1659535024906,
"originalKey": "e3cd494a-536b-48bc-8290-c048118c82eb",
"requestMsgId": "e3cd494a-536b-48bc-8290-c048118c82eb",
"showInput": false
}, },
"source": [ "source": [
"## 5. Evaluate the module\n", "## 5. Evaluate the module\n",
"\n", "\n",
"We generate complete images from all the viewpoints to see how they look." "We generate complete images from all the viewpoints to see how they look."
], ]
"attachments": {}
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"originalKey": "fbe1b2ea-cc24-4b20-a2d7-0249185e34a5", "code_folding": [],
"showInput": true,
"customInput": null,
"collapsed": false, "collapsed": false,
"requestMsgId": "771ef1f8-5eee-4932-9e81-33604bf0512a", "customInput": null,
"customOutput": null, "customOutput": null,
"executionStartTime": 1659621299859, "executionStartTime": 1659621299859,
"executionStopTime": 1659621311133, "executionStopTime": 1659621311133,
"code_folding": [], "hidden_ranges": [],
"hidden_ranges": [] "originalKey": "fbe1b2ea-cc24-4b20-a2d7-0249185e34a5",
"requestMsgId": "771ef1f8-5eee-4932-9e81-33604bf0512a",
"showInput": true
}, },
"outputs": [],
"source": [ "source": [
"def to_numpy_image(image):\n", "def to_numpy_image(image):\n",
" # Takes an image of shape (C, H, W) in [0,1], where C=3 or 1\n", " # Takes an image of shape (C, H, W) in [0,1], where C=3 or 1\n",
...@@ -731,21 +688,20 @@ ...@@ -731,21 +688,20 @@
" masks.append(mask)\n", " masks.append(mask)\n",
" expected.append(expd)\n", " expected.append(expd)\n",
" masks_expected.append(mask_expected)" " masks_expected.append(mask_expected)"
], ]
"execution_count": null,
"outputs": []
}, },
{ {
"attachments": {},
"cell_type": "markdown", "cell_type": "markdown",
"metadata": { "metadata": {
"originalKey": "24953039-9780-40fd-bd81-5d63e9f40069",
"showInput": false,
"customInput": null,
"collapsed": false, "collapsed": false,
"requestMsgId": "7af895a3-dfe4-4c28-ac3b-4ff0fbb40c7f", "customInput": null,
"customOutput": null, "customOutput": null,
"executionStartTime": 1659614622542, "executionStartTime": 1659614622542,
"executionStopTime": 1659614622757 "executionStopTime": 1659614622757,
"originalKey": "24953039-9780-40fd-bd81-5d63e9f40069",
"requestMsgId": "7af895a3-dfe4-4c28-ac3b-4ff0fbb40c7f",
"showInput": false
}, },
"source": [ "source": [
"We draw a grid showing predicted image and expected image, followed by predicted mask and expected mask, from each viewpoint. \n", "We draw a grid showing predicted image and expected image, followed by predicted mask and expected mask, from each viewpoint. \n",
...@@ -788,23 +744,24 @@ ...@@ -788,23 +744,24 @@
" ...\n", " ...\n",
"```\n", "```\n",
"</center></small>" "</center></small>"
], ]
"attachments": {}
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"originalKey": "c488a34a-e46d-4649-93fb-4b1bb5a0e439", "code_folding": [],
"showInput": true,
"customInput": null,
"collapsed": false, "collapsed": false,
"requestMsgId": "4221e632-fca1-4fe5-b2e3-f92c37aa40e4", "customInput": null,
"customOutput": null, "customOutput": null,
"executionStartTime": 1659621313894, "executionStartTime": 1659621313894,
"executionStopTime": 1659621314042, "executionStopTime": 1659621314042,
"code_folding": [], "hidden_ranges": [],
"hidden_ranges": [] "originalKey": "c488a34a-e46d-4649-93fb-4b1bb5a0e439",
"requestMsgId": "4221e632-fca1-4fe5-b2e3-f92c37aa40e4",
"showInput": true
}, },
"outputs": [],
"source": [ "source": [
"images_to_display = [images.copy(), expected.copy(), masks.copy(), masks_expected.copy()]\n", "images_to_display = [images.copy(), expected.copy(), masks.copy(), masks_expected.copy()]\n",
"n_rows = 4\n", "n_rows = 4\n",
...@@ -821,47 +778,46 @@ ...@@ -821,47 +778,46 @@
" for group in images_to_display_listed:\n", " for group in images_to_display_listed:\n",
" split.append(group[row*n_per_row:(row+1)*n_per_row]) \n", " split.append(group[row*n_per_row:(row+1)*n_per_row]) \n",
"\n", "\n",
"Image.fromarray(np.block(split))\n", "Image.fromarray(np.block(split))\n"
"" ]
],
"execution_count": null,
"outputs": []
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"originalKey": "49eab9e1-4fe2-4fbe-b4f3-7b6953340170", "code_folding": [],
"showInput": true,
"customInput": null,
"collapsed": false, "collapsed": false,
"requestMsgId": "85b402ad-f903-431f-a13e-c2d697e869bb", "customInput": null,
"customOutput": null, "customOutput": null,
"executionStartTime": 1659621323795, "executionStartTime": 1659621323795,
"executionStopTime": 1659621323820, "executionStopTime": 1659621323820,
"code_folding": [], "hidden_ranges": [],
"hidden_ranges": [] "originalKey": "49eab9e1-4fe2-4fbe-b4f3-7b6953340170",
"requestMsgId": "85b402ad-f903-431f-a13e-c2d697e869bb",
"showInput": true
}, },
"outputs": [],
"source": [ "source": [
"# Print the maximum channel intensity in the first image.\n", "# Print the maximum channel intensity in the first image.\n",
"print(images[1].max()/255)" "print(images[1].max()/255)"
], ]
"execution_count": null,
"outputs": []
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"originalKey": "137d2c43-d39d-4266-ac5e-2b714da5e0ee", "code_folding": [],
"showInput": true, "collapsed": false,
"customInput": null, "customInput": null,
"customOutput": null, "customOutput": null,
"code_folding": [], "executionStartTime": 1659621408642,
"executionStopTime": 1659621409559,
"hidden_ranges": [], "hidden_ranges": [],
"collapsed": false, "originalKey": "137d2c43-d39d-4266-ac5e-2b714da5e0ee",
"requestMsgId": "8e27ec57-c2d6-4ae0-be69-b63b6af929ff", "requestMsgId": "8e27ec57-c2d6-4ae0-be69-b63b6af929ff",
"executionStartTime": 1659621408642, "showInput": true
"executionStopTime": 1659621409559
}, },
"outputs": [],
"source": [ "source": [
"plt.ioff()\n", "plt.ioff()\n",
"fig, ax = plt.subplots(figsize=(3,3))\n", "fig, ax = plt.subplots(figsize=(3,3))\n",
...@@ -869,51 +825,89 @@ ...@@ -869,51 +825,89 @@
"ax.grid(None)\n", "ax.grid(None)\n",
"ims = [[ax.imshow(im, animated=True)] for im in images]\n", "ims = [[ax.imshow(im, animated=True)] for im in images]\n",
"ani = animation.ArtistAnimation(fig, ims, interval=80, blit=True)\n", "ani = animation.ArtistAnimation(fig, ims, interval=80, blit=True)\n",
"ani_html = ani.to_jshtml()\n", "ani_html = ani.to_jshtml()\n"
"" ]
],
"execution_count": null,
"outputs": []
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"originalKey": "783e70d6-7cf1-4d76-a126-ba11ffc2f5be",
"showInput": true,
"customInput": null,
"collapsed": false, "collapsed": false,
"requestMsgId": "b6843506-c5fa-4508-80fc-8ecae51a934a", "customInput": null,
"customOutput": null, "customOutput": null,
"executionStartTime": 1659621409620, "executionStartTime": 1659621409620,
"executionStopTime": 1659621409725 "executionStopTime": 1659621409725,
"originalKey": "783e70d6-7cf1-4d76-a126-ba11ffc2f5be",
"requestMsgId": "b6843506-c5fa-4508-80fc-8ecae51a934a",
"showInput": true
}, },
"outputs": [],
"source": [ "source": [
"HTML(ani_html)" "HTML(ani_html)"
], ]
"execution_count": null,
"outputs": []
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null,
"metadata": { "metadata": {
"originalKey": "0286c350-2362-4f47-8181-2fc2ba51cfcf",
"showInput": true,
"customInput": null,
"collapsed": false, "collapsed": false,
"requestMsgId": "976f4db9-d4c7-466c-bcfd-218234400226", "customInput": null,
"customOutput": null, "customOutput": null,
"executionStartTime": 1659614670081, "executionStartTime": 1659614670081,
"executionStopTime": 1659614670168 "executionStopTime": 1659614670168,
"originalKey": "0286c350-2362-4f47-8181-2fc2ba51cfcf",
"requestMsgId": "976f4db9-d4c7-466c-bcfd-218234400226",
"showInput": true
}, },
"outputs": [],
"source": [ "source": [
"# If you want to see the output of the model with the volume forced to opaque white, run this and re-evaluate\n", "# If you want to see the output of the model with the volume forced to opaque white, run this and re-evaluate\n",
"# with torch.no_grad():\n", "# with torch.no_grad():\n",
"# gm._implicit_functions[0]._fn.density.fill_(9.0)\n", "# gm._implicit_functions[0]._fn.density.fill_(9.0)\n",
"# gm._implicit_functions[0]._fn.color.fill_(9.0)\n", "# gm._implicit_functions[0]._fn.color.fill_(9.0)\n"
"" ]
],
"execution_count": null,
"outputs": []
} }
] ],
"metadata": {
"bento_stylesheets": {
"bento/extensions/flow/main.css": true,
"bento/extensions/kernel_selector/main.css": true,
"bento/extensions/kernel_ui/main.css": true,
"bento/extensions/new_kernel/main.css": true,
"bento/extensions/system_usage/main.css": true,
"bento/extensions/theme/main.css": true
},
"captumWidgetMessage": {},
"dataExplorerConfig": {},
"kernelspec": {
"display_name": "pytorch3d",
"language": "python",
"metadata": {
"cinder_runtime": false,
"fbpkg_supported": true,
"is_prebuilt": true,
"kernel_name": "bento_kernel_pytorch3d",
"nightly_builds": true
},
"name": "bento_kernel_pytorch3d"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3"
},
"last_base_url": "https://9177.od.fbinfra.net:443/",
"last_kernel_id": "bb33cd83-7924-489a-8bd8-2d9d62eb0126",
"last_msg_id": "99f7088e-d22b355b859660479ef0574e_5743",
"last_server_session_id": "2944b203-9ea8-4c0e-9634-645dfea5f26b",
"outputWidgetContext": {}
},
"nbformat": 4,
"nbformat_minor": 2
} }
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment