Skip to content

GitLab

Unverified Commit c9eb3554 authored by vivek rathod's avatar vivek rathod Committed by GitHub
Browse files

Merged commit includes the following changes: (#8829) 



320618558  by rathodv:

    Internal Change.

--
320597532  by ronnyvotel:

    Exposing DensePose visualizations to model_lib_v2.py.

--
320533669  by ronnyvotel:

    Adding utilities to visualize DensePose outputs.

--
320529647  by lzc:

    Fix saved_model issue in object_detection_tutorial notebook.

--
320510127  by aom:

    Internal change.

--
320490236  by derekjchow:

    Update Dockerfiles to use setup.py

--
320443572  by rathodv:

    Update `Tensorflow` to `TensorFlow` in documentation.

--
320426460  by ronnyvotel:

    DensePose proto and model builder.

--
320352611  by rathodv:

    Update documentation to reflect the support for TF1 and TF2. Provide separate sets of instructions to reduce confusion.

--
320350724  by rathodv:

    Internal Change.

--

PiperOrigin-RevId: 320618558
Co-authored-by: default avatarkmindspark <kaushikshiv@google.com>
parent 950ebc45
Hide whitespace changes
Inline Side-by-side
Showing with 600 additions and 332 deletions
research/object_detection/CONTRIBUTING.md
View file @ c9eb3554
# Contributing to the Tensorflow Object Detection API
# Contributing to the TensorFlow Object Detection API
Patches to Tensorflow Object Detection API are welcome!
Patches to TensorFlow Object Detection API are welcome!
We require contributors to fill out either the individual or corporate
Contributor License Agreement (CLA).
......@@ -9,5 +9,5 @@ Contributor License Agreement (CLA).
* If you work for a company that wants to allow you to contribute your work, then you'll need to sign a [corporate CLA](http://code.google.com/legal/corporate-cla-v1.0.html).
Please follow the
[Tensorflow contributing guidelines](https://github.com/tensorflow/tensorflow/blob/master/CONTRIBUTING.md)
[TensorFlow contributing guidelines](https://github.com/tensorflow/tensorflow/blob/master/CONTRIBUTING.md)
when submitting pull requests.
research/object_detection/builders/model_builder.py
View file @ c9eb3554
......@@ -870,6 +870,22 @@ def mask_proto_to_params(mask_config):
heatmap_bias_init=mask_config.heatmap_bias_init)
def densepose_proto_to_params(densepose_config):
"""Converts CenterNet.DensePoseEstimation proto to parameter namedtuple."""
classification_loss, localization_loss, _, _, _, _, _ = (
losses_builder.build(densepose_config.loss))
return center_net_meta_arch.DensePoseParams(
class_id=densepose_config.class_id,
classification_loss=classification_loss,
localization_loss=localization_loss,
part_loss_weight=densepose_config.part_loss_weight,
coordinate_loss_weight=densepose_config.coordinate_loss_weight,
num_parts=densepose_config.num_parts,
task_loss_weight=densepose_config.task_loss_weight,
upsample_to_input_res=densepose_config.upsample_to_input_res,
heatmap_bias_init=densepose_config.heatmap_bias_init)
def _build_center_net_model(center_net_config, is_training, add_summaries):
"""Build a CenterNet detection model.
......@@ -922,6 +938,11 @@ def _build_center_net_model(center_net_config, is_training, add_summaries):
if center_net_config.HasField('mask_estimation_task'):
mask_params = mask_proto_to_params(center_net_config.mask_estimation_task)
densepose_params = None
if center_net_config.HasField('densepose_estimation_task'):
densepose_params = densepose_proto_to_params(
center_net_config.densepose_estimation_task)
return center_net_meta_arch.CenterNetMetaArch(
is_training=is_training,
add_summaries=add_summaries,
......@@ -931,7 +952,8 @@ def _build_center_net_model(center_net_config, is_training, add_summaries):
object_center_params=object_center_params,
object_detection_params=object_detection_params,
keypoint_params_dict=keypoint_params_dict,
mask_params=mask_params)
mask_params=mask_params,
densepose_params=densepose_params)
def _build_center_net_feature_extractor(
......
research/object_detection/builders/model_builder_tf2_test.py
View file @ c9eb3554
......@@ -164,6 +164,28 @@ class ModelBuilderTF2Test(model_builder_test.ModelBuilderTest):
return text_format.Merge(proto_txt,
center_net_pb2.CenterNet.MaskEstimation())
def get_fake_densepose_proto(self):
proto_txt = """
task_loss_weight: 0.5
class_id: 0
loss {
classification_loss {
weighted_softmax {}
}
localization_loss {
l1_localization_loss {
}
}
}
num_parts: 24
part_loss_weight: 1.0
coordinate_loss_weight: 2.0
upsample_to_input_res: true
heatmap_bias_init: -2.0
"""
return text_format.Merge(proto_txt,
center_net_pb2.CenterNet.DensePoseEstimation())
def test_create_center_net_model(self):
"""Test building a CenterNet model from proto txt."""
proto_txt = """
......@@ -195,6 +217,8 @@ class ModelBuilderTF2Test(model_builder_test.ModelBuilderTest):
self.get_fake_label_map_file_path())
config.center_net.mask_estimation_task.CopyFrom(
self.get_fake_mask_proto())
config.center_net.densepose_estimation_task.CopyFrom(
self.get_fake_densepose_proto())
# Build the model from the configuration.
model = model_builder.build(config, is_training=True)
......@@ -251,6 +275,21 @@ class ModelBuilderTF2Test(model_builder_test.ModelBuilderTest):
self.assertAlmostEqual(
model._mask_params.heatmap_bias_init, -2.0, places=4)
# Check DensePose related parameters.
self.assertEqual(model._densepose_params.class_id, 0)
self.assertIsInstance(model._densepose_params.classification_loss,
losses.WeightedSoftmaxClassificationLoss)
self.assertIsInstance(model._densepose_params.localization_loss,
losses.L1LocalizationLoss)
self.assertAlmostEqual(model._densepose_params.part_loss_weight, 1.0)
self.assertAlmostEqual(model._densepose_params.coordinate_loss_weight, 2.0)
self.assertEqual(model._densepose_params.num_parts, 24)
self.assertAlmostEqual(model._densepose_params.task_loss_weight, 0.5)
self.assertTrue(model._densepose_params.upsample_to_input_res)
self.assertEqual(model._densepose_params.upsample_method, 'bilinear')
self.assertAlmostEqual(
model._densepose_params.heatmap_bias_init, -2.0, places=4)
# Check feature extractor parameters.
self.assertIsInstance(
model._feature_extractor,
......
research/object_detection/colab_tutorials/eager_few_shot_od_training_tf2_colab.ipynb
View file @ c9eb3554
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "interactive_eager_few_shot_od_training_colab.ipynb",
"provenance": [],
"collapsed_sections": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"accelerator": "GPU"
},
"cells": [
{
"cell_type": "markdown",
......@@ -29,7 +15,7 @@
"To run this colab, you will need to connect to a borg runtime (we recommend\n",
"Tensorflow GPU with Python 3).\n",
"\n",
"Estimated time to run through this colab: < 5 minutes."
"Estimated time to run through this colab: \u003c 5 minutes."
]
},
{
......@@ -44,24 +30,26 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "LBZ9VWZZFUCT",
"colab": {},
"colab_type": "code",
"colab": {}
"id": "LBZ9VWZZFUCT"
},
"outputs": [],
"source": [
"!pip install -U --pre tensorflow==\"2.2.0\""
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "oi28cqGGFWnY",
"colab": {},
"colab_type": "code",
"colab": {}
"id": "oi28cqGGFWnY"
},
"outputs": [],
"source": [
"import os\n",
"import pathlib\n",
......@@ -72,17 +60,17 @@
" os.chdir('..')\n",
"elif not pathlib.Path('models').exists():\n",
" !git clone --depth 1 https://github.com/tensorflow/models"
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "NwdsBdGhFanc",
"colab": {},
"colab_type": "code",
"colab": {}
"id": "NwdsBdGhFanc"
},
"outputs": [],
"source": [
"# Install the Object Detection API\n",
"%%bash\n",
......@@ -90,17 +78,17 @@
"protoc object_detection/protos/*.proto --python_out=.\n",
"cp object_detection/packages/tf2/setup.py .\n",
"python -m pip install ."
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "uZcqD4NLdnf4",
"colab": {}
"id": "uZcqD4NLdnf4"
},
"outputs": [],
"source": [
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
......@@ -126,9 +114,7 @@
"from object_detection.builders import model_builder\n",
"\n",
"%matplotlib inline"
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
......@@ -142,11 +128,13 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "-y9R0Xllefec",
"colab": {}
"id": "-y9R0Xllefec"
},
"outputs": [],
"source": [
"def load_image_into_numpy_array(path):\n",
" \"\"\"Load an image from file into a numpy array.\n",
......@@ -156,7 +144,7 @@
" (height, width, channels), where channels=3 for RGB.\n",
"\n",
" Args:\n",
" path: a file path (this can be local or on colossus)\n",
" path: a file path.\n",
"\n",
" Returns:\n",
" uint8 numpy array with shape (img_height, img_width, 3)\n",
......@@ -171,7 +159,7 @@
" boxes,\n",
" classes,\n",
" scores,\n",
" category_index, \n",
" category_index,\n",
" figsize=(12, 16),\n",
" image_name=None):\n",
" \"\"\"Wrapper function to visualize detections.\n",
......@@ -186,6 +174,8 @@
" boxes and plot all boxes as black with no classes or scores.\n",
" category_index: a dict containing category dictionaries (each holding\n",
" category index `id` and category name `name`) keyed by category indices.\n",
" figsize: size for the figure.\n",
" image_name: a name for the image file.\n",
" \"\"\"\n",
" image_np_with_annotations = image_np.copy()\n",
" viz_utils.visualize_boxes_and_labels_on_image_array(\n",
......@@ -196,13 +186,11 @@
" category_index,\n",
" use_normalized_coordinates=True,\n",
" min_score_thresh=0.8)\n",
" if (image_name):\n",
" if image_name:\n",
" plt.imsave(image_name, image_np_with_annotations)\n",
" else:\n",
" plt.imshow(image_np_with_annotations)\n"
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
......@@ -219,11 +207,13 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "SQy3ND7EpFQM",
"colab": {}
"id": "SQy3ND7EpFQM"
},
"outputs": [],
"source": [
"# Load images and visualize\n",
"train_image_dir = 'models/research/object_detection/test_images/ducky/train/'\n",
......@@ -242,12 +232,10 @@
"plt.rcParams['figure.figsize'] = [14, 7]\n",
"\n",
"for idx, train_image_np in enumerate(train_images_np):\n",
" plt.subplot(2,3, idx+1)\n",
" plt.subplot(2, 3, idx+1)\n",
" plt.imshow(train_image_np)\n",
"plt.show()"
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
......@@ -266,17 +254,17 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "-nEDRoUEcUgL",
"colab": {}
"id": "-nEDRoUEcUgL"
},
"outputs": [],
"source": [
"gt_boxes = []\n",
"colab_utils.annotate(train_images_np, box_storage_pointer=gt_boxes)"
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
......@@ -294,11 +282,13 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "wIAT6ZUmdHOC",
"colab": {},
"colab_type": "code",
"colab": {}
"id": "wIAT6ZUmdHOC"
},
"outputs": [],
"source": [
"# gt_boxes = [\n",
"# np.array([[0.436, 0.591, 0.629, 0.712]], dtype=np.float32),\n",
......@@ -307,9 +297,7 @@
"# np.array([[0.313, 0.308, 0.648, 0.526]], dtype=np.float32),\n",
"# np.array([[0.256, 0.444, 0.484, 0.629]], dtype=np.float32)\n",
"# ]"
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
......@@ -326,11 +314,13 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "HWBqFVMcweF-",
"colab": {}
"id": "HWBqFVMcweF-"
},
"outputs": [],
"source": [
"\n",
"# By convention, our non-background classes start counting at 1. Given\n",
......@@ -360,9 +350,7 @@
" gt_classes_one_hot_tensors.append(tf.one_hot(\n",
" zero_indexed_groundtruth_classes, num_classes))\n",
"print('Done prepping data.')\n"
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
......@@ -376,26 +364,26 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "YBD6l-E4N71y",
"colab": {}
"id": "YBD6l-E4N71y"
},
"outputs": [],
"source": [
"dummy_scores = np.array([1.0], dtype=np.float32) # give boxes a score of 100%\n",
"\n",
"plt.figure(figsize=(30, 15))\n",
"for idx in range(5):\n",
" plt.subplot(2,3, idx+1)\n",
" plt.subplot(2, 3, idx+1)\n",
" plot_detections(\n",
" train_images_np[idx],\n",
" gt_boxes[idx],\n",
" np.ones(shape=[gt_boxes[idx].shape[0]], dtype=np.int32),\n",
" dummy_scores, category_index)\n",
"plt.show()"
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
......@@ -413,28 +401,30 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "9J16r3NChD-7",
"colab": {},
"colab_type": "code",
"colab": {}
"id": "9J16r3NChD-7"
},
"outputs": [],
"source": [
"# Download the checkpoint and put it into models/research/object_detection/test_data/\n",
"\n",
"!wget http://download.tensorflow.org/models/object_detection/tf2/20200710/ssd_resnet50_v1_fpn_640x640_coco17_tpu-8.tar.gz\n",
"!tar -xf ssd_resnet50_v1_fpn_640x640_coco17_tpu-8.tar.gz\n",
"!mv ssd_resnet50_v1_fpn_640x640_coco17_tpu-8/checkpoint models/research/object_detection/test_data/"
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "RyT4BUbaMeG-",
"colab": {}
"id": "RyT4BUbaMeG-"
},
"outputs": [],
"source": [
"tf.keras.backend.clear_session()\n",
"\n",
......@@ -477,9 +467,7 @@
"prediction_dict = detection_model.predict(image, shapes)\n",
"_ = detection_model.postprocess(prediction_dict, shapes)\n",
"print('Weights restored!')"
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
......@@ -494,11 +482,13 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "nyHoF4mUrv5-",
"colab": {}
"id": "nyHoF4mUrv5-"
},
"outputs": [],
"source": [
"tf.keras.backend.set_learning_phase(True)\n",
"\n",
......@@ -588,9 +578,7 @@
" + ', loss=' + str(total_loss.numpy()), flush=True)\n",
"\n",
"print('Done fine-tuning!')"
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
......@@ -604,11 +592,13 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "WcE6OwrHQJya",
"colab": {}
"id": "WcE6OwrHQJya"
},
"outputs": [],
"source": [
"test_image_dir = 'models/research/object_detection/test_images/ducky/test/'\n",
"test_images_np = []\n",
......@@ -651,17 +641,17 @@
" + label_id_offset,\n",
" detections['detection_scores'][0].numpy(),\n",
" category_index, figsize=(15, 20), image_name=\"gif_frame_\" + ('%02d' % i) + \".jpg\")"
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "RW1FrT2iNnpy",
"colab": {},
"colab_type": "code",
"colab": {}
"id": "RW1FrT2iNnpy"
},
"outputs": [],
"source": [
"imageio.plugins.freeimage.download()\n",
"\n",
......@@ -671,16 +661,28 @@
"filenames = sorted(filenames)\n",
"last = -1\n",
"images = []\n",
"for i,filename in enumerate(filenames):\n",
"for filename in filenames:\n",
" image = imageio.imread(filename)\n",
" images.append(image)\n",
"\n",
"imageio.mimsave(anim_file, images, 'GIF-FI', fps=5)\n",
"\n",
"display(IPyImage(open(anim_file, 'rb').read()))"
],
"execution_count": null,
"outputs": []
]
}
],
"metadata": {
"accelerator": "GPU",
"colab": {
"collapsed_sections": [],
"name": "interactive_eager_few_shot_od_training_colab.ipynb",
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
}
]
}
\ No newline at end of file
},
"nbformat": 4,
"nbformat_minor": 0
}
research/object_detection/colab_tutorials/inference_tf2_colab.ipynb
View file @ c9eb3554
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "inference_tf2_colab.ipynb",
"provenance": [],
"collapsed_sections": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
}
},
"cells": [
{
"cell_type": "markdown",
......@@ -37,24 +24,26 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "LBZ9VWZZFUCT",
"colab": {},
"colab_type": "code",
"colab": {}
"id": "LBZ9VWZZFUCT"
},
"outputs": [],
"source": [
"!pip install -U --pre tensorflow==\"2.2.0\""
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "oi28cqGGFWnY",
"colab": {},
"colab_type": "code",
"colab": {}
"id": "oi28cqGGFWnY"
},
"outputs": [],
"source": [
"import os\n",
"import pathlib\n",
......@@ -65,17 +54,17 @@
" os.chdir('..')\n",
"elif not pathlib.Path('models').exists():\n",
" !git clone --depth 1 https://github.com/tensorflow/models"
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "NwdsBdGhFanc",
"colab": {},
"colab_type": "code",
"colab": {}
"id": "NwdsBdGhFanc"
},
"outputs": [],
"source": [
"# Install the Object Detection API\n",
"%%bash\n",
......@@ -83,33 +72,33 @@
"protoc object_detection/protos/*.proto --python_out=.\n",
"cp object_detection/packages/tf2/setup.py .\n",
"python -m pip install ."
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "tNR8YgZVFhPm",
"colab": {},
"colab_type": "code",
"colab": {}
"id": "tNR8YgZVFhPm"
},
"outputs": [],
"source": [
"# Test the Object Detection API installation\n",
"%%bash\n",
"cd models/research\n",
"python object_detection/builders/model_builder_tf2_test.py"
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "yn5_uV1HLvaz",
"colab": {}
"id": "yn5_uV1HLvaz"
},
"outputs": [],
"source": [
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
......@@ -128,9 +117,7 @@
"from object_detection.builders import model_builder\n",
"\n",
"%matplotlib inline"
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
......@@ -144,11 +131,13 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "-y9R0Xllefec",
"colab": {}
"id": "-y9R0Xllefec"
},
"outputs": [],
"source": [
"def load_image_into_numpy_array(path):\n",
" \"\"\"Load an image from file into a numpy array.\n",
......@@ -158,7 +147,7 @@
" (height, width, channels), where channels=3 for RGB.\n",
"\n",
" Args:\n",
" path: a file path (this can be local or on colossus)\n",
" path: a file path.\n",
"\n",
" Returns:\n",
" uint8 numpy array with shape (img_height, img_width, 3)\n",
......@@ -183,26 +172,24 @@
" for edge in kp_list:\n",
" tuple_list.append((edge.start, edge.end))\n",
" return tuple_list"
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "R4YjnOjME1gy",
"colab": {},
"colab_type": "code",
"colab": {}
"id": "R4YjnOjME1gy"
},
"outputs": [],
"source": [
"# @title Choose the model to use, then evaluate the cell.\n",
"MODELS = {'centernet_with_keypoints': 'center_net_resnet101_v1_fpn_512x512_kpts_coco17_tpu-8', 'centernet_without_keypoints': 'center_net_resnet101_v1_fpn_512x512_coco17_tpu-8'}\n",
"\n",
"model_display_name = 'centernet_with_keypoints' # @param ['centernet_with_keypoints', 'centernet_without_keypoints']\n",
"model_name = MODELS[model_display_name]"
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
......@@ -218,24 +205,26 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "ctPavqlyPuU_",
"colab": {},
"colab_type": "code",
"colab": {}
"id": "ctPavqlyPuU_"
},
"outputs": [],
"source": [
"# Download the checkpoint/ and put it into models/research/object_detection/test_data/"
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "4cni4SSocvP_",
"colab": {}
"id": "4cni4SSocvP_"
},
"outputs": [],
"source": [
"pipeline_config = os.path.join('models/research/object_detection/configs/tf2/',\n",
" model_name + '.config')\n",
......@@ -268,9 +257,7 @@
" return detect_fn\n",
"\n",
"detect_fn = get_model_detection_function(detection_model)"
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
......@@ -286,11 +273,13 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "5mucYUS6exUJ",
"colab": {}
"id": "5mucYUS6exUJ"
},
"outputs": [],
"source": [
"label_map_path = configs['eval_input_config'].label_map_path\n",
"label_map = label_map_util.load_labelmap(label_map_path)\n",
......@@ -300,9 +289,7 @@
" use_display_name=True)\n",
"category_index = label_map_util.create_category_index(categories)\n",
"label_map_dict = label_map_util.get_label_map_dict(label_map, use_display_name=True)"
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
......@@ -328,11 +315,13 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "vr_Fux-gfaG9",
"colab": {}
"id": "vr_Fux-gfaG9"
},
"outputs": [],
"source": [
"image_dir = 'models/research/object_detection/test_images/'\n",
"image_path = os.path.join(image_dir, 'image2.jpg')\n",
......@@ -376,9 +365,7 @@
"plt.figure(figsize=(12,16))\n",
"plt.imshow(image_np_with_detections)\n",
"plt.show()"
],
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
......@@ -396,11 +383,13 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "xBgYgSGMhHVi",
"colab": {}
"id": "xBgYgSGMhHVi"
},
"outputs": [],
"source": [
"if detection_model.__class__.__name__ != 'CenterNetMetaArch':\n",
" raise AssertionError('The meta-architecture for this section '\n",
......@@ -469,9 +458,20 @@
"plt.imshow(resized_heatmap_unpadded, alpha=0.7,vmin=0, vmax=160, cmap='viridis')\n",
"plt.title('Object center heatmap (class: ' + class_name + ')')\n",
"plt.show()"
],
"execution_count": null,
"outputs": []
]
}
],
"metadata": {
"colab": {
"collapsed_sections": [],
"name": "inference_tf2_colab.ipynb",
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
}
]
}
\ No newline at end of file
},
"nbformat": 4,
"nbformat_minor": 0
}
research/object_detection/colab_tutorials/object_detection_tutorial.ipynb
View file @ c9eb3554
......@@ -71,7 +71,7 @@
},
{
"cell_type": "code",
"execution_count": 0,
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
......@@ -95,7 +95,7 @@
},
{
"cell_type": "code",
"execution_count": 0,
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
......@@ -118,7 +118,7 @@
},
{
"cell_type": "code",
"execution_count": 0,
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
......@@ -149,7 +149,7 @@
},
{
"cell_type": "code",
"execution_count": 0,
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
......@@ -164,7 +164,7 @@
},
{
"cell_type": "code",
"execution_count": 0,
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
......@@ -189,7 +189,7 @@
},
{
"cell_type": "code",
"execution_count": 0,
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
......@@ -224,7 +224,7 @@
},
{
"cell_type": "code",
"execution_count": 0,
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
......@@ -249,7 +249,7 @@
},
{
"cell_type": "code",
"execution_count": 0,
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
......@@ -300,7 +300,7 @@
},
{
"cell_type": "code",
"execution_count": 0,
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
......@@ -319,7 +319,6 @@
" model_dir = pathlib.Path(model_dir)/\"saved_model\"\n",
"\n",
" model = tf.saved_model.load(str(model_dir))\n",
" model = model.signatures['serving_default']\n",
"\n",
" return model"
]
......@@ -337,7 +336,7 @@
},
{
"cell_type": "code",
"execution_count": 0,
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
......@@ -362,7 +361,7 @@
},
{
"cell_type": "code",
"execution_count": 0,
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
......@@ -398,7 +397,7 @@
},
{
"cell_type": "code",
"execution_count": 0,
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
......@@ -417,12 +416,12 @@
"id": "yN1AYfAEJIGp"
},
"source": [
"Check the model's input signature, it expects a batch of 3-color images of type uint8: "
"Check the model's input signature, it expects a batch of 3-color images of type uint8:"
]
},
{
"cell_type": "code",
"execution_count": 0,
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
......@@ -430,7 +429,7 @@
},
"outputs": [],
"source": [
"print(detection_model.inputs)"
"print(detection_model.signatures['serving_default'].inputs)"
]
},
{
......@@ -445,7 +444,7 @@
},
{
"cell_type": "code",
"execution_count": 0,
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
......@@ -453,12 +452,12 @@
},
"outputs": [],
"source": [
"detection_model.output_dtypes"
"detection_model.signatures['serving_default'].output_dtypes"
]
},
{
"cell_type": "code",
"execution_count": 0,
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
......@@ -466,7 +465,7 @@
},
"outputs": [],
"source": [
"detection_model.output_shapes"
"detection_model.signatures['serving_default'].output_shapes"
]
},
{
......@@ -481,7 +480,7 @@
},
{
"cell_type": "code",
"execution_count": 0,
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
......@@ -497,7 +496,8 @@
" input_tensor = input_tensor[tf.newaxis,...]\n",
"\n",
" # Run inference\n",
" output_dict = model(input_tensor)\n",
" model_fn = model.signatures['serving_default']\n",
" output_dict = model_fn(input_tensor)\n",
"\n",
" # All outputs are batches tensors.\n",
" # Convert to numpy arrays, and take index [0] to remove the batch dimension.\n",
......@@ -535,7 +535,7 @@
},
{
"cell_type": "code",
"execution_count": 0,
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
......@@ -565,7 +565,7 @@
},
{
"cell_type": "code",
"execution_count": 0,
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
......@@ -589,7 +589,7 @@
},
{
"cell_type": "code",
"execution_count": 0,
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
......@@ -613,7 +613,7 @@
},
{
"cell_type": "code",
"execution_count": 0,
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
......@@ -626,7 +626,7 @@
},
{
"cell_type": "code",
"execution_count": 0,
"execution_count": null,
"metadata": {
"colab": {},
"colab_type": "code",
......@@ -637,19 +637,6 @@
"for image_path in TEST_IMAGE_PATHS:\n",
" show_inference(masking_model, image_path)"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "nLlmm9JojEKm"
},
"outputs": [],
"source": [
""
]
}
],
"metadata": {
......@@ -663,6 +650,10 @@
"name": "object_detection_tutorial.ipynb",
"private_outputs": true,
"provenance": [
{
"file_id": "/piper/depot/google3/third_party/tensorflow_models/object_detection/colab_tutorials/object_detection_tutorial.ipynb",
"timestamp": 1594335690840
},
{
"file_id": "1LNYL6Zsn9Xlil2CVNOTsgDZQSBKeOjCh",
"timestamp": 1566498233247
......@@ -699,8 +690,7 @@
"file_id": "https://github.com/tensorflow/models/blob/master/research/object_detection/object_detection_tutorial.ipynb",
"timestamp": 1556150293326
}
],
"version": "0.3.2"
]
},
"kernelspec": {
"display_name": "Python 3",
......
research/object_detection/dockerfiles/tf1/Dockerfile
View file @ c9eb3554
......@@ -25,20 +25,17 @@ RUN useradd -ms /bin/bash tensorflow
USER tensorflow
WORKDIR /home/tensorflow
# Install pip dependencies
RUN pip3 install --user absl-py
RUN pip3 install --user contextlib2
RUN pip3 install --user Cython
RUN pip3 install --user jupyter
RUN pip3 install --user matplotlib
RUN pip3 install --user pycocotools
RUN pip3 install --user tf-slim
# Copy this version of of the model garden into the image
COPY --chown=tensorflow . /home/tensorflow/models
# Compile protobuf configs
RUN (cd /home/tensorflow/models/research/ && protoc object_detection/protos/*.proto --python_out=.)
WORKDIR /home/tensorflow/models/research/
RUN cp object_detection/packages/tf1/setup.py ./
ENV PATH="/home/tensorflow/.local/bin:${PATH}"
RUN python -m pip install --user -U pip
RUN python -m pip install --user .
ENV PYTHONPATH $PYTHONPATH:/home/tensorflow/models/research/:/home/tensorflow/models/research/slim
ENV TF_CPP_MIN_LOG_LEVEL 3
research/object_detection/dockerfiles/tf1/README.md
View file @ c9eb3554
# Tensorflow Object Detection on Docker
# TensorFlow Object Detection on Docker
These instructions are experimental.
......@@ -6,6 +6,6 @@ These instructions are experimental.
```bash
# From the root of the git repository
docker build -f research/object_detection/dockerfiles/1.15/Dockerfile -t od .
docker build -f research/object_detection/dockerfiles/tf1/Dockerfile -t od .
docker run -it od
```
research/object_detection/dockerfiles/tf2/Dockerfile
View file @ c9eb3554
......@@ -25,20 +25,17 @@ RUN useradd -ms /bin/bash tensorflow
USER tensorflow
WORKDIR /home/tensorflow
# Install pip dependencies
RUN pip3 install --user absl-py
RUN pip3 install --user contextlib2
RUN pip3 install --user Cython
RUN pip3 install --user jupyter
RUN pip3 install --user matplotlib
RUN pip3 install --user pycocotools
RUN pip3 install --user tf-slim
# Copy this version of of the model garden into the image
COPY --chown=tensorflow . /home/tensorflow/models
# Compile protobuf configs
RUN (cd /home/tensorflow/models/research/ && protoc object_detection/protos/*.proto --python_out=.)
WORKDIR /home/tensorflow/models/research/
RUN cp object_detection/packages/tf2/setup.py ./
ENV PATH="/home/tensorflow/.local/bin:${PATH}"
RUN python -m pip install -U pip
RUN python -m pip install .
ENV PYTHONPATH $PYTHONPATH:/home/tensorflow/models/research/:/home/tensorflow/models/research/slim
ENV TF_CPP_MIN_LOG_LEVEL 3
research/object_detection/dockerfiles/tf2/README.md
View file @ c9eb3554
# Tensorflow Object Detection on Docker
# TensorFlow Object Detection on Docker
These instructions are experimental.
......@@ -6,6 +6,6 @@ These instructions are experimental.
```bash
# From the root of the git repository
docker build -f research/object_detection/dockerfiles/2.2/Dockerfile -t od .
docker build -f research/object_detection/dockerfiles/tf2/Dockerfile -t od .
docker run -it od
```
research/object_detection/model_lib_v2.py
View file @ c9eb3554
......@@ -784,7 +784,16 @@ def eager_eval_loop(
name='eval_side_by_side_' + str(i),
step=global_step,
data=sbys_images,
max_outputs=1)
max_outputs=eval_config.num_visualizations)
if eval_util.has_densepose(eval_dict):
dp_image_list = vutils.draw_densepose_visualizations(
eval_dict)
dp_images = tf.concat(dp_image_list, axis=0)
tf.compat.v2.summary.image(
name='densepose_detections_' + str(i),
step=global_step,
data=dp_images,
max_outputs=eval_config.num_visualizations)
if evaluators is None:
if class_agnostic:
......
research/object_detection/models/ssd_efficientnet_bifpn_feature_extractor.py
View file @ c9eb3554
......@@ -221,7 +221,7 @@ class SSDEfficientNetBiFPNKerasFeatureExtractor(
ops.pad_to_multiple(preprocessed_inputs, self._pad_to_multiple))
output_feature_map_dict = self._bifpn_stage(
zip(self._output_layer_alias, base_feature_maps))
list(zip(self._output_layer_alias, base_feature_maps)))
return list(output_feature_map_dict.values())
......
research/object_detection/protos/center_net.proto
View file @ c9eb3554
......@@ -183,6 +183,41 @@ message CenterNet {
optional float heatmap_bias_init = 3 [default = -2.19];
}
optional MaskEstimation mask_estimation_task = 8;
// Parameters which are related to DensePose estimation task.
// http://densepose.org/
message DensePoseEstimation {
// Weight of the task loss. The total loss of the model will be their
// summation of task losses weighted by the weights.
optional float task_loss_weight = 1 [default = 1.0];
// Class ID (0-indexed) that corresponds to the object in the label map that
// contains DensePose data.
optional int32 class_id = 2;
// Loss configuration for DensePose heatmap and regression losses. Note
// that the localization loss is used for surface coordinate losses and
// classification loss is used for part classification losses.
optional Loss loss = 3;
// The number of body parts.
optional int32 num_parts = 4 [default = 24];
// Loss weights for the two DensePose heads.
optional float part_loss_weight = 5 [default = 1.0];
optional float coordinate_loss_weight = 6 [default = 1.0];
// Whether to upsample the prediction feature maps back to the original
// input dimension prior to applying loss. This has the benefit of
// maintaining finer groundtruth location information.
optional bool upsample_to_input_res = 7 [default = true];
// The initial bias value of the convlution kernel of the class heatmap
// prediction head. -2.19 corresponds to predicting foreground with
// a probability of 0.1.
optional float heatmap_bias_init = 8 [default = -2.19];
}
optional DensePoseEstimation densepose_estimation_task = 9;
}
message CenterNetFeatureExtractor {
......
research/object_detection/utils/colab_utils.py
View file @ c9eb3554
# Lint as: python3
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
......@@ -12,59 +13,53 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Utils for colab tutorials located in object_detection/colab_tutorials/"""
import os
import random
import uuid
"""Utils for colab tutorials located in object_detection/colab_tutorials/..."""
import base64
import io
import operator
import json
from typing import Dict
from typing import List
from typing import Union
import uuid
from IPython.display import display
from IPython.display import Javascript
import numpy as np
from PIL import Image
from typing import List, Dict, Union
from base64 import b64decode, b64encode
from IPython.display import display, Javascript
from google.colab import output
from google.colab.output import eval_js
import tensorflow as tf
import numpy as np
from six import BytesIO
from PIL import Image, ImageDraw, ImageFont
def image_from_numpy(image):
"""
Open an image at the specified path and encode it in Base64.
"""Open an image at the specified path and encode it in Base64.
Parameters
----------
image: np.ndarray
Args:
image: np.ndarray
Image represented as a numpy array
Returns
-------
str
Encoded Base64 representation of the image
Returns:
An encoded Base64 representation of the image
"""
with io.BytesIO() as output:
Image.fromarray(image).save(output, format="JPEG")
data = output.getvalue()
data = str(b64encode(data))[2:-1]
with io.BytesIO() as img_output:
Image.fromarray(image).save(img_output, format='JPEG')
data = output.getvalue()
data = str(base64.b64encode(data))[2:-1]
return data
def draw_bbox(image_urls, callbackId):
"""
Open the bounding box UI and send the results to a callback function.
Parameters
----------
image_urls: list[str | np.ndarray]
def draw_bbox(image_urls, callbackId): # pylint: disable=invalid-name
"""Open the bounding box UI and send the results to a callback function.
Args:
image_urls: list[str | np.ndarray]
List of locations from where to load the images from. If a np.ndarray is
given, the array is interpretted as an image and sent to the frontend. If
a str is given, the string is interpreted as a path and is read as a
given, the array is interpretted as an image and sent to the frontend.
If a str is given, the string is interpreted as a path and is read as a
np.ndarray before being sent to the frontend.
callbackId: str
callbackId: str
The ID for the callback function to send the bounding box results to
when the user hits submit.
"""
......@@ -126,7 +121,6 @@ def draw_bbox(image_urls, callbackId):
for (var i = 0; i < imgs.length; i++) {
allBoundingBoxes[i] = [];
}
//initialize image view
errorlog.id = 'errorlog';
image.style.display = 'block';
......@@ -175,7 +169,6 @@ def draw_bbox(image_urls, callbackId):
}
resetcanvas();
}
// on delete, deletes the last bounding box
deleteButton.textContent = "undo bbox";
deleteButton.onclick = function(){
......@@ -187,7 +180,6 @@ def draw_bbox(image_urls, callbackId):
boundingBoxes.map(r => {drawRect(r)});
};
}
// on all delete, deletes all of the bounding box
deleteAllbutton.textContent = "delete all"
deleteAllbutton.onclick = function(){
......@@ -239,7 +231,7 @@ def draw_bbox(image_urls, callbackId):
start = oMousePos(canvas_img, e);
// configure is drawing to true
isDrawing = true;
isDrawing = true;
}
function handleMouseMove(e) {
......@@ -268,15 +260,15 @@ def draw_bbox(image_urls, callbackId):
box.x = o.x;
}
else{
box.x = o.x + o.w;
box.x = o.x + o.w;
}
if (o.h > 0){
box.y = o.y;
}
else{
box.y = o.y + o.h;
box.y = o.y + o.h;
}
box.w = Math.abs(o.w);
box.w = Math.abs(o.w);
box.h = Math.abs(o.h);
// add the bounding box to the image
......@@ -285,7 +277,7 @@ def draw_bbox(image_urls, callbackId):
}
}
function draw() {
function draw() {
o.x = (start.x)/image.width; // start position of x
o.y = (start.y)/image.height; // start position of y
o.w = (m.x - start.x)/image.width; // width
......@@ -296,7 +288,7 @@ def draw_bbox(image_urls, callbackId):
// draw all the rectangles saved in the rectsRy
boundingBoxes.map(r => {drawRect(r)});
// draw the actual rectangle
drawRect(o);
drawRect(o);
}
// add the handlers needed for dragging
......@@ -312,7 +304,7 @@ def draw_bbox(image_urls, callbackId):
img = imgs[curr_image]
image.src = "data:image/png;base64," + img;
// onload init new canvas and display image
// onload init new canvas and display image
image.onload = function() {
// normalize display height and canvas
image.height = height;
......@@ -331,7 +323,7 @@ def draw_bbox(image_urls, callbackId):
}
function drawRect(o){
// draw a predefined rectangle
// draw a predefined rectangle
ctx.strokeStyle = "red";
ctx.lineWidth = 2;
ctx.beginPath(o);
......@@ -342,7 +334,7 @@ def draw_bbox(image_urls, callbackId):
// Function to detect the mouse position
function oMousePos(canvas_img, evt) {
let ClientRect = canvas_img.getBoundingClientRect();
return {
return {
x: evt.clientX - ClientRect.left,
y: evt.clientY - ClientRect.top
};
......@@ -370,46 +362,45 @@ def draw_bbox(image_urls, callbackId):
document.querySelector("#output-area").appendChild(div);
return
}''')
#load the images as a byte array
# load the images as a byte array
bytearrays = []
for image in image_urls:
if isinstance(image, str):
bytearrays.append(image_from_path(image))
elif isinstance(image, np.ndarray):
bytearrays.append(image_from_numpy(image))
else:
raise TypeError(f"Image has unsupported type {type(image)}. Only str and np.ndarray are supported.")
if isinstance(image, np.ndarray):
bytearrays.append(image_from_numpy(image))
else:
raise TypeError('Image has unsupported type {}.'.format(type(image)))
#format arrays for input
# format arrays for input
image_data = json.dumps(bytearrays)
del bytearrays
#call java script function pass string byte array(image_data) as input
# call java script function pass string byte array(image_data) as input
display(js)
eval_js(f"load_image({image_data}, '{callbackId}')")
eval_js('load_image({}, \'{}\')'.format(image_data, callbackId))
return
def annotate(imgs: List[Union[str, np.ndarray]], box_storage_pointer: List[np.ndarray], callbackId: str = None):
"""
Open the bounding box UI and prompt the user for input.
Parameters
----------
imgs: list[str | np.ndarray]
def annotate(imgs: List[Union[str, np.ndarray]], # pylint: disable=invalid-name
box_storage_pointer: List[np.ndarray],
callbackId: str = None):
"""Open the bounding box UI and prompt the user for input.
Args:
imgs: list[str | np.ndarray]
List of locations from where to load the images from. If a np.ndarray is
given, the array is interpretted as an image and sent to the frontend. If
a str is given, the string is interpreted as a path and is read as a
np.ndarray before being sent to the frontend.
box_storage_pointer: list[np.ndarray]
box_storage_pointer: list[np.ndarray]
Destination list for bounding box arrays. Each array in this list
corresponds to one of the images given in imgs. The array is a
N x 4 array where N is the number of bounding boxes given by the user
for that particular image. If there are no bounding boxes for an image,
None is used instead of an empty array.
callbackId: str, optional
callbackId: str, optional
The ID for the callback function that communicates between the fontend
and the backend. If no ID is given, a random UUID string is used instead.
"""
......@@ -418,23 +409,30 @@ def annotate(imgs: List[Union[str, np.ndarray]], box_storage_pointer: List[np.nd
if callbackId is None:
callbackId = str(uuid.uuid1()).replace('-', '')
def dictToList(input):
'''
def dictToList(input_bbox): # pylint: disable=invalid-name
"""Convert bbox.
This function converts the dictionary from the frontend (if the format
{x, y, w, h} as shown in callbackFunction) into a list
([y_min, x_min, y_max, x_max])
'''
return (input['y'], input['x'], input['y'] + input['h'], input['x'] + input['w'])
Args:
input_bbox:
def callbackFunction(annotations: List[List[Dict[str, float]]]):
Returns:
A list with bbox coordinates in the form [ymin, xmin, ymax, xmax].
"""
return (input_bbox['y'], input_bbox['x'], input_bbox['y'] + input_bbox['h'],
input_bbox['x'] + input_bbox['w'])
def callbackFunction(annotations: List[List[Dict[str, float]]]): # pylint: disable=invalid-name
"""Callback function.
This is the call back function to capture the data from the frontend and
convert the data into a numpy array.
Parameters
----------
annotations: list[list[dict[str, float]]]
Args:
annotations: list[list[dict[str, float]]]
The input of the call back function is a list of list of objects
corresponding to the annotations. The format of annotations is shown
below
......@@ -466,16 +464,17 @@ def annotate(imgs: List[Union[str, np.ndarray]], box_storage_pointer: List[np.nd
boxes.clear()
# load the new annotations into the boxes list
for annotationsPerImg in annotations:
rectanglesAsArrays = [np.clip(dictToList(annotation), 0, 1) for annotation in annotationsPerImg]
if rectanglesAsArrays:
boxes.append(np.stack(rectanglesAsArrays))
for annotations_per_img in annotations:
rectangles_as_arrays = [np.clip(dictToList(annotation), 0, 1)
for annotation in annotations_per_img]
if rectangles_as_arrays:
boxes.append(np.stack(rectangles_as_arrays))
else:
boxes.append(None)
# output the annotations to the errorlog
with output.redirect_to_element('#errorlog'):
display("--boxes array populated--")
display('--boxes array populated--')
output.register_callback(callbackId, callbackFunction)
draw_bbox(imgs, callbackId)
\ No newline at end of file
draw_bbox(imgs, callbackId)
research/object_detection/utils/visualization_utils.py
View file @ c9eb3554
......@@ -790,6 +790,81 @@ def draw_side_by_side_evaluation_image(eval_dict,
return images_with_detections_list
def draw_densepose_visualizations(eval_dict,
max_boxes_to_draw=20,
min_score_thresh=0.2,
num_parts=24,
dp_coord_to_visualize=0):
"""Draws DensePose visualizations.
Args:
eval_dict: The evaluation dictionary returned by
eval_util.result_dict_for_batched_example().
max_boxes_to_draw: The maximum number of boxes to draw for detections.
min_score_thresh: The minimum score threshold for showing detections.
num_parts: The number of different densepose parts.
dp_coord_to_visualize: Whether to visualize v-coordinates (0) or
u-coordinates (0) overlaid on the person masks.
Returns:
A list of [1, H, W, C] uint8 tensor, each element corresponding to an image
in the batch.
Raises:
ValueError: If `dp_coord_to_visualize` is not 0 or 1.
"""
if dp_coord_to_visualize not in (0, 1):
raise ValueError('`dp_coord_to_visualize` must be either 0 for v '
'coordinates), or 1 for u coordinates, but instead got '
'{}'.format(dp_coord_to_visualize))
detection_fields = fields.DetectionResultFields()
input_data_fields = fields.InputDataFields()
if detection_fields.detection_masks not in eval_dict:
raise ValueError('Expected `detection_masks` in `eval_dict`.')
if detection_fields.detection_surface_coords not in eval_dict:
raise ValueError('Expected `detection_surface_coords` in `eval_dict`.')
images_with_detections_list = []
for indx in range(eval_dict[input_data_fields.original_image].shape[0]):
# Note that detection masks have already been resized to the original image
# shapes, but `original_image` has not.
# TODO(ronnyvotel): Consider resizing `original_image` in
# eval_util.result_dict_for_batched_example().
true_shape = eval_dict[input_data_fields.true_image_shape][indx]
original_shape = eval_dict[
input_data_fields.original_image_spatial_shape][indx]
image = eval_dict[input_data_fields.original_image][indx]
image = shape_utils.pad_or_clip_nd(image, [true_shape[0], true_shape[1], 3])
image = _resize_original_image(image, original_shape)
scores = eval_dict[detection_fields.detection_scores][indx]
detection_masks = eval_dict[detection_fields.detection_masks][indx]
surface_coords = eval_dict[detection_fields.detection_surface_coords][indx]
def draw_densepose_py_func(image, detection_masks, surface_coords, scores):
"""Overlays part masks and surface coords on original images."""
surface_coord_image = np.copy(image)
for i, (score, surface_coord, mask) in enumerate(
zip(scores, surface_coords, detection_masks)):
if i == max_boxes_to_draw:
break
if score > min_score_thresh:
draw_part_mask_on_image_array(image, mask, num_parts=num_parts)
draw_float_channel_on_image_array(
surface_coord_image, surface_coord[:, :, dp_coord_to_visualize],
mask)
return np.concatenate([image, surface_coord_image], axis=1)
image_with_densepose = tf.py_func(
draw_densepose_py_func,
[image, detection_masks, surface_coords, scores],
tf.uint8)
images_with_detections_list.append(
image_with_densepose[tf.newaxis, :, :, :])
return images_with_detections_list
def draw_keypoints_on_image_array(image,
keypoints,
keypoint_scores=None,
......@@ -918,8 +993,6 @@ def draw_mask_on_image_array(image, mask, color='red', alpha=0.4):
raise ValueError('`image` not of type np.uint8')
if mask.dtype != np.uint8:
raise ValueError('`mask` not of type np.uint8')
if np.any(np.logical_and(mask != 1, mask != 0)):
raise ValueError('`mask` elements should be in [0, 1]')
if image.shape[:2] != mask.shape:
raise ValueError('The image has spatial dimensions %s but the mask has '
'dimensions %s' % (image.shape[:2], mask.shape))
......@@ -929,11 +1002,85 @@ def draw_mask_on_image_array(image, mask, color='red', alpha=0.4):
solid_color = np.expand_dims(
np.ones_like(mask), axis=2) * np.reshape(list(rgb), [1, 1, 3])
pil_solid_color = Image.fromarray(np.uint8(solid_color)).convert('RGBA')
pil_mask = Image.fromarray(np.uint8(255.0*alpha*mask)).convert('L')
pil_mask = Image.fromarray(np.uint8(255.0*alpha*(mask > 0))).convert('L')
pil_image = Image.composite(pil_solid_color, pil_image, pil_mask)
np.copyto(image, np.array(pil_image.convert('RGB')))
def draw_part_mask_on_image_array(image, mask, alpha=0.4, num_parts=24):
"""Draws part mask on an image.
Args:
image: uint8 numpy array with shape (img_height, img_height, 3)
mask: a uint8 numpy array of shape (img_height, img_height) with
1-indexed parts (0 for background).
alpha: transparency value between 0 and 1 (default: 0.4)
num_parts: the maximum number of parts that may exist in the image (default
24 for DensePose).
Raises:
ValueError: On incorrect data type for image or masks.
"""
if image.dtype != np.uint8:
raise ValueError('`image` not of type np.uint8')
if mask.dtype != np.uint8:
raise ValueError('`mask` not of type np.uint8')
if image.shape[:2] != mask.shape:
raise ValueError('The image has spatial dimensions %s but the mask has '
'dimensions %s' % (image.shape[:2], mask.shape))
pil_image = Image.fromarray(image)
part_colors = np.zeros_like(image)
mask_1_channel = mask[:, :, np.newaxis]
for i, color in enumerate(STANDARD_COLORS[:num_parts]):
rgb = np.array(ImageColor.getrgb(color), dtype=np.uint8)
part_colors += (mask_1_channel == i + 1) * rgb[np.newaxis, np.newaxis, :]
pil_part_colors = Image.fromarray(np.uint8(part_colors)).convert('RGBA')
pil_mask = Image.fromarray(np.uint8(255.0 * alpha * (mask > 0))).convert('L')
pil_image = Image.composite(pil_part_colors, pil_image, pil_mask)
np.copyto(image, np.array(pil_image.convert('RGB')))
def draw_float_channel_on_image_array(image, channel, mask, alpha=0.9,
cmap='YlGn'):
"""Draws a floating point channel on an image array.
Args:
image: uint8 numpy array with shape (img_height, img_height, 3)
channel: float32 numpy array with shape (img_height, img_height). The values
should be in the range [0, 1], and will be mapped to colors using the
provided colormap `cmap` argument.
mask: a uint8 numpy array of shape (img_height, img_height) with
1-indexed parts (0 for background).
alpha: transparency value between 0 and 1 (default: 0.9)
cmap: string with the colormap to use.
Raises:
ValueError: On incorrect data type for image or masks.
"""
if image.dtype != np.uint8:
raise ValueError('`image` not of type np.uint8')
if channel.dtype != np.float32:
raise ValueError('`channel` not of type np.float32')
if mask.dtype != np.uint8:
raise ValueError('`mask` not of type np.uint8')
if image.shape[:2] != channel.shape:
raise ValueError('The image has spatial dimensions %s but the channel has '
'dimensions %s' % (image.shape[:2], channel.shape))
if image.shape[:2] != mask.shape:
raise ValueError('The image has spatial dimensions %s but the mask has '
'dimensions %s' % (image.shape[:2], mask.shape))
cm = plt.get_cmap(cmap)
pil_image = Image.fromarray(image)
colored_channel = cm(channel)[:, :, :3]
pil_colored_channel = Image.fromarray(
np.uint8(colored_channel * 255)).convert('RGBA')
pil_mask = Image.fromarray(np.uint8(255.0 * alpha * (mask > 0))).convert('L')
pil_image = Image.composite(pil_colored_channel, pil_image, pil_mask)
np.copyto(image, np.array(pil_image.convert('RGB')))
def visualize_boxes_and_labels_on_image_array(
image,
boxes,
......@@ -973,8 +1120,8 @@ def visualize_boxes_and_labels_on_image_array(
boxes and plot all boxes as black with no classes or scores.
category_index: a dict containing category dictionaries (each holding
category index `id` and category name `name`) keyed by category indices.
instance_masks: a numpy array of shape [N, image_height, image_width] with
values ranging between 0 and 1, can be None.
instance_masks: a uint8 numpy array of shape [N, image_height, image_width],
can be None.
instance_boundaries: a numpy array of shape [N, image_height, image_width]
with values ranging between 0 and 1, can be None.
keypoints: a numpy array of shape [N, num_keypoints, 2], can
......
research/object_detection/utils/visualization_utils_test.py
View file @ c9eb3554
......@@ -373,6 +373,38 @@ class VisualizationUtilsTest(test_case.TestCase):
color='Blue', alpha=.5)
self.assertAllEqual(test_image, expected_result)
def test_draw_part_mask_on_image_array(self):
test_image = np.asarray([[[0, 0, 0], [0, 0, 0]],
[[0, 0, 0], [0, 0, 0]]], dtype=np.uint8)
mask = np.asarray([[0, 1],
[1, 6]], dtype=np.uint8)
visualization_utils.draw_part_mask_on_image_array(test_image, mask,
alpha=.5)
self.assertAllEqual([0, 0, 0], test_image[0, 0])
self.assertAllGreater(test_image[0, 1], 0)
self.assertAllGreater(test_image[1, 0], 0)
self.assertAllGreater(test_image[1, 1], 0)
self.assertAllEqual(test_image[0, 1], test_image[1, 0])
def test_draw_float_channel_on_image_array(self):
test_image = np.asarray([[[0, 0, 0], [0, 0, 0]],
[[0, 0, 0], [0, 0, 0]]], dtype=np.uint8)
channel = np.asarray([[0., 0.5],
[0., 1.]], dtype=np.float32)
mask = np.asarray([[0, 1],
[1, 1]], dtype=np.uint8)
# The colormap ('bwr') maps the values as follows:
# 0.0 -> Blue
# 0.5 -> White
# 1.0 -> Red
visualization_utils.draw_float_channel_on_image_array(
test_image, channel, mask, alpha=1.0, cmap='bwr')
expected_result = np.asarray([[[0, 0, 0], [255, 254, 254]],
[[0, 0, 255], [255, 0, 0]]], dtype=np.uint8)
self.assertAllEqual(test_image, expected_result)
def test_draw_heatmaps_on_image(self):
test_image = self.create_colorful_test_image()
test_image = Image.fromarray(test_image)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment