Merged commit includes the following changes: (#8803)

320117767  by ronnyvotel:

    DensePose postprocessing implementation.

--
320065853  by ronnyvotel:

    Updating how masks are reframed, so that it works on float and uint8 masks.

--
320061717  by yuhuic:

    Updated CenterNet restore_from_objects to allow the model to load the
    checkpoints saved during training.

--
319835172  by ronnyvotel:

    Updating how the DensePose UV Symmetries MAT file path is constructed and loaded.

--
319834678  by ronnyvotel:

    First update to CenterNetMetaArch for DensePose. Adding prediction and loss functionality.

--
319810261  by rathodv:

    Create a setup.py file to simplify installation.

    Usage:
    "python object_detection/packages/tf1/setup.py install" for TF1.
    "python object_detection/packages/tf2/setup.py install" for TF2.

    or to create source distribution
    "python object_detection/packages/tf1/setup.py sdist" for TF1.
    "python object_detection/packages/tf2/setup.py sdist" for TF2.

--
319803041  by sbeery:

    Updating documentation for export

--
319688087  by rathodv:

    Update as_matrix() to to_numpy() to avoid failures with python3.6

--
319686183  by vighneshb:

    Require tpu_name when use_tpu is set.

--
319613327  by aom:

    EfficientDet-style Data Augmentation.

--
319572180  by rathodv:

    Add TF2 SSD FPN (a.k.a RetinaNet) configs.

--
319553823  by rathodv:

    Internal Change.

--

PiperOrigin-RevId: 320117767
Co-authored-by: TF Object Detection Team <no-reply@google.com>

research/object_detection/meta_architectures/context_rcnn_meta_arch_tf1_test.py

@@ -20,8 +20,8 @@ from __future__ import print_function
 
 import functools
 import unittest
+from unittest import mock  # pylint: disable=g-importing-member
 from absl.testing import parameterized
-import mock
 import tensorflow.compat.v1 as tf
 import tf_slim as slim
 

research/object_detection/metrics/oid_challenge_evaluation_utils.py

@@ -136,15 +136,15 @@ def build_groundtruth_dictionary(data, class_label_map):
 
   dictionary = {
       standard_fields.InputDataFields.groundtruth_boxes:
-          data_location[['YMin', 'XMin', 'YMax', 'XMax']].as_matrix(),
+          data_location[['YMin', 'XMin', 'YMax', 'XMax']].to_numpy(),
       standard_fields.InputDataFields.groundtruth_classes:
           data_location['LabelName'].map(lambda x: class_label_map[x]
-                                        ).as_matrix(),
+                                        ).to_numpy(),
       standard_fields.InputDataFields.groundtruth_group_of:
-          data_location['IsGroupOf'].as_matrix().astype(int),
+          data_location['IsGroupOf'].to_numpy().astype(int),
       standard_fields.InputDataFields.groundtruth_image_classes:
           data_labels['LabelName'].map(lambda x: class_label_map[x]
-                                      ).as_matrix(),
+                                      ).to_numpy(),
   }
 
   if 'Mask' in data_location:
@@ -179,9 +179,9 @@ def build_predictions_dictionary(data, class_label_map):
   """
   dictionary = {
       standard_fields.DetectionResultFields.detection_classes:
-          data['LabelName'].map(lambda x: class_label_map[x]).as_matrix(),
+          data['LabelName'].map(lambda x: class_label_map[x]).to_numpy(),
       standard_fields.DetectionResultFields.detection_scores:
-          data['Score'].as_matrix()
+          data['Score'].to_numpy()
   }
 
   if 'Mask' in data:
@@ -192,6 +192,6 @@ def build_predictions_dictionary(data, class_label_map):
   else:
     dictionary[standard_fields.DetectionResultFields.detection_boxes] = data[[
         'YMin', 'XMin', 'YMax', 'XMax'
-    ]].as_matrix()
+    ]].to_numpy()
 
   return dictionary

research/object_detection/metrics/oid_vrd_challenge_evaluation_utils.py

@@ -53,16 +53,16 @@ def build_groundtruth_vrd_dictionary(data, class_label_map,
 
   boxes = np.zeros(data_boxes.shape[0], dtype=vrd_evaluation.vrd_box_data_type)
   boxes['subject'] = data_boxes[['YMin1', 'XMin1', 'YMax1',
-                                 'XMax1']].as_matrix()
-  boxes['object'] = data_boxes[['YMin2', 'XMin2', 'YMax2', 'XMax2']].as_matrix()
+                                 'XMax1']].to_numpy()
+  boxes['object'] = data_boxes[['YMin2', 'XMin2', 'YMax2', 'XMax2']].to_numpy()
 
   labels = np.zeros(data_boxes.shape[0], dtype=vrd_evaluation.label_data_type)
   labels['subject'] = data_boxes['LabelName1'].map(
-      lambda x: class_label_map[x]).as_matrix()
+      lambda x: class_label_map[x]).to_numpy()
   labels['object'] = data_boxes['LabelName2'].map(
-      lambda x: class_label_map[x]).as_matrix()
+      lambda x: class_label_map[x]).to_numpy()
   labels['relation'] = data_boxes['RelationshipLabel'].map(
-      lambda x: relationship_label_map[x]).as_matrix()
+      lambda x: relationship_label_map[x]).to_numpy()
 
   return {
       standard_fields.InputDataFields.groundtruth_boxes:
@@ -71,7 +71,7 @@ def build_groundtruth_vrd_dictionary(data, class_label_map,
           labels,
       standard_fields.InputDataFields.groundtruth_image_classes:
           data_labels['LabelName'].map(lambda x: class_label_map[x])
-          .as_matrix(),
+          .to_numpy(),
   }
 
 
@@ -104,16 +104,16 @@ def build_predictions_vrd_dictionary(data, class_label_map,
 
   boxes = np.zeros(data_boxes.shape[0], dtype=vrd_evaluation.vrd_box_data_type)
   boxes['subject'] = data_boxes[['YMin1', 'XMin1', 'YMax1',
-                                 'XMax1']].as_matrix()
-  boxes['object'] = data_boxes[['YMin2', 'XMin2', 'YMax2', 'XMax2']].as_matrix()
+                                 'XMax1']].to_numpy()
+  boxes['object'] = data_boxes[['YMin2', 'XMin2', 'YMax2', 'XMax2']].to_numpy()
 
   labels = np.zeros(data_boxes.shape[0], dtype=vrd_evaluation.label_data_type)
   labels['subject'] = data_boxes['LabelName1'].map(
-      lambda x: class_label_map[x]).as_matrix()
+      lambda x: class_label_map[x]).to_numpy()
   labels['object'] = data_boxes['LabelName2'].map(
-      lambda x: class_label_map[x]).as_matrix()
+      lambda x: class_label_map[x]).to_numpy()
   labels['relation'] = data_boxes['RelationshipLabel'].map(
-      lambda x: relationship_label_map[x]).as_matrix()
+      lambda x: relationship_label_map[x]).to_numpy()
 
   return {
       standard_fields.DetectionResultFields.detection_boxes:
@@ -121,5 +121,5 @@ def build_predictions_vrd_dictionary(data, class_label_map,
       standard_fields.DetectionResultFields.detection_classes:
           labels,
       standard_fields.DetectionResultFields.detection_scores:
-          data_boxes['Score'].as_matrix()
+          data_boxes['Score'].to_numpy()
   }

research/object_detection/model_main_tf2.py

@@ -54,6 +54,10 @@ flags.DEFINE_integer('eval_timeout', 3600, 'Number of seconds to wait for an'
                      'evaluation checkpoint before exiting.')
 
 flags.DEFINE_bool('use_tpu', False, 'Whether the job is executing on a TPU.')
+flags.DEFINE_string(
+    'tpu_name',
+    default=None,
+    help='Name of the Cloud TPU for Cluster Resolvers.')
 flags.DEFINE_integer(
     'num_workers', 1, 'When num_workers > 1, training uses '
     'MultiWorkerMirroredStrategy. When num_workers = 1 it uses '
@@ -79,7 +83,11 @@ def main(unused_argv):
         wait_interval=300, timeout=FLAGS.eval_timeout)
   else:
     if FLAGS.use_tpu:
-      resolver = tf.distribute.cluster_resolver.TPUClusterResolver()
+      if FLAGS.tpu_name is None:
+        raise ValueError('--tpu_name needs to be specified when use_tpu'
+                         ' is set.')
+      resolver = tf.distribute.cluster_resolver.TPUClusterResolver(
+          FLAGS.tpu_name)
       tf.config.experimental_connect_to_cluster(resolver)
       tf.tpu.experimental.initialize_tpu_system(resolver)
       strategy = tf.distribute.experimental.TPUStrategy(resolver)

research/object_detection/packages/tf1/setup.py

+"""Setup script for object_detection with TF1.0."""
+import os
+from setuptools import find_packages
+from setuptools import setup
+
+REQUIRED_PACKAGES = ['apache-beam', 'pillow', 'lxml', 'matplotlib', 'Cython',
+                     'contextlib2', 'tf-slim', 'six', 'pycocotools', 'scipy',
+                     'pandas']
+
+setup(
+    name='object_detection',
+    version='0.1',
+    install_requires=REQUIRED_PACKAGES,
+    include_package_data=True,
+    packages=(
+        [p for p in find_packages() if p.startswith('object_detection')] +
+        find_packages(where=os.path.join('.', 'slim'))),
+    package_dir={
+        'datasets': os.path.join('slim', 'datasets'),
+        'nets': os.path.join('slim', 'nets'),
+        'preprocessing': os.path.join('slim', 'preprocessing'),
+        'deployment': os.path.join('slim', 'deployment'),
+        'scripts': os.path.join('slim', 'scripts'),
+    },
+    description='Tensorflow Object Detection Library with TF1.0',
+    python_requires='>3.6',
+)

research/object_detection/packages/tf2/setup.py

+"""Setup script for object_detection with TF2.0."""
+import os
+from setuptools import find_packages
+from setuptools import setup
+
+# Note: adding apache-beam to required packages causes conflict with
+# tf-models-offical requirements. These packages request for incompatible
+# oauth2client package.
+REQUIRED_PACKAGES = ['pillow', 'lxml', 'matplotlib', 'Cython', 'contextlib2',
+                     'tf-slim', 'six', 'pycocotools', 'scipy', 'pandas',
+                     'tf-models-official']
+
+setup(
+    name='object_detection',
+    version='0.1',
+    install_requires=REQUIRED_PACKAGES,
+    include_package_data=True,
+    packages=(
+        [p for p in find_packages() if p.startswith('object_detection')] +
+        find_packages(where=os.path.join('.', 'slim'))),
+    package_dir={
+        'datasets': os.path.join('slim', 'datasets'),
+        'nets': os.path.join('slim', 'nets'),
+        'preprocessing': os.path.join('slim', 'preprocessing'),
+        'deployment': os.path.join('slim', 'deployment'),
+        'scripts': os.path.join('slim', 'scripts'),
+    },
+    description='Tensorflow Object Detection Library',
+    python_requires='>3.6',
+)

research/object_detection/protos/preprocessor.proto

@@ -4,7 +4,7 @@ package object_detection.protos;
 
 // Message for defining a preprocessing operation on input data.
 // See: //third_party/tensorflow_models/object_detection/core/preprocessor.py
-// Next ID: 38
+// Next ID: 39
 message PreprocessingStep {
   oneof preprocessing_step {
     NormalizeImage normalize_image = 1;
@@ -44,6 +44,7 @@ message PreprocessingStep {
     RandomDownscaleToTargetPixels random_downscale_to_target_pixels = 35;
     RandomPatchGaussian random_patch_gaussian = 36;
     RandomSquareCropByScale random_square_crop_by_scale = 37;
+    RandomScaleCropAndPadToSquare random_scale_crop_and_pad_to_square = 38;
   }
 }
 
@@ -572,3 +573,20 @@ message RandomSquareCropByScale {
   // [min_scale, max_scale]
   optional int32 num_scales = 4 [default=8];
 }
+
+// Randomly scale, crop, and then pad an image to the desired square output
+// dimensions. Specifically, this method first samples a random_scale factor
+// from a uniform distribution between scale_min and scale_max, and then resizes
+// the image such that it's maximum dimension is (output_size * random_scale).
+// Secondly, a square output_size crop is extracted from the resized image, and
+// finally the cropped region is padded to the desired square output_size.
+// The augmentation is borrowed from [1]
+// [1]: https://arxiv.org/abs/1911.09070
+message RandomScaleCropAndPadToSquare {
+  // The (square) output image size
+  optional int32 output_size = 1 [default = 512];
+
+  // The minimum and maximum values from which to sample the random scale.
+  optional float scale_min = 2 [default=0.1];
+  optional float scale_max = 3 [default=2.0];
+}

research/object_detection/utils/ops.py

@@ -799,14 +799,14 @@ def position_sensitive_crop_regions(image,
 
 
 def reframe_box_masks_to_image_masks(box_masks, boxes, image_height,
-                                     image_width):
+                                     image_width, resize_method='bilinear'):
   """Transforms the box masks back to full image masks.
 
   Embeds masks in bounding boxes of larger masks whose shapes correspond to
   image shape.
 
   Args:
-    box_masks: A tf.float32 tensor of size [num_masks, mask_height, mask_width].
+    box_masks: A tensor of size [num_masks, mask_height, mask_width].
     boxes: A tf.float32 tensor of size [num_masks, 4] containing the box
            corners. Row i contains [ymin, xmin, ymax, xmax] of the box
            corresponding to mask i. Note that the box corners are in
@@ -815,10 +815,14 @@ def reframe_box_masks_to_image_masks(box_masks, boxes, image_height,
                   the image height.
     image_width: Image width. The output mask will have the same width as the
                  image width.
+    resize_method: The resize method, either 'bilinear' or 'nearest'. Note that
+      'bilinear' is only respected if box_masks is a float.
 
   Returns:
-    A tf.float32 tensor of size [num_masks, image_height, image_width].
+    A tensor of size [num_masks, image_height, image_width] with the same dtype
+    as `box_masks`.
   """
+  resize_method = 'nearest' if box_masks.dtype == tf.uint8 else resize_method
   # TODO(rathodv): Make this a public function.
   def reframe_box_masks_to_image_masks_default():
     """The default function when there are more than 0 box masks."""
@@ -840,16 +844,19 @@ def reframe_box_masks_to_image_masks(box_masks, boxes, image_height,
 
     # TODO(vighneshb) Use matmul_crop_and_resize so that the output shape
     # is static. This will help us run and test on TPUs.
-    return tf.image.crop_and_resize(
+    resized_crops = tf.image.crop_and_resize(
         image=box_masks_expanded,
         boxes=reverse_boxes,
         box_ind=tf.range(num_boxes),
         crop_size=[image_height, image_width],
-        extrapolation_value=0.0)
+        method=resize_method,
+        extrapolation_value=0)
+    return tf.cast(resized_crops, box_masks.dtype)
+
   image_masks = tf.cond(
       tf.shape(box_masks)[0] > 0,
       reframe_box_masks_to_image_masks_default,
-      lambda: tf.zeros([0, image_height, image_width, 1], dtype=tf.float32))
+      lambda: tf.zeros([0, image_height, image_width, 1], box_masks.dtype))
   return tf.squeeze(image_masks, axis=3)
 
 

research/object_detection/utils/ops_test.py

@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+
+from absl.testing import parameterized
 import numpy as np
 import six
 from six.moves import range
@@ -1190,36 +1192,59 @@ class OpsTestBatchPositionSensitiveCropRegions(test_case.TestCase):
 
 # The following tests are only executed on CPU because the output
 # shape is not constant.
-class ReframeBoxMasksToImageMasksTest(test_case.TestCase):
-
-  def testZeroImageOnEmptyMask(self):
+class ReframeBoxMasksToImageMasksTest(test_case.TestCase,
+                                      parameterized.TestCase):
+
+  @parameterized.parameters(
+      {'mask_dtype': tf.float32, 'mask_dtype_np': np.float32,
+       'resize_method': 'bilinear'},
+      {'mask_dtype': tf.float32, 'mask_dtype_np': np.float32,
+       'resize_method': 'nearest'},
+      {'mask_dtype': tf.uint8, 'mask_dtype_np': np.uint8,
+       'resize_method': 'bilinear'},
+      {'mask_dtype': tf.uint8, 'mask_dtype_np': np.uint8,
+       'resize_method': 'nearest'},
+  )
+  def testZeroImageOnEmptyMask(self, mask_dtype, mask_dtype_np, resize_method):
     np_expected_image_masks = np.array([[[0, 0, 0, 0],
                                          [0, 0, 0, 0],
                                          [0, 0, 0, 0],
-                                         [0, 0, 0, 0]]], dtype=np.float32)
+                                         [0, 0, 0, 0]]])
     def graph_fn():
       box_masks = tf.constant([[[0, 0],
-                                [0, 0]]], dtype=tf.float32)
+                                [0, 0]]], dtype=mask_dtype)
       boxes = tf.constant([[0.0, 0.0, 1.0, 1.0]], dtype=tf.float32)
-      image_masks = ops.reframe_box_masks_to_image_masks(box_masks, boxes,
-                                                         image_height=4,
-                                                         image_width=4)
+      image_masks = ops.reframe_box_masks_to_image_masks(
+          box_masks, boxes, image_height=4, image_width=4,
+          resize_method=resize_method)
       return image_masks
 
     np_image_masks = self.execute_cpu(graph_fn, [])
+    self.assertEqual(np_image_masks.dtype, mask_dtype_np)
     self.assertAllClose(np_image_masks, np_expected_image_masks)
 
-  def testZeroBoxMasks(self):
+  @parameterized.parameters(
+      {'mask_dtype': tf.float32, 'mask_dtype_np': np.float32,
+       'resize_method': 'bilinear'},
+      {'mask_dtype': tf.float32, 'mask_dtype_np': np.float32,
+       'resize_method': 'nearest'},
+      {'mask_dtype': tf.uint8, 'mask_dtype_np': np.uint8,
+       'resize_method': 'bilinear'},
+      {'mask_dtype': tf.uint8, 'mask_dtype_np': np.uint8,
+       'resize_method': 'nearest'},
+  )
+  def testZeroBoxMasks(self, mask_dtype, mask_dtype_np, resize_method):
 
     def graph_fn():
-      box_masks = tf.zeros([0, 3, 3], dtype=tf.float32)
+      box_masks = tf.zeros([0, 3, 3], dtype=mask_dtype)
       boxes = tf.zeros([0, 4], dtype=tf.float32)
-      image_masks = ops.reframe_box_masks_to_image_masks(box_masks, boxes,
-                                                         image_height=4,
-                                                         image_width=4)
+      image_masks = ops.reframe_box_masks_to_image_masks(
+          box_masks, boxes, image_height=4, image_width=4,
+          resize_method=resize_method)
       return image_masks
 
     np_image_masks = self.execute_cpu(graph_fn, [])
+    self.assertEqual(np_image_masks.dtype, mask_dtype_np)
     self.assertAllEqual(np_image_masks.shape, np.array([0, 4, 4]))
 
   def testBoxWithZeroArea(self):
@@ -1235,40 +1260,70 @@ class ReframeBoxMasksToImageMasksTest(test_case.TestCase):
     np_image_masks = self.execute_cpu(graph_fn, [])
     self.assertAllEqual(np_image_masks.shape, np.array([1, 4, 4]))
 
-  def testMaskIsCenteredInImageWhenBoxIsCentered(self):
+  @parameterized.parameters(
+      {'mask_dtype': tf.float32, 'mask_dtype_np': np.float32,
+       'resize_method': 'bilinear'},
+      {'mask_dtype': tf.float32, 'mask_dtype_np': np.float32,
+       'resize_method': 'nearest'},
+      {'mask_dtype': tf.uint8, 'mask_dtype_np': np.uint8,
+       'resize_method': 'bilinear'},
+      {'mask_dtype': tf.uint8, 'mask_dtype_np': np.uint8,
+       'resize_method': 'nearest'},
+  )
+  def testMaskIsCenteredInImageWhenBoxIsCentered(self, mask_dtype,
+                                                 mask_dtype_np, resize_method):
 
     def graph_fn():
-      box_masks = tf.constant([[[1, 1],
-                                [1, 1]]], dtype=tf.float32)
+      box_masks = tf.constant([[[4, 4],
+                                [4, 4]]], dtype=mask_dtype)
       boxes = tf.constant([[0.25, 0.25, 0.75, 0.75]], dtype=tf.float32)
-      image_masks = ops.reframe_box_masks_to_image_masks(box_masks, boxes,
-                                                         image_height=4,
-                                                         image_width=4)
+      image_masks = ops.reframe_box_masks_to_image_masks(
+          box_masks, boxes, image_height=4, image_width=4,
+          resize_method=resize_method)
       return image_masks
 
     np_expected_image_masks = np.array([[[0, 0, 0, 0],
-                                         [0, 1, 1, 0],
-                                         [0, 1, 1, 0],
-                                         [0, 0, 0, 0]]], dtype=np.float32)
+                                         [0, 4, 4, 0],
+                                         [0, 4, 4, 0],
+                                         [0, 0, 0, 0]]], dtype=mask_dtype_np)
     np_image_masks = self.execute_cpu(graph_fn, [])
+    self.assertEqual(np_image_masks.dtype, mask_dtype_np)
     self.assertAllClose(np_image_masks, np_expected_image_masks)
 
-  def testMaskOffCenterRemainsOffCenterInImage(self):
+  @parameterized.parameters(
+      {'mask_dtype': tf.float32, 'mask_dtype_np': np.float32,
+       'resize_method': 'bilinear'},
+      {'mask_dtype': tf.float32, 'mask_dtype_np': np.float32,
+       'resize_method': 'nearest'},
+      {'mask_dtype': tf.uint8, 'mask_dtype_np': np.uint8,
+       'resize_method': 'bilinear'},
+      {'mask_dtype': tf.uint8, 'mask_dtype_np': np.uint8,
+       'resize_method': 'nearest'},
+  )
+  def testMaskOffCenterRemainsOffCenterInImage(self, mask_dtype,
+                                               mask_dtype_np, resize_method):
 
     def graph_fn():
       box_masks = tf.constant([[[1, 0],
-                                [0, 1]]], dtype=tf.float32)
+                                [0, 1]]], dtype=mask_dtype)
       boxes = tf.constant([[0.25, 0.5, 0.75, 1.0]], dtype=tf.float32)
-      image_masks = ops.reframe_box_masks_to_image_masks(box_masks, boxes,
-                                                         image_height=4,
-                                                         image_width=4)
+      image_masks = ops.reframe_box_masks_to_image_masks(
+          box_masks, boxes, image_height=4, image_width=4,
+          resize_method=resize_method)
       return image_masks
 
-    np_expected_image_masks = np.array([[[0, 0, 0, 0],
-                                         [0, 0, 0.6111111, 0.16666669],
-                                         [0, 0, 0.3888889, 0.83333337],
-                                         [0, 0, 0, 0]]], dtype=np.float32)
+    if mask_dtype == tf.float32 and resize_method == 'bilinear':
+      np_expected_image_masks = np.array([[[0, 0, 0, 0],
+                                           [0, 0, 0.6111111, 0.16666669],
+                                           [0, 0, 0.3888889, 0.83333337],
+                                           [0, 0, 0, 0]]], dtype=np.float32)
+    else:
+      np_expected_image_masks = np.array([[[0, 0, 0, 0],
+                                           [0, 0, 1, 0],
+                                           [0, 0, 0, 1],
+                                           [0, 0, 0, 0]]], dtype=mask_dtype_np)
     np_image_masks = self.execute_cpu(graph_fn, [])
+    self.assertEqual(np_image_masks.dtype, mask_dtype_np)
     self.assertAllClose(np_image_masks, np_expected_image_masks)