Merge branch 'master' of https://github.com/tensorflow/models

research/object_detection/models/keras_models/nonlocal_block_tf2_test.py

+"""Tests for google3.third_party.tensorflow_models.object_detection.models.keras_models.nonlocal_block."""
+import unittest
+from absl.testing import parameterized
+import tensorflow as tf
+
+from object_detection.models.keras_models import nonlocal_block
+from object_detection.utils import test_case
+from object_detection.utils import tf_version
+
+
+@unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
+class NonlocalTest(test_case.TestCase, parameterized.TestCase):
+
+  @parameterized.parameters([{'pool_size': None,
+                              'add_coord_conv': False},
+                             {'pool_size': None,
+                              'add_coord_conv': True},
+                             {'pool_size': 2,
+                              'add_coord_conv': False},
+                             {'pool_size': 2,
+                              'add_coord_conv': True}])
+  def test_run_nonlocal_block(self, pool_size, add_coord_conv):
+    nonlocal_op = nonlocal_block.NonLocalBlock(
+        8, pool_size=pool_size, add_coord_conv=add_coord_conv)
+    def graph_fn():
+      inputs = tf.zeros((4, 16, 16, 32), dtype=tf.float32)
+      outputs = nonlocal_op(inputs)
+      return outputs
+    outputs = self.execute(graph_fn, [])
+    self.assertAllEqual([4, 16, 16, 32], outputs.shape)
+
+
+if __name__ == '__main__':
+  tf.test.main()

research/object_detection/models/keras_models/resnet_v1.py

@@ -19,7 +19,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from keras.applications import resnet
+from tensorflow.python.keras.applications import resnet
 
 import tensorflow.compat.v1 as tf
 

research/object_detection/packages/tf2/setup.py

@@ -21,7 +21,11 @@ REQUIRED_PACKAGES = [
     'lvis',
     'scipy',
     'pandas',
-    'tf-models-official'
+    # tensorflow 2.5.0 requires grpcio~=1.34.0.
+    # tf-models-official (which requires google-could-bigquery)  ends
+    # up installing the latest grpcio which causes problems later.
+    'google-cloud-bigquery==1.21.0',
+    'tf-models-official',
 ]
 
 setup(

research/object_detection/protos/center_net.proto

@@ -187,8 +187,9 @@ message CenterNet {
     // vicinity of each regressed keypoint.
     optional float candidate_search_scale = 15 [default = 0.3];
 
-    // One of ['min_distance', 'score_distance_ratio'] indicating how to select
-    // the keypoint candidate.
+    // One of ['min_distance', 'score_distance_ratio',
+    // 'score_scaled_distance_ratio', 'gaussian_weighted'] indicating how to
+    // select the keypoint candidate.
     optional string candidate_ranking_mode = 16 [default = "min_distance"];
 
     // The score distance ratio offset, only used if candidate_ranking_mode is
@@ -197,6 +198,28 @@ message CenterNet {
     // keypoint_score / (distance + score_distance_offset)
     optional float score_distance_offset = 22 [default = 1.0];
 
+    // A scalar used to multiply the bounding box size to be used as the offset
+    // in the score-to-distance-ratio formula. Only applicable when the
+    // candidate_ranking_mode is score_scaled_distance_ratio.
+    // The keypoint candidates are ranked using the formula:
+    //   ranking_score = score / (distance + offset)
+    // where 'score' is the keypoint heatmap scores, 'distance' is the distance
+    // between the heatmap peak location and the regressed joint location,
+    // 'offset' is a function of the predicted bounding box:
+    //   offset = max(bbox height, bbox width) * score_distance_multiplier
+    optional float score_distance_multiplier = 28 [default = 0.1];
+
+    // A scalar used to multiply the Gaussian standard deviation to control the
+    // Gaussian kernel which is used to weight the candidates. Only applicable
+    // when the candidate_ranking_mode is gaussian_weighted.
+    // The keypoint candidates are ranked using the formula:
+    //   scores * exp((-distances^2) / (2 * sigma^2))
+    // where 'distances' is the distance between the heatmap peak location and
+    // the regressed joint location and 'sigma' is the Gaussian standard
+    // deviation used in generating the Gaussian heatmap target multiplied by the
+    // 'std_dev_multiplier'.
+    optional float std_dev_multiplier = 29 [default = 1.0];
+
     // The radius (in the unit of output pixel) around heatmap peak to assign
     // the offset targets. If set 0, then the offset target will only be
     // assigned to the heatmap peak (same behavior as the original paper).
@@ -238,6 +261,11 @@ message CenterNet {
     // the number of keypoints for that class.
     optional bool rescore_instances = 24 [default = false];
 
+    // A scalar used when "rescore_instances" is set to True. The detection
+    // score of an instance is set to be the average score among those keypoints
+    // with scores higher than the threshold.
+    optional float rescoring_threshold = 30 [default = 0.0];
+
     // Parameters to determine the architecture of the keypoint heatmap
     // prediction head.
     optional PredictionHeadParams heatmap_head_params = 25;
@@ -418,6 +446,10 @@ message CenterNet {
     // The mode for jitterting box ROIs. See RandomJitterBoxes in
     // preprocessor.proto for more details
     optional RandomJitterBoxes.JitterMode jitter_mode = 15 [default=DEFAULT];
+
+    // Weight for the box consistency loss as described in the BoxInst paper
+    // https://arxiv.org/abs/2012.02310
+    optional float box_consistency_loss_weight = 16 [default=0.0];
   }
 
   optional DeepMACMaskEstimation deepmac_mask_estimation = 14;
@@ -466,3 +498,4 @@ message CenterNetFeatureExtractor {
   optional string upsampling_interpolation = 11 [default = 'nearest'];
 }
 
+

research/object_detection/utils/config_util.py

@@ -639,6 +639,12 @@ def _maybe_update_config_with_key_value(configs, key, value):
     _update_rescore_instances(configs["model"], value)
   elif field_name == "unmatched_keypoint_score":
     _update_unmatched_keypoint_score(configs["model"], value)
+  elif field_name == "score_distance_multiplier":
+    _update_score_distance_multiplier(configs["model"], value)
+  elif field_name == "std_dev_multiplier":
+    _update_std_dev_multiplier(configs["model"], value)
+  elif field_name == "rescoring_threshold":
+    _update_rescoring_threshold(configs["model"], value)
   else:
     return False
   return True
@@ -1135,10 +1141,12 @@ def _update_candidate_search_scale(model_config, search_scale):
 
 def _update_candidate_ranking_mode(model_config, mode):
   """Updates how keypoints are snapped to candidates in CenterNet."""
-  if mode not in ("min_distance", "score_distance_ratio"):
+  if mode not in ("min_distance", "score_distance_ratio",
+                  "score_scaled_distance_ratio", "gaussian_weighted"):
     raise ValueError("Attempting to set the keypoint candidate ranking mode "
-                     "to {}, but the only options are 'min_distance' and "
-                     "'score_distance_ratio'.".format(mode))
+                     "to {}, but the only options are 'min_distance', "
+                     "'score_distance_ratio', 'score_scaled_distance_ratio', "
+                     "'gaussian_weighted'.".format(mode))
   meta_architecture = model_config.WhichOneof("model")
   if meta_architecture == "center_net":
     if len(model_config.center_net.keypoint_estimation_task) == 1:
@@ -1214,3 +1222,50 @@ def _update_unmatched_keypoint_score(model_config, score):
                          "unmatched_keypoint_score since there are multiple "
                          "keypoint estimation tasks")
 
+
+def _update_score_distance_multiplier(model_config, score_distance_multiplier):
+  """Updates the keypoint candidate selection metric. See CenterNet proto."""
+  meta_architecture = model_config.WhichOneof("model")
+  if meta_architecture == "center_net":
+    if len(model_config.center_net.keypoint_estimation_task) == 1:
+      kpt_estimation_task = model_config.center_net.keypoint_estimation_task[0]
+      kpt_estimation_task.score_distance_multiplier = score_distance_multiplier
+    else:
+      tf.logging.warning("Ignoring config override key for "
+                         "score_distance_multiplier since there are multiple "
+                         "keypoint estimation tasks")
+  else:
+    raise ValueError(
+        "Unsupported meta_architecture type: %s" % meta_architecture)
+
+
+def _update_std_dev_multiplier(model_config, std_dev_multiplier):
+  """Updates the keypoint candidate selection metric. See CenterNet proto."""
+  meta_architecture = model_config.WhichOneof("model")
+  if meta_architecture == "center_net":
+    if len(model_config.center_net.keypoint_estimation_task) == 1:
+      kpt_estimation_task = model_config.center_net.keypoint_estimation_task[0]
+      kpt_estimation_task.std_dev_multiplier = std_dev_multiplier
+    else:
+      tf.logging.warning("Ignoring config override key for "
+                         "std_dev_multiplier since there are multiple "
+                         "keypoint estimation tasks")
+  else:
+    raise ValueError(
+        "Unsupported meta_architecture type: %s" % meta_architecture)
+
+
+def _update_rescoring_threshold(model_config, rescoring_threshold):
+  """Updates the keypoint candidate selection metric. See CenterNet proto."""
+  meta_architecture = model_config.WhichOneof("model")
+  if meta_architecture == "center_net":
+    if len(model_config.center_net.keypoint_estimation_task) == 1:
+      kpt_estimation_task = model_config.center_net.keypoint_estimation_task[0]
+      kpt_estimation_task.rescoring_threshold = rescoring_threshold
+    else:
+      tf.logging.warning("Ignoring config override key for "
+                         "rescoring_threshold since there are multiple "
+                         "keypoint estimation tasks")
+  else:
+    raise ValueError(
+        "Unsupported meta_architecture type: %s" % meta_architecture)

research/object_detection/utils/np_box_ops.py

@@ -70,7 +70,7 @@ def iou(boxes1, boxes2):
 
   Args:
     boxes1: a numpy array with shape [N, 4] holding N boxes.
-    boxes2: a numpy array with shape [M, 4] holding N boxes.
+    boxes2: a numpy array with shape [M, 4] holding M boxes.
 
   Returns:
     a numpy array with shape [N, M] representing pairwise iou scores.
@@ -92,7 +92,7 @@ def ioa(boxes1, boxes2):
 
   Args:
     boxes1: a numpy array with shape [N, 4] holding N boxes.
-    boxes2: a numpy array with shape [M, 4] holding N boxes.
+    boxes2: a numpy array with shape [M, 4] holding M boxes.
 
   Returns:
     a numpy array with shape [N, M] representing pairwise ioa scores.

research/object_detection/utils/target_assigner_utils.py

@@ -16,6 +16,8 @@
 
 import tensorflow.compat.v1 as tf
 
+from object_detection.core import box_list
+from object_detection.core import box_list_ops
 from object_detection.utils import shape_utils
 
 
@@ -290,7 +292,8 @@ def get_valid_keypoint_mask_for_class(keypoint_coordinates,
 
 
 def blackout_pixel_weights_by_box_regions(height, width, boxes, blackout,
-                                          weights=None):
+                                          weights=None,
+                                          boxes_scale=1.0):
   """Apply weights at pixel locations.
 
   This function is used to generate the pixel weight mask (usually in the output
@@ -332,6 +335,10 @@ def blackout_pixel_weights_by_box_regions(height, width, boxes, blackout,
       a value to apply in each box region. Note that if blackout=True for a
       given box, the weight will be zero. If None, all weights are assumed to be
       1.
+    boxes_scale: The amount to scale the height/width of the boxes before
+      constructing the blackout regions. This is often useful to guarantee that
+      the proper weight fully covers the object boxes/masks during supervision,
+      as shifting might occur during image resizing, network stride, etc.
 
   Returns:
     A float tensor with shape [height, width] where all values within the
@@ -347,6 +354,10 @@ def blackout_pixel_weights_by_box_regions(height, width, boxes, blackout,
   (y_grid, x_grid) = image_shape_to_grids(height, width)
   y_grid = tf.expand_dims(y_grid, axis=0)
   x_grid = tf.expand_dims(x_grid, axis=0)
+  boxlist = box_list.BoxList(boxes)
+  boxlist = box_list_ops.scale_height_width(
+      boxlist, y_scale=boxes_scale, x_scale=boxes_scale)
+  boxes = boxlist.get()
   y_min = tf.expand_dims(boxes[:, 0:1], axis=-1)
   x_min = tf.expand_dims(boxes[:, 1:2], axis=-1)
   y_max = tf.expand_dims(boxes[:, 2:3], axis=-1)