Merge remote-tracking branch 'upstream/master'

research/object_detection/builders/hyperparams_builder_test.py

@@ -460,6 +460,11 @@ class HyperparamsBuilderTest(tf.test.TestCase):
     keras_config = hyperparams_builder.KerasLayerHyperparams(
         conv_hyperparams_proto)
     self.assertEqual(keras_config.params()['activation'], None)
+    self.assertEqual(
+        keras_config.params(include_activation=True)['activation'], None)
+    activation_layer = keras_config.build_activation_layer()
+    self.assertTrue(isinstance(activation_layer, tf.keras.layers.Lambda))
+    self.assertEqual(activation_layer.function, tf.identity)
 
   def test_use_relu_activation(self):
     conv_hyperparams_text_proto = """
@@ -497,7 +502,12 @@ class HyperparamsBuilderTest(tf.test.TestCase):
     text_format.Merge(conv_hyperparams_text_proto, conv_hyperparams_proto)
     keras_config = hyperparams_builder.KerasLayerHyperparams(
         conv_hyperparams_proto)
-    self.assertEqual(keras_config.params()['activation'], tf.nn.relu)
+    self.assertEqual(keras_config.params()['activation'], None)
+    self.assertEqual(
+        keras_config.params(include_activation=True)['activation'], tf.nn.relu)
+    activation_layer = keras_config.build_activation_layer()
+    self.assertTrue(isinstance(activation_layer, tf.keras.layers.Lambda))
+    self.assertEqual(activation_layer.function, tf.nn.relu)
 
   def test_use_relu_6_activation(self):
     conv_hyperparams_text_proto = """
@@ -535,7 +545,12 @@ class HyperparamsBuilderTest(tf.test.TestCase):
     text_format.Merge(conv_hyperparams_text_proto, conv_hyperparams_proto)
     keras_config = hyperparams_builder.KerasLayerHyperparams(
         conv_hyperparams_proto)
-    self.assertEqual(keras_config.params()['activation'], tf.nn.relu6)
+    self.assertEqual(keras_config.params()['activation'], None)
+    self.assertEqual(
+        keras_config.params(include_activation=True)['activation'], tf.nn.relu6)
+    activation_layer = keras_config.build_activation_layer()
+    self.assertTrue(isinstance(activation_layer, tf.keras.layers.Lambda))
+    self.assertEqual(activation_layer.function, tf.nn.relu6)
 
   def test_override_activation_keras(self):
     conv_hyperparams_text_proto = """

research/object_detection/builders/input_reader_builder_test.py

@@ -21,11 +21,10 @@ import tensorflow as tf
 
 from google.protobuf import text_format
 
-from tensorflow.core.example import example_pb2
-from tensorflow.core.example import feature_pb2
 from object_detection.builders import input_reader_builder
 from object_detection.core import standard_fields as fields
 from object_detection.protos import input_reader_pb2
+from object_detection.utils import dataset_util
 
 
 class InputReaderBuilderTest(tf.test.TestCase):
@@ -38,27 +37,17 @@ class InputReaderBuilderTest(tf.test.TestCase):
     flat_mask = (4 * 5) * [1.0]
     with self.test_session():
       encoded_jpeg = tf.image.encode_jpeg(tf.constant(image_tensor)).eval()
-    example = example_pb2.Example(features=feature_pb2.Features(feature={
-        'image/encoded': feature_pb2.Feature(
-            bytes_list=feature_pb2.BytesList(value=[encoded_jpeg])),
-        'image/format': feature_pb2.Feature(
-            bytes_list=feature_pb2.BytesList(value=['jpeg'.encode('utf-8')])),
-        'image/height': feature_pb2.Feature(
-            int64_list=feature_pb2.Int64List(value=[4])),
-        'image/width': feature_pb2.Feature(
-            int64_list=feature_pb2.Int64List(value=[5])),
-        'image/object/bbox/xmin': feature_pb2.Feature(
-            float_list=feature_pb2.FloatList(value=[0.0])),
-        'image/object/bbox/xmax': feature_pb2.Feature(
-            float_list=feature_pb2.FloatList(value=[1.0])),
-        'image/object/bbox/ymin': feature_pb2.Feature(
-            float_list=feature_pb2.FloatList(value=[0.0])),
-        'image/object/bbox/ymax': feature_pb2.Feature(
-            float_list=feature_pb2.FloatList(value=[1.0])),
-        'image/object/class/label': feature_pb2.Feature(
-            int64_list=feature_pb2.Int64List(value=[2])),
-        'image/object/mask': feature_pb2.Feature(
-            float_list=feature_pb2.FloatList(value=flat_mask)),
+    example = tf.train.Example(features=tf.train.Features(feature={
+        'image/encoded': dataset_util.bytes_feature(encoded_jpeg),
+        'image/format': dataset_util.bytes_feature('jpeg'.encode('utf8')),
+        'image/height': dataset_util.int64_feature(4),
+        'image/width': dataset_util.int64_feature(5),
+        'image/object/bbox/xmin': dataset_util.float_list_feature([0.0]),
+        'image/object/bbox/xmax': dataset_util.float_list_feature([1.0]),
+        'image/object/bbox/ymin': dataset_util.float_list_feature([0.0]),
+        'image/object/bbox/ymax': dataset_util.float_list_feature([1.0]),
+        'image/object/class/label': dataset_util.int64_list_feature([2]),
+        'image/object/mask': dataset_util.float_list_feature(flat_mask),
     }))
     writer.write(example.SerializeToString())
     writer.close()
@@ -79,9 +68,7 @@ class InputReaderBuilderTest(tf.test.TestCase):
     text_format.Merge(input_reader_text_proto, input_reader_proto)
     tensor_dict = input_reader_builder.build(input_reader_proto)
 
-    sv = tf.train.Supervisor(logdir=self.get_temp_dir())
-    with sv.prepare_or_wait_for_session() as sess:
-      sv.start_queue_runners(sess)
+    with tf.train.MonitoredSession() as sess:
       output_dict = sess.run(tensor_dict)
 
     self.assertTrue(fields.InputDataFields.groundtruth_instance_masks
@@ -111,9 +98,7 @@ class InputReaderBuilderTest(tf.test.TestCase):
     text_format.Merge(input_reader_text_proto, input_reader_proto)
     tensor_dict = input_reader_builder.build(input_reader_proto)
 
-    sv = tf.train.Supervisor(logdir=self.get_temp_dir())
-    with sv.prepare_or_wait_for_session() as sess:
-      sv.start_queue_runners(sess)
+    with tf.train.MonitoredSession() as sess:
       output_dict = sess.run(tensor_dict)
 
     self.assertEquals(

research/object_detection/builders/model_builder.py

@@ -14,7 +14,9 @@
 # ==============================================================================
 
 """A function to build a DetectionModel from configuration."""
+
 import functools
+
 from object_detection.builders import anchor_generator_builder
 from object_detection.builders import box_coder_builder
 from object_detection.builders import box_predictor_builder
@@ -25,6 +27,7 @@ from object_detection.builders import matcher_builder
 from object_detection.builders import post_processing_builder
 from object_detection.builders import region_similarity_calculator_builder as sim_calc
 from object_detection.core import balanced_positive_negative_sampler as sampler
+from object_detection.core import post_processing
 from object_detection.core import target_assigner
 from object_detection.meta_architectures import faster_rcnn_meta_arch
 from object_detection.meta_architectures import rfcn_meta_arch
@@ -43,11 +46,11 @@ from object_detection.models.ssd_mobilenet_v1_feature_extractor import SSDMobile
 from object_detection.models.ssd_mobilenet_v1_fpn_feature_extractor import SSDMobileNetV1FpnFeatureExtractor
 from object_detection.models.ssd_mobilenet_v1_ppn_feature_extractor import SSDMobileNetV1PpnFeatureExtractor
 from object_detection.models.ssd_mobilenet_v2_feature_extractor import SSDMobileNetV2FeatureExtractor
+from object_detection.models.ssd_mobilenet_v2_fpn_feature_extractor import SSDMobileNetV2FpnFeatureExtractor
 from object_detection.predictors import rfcn_box_predictor
 from object_detection.protos import model_pb2
 from object_detection.utils import ops
 
-
 # A map of names to SSD feature extractors.
 SSD_FEATURE_EXTRACTOR_CLASS_MAP = {
     'ssd_inception_v2': SSDInceptionV2FeatureExtractor,
@@ -56,6 +59,7 @@ SSD_FEATURE_EXTRACTOR_CLASS_MAP = {
     'ssd_mobilenet_v1_fpn': SSDMobileNetV1FpnFeatureExtractor,
     'ssd_mobilenet_v1_ppn': SSDMobileNetV1PpnFeatureExtractor,
     'ssd_mobilenet_v2': SSDMobileNetV2FeatureExtractor,
+    'ssd_mobilenet_v2_fpn': SSDMobileNetV2FpnFeatureExtractor,
     'ssd_resnet50_v1_fpn': ssd_resnet_v1_fpn.SSDResnet50V1FpnFeatureExtractor,
     'ssd_resnet101_v1_fpn': ssd_resnet_v1_fpn.SSDResnet101V1FpnFeatureExtractor,
     'ssd_resnet152_v1_fpn': ssd_resnet_v1_fpn.SSDResnet152V1FpnFeatureExtractor,
@@ -170,8 +174,12 @@ def _build_ssd_feature_extractor(feature_extractor_config, is_training,
 
   if feature_extractor_config.HasField('fpn'):
     kwargs.update({
-        'fpn_min_level': feature_extractor_config.fpn.min_level,
-        'fpn_max_level': feature_extractor_config.fpn.max_level,
+        'fpn_min_level':
+            feature_extractor_config.fpn.min_level,
+        'fpn_max_level':
+            feature_extractor_config.fpn.max_level,
+        'additional_layer_depth':
+            feature_extractor_config.fpn.additional_layer_depth,
     })
 
   return feature_extractor_class(**kwargs)
@@ -240,25 +248,24 @@ def _build_ssd_model(ssd_config, is_training, add_summaries,
         desired_negative_sampling_ratio=ssd_config.
         desired_negative_sampling_ratio)
 
-  return ssd_meta_arch.SSDMetaArch(
-      is_training,
-      anchor_generator,
-      ssd_box_predictor,
-      box_coder,
-      feature_extractor,
-      matcher,
-      region_similarity_calculator,
-      encode_background_as_zeros,
-      negative_class_weight,
-      image_resizer_fn,
-      non_max_suppression_fn,
-      score_conversion_fn,
-      classification_loss,
-      localization_loss,
-      classification_weight,
-      localization_weight,
-      normalize_loss_by_num_matches,
-      hard_example_miner,
+  ssd_meta_arch_fn = ssd_meta_arch.SSDMetaArch
+
+  return ssd_meta_arch_fn(
+      is_training=is_training,
+      anchor_generator=anchor_generator,
+      box_predictor=ssd_box_predictor,
+      box_coder=box_coder,
+      feature_extractor=feature_extractor,
+      encode_background_as_zeros=encode_background_as_zeros,
+      image_resizer_fn=image_resizer_fn,
+      non_max_suppression_fn=non_max_suppression_fn,
+      score_conversion_fn=score_conversion_fn,
+      classification_loss=classification_loss,
+      localization_loss=localization_loss,
+      classification_loss_weight=classification_weight,
+      localization_loss_weight=localization_weight,
+      normalize_loss_by_num_matches=normalize_loss_by_num_matches,
+      hard_example_miner=hard_example_miner,
       target_assigner_instance=target_assigner_instance,
       add_summaries=add_summaries,
       normalize_loc_loss_by_codesize=normalize_loc_loss_by_codesize,
@@ -350,12 +357,27 @@ def _build_faster_rcnn_model(frcnn_config, is_training, add_summaries):
       frcnn_config.first_stage_box_predictor_kernel_size)
   first_stage_box_predictor_depth = frcnn_config.first_stage_box_predictor_depth
   first_stage_minibatch_size = frcnn_config.first_stage_minibatch_size
+  # TODO(bhattad): When eval is supported using static shapes, add separate
+  # use_static_shapes_for_trainig and use_static_shapes_for_evaluation.
+  use_static_shapes = frcnn_config.use_static_shapes and is_training
   first_stage_sampler = sampler.BalancedPositiveNegativeSampler(
       positive_fraction=frcnn_config.first_stage_positive_balance_fraction,
-      is_static=frcnn_config.use_static_balanced_label_sampler)
-  first_stage_nms_score_threshold = frcnn_config.first_stage_nms_score_threshold
-  first_stage_nms_iou_threshold = frcnn_config.first_stage_nms_iou_threshold
+      is_static=frcnn_config.use_static_balanced_label_sampler and is_training)
   first_stage_max_proposals = frcnn_config.first_stage_max_proposals
+  if (frcnn_config.first_stage_nms_iou_threshold < 0 or
+      frcnn_config.first_stage_nms_iou_threshold > 1.0):
+    raise ValueError('iou_threshold not in [0, 1.0].')
+  if (is_training and frcnn_config.second_stage_batch_size >
+      first_stage_max_proposals):
+    raise ValueError('second_stage_batch_size should be no greater than '
+                     'first_stage_max_proposals.')
+  first_stage_non_max_suppression_fn = functools.partial(
+      post_processing.batch_multiclass_non_max_suppression,
+      score_thresh=frcnn_config.first_stage_nms_score_threshold,
+      iou_thresh=frcnn_config.first_stage_nms_iou_threshold,
+      max_size_per_class=frcnn_config.first_stage_max_proposals,
+      max_total_size=frcnn_config.first_stage_max_proposals,
+      use_static_shapes=use_static_shapes and is_training)
   first_stage_loc_loss_weight = (
       frcnn_config.first_stage_localization_loss_weight)
   first_stage_obj_loss_weight = frcnn_config.first_stage_objectness_loss_weight
@@ -376,7 +398,7 @@ def _build_faster_rcnn_model(frcnn_config, is_training, add_summaries):
   second_stage_batch_size = frcnn_config.second_stage_batch_size
   second_stage_sampler = sampler.BalancedPositiveNegativeSampler(
       positive_fraction=frcnn_config.second_stage_balance_fraction,
-      is_static=frcnn_config.use_static_balanced_label_sampler)
+      is_static=frcnn_config.use_static_balanced_label_sampler and is_training)
   (second_stage_non_max_suppression_fn, second_stage_score_conversion_fn
   ) = post_processing_builder.build(frcnn_config.second_stage_post_processing)
   second_stage_localization_loss_weight = (
@@ -396,7 +418,9 @@ def _build_faster_rcnn_model(frcnn_config, is_training, add_summaries):
         second_stage_classification_loss_weight,
         second_stage_localization_loss_weight)
 
-  use_matmul_crop_and_resize = (frcnn_config.use_matmul_crop_and_resize)
+  crop_and_resize_fn = (
+      ops.matmul_crop_and_resize if frcnn_config.use_matmul_crop_and_resize
+      else ops.native_crop_and_resize)
   clip_anchors_to_image = (
       frcnn_config.clip_anchors_to_image)
 
@@ -416,8 +440,7 @@ def _build_faster_rcnn_model(frcnn_config, is_training, add_summaries):
       'first_stage_box_predictor_depth': first_stage_box_predictor_depth,
       'first_stage_minibatch_size': first_stage_minibatch_size,
       'first_stage_sampler': first_stage_sampler,
-      'first_stage_nms_score_threshold': first_stage_nms_score_threshold,
-      'first_stage_nms_iou_threshold': first_stage_nms_iou_threshold,
+      'first_stage_non_max_suppression_fn': first_stage_non_max_suppression_fn,
       'first_stage_max_proposals': first_stage_max_proposals,
       'first_stage_localization_loss_weight': first_stage_loc_loss_weight,
       'first_stage_objectness_loss_weight': first_stage_obj_loss_weight,
@@ -435,8 +458,10 @@ def _build_faster_rcnn_model(frcnn_config, is_training, add_summaries):
       second_stage_classification_loss_weight,
       'hard_example_miner': hard_example_miner,
       'add_summaries': add_summaries,
-      'use_matmul_crop_and_resize': use_matmul_crop_and_resize,
-      'clip_anchors_to_image': clip_anchors_to_image
+      'crop_and_resize_fn': crop_and_resize_fn,
+      'clip_anchors_to_image': clip_anchors_to_image,
+      'use_static_shapes': use_static_shapes,
+      'resize_masks': frcnn_config.resize_masks
   }
 
   if isinstance(second_stage_box_predictor,

research/object_detection/builders/model_builder_test.py

@@ -15,6 +15,8 @@
 
 """Tests for object_detection.models.model_builder."""
 
+from absl.testing import parameterized
+
 import tensorflow as tf
 
 from google.protobuf import text_format
@@ -36,6 +38,7 @@ from object_detection.models.ssd_mobilenet_v1_feature_extractor import SSDMobile
 from object_detection.models.ssd_mobilenet_v1_fpn_feature_extractor import SSDMobileNetV1FpnFeatureExtractor
 from object_detection.models.ssd_mobilenet_v1_ppn_feature_extractor import SSDMobileNetV1PpnFeatureExtractor
 from object_detection.models.ssd_mobilenet_v2_feature_extractor import SSDMobileNetV2FeatureExtractor
+from object_detection.models.ssd_mobilenet_v2_fpn_feature_extractor import SSDMobileNetV2FpnFeatureExtractor
 from object_detection.protos import model_pb2
 
 FRCNN_RESNET_FEAT_MAPS = {
@@ -66,7 +69,7 @@ SSD_RESNET_V1_PPN_FEAT_MAPS = {
 }
 
 
-class ModelBuilderTest(tf.test.TestCase):
+class ModelBuilderTest(tf.test.TestCase, parameterized.TestCase):
 
   def create_model(self, model_config):
     """Builds a DetectionModel based on the model config.
@@ -161,6 +164,7 @@ class ModelBuilderTest(tf.test.TestCase):
             'desired_negative_sampling_ratio': 2
         })
 
+
   def test_create_ssd_inception_v3_model_from_config(self):
     model_text_proto = """
       ssd {
@@ -712,6 +716,170 @@ class ModelBuilderTest(tf.test.TestCase):
     self.assertTrue(model._normalize_loc_loss_by_codesize)
     self.assertTrue(model._target_assigner._weight_regression_loss_by_score)
 
+  def test_create_ssd_mobilenet_v2_fpn_model_from_config(self):
+    model_text_proto = """
+      ssd {
+        freeze_batchnorm: true
+        inplace_batchnorm_update: true
+        feature_extractor {
+          type: 'ssd_mobilenet_v2_fpn'
+          fpn {
+            min_level: 3
+            max_level: 7
+          }
+          conv_hyperparams {
+            regularizer {
+                l2_regularizer {
+                }
+              }
+              initializer {
+                truncated_normal_initializer {
+                }
+              }
+          }
+        }
+        box_coder {
+          faster_rcnn_box_coder {
+          }
+        }
+        matcher {
+          argmax_matcher {
+          }
+        }
+        similarity_calculator {
+          iou_similarity {
+          }
+        }
+        anchor_generator {
+          ssd_anchor_generator {
+            aspect_ratios: 1.0
+          }
+        }
+        image_resizer {
+          fixed_shape_resizer {
+            height: 320
+            width: 320
+          }
+        }
+        box_predictor {
+          convolutional_box_predictor {
+            conv_hyperparams {
+              regularizer {
+                l2_regularizer {
+                }
+              }
+              initializer {
+                truncated_normal_initializer {
+                }
+              }
+            }
+          }
+        }
+        normalize_loc_loss_by_codesize: true
+        loss {
+          classification_loss {
+            weighted_softmax {
+            }
+          }
+          localization_loss {
+            weighted_smooth_l1 {
+            }
+          }
+        }
+      }"""
+    model_proto = model_pb2.DetectionModel()
+    text_format.Merge(model_text_proto, model_proto)
+    model = self.create_model(model_proto)
+    self.assertIsInstance(model, ssd_meta_arch.SSDMetaArch)
+    self.assertIsInstance(model._feature_extractor,
+                          SSDMobileNetV2FpnFeatureExtractor)
+    self.assertTrue(model._normalize_loc_loss_by_codesize)
+    self.assertTrue(model._freeze_batchnorm)
+    self.assertTrue(model._inplace_batchnorm_update)
+
+  def test_create_ssd_mobilenet_v2_fpnlite_model_from_config(self):
+    model_text_proto = """
+      ssd {
+        freeze_batchnorm: true
+        inplace_batchnorm_update: true
+        feature_extractor {
+          type: 'ssd_mobilenet_v2_fpn'
+          use_depthwise: true
+          fpn {
+            min_level: 3
+            max_level: 7
+            additional_layer_depth: 128
+          }
+          conv_hyperparams {
+            regularizer {
+                l2_regularizer {
+                }
+              }
+              initializer {
+                truncated_normal_initializer {
+                }
+              }
+          }
+        }
+        box_coder {
+          faster_rcnn_box_coder {
+          }
+        }
+        matcher {
+          argmax_matcher {
+          }
+        }
+        similarity_calculator {
+          iou_similarity {
+          }
+        }
+        anchor_generator {
+          ssd_anchor_generator {
+            aspect_ratios: 1.0
+          }
+        }
+        image_resizer {
+          fixed_shape_resizer {
+            height: 320
+            width: 320
+          }
+        }
+        box_predictor {
+          convolutional_box_predictor {
+            conv_hyperparams {
+              regularizer {
+                l2_regularizer {
+                }
+              }
+              initializer {
+                truncated_normal_initializer {
+                }
+              }
+            }
+          }
+        }
+        normalize_loc_loss_by_codesize: true
+        loss {
+          classification_loss {
+            weighted_softmax {
+            }
+          }
+          localization_loss {
+            weighted_smooth_l1 {
+            }
+          }
+        }
+      }"""
+    model_proto = model_pb2.DetectionModel()
+    text_format.Merge(model_text_proto, model_proto)
+    model = self.create_model(model_proto)
+    self.assertIsInstance(model, ssd_meta_arch.SSDMetaArch)
+    self.assertIsInstance(model._feature_extractor,
+                          SSDMobileNetV2FpnFeatureExtractor)
+    self.assertTrue(model._normalize_loc_loss_by_codesize)
+    self.assertTrue(model._freeze_batchnorm)
+    self.assertTrue(model._inplace_batchnorm_update)
+
   def test_create_embedded_ssd_mobilenet_v1_model_from_config(self):
     model_text_proto = """
       ssd {
@@ -845,13 +1013,19 @@ class ModelBuilderTest(tf.test.TestCase):
       }"""
     model_proto = model_pb2.DetectionModel()
     text_format.Merge(model_text_proto, model_proto)
+
     for extractor_type, extractor_class in FRCNN_RESNET_FEAT_MAPS.items():
       model_proto.faster_rcnn.feature_extractor.type = extractor_type
       model = model_builder.build(model_proto, is_training=True)
       self.assertIsInstance(model, faster_rcnn_meta_arch.FasterRCNNMetaArch)
       self.assertIsInstance(model._feature_extractor, extractor_class)
 
-  def test_create_faster_rcnn_resnet101_with_mask_prediction_enabled(self):
+  @parameterized.parameters(
+      {'use_matmul_crop_and_resize': False},
+      {'use_matmul_crop_and_resize': True},
+  )
+  def test_create_faster_rcnn_resnet101_with_mask_prediction_enabled(
+      self, use_matmul_crop_and_resize):
     model_text_proto = """
       faster_rcnn {
         num_classes: 3
@@ -924,6 +1098,8 @@ class ModelBuilderTest(tf.test.TestCase):
       }"""
     model_proto = model_pb2.DetectionModel()
     text_format.Merge(model_text_proto, model_proto)
+    model_proto.faster_rcnn.use_matmul_crop_and_resize = (
+        use_matmul_crop_and_resize)
     model = model_builder.build(model_proto, is_training=True)
     self.assertAlmostEqual(model._second_stage_mask_loss_weight, 3.0)
 

research/object_detection/builders/post_processing_builder.py

@@ -84,7 +84,8 @@ def _build_non_max_suppressor(nms_config):
       score_thresh=nms_config.score_threshold,
       iou_thresh=nms_config.iou_threshold,
       max_size_per_class=nms_config.max_detections_per_class,
-      max_total_size=nms_config.max_total_detections)
+      max_total_size=nms_config.max_total_detections,
+      use_static_shapes=nms_config.use_static_shapes)
   return non_max_suppressor_fn
 
 

research/object_detection/core/balanced_positive_negative_sampler.py

@@ -24,6 +24,10 @@ for obtaining the desired batch_size, it returns fewer examples.
 The main function to call is Subsample(self, indicator, labels). For convenience
 one can also call SubsampleWeights(self, weights, labels) which is defined in
 the minibatch_sampler base class.
+
+When is_static is True, it implements a method that guarantees static shapes.
+It also ensures the length of output of the subsample is always batch_size, even
+when number of examples set to True in indicator is less than batch_size.
 """
 
 import tensorflow as tf
@@ -102,13 +106,14 @@ class BalancedPositiveNegativeSampler(minibatch_sampler.MinibatchSampler):
     end_positions = tf.greater_equal(
         tf.range(input_length), input_length - num_end_samples)
     selected_positions = tf.logical_or(start_positions, end_positions)
-    selected_positions = tf.cast(selected_positions, tf.int32)
+    selected_positions = tf.cast(selected_positions, tf.float32)
     indexed_positions = tf.multiply(tf.cumsum(selected_positions),
                                     selected_positions)
-    one_hot_selector = tf.one_hot(indexed_positions - 1,
+    one_hot_selector = tf.one_hot(tf.cast(indexed_positions, tf.int32) - 1,
                                   total_num_samples,
-                                  dtype=tf.int32)
-    return tf.tensordot(input_tensor, one_hot_selector, axes=[0, 0])
+                                  dtype=tf.float32)
+    return tf.cast(tf.tensordot(tf.cast(input_tensor, tf.float32),
+                                one_hot_selector, axes=[0, 0]), tf.int32)
 
   def _static_subsample(self, indicator, batch_size, labels):
     """Returns subsampled minibatch.
@@ -122,7 +127,9 @@ class BalancedPositiveNegativeSampler(minibatch_sampler.MinibatchSampler):
 
     Returns:
       sampled_idx_indicator: boolean tensor of shape [N], True for entries which
-        are sampled.
+        are sampled. It ensures the length of output of the subsample is always
+        batch_size, even when number of examples set to True in indicator is
+        less than batch_size.
 
     Raises:
       ValueError: if labels and indicator are not 1D boolean tensors.
@@ -140,6 +147,14 @@ class BalancedPositiveNegativeSampler(minibatch_sampler.MinibatchSampler):
 
     input_length = tf.shape(indicator)[0]
 
+    # Set the number of examples set True in indicator to be at least
+    # batch_size.
+    num_true_sampled = tf.reduce_sum(tf.cast(indicator, tf.float32))
+    additional_false_sample = tf.less_equal(
+        tf.cumsum(tf.cast(tf.logical_not(indicator), tf.float32)),
+        batch_size - num_true_sampled)
+    indicator = tf.logical_or(indicator, additional_false_sample)
+
     # Shuffle indicator and label. Need to store the permutation to restore the
     # order post sampling.
     permutation = tf.random_shuffle(tf.range(input_length))
@@ -148,7 +163,7 @@ class BalancedPositiveNegativeSampler(minibatch_sampler.MinibatchSampler):
     labels = ops.matmul_gather_on_zeroth_axis(
         tf.cast(labels, tf.float32), permutation)
 
-    # index (starting from 1) when cls_weight is True, 0 when False
+    # index (starting from 1) when indicator is True, 0 when False
     indicator_idx = tf.where(
         tf.cast(indicator, tf.bool), tf.range(1, input_length + 1),
         tf.zeros(input_length, tf.int32))
@@ -183,9 +198,10 @@ class BalancedPositiveNegativeSampler(minibatch_sampler.MinibatchSampler):
         axis=0), tf.bool)
 
     # project back the order based on stored permutations
-    reprojections = tf.one_hot(permutation, depth=input_length, dtype=tf.int32)
+    reprojections = tf.one_hot(permutation, depth=input_length,
+                               dtype=tf.float32)
     return tf.cast(tf.tensordot(
-        tf.cast(sampled_idx_indicator, tf.int32),
+        tf.cast(sampled_idx_indicator, tf.float32),
         reprojections, axes=[0, 0]), tf.bool)
 
   def subsample(self, indicator, batch_size, labels, scope=None):

research/object_detection/core/balanced_positive_negative_sampler_test.py

@@ -24,7 +24,7 @@ from object_detection.utils import test_case
 
 class BalancedPositiveNegativeSamplerTest(test_case.TestCase):
 
-  def _test_subsample_all_examples(self, is_static=False):
+  def test_subsample_all_examples_dynamic(self):
     numpy_labels = np.random.permutation(300)
     indicator = tf.constant(np.ones(300) == 1)
     numpy_labels = (numpy_labels - 200) > 0
@@ -32,8 +32,7 @@ class BalancedPositiveNegativeSamplerTest(test_case.TestCase):
     labels = tf.constant(numpy_labels)
 
     sampler = (
-        balanced_positive_negative_sampler.BalancedPositiveNegativeSampler(
-            is_static=is_static))
+        balanced_positive_negative_sampler.BalancedPositiveNegativeSampler())
     is_sampled = sampler.subsample(indicator, 64, labels)
     with self.test_session() as sess:
       is_sampled = sess.run(is_sampled)
@@ -42,13 +41,26 @@ class BalancedPositiveNegativeSamplerTest(test_case.TestCase):
       self.assertTrue(sum(np.logical_and(
           np.logical_not(numpy_labels), is_sampled)) == 32)
 
-  def test_subsample_all_examples_dynamic(self):
-    self._test_subsample_all_examples()
-
   def test_subsample_all_examples_static(self):
-    self._test_subsample_all_examples(is_static=True)
+    numpy_labels = np.random.permutation(300)
+    indicator = np.array(np.ones(300) == 1, np.bool)
+    numpy_labels = (numpy_labels - 200) > 0
+
+    labels = np.array(numpy_labels, np.bool)
+
+    def graph_fn(indicator, labels):
+      sampler = (
+          balanced_positive_negative_sampler.BalancedPositiveNegativeSampler(
+              is_static=True))
+      return sampler.subsample(indicator, 64, labels)
+
+    is_sampled = self.execute(graph_fn, [indicator, labels])
+    self.assertTrue(sum(is_sampled) == 64)
+    self.assertTrue(sum(np.logical_and(numpy_labels, is_sampled)) == 32)
+    self.assertTrue(sum(np.logical_and(
+        np.logical_not(numpy_labels), is_sampled)) == 32)
 
-  def _test_subsample_selection(self, is_static=False):
+  def test_subsample_selection_dynamic(self):
     # Test random sampling when only some examples can be sampled:
     # 100 samples, 20 positives, 10 positives cannot be sampled
     numpy_labels = np.arange(100)
@@ -59,8 +71,7 @@ class BalancedPositiveNegativeSamplerTest(test_case.TestCase):
     labels = tf.constant(numpy_labels)
 
     sampler = (
-        balanced_positive_negative_sampler.BalancedPositiveNegativeSampler(
-            is_static=is_static))
+        balanced_positive_negative_sampler.BalancedPositiveNegativeSampler())
     is_sampled = sampler.subsample(indicator, 64, labels)
     with self.test_session() as sess:
       is_sampled = sess.run(is_sampled)
@@ -71,13 +82,30 @@ class BalancedPositiveNegativeSamplerTest(test_case.TestCase):
       self.assertAllEqual(is_sampled, np.logical_and(is_sampled,
                                                      numpy_indicator))
 
-  def test_subsample_selection_dynamic(self):
-    self._test_subsample_selection()
-
   def test_subsample_selection_static(self):
-    self._test_subsample_selection(is_static=True)
+    # Test random sampling when only some examples can be sampled:
+    # 100 samples, 20 positives, 10 positives cannot be sampled.
+    numpy_labels = np.arange(100)
+    numpy_indicator = numpy_labels < 90
+    indicator = np.array(numpy_indicator, np.bool)
+    numpy_labels = (numpy_labels - 80) >= 0
 
-  def _test_subsample_selection_larger_batch_size(self, is_static=False):
+    labels = np.array(numpy_labels, np.bool)
+
+    def graph_fn(indicator, labels):
+      sampler = (
+          balanced_positive_negative_sampler.BalancedPositiveNegativeSampler(
+              is_static=True))
+      return sampler.subsample(indicator, 64, labels)
+
+    is_sampled = self.execute(graph_fn, [indicator, labels])
+    self.assertTrue(sum(is_sampled) == 64)
+    self.assertTrue(sum(np.logical_and(numpy_labels, is_sampled)) == 10)
+    self.assertTrue(sum(np.logical_and(
+        np.logical_not(numpy_labels), is_sampled)) == 54)
+    self.assertAllEqual(is_sampled, np.logical_and(is_sampled, numpy_indicator))
+
+  def test_subsample_selection_larger_batch_size_dynamic(self):
     # Test random sampling when total number of examples that can be sampled are
     # less than batch size:
     # 100 samples, 50 positives, 40 positives cannot be sampled, batch size 64.
@@ -89,8 +117,7 @@ class BalancedPositiveNegativeSamplerTest(test_case.TestCase):
     labels = tf.constant(numpy_labels)
 
     sampler = (
-        balanced_positive_negative_sampler.BalancedPositiveNegativeSampler(
-            is_static=is_static))
+        balanced_positive_negative_sampler.BalancedPositiveNegativeSampler())
     is_sampled = sampler.subsample(indicator, 64, labels)
     with self.test_session() as sess:
       is_sampled = sess.run(is_sampled)
@@ -101,11 +128,31 @@ class BalancedPositiveNegativeSamplerTest(test_case.TestCase):
       self.assertAllEqual(is_sampled, np.logical_and(is_sampled,
                                                      numpy_indicator))
 
-  def test_subsample_selection_larger_batch_size_dynamic(self):
-    self._test_subsample_selection_larger_batch_size()
-
   def test_subsample_selection_larger_batch_size_static(self):
-    self._test_subsample_selection_larger_batch_size(is_static=True)
+    # Test random sampling when total number of examples that can be sampled are
+    # less than batch size:
+    # 100 samples, 50 positives, 40 positives cannot be sampled, batch size 64.
+    # It should still return 64 samples, with 4 of them that couldn't have been
+    # sampled.
+    numpy_labels = np.arange(100)
+    numpy_indicator = numpy_labels < 60
+    indicator = np.array(numpy_indicator, np.bool)
+    numpy_labels = (numpy_labels - 50) >= 0
+
+    labels = np.array(numpy_labels, np.bool)
+
+    def graph_fn(indicator, labels):
+      sampler = (
+          balanced_positive_negative_sampler.BalancedPositiveNegativeSampler(
+              is_static=True))
+      return sampler.subsample(indicator, 64, labels)
+
+    is_sampled = self.execute(graph_fn, [indicator, labels])
+    self.assertTrue(sum(is_sampled) == 64)
+    self.assertTrue(sum(np.logical_and(numpy_labels, is_sampled)) >= 10)
+    self.assertTrue(
+        sum(np.logical_and(np.logical_not(numpy_labels), is_sampled)) >= 50)
+    self.assertTrue(sum(np.logical_and(is_sampled, numpy_indicator)) == 60)
 
   def test_subsample_selection_no_batch_size(self):
     # Test random sampling when only some examples can be sampled:

research/object_detection/core/box_list_ops.py

@@ -26,6 +26,7 @@ BoxList are retained unless documented otherwise.
 import tensorflow as tf
 
 from object_detection.core import box_list
+from object_detection.utils import ops
 from object_detection.utils import shape_utils
 
 
@@ -420,7 +421,8 @@ def sq_dist(boxlist1, boxlist2, scope=None):
     return sqnorm1 + tf.transpose(sqnorm2) - 2.0 * innerprod
 
 
-def boolean_mask(boxlist, indicator, fields=None, scope=None):
+def boolean_mask(boxlist, indicator, fields=None, scope=None,
+                 use_static_shapes=False, indicator_sum=None):
   """Select boxes from BoxList according to indicator and return new BoxList.
 
   `boolean_mask` returns the subset of boxes that are marked as "True" by the
@@ -436,6 +438,10 @@ def boolean_mask(boxlist, indicator, fields=None, scope=None):
       all fields are gathered from.  Pass an empty fields list to only gather
       the box coordinates.
     scope: name scope.
+    use_static_shapes: Whether to use an implementation with static shape
+      gurantees.
+    indicator_sum: An integer containing the sum of `indicator` vector. Only
+      required if `use_static_shape` is True.
 
   Returns:
     subboxlist: a BoxList corresponding to the subset of the input BoxList
@@ -448,18 +454,36 @@ def boolean_mask(boxlist, indicator, fields=None, scope=None):
       raise ValueError('indicator should have rank 1')
     if indicator.dtype != tf.bool:
       raise ValueError('indicator should be a boolean tensor')
-    subboxlist = box_list.BoxList(tf.boolean_mask(boxlist.get(), indicator))
-    if fields is None:
-      fields = boxlist.get_extra_fields()
-    for field in fields:
-      if not boxlist.has_field(field):
-        raise ValueError('boxlist must contain all specified fields')
-      subfieldlist = tf.boolean_mask(boxlist.get_field(field), indicator)
-      subboxlist.add_field(field, subfieldlist)
-    return subboxlist
+    if use_static_shapes:
+      if not (indicator_sum and isinstance(indicator_sum, int)):
+        raise ValueError('`indicator_sum` must be a of type int')
+      selected_positions = tf.to_float(indicator)
+      indexed_positions = tf.cast(
+          tf.multiply(
+              tf.cumsum(selected_positions), selected_positions),
+          dtype=tf.int32)
+      one_hot_selector = tf.one_hot(
+          indexed_positions - 1, indicator_sum, dtype=tf.float32)
+      sampled_indices = tf.cast(
+          tf.tensordot(
+              tf.to_float(tf.range(tf.shape(indicator)[0])),
+              one_hot_selector,
+              axes=[0, 0]),
+          dtype=tf.int32)
+      return gather(boxlist, sampled_indices, use_static_shapes=True)
+    else:
+      subboxlist = box_list.BoxList(tf.boolean_mask(boxlist.get(), indicator))
+      if fields is None:
+        fields = boxlist.get_extra_fields()
+      for field in fields:
+        if not boxlist.has_field(field):
+          raise ValueError('boxlist must contain all specified fields')
+        subfieldlist = tf.boolean_mask(boxlist.get_field(field), indicator)
+        subboxlist.add_field(field, subfieldlist)
+      return subboxlist
 
 
-def gather(boxlist, indices, fields=None, scope=None):
+def gather(boxlist, indices, fields=None, scope=None, use_static_shapes=False):
   """Gather boxes from BoxList according to indices and return new BoxList.
 
   By default, `gather` returns boxes corresponding to the input index list, as
@@ -474,6 +498,8 @@ def gather(boxlist, indices, fields=None, scope=None):
       all fields are gathered from.  Pass an empty fields list to only gather
       the box coordinates.
     scope: name scope.
+    use_static_shapes: Whether to use an implementation with static shape
+      gurantees.
 
   Returns:
     subboxlist: a BoxList corresponding to the subset of the input BoxList
@@ -487,13 +513,17 @@ def gather(boxlist, indices, fields=None, scope=None):
       raise ValueError('indices should have rank 1')
     if indices.dtype != tf.int32 and indices.dtype != tf.int64:
       raise ValueError('indices should be an int32 / int64 tensor')
-    subboxlist = box_list.BoxList(tf.gather(boxlist.get(), indices))
+    gather_op = tf.gather
+    if use_static_shapes:
+      gather_op = ops.matmul_gather_on_zeroth_axis
+    subboxlist = box_list.BoxList(gather_op(boxlist.get(), indices))
     if fields is None:
       fields = boxlist.get_extra_fields()
+    fields += ['boxes']
     for field in fields:
       if not boxlist.has_field(field):
         raise ValueError('boxlist must contain all specified fields')
-      subfieldlist = tf.gather(boxlist.get_field(field), indices)
+      subfieldlist = gather_op(boxlist.get_field(field), indices)
       subboxlist.add_field(field, subfieldlist)
     return subboxlist
 
@@ -585,10 +615,7 @@ def sort_by_field(boxlist, field, order=SortOrder.descend, scope=None):
         ['Incorrect field size: actual vs expected.', num_entries, num_boxes])
 
     with tf.control_dependencies([length_assert]):
-      # TODO(derekjchow): Remove with tf.device when top_k operation runs
-      # correctly on GPU.
-      with tf.device('/cpu:0'):
-        _, sorted_indices = tf.nn.top_k(field_to_sort, num_boxes, sorted=True)
+      _, sorted_indices = tf.nn.top_k(field_to_sort, num_boxes, sorted=True)
 
     if order == SortOrder.ascend:
       sorted_indices = tf.reverse_v2(sorted_indices, [0])
@@ -1059,3 +1086,51 @@ def get_minimal_coverage_box(boxlist,
         tf.greater_equal(num_boxes, 1),
         true_fn=lambda: coverage_box(boxlist.get()),
         false_fn=lambda: default_box)
+
+
+def sample_boxes_by_jittering(boxlist,
+                              num_boxes_to_sample,
+                              stddev=0.1,
+                              scope=None):
+  """Samples num_boxes_to_sample boxes by jittering around boxlist boxes.
+
+  It is possible that this function might generate boxes with size 0. The larger
+  the stddev, this is more probable. For a small stddev of 0.1 this probability
+  is very small.
+
+  Args:
+    boxlist: A boxlist containing N boxes in normalized coordinates.
+    num_boxes_to_sample: A positive integer containing the number of boxes to
+      sample.
+    stddev: Standard deviation. This is used to draw random offsets for the
+      box corners from a normal distribution. The offset is multiplied by the
+      box size so will be larger in terms of pixels for larger boxes.
+    scope: Name scope.
+
+  Returns:
+    sampled_boxlist: A boxlist containing num_boxes_to_sample boxes in
+      normalized coordinates.
+  """
+  with tf.name_scope(scope, 'SampleBoxesByJittering'):
+    num_boxes = boxlist.num_boxes()
+    box_indices = tf.random_uniform(
+        [num_boxes_to_sample],
+        minval=0,
+        maxval=num_boxes,
+        dtype=tf.int32)
+    sampled_boxes = tf.gather(boxlist.get(), box_indices)
+    sampled_boxes_height = sampled_boxes[:, 2] - sampled_boxes[:, 0]
+    sampled_boxes_width = sampled_boxes[:, 3] - sampled_boxes[:, 1]
+    rand_miny_gaussian = tf.random_normal([num_boxes_to_sample], stddev=stddev)
+    rand_minx_gaussian = tf.random_normal([num_boxes_to_sample], stddev=stddev)
+    rand_maxy_gaussian = tf.random_normal([num_boxes_to_sample], stddev=stddev)
+    rand_maxx_gaussian = tf.random_normal([num_boxes_to_sample], stddev=stddev)
+    miny = rand_miny_gaussian * sampled_boxes_height + sampled_boxes[:, 0]
+    minx = rand_minx_gaussian * sampled_boxes_width + sampled_boxes[:, 1]
+    maxy = rand_maxy_gaussian * sampled_boxes_height + sampled_boxes[:, 2]
+    maxx = rand_maxx_gaussian * sampled_boxes_width + sampled_boxes[:, 3]
+    maxy = tf.maximum(miny, maxy)
+    maxx = tf.maximum(minx, maxx)
+    sampled_boxes = tf.stack([miny, minx, maxy, maxx], axis=1)
+    sampled_boxes = tf.maximum(tf.minimum(sampled_boxes, 1.0), 0.0)
+    return box_list.BoxList(sampled_boxes)

research/object_detection/core/box_list_ops_test.py

@@ -16,14 +16,13 @@
 """Tests for object_detection.core.box_list_ops."""
 import numpy as np
 import tensorflow as tf
-from tensorflow.python.framework import errors
-from tensorflow.python.framework import ops
 
 from object_detection.core import box_list
 from object_detection.core import box_list_ops
+from object_detection.utils import test_case
 
 
-class BoxListOpsTest(tf.test.TestCase):
+class BoxListOpsTest(test_case.TestCase):
   """Tests for common bounding box operations."""
 
   def test_area(self):
@@ -364,11 +363,35 @@ class BoxListOpsTest(tf.test.TestCase):
       subset_output = sess.run(subset.get())
       self.assertAllClose(subset_output, expected_subset)
 
-  def test_boolean_mask_with_field(self):
-    corners = tf.constant(
-        [4 * [0.0], 4 * [1.0], 4 * [2.0], 4 * [3.0], 4 * [4.0]])
-    indicator = tf.constant([True, False, True, False, True], tf.bool)
-    weights = tf.constant([[.1], [.3], [.5], [.7], [.9]], tf.float32)
+  def test_static_boolean_mask_with_field(self):
+
+    def graph_fn(corners, weights, indicator):
+      boxes = box_list.BoxList(corners)
+      boxes.add_field('weights', weights)
+      subset = box_list_ops.boolean_mask(
+          boxes,
+          indicator, ['weights'],
+          use_static_shapes=True,
+          indicator_sum=3)
+      return (subset.get_field('boxes'), subset.get_field('weights'))
+
+    corners = np.array(
+        [4 * [0.0], 4 * [1.0], 4 * [2.0], 4 * [3.0], 4 * [4.0]],
+        dtype=np.float32)
+    indicator = np.array([True, False, True, False, True], dtype=np.bool)
+    weights = np.array([[.1], [.3], [.5], [.7], [.9]], dtype=np.float32)
+    result_boxes, result_weights = self.execute(graph_fn,
+                                                [corners, weights, indicator])
+    expected_boxes = [4 * [0.0], 4 * [2.0], 4 * [4.0]]
+    expected_weights = [[.1], [.5], [.9]]
+
+    self.assertAllClose(result_boxes, expected_boxes)
+    self.assertAllClose(result_weights, expected_weights)
+
+  def test_dynamic_boolean_mask_with_field(self):
+    corners = tf.placeholder(tf.float32, [None, 4])
+    indicator = tf.placeholder(tf.bool, [None])
+    weights = tf.placeholder(tf.float32, [None, 1])
     expected_subset = [4 * [0.0], 4 * [2.0], 4 * [4.0]]
     expected_weights = [[.1], [.5], [.9]]
 
@@ -377,7 +400,16 @@ class BoxListOpsTest(tf.test.TestCase):
     subset = box_list_ops.boolean_mask(boxes, indicator, ['weights'])
     with self.test_session() as sess:
       subset_output, weights_output = sess.run(
-          [subset.get(), subset.get_field('weights')])
+          [subset.get(), subset.get_field('weights')],
+          feed_dict={
+              corners:
+                  np.array(
+                      [4 * [0.0], 4 * [1.0], 4 * [2.0], 4 * [3.0], 4 * [4.0]]),
+              indicator:
+                  np.array([True, False, True, False, True]).astype(np.bool),
+              weights:
+                  np.array([[.1], [.3], [.5], [.7], [.9]])
+          })
       self.assertAllClose(subset_output, expected_subset)
       self.assertAllClose(weights_output, expected_weights)
 
@@ -392,19 +424,50 @@ class BoxListOpsTest(tf.test.TestCase):
       subset_output = sess.run(subset.get())
       self.assertAllClose(subset_output, expected_subset)
 
-  def test_gather_with_field(self):
-    corners = tf.constant([4*[0.0], 4*[1.0], 4*[2.0], 4*[3.0], 4*[4.0]])
-    indices = tf.constant([0, 2, 4], tf.int32)
-    weights = tf.constant([[.1], [.3], [.5], [.7], [.9]], tf.float32)
+  def test_static_gather_with_field(self):
+
+    def graph_fn(corners, weights, indices):
+      boxes = box_list.BoxList(corners)
+      boxes.add_field('weights', weights)
+      subset = box_list_ops.gather(
+          boxes, indices, ['weights'], use_static_shapes=True)
+      return (subset.get_field('boxes'), subset.get_field('weights'))
+
+    corners = np.array([4 * [0.0], 4 * [1.0], 4 * [2.0], 4 * [3.0],
+                        4 * [4.0]], dtype=np.float32)
+    weights = np.array([[.1], [.3], [.5], [.7], [.9]], dtype=np.float32)
+    indices = np.array([0, 2, 4], dtype=np.int32)
+
+    result_boxes, result_weights = self.execute(graph_fn,
+                                                [corners, weights, indices])
+    expected_boxes = [4 * [0.0], 4 * [2.0], 4 * [4.0]]
+    expected_weights = [[.1], [.5], [.9]]
+    self.assertAllClose(result_boxes, expected_boxes)
+    self.assertAllClose(result_weights, expected_weights)
+
+  def test_dynamic_gather_with_field(self):
+    corners = tf.placeholder(tf.float32, [None, 4])
+    indices = tf.placeholder(tf.int32, [None])
+    weights = tf.placeholder(tf.float32, [None, 1])
     expected_subset = [4 * [0.0], 4 * [2.0], 4 * [4.0]]
     expected_weights = [[.1], [.5], [.9]]
 
     boxes = box_list.BoxList(corners)
     boxes.add_field('weights', weights)
-    subset = box_list_ops.gather(boxes, indices, ['weights'])
+    subset = box_list_ops.gather(boxes, indices, ['weights'],
+                                 use_static_shapes=True)
     with self.test_session() as sess:
       subset_output, weights_output = sess.run(
-          [subset.get(), subset.get_field('weights')])
+          [subset.get(), subset.get_field('weights')],
+          feed_dict={
+              corners:
+                  np.array(
+                      [4 * [0.0], 4 * [1.0], 4 * [2.0], 4 * [3.0], 4 * [4.0]]),
+              indices:
+                  np.array([0, 2, 4]).astype(np.int32),
+              weights:
+                  np.array([[.1], [.3], [.5], [.7], [.9]])
+          })
       self.assertAllClose(subset_output, expected_subset)
       self.assertAllClose(weights_output, expected_weights)
 
@@ -503,20 +566,14 @@ class BoxListOpsTest(tf.test.TestCase):
     boxes.add_field('misc', misc)
     boxes.add_field('weights', weights)
 
-    with self.test_session() as sess:
-      with self.assertRaises(ValueError):
-        box_list_ops.sort_by_field(boxes, 'area')
+    with self.assertRaises(ValueError):
+      box_list_ops.sort_by_field(boxes, 'area')
 
-      with self.assertRaises(ValueError):
-        box_list_ops.sort_by_field(boxes, 'misc')
+    with self.assertRaises(ValueError):
+      box_list_ops.sort_by_field(boxes, 'misc')
 
-      if ops._USE_C_API:
-        with self.assertRaises(ValueError):
-          box_list_ops.sort_by_field(boxes, 'weights')
-      else:
-        with self.assertRaisesWithPredicateMatch(errors.InvalidArgumentError,
-                                                 'Incorrect field size'):
-          sess.run(box_list_ops.sort_by_field(boxes, 'weights').get())
+    with self.assertRaises(ValueError):
+      box_list_ops.sort_by_field(boxes, 'weights')
 
   def test_visualize_boxes_in_image(self):
     image = tf.zeros((6, 4, 3))
@@ -1031,6 +1088,21 @@ class BoxRefinementTest(tf.test.TestCase):
       self.assertAllClose(expected_scores, scores_out)
       self.assertAllEqual(extra_field_out, [0, 1, 1])
 
+  def test_sample_boxes_by_jittering(self):
+    boxes = box_list.BoxList(
+        tf.constant([[0.1, 0.1, 0.4, 0.4],
+                     [0.1, 0.1, 0.5, 0.5],
+                     [0.6, 0.6, 0.8, 0.8],
+                     [0.2, 0.2, 0.3, 0.3]], tf.float32))
+    sampled_boxes = box_list_ops.sample_boxes_by_jittering(
+        boxlist=boxes, num_boxes_to_sample=10)
+    iou = box_list_ops.iou(boxes, sampled_boxes)
+    iou_max = tf.reduce_max(iou, axis=0)
+    with self.test_session() as sess:
+      (np_sampled_boxes, np_iou_max) = sess.run([sampled_boxes.get(), iou_max])
+      self.assertAllEqual(np_sampled_boxes.shape, [10, 4])
+      self.assertAllGreater(np_iou_max, 0.5)
+
 
 if __name__ == '__main__':
   tf.test.main()

research/object_detection/core/box_predictor.py

@@ -138,7 +138,7 @@ class KerasBoxPredictor(tf.keras.Model):
   """Keras-based BoxPredictor."""
 
   def __init__(self, is_training, num_classes, freeze_batchnorm,
-               inplace_batchnorm_update):
+               inplace_batchnorm_update, name=None):
     """Constructor.
 
     Args:
@@ -155,8 +155,10 @@ class KerasBoxPredictor(tf.keras.Model):
         values inplace. When this is false train op must add a control
         dependency on tf.graphkeys.UPDATE_OPS collection in order to update
         batch norm statistics.
+      name: A string name scope to assign to the model. If `None`, Keras
+        will auto-generate one from the class name.
     """
-    super(KerasBoxPredictor, self).__init__()
+    super(KerasBoxPredictor, self).__init__(name=name)
 
     self._is_training = is_training
     self._num_classes = num_classes
@@ -171,7 +173,7 @@ class KerasBoxPredictor(tf.keras.Model):
   def num_classes(self):
     return self._num_classes
 
-  def call(self, image_features, scope=None, **kwargs):
+  def call(self, image_features, **kwargs):
     """Computes encoded object locations and corresponding confidences.
 
     Takes a list of high level image feature maps as input and produces a list
@@ -181,9 +183,8 @@ class KerasBoxPredictor(tf.keras.Model):
     Args:
       image_features: A list of float tensors of shape [batch_size, height_i,
       width_i, channels_i] containing features for a batch of images.
-      scope: Variable and Op scope name.
       **kwargs: Additional keyword arguments for specific implementations of
-              BoxPredictor.
+            BoxPredictor.
 
     Returns:
       A dictionary containing at least the following tensors.

research/object_detection/core/losses.py

@@ -46,6 +46,7 @@ class Loss(object):
                prediction_tensor,
                target_tensor,
                ignore_nan_targets=False,
+               losses_mask=None,
                scope=None,
                **params):
     """Call the loss function.
@@ -58,6 +59,11 @@ class Loss(object):
       ignore_nan_targets: whether to ignore nan targets in the loss computation.
         E.g. can be used if the target tensor is missing groundtruth data that
         shouldn't be factored into the loss.
+      losses_mask: A [batch] boolean tensor that indicates whether losses should
+        be applied to individual images in the batch. For elements that
+        are True, corresponding prediction, target, and weight tensors will be
+        removed prior to loss computation. If None, no filtering will take place
+        prior to loss computation.
       scope: Op scope name. Defaults to 'Loss' if None.
       **params: Additional keyword arguments for specific implementations of
               the Loss.
@@ -71,8 +77,25 @@ class Loss(object):
         target_tensor = tf.where(tf.is_nan(target_tensor),
                                  prediction_tensor,
                                  target_tensor)
+      if losses_mask is not None:
+        tensor_multiplier = self._get_loss_multiplier_for_tensor(
+            prediction_tensor,
+            losses_mask)
+        prediction_tensor *= tensor_multiplier
+        target_tensor *= tensor_multiplier
+
+        if 'weights' in params:
+          params['weights'] = tf.convert_to_tensor(params['weights'])
+          weights_multiplier = self._get_loss_multiplier_for_tensor(
+              params['weights'],
+              losses_mask)
+          params['weights'] *= weights_multiplier
       return self._compute_loss(prediction_tensor, target_tensor, **params)
 
+  def _get_loss_multiplier_for_tensor(self, tensor, losses_mask):
+    loss_multiplier_shape = tf.stack([-1] + [1] * (len(tensor.shape) - 1))
+    return tf.cast(tf.reshape(losses_mask, loss_multiplier_shape), tf.float32)
+
   @abstractmethod
   def _compute_loss(self, prediction_tensor, target_tensor, **params):
     """Method to be overridden by implementations.

research/object_detection/core/losses_test.py

@@ -79,6 +79,26 @@ class WeightedL2LocalizationLossTest(tf.test.TestCase):
       loss_output = sess.run(loss)
       self.assertAllClose(loss_output, expected_loss)
 
+  def testReturnsCorrectWeightedLossWithLossesMask(self):
+    batch_size = 4
+    num_anchors = 10
+    code_size = 4
+    prediction_tensor = tf.ones([batch_size, num_anchors, code_size])
+    target_tensor = tf.zeros([batch_size, num_anchors, code_size])
+    weights = tf.constant([[1, 1, 1, 1, 1, 0, 0, 0, 0, 0],
+                           [1, 1, 1, 1, 1, 1, 1, 1, 0, 0],
+                           [1, 1, 1, 1, 1, 0, 0, 0, 0, 0],
+                           [1, 1, 1, 1, 1, 0, 0, 0, 0, 0]], tf.float32)
+    losses_mask = tf.constant([True, False, True, True], tf.bool)
+    loss_op = losses.WeightedL2LocalizationLoss()
+    loss = tf.reduce_sum(loss_op(prediction_tensor, target_tensor,
+                                 weights=weights, losses_mask=losses_mask))
+
+    expected_loss = (3 * 5 * 4) / 2.0
+    with self.test_session() as sess:
+      loss_output = sess.run(loss)
+      self.assertAllClose(loss_output, expected_loss)
+
 
 class WeightedSmoothL1LocalizationLossTest(tf.test.TestCase):
 
@@ -104,6 +124,34 @@ class WeightedSmoothL1LocalizationLossTest(tf.test.TestCase):
       loss_output = sess.run(loss)
       self.assertAllClose(loss_output, exp_loss)
 
+  def testReturnsCorrectLossWithLossesMask(self):
+    batch_size = 3
+    num_anchors = 3
+    code_size = 4
+    prediction_tensor = tf.constant([[[2.5, 0, .4, 0],
+                                      [0, 0, 0, 0],
+                                      [0, 2.5, 0, .4]],
+                                     [[3.5, 0, 0, 0],
+                                      [0, .4, 0, .9],
+                                      [0, 0, 1.5, 0]],
+                                     [[3.5, 7., 0, 0],
+                                      [0, .4, 0, .9],
+                                      [2.2, 2.2, 1.5, 0]]], tf.float32)
+    target_tensor = tf.zeros([batch_size, num_anchors, code_size])
+    weights = tf.constant([[2, 1, 1],
+                           [0, 3, 0],
+                           [4, 3, 0]], tf.float32)
+    losses_mask = tf.constant([True, True, False], tf.bool)
+    loss_op = losses.WeightedSmoothL1LocalizationLoss()
+    loss = loss_op(prediction_tensor, target_tensor, weights=weights,
+                   losses_mask=losses_mask)
+    loss = tf.reduce_sum(loss)
+
+    exp_loss = 7.695
+    with self.test_session() as sess:
+      loss_output = sess.run(loss)
+      self.assertAllClose(loss_output, exp_loss)
+
 
 class WeightedIOULocalizationLossTest(tf.test.TestCase):
 
@@ -123,6 +171,24 @@ class WeightedIOULocalizationLossTest(tf.test.TestCase):
       loss_output = sess.run(loss)
       self.assertAllClose(loss_output, exp_loss)
 
+  def testReturnsCorrectLossWithNoLabels(self):
+    prediction_tensor = tf.constant([[[1.5, 0, 2.4, 1],
+                                      [0, 0, 1, 1],
+                                      [0, 0, .5, .25]]])
+    target_tensor = tf.constant([[[1.5, 0, 2.4, 1],
+                                  [0, 0, 1, 1],
+                                  [50, 50, 500.5, 100.25]]])
+    weights = [[1.0, .5, 2.0]]
+    losses_mask = tf.constant([False], tf.bool)
+    loss_op = losses.WeightedIOULocalizationLoss()
+    loss = loss_op(prediction_tensor, target_tensor, weights=weights,
+                   losses_mask=losses_mask)
+    loss = tf.reduce_sum(loss)
+    exp_loss = 0.0
+    with self.test_session() as sess:
+      loss_output = sess.run(loss)
+      self.assertAllClose(loss_output, exp_loss)
+
 
 class WeightedSigmoidClassificationLossTest(tf.test.TestCase):
 
@@ -215,6 +281,50 @@ class WeightedSigmoidClassificationLossTest(tf.test.TestCase):
       loss_output = sess.run(loss)
       self.assertAllClose(loss_output, exp_loss)
 
+  def testReturnsCorrectLossWithLossesMask(self):
+    prediction_tensor = tf.constant([[[-100, 100, -100],
+                                      [100, -100, -100],
+                                      [100, 0, -100],
+                                      [-100, -100, 100]],
+                                     [[-100, 0, 100],
+                                      [-100, 100, -100],
+                                      [100, 100, 100],
+                                      [0, 0, -1]],
+                                     [[-100, 0, 100],
+                                      [-100, 100, -100],
+                                      [100, 100, 100],
+                                      [0, 0, -100]]], tf.float32)
+    target_tensor = tf.constant([[[0, 1, 0],
+                                  [1, 0, 0],
+                                  [1, 0, 0],
+                                  [0, 0, 1]],
+                                 [[0, 0, 1],
+                                  [0, 1, 0],
+                                  [1, 1, 1],
+                                  [1, 0, 0]],
+                                 [[0, 0, 0],
+                                  [0, 0, 0],
+                                  [0, 0, 0],
+                                  [0, 0, 0]]], tf.float32)
+    weights = tf.constant([[1, 1, 1, 1],
+                           [1, 1, 1, 0],
+                           [1, 1, 1, 1]], tf.float32)
+    losses_mask = tf.constant([True, True, False], tf.bool)
+
+    loss_op = losses.WeightedSigmoidClassificationLoss()
+    loss_per_anchor = loss_op(prediction_tensor, target_tensor, weights=weights,
+                              losses_mask=losses_mask)
+    loss = tf.reduce_sum(loss_per_anchor)
+
+    exp_loss = -2 * math.log(.5)
+    with self.test_session() as sess:
+      loss_output = sess.run(loss)
+      self.assertAllEqual(prediction_tensor.shape.as_list(),
+                          loss_per_anchor.shape.as_list())
+      self.assertAllEqual(target_tensor.shape.as_list(),
+                          loss_per_anchor.shape.as_list())
+      self.assertAllClose(loss_output, exp_loss)
+
 
 def _logit(probability):
   return math.log(probability / (1. - probability))
@@ -484,6 +594,51 @@ class SigmoidFocalClassificationLossTest(tf.test.TestCase):
              8 * 2))),      # negatives from 8 anchors for two classes.
           focal_loss)
 
+  def testExpectedLossWithLossesMask(self):
+    # All zeros correspond to 0.5 probability.
+    prediction_tensor = tf.constant([[[0, 0, 0],
+                                      [0, 0, 0],
+                                      [0, 0, 0],
+                                      [0, 0, 0]],
+                                     [[0, 0, 0],
+                                      [0, 0, 0],
+                                      [0, 0, 0],
+                                      [0, 0, 0]],
+                                     [[0, 0, 0],
+                                      [0, 0, 0],
+                                      [0, 0, 0],
+                                      [0, 0, 0]]], tf.float32)
+    target_tensor = tf.constant([[[0, 1, 0],
+                                  [1, 0, 0],
+                                  [1, 0, 0],
+                                  [0, 0, 1]],
+                                 [[0, 0, 1],
+                                  [0, 1, 0],
+                                  [1, 0, 0],
+                                  [1, 0, 0]],
+                                 [[1, 0, 0],
+                                  [1, 0, 0],
+                                  [1, 0, 0],
+                                  [1, 0, 0]]], tf.float32)
+    weights = tf.constant([[1, 1, 1, 1],
+                           [1, 1, 1, 1],
+                           [1, 1, 1, 1]], tf.float32)
+    losses_mask = tf.constant([True, True, False], tf.bool)
+    focal_loss_op = losses.SigmoidFocalClassificationLoss(alpha=0.75, gamma=0.0)
+
+    focal_loss = tf.reduce_sum(focal_loss_op(prediction_tensor, target_tensor,
+                                             weights=weights,
+                                             losses_mask=losses_mask))
+    with self.test_session() as sess:
+      focal_loss = sess.run(focal_loss)
+      self.assertAllClose(
+          (-math.log(.5) *  # x-entropy per class per anchor.
+           ((0.75 *         # alpha for positives.
+             8) +           # positives from 8 anchors.
+            (0.25 *         # alpha for negatives.
+             8 * 2))),      # negatives from 8 anchors for two classes.
+          focal_loss)
+
 
 class WeightedSoftmaxClassificationLossTest(tf.test.TestCase):
 
@@ -575,6 +730,45 @@ class WeightedSoftmaxClassificationLossTest(tf.test.TestCase):
       loss_output = sess.run(loss)
       self.assertAllClose(loss_output, exp_loss)
 
+  def testReturnsCorrectLossWithLossesMask(self):
+    prediction_tensor = tf.constant([[[-100, 100, -100],
+                                      [100, -100, -100],
+                                      [0, 0, -100],
+                                      [-100, -100, 100]],
+                                     [[-100, 0, 0],
+                                      [-100, 100, -100],
+                                      [-100, 100, -100],
+                                      [100, -100, -100]],
+                                     [[-100, 0, 0],
+                                      [-100, 100, -100],
+                                      [-100, 100, -100],
+                                      [100, -100, -100]]], tf.float32)
+    target_tensor = tf.constant([[[0, 1, 0],
+                                  [1, 0, 0],
+                                  [1, 0, 0],
+                                  [0, 0, 1]],
+                                 [[0, 0, 1],
+                                  [0, 1, 0],
+                                  [0, 1, 0],
+                                  [1, 0, 0]],
+                                 [[1, 0, 0],
+                                  [1, 0, 0],
+                                  [1, 0, 0],
+                                  [1, 0, 0]]], tf.float32)
+    weights = tf.constant([[1, 1, .5, 1],
+                           [1, 1, 1, 0],
+                           [1, 1, 1, 1]], tf.float32)
+    losses_mask = tf.constant([True, True, False], tf.bool)
+    loss_op = losses.WeightedSoftmaxClassificationLoss()
+    loss = loss_op(prediction_tensor, target_tensor, weights=weights,
+                   losses_mask=losses_mask)
+    loss = tf.reduce_sum(loss)
+
+    exp_loss = - 1.5 * math.log(.5)
+    with self.test_session() as sess:
+      loss_output = sess.run(loss)
+      self.assertAllClose(loss_output, exp_loss)
+
 
 class WeightedSoftmaxClassificationAgainstLogitsLossTest(tf.test.TestCase):
 

research/object_detection/core/matcher.py

@@ -219,7 +219,7 @@ class Matcher(object):
     """
     self._use_matmul_gather = use_matmul_gather
 
-  def match(self, similarity_matrix, scope=None, **params):
+  def match(self, similarity_matrix, valid_rows=None, scope=None):
     """Computes matches among row and column indices and returns the result.
 
     Computes matches among the row and column indices based on the similarity
@@ -228,27 +228,28 @@ class Matcher(object):
     Args:
       similarity_matrix: Float tensor of shape [N, M] with pairwise similarity
         where higher value means more similar.
+      valid_rows: A boolean tensor of shape [N] indicating the rows that are
+        valid for matching.
       scope: Op scope name. Defaults to 'Match' if None.
-      **params: Additional keyword arguments for specific implementations of
-        the Matcher.
 
     Returns:
       A Match object with the results of matching.
     """
-    with tf.name_scope(scope, 'Match', [similarity_matrix, params]) as scope:
-      return Match(self._match(similarity_matrix, **params),
+    with tf.name_scope(scope, 'Match') as scope:
+      if valid_rows is None:
+        valid_rows = tf.ones(tf.shape(similarity_matrix)[0], dtype=tf.bool)
+      return Match(self._match(similarity_matrix, valid_rows),
                    self._use_matmul_gather)
 
   @abstractmethod
-  def _match(self, similarity_matrix, **params):
+  def _match(self, similarity_matrix, valid_rows):
     """Method to be overridden by implementations.
 
     Args:
       similarity_matrix: Float tensor of shape [N, M] with pairwise similarity
         where higher value means more similar.
-      **params: Additional keyword arguments for specific implementations of
-        the Matcher.
-
+      valid_rows: A boolean tensor of shape [N] indicating the rows that are
+        valid for matching.
     Returns:
       match_results: Integer tensor of shape [M]: match_results[i]>=0 means
         that column i is matched to row match_results[i], match_results[i]=-1

research/object_detection/core/model.py

@@ -84,7 +84,8 @@ class DetectionModel(object):
 
     Args:
       field: a string key, options are
-        fields.BoxListFields.{boxes,classes,masks,keypoints}
+        fields.BoxListFields.{boxes,classes,masks,keypoints} or
+        fields.InputDataFields.is_annotated.
 
     Returns:
       a list of tensors holding groundtruth information (see also
@@ -94,7 +95,8 @@ class DetectionModel(object):
       RuntimeError: if the field has not been provided via provide_groundtruth.
     """
     if field not in self._groundtruth_lists:
-      raise RuntimeError('Groundtruth tensor %s has not been provided', field)
+      raise RuntimeError('Groundtruth tensor {} has not been provided'.format(
+          field))
     return self._groundtruth_lists[field]
 
   def groundtruth_has_field(self, field):
@@ -102,7 +104,8 @@ class DetectionModel(object):
 
     Args:
       field: a string key, options are
-        fields.BoxListFields.{boxes,classes,masks,keypoints}
+        fields.BoxListFields.{boxes,classes,masks,keypoints} or
+        fields.InputDataFields.is_annotated.
 
     Returns:
       True if the groundtruth includes the given field, False otherwise.
@@ -238,7 +241,8 @@ class DetectionModel(object):
                           groundtruth_masks_list=None,
                           groundtruth_keypoints_list=None,
                           groundtruth_weights_list=None,
-                          groundtruth_is_crowd_list=None):
+                          groundtruth_is_crowd_list=None,
+                          is_annotated_list=None):
     """Provide groundtruth tensors.
 
     Args:
@@ -263,6 +267,8 @@ class DetectionModel(object):
         [num_boxes] containing weights for groundtruth boxes.
       groundtruth_is_crowd_list: A list of 1-D tf.bool tensors of shape
         [num_boxes] containing is_crowd annotations
+      is_annotated_list: A list of scalar tf.bool tensors indicating whether
+        images have been labeled or not.
     """
     self._groundtruth_lists[fields.BoxListFields.boxes] = groundtruth_boxes_list
     self._groundtruth_lists[
@@ -279,6 +285,9 @@ class DetectionModel(object):
     if groundtruth_is_crowd_list:
       self._groundtruth_lists[
           fields.BoxListFields.is_crowd] = groundtruth_is_crowd_list
+    if is_annotated_list:
+      self._groundtruth_lists[
+          fields.InputDataFields.is_annotated] = is_annotated_list
 
   @abstractmethod
   def restore_map(self, fine_tune_checkpoint_type='detection'):

research/object_detection/core/post_processing.py

@@ -33,6 +33,7 @@ def multiclass_non_max_suppression(boxes,
                                    change_coordinate_frame=False,
                                    masks=None,
                                    boundaries=None,
+                                   pad_to_max_output_size=False,
                                    additional_fields=None,
                                    scope=None):
   """Multi-class version of non maximum suppression.
@@ -55,7 +56,8 @@ def multiclass_non_max_suppression(boxes,
       number of classes or 1 depending on whether a separate box is predicted
       per class.
     scores: A [k, num_classes] float32 tensor containing the scores for each of
-      the k detections.
+      the k detections. The scores have to be non-negative when
+      pad_to_max_output_size is True.
     score_thresh: scalar threshold for score (low scoring boxes are removed).
     iou_thresh: scalar threshold for IOU (new boxes that have high IOU overlap
       with previously selected boxes are removed).
@@ -74,6 +76,8 @@ def multiclass_non_max_suppression(boxes,
     boundaries: (optional) a [k, q, boundary_height, boundary_width] float32
       tensor containing box boundaries. `q` can be either number of classes or 1
       depending on whether a separate boundary is predicted per class.
+    pad_to_max_output_size: If true, the output nmsed boxes are padded to be of
+      length `max_size_per_class`. Defaults to false.
     additional_fields: (optional) If not None, a dictionary that maps keys to
       tensors whose first dimensions are all of size `k`. After non-maximum
       suppression, all tensors corresponding to the selected boxes will be
@@ -81,9 +85,12 @@ def multiclass_non_max_suppression(boxes,
     scope: name scope.
 
   Returns:
-    a BoxList holding M boxes with a rank-1 scores field representing
+    A tuple of sorted_boxes and num_valid_nms_boxes. The sorted_boxes is a
+      BoxList holds M boxes with a rank-1 scores field representing
       corresponding scores for each box with scores sorted in decreasing order
-      and a rank-1 classes field representing a class label for each box.
+      and a rank-1 classes field representing a class label for each box. The
+      num_valid_nms_boxes is a 0-D integer tensor representing the number of
+      valid elements in `BoxList`, with the valid elements appearing first.
 
   Raises:
     ValueError: if iou_thresh is not in [0, 1] or if input boxlist does not have
@@ -113,6 +120,7 @@ def multiclass_non_max_suppression(boxes,
     num_classes = scores.get_shape()[1]
 
     selected_boxes_list = []
+    num_valid_nms_boxes_cumulative = tf.constant(0)
     per_class_boxes_list = tf.unstack(boxes, axis=1)
     if masks is not None:
       per_class_masks_list = tf.unstack(masks, axis=1)
@@ -140,16 +148,40 @@ def multiclass_non_max_suppression(boxes,
         for key, tensor in additional_fields.items():
           boxlist_and_class_scores.add_field(key, tensor)
 
-      max_selection_size = tf.minimum(max_size_per_class,
-                                      boxlist_and_class_scores.num_boxes())
-      selected_indices = tf.image.non_max_suppression(
-          boxlist_and_class_scores.get(),
-          boxlist_and_class_scores.get_field(fields.BoxListFields.scores),
-          max_selection_size,
-          iou_threshold=iou_thresh,
-          score_threshold=score_thresh)
+      if pad_to_max_output_size:
+        max_selection_size = max_size_per_class
+        selected_indices, num_valid_nms_boxes = (
+            tf.image.non_max_suppression_padded(
+                boxlist_and_class_scores.get(),
+                boxlist_and_class_scores.get_field(fields.BoxListFields.scores),
+                max_selection_size,
+                iou_threshold=iou_thresh,
+                score_threshold=score_thresh,
+                pad_to_max_output_size=True))
+      else:
+        max_selection_size = tf.minimum(max_size_per_class,
+                                        boxlist_and_class_scores.num_boxes())
+        selected_indices = tf.image.non_max_suppression(
+            boxlist_and_class_scores.get(),
+            boxlist_and_class_scores.get_field(fields.BoxListFields.scores),
+            max_selection_size,
+            iou_threshold=iou_thresh,
+            score_threshold=score_thresh)
+        num_valid_nms_boxes = tf.shape(selected_indices)[0]
+        selected_indices = tf.concat(
+            [selected_indices,
+             tf.zeros(max_selection_size-num_valid_nms_boxes, tf.int32)], 0)
       nms_result = box_list_ops.gather(boxlist_and_class_scores,
                                        selected_indices)
+      # Make the scores -1 for invalid boxes.
+      valid_nms_boxes_indx = tf.less(
+          tf.range(max_selection_size), num_valid_nms_boxes)
+      nms_scores = nms_result.get_field(fields.BoxListFields.scores)
+      nms_result.add_field(fields.BoxListFields.scores,
+                           tf.where(valid_nms_boxes_indx,
+                                    nms_scores, -1*tf.ones(max_selection_size)))
+      num_valid_nms_boxes_cumulative += num_valid_nms_boxes
+
       nms_result.add_field(
           fields.BoxListFields.classes, (tf.zeros_like(
               nms_result.get_field(fields.BoxListFields.scores)) + class_idx))
@@ -158,16 +190,43 @@ def multiclass_non_max_suppression(boxes,
     sorted_boxes = box_list_ops.sort_by_field(selected_boxes,
                                               fields.BoxListFields.scores)
     if clip_window is not None:
-      sorted_boxes = box_list_ops.clip_to_window(sorted_boxes, clip_window)
+      # When pad_to_max_output_size is False, it prunes the boxes with zero
+      # area.
+      sorted_boxes = box_list_ops.clip_to_window(
+          sorted_boxes,
+          clip_window,
+          filter_nonoverlapping=not pad_to_max_output_size)
+      # Set the scores of boxes with zero area to -1 to keep the default
+      # behaviour of pruning out zero area boxes.
+      sorted_boxes_size = tf.shape(sorted_boxes.get())[0]
+      non_zero_box_area = tf.cast(box_list_ops.area(sorted_boxes), tf.bool)
+      sorted_boxes_scores = tf.where(
+          non_zero_box_area,
+          sorted_boxes.get_field(fields.BoxListFields.scores),
+          -1*tf.ones(sorted_boxes_size))
+      sorted_boxes.add_field(fields.BoxListFields.scores, sorted_boxes_scores)
+      num_valid_nms_boxes_cumulative = tf.reduce_sum(
+          tf.cast(tf.greater_equal(sorted_boxes_scores, 0), tf.int32))
+      sorted_boxes = box_list_ops.sort_by_field(sorted_boxes,
+                                                fields.BoxListFields.scores)
       if change_coordinate_frame:
         sorted_boxes = box_list_ops.change_coordinate_frame(
             sorted_boxes, clip_window)
+
     if max_total_size:
       max_total_size = tf.minimum(max_total_size,
                                   sorted_boxes.num_boxes())
       sorted_boxes = box_list_ops.gather(sorted_boxes,
                                          tf.range(max_total_size))
-    return sorted_boxes
+      num_valid_nms_boxes_cumulative = tf.where(
+          max_total_size > num_valid_nms_boxes_cumulative,
+          num_valid_nms_boxes_cumulative, max_total_size)
+    # Select only the valid boxes if pad_to_max_output_size is False.
+    if not pad_to_max_output_size:
+      sorted_boxes = box_list_ops.gather(
+          sorted_boxes, tf.range(num_valid_nms_boxes_cumulative))
+
+    return sorted_boxes, num_valid_nms_boxes_cumulative
 
 
 def batch_multiclass_non_max_suppression(boxes,
@@ -182,6 +241,7 @@ def batch_multiclass_non_max_suppression(boxes,
                                          masks=None,
                                          additional_fields=None,
                                          scope=None,
+                                         use_static_shapes=False,
                                          parallel_iterations=32):
   """Multi-class version of non maximum suppression that operates on a batch.
 
@@ -195,7 +255,8 @@ def batch_multiclass_non_max_suppression(boxes,
         otherwise, if `q` is equal to number of classes, class-specific boxes
         are used.
     scores: A [batch_size, num_anchors, num_classes] float32 tensor containing
-      the scores for each of the `num_anchors` detections.
+      the scores for each of the `num_anchors` detections. The scores have to be
+      non-negative when use_static_shapes is set True.
     score_thresh: scalar threshold for score (low scoring boxes are removed).
     iou_thresh: scalar threshold for IOU (new boxes that have high IOU overlap
       with previously selected boxes are removed).
@@ -221,6 +282,9 @@ def batch_multiclass_non_max_suppression(boxes,
     additional_fields: (optional) If not None, a dictionary that maps keys to
       tensors whose dimensions are [batch_size, num_anchors, ...].
     scope: tf scope name.
+    use_static_shapes: If true, the output nmsed boxes are padded to be of
+      length `max_size_per_class` and it doesn't clip boxes to max_total_size.
+      Defaults to false.
     parallel_iterations: (optional) number of batch items to process in
       parallel.
 
@@ -276,7 +340,7 @@ def batch_multiclass_non_max_suppression(boxes,
     # If masks aren't provided, create dummy masks so we can only have one copy
     # of _single_image_nms_fn and discard the dummy masks after map_fn.
     if masks is None:
-      masks_shape = tf.stack([batch_size, num_anchors, 1, 0, 0])
+      masks_shape = tf.stack([batch_size, num_anchors, q, 1, 1])
       masks = tf.zeros(masks_shape)
 
     if clip_window is None:
@@ -365,7 +429,7 @@ def batch_multiclass_non_max_suppression(boxes,
                        tf.stack([per_image_num_valid_boxes] +
                                 (additional_field_dim - 1) * [-1])),
               [-1] + [dim.value for dim in additional_field_shape[1:]])
-      nmsed_boxlist = multiclass_non_max_suppression(
+      nmsed_boxlist, num_valid_nms_boxes = multiclass_non_max_suppression(
           per_image_boxes,
           per_image_scores,
           score_thresh,
@@ -375,16 +439,19 @@ def batch_multiclass_non_max_suppression(boxes,
           clip_window=per_image_clip_window,
           change_coordinate_frame=change_coordinate_frame,
           masks=per_image_masks,
+          pad_to_max_output_size=use_static_shapes,
           additional_fields=per_image_additional_fields)
-      padded_boxlist = box_list_ops.pad_or_clip_box_list(nmsed_boxlist,
-                                                         max_total_size)
-      num_detections = nmsed_boxlist.num_boxes()
-      nmsed_boxes = padded_boxlist.get()
-      nmsed_scores = padded_boxlist.get_field(fields.BoxListFields.scores)
-      nmsed_classes = padded_boxlist.get_field(fields.BoxListFields.classes)
-      nmsed_masks = padded_boxlist.get_field(fields.BoxListFields.masks)
+
+      if not use_static_shapes:
+        nmsed_boxlist = box_list_ops.pad_or_clip_box_list(
+            nmsed_boxlist, max_total_size)
+      num_detections = num_valid_nms_boxes
+      nmsed_boxes = nmsed_boxlist.get()
+      nmsed_scores = nmsed_boxlist.get_field(fields.BoxListFields.scores)
+      nmsed_classes = nmsed_boxlist.get_field(fields.BoxListFields.classes)
+      nmsed_masks = nmsed_boxlist.get_field(fields.BoxListFields.masks)
       nmsed_additional_fields = [
-          padded_boxlist.get_field(key) for key in per_image_additional_fields
+          nmsed_boxlist.get_field(key) for key in per_image_additional_fields
       ]
       return ([nmsed_boxes, nmsed_scores, nmsed_classes, nmsed_masks] +
               nmsed_additional_fields + [num_detections])

research/object_detection/core/post_processing_test.py

@@ -18,9 +18,10 @@ import numpy as np
 import tensorflow as tf
 from object_detection.core import post_processing
 from object_detection.core import standard_fields as fields
+from object_detection.utils import test_case
 
 
-class MulticlassNonMaxSuppressionTest(tf.test.TestCase):
+class MulticlassNonMaxSuppressionTest(test_case.TestCase):
 
   def test_multiclass_nms_select_with_shared_boxes(self):
     boxes = tf.constant([[[0, 0, 1, 1]],
@@ -46,7 +47,7 @@ class MulticlassNonMaxSuppressionTest(tf.test.TestCase):
     exp_nms_scores = [.95, .9, .85, .3]
     exp_nms_classes = [0, 0, 1, 0]
 
-    nms = post_processing.multiclass_non_max_suppression(
+    nms, _ = post_processing.multiclass_non_max_suppression(
         boxes, scores, score_thresh, iou_thresh, max_output_size)
     with self.test_session() as sess:
       nms_corners_output, nms_scores_output, nms_classes_output = sess.run(
@@ -56,6 +57,8 @@ class MulticlassNonMaxSuppressionTest(tf.test.TestCase):
       self.assertAllClose(nms_scores_output, exp_nms_scores)
       self.assertAllClose(nms_classes_output, exp_nms_classes)
 
+  # TODO(bhattad): Remove conditional after CMLE moves to TF 1.9
+
   def test_multiclass_nms_select_with_shared_boxes_given_keypoints(self):
     boxes = tf.constant([[[0, 0, 1, 1]],
                          [[0, 0.1, 1, 1.1]],
@@ -87,10 +90,13 @@ class MulticlassNonMaxSuppressionTest(tf.test.TestCase):
         tf.reshape(tf.constant([3, 0, 6, 5], dtype=tf.float32), [4, 1, 1]),
         [1, num_keypoints, 2])
 
-    nms = post_processing.multiclass_non_max_suppression(
-        boxes, scores, score_thresh, iou_thresh, max_output_size,
-        additional_fields={
-            fields.BoxListFields.keypoints: keypoints})
+    nms, _ = post_processing.multiclass_non_max_suppression(
+        boxes,
+        scores,
+        score_thresh,
+        iou_thresh,
+        max_output_size,
+        additional_fields={fields.BoxListFields.keypoints: keypoints})
 
     with self.test_session() as sess:
       (nms_corners_output,
@@ -145,10 +151,15 @@ class MulticlassNonMaxSuppressionTest(tf.test.TestCase):
     exp_nms_keypoint_heatmaps = np.ones(
         (4, heatmap_height, heatmap_width, num_keypoints), dtype=np.float32)
 
-    nms = post_processing.multiclass_non_max_suppression(
-        boxes, scores, score_thresh, iou_thresh, max_output_size,
+    nms, _ = post_processing.multiclass_non_max_suppression(
+        boxes,
+        scores,
+        score_thresh,
+        iou_thresh,
+        max_output_size,
         additional_fields={
-            fields.BoxListFields.keypoint_heatmaps: keypoint_heatmaps})
+            fields.BoxListFields.keypoint_heatmaps: keypoint_heatmaps
+        })
 
     with self.test_session() as sess:
       (nms_corners_output,
@@ -208,8 +219,12 @@ class MulticlassNonMaxSuppressionTest(tf.test.TestCase):
     exp_nms_scores = [.95, .9, .85, .3]
     exp_nms_classes = [0, 0, 1, 0]
 
-    nms = post_processing.multiclass_non_max_suppression(
-        boxes, scores, score_thresh, iou_thresh, max_output_size,
+    nms, _ = post_processing.multiclass_non_max_suppression(
+        boxes,
+        scores,
+        score_thresh,
+        iou_thresh,
+        max_output_size,
         additional_fields={coarse_boxes_key: coarse_boxes})
 
     with self.test_session() as sess:
@@ -260,11 +275,8 @@ class MulticlassNonMaxSuppressionTest(tf.test.TestCase):
         tf.reshape(tf.constant([3, 0, 6, 5], dtype=tf.float32), [4, 1, 1]),
         [1, mask_height, mask_width])
 
-    nms = post_processing.multiclass_non_max_suppression(boxes, scores,
-                                                         score_thresh,
-                                                         iou_thresh,
-                                                         max_output_size,
-                                                         masks=masks)
+    nms, _ = post_processing.multiclass_non_max_suppression(
+        boxes, scores, score_thresh, iou_thresh, max_output_size, masks=masks)
     with self.test_session() as sess:
       (nms_corners_output,
        nms_scores_output,
@@ -293,8 +305,12 @@ class MulticlassNonMaxSuppressionTest(tf.test.TestCase):
     exp_nms_scores = [.9]
     exp_nms_classes = [0]
 
-    nms = post_processing.multiclass_non_max_suppression(
-        boxes, scores, score_thresh, iou_thresh, max_output_size,
+    nms, _ = post_processing.multiclass_non_max_suppression(
+        boxes,
+        scores,
+        score_thresh,
+        iou_thresh,
+        max_output_size,
         clip_window=clip_window)
     with self.test_session() as sess:
       nms_corners_output, nms_scores_output, nms_classes_output = sess.run(
@@ -317,9 +333,14 @@ class MulticlassNonMaxSuppressionTest(tf.test.TestCase):
     exp_nms_scores = [.9]
     exp_nms_classes = [0]
 
-    nms = post_processing.multiclass_non_max_suppression(
-        boxes, scores, score_thresh, iou_thresh, max_output_size,
-        clip_window=clip_window, change_coordinate_frame=True)
+    nms, _ = post_processing.multiclass_non_max_suppression(
+        boxes,
+        scores,
+        score_thresh,
+        iou_thresh,
+        max_output_size,
+        clip_window=clip_window,
+        change_coordinate_frame=True)
     with self.test_session() as sess:
       nms_corners_output, nms_scores_output, nms_classes_output = sess.run(
           [nms.get(), nms.get_field(fields.BoxListFields.scores),
@@ -351,7 +372,7 @@ class MulticlassNonMaxSuppressionTest(tf.test.TestCase):
     exp_nms_scores = [.95, .9, .85]
     exp_nms_classes = [0, 0, 1]
 
-    nms = post_processing.multiclass_non_max_suppression(
+    nms, _ = post_processing.multiclass_non_max_suppression(
         boxes, scores, score_thresh, iou_thresh, max_size_per_class)
     with self.test_session() as sess:
       nms_corners_output, nms_scores_output, nms_classes_output = sess.run(
@@ -384,7 +405,7 @@ class MulticlassNonMaxSuppressionTest(tf.test.TestCase):
     exp_nms_scores = [.95, .9]
     exp_nms_classes = [0, 0]
 
-    nms = post_processing.multiclass_non_max_suppression(
+    nms, _ = post_processing.multiclass_non_max_suppression(
         boxes, scores, score_thresh, iou_thresh, max_size_per_class,
         max_total_size)
     with self.test_session() as sess:
@@ -412,7 +433,7 @@ class MulticlassNonMaxSuppressionTest(tf.test.TestCase):
     exp_nms = [[0, 10, 1, 11],
                [0, 0, 1, 1],
                [0, 100, 1, 101]]
-    nms = post_processing.multiclass_non_max_suppression(
+    nms, _ = post_processing.multiclass_non_max_suppression(
         boxes, scores, score_thresh, iou_thresh, max_output_size)
     with self.test_session() as sess:
       nms_output = sess.run(nms.get())
@@ -443,7 +464,7 @@ class MulticlassNonMaxSuppressionTest(tf.test.TestCase):
     exp_nms_scores = [.95, .9, .85, .3]
     exp_nms_classes = [0, 0, 1, 0]
 
-    nms = post_processing.multiclass_non_max_suppression(
+    nms, _ = post_processing.multiclass_non_max_suppression(
         boxes, scores, score_thresh, iou_thresh, max_output_size)
     with self.test_session() as sess:
       nms_corners_output, nms_scores_output, nms_classes_output = sess.run(
@@ -1055,6 +1076,7 @@ class MulticlassNonMaxSuppressionTest(tf.test.TestCase):
                             exp_nms_additional_fields[key])
       self.assertAllClose(num_detections, [1, 1])
 
+  # TODO(bhattad): Remove conditional after CMLE moves to TF 1.9
 
 if __name__ == '__main__':
   tf.test.main()

research/object_detection/core/preprocessor.py

@@ -810,9 +810,11 @@ def random_image_scale(image,
     image = tf.image.resize_images(
         image, [image_newysize, image_newxsize], align_corners=True)
     result.append(image)
-    if masks:
-      masks = tf.image.resize_nearest_neighbor(
-          masks, [image_newysize, image_newxsize], align_corners=True)
+    if masks is not None:
+      masks = tf.image.resize_images(
+          masks, [image_newysize, image_newxsize],
+          method=tf.image.ResizeMethod.NEAREST_NEIGHBOR,
+          align_corners=True)
       result.append(masks)
     return tuple(result)
 
@@ -2969,7 +2971,8 @@ def get_default_func_arg_map(include_label_scores=False,
   """
   groundtruth_label_scores = None
   if include_label_scores:
-    groundtruth_label_scores = (fields.InputDataFields.groundtruth_label_scores)
+    groundtruth_label_scores = (
+        fields.InputDataFields.groundtruth_confidences)
 
   multiclass_scores = None
   if include_multiclass_scores:

research/object_detection/core/preprocessor_cache.py

@@ -67,7 +67,7 @@ class PreprocessorCache(object):
 
   def clear(self):
     """Resets cache."""
-    self._history = {}
+    self._history = defaultdict(dict)
 
   def get(self, function_id, key):
     """Gets stored value given a function id and key.

research/object_detection/core/preprocessor_test.py

@@ -1615,7 +1615,7 @@ class PreprocessorTest(tf.test.TestCase):
     tensor_dict = {
         fields.InputDataFields.groundtruth_boxes: boxes,
         fields.InputDataFields.groundtruth_classes: labels,
-        fields.InputDataFields.groundtruth_label_scores: label_scores
+        fields.InputDataFields.groundtruth_confidences: label_scores
     }
 
     preprocessing_options = [
@@ -1630,7 +1630,7 @@ class PreprocessorTest(tf.test.TestCase):
     retained_labels = retained_tensor_dict[
         fields.InputDataFields.groundtruth_classes]
     retained_label_scores = retained_tensor_dict[
-        fields.InputDataFields.groundtruth_label_scores]
+        fields.InputDataFields.groundtruth_confidences]
 
     with self.test_session() as sess:
       (retained_boxes_, retained_labels_,
@@ -1655,7 +1655,7 @@ class PreprocessorTest(tf.test.TestCase):
     tensor_dict = {
         fields.InputDataFields.groundtruth_boxes: boxes,
         fields.InputDataFields.groundtruth_classes: labels,
-        fields.InputDataFields.groundtruth_label_scores: label_scores,
+        fields.InputDataFields.groundtruth_confidences: label_scores,
         fields.InputDataFields.groundtruth_instance_masks: masks
     }
 
@@ -1687,7 +1687,7 @@ class PreprocessorTest(tf.test.TestCase):
     tensor_dict = {
         fields.InputDataFields.groundtruth_boxes: boxes,
         fields.InputDataFields.groundtruth_classes: labels,
-        fields.InputDataFields.groundtruth_label_scores: label_scores,
+        fields.InputDataFields.groundtruth_confidences: label_scores,
         fields.InputDataFields.groundtruth_keypoints: keypoints
     }
 
@@ -2784,7 +2784,7 @@ class PreprocessorTest(tf.test.TestCase):
     }
     if include_label_scores:
       label_scores = self.createTestLabelScores()
-      tensor_dict[fields.InputDataFields.groundtruth_label_scores] = (
+      tensor_dict[fields.InputDataFields.groundtruth_confidences] = (
           label_scores)
     if include_multiclass_scores:
       multiclass_scores = self.createTestMultiClassScores()

research/object_detection/core/standard_fields.py

@@ -36,12 +36,15 @@ class InputDataFields(object):
     image: image.
     image_additional_channels: additional channels.
     original_image: image in the original input size.
+    original_image_spatial_shape: image in the original input size.
     key: unique key corresponding to image.
     source_id: source of the original image.
     filename: original filename of the dataset (without common path).
     groundtruth_image_classes: image-level class labels.
+    groundtruth_image_confidences: image-level class confidences.
     groundtruth_boxes: coordinates of the ground truth boxes in the image.
     groundtruth_classes: box-level class labels.
+    groundtruth_confidences: box-level class confidences.
     groundtruth_label_types: box-level label types (e.g. explicit negative).
     groundtruth_is_crowd: [DEPRECATED, use groundtruth_group_of instead]
       is the groundtruth a single object or a crowd.
@@ -60,6 +63,7 @@ class InputDataFields(object):
     groundtruth_label_scores: groundtruth label scores.
     groundtruth_weights: groundtruth weight factor for bounding boxes.
     num_groundtruth_boxes: number of groundtruth boxes.
+    is_annotated: whether an image has been labeled or not.
     true_image_shapes: true shapes of images in the resized images, as resized
       images can be padded with zeros.
     multiclass_scores: the label score per class for each box.
@@ -67,12 +71,15 @@ class InputDataFields(object):
   image = 'image'
   image_additional_channels = 'image_additional_channels'
   original_image = 'original_image'
+  original_image_spatial_shape = 'original_image_spatial_shape'
   key = 'key'
   source_id = 'source_id'
   filename = 'filename'
   groundtruth_image_classes = 'groundtruth_image_classes'
+  groundtruth_image_confidences = 'groundtruth_image_confidences'
   groundtruth_boxes = 'groundtruth_boxes'
   groundtruth_classes = 'groundtruth_classes'
+  groundtruth_confidences = 'groundtruth_confidences'
   groundtruth_label_types = 'groundtruth_label_types'
   groundtruth_is_crowd = 'groundtruth_is_crowd'
   groundtruth_area = 'groundtruth_area'
@@ -88,6 +95,7 @@ class InputDataFields(object):
   groundtruth_label_scores = 'groundtruth_label_scores'
   groundtruth_weights = 'groundtruth_weights'
   num_groundtruth_boxes = 'num_groundtruth_boxes'
+  is_annotated = 'is_annotated'
   true_image_shape = 'true_image_shape'
   multiclass_scores = 'multiclass_scores'