Merge remote-tracking branch 'upstream/master'

object_detection/builders/preprocessor_builder_test.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Tests for preprocessor_builder."""
+
+import tensorflow as tf
+
+from google.protobuf import text_format
+
+from object_detection.builders import preprocessor_builder
+from object_detection.core import preprocessor
+from object_detection.protos import preprocessor_pb2
+
+
+class PreprocessorBuilderTest(tf.test.TestCase):
+
+  def assert_dictionary_close(self, dict1, dict2):
+    """Helper to check if two dicts with floatst or integers are close."""
+    self.assertEqual(sorted(dict1.keys()), sorted(dict2.keys()))
+    for key in dict1:
+      value = dict1[key]
+      if isinstance(value, float):
+        self.assertAlmostEqual(value, dict2[key])
+      else:
+        self.assertEqual(value, dict2[key])
+
+  def test_build_normalize_image(self):
+    preprocessor_text_proto = """
+    normalize_image {
+      original_minval: 0.0
+      original_maxval: 255.0
+      target_minval: -1.0
+      target_maxval: 1.0
+    }
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.normalize_image)
+    self.assertEqual(args, {
+        'original_minval': 0.0,
+        'original_maxval': 255.0,
+        'target_minval': -1.0,
+        'target_maxval': 1.0,
+    })
+
+  def test_build_random_horizontal_flip(self):
+    preprocessor_text_proto = """
+    random_horizontal_flip {
+    }
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.random_horizontal_flip)
+    self.assertEqual(args, {})
+
+  def test_build_random_pixel_value_scale(self):
+    preprocessor_text_proto = """
+    random_pixel_value_scale {
+      minval: 0.8
+      maxval: 1.2
+    }
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.random_pixel_value_scale)
+    self.assert_dictionary_close(args, {'minval': 0.8, 'maxval': 1.2})
+
+  def test_build_random_image_scale(self):
+    preprocessor_text_proto = """
+    random_image_scale {
+      min_scale_ratio: 0.8
+      max_scale_ratio: 2.2
+    }
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.random_image_scale)
+    self.assert_dictionary_close(args, {'min_scale_ratio': 0.8,
+                                        'max_scale_ratio': 2.2})
+
+  def test_build_random_rgb_to_gray(self):
+    preprocessor_text_proto = """
+    random_rgb_to_gray {
+      probability: 0.8
+    }
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.random_rgb_to_gray)
+    self.assert_dictionary_close(args, {'probability': 0.8})
+
+  def test_build_random_adjust_brightness(self):
+    preprocessor_text_proto = """
+    random_adjust_brightness {
+      max_delta: 0.2
+    }
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.random_adjust_brightness)
+    self.assert_dictionary_close(args, {'max_delta': 0.2})
+
+  def test_build_random_adjust_contrast(self):
+    preprocessor_text_proto = """
+    random_adjust_contrast {
+      min_delta: 0.7
+      max_delta: 1.1
+    }
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.random_adjust_contrast)
+    self.assert_dictionary_close(args, {'min_delta': 0.7, 'max_delta': 1.1})
+
+  def test_build_random_adjust_hue(self):
+    preprocessor_text_proto = """
+    random_adjust_hue {
+      max_delta: 0.01
+    }
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.random_adjust_hue)
+    self.assert_dictionary_close(args, {'max_delta': 0.01})
+
+  def test_build_random_adjust_saturation(self):
+    preprocessor_text_proto = """
+    random_adjust_saturation {
+      min_delta: 0.75
+      max_delta: 1.15
+    }
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.random_adjust_saturation)
+    self.assert_dictionary_close(args, {'min_delta': 0.75, 'max_delta': 1.15})
+
+  def test_build_random_distort_color(self):
+    preprocessor_text_proto = """
+    random_distort_color {
+      color_ordering: 1
+    }
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.random_distort_color)
+    self.assertEqual(args, {'color_ordering': 1})
+
+  def test_build_random_jitter_boxes(self):
+    preprocessor_text_proto = """
+    random_jitter_boxes {
+      ratio: 0.1
+    }
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.random_jitter_boxes)
+    self.assert_dictionary_close(args, {'ratio': 0.1})
+
+  def test_build_random_crop_image(self):
+    preprocessor_text_proto = """
+    random_crop_image {
+      min_object_covered: 0.75
+      min_aspect_ratio: 0.75
+      max_aspect_ratio: 1.5
+      min_area: 0.25
+      max_area: 0.875
+      overlap_thresh: 0.5
+      random_coef: 0.125
+    }
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.random_crop_image)
+    self.assertEqual(args, {
+        'min_object_covered': 0.75,
+        'aspect_ratio_range': (0.75, 1.5),
+        'area_range': (0.25, 0.875),
+        'overlap_thresh': 0.5,
+        'random_coef': 0.125,
+    })
+
+  def test_build_random_pad_image(self):
+    preprocessor_text_proto = """
+    random_pad_image {
+    }
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.random_pad_image)
+    self.assertEqual(args, {
+        'min_image_size': None,
+        'max_image_size': None,
+        'pad_color': None,
+    })
+
+  def test_build_random_crop_pad_image(self):
+    preprocessor_text_proto = """
+    random_crop_pad_image {
+      min_object_covered: 0.75
+      min_aspect_ratio: 0.75
+      max_aspect_ratio: 1.5
+      min_area: 0.25
+      max_area: 0.875
+      overlap_thresh: 0.5
+      random_coef: 0.125
+    }
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.random_crop_pad_image)
+    self.assertEqual(args, {
+        'min_object_covered': 0.75,
+        'aspect_ratio_range': (0.75, 1.5),
+        'area_range': (0.25, 0.875),
+        'overlap_thresh': 0.5,
+        'random_coef': 0.125,
+        'min_padded_size_ratio': None,
+        'max_padded_size_ratio': None,
+        'pad_color': None,
+    })
+
+  def test_build_random_crop_to_aspect_ratio(self):
+    preprocessor_text_proto = """
+    random_crop_to_aspect_ratio {
+      aspect_ratio: 0.85
+      overlap_thresh: 0.35
+    }
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.random_crop_to_aspect_ratio)
+    self.assert_dictionary_close(args, {'aspect_ratio': 0.85,
+                                        'overlap_thresh': 0.35})
+
+  def test_build_random_black_patches(self):
+    preprocessor_text_proto = """
+    random_black_patches {
+      max_black_patches: 20
+      probability: 0.95
+      size_to_image_ratio: 0.12
+    }
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.random_black_patches)
+    self.assert_dictionary_close(args, {'max_black_patches': 20,
+                                        'probability': 0.95,
+                                        'size_to_image_ratio': 0.12})
+
+  def test_build_random_resize_method(self):
+    preprocessor_text_proto = """
+    random_resize_method {
+      target_height: 75
+      target_width: 100
+    }
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.random_resize_method)
+    self.assert_dictionary_close(args, {'target_size': [75, 100]})
+
+  def test_build_scale_boxes_to_pixel_coordinates(self):
+    preprocessor_text_proto = """
+    scale_boxes_to_pixel_coordinates {}
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.scale_boxes_to_pixel_coordinates)
+    self.assertEqual(args, {})
+
+  def test_build_resize_image(self):
+    preprocessor_text_proto = """
+    resize_image {
+      new_height: 75
+      new_width: 100
+      method: BICUBIC
+    }
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.resize_image)
+    self.assertEqual(args, {'new_height': 75,
+                            'new_width': 100,
+                            'method': tf.image.ResizeMethod.BICUBIC})
+
+  def test_build_subtract_channel_mean(self):
+    preprocessor_text_proto = """
+    subtract_channel_mean {
+      means: [1.0, 2.0, 3.0]
+    }
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.subtract_channel_mean)
+    self.assertEqual(args, {'means': [1.0, 2.0, 3.0]})
+
+  def test_build_ssd_random_crop(self):
+    preprocessor_text_proto = """
+    ssd_random_crop {
+      operations {
+        min_object_covered: 0.0
+        min_aspect_ratio: 0.875
+        max_aspect_ratio: 1.125
+        min_area: 0.5
+        max_area: 1.0
+        overlap_thresh: 0.0
+        random_coef: 0.375
+      }
+      operations {
+        min_object_covered: 0.25
+        min_aspect_ratio: 0.75
+        max_aspect_ratio: 1.5
+        min_area: 0.5
+        max_area: 1.0
+        overlap_thresh: 0.25
+        random_coef: 0.375
+      }
+    }
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.ssd_random_crop)
+    self.assertEqual(args, {'min_object_covered': [0.0, 0.25],
+                            'aspect_ratio_range': [(0.875, 1.125), (0.75, 1.5)],
+                            'area_range': [(0.5, 1.0), (0.5, 1.0)],
+                            'overlap_thresh': [0.0, 0.25],
+                            'random_coef': [0.375, 0.375]})
+
+  def test_build_ssd_random_crop_empty_operations(self):
+    preprocessor_text_proto = """
+    ssd_random_crop {
+    }
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.ssd_random_crop)
+    self.assertEqual(args, {})
+
+  def test_build_ssd_random_crop_pad(self):
+    preprocessor_text_proto = """
+    ssd_random_crop_pad {
+      operations {
+        min_object_covered: 0.0
+        min_aspect_ratio: 0.875
+        max_aspect_ratio: 1.125
+        min_area: 0.5
+        max_area: 1.0
+        overlap_thresh: 0.0
+        random_coef: 0.375
+        min_padded_size_ratio: [0.0, 0.0]
+        max_padded_size_ratio: [2.0, 2.0]
+        pad_color_r: 0.5
+        pad_color_g: 0.5
+        pad_color_b: 0.5
+      }
+      operations {
+        min_object_covered: 0.25
+        min_aspect_ratio: 0.75
+        max_aspect_ratio: 1.5
+        min_area: 0.5
+        max_area: 1.0
+        overlap_thresh: 0.25
+        random_coef: 0.375
+        min_padded_size_ratio: [0.0, 0.0]
+        max_padded_size_ratio: [2.0, 2.0]
+        pad_color_r: 0.5
+        pad_color_g: 0.5
+        pad_color_b: 0.5
+      }
+    }
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.ssd_random_crop_pad)
+    self.assertEqual(args, {'min_object_covered': [0.0, 0.25],
+                            'aspect_ratio_range': [(0.875, 1.125), (0.75, 1.5)],
+                            'area_range': [(0.5, 1.0), (0.5, 1.0)],
+                            'overlap_thresh': [0.0, 0.25],
+                            'random_coef': [0.375, 0.375],
+                            'min_padded_size_ratio': [(0.0, 0.0), (0.0, 0.0)],
+                            'max_padded_size_ratio': [(2.0, 2.0), (2.0, 2.0)],
+                            'pad_color': [(0.5, 0.5, 0.5), (0.5, 0.5, 0.5)]})
+
+  def test_build_ssd_random_crop_fixed_aspect_ratio(self):
+    preprocessor_text_proto = """
+    ssd_random_crop_fixed_aspect_ratio {
+      operations {
+        min_object_covered: 0.0
+        min_area: 0.5
+        max_area: 1.0
+        overlap_thresh: 0.0
+        random_coef: 0.375
+      }
+      operations {
+        min_object_covered: 0.25
+        min_area: 0.5
+        max_area: 1.0
+        overlap_thresh: 0.25
+        random_coef: 0.375
+      }
+      aspect_ratio: 0.875
+    }
+    """
+    preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+    text_format.Merge(preprocessor_text_proto, preprocessor_proto)
+    function, args = preprocessor_builder.build(preprocessor_proto)
+    self.assertEqual(function, preprocessor.ssd_random_crop_fixed_aspect_ratio)
+    self.assertEqual(args, {'min_object_covered': [0.0, 0.25],
+                            'aspect_ratio': 0.875,
+                            'area_range': [(0.5, 1.0), (0.5, 1.0)],
+                            'overlap_thresh': [0.0, 0.25],
+                            'random_coef': [0.375, 0.375]})
+
+
+if __name__ == '__main__':
+  tf.test.main()

object_detection/builders/region_similarity_calculator_builder.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Builder for region similarity calculators."""
+
+from object_detection.core import region_similarity_calculator
+from object_detection.protos import region_similarity_calculator_pb2
+
+
+def build(region_similarity_calculator_config):
+  """Builds region similarity calculator based on the configuration.
+
+  Builds one of [IouSimilarity, IoaSimilarity, NegSqDistSimilarity] objects. See
+  core/region_similarity_calculator.proto for details.
+
+  Args:
+    region_similarity_calculator_config: RegionSimilarityCalculator
+      configuration proto.
+
+  Returns:
+    region_similarity_calculator: RegionSimilarityCalculator object.
+
+  Raises:
+    ValueError: On unknown region similarity calculator.
+  """
+
+  if not isinstance(
+      region_similarity_calculator_config,
+      region_similarity_calculator_pb2.RegionSimilarityCalculator):
+    raise ValueError(
+        'region_similarity_calculator_config not of type '
+        'region_similarity_calculator_pb2.RegionsSimilarityCalculator')
+
+  similarity_calculator = region_similarity_calculator_config.WhichOneof(
+      'region_similarity')
+  if similarity_calculator == 'iou_similarity':
+    return region_similarity_calculator.IouSimilarity()
+  if similarity_calculator == 'ioa_similarity':
+    return region_similarity_calculator.IoaSimilarity()
+  if similarity_calculator == 'neg_sq_dist_similarity':
+    return region_similarity_calculator.NegSqDistSimilarity()
+
+  raise ValueError('Unknown region similarity calculator.')
+

object_detection/builders/region_similarity_calculator_builder_test.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Tests for region_similarity_calculator_builder."""
+
+import tensorflow as tf
+
+from google.protobuf import text_format
+from object_detection.builders import region_similarity_calculator_builder
+from object_detection.core import region_similarity_calculator
+from object_detection.protos import region_similarity_calculator_pb2 as sim_calc_pb2
+
+
+class RegionSimilarityCalculatorBuilderTest(tf.test.TestCase):
+
+  def testBuildIoaSimilarityCalculator(self):
+    similarity_calc_text_proto = """
+      ioa_similarity {
+      }
+    """
+    similarity_calc_proto = sim_calc_pb2.RegionSimilarityCalculator()
+    text_format.Merge(similarity_calc_text_proto, similarity_calc_proto)
+    similarity_calc = region_similarity_calculator_builder.build(
+        similarity_calc_proto)
+    self.assertTrue(isinstance(similarity_calc,
+                               region_similarity_calculator.IoaSimilarity))
+
+  def testBuildIouSimilarityCalculator(self):
+    similarity_calc_text_proto = """
+      iou_similarity {
+      }
+    """
+    similarity_calc_proto = sim_calc_pb2.RegionSimilarityCalculator()
+    text_format.Merge(similarity_calc_text_proto, similarity_calc_proto)
+    similarity_calc = region_similarity_calculator_builder.build(
+        similarity_calc_proto)
+    self.assertTrue(isinstance(similarity_calc,
+                               region_similarity_calculator.IouSimilarity))
+
+  def testBuildNegSqDistSimilarityCalculator(self):
+    similarity_calc_text_proto = """
+      neg_sq_dist_similarity {
+      }
+    """
+    similarity_calc_proto = sim_calc_pb2.RegionSimilarityCalculator()
+    text_format.Merge(similarity_calc_text_proto, similarity_calc_proto)
+    similarity_calc = region_similarity_calculator_builder.build(
+        similarity_calc_proto)
+    self.assertTrue(isinstance(similarity_calc,
+                               region_similarity_calculator.
+                               NegSqDistSimilarity))
+
+
+if __name__ == '__main__':
+  tf.test.main()

object_detection/core/BUILD

+# Tensorflow Object Detection API: Core.
+
+package(
+    default_visibility = ["//visibility:public"],
+)
+
+licenses(["notice"])
+# Apache 2.0
+
+py_library(
+    name = "batcher",
+    srcs = ["batcher.py"],
+    deps = [
+        ":prefetcher",
+        ":preprocessor",
+        ":standard_fields",
+        "//tensorflow",
+    ],
+)
+
+py_test(
+    name = "batcher_test",
+    srcs = ["batcher_test.py"],
+    deps = [
+        ":batcher",
+        "//tensorflow",
+    ],
+)
+
+py_library(
+    name = "box_list",
+    srcs = [
+        "box_list.py",
+    ],
+    deps = [
+        "//tensorflow",
+    ],
+)
+
+py_test(
+    name = "box_list_test",
+    srcs = ["box_list_test.py"],
+    deps = [
+        ":box_list",
+    ],
+)
+
+py_library(
+    name = "box_list_ops",
+    srcs = [
+        "box_list_ops.py",
+    ],
+    deps = [
+        ":box_list",
+        "//tensorflow",
+        "//tensorflow_models/object_detection/utils:shape_utils",
+    ],
+)
+
+py_test(
+    name = "box_list_ops_test",
+    srcs = ["box_list_ops_test.py"],
+    deps = [
+        ":box_list",
+        ":box_list_ops",
+    ],
+)
+
+py_library(
+    name = "box_coder",
+    srcs = [
+        "box_coder.py",
+    ],
+    deps = [
+        "//tensorflow",
+    ],
+)
+
+py_test(
+    name = "box_coder_test",
+    srcs = [
+        "box_coder_test.py",
+    ],
+    deps = [
+        ":box_coder",
+        ":box_list",
+        "//tensorflow",
+    ],
+)
+
+py_library(
+    name = "keypoint_ops",
+    srcs = [
+        "keypoint_ops.py",
+    ],
+    deps = [
+        "//tensorflow",
+    ],
+)
+
+py_test(
+    name = "keypoint_ops_test",
+    srcs = ["keypoint_ops_test.py"],
+    deps = [
+        ":keypoint_ops",
+    ],
+)
+
+py_library(
+    name = "losses",
+    srcs = ["losses.py"],
+    deps = [
+        ":box_list",
+        ":box_list_ops",
+        "//tensorflow",
+        "//tensorflow_models/object_detection/utils:ops",
+    ],
+)
+
+py_library(
+    name = "matcher",
+    srcs = [
+        "matcher.py",
+    ],
+    deps = [
+    ],
+)
+
+py_library(
+    name = "model",
+    srcs = ["model.py"],
+    deps = [
+        ":standard_fields",
+    ],
+)
+
+py_test(
+    name = "matcher_test",
+    srcs = [
+        "matcher_test.py",
+    ],
+    deps = [
+        ":matcher",
+        "//tensorflow",
+    ],
+)
+
+py_library(
+    name = "prefetcher",
+    srcs = ["prefetcher.py"],
+    deps = ["//tensorflow"],
+)
+
+py_library(
+    name = "preprocessor",
+    srcs = [
+        "preprocessor.py",
+    ],
+    deps = [
+        ":box_list",
+        ":box_list_ops",
+        ":keypoint_ops",
+        ":standard_fields",
+        "//tensorflow",
+    ],
+)
+
+py_test(
+    name = "preprocessor_test",
+    srcs = [
+        "preprocessor_test.py",
+    ],
+    deps = [
+        ":preprocessor",
+        "//tensorflow",
+    ],
+)
+
+py_test(
+    name = "losses_test",
+    srcs = ["losses_test.py"],
+    deps = [
+        ":box_list",
+        ":losses",
+        ":matcher",
+        "//tensorflow",
+    ],
+)
+
+py_test(
+    name = "prefetcher_test",
+    srcs = ["prefetcher_test.py"],
+    deps = [
+        ":prefetcher",
+        "//tensorflow",
+    ],
+)
+
+py_library(
+    name = "standard_fields",
+    srcs = [
+        "standard_fields.py",
+    ],
+)
+
+py_library(
+    name = "post_processing",
+    srcs = ["post_processing.py"],
+    deps = [
+        ":box_list",
+        ":box_list_ops",
+        ":standard_fields",
+        "//tensorflow",
+    ],
+)
+
+py_test(
+    name = "post_processing_test",
+    srcs = ["post_processing_test.py"],
+    deps = [
+        ":box_list",
+        ":box_list_ops",
+        ":post_processing",
+        "//tensorflow",
+    ],
+)
+
+py_library(
+    name = "target_assigner",
+    srcs = [
+        "target_assigner.py",
+    ],
+    deps = [
+        ":box_list",
+        ":box_list_ops",
+        ":matcher",
+        ":region_similarity_calculator",
+        "//tensorflow",
+        "//tensorflow_models/object_detection/box_coders:faster_rcnn_box_coder",
+        "//tensorflow_models/object_detection/box_coders:mean_stddev_box_coder",
+        "//tensorflow_models/object_detection/core:box_coder",
+        "//tensorflow_models/object_detection/matchers:argmax_matcher",
+        "//tensorflow_models/object_detection/matchers:bipartite_matcher",
+    ],
+)
+
+py_test(
+    name = "target_assigner_test",
+    size = "large",
+    timeout = "long",
+    srcs = ["target_assigner_test.py"],
+    deps = [
+        ":box_list",
+        ":region_similarity_calculator",
+        ":target_assigner",
+        "//tensorflow",
+        "//tensorflow_models/object_detection/box_coders:mean_stddev_box_coder",
+        "//tensorflow_models/object_detection/matchers:bipartite_matcher",
+    ],
+)
+
+py_library(
+    name = "data_decoder",
+    srcs = ["data_decoder.py"],
+)
+
+py_library(
+    name = "box_predictor",
+    srcs = ["box_predictor.py"],
+    deps = [
+        "//tensorflow",
+        "//tensorflow_models/object_detection/utils:ops",
+        "//tensorflow_models/object_detection/utils:static_shape",
+    ],
+)
+
+py_test(
+    name = "box_predictor_test",
+    srcs = ["box_predictor_test.py"],
+    deps = [
+        ":box_predictor",
+        "//tensorflow",
+        "//tensorflow_models/object_detection/builders:hyperparams_builder",
+        "//tensorflow_models/object_detection/protos:hyperparams_py_pb2",
+    ],
+)
+
+py_library(
+    name = "region_similarity_calculator",
+    srcs = [
+        "region_similarity_calculator.py",
+    ],
+    deps = [
+        "//tensorflow",
+        "//tensorflow_models/object_detection/core:box_list_ops",
+    ],
+)
+
+py_test(
+    name = "region_similarity_calculator_test",
+    srcs = [
+        "region_similarity_calculator_test.py",
+    ],
+    deps = [
+        ":region_similarity_calculator",
+        "//tensorflow_models/object_detection/core:box_list",
+    ],
+)
+
+py_library(
+    name = "anchor_generator",
+    srcs = [
+        "anchor_generator.py",
+    ],
+    deps = [
+        "//tensorflow",
+    ],
+)
+
+py_library(
+    name = "minibatch_sampler",
+    srcs = [
+        "minibatch_sampler.py",
+    ],
+    deps = [
+        "//tensorflow",
+        "//tensorflow_models/object_detection/utils:ops",
+    ],
+)
+
+py_test(
+    name = "minibatch_sampler_test",
+    srcs = [
+        "minibatch_sampler_test.py",
+    ],
+    deps = [
+        ":minibatch_sampler",
+        "//tensorflow",
+    ],
+)
+
+py_library(
+    name = "balanced_positive_negative_sampler",
+    srcs = [
+        "balanced_positive_negative_sampler.py",
+    ],
+    deps = [
+        ":minibatch_sampler",
+        "//tensorflow",
+    ],
+)
+
+py_test(
+    name = "balanced_positive_negative_sampler_test",
+    srcs = [
+        "balanced_positive_negative_sampler_test.py",
+    ],
+    deps = [
+        ":balanced_positive_negative_sampler",
+        "//tensorflow",
+    ],
+)

object_detection/core/anchor_generator.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Base anchor generator.
+
+The job of the anchor generator is to create (or load) a collection
+of bounding boxes to be used as anchors.
+
+Generated anchors are assumed to match some convolutional grid or list of grid
+shapes.  For example, we might want to generate anchors matching an 8x8
+feature map and a 4x4 feature map.  If we place 3 anchors per grid location
+on the first feature map and 6 anchors per grid location on the second feature
+map, then 3*8*8 + 6*4*4 = 288 anchors are generated in total.
+
+To support fully convolutional settings, feature map shapes are passed
+dynamically at generation time.  The number of anchors to place at each location
+is static --- implementations of AnchorGenerator must always be able return
+the number of anchors that it uses per location for each feature map.
+"""
+from abc import ABCMeta
+from abc import abstractmethod
+
+import tensorflow as tf
+
+
+class AnchorGenerator(object):
+  """Abstract base class for anchor generators."""
+  __metaclass__ = ABCMeta
+
+  @abstractmethod
+  def name_scope(self):
+    """Name scope.
+
+    Must be defined by implementations.
+
+    Returns:
+      a string representing the name scope of the anchor generation operation.
+    """
+    pass
+
+  @property
+  def check_num_anchors(self):
+    """Whether to dynamically check the number of anchors generated.
+
+    Can be overridden by implementations that would like to disable this
+    behavior.
+
+    Returns:
+      a boolean controlling whether the Generate function should dynamically
+      check the number of anchors generated against the mathematically
+      expected number of anchors.
+    """
+    return True
+
+  @abstractmethod
+  def num_anchors_per_location(self):
+    """Returns the number of anchors per spatial location.
+
+    Returns:
+      a list of integers, one for each expected feature map to be passed to
+      the `generate` function.
+    """
+    pass
+
+  def generate(self, feature_map_shape_list, **params):
+    """Generates a collection of bounding boxes to be used as anchors.
+
+    TODO: remove **params from argument list and make stride and offsets (for
+        multiple_grid_anchor_generator) constructor arguments.
+
+    Args:
+      feature_map_shape_list: list of (height, width) pairs in the format
+        [(height_0, width_0), (height_1, width_1), ...] that the generated
+        anchors must align with.  Pairs can be provided as 1-dimensional
+        integer tensors of length 2 or simply as tuples of integers.
+      **params: parameters for anchor generation op
+
+    Returns:
+      boxes: a BoxList holding a collection of N anchor boxes
+    Raises:
+      ValueError: if the number of feature map shapes does not match the length
+        of NumAnchorsPerLocation.
+    """
+    if self.check_num_anchors and (
+        len(feature_map_shape_list) != len(self.num_anchors_per_location())):
+      raise ValueError('Number of feature maps is expected to equal the length '
+                       'of `num_anchors_per_location`.')
+    with tf.name_scope(self.name_scope()):
+      anchors = self._generate(feature_map_shape_list, **params)
+      if self.check_num_anchors:
+        with tf.control_dependencies([
+            self._assert_correct_number_of_anchors(
+                anchors, feature_map_shape_list)]):
+          anchors.set(tf.identity(anchors.get()))
+      return anchors
+
+  @abstractmethod
+  def _generate(self, feature_map_shape_list, **params):
+    """To be overridden by implementations.
+
+    Args:
+      feature_map_shape_list: list of (height, width) pairs in the format
+        [(height_0, width_0), (height_1, width_1), ...] that the generated
+        anchors must align with.
+      **params: parameters for anchor generation op
+
+    Returns:
+      boxes: a BoxList holding a collection of N anchor boxes
+    """
+    pass
+
+  def _assert_correct_number_of_anchors(self, anchors, feature_map_shape_list):
+    """Assert that correct number of anchors was generated.
+
+    Args:
+      anchors: box_list.BoxList object holding anchors generated
+      feature_map_shape_list: list of (height, width) pairs in the format
+        [(height_0, width_0), (height_1, width_1), ...] that the generated
+        anchors must align with.
+    Returns:
+      Op that raises InvalidArgumentError if the number of anchors does not
+        match the number of expected anchors.
+    """
+    expected_num_anchors = 0
+    for num_anchors_per_location, feature_map_shape in zip(
+        self.num_anchors_per_location(), feature_map_shape_list):
+      expected_num_anchors += (num_anchors_per_location
+                               * feature_map_shape[0]
+                               * feature_map_shape[1])
+    return tf.assert_equal(expected_num_anchors, anchors.num_boxes())

object_detection/core/balanced_positive_negative_sampler.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Class to subsample minibatches by balancing positives and negatives.
+
+Subsamples minibatches based on a pre-specified positive fraction in range
+[0,1]. The class presumes there are many more negatives than positive examples:
+if the desired batch_size cannot be achieved with the pre-specified positive
+fraction, it fills the rest with negative examples. If this is not sufficient
+for obtaining the desired batch_size, it returns fewer examples.
+
+The main function to call is Subsample(self, indicator, labels). For convenience
+one can also call SubsampleWeights(self, weights, labels) which is defined in
+the minibatch_sampler base class.
+"""
+
+import tensorflow as tf
+
+from object_detection.core import minibatch_sampler
+
+
+class BalancedPositiveNegativeSampler(minibatch_sampler.MinibatchSampler):
+  """Subsamples minibatches to a desired balance of positives and negatives."""
+
+  def __init__(self, positive_fraction=0.5):
+    """Constructs a minibatch sampler.
+
+    Args:
+      positive_fraction: desired fraction of positive examples (scalar in [0,1])
+
+    Raises:
+      ValueError: if positive_fraction < 0, or positive_fraction > 1
+    """
+    if positive_fraction < 0 or positive_fraction > 1:
+      raise ValueError('positive_fraction should be in range [0,1]. '
+                       'Received: %s.' % positive_fraction)
+    self._positive_fraction = positive_fraction
+
+  def subsample(self, indicator, batch_size, labels):
+    """Returns subsampled minibatch.
+
+    Args:
+      indicator: boolean tensor of shape [N] whose True entries can be sampled.
+      batch_size: desired batch size.
+      labels: boolean tensor of shape [N] denoting positive(=True) and negative
+          (=False) examples.
+
+    Returns:
+      is_sampled: boolean tensor of shape [N], True for entries which are
+          sampled.
+
+    Raises:
+      ValueError: if labels and indicator are not 1D boolean tensors.
+    """
+    if len(indicator.get_shape().as_list()) != 1:
+      raise ValueError('indicator must be 1 dimensional, got a tensor of '
+                       'shape %s' % indicator.get_shape())
+    if len(labels.get_shape().as_list()) != 1:
+      raise ValueError('labels must be 1 dimensional, got a tensor of '
+                       'shape %s' % labels.get_shape())
+    if labels.dtype != tf.bool:
+      raise ValueError('labels should be of type bool. Received: %s' %
+                       labels.dtype)
+    if indicator.dtype != tf.bool:
+      raise ValueError('indicator should be of type bool. Received: %s' %
+                       indicator.dtype)
+
+    # Only sample from indicated samples
+    negative_idx = tf.logical_not(labels)
+    positive_idx = tf.logical_and(labels, indicator)
+    negative_idx = tf.logical_and(negative_idx, indicator)
+
+    # Sample positive and negative samples separately
+    max_num_pos = int(self._positive_fraction * batch_size)
+    sampled_pos_idx = self.subsample_indicator(positive_idx, max_num_pos)
+    max_num_neg = batch_size - tf.reduce_sum(tf.cast(sampled_pos_idx, tf.int32))
+    sampled_neg_idx = self.subsample_indicator(negative_idx, max_num_neg)
+
+    sampled_idx = tf.logical_or(sampled_pos_idx, sampled_neg_idx)
+    return sampled_idx

object_detection/core/balanced_positive_negative_sampler_test.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Tests for object_detection.core.balanced_positive_negative_sampler."""
+
+import numpy as np
+import tensorflow as tf
+
+from object_detection.core import balanced_positive_negative_sampler
+
+
+class BalancedPositiveNegativeSamplerTest(tf.test.TestCase):
+
+  def test_subsample_all_examples(self):
+    numpy_labels = np.random.permutation(300)
+    indicator = tf.constant(np.ones(300) == 1)
+    numpy_labels = (numpy_labels - 200) > 0
+
+    labels = tf.constant(numpy_labels)
+
+    sampler = (balanced_positive_negative_sampler.
+               BalancedPositiveNegativeSampler())
+    is_sampled = sampler.subsample(indicator, 64, labels)
+    with self.test_session() as sess:
+      is_sampled = sess.run(is_sampled)
+      self.assertTrue(sum(is_sampled) == 64)
+      self.assertTrue(sum(np.logical_and(numpy_labels, is_sampled)) == 32)
+      self.assertTrue(sum(np.logical_and(
+          np.logical_not(numpy_labels), is_sampled)) == 32)
+
+  def test_subsample_selection(self):
+    # Test random sampling when only some examples can be sampled:
+    # 100 samples, 20 positives, 10 positives cannot be sampled
+    numpy_labels = np.arange(100)
+    numpy_indicator = numpy_labels < 90
+    indicator = tf.constant(numpy_indicator)
+    numpy_labels = (numpy_labels - 80) >= 0
+
+    labels = tf.constant(numpy_labels)
+
+    sampler = (balanced_positive_negative_sampler.
+               BalancedPositiveNegativeSampler())
+    is_sampled = sampler.subsample(indicator, 64, labels)
+    with self.test_session() as sess:
+      is_sampled = sess.run(is_sampled)
+      self.assertTrue(sum(is_sampled) == 64)
+      self.assertTrue(sum(np.logical_and(numpy_labels, is_sampled)) == 10)
+      self.assertTrue(sum(np.logical_and(
+          np.logical_not(numpy_labels), is_sampled)) == 54)
+      self.assertAllEqual(is_sampled, np.logical_and(is_sampled,
+                                                     numpy_indicator))
+
+  def test_raises_error_with_incorrect_label_shape(self):
+    labels = tf.constant([[True, False, False]])
+    indicator = tf.constant([True, False, True])
+    sampler = (balanced_positive_negative_sampler.
+               BalancedPositiveNegativeSampler())
+    with self.assertRaises(ValueError):
+      sampler.subsample(indicator, 64, labels)
+
+  def test_raises_error_with_incorrect_indicator_shape(self):
+    labels = tf.constant([True, False, False])
+    indicator = tf.constant([[True, False, True]])
+    sampler = (balanced_positive_negative_sampler.
+               BalancedPositiveNegativeSampler())
+    with self.assertRaises(ValueError):
+      sampler.subsample(indicator, 64, labels)
+
+
+if __name__ == '__main__':
+  tf.test.main()

object_detection/core/batcher.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Provides functions to batch a dictionary of input tensors."""
+import collections
+
+import tensorflow as tf
+
+from object_detection.core import prefetcher
+
+rt_shape_str = '_runtime_shapes'
+
+
+class BatchQueue(object):
+  """BatchQueue class.
+
+  This class creates a batch queue to asynchronously enqueue tensors_dict.
+  It also adds a FIFO prefetcher so that the batches are readily available
+  for the consumers.  Dequeue ops for a BatchQueue object can be created via
+  the Dequeue method which evaluates to a batch of tensor_dict.
+
+  Example input pipeline with batching:
+  ------------------------------------
+  key, string_tensor = slim.parallel_reader.parallel_read(...)
+  tensor_dict = decoder.decode(string_tensor)
+  tensor_dict = preprocessor.preprocess(tensor_dict, ...)
+  batch_queue = batcher.BatchQueue(tensor_dict,
+                                   batch_size=32,
+                                   batch_queue_capacity=2000,
+                                   num_batch_queue_threads=8,
+                                   prefetch_queue_capacity=20)
+  tensor_dict = batch_queue.dequeue()
+  outputs = Model(tensor_dict)
+  ...
+  -----------------------------------
+
+  Notes:
+  -----
+  This class batches tensors of unequal sizes by zero padding and unpadding
+  them after generating a batch. This can be computationally expensive when
+  batching tensors (such as images) that are of vastly different sizes. So it is
+  recommended that the shapes of such tensors be fully defined in tensor_dict
+  while other lightweight tensors such as bounding box corners and class labels
+  can be of varying sizes. Use either crop or resize operations to fully define
+  the shape of an image in tensor_dict.
+
+  It is also recommended to perform any preprocessing operations on tensors
+  before passing to BatchQueue and subsequently calling the Dequeue method.
+
+  Another caveat is that this class does not read the last batch if it is not
+  full. The current implementation makes it hard to support that use case. So,
+  for evaluation, when it is critical to run all the examples through your
+  network use the input pipeline example mentioned in core/prefetcher.py.
+  """
+
+  def __init__(self, tensor_dict, batch_size, batch_queue_capacity,
+               num_batch_queue_threads, prefetch_queue_capacity):
+    """Constructs a batch queue holding tensor_dict.
+
+    Args:
+      tensor_dict: dictionary of tensors to batch.
+      batch_size: batch size.
+      batch_queue_capacity: max capacity of the queue from which the tensors are
+        batched.
+      num_batch_queue_threads: number of threads to use for batching.
+      prefetch_queue_capacity: max capacity of the queue used to prefetch
+        assembled batches.
+    """
+    # Remember static shapes to set shapes of batched tensors.
+    static_shapes = collections.OrderedDict(
+        {key: tensor.get_shape() for key, tensor in tensor_dict.items()})
+    # Remember runtime shapes to unpad tensors after batching.
+    runtime_shapes = collections.OrderedDict(
+        {(key + rt_shape_str): tf.shape(tensor)
+         for key, tensor in tensor_dict.iteritems()})
+
+    all_tensors = tensor_dict
+    all_tensors.update(runtime_shapes)
+    batched_tensors = tf.train.batch(
+        all_tensors,
+        capacity=batch_queue_capacity,
+        batch_size=batch_size,
+        dynamic_pad=True,
+        num_threads=num_batch_queue_threads)
+
+    self._queue = prefetcher.prefetch(batched_tensors,
+                                      prefetch_queue_capacity)
+    self._static_shapes = static_shapes
+    self._batch_size = batch_size
+
+  def dequeue(self):
+    """Dequeues a batch of tensor_dict from the BatchQueue.
+
+    TODO: use allow_smaller_final_batch to allow running over the whole eval set
+
+    Returns:
+      A list of tensor_dicts of the requested batch_size.
+    """
+    batched_tensors = self._queue.dequeue()
+    # Separate input tensors from tensors containing their runtime shapes.
+    tensors = {}
+    shapes = {}
+    for key, batched_tensor in batched_tensors.items():
+      unbatched_tensor_list = tf.unstack(batched_tensor)
+      for i, unbatched_tensor in enumerate(unbatched_tensor_list):
+        if rt_shape_str in key:
+          shapes[(key[:-len(rt_shape_str)], i)] = unbatched_tensor
+        else:
+          tensors[(key, i)] = unbatched_tensor
+
+    # Undo that padding using shapes and create a list of size `batch_size` that
+    # contains tensor dictionaries.
+    tensor_dict_list = []
+    batch_size = self._batch_size
+    for batch_id in range(batch_size):
+      tensor_dict = {}
+      for key in self._static_shapes:
+        tensor_dict[key] = tf.slice(tensors[(key, batch_id)],
+                                    tf.zeros_like(shapes[(key, batch_id)]),
+                                    shapes[(key, batch_id)])
+        tensor_dict[key].set_shape(self._static_shapes[key])
+      tensor_dict_list.append(tensor_dict)
+
+    return tensor_dict_list

object_detection/core/batcher_test.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Tests for object_detection.core.batcher."""
+
+import numpy as np
+import tensorflow as tf
+
+from object_detection.core import batcher
+
+slim = tf.contrib.slim
+
+
+class BatcherTest(tf.test.TestCase):
+
+  def test_batch_and_unpad_2d_tensors_of_different_sizes_in_1st_dimension(self):
+    with self.test_session() as sess:
+      batch_size = 3
+      num_batches = 2
+      examples = tf.Variable(tf.constant(2, dtype=tf.int32))
+      counter = examples.count_up_to(num_batches * batch_size + 2)
+      boxes = tf.tile(
+          tf.reshape(tf.range(4), [1, 4]), tf.stack([counter, tf.constant(1)]))
+      batch_queue = batcher.BatchQueue(
+          tensor_dict={'boxes': boxes},
+          batch_size=batch_size,
+          batch_queue_capacity=100,
+          num_batch_queue_threads=1,
+          prefetch_queue_capacity=100)
+      batch = batch_queue.dequeue()
+
+      for tensor_dict in batch:
+        for tensor in tensor_dict.values():
+          self.assertAllEqual([None, 4], tensor.get_shape().as_list())
+
+      tf.initialize_all_variables().run()
+      with slim.queues.QueueRunners(sess):
+        i = 2
+        for _ in range(num_batches):
+          batch_np = sess.run(batch)
+          for tensor_dict in batch_np:
+            for tensor in tensor_dict.values():
+              self.assertAllEqual(tensor, np.tile(np.arange(4), (i, 1)))
+              i += 1
+        with self.assertRaises(tf.errors.OutOfRangeError):
+          sess.run(batch)
+
+  def test_batch_and_unpad_2d_tensors_of_different_sizes_in_all_dimensions(
+      self):
+    with self.test_session() as sess:
+      batch_size = 3
+      num_batches = 2
+      examples = tf.Variable(tf.constant(2, dtype=tf.int32))
+      counter = examples.count_up_to(num_batches * batch_size + 2)
+      image = tf.reshape(
+          tf.range(counter * counter), tf.stack([counter, counter]))
+      batch_queue = batcher.BatchQueue(
+          tensor_dict={'image': image},
+          batch_size=batch_size,
+          batch_queue_capacity=100,
+          num_batch_queue_threads=1,
+          prefetch_queue_capacity=100)
+      batch = batch_queue.dequeue()
+
+      for tensor_dict in batch:
+        for tensor in tensor_dict.values():
+          self.assertAllEqual([None, None], tensor.get_shape().as_list())
+
+      tf.initialize_all_variables().run()
+      with slim.queues.QueueRunners(sess):
+        i = 2
+        for _ in range(num_batches):
+          batch_np = sess.run(batch)
+          for tensor_dict in batch_np:
+            for tensor in tensor_dict.values():
+              self.assertAllEqual(tensor, np.arange(i * i).reshape((i, i)))
+              i += 1
+        with self.assertRaises(tf.errors.OutOfRangeError):
+          sess.run(batch)
+
+  def test_batch_and_unpad_2d_tensors_of_same_size_in_all_dimensions(self):
+    with self.test_session() as sess:
+      batch_size = 3
+      num_batches = 2
+      examples = tf.Variable(tf.constant(1, dtype=tf.int32))
+      counter = examples.count_up_to(num_batches * batch_size + 1)
+      image = tf.reshape(tf.range(1, 13), [4, 3]) * counter
+      batch_queue = batcher.BatchQueue(
+          tensor_dict={'image': image},
+          batch_size=batch_size,
+          batch_queue_capacity=100,
+          num_batch_queue_threads=1,
+          prefetch_queue_capacity=100)
+      batch = batch_queue.dequeue()
+
+      for tensor_dict in batch:
+        for tensor in tensor_dict.values():
+          self.assertAllEqual([4, 3], tensor.get_shape().as_list())
+
+      tf.initialize_all_variables().run()
+      with slim.queues.QueueRunners(sess):
+        i = 1
+        for _ in range(num_batches):
+          batch_np = sess.run(batch)
+          for tensor_dict in batch_np:
+            for tensor in tensor_dict.values():
+              self.assertAllEqual(tensor, np.arange(1, 13).reshape((4, 3)) * i)
+              i += 1
+        with self.assertRaises(tf.errors.OutOfRangeError):
+          sess.run(batch)
+
+  def test_batcher_when_batch_size_is_one(self):
+    with self.test_session() as sess:
+      batch_size = 1
+      num_batches = 2
+      examples = tf.Variable(tf.constant(2, dtype=tf.int32))
+      counter = examples.count_up_to(num_batches * batch_size + 2)
+      image = tf.reshape(
+          tf.range(counter * counter), tf.stack([counter, counter]))
+      batch_queue = batcher.BatchQueue(
+          tensor_dict={'image': image},
+          batch_size=batch_size,
+          batch_queue_capacity=100,
+          num_batch_queue_threads=1,
+          prefetch_queue_capacity=100)
+      batch = batch_queue.dequeue()
+
+      for tensor_dict in batch:
+        for tensor in tensor_dict.values():
+          self.assertAllEqual([None, None], tensor.get_shape().as_list())
+
+      tf.initialize_all_variables().run()
+      with slim.queues.QueueRunners(sess):
+        i = 2
+        for _ in range(num_batches):
+          batch_np = sess.run(batch)
+          for tensor_dict in batch_np:
+            for tensor in tensor_dict.values():
+              self.assertAllEqual(tensor, np.arange(i * i).reshape((i, i)))
+              i += 1
+        with self.assertRaises(tf.errors.OutOfRangeError):
+          sess.run(batch)
+
+
+if __name__ == '__main__':
+  tf.test.main()

object_detection/core/box_coder.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Base box coder.
+
+Box coders convert between coordinate frames, namely image-centric
+(with (0,0) on the top left of image) and anchor-centric (with (0,0) being
+defined by a specific anchor).
+
+Users of a BoxCoder can call two methods:
+ encode: which encodes a box with respect to a given anchor
+  (or rather, a tensor of boxes wrt a corresponding tensor of anchors) and
+ decode: which inverts this encoding with a decode operation.
+In both cases, the arguments are assumed to be in 1-1 correspondence already;
+it is not the job of a BoxCoder to perform matching.
+"""
+from abc import ABCMeta
+from abc import abstractmethod
+from abc import abstractproperty
+
+import tensorflow as tf
+
+
+# Box coder types.
+FASTER_RCNN = 'faster_rcnn'
+KEYPOINT = 'keypoint'
+MEAN_STDDEV = 'mean_stddev'
+SQUARE = 'square'
+
+
+class BoxCoder(object):
+  """Abstract base class for box coder."""
+  __metaclass__ = ABCMeta
+
+  @abstractproperty
+  def code_size(self):
+    """Return the size of each code.
+
+    This number is a constant and should agree with the output of the `encode`
+    op (e.g. if rel_codes is the output of self.encode(...), then it should have
+    shape [N, code_size()]).  This abstractproperty should be overridden by
+    implementations.
+
+    Returns:
+      an integer constant
+    """
+    pass
+
+  def encode(self, boxes, anchors):
+    """Encode a box list relative to an anchor collection.
+
+    Args:
+      boxes: BoxList holding N boxes to be encoded
+      anchors: BoxList of N anchors
+
+    Returns:
+      a tensor representing N relative-encoded boxes
+    """
+    with tf.name_scope('Encode'):
+      return self._encode(boxes, anchors)
+
+  def decode(self, rel_codes, anchors):
+    """Decode boxes that are encoded relative to an anchor collection.
+
+    Args:
+      rel_codes: a tensor representing N relative-encoded boxes
+      anchors: BoxList of anchors
+
+    Returns:
+      boxlist: BoxList holding N boxes encoded in the ordinary way (i.e.,
+        with corners y_min, x_min, y_max, x_max)
+    """
+    with tf.name_scope('Decode'):
+      return self._decode(rel_codes, anchors)
+
+  @abstractmethod
+  def _encode(self, boxes, anchors):
+    """Method to be overriden by implementations.
+
+    Args:
+      boxes: BoxList holding N boxes to be encoded
+      anchors: BoxList of N anchors
+
+    Returns:
+      a tensor representing N relative-encoded boxes
+    """
+    pass
+
+  @abstractmethod
+  def _decode(self, rel_codes, anchors):
+    """Method to be overriden by implementations.
+
+    Args:
+      rel_codes: a tensor representing N relative-encoded boxes
+      anchors: BoxList of anchors
+
+    Returns:
+      boxlist: BoxList holding N boxes encoded in the ordinary way (i.e.,
+        with corners y_min, x_min, y_max, x_max)
+    """
+    pass
+
+
+def batch_decode(encoded_boxes, box_coder, anchors):
+  """Decode a batch of encoded boxes.
+
+  This op takes a batch of encoded bounding boxes and transforms
+  them to a batch of bounding boxes specified by their corners in
+  the order of [y_min, x_min, y_max, x_max].
+
+  Args:
+    encoded_boxes: a float32 tensor of shape [batch_size, num_anchors,
+      code_size] representing the location of the objects.
+    box_coder: a BoxCoder object.
+    anchors: a BoxList of anchors used to encode `encoded_boxes`.
+
+  Returns:
+    decoded_boxes: a float32 tensor of shape [batch_size, num_anchors,
+      coder_size] representing the corners of the objects in the order
+      of [y_min, x_min, y_max, x_max].
+
+  Raises:
+    ValueError: if batch sizes of the inputs are inconsistent, or if
+    the number of anchors inferred from encoded_boxes and anchors are
+    inconsistent.
+  """
+  encoded_boxes.get_shape().assert_has_rank(3)
+  if encoded_boxes.get_shape()[1].value != anchors.num_boxes_static():
+    raise ValueError('The number of anchors inferred from encoded_boxes'
+                     ' and anchors are inconsistent: shape[1] of encoded_boxes'
+                     ' %s should be equal to the number of anchors: %s.' %
+                     (encoded_boxes.get_shape()[1].value,
+                      anchors.num_boxes_static()))
+
+  decoded_boxes = tf.stack([
+      box_coder.decode(boxes, anchors).get()
+      for boxes in tf.unstack(encoded_boxes)
+  ])
+  return decoded_boxes

object_detection/core/box_coder_test.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Tests for object_detection.core.box_coder."""
+
+import tensorflow as tf
+
+from object_detection.core import box_coder
+from object_detection.core import box_list
+
+
+class MockBoxCoder(box_coder.BoxCoder):
+  """Test BoxCoder that encodes/decodes using the multiply-by-two function."""
+
+  def code_size(self):
+    return 4
+
+  def _encode(self, boxes, anchors):
+    return 2.0 * boxes.get()
+
+  def _decode(self, rel_codes, anchors):
+    return box_list.BoxList(rel_codes / 2.0)
+
+
+class BoxCoderTest(tf.test.TestCase):
+
+  def test_batch_decode(self):
+    mock_anchor_corners = tf.constant(
+        [[0, 0.1, 0.2, 0.3], [0.2, 0.4, 0.4, 0.6]], tf.float32)
+    mock_anchors = box_list.BoxList(mock_anchor_corners)
+    mock_box_coder = MockBoxCoder()
+
+    expected_boxes = [[[0.0, 0.1, 0.5, 0.6], [0.5, 0.6, 0.7, 0.8]],
+                      [[0.1, 0.2, 0.3, 0.4], [0.7, 0.8, 0.9, 1.0]]]
+
+    encoded_boxes_list = [mock_box_coder.encode(
+        box_list.BoxList(tf.constant(boxes)), mock_anchors)
+                          for boxes in expected_boxes]
+    encoded_boxes = tf.stack(encoded_boxes_list)
+    decoded_boxes = box_coder.batch_decode(
+        encoded_boxes, mock_box_coder, mock_anchors)
+
+    with self.test_session() as sess:
+      decoded_boxes_result = sess.run(decoded_boxes)
+      self.assertAllClose(expected_boxes, decoded_boxes_result)
+
+
+if __name__ == '__main__':
+  tf.test.main()

object_detection/core/box_list.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Bounding Box List definition.
+
+BoxList represents a list of bounding boxes as tensorflow
+tensors, where each bounding box is represented as a row of 4 numbers,
+[y_min, x_min, y_max, x_max].  It is assumed that all bounding boxes
+within a given list correspond to a single image.  See also
+box_list_ops.py for common box related operations (such as area, iou, etc).
+
+Optionally, users can add additional related fields (such as weights).
+We assume the following things to be true about fields:
+* they correspond to boxes in the box_list along the 0th dimension
+* they have inferrable rank at graph construction time
+* all dimensions except for possibly the 0th can be inferred
+  (i.e., not None) at graph construction time.
+
+Some other notes:
+  * Following tensorflow conventions, we use height, width ordering,
+  and correspondingly, y,x (or ymin, xmin, ymax, xmax) ordering
+  * Tensors are always provided as (flat) [N, 4] tensors.
+"""
+
+import tensorflow as tf
+
+
+class BoxList(object):
+  """Box collection."""
+
+  def __init__(self, boxes):
+    """Constructs box collection.
+
+    Args:
+      boxes: a tensor of shape [N, 4] representing box corners
+
+    Raises:
+      ValueError: if invalid dimensions for bbox data or if bbox data is not in
+          float32 format.
+    """
+    if len(boxes.get_shape()) != 2 or boxes.get_shape()[-1] != 4:
+      raise ValueError('Invalid dimensions for box data.')
+    if boxes.dtype != tf.float32:
+      raise ValueError('Invalid tensor type: should be tf.float32')
+    self.data = {'boxes': boxes}
+
+  def num_boxes(self):
+    """Returns number of boxes held in collection.
+
+    Returns:
+      a tensor representing the number of boxes held in the collection.
+    """
+    return tf.shape(self.data['boxes'])[0]
+
+  def num_boxes_static(self):
+    """Returns number of boxes held in collection.
+
+    This number is inferred at graph construction time rather than run-time.
+
+    Returns:
+      Number of boxes held in collection (integer) or None if this is not
+        inferrable at graph construction time.
+    """
+    return self.data['boxes'].get_shape()[0].value
+
+  def get_all_fields(self):
+    """Returns all fields."""
+    return self.data.keys()
+
+  def get_extra_fields(self):
+    """Returns all non-box fields (i.e., everything not named 'boxes')."""
+    return [k for k in self.data.keys() if k != 'boxes']
+
+  def add_field(self, field, field_data):
+    """Add field to box list.
+
+    This method can be used to add related box data such as
+    weights/labels, etc.
+
+    Args:
+      field: a string key to access the data via `get`
+      field_data: a tensor containing the data to store in the BoxList
+    """
+    self.data[field] = field_data
+
+  def has_field(self, field):
+    return field in self.data
+
+  def get(self):
+    """Convenience function for accessing box coordinates.
+
+    Returns:
+      a tensor with shape [N, 4] representing box coordinates.
+    """
+    return self.get_field('boxes')
+
+  def set(self, boxes):
+    """Convenience function for setting box coordinates.
+
+    Args:
+      boxes: a tensor of shape [N, 4] representing box corners
+
+    Raises:
+      ValueError: if invalid dimensions for bbox data
+    """
+    if len(boxes.get_shape()) != 2 or boxes.get_shape()[-1] != 4:
+      raise ValueError('Invalid dimensions for box data.')
+    self.data['boxes'] = boxes
+
+  def get_field(self, field):
+    """Accesses a box collection and associated fields.
+
+    This function returns specified field with object; if no field is specified,
+    it returns the box coordinates.
+
+    Args:
+      field: this optional string parameter can be used to specify
+        a related field to be accessed.
+
+    Returns:
+      a tensor representing the box collection or an associated field.
+
+    Raises:
+      ValueError: if invalid field
+    """
+    if not self.has_field(field):
+      raise ValueError('field ' + str(field) + ' does not exist')
+    return self.data[field]
+
+  def set_field(self, field, value):
+    """Sets the value of a field.
+
+    Updates the field of a box_list with a given value.
+
+    Args:
+      field: (string) name of the field to set value.
+      value: the value to assign to the field.
+
+    Raises:
+      ValueError: if the box_list does not have specified field.
+    """
+    if not self.has_field(field):
+      raise ValueError('field %s does not exist' % field)
+    self.data[field] = value
+
+  def get_center_coordinates_and_sizes(self, scope=None):
+    """Computes the center coordinates, height and width of the boxes.
+
+    Args:
+      scope: name scope of the function.
+
+    Returns:
+      a list of 4 1-D tensors [ycenter, xcenter, height, width].
+    """
+    with tf.name_scope(scope, 'get_center_coordinates_and_sizes'):
+      box_corners = self.get()
+      ymin, xmin, ymax, xmax = tf.unstack(tf.transpose(box_corners))
+      width = xmax - xmin
+      height = ymax - ymin
+      ycenter = ymin + height / 2.
+      xcenter = xmin + width / 2.
+      return [ycenter, xcenter, height, width]
+
+  def transpose_coordinates(self, scope=None):
+    """Transpose the coordinate representation in a boxlist.
+
+    Args:
+      scope: name scope of the function.
+    """
+    with tf.name_scope(scope, 'transpose_coordinates'):
+      y_min, x_min, y_max, x_max = tf.split(
+          value=self.get(), num_or_size_splits=4, axis=1)
+      self.set(tf.concat([x_min, y_min, x_max, y_max], 1))
+
+  def as_tensor_dict(self, fields=None):
+    """Retrieves specified fields as a dictionary of tensors.
+
+    Args:
+      fields: (optional) list of fields to return in the dictionary.
+        If None (default), all fields are returned.
+
+    Returns:
+      tensor_dict: A dictionary of tensors specified by fields.
+
+    Raises:
+      ValueError: if specified field is not contained in boxlist.
+    """
+    tensor_dict = {}
+    if fields is None:
+      fields = self.get_all_fields()
+    for field in fields:
+      if not self.has_field(field):
+        raise ValueError('boxlist must contain all specified fields')
+      tensor_dict[field] = self.get_field(field)
+    return tensor_dict

object_detection/core/box_list_ops.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Bounding Box List operations.
+
+Example box operations that are supported:
+  * areas: compute bounding box areas
+  * iou: pairwise intersection-over-union scores
+  * sq_dist: pairwise distances between bounding boxes
+
+Whenever box_list_ops functions output a BoxList, the fields of the incoming
+BoxList are retained unless documented otherwise.
+"""
+import tensorflow as tf
+
+from object_detection.core import box_list
+from object_detection.utils import shape_utils
+
+
+class SortOrder(object):
+  """Enum class for sort order.
+
+  Attributes:
+    ascend: ascend order.
+    descend: descend order.
+  """
+  ascend = 1
+  descend = 2
+
+
+def area(boxlist, scope=None):
+  """Computes area of boxes.
+
+  Args:
+    boxlist: BoxList holding N boxes
+    scope: name scope.
+
+  Returns:
+    a tensor with shape [N] representing box areas.
+  """
+  with tf.name_scope(scope, 'Area'):
+    y_min, x_min, y_max, x_max = tf.split(
+        value=boxlist.get(), num_or_size_splits=4, axis=1)
+    return tf.squeeze((y_max - y_min) * (x_max - x_min), [1])
+
+
+def height_width(boxlist, scope=None):
+  """Computes height and width of boxes in boxlist.
+
+  Args:
+    boxlist: BoxList holding N boxes
+    scope: name scope.
+
+  Returns:
+    Height: A tensor with shape [N] representing box heights.
+    Width: A tensor with shape [N] representing box widths.
+  """
+  with tf.name_scope(scope, 'HeightWidth'):
+    y_min, x_min, y_max, x_max = tf.split(
+        value=boxlist.get(), num_or_size_splits=4, axis=1)
+    return tf.squeeze(y_max - y_min, [1]), tf.squeeze(x_max - x_min, [1])
+
+
+def scale(boxlist, y_scale, x_scale, scope=None):
+  """scale box coordinates in x and y dimensions.
+
+  Args:
+    boxlist: BoxList holding N boxes
+    y_scale: (float) scalar tensor
+    x_scale: (float) scalar tensor
+    scope: name scope.
+
+  Returns:
+    boxlist: BoxList holding N boxes
+  """
+  with tf.name_scope(scope, 'Scale'):
+    y_scale = tf.cast(y_scale, tf.float32)
+    x_scale = tf.cast(x_scale, tf.float32)
+    y_min, x_min, y_max, x_max = tf.split(
+        value=boxlist.get(), num_or_size_splits=4, axis=1)
+    y_min = y_scale * y_min
+    y_max = y_scale * y_max
+    x_min = x_scale * x_min
+    x_max = x_scale * x_max
+    scaled_boxlist = box_list.BoxList(
+        tf.concat([y_min, x_min, y_max, x_max], 1))
+    return _copy_extra_fields(scaled_boxlist, boxlist)
+
+
+def clip_to_window(boxlist, window, filter_nonoverlapping=True, scope=None):
+  """Clip bounding boxes to a window.
+
+  This op clips any input bounding boxes (represented by bounding box
+  corners) to a window, optionally filtering out boxes that do not
+  overlap at all with the window.
+
+  Args:
+    boxlist: BoxList holding M_in boxes
+    window: a tensor of shape [4] representing the [y_min, x_min, y_max, x_max]
+      window to which the op should clip boxes.
+    filter_nonoverlapping: whether to filter out boxes that do not overlap at
+      all with the window.
+    scope: name scope.
+
+  Returns:
+    a BoxList holding M_out boxes where M_out <= M_in
+  """
+  with tf.name_scope(scope, 'ClipToWindow'):
+    y_min, x_min, y_max, x_max = tf.split(
+        value=boxlist.get(), num_or_size_splits=4, axis=1)
+    win_y_min, win_x_min, win_y_max, win_x_max = tf.unstack(window)
+    y_min_clipped = tf.maximum(tf.minimum(y_min, win_y_max), win_y_min)
+    y_max_clipped = tf.maximum(tf.minimum(y_max, win_y_max), win_y_min)
+    x_min_clipped = tf.maximum(tf.minimum(x_min, win_x_max), win_x_min)
+    x_max_clipped = tf.maximum(tf.minimum(x_max, win_x_max), win_x_min)
+    clipped = box_list.BoxList(
+        tf.concat([y_min_clipped, x_min_clipped, y_max_clipped, x_max_clipped],
+                  1))
+    clipped = _copy_extra_fields(clipped, boxlist)
+    if filter_nonoverlapping:
+      areas = area(clipped)
+      nonzero_area_indices = tf.cast(
+          tf.reshape(tf.where(tf.greater(areas, 0.0)), [-1]), tf.int32)
+      clipped = gather(clipped, nonzero_area_indices)
+    return clipped
+
+
+def prune_outside_window(boxlist, window, scope=None):
+  """Prunes bounding boxes that fall outside a given window.
+
+  This function prunes bounding boxes that even partially fall outside the given
+  window. See also clip_to_window which only prunes bounding boxes that fall
+  completely outside the window, and clips any bounding boxes that partially
+  overflow.
+
+  Args:
+    boxlist: a BoxList holding M_in boxes.
+    window: a float tensor of shape [4] representing [ymin, xmin, ymax, xmax]
+      of the window
+    scope: name scope.
+
+  Returns:
+    pruned_corners: a tensor with shape [M_out, 4] where M_out <= M_in
+    valid_indices: a tensor with shape [M_out] indexing the valid bounding boxes
+     in the input tensor.
+  """
+  with tf.name_scope(scope, 'PruneOutsideWindow'):
+    y_min, x_min, y_max, x_max = tf.split(
+        value=boxlist.get(), num_or_size_splits=4, axis=1)
+    win_y_min, win_x_min, win_y_max, win_x_max = tf.unstack(window)
+    coordinate_violations = tf.concat([
+        tf.less(y_min, win_y_min), tf.less(x_min, win_x_min),
+        tf.greater(y_max, win_y_max), tf.greater(x_max, win_x_max)
+    ], 1)
+    valid_indices = tf.reshape(
+        tf.where(tf.logical_not(tf.reduce_any(coordinate_violations, 1))), [-1])
+    return gather(boxlist, valid_indices), valid_indices
+
+
+def prune_completely_outside_window(boxlist, window, scope=None):
+  """Prunes bounding boxes that fall completely outside of the given window.
+
+  The function clip_to_window prunes bounding boxes that fall
+  completely outside the window, but also clips any bounding boxes that
+  partially overflow. This function does not clip partially overflowing boxes.
+
+  Args:
+    boxlist: a BoxList holding M_in boxes.
+    window: a float tensor of shape [4] representing [ymin, xmin, ymax, xmax]
+      of the window
+    scope: name scope.
+
+  Returns:
+    pruned_corners: a tensor with shape [M_out, 4] where M_out <= M_in
+    valid_indices: a tensor with shape [M_out] indexing the valid bounding boxes
+     in the input tensor.
+  """
+  with tf.name_scope(scope, 'PruneCompleteleyOutsideWindow'):
+    y_min, x_min, y_max, x_max = tf.split(
+        value=boxlist.get(), num_or_size_splits=4, axis=1)
+    win_y_min, win_x_min, win_y_max, win_x_max = tf.unstack(window)
+    coordinate_violations = tf.concat([
+        tf.greater_equal(y_min, win_y_max), tf.greater_equal(x_min, win_x_max),
+        tf.less_equal(y_max, win_y_min), tf.less_equal(x_max, win_x_min)
+    ], 1)
+    valid_indices = tf.reshape(
+        tf.where(tf.logical_not(tf.reduce_any(coordinate_violations, 1))), [-1])
+    return gather(boxlist, valid_indices), valid_indices
+
+
+def intersection(boxlist1, boxlist2, scope=None):
+  """Compute pairwise intersection areas between boxes.
+
+  Args:
+    boxlist1: BoxList holding N boxes
+    boxlist2: BoxList holding M boxes
+    scope: name scope.
+
+  Returns:
+    a tensor with shape [N, M] representing pairwise intersections
+  """
+  with tf.name_scope(scope, 'Intersection'):
+    y_min1, x_min1, y_max1, x_max1 = tf.split(
+        value=boxlist1.get(), num_or_size_splits=4, axis=1)
+    y_min2, x_min2, y_max2, x_max2 = tf.split(
+        value=boxlist2.get(), num_or_size_splits=4, axis=1)
+    all_pairs_min_ymax = tf.minimum(y_max1, tf.transpose(y_max2))
+    all_pairs_max_ymin = tf.maximum(y_min1, tf.transpose(y_min2))
+    intersect_heights = tf.maximum(0.0, all_pairs_min_ymax - all_pairs_max_ymin)
+    all_pairs_min_xmax = tf.minimum(x_max1, tf.transpose(x_max2))
+    all_pairs_max_xmin = tf.maximum(x_min1, tf.transpose(x_min2))
+    intersect_widths = tf.maximum(0.0, all_pairs_min_xmax - all_pairs_max_xmin)
+    return intersect_heights * intersect_widths
+
+
+def matched_intersection(boxlist1, boxlist2, scope=None):
+  """Compute intersection areas between corresponding boxes in two boxlists.
+
+  Args:
+    boxlist1: BoxList holding N boxes
+    boxlist2: BoxList holding N boxes
+    scope: name scope.
+
+  Returns:
+    a tensor with shape [N] representing pairwise intersections
+  """
+  with tf.name_scope(scope, 'MatchedIntersection'):
+    y_min1, x_min1, y_max1, x_max1 = tf.split(
+        value=boxlist1.get(), num_or_size_splits=4, axis=1)
+    y_min2, x_min2, y_max2, x_max2 = tf.split(
+        value=boxlist2.get(), num_or_size_splits=4, axis=1)
+    min_ymax = tf.minimum(y_max1, y_max2)
+    max_ymin = tf.maximum(y_min1, y_min2)
+    intersect_heights = tf.maximum(0.0, min_ymax - max_ymin)
+    min_xmax = tf.minimum(x_max1, x_max2)
+    max_xmin = tf.maximum(x_min1, x_min2)
+    intersect_widths = tf.maximum(0.0, min_xmax - max_xmin)
+    return tf.reshape(intersect_heights * intersect_widths, [-1])
+
+
+def iou(boxlist1, boxlist2, scope=None):
+  """Computes pairwise intersection-over-union between box collections.
+
+  Args:
+    boxlist1: BoxList holding N boxes
+    boxlist2: BoxList holding M boxes
+    scope: name scope.
+
+  Returns:
+    a tensor with shape [N, M] representing pairwise iou scores.
+  """
+  with tf.name_scope(scope, 'IOU'):
+    intersections = intersection(boxlist1, boxlist2)
+    areas1 = area(boxlist1)
+    areas2 = area(boxlist2)
+    unions = (
+        tf.expand_dims(areas1, 1) + tf.expand_dims(areas2, 0) - intersections)
+    return tf.where(
+        tf.equal(intersections, 0.0),
+        tf.zeros_like(intersections), tf.truediv(intersections, unions))
+
+
+def matched_iou(boxlist1, boxlist2, scope=None):
+  """Compute intersection-over-union between corresponding boxes in boxlists.
+
+  Args:
+    boxlist1: BoxList holding N boxes
+    boxlist2: BoxList holding N boxes
+    scope: name scope.
+
+  Returns:
+    a tensor with shape [N] representing pairwise iou scores.
+  """
+  with tf.name_scope(scope, 'MatchedIOU'):
+    intersections = matched_intersection(boxlist1, boxlist2)
+    areas1 = area(boxlist1)
+    areas2 = area(boxlist2)
+    unions = areas1 + areas2 - intersections
+    return tf.where(
+        tf.equal(intersections, 0.0),
+        tf.zeros_like(intersections), tf.truediv(intersections, unions))
+
+
+def ioa(boxlist1, boxlist2, scope=None):
+  """Computes pairwise intersection-over-area between box collections.
+
+  intersection-over-area (IOA) between two boxes box1 and box2 is defined as
+  their intersection area over box2's area. Note that ioa is not symmetric,
+  that is, ioa(box1, box2) != ioa(box2, box1).
+
+  Args:
+    boxlist1: BoxList holding N boxes
+    boxlist2: BoxList holding M boxes
+    scope: name scope.
+
+  Returns:
+    a tensor with shape [N, M] representing pairwise ioa scores.
+  """
+  with tf.name_scope(scope, 'IOA'):
+    intersections = intersection(boxlist1, boxlist2)
+    areas = tf.expand_dims(area(boxlist2), 0)
+    return tf.truediv(intersections, areas)
+
+
+def prune_non_overlapping_boxes(
+    boxlist1, boxlist2, min_overlap=0.0, scope=None):
+  """Prunes the boxes in boxlist1 that overlap less than thresh with boxlist2.
+
+  For each box in boxlist1, we want its IOA to be more than minoverlap with
+  at least one of the boxes in boxlist2. If it does not, we remove it.
+
+  Args:
+    boxlist1: BoxList holding N boxes.
+    boxlist2: BoxList holding M boxes.
+    min_overlap: Minimum required overlap between boxes, to count them as
+                overlapping.
+    scope: name scope.
+
+  Returns:
+    new_boxlist1: A pruned boxlist with size [N', 4].
+    keep_inds: A tensor with shape [N'] indexing kept bounding boxes in the
+      first input BoxList `boxlist1`.
+  """
+  with tf.name_scope(scope, 'PruneNonOverlappingBoxes'):
+    ioa_ = ioa(boxlist2, boxlist1)  # [M, N] tensor
+    ioa_ = tf.reduce_max(ioa_, reduction_indices=[0])  # [N] tensor
+    keep_bool = tf.greater_equal(ioa_, tf.constant(min_overlap))
+    keep_inds = tf.squeeze(tf.where(keep_bool), squeeze_dims=[1])
+    new_boxlist1 = gather(boxlist1, keep_inds)
+    return new_boxlist1, keep_inds
+
+
+def prune_small_boxes(boxlist, min_side, scope=None):
+  """Prunes small boxes in the boxlist which have a side smaller than min_side.
+
+  Args:
+    boxlist: BoxList holding N boxes.
+    min_side: Minimum width AND height of box to survive pruning.
+    scope: name scope.
+
+  Returns:
+    A pruned boxlist.
+  """
+  with tf.name_scope(scope, 'PruneSmallBoxes'):
+    height, width = height_width(boxlist)
+    is_valid = tf.logical_and(tf.greater_equal(width, min_side),
+                              tf.greater_equal(height, min_side))
+    return gather(boxlist, tf.reshape(tf.where(is_valid), [-1]))
+
+
+def change_coordinate_frame(boxlist, window, scope=None):
+  """Change coordinate frame of the boxlist to be relative to window's frame.
+
+  Given a window of the form [ymin, xmin, ymax, xmax],
+  changes bounding box coordinates from boxlist to be relative to this window
+  (e.g., the min corner maps to (0,0) and the max corner maps to (1,1)).
+
+  An example use case is data augmentation: where we are given groundtruth
+  boxes (boxlist) and would like to randomly crop the image to some
+  window (window). In this case we need to change the coordinate frame of
+  each groundtruth box to be relative to this new window.
+
+  Args:
+    boxlist: A BoxList object holding N boxes.
+    window: A rank 1 tensor [4].
+    scope: name scope.
+
+  Returns:
+    Returns a BoxList object with N boxes.
+  """
+  with tf.name_scope(scope, 'ChangeCoordinateFrame'):
+    win_height = window[2] - window[0]
+    win_width = window[3] - window[1]
+    boxlist_new = scale(box_list.BoxList(
+        boxlist.get() - [window[0], window[1], window[0], window[1]]),
+                        1.0 / win_height, 1.0 / win_width)
+    boxlist_new = _copy_extra_fields(boxlist_new, boxlist)
+    return boxlist_new
+
+
+def sq_dist(boxlist1, boxlist2, scope=None):
+  """Computes the pairwise squared distances between box corners.
+
+  This op treats each box as if it were a point in a 4d Euclidean space and
+  computes pairwise squared distances.
+
+  Mathematically, we are given two matrices of box coordinates X and Y,
+  where X(i,:) is the i'th row of X, containing the 4 numbers defining the
+  corners of the i'th box in boxlist1. Similarly Y(j,:) corresponds to
+  boxlist2.  We compute
+  Z(i,j) = ||X(i,:) - Y(j,:)||^2
+         = ||X(i,:)||^2 + ||Y(j,:)||^2 - 2 X(i,:)' * Y(j,:),
+
+  Args:
+    boxlist1: BoxList holding N boxes
+    boxlist2: BoxList holding M boxes
+    scope: name scope.
+
+  Returns:
+    a tensor with shape [N, M] representing pairwise distances
+  """
+  with tf.name_scope(scope, 'SqDist'):
+    sqnorm1 = tf.reduce_sum(tf.square(boxlist1.get()), 1, keep_dims=True)
+    sqnorm2 = tf.reduce_sum(tf.square(boxlist2.get()), 1, keep_dims=True)
+    innerprod = tf.matmul(boxlist1.get(), boxlist2.get(),
+                          transpose_a=False, transpose_b=True)
+    return sqnorm1 + tf.transpose(sqnorm2) - 2.0 * innerprod
+
+
+def boolean_mask(boxlist, indicator, fields=None, scope=None):
+  """Select boxes from BoxList according to indicator and return new BoxList.
+
+  `boolean_mask` returns the subset of boxes that are marked as "True" by the
+  indicator tensor. By default, `boolean_mask` returns boxes corresponding to
+  the input index list, as well as all additional fields stored in the boxlist
+  (indexing into the first dimension).  However one can optionally only draw
+  from a subset of fields.
+
+  Args:
+    boxlist: BoxList holding N boxes
+    indicator: a rank-1 boolean tensor
+    fields: (optional) list of fields to also gather from.  If None (default),
+      all fields are gathered from.  Pass an empty fields list to only gather
+      the box coordinates.
+    scope: name scope.
+
+  Returns:
+    subboxlist: a BoxList corresponding to the subset of the input BoxList
+      specified by indicator
+  Raises:
+    ValueError: if `indicator` is not a rank-1 boolean tensor.
+  """
+  with tf.name_scope(scope, 'BooleanMask'):
+    if indicator.shape.ndims != 1:
+      raise ValueError('indicator should have rank 1')
+    if indicator.dtype != tf.bool:
+      raise ValueError('indicator should be a boolean tensor')
+    subboxlist = box_list.BoxList(tf.boolean_mask(boxlist.get(), indicator))
+    if fields is None:
+      fields = boxlist.get_extra_fields()
+    for field in fields:
+      if not boxlist.has_field(field):
+        raise ValueError('boxlist must contain all specified fields')
+      subfieldlist = tf.boolean_mask(boxlist.get_field(field), indicator)
+      subboxlist.add_field(field, subfieldlist)
+    return subboxlist
+
+
+def gather(boxlist, indices, fields=None, scope=None):
+  """Gather boxes from BoxList according to indices and return new BoxList.
+
+  By default, `gather` returns boxes corresponding to the input index list, as
+  well as all additional fields stored in the boxlist (indexing into the
+  first dimension).  However one can optionally only gather from a
+  subset of fields.
+
+  Args:
+    boxlist: BoxList holding N boxes
+    indices: a rank-1 tensor of type int32 / int64
+    fields: (optional) list of fields to also gather from.  If None (default),
+      all fields are gathered from.  Pass an empty fields list to only gather
+      the box coordinates.
+    scope: name scope.
+
+  Returns:
+    subboxlist: a BoxList corresponding to the subset of the input BoxList
+    specified by indices
+  Raises:
+    ValueError: if specified field is not contained in boxlist or if the
+      indices are not of type int32
+  """
+  with tf.name_scope(scope, 'Gather'):
+    if len(indices.shape.as_list()) != 1:
+      raise ValueError('indices should have rank 1')
+    if indices.dtype != tf.int32 and indices.dtype != tf.int64:
+      raise ValueError('indices should be an int32 / int64 tensor')
+    subboxlist = box_list.BoxList(tf.gather(boxlist.get(), indices))
+    if fields is None:
+      fields = boxlist.get_extra_fields()
+    for field in fields:
+      if not boxlist.has_field(field):
+        raise ValueError('boxlist must contain all specified fields')
+      subfieldlist = tf.gather(boxlist.get_field(field), indices)
+      subboxlist.add_field(field, subfieldlist)
+    return subboxlist
+
+
+def concatenate(boxlists, fields=None, scope=None):
+  """Concatenate list of BoxLists.
+
+  This op concatenates a list of input BoxLists into a larger BoxList.  It also
+  handles concatenation of BoxList fields as long as the field tensor shapes
+  are equal except for the first dimension.
+
+  Args:
+    boxlists: list of BoxList objects
+    fields: optional list of fields to also concatenate.  By default, all
+      fields from the first BoxList in the list are included in the
+      concatenation.
+    scope: name scope.
+
+  Returns:
+    a BoxList with number of boxes equal to
+      sum([boxlist.num_boxes() for boxlist in BoxList])
+  Raises:
+    ValueError: if boxlists is invalid (i.e., is not a list, is empty, or
+      contains non BoxList objects), or if requested fields are not contained in
+      all boxlists
+  """
+  with tf.name_scope(scope, 'Concatenate'):
+    if not isinstance(boxlists, list):
+      raise ValueError('boxlists should be a list')
+    if not boxlists:
+      raise ValueError('boxlists should have nonzero length')
+    for boxlist in boxlists:
+      if not isinstance(boxlist, box_list.BoxList):
+        raise ValueError('all elements of boxlists should be BoxList objects')
+    concatenated = box_list.BoxList(
+        tf.concat([boxlist.get() for boxlist in boxlists], 0))
+    if fields is None:
+      fields = boxlists[0].get_extra_fields()
+    for field in fields:
+      first_field_shape = boxlists[0].get_field(field).get_shape().as_list()
+      first_field_shape[0] = -1
+      if None in first_field_shape:
+        raise ValueError('field %s must have fully defined shape except for the'
+                         ' 0th dimension.' % field)
+      for boxlist in boxlists:
+        if not boxlist.has_field(field):
+          raise ValueError('boxlist must contain all requested fields')
+        field_shape = boxlist.get_field(field).get_shape().as_list()
+        field_shape[0] = -1
+        if field_shape != first_field_shape:
+          raise ValueError('field %s must have same shape for all boxlists '
+                           'except for the 0th dimension.' % field)
+      concatenated_field = tf.concat(
+          [boxlist.get_field(field) for boxlist in boxlists], 0)
+      concatenated.add_field(field, concatenated_field)
+    return concatenated
+
+
+def sort_by_field(boxlist, field, order=SortOrder.descend, scope=None):
+  """Sort boxes and associated fields according to a scalar field.
+
+  A common use case is reordering the boxes according to descending scores.
+
+  Args:
+    boxlist: BoxList holding N boxes.
+    field: A BoxList field for sorting and reordering the BoxList.
+    order: (Optional) descend or ascend. Default is descend.
+    scope: name scope.
+
+  Returns:
+    sorted_boxlist: A sorted BoxList with the field in the specified order.
+
+  Raises:
+    ValueError: if specified field does not exist
+    ValueError: if the order is not either descend or ascend
+  """
+  with tf.name_scope(scope, 'SortByField'):
+    if order != SortOrder.descend and order != SortOrder.ascend:
+      raise ValueError('Invalid sort order')
+
+    field_to_sort = boxlist.get_field(field)
+    if len(field_to_sort.shape.as_list()) != 1:
+      raise ValueError('Field should have rank 1')
+
+    num_boxes = boxlist.num_boxes()
+    num_entries = tf.size(field_to_sort)
+    length_assert = tf.Assert(
+        tf.equal(num_boxes, num_entries),
+        ['Incorrect field size: actual vs expected.', num_entries, num_boxes])
+
+    with tf.control_dependencies([length_assert]):
+      # TODO: Remove with tf.device when top_k operation runs correctly on GPU.
+      with tf.device('/cpu:0'):
+        _, sorted_indices = tf.nn.top_k(field_to_sort, num_boxes, sorted=True)
+
+    if order == SortOrder.ascend:
+      sorted_indices = tf.reverse_v2(sorted_indices, [0])
+
+    return gather(boxlist, sorted_indices)
+
+
+def visualize_boxes_in_image(image, boxlist, normalized=False, scope=None):
+  """Overlay bounding box list on image.
+
+  Currently this visualization plots a 1 pixel thick red bounding box on top
+  of the image.  Note that tf.image.draw_bounding_boxes essentially is
+  1 indexed.
+
+  Args:
+    image: an image tensor with shape [height, width, 3]
+    boxlist: a BoxList
+    normalized: (boolean) specify whether corners are to be interpreted
+      as absolute coordinates in image space or normalized with respect to the
+      image size.
+    scope: name scope.
+
+  Returns:
+    image_and_boxes: an image tensor with shape [height, width, 3]
+  """
+  with tf.name_scope(scope, 'VisualizeBoxesInImage'):
+    if not normalized:
+      height, width, _ = tf.unstack(tf.shape(image))
+      boxlist = scale(boxlist,
+                      1.0 / tf.cast(height, tf.float32),
+                      1.0 / tf.cast(width, tf.float32))
+    corners = tf.expand_dims(boxlist.get(), 0)
+    image = tf.expand_dims(image, 0)
+    return tf.squeeze(tf.image.draw_bounding_boxes(image, corners), [0])
+
+
+def filter_field_value_equals(boxlist, field, value, scope=None):
+  """Filter to keep only boxes with field entries equal to the given value.
+
+  Args:
+    boxlist: BoxList holding N boxes.
+    field: field name for filtering.
+    value: scalar value.
+    scope: name scope.
+
+  Returns:
+    a BoxList holding M boxes where M <= N
+
+  Raises:
+    ValueError: if boxlist not a BoxList object or if it does not have
+      the specified field.
+  """
+  with tf.name_scope(scope, 'FilterFieldValueEquals'):
+    if not isinstance(boxlist, box_list.BoxList):
+      raise ValueError('boxlist must be a BoxList')
+    if not boxlist.has_field(field):
+      raise ValueError('boxlist must contain the specified field')
+    filter_field = boxlist.get_field(field)
+    gather_index = tf.reshape(tf.where(tf.equal(filter_field, value)), [-1])
+    return gather(boxlist, gather_index)
+
+
+def filter_greater_than(boxlist, thresh, scope=None):
+  """Filter to keep only boxes with score exceeding a given threshold.
+
+  This op keeps the collection of boxes whose corresponding scores are
+  greater than the input threshold.
+
+  TODO: Change function name to FilterScoresGreaterThan
+
+  Args:
+    boxlist: BoxList holding N boxes.  Must contain a 'scores' field
+      representing detection scores.
+    thresh: scalar threshold
+    scope: name scope.
+
+  Returns:
+    a BoxList holding M boxes where M <= N
+
+  Raises:
+    ValueError: if boxlist not a BoxList object or if it does not
+      have a scores field
+  """
+  with tf.name_scope(scope, 'FilterGreaterThan'):
+    if not isinstance(boxlist, box_list.BoxList):
+      raise ValueError('boxlist must be a BoxList')
+    if not boxlist.has_field('scores'):
+      raise ValueError('input boxlist must have \'scores\' field')
+    scores = boxlist.get_field('scores')
+    if len(scores.shape.as_list()) > 2:
+      raise ValueError('Scores should have rank 1 or 2')
+    if len(scores.shape.as_list()) == 2 and scores.shape.as_list()[1] != 1:
+      raise ValueError('Scores should have rank 1 or have shape '
+                       'consistent with [None, 1]')
+    high_score_indices = tf.cast(tf.reshape(
+        tf.where(tf.greater(scores, thresh)),
+        [-1]), tf.int32)
+    return gather(boxlist, high_score_indices)
+
+
+def non_max_suppression(boxlist, thresh, max_output_size, scope=None):
+  """Non maximum suppression.
+
+  This op greedily selects a subset of detection bounding boxes, pruning
+  away boxes that have high IOU (intersection over union) overlap (> thresh)
+  with already selected boxes.  Note that this only works for a single class ---
+  to apply NMS to multi-class predictions, use MultiClassNonMaxSuppression.
+
+  Args:
+    boxlist: BoxList holding N boxes.  Must contain a 'scores' field
+      representing detection scores.
+    thresh: scalar threshold
+    max_output_size: maximum number of retained boxes
+    scope: name scope.
+
+  Returns:
+    a BoxList holding M boxes where M <= max_output_size
+  Raises:
+    ValueError: if thresh is not in [0, 1]
+  """
+  with tf.name_scope(scope, 'NonMaxSuppression'):
+    if not 0 <= thresh <= 1.0:
+      raise ValueError('thresh must be between 0 and 1')
+    if not isinstance(boxlist, box_list.BoxList):
+      raise ValueError('boxlist must be a BoxList')
+    if not boxlist.has_field('scores'):
+      raise ValueError('input boxlist must have \'scores\' field')
+    selected_indices = tf.image.non_max_suppression(
+        boxlist.get(), boxlist.get_field('scores'),
+        max_output_size, iou_threshold=thresh)
+    return gather(boxlist, selected_indices)
+
+
+def _copy_extra_fields(boxlist_to_copy_to, boxlist_to_copy_from):
+  """Copies the extra fields of boxlist_to_copy_from to boxlist_to_copy_to.
+
+  Args:
+    boxlist_to_copy_to: BoxList to which extra fields are copied.
+    boxlist_to_copy_from: BoxList from which fields are copied.
+
+  Returns:
+    boxlist_to_copy_to with extra fields.
+  """
+  for field in boxlist_to_copy_from.get_extra_fields():
+    boxlist_to_copy_to.add_field(field, boxlist_to_copy_from.get_field(field))
+  return boxlist_to_copy_to
+
+
+def to_normalized_coordinates(boxlist, height, width,
+                              check_range=True, scope=None):
+  """Converts absolute box coordinates to normalized coordinates in [0, 1].
+
+  Usually one uses the dynamic shape of the image or conv-layer tensor:
+    boxlist = box_list_ops.to_normalized_coordinates(boxlist,
+                                                     tf.shape(images)[1],
+                                                     tf.shape(images)[2]),
+
+  This function raises an assertion failed error at graph execution time when
+  the maximum coordinate is smaller than 1.01 (which means that coordinates are
+  already normalized). The value 1.01 is to deal with small rounding errors.
+
+  Args:
+    boxlist: BoxList with coordinates in terms of pixel-locations.
+    height: Maximum value for height of absolute box coordinates.
+    width: Maximum value for width of absolute box coordinates.
+    check_range: If True, checks if the coordinates are normalized or not.
+    scope: name scope.
+
+  Returns:
+    boxlist with normalized coordinates in [0, 1].
+  """
+  with tf.name_scope(scope, 'ToNormalizedCoordinates'):
+    height = tf.cast(height, tf.float32)
+    width = tf.cast(width, tf.float32)
+
+    if check_range:
+      max_val = tf.reduce_max(boxlist.get())
+      max_assert = tf.Assert(tf.greater(max_val, 1.01),
+                             ['max value is lower than 1.01: ', max_val])
+      with tf.control_dependencies([max_assert]):
+        width = tf.identity(width)
+
+    return scale(boxlist, 1 / height, 1 / width)
+
+
+def to_absolute_coordinates(boxlist, height, width,
+                            check_range=True, scope=None):
+  """Converts normalized box coordinates to absolute pixel coordinates.
+
+  This function raises an assertion failed error when the maximum box coordinate
+  value is larger than 1.01 (in which case coordinates are already absolute).
+
+  Args:
+    boxlist: BoxList with coordinates in range [0, 1].
+    height: Maximum value for height of absolute box coordinates.
+    width: Maximum value for width of absolute box coordinates.
+    check_range: If True, checks if the coordinates are normalized or not.
+    scope: name scope.
+
+  Returns:
+    boxlist with absolute coordinates in terms of the image size.
+
+  """
+  with tf.name_scope(scope, 'ToAbsoluteCoordinates'):
+    height = tf.cast(height, tf.float32)
+    width = tf.cast(width, tf.float32)
+
+    # Ensure range of input boxes is correct.
+    if check_range:
+      box_maximum = tf.reduce_max(boxlist.get())
+      max_assert = tf.Assert(tf.greater_equal(1.01, box_maximum),
+                             ['maximum box coordinate value is larger '
+                              'than 1.01: ', box_maximum])
+      with tf.control_dependencies([max_assert]):
+        width = tf.identity(width)
+
+    return scale(boxlist, height, width)
+
+
+def refine_boxes_multi_class(pool_boxes,
+                             num_classes,
+                             nms_iou_thresh,
+                             nms_max_detections,
+                             voting_iou_thresh=0.5):
+  """Refines a pool of boxes using non max suppression and box voting.
+
+  Box refinement is done independently for each class.
+
+  Args:
+    pool_boxes: (BoxList) A collection of boxes to be refined. pool_boxes must
+      have a rank 1 'scores' field and a rank 1 'classes' field.
+    num_classes: (int scalar) Number of classes.
+    nms_iou_thresh: (float scalar) iou threshold for non max suppression (NMS).
+    nms_max_detections: (int scalar) maximum output size for NMS.
+    voting_iou_thresh: (float scalar) iou threshold for box voting.
+
+  Returns:
+    BoxList of refined boxes.
+
+  Raises:
+    ValueError: if
+      a) nms_iou_thresh or voting_iou_thresh is not in [0, 1].
+      b) pool_boxes is not a BoxList.
+      c) pool_boxes does not have a scores and classes field.
+  """
+  if not 0.0 <= nms_iou_thresh <= 1.0:
+    raise ValueError('nms_iou_thresh must be between 0 and 1')
+  if not 0.0 <= voting_iou_thresh <= 1.0:
+    raise ValueError('voting_iou_thresh must be between 0 and 1')
+  if not isinstance(pool_boxes, box_list.BoxList):
+    raise ValueError('pool_boxes must be a BoxList')
+  if not pool_boxes.has_field('scores'):
+    raise ValueError('pool_boxes must have a \'scores\' field')
+  if not pool_boxes.has_field('classes'):
+    raise ValueError('pool_boxes must have a \'classes\' field')
+
+  refined_boxes = []
+  for i in range(num_classes):
+    boxes_class = filter_field_value_equals(pool_boxes, 'classes', i)
+    refined_boxes_class = refine_boxes(boxes_class, nms_iou_thresh,
+                                       nms_max_detections, voting_iou_thresh)
+    refined_boxes.append(refined_boxes_class)
+  return sort_by_field(concatenate(refined_boxes), 'scores')
+
+
+def refine_boxes(pool_boxes,
+                 nms_iou_thresh,
+                 nms_max_detections,
+                 voting_iou_thresh=0.5):
+  """Refines a pool of boxes using non max suppression and box voting.
+
+  Args:
+    pool_boxes: (BoxList) A collection of boxes to be refined. pool_boxes must
+      have a rank 1 'scores' field.
+    nms_iou_thresh: (float scalar) iou threshold for non max suppression (NMS).
+    nms_max_detections: (int scalar) maximum output size for NMS.
+    voting_iou_thresh: (float scalar) iou threshold for box voting.
+
+  Returns:
+    BoxList of refined boxes.
+
+  Raises:
+    ValueError: if
+      a) nms_iou_thresh or voting_iou_thresh is not in [0, 1].
+      b) pool_boxes is not a BoxList.
+      c) pool_boxes does not have a scores field.
+  """
+  if not 0.0 <= nms_iou_thresh <= 1.0:
+    raise ValueError('nms_iou_thresh must be between 0 and 1')
+  if not 0.0 <= voting_iou_thresh <= 1.0:
+    raise ValueError('voting_iou_thresh must be between 0 and 1')
+  if not isinstance(pool_boxes, box_list.BoxList):
+    raise ValueError('pool_boxes must be a BoxList')
+  if not pool_boxes.has_field('scores'):
+    raise ValueError('pool_boxes must have a \'scores\' field')
+
+  nms_boxes = non_max_suppression(
+      pool_boxes, nms_iou_thresh, nms_max_detections)
+  return box_voting(nms_boxes, pool_boxes, voting_iou_thresh)
+
+
+def box_voting(selected_boxes, pool_boxes, iou_thresh=0.5):
+  """Performs box voting as described in S. Gidaris and N. Komodakis, ICCV 2015.
+
+  Performs box voting as described in 'Object detection via a multi-region &
+  semantic segmentation-aware CNN model', Gidaris and Komodakis, ICCV 2015. For
+  each box 'B' in selected_boxes, we find the set 'S' of boxes in pool_boxes
+  with iou overlap >= iou_thresh. The location of B is set to the weighted
+  average location of boxes in S (scores are used for weighting). And the score
+  of B is set to the average score of boxes in S.
+
+  Args:
+    selected_boxes: BoxList containing a subset of boxes in pool_boxes. These
+      boxes are usually selected from pool_boxes using non max suppression.
+    pool_boxes: BoxList containing a set of (possibly redundant) boxes.
+    iou_thresh: (float scalar) iou threshold for matching boxes in
+      selected_boxes and pool_boxes.
+
+  Returns:
+    BoxList containing averaged locations and scores for each box in
+    selected_boxes.
+
+  Raises:
+    ValueError: if
+      a) selected_boxes or pool_boxes is not a BoxList.
+      b) if iou_thresh is not in [0, 1].
+      c) pool_boxes does not have a scores field.
+  """
+  if not 0.0 <= iou_thresh <= 1.0:
+    raise ValueError('iou_thresh must be between 0 and 1')
+  if not isinstance(selected_boxes, box_list.BoxList):
+    raise ValueError('selected_boxes must be a BoxList')
+  if not isinstance(pool_boxes, box_list.BoxList):
+    raise ValueError('pool_boxes must be a BoxList')
+  if not pool_boxes.has_field('scores'):
+    raise ValueError('pool_boxes must have a \'scores\' field')
+
+  iou_ = iou(selected_boxes, pool_boxes)
+  match_indicator = tf.to_float(tf.greater(iou_, iou_thresh))
+  num_matches = tf.reduce_sum(match_indicator, 1)
+  # TODO: Handle the case where some boxes in selected_boxes do not match to any
+  # boxes in pool_boxes. For such boxes without any matches, we should return
+  # the original boxes without voting.
+  match_assert = tf.Assert(
+      tf.reduce_all(tf.greater(num_matches, 0)),
+      ['Each box in selected_boxes must match with at least one box '
+       'in pool_boxes.'])
+
+  scores = tf.expand_dims(pool_boxes.get_field('scores'), 1)
+  scores_assert = tf.Assert(
+      tf.reduce_all(tf.greater_equal(scores, 0)),
+      ['Scores must be non negative.'])
+
+  with tf.control_dependencies([scores_assert, match_assert]):
+    sum_scores = tf.matmul(match_indicator, scores)
+  averaged_scores = tf.reshape(sum_scores, [-1]) / num_matches
+
+  box_locations = tf.matmul(match_indicator,
+                            pool_boxes.get() * scores) / sum_scores
+  averaged_boxes = box_list.BoxList(box_locations)
+  _copy_extra_fields(averaged_boxes, selected_boxes)
+  averaged_boxes.add_field('scores', averaged_scores)
+  return averaged_boxes
+
+
+def pad_or_clip_box_list(boxlist, num_boxes, scope=None):
+  """Pads or clips all fields of a BoxList.
+
+  Args:
+    boxlist: A BoxList with arbitrary of number of boxes.
+    num_boxes: First num_boxes in boxlist are kept.
+      The fields are zero-padded if num_boxes is bigger than the
+      actual number of boxes.
+    scope: name scope.
+
+  Returns:
+    BoxList with all fields padded or clipped.
+  """
+  with tf.name_scope(scope, 'PadOrClipBoxList'):
+    subboxlist = box_list.BoxList(shape_utils.pad_or_clip_tensor(
+        boxlist.get(), num_boxes))
+    for field in boxlist.get_extra_fields():
+      subfield = shape_utils.pad_or_clip_tensor(
+          boxlist.get_field(field), num_boxes)
+      subboxlist.add_field(field, subfield)
+    return subboxlist

object_detection/core/box_list_ops_test.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Tests for object_detection.core.box_list_ops."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.python.framework import errors
+
+from object_detection.core import box_list
+from object_detection.core import box_list_ops
+
+
+class BoxListOpsTest(tf.test.TestCase):
+  """Tests for common bounding box operations."""
+
+  def test_area(self):
+    corners = tf.constant([[0.0, 0.0, 10.0, 20.0], [1.0, 2.0, 3.0, 4.0]])
+    exp_output = [200.0, 4.0]
+    boxes = box_list.BoxList(corners)
+    areas = box_list_ops.area(boxes)
+    with self.test_session() as sess:
+      areas_output = sess.run(areas)
+      self.assertAllClose(areas_output, exp_output)
+
+  def test_height_width(self):
+    corners = tf.constant([[0.0, 0.0, 10.0, 20.0], [1.0, 2.0, 3.0, 4.0]])
+    exp_output_heights = [10., 2.]
+    exp_output_widths = [20., 2.]
+    boxes = box_list.BoxList(corners)
+    heights, widths = box_list_ops.height_width(boxes)
+    with self.test_session() as sess:
+      output_heights, output_widths = sess.run([heights, widths])
+      self.assertAllClose(output_heights, exp_output_heights)
+      self.assertAllClose(output_widths, exp_output_widths)
+
+  def test_scale(self):
+    corners = tf.constant([[0, 0, 100, 200], [50, 120, 100, 140]],
+                          dtype=tf.float32)
+    boxes = box_list.BoxList(corners)
+    boxes.add_field('extra_data', tf.constant([[1], [2]]))
+
+    y_scale = tf.constant(1.0/100)
+    x_scale = tf.constant(1.0/200)
+    scaled_boxes = box_list_ops.scale(boxes, y_scale, x_scale)
+    exp_output = [[0, 0, 1, 1], [0.5, 0.6, 1.0, 0.7]]
+    with self.test_session() as sess:
+      scaled_corners_out = sess.run(scaled_boxes.get())
+      self.assertAllClose(scaled_corners_out, exp_output)
+      extra_data_out = sess.run(scaled_boxes.get_field('extra_data'))
+      self.assertAllEqual(extra_data_out, [[1], [2]])
+
+  def test_clip_to_window_filter_boxes_which_fall_outside_the_window(
+      self):
+    window = tf.constant([0, 0, 9, 14], tf.float32)
+    corners = tf.constant([[5.0, 5.0, 6.0, 6.0],
+                           [-1.0, -2.0, 4.0, 5.0],
+                           [2.0, 3.0, 5.0, 9.0],
+                           [0.0, 0.0, 9.0, 14.0],
+                           [-100.0, -100.0, 300.0, 600.0],
+                           [-10.0, -10.0, -9.0, -9.0]])
+    boxes = box_list.BoxList(corners)
+    boxes.add_field('extra_data', tf.constant([[1], [2], [3], [4], [5], [6]]))
+    exp_output = [[5.0, 5.0, 6.0, 6.0], [0.0, 0.0, 4.0, 5.0],
+                  [2.0, 3.0, 5.0, 9.0], [0.0, 0.0, 9.0, 14.0],
+                  [0.0, 0.0, 9.0, 14.0]]
+    pruned = box_list_ops.clip_to_window(
+        boxes, window, filter_nonoverlapping=True)
+    with self.test_session() as sess:
+      pruned_output = sess.run(pruned.get())
+      self.assertAllClose(pruned_output, exp_output)
+      extra_data_out = sess.run(pruned.get_field('extra_data'))
+      self.assertAllEqual(extra_data_out, [[1], [2], [3], [4], [5]])
+
+  def test_clip_to_window_without_filtering_boxes_which_fall_outside_the_window(
+      self):
+    window = tf.constant([0, 0, 9, 14], tf.float32)
+    corners = tf.constant([[5.0, 5.0, 6.0, 6.0],
+                           [-1.0, -2.0, 4.0, 5.0],
+                           [2.0, 3.0, 5.0, 9.0],
+                           [0.0, 0.0, 9.0, 14.0],
+                           [-100.0, -100.0, 300.0, 600.0],
+                           [-10.0, -10.0, -9.0, -9.0]])
+    boxes = box_list.BoxList(corners)
+    boxes.add_field('extra_data', tf.constant([[1], [2], [3], [4], [5], [6]]))
+    exp_output = [[5.0, 5.0, 6.0, 6.0], [0.0, 0.0, 4.0, 5.0],
+                  [2.0, 3.0, 5.0, 9.0], [0.0, 0.0, 9.0, 14.0],
+                  [0.0, 0.0, 9.0, 14.0], [0.0, 0.0, 0.0, 0.0]]
+    pruned = box_list_ops.clip_to_window(
+        boxes, window, filter_nonoverlapping=False)
+    with self.test_session() as sess:
+      pruned_output = sess.run(pruned.get())
+      self.assertAllClose(pruned_output, exp_output)
+      extra_data_out = sess.run(pruned.get_field('extra_data'))
+      self.assertAllEqual(extra_data_out, [[1], [2], [3], [4], [5], [6]])
+
+  def test_prune_outside_window_filters_boxes_which_fall_outside_the_window(
+      self):
+    window = tf.constant([0, 0, 9, 14], tf.float32)
+    corners = tf.constant([[5.0, 5.0, 6.0, 6.0],
+                           [-1.0, -2.0, 4.0, 5.0],
+                           [2.0, 3.0, 5.0, 9.0],
+                           [0.0, 0.0, 9.0, 14.0],
+                           [-10.0, -10.0, -9.0, -9.0],
+                           [-100.0, -100.0, 300.0, 600.0]])
+    boxes = box_list.BoxList(corners)
+    boxes.add_field('extra_data', tf.constant([[1], [2], [3], [4], [5], [6]]))
+    exp_output = [[5.0, 5.0, 6.0, 6.0],
+                  [2.0, 3.0, 5.0, 9.0],
+                  [0.0, 0.0, 9.0, 14.0]]
+    pruned, keep_indices = box_list_ops.prune_outside_window(boxes, window)
+    with self.test_session() as sess:
+      pruned_output = sess.run(pruned.get())
+      self.assertAllClose(pruned_output, exp_output)
+      keep_indices_out = sess.run(keep_indices)
+      self.assertAllEqual(keep_indices_out, [0, 2, 3])
+      extra_data_out = sess.run(pruned.get_field('extra_data'))
+      self.assertAllEqual(extra_data_out, [[1], [3], [4]])
+
+  def test_prune_completely_outside_window(self):
+    window = tf.constant([0, 0, 9, 14], tf.float32)
+    corners = tf.constant([[5.0, 5.0, 6.0, 6.0],
+                           [-1.0, -2.0, 4.0, 5.0],
+                           [2.0, 3.0, 5.0, 9.0],
+                           [0.0, 0.0, 9.0, 14.0],
+                           [-10.0, -10.0, -9.0, -9.0],
+                           [-100.0, -100.0, 300.0, 600.0]])
+    boxes = box_list.BoxList(corners)
+    boxes.add_field('extra_data', tf.constant([[1], [2], [3], [4], [5], [6]]))
+    exp_output = [[5.0, 5.0, 6.0, 6.0],
+                  [-1.0, -2.0, 4.0, 5.0],
+                  [2.0, 3.0, 5.0, 9.0],
+                  [0.0, 0.0, 9.0, 14.0],
+                  [-100.0, -100.0, 300.0, 600.0]]
+    pruned, keep_indices = box_list_ops.prune_completely_outside_window(boxes,
+                                                                        window)
+    with self.test_session() as sess:
+      pruned_output = sess.run(pruned.get())
+      self.assertAllClose(pruned_output, exp_output)
+      keep_indices_out = sess.run(keep_indices)
+      self.assertAllEqual(keep_indices_out, [0, 1, 2, 3, 5])
+      extra_data_out = sess.run(pruned.get_field('extra_data'))
+      self.assertAllEqual(extra_data_out, [[1], [2], [3], [4], [6]])
+
+  def test_intersection(self):
+    corners1 = tf.constant([[4.0, 3.0, 7.0, 5.0], [5.0, 6.0, 10.0, 7.0]])
+    corners2 = tf.constant([[3.0, 4.0, 6.0, 8.0], [14.0, 14.0, 15.0, 15.0],
+                            [0.0, 0.0, 20.0, 20.0]])
+    exp_output = [[2.0, 0.0, 6.0], [1.0, 0.0, 5.0]]
+    boxes1 = box_list.BoxList(corners1)
+    boxes2 = box_list.BoxList(corners2)
+    intersect = box_list_ops.intersection(boxes1, boxes2)
+    with self.test_session() as sess:
+      intersect_output = sess.run(intersect)
+      self.assertAllClose(intersect_output, exp_output)
+
+  def test_matched_intersection(self):
+    corners1 = tf.constant([[4.0, 3.0, 7.0, 5.0], [5.0, 6.0, 10.0, 7.0]])
+    corners2 = tf.constant([[3.0, 4.0, 6.0, 8.0], [14.0, 14.0, 15.0, 15.0]])
+    exp_output = [2.0, 0.0]
+    boxes1 = box_list.BoxList(corners1)
+    boxes2 = box_list.BoxList(corners2)
+    intersect = box_list_ops.matched_intersection(boxes1, boxes2)
+    with self.test_session() as sess:
+      intersect_output = sess.run(intersect)
+      self.assertAllClose(intersect_output, exp_output)
+
+  def test_iou(self):
+    corners1 = tf.constant([[4.0, 3.0, 7.0, 5.0], [5.0, 6.0, 10.0, 7.0]])
+    corners2 = tf.constant([[3.0, 4.0, 6.0, 8.0], [14.0, 14.0, 15.0, 15.0],
+                            [0.0, 0.0, 20.0, 20.0]])
+    exp_output = [[2.0 / 16.0, 0, 6.0 / 400.0], [1.0 / 16.0, 0.0, 5.0 / 400.0]]
+    boxes1 = box_list.BoxList(corners1)
+    boxes2 = box_list.BoxList(corners2)
+    iou = box_list_ops.iou(boxes1, boxes2)
+    with self.test_session() as sess:
+      iou_output = sess.run(iou)
+      self.assertAllClose(iou_output, exp_output)
+
+  def test_matched_iou(self):
+    corners1 = tf.constant([[4.0, 3.0, 7.0, 5.0], [5.0, 6.0, 10.0, 7.0]])
+    corners2 = tf.constant([[3.0, 4.0, 6.0, 8.0], [14.0, 14.0, 15.0, 15.0]])
+    exp_output = [2.0 / 16.0, 0]
+    boxes1 = box_list.BoxList(corners1)
+    boxes2 = box_list.BoxList(corners2)
+    iou = box_list_ops.matched_iou(boxes1, boxes2)
+    with self.test_session() as sess:
+      iou_output = sess.run(iou)
+      self.assertAllClose(iou_output, exp_output)
+
+  def test_iouworks_on_empty_inputs(self):
+    corners1 = tf.constant([[4.0, 3.0, 7.0, 5.0], [5.0, 6.0, 10.0, 7.0]])
+    corners2 = tf.constant([[3.0, 4.0, 6.0, 8.0], [14.0, 14.0, 15.0, 15.0],
+                            [0.0, 0.0, 20.0, 20.0]])
+    boxes1 = box_list.BoxList(corners1)
+    boxes2 = box_list.BoxList(corners2)
+    boxes_empty = box_list.BoxList(tf.zeros((0, 4)))
+    iou_empty_1 = box_list_ops.iou(boxes1, boxes_empty)
+    iou_empty_2 = box_list_ops.iou(boxes_empty, boxes2)
+    iou_empty_3 = box_list_ops.iou(boxes_empty, boxes_empty)
+    with self.test_session() as sess:
+      iou_output_1, iou_output_2, iou_output_3 = sess.run(
+          [iou_empty_1, iou_empty_2, iou_empty_3])
+      self.assertAllEqual(iou_output_1.shape, (2, 0))
+      self.assertAllEqual(iou_output_2.shape, (0, 3))
+      self.assertAllEqual(iou_output_3.shape, (0, 0))
+
+  def test_ioa(self):
+    corners1 = tf.constant([[4.0, 3.0, 7.0, 5.0], [5.0, 6.0, 10.0, 7.0]])
+    corners2 = tf.constant([[3.0, 4.0, 6.0, 8.0], [14.0, 14.0, 15.0, 15.0],
+                            [0.0, 0.0, 20.0, 20.0]])
+    exp_output_1 = [[2.0 / 12.0, 0, 6.0 / 400.0],
+                    [1.0 / 12.0, 0.0, 5.0 / 400.0]]
+    exp_output_2 = [[2.0 / 6.0, 1.0 / 5.0],
+                    [0, 0],
+                    [6.0 / 6.0, 5.0 / 5.0]]
+    boxes1 = box_list.BoxList(corners1)
+    boxes2 = box_list.BoxList(corners2)
+    ioa_1 = box_list_ops.ioa(boxes1, boxes2)
+    ioa_2 = box_list_ops.ioa(boxes2, boxes1)
+    with self.test_session() as sess:
+      ioa_output_1, ioa_output_2 = sess.run([ioa_1, ioa_2])
+      self.assertAllClose(ioa_output_1, exp_output_1)
+      self.assertAllClose(ioa_output_2, exp_output_2)
+
+  def test_prune_non_overlapping_boxes(self):
+    corners1 = tf.constant([[4.0, 3.0, 7.0, 5.0], [5.0, 6.0, 10.0, 7.0]])
+    corners2 = tf.constant([[3.0, 4.0, 6.0, 8.0], [14.0, 14.0, 15.0, 15.0],
+                            [0.0, 0.0, 20.0, 20.0]])
+    boxes1 = box_list.BoxList(corners1)
+    boxes2 = box_list.BoxList(corners2)
+    minoverlap = 0.5
+
+    exp_output_1 = boxes1
+    exp_output_2 = box_list.BoxList(tf.constant(0.0, shape=[0, 4]))
+    output_1, keep_indices_1 = box_list_ops.prune_non_overlapping_boxes(
+        boxes1, boxes2, min_overlap=minoverlap)
+    output_2, keep_indices_2 = box_list_ops.prune_non_overlapping_boxes(
+        boxes2, boxes1, min_overlap=minoverlap)
+    with self.test_session() as sess:
+      (output_1_, keep_indices_1_, output_2_, keep_indices_2_, exp_output_1_,
+       exp_output_2_) = sess.run(
+           [output_1.get(), keep_indices_1,
+            output_2.get(), keep_indices_2,
+            exp_output_1.get(), exp_output_2.get()])
+      self.assertAllClose(output_1_, exp_output_1_)
+      self.assertAllClose(output_2_, exp_output_2_)
+      self.assertAllEqual(keep_indices_1_, [0, 1])
+      self.assertAllEqual(keep_indices_2_, [])
+
+  def test_prune_small_boxes(self):
+    boxes = tf.constant([[4.0, 3.0, 7.0, 5.0],
+                         [5.0, 6.0, 10.0, 7.0],
+                         [3.0, 4.0, 6.0, 8.0],
+                         [14.0, 14.0, 15.0, 15.0],
+                         [0.0, 0.0, 20.0, 20.0]])
+    exp_boxes = [[3.0, 4.0, 6.0, 8.0],
+                 [0.0, 0.0, 20.0, 20.0]]
+    boxes = box_list.BoxList(boxes)
+    pruned_boxes = box_list_ops.prune_small_boxes(boxes, 3)
+    with self.test_session() as sess:
+      pruned_boxes = sess.run(pruned_boxes.get())
+      self.assertAllEqual(pruned_boxes, exp_boxes)
+
+  def test_prune_small_boxes_prunes_boxes_with_negative_side(self):
+    boxes = tf.constant([[4.0, 3.0, 7.0, 5.0],
+                         [5.0, 6.0, 10.0, 7.0],
+                         [3.0, 4.0, 6.0, 8.0],
+                         [14.0, 14.0, 15.0, 15.0],
+                         [0.0, 0.0, 20.0, 20.0],
+                         [2.0, 3.0, 1.5, 7.0],  # negative height
+                         [2.0, 3.0, 5.0, 1.7]])  # negative width
+    exp_boxes = [[3.0, 4.0, 6.0, 8.0],
+                 [0.0, 0.0, 20.0, 20.0]]
+    boxes = box_list.BoxList(boxes)
+    pruned_boxes = box_list_ops.prune_small_boxes(boxes, 3)
+    with self.test_session() as sess:
+      pruned_boxes = sess.run(pruned_boxes.get())
+      self.assertAllEqual(pruned_boxes, exp_boxes)
+
+  def test_change_coordinate_frame(self):
+    corners = tf.constant([[0.25, 0.5, 0.75, 0.75], [0.5, 0.0, 1.0, 1.0]])
+    window = tf.constant([0.25, 0.25, 0.75, 0.75])
+    boxes = box_list.BoxList(corners)
+
+    expected_corners = tf.constant([[0, 0.5, 1.0, 1.0], [0.5, -0.5, 1.5, 1.5]])
+    expected_boxes = box_list.BoxList(expected_corners)
+    output = box_list_ops.change_coordinate_frame(boxes, window)
+
+    with self.test_session() as sess:
+      output_, expected_boxes_ = sess.run([output.get(), expected_boxes.get()])
+      self.assertAllClose(output_, expected_boxes_)
+
+  def test_ioaworks_on_empty_inputs(self):
+    corners1 = tf.constant([[4.0, 3.0, 7.0, 5.0], [5.0, 6.0, 10.0, 7.0]])
+    corners2 = tf.constant([[3.0, 4.0, 6.0, 8.0], [14.0, 14.0, 15.0, 15.0],
+                            [0.0, 0.0, 20.0, 20.0]])
+    boxes1 = box_list.BoxList(corners1)
+    boxes2 = box_list.BoxList(corners2)
+    boxes_empty = box_list.BoxList(tf.zeros((0, 4)))
+    ioa_empty_1 = box_list_ops.ioa(boxes1, boxes_empty)
+    ioa_empty_2 = box_list_ops.ioa(boxes_empty, boxes2)
+    ioa_empty_3 = box_list_ops.ioa(boxes_empty, boxes_empty)
+    with self.test_session() as sess:
+      ioa_output_1, ioa_output_2, ioa_output_3 = sess.run(
+          [ioa_empty_1, ioa_empty_2, ioa_empty_3])
+      self.assertAllEqual(ioa_output_1.shape, (2, 0))
+      self.assertAllEqual(ioa_output_2.shape, (0, 3))
+      self.assertAllEqual(ioa_output_3.shape, (0, 0))
+
+  def test_pairwise_distances(self):
+    corners1 = tf.constant([[0.0, 0.0, 0.0, 0.0],
+                            [1.0, 1.0, 0.0, 2.0]])
+    corners2 = tf.constant([[3.0, 4.0, 1.0, 0.0],
+                            [-4.0, 0.0, 0.0, 3.0],
+                            [0.0, 0.0, 0.0, 0.0]])
+    exp_output = [[26, 25, 0], [18, 27, 6]]
+    boxes1 = box_list.BoxList(corners1)
+    boxes2 = box_list.BoxList(corners2)
+    dist_matrix = box_list_ops.sq_dist(boxes1, boxes2)
+    with self.test_session() as sess:
+      dist_output = sess.run(dist_matrix)
+      self.assertAllClose(dist_output, exp_output)
+
+  def test_boolean_mask(self):
+    corners = tf.constant(
+        [4 * [0.0], 4 * [1.0], 4 * [2.0], 4 * [3.0], 4 * [4.0]])
+    indicator = tf.constant([True, False, True, False, True], tf.bool)
+    expected_subset = [4 * [0.0], 4 * [2.0], 4 * [4.0]]
+    boxes = box_list.BoxList(corners)
+    subset = box_list_ops.boolean_mask(boxes, indicator)
+    with self.test_session() as sess:
+      subset_output = sess.run(subset.get())
+      self.assertAllClose(subset_output, expected_subset)
+
+  def test_boolean_mask_with_field(self):
+    corners = tf.constant(
+        [4 * [0.0], 4 * [1.0], 4 * [2.0], 4 * [3.0], 4 * [4.0]])
+    indicator = tf.constant([True, False, True, False, True], tf.bool)
+    weights = tf.constant([[.1], [.3], [.5], [.7], [.9]], tf.float32)
+    expected_subset = [4 * [0.0], 4 * [2.0], 4 * [4.0]]
+    expected_weights = [[.1], [.5], [.9]]
+
+    boxes = box_list.BoxList(corners)
+    boxes.add_field('weights', weights)
+    subset = box_list_ops.boolean_mask(boxes, indicator, ['weights'])
+    with self.test_session() as sess:
+      subset_output, weights_output = sess.run(
+          [subset.get(), subset.get_field('weights')])
+      self.assertAllClose(subset_output, expected_subset)
+      self.assertAllClose(weights_output, expected_weights)
+
+  def test_gather(self):
+    corners = tf.constant(
+        [4 * [0.0], 4 * [1.0], 4 * [2.0], 4 * [3.0], 4 * [4.0]])
+    indices = tf.constant([0, 2, 4], tf.int32)
+    expected_subset = [4 * [0.0], 4 * [2.0], 4 * [4.0]]
+    boxes = box_list.BoxList(corners)
+    subset = box_list_ops.gather(boxes, indices)
+    with self.test_session() as sess:
+      subset_output = sess.run(subset.get())
+      self.assertAllClose(subset_output, expected_subset)
+
+  def test_gather_with_field(self):
+    corners = tf.constant([4*[0.0], 4*[1.0], 4*[2.0], 4*[3.0], 4*[4.0]])
+    indices = tf.constant([0, 2, 4], tf.int32)
+    weights = tf.constant([[.1], [.3], [.5], [.7], [.9]], tf.float32)
+    expected_subset = [4 * [0.0], 4 * [2.0], 4 * [4.0]]
+    expected_weights = [[.1], [.5], [.9]]
+
+    boxes = box_list.BoxList(corners)
+    boxes.add_field('weights', weights)
+    subset = box_list_ops.gather(boxes, indices, ['weights'])
+    with self.test_session() as sess:
+      subset_output, weights_output = sess.run(
+          [subset.get(), subset.get_field('weights')])
+      self.assertAllClose(subset_output, expected_subset)
+      self.assertAllClose(weights_output, expected_weights)
+
+  def test_gather_with_invalid_field(self):
+    corners = tf.constant([4 * [0.0], 4 * [1.0]])
+    indices = tf.constant([0, 1], tf.int32)
+    weights = tf.constant([[.1], [.3]], tf.float32)
+
+    boxes = box_list.BoxList(corners)
+    boxes.add_field('weights', weights)
+    with self.assertRaises(ValueError):
+      box_list_ops.gather(boxes, indices, ['foo', 'bar'])
+
+  def test_gather_with_invalid_inputs(self):
+    corners = tf.constant(
+        [4 * [0.0], 4 * [1.0], 4 * [2.0], 4 * [3.0], 4 * [4.0]])
+    indices_float32 = tf.constant([0, 2, 4], tf.float32)
+    boxes = box_list.BoxList(corners)
+    with self.assertRaises(ValueError):
+      _ = box_list_ops.gather(boxes, indices_float32)
+    indices_2d = tf.constant([[0, 2, 4]], tf.int32)
+    boxes = box_list.BoxList(corners)
+    with self.assertRaises(ValueError):
+      _ = box_list_ops.gather(boxes, indices_2d)
+
+  def test_gather_with_dynamic_indexing(self):
+    corners = tf.constant([4 * [0.0], 4 * [1.0], 4 * [2.0], 4 * [3.0], 4 * [4.0]
+                          ])
+    weights = tf.constant([.5, .3, .7, .1, .9], tf.float32)
+    indices = tf.reshape(tf.where(tf.greater(weights, 0.4)), [-1])
+    expected_subset = [4 * [0.0], 4 * [2.0], 4 * [4.0]]
+    expected_weights = [.5, .7, .9]
+
+    boxes = box_list.BoxList(corners)
+    boxes.add_field('weights', weights)
+    subset = box_list_ops.gather(boxes, indices, ['weights'])
+    with self.test_session() as sess:
+      subset_output, weights_output = sess.run([subset.get(), subset.get_field(
+          'weights')])
+      self.assertAllClose(subset_output, expected_subset)
+      self.assertAllClose(weights_output, expected_weights)
+
+  def test_sort_by_field_ascending_order(self):
+    exp_corners = [[0, 0, 1, 1], [0, 0.1, 1, 1.1], [0, -0.1, 1, 0.9],
+                   [0, 10, 1, 11], [0, 10.1, 1, 11.1], [0, 100, 1, 101]]
+    exp_scores = [.95, .9, .75, .6, .5, .3]
+    exp_weights = [.2, .45, .6, .75, .8, .92]
+    shuffle = [2, 4, 0, 5, 1, 3]
+    corners = tf.constant([exp_corners[i] for i in shuffle], tf.float32)
+    boxes = box_list.BoxList(corners)
+    boxes.add_field('scores', tf.constant(
+        [exp_scores[i] for i in shuffle], tf.float32))
+    boxes.add_field('weights', tf.constant(
+        [exp_weights[i] for i in shuffle], tf.float32))
+    sort_by_weight = box_list_ops.sort_by_field(
+        boxes,
+        'weights',
+        order=box_list_ops.SortOrder.ascend)
+    with self.test_session() as sess:
+      corners_out, scores_out, weights_out = sess.run([
+          sort_by_weight.get(),
+          sort_by_weight.get_field('scores'),
+          sort_by_weight.get_field('weights')])
+      self.assertAllClose(corners_out, exp_corners)
+      self.assertAllClose(scores_out, exp_scores)
+      self.assertAllClose(weights_out, exp_weights)
+
+  def test_sort_by_field_descending_order(self):
+    exp_corners = [[0, 0, 1, 1], [0, 0.1, 1, 1.1], [0, -0.1, 1, 0.9],
+                   [0, 10, 1, 11], [0, 10.1, 1, 11.1], [0, 100, 1, 101]]
+    exp_scores = [.95, .9, .75, .6, .5, .3]
+    exp_weights = [.2, .45, .6, .75, .8, .92]
+    shuffle = [2, 4, 0, 5, 1, 3]
+
+    corners = tf.constant([exp_corners[i] for i in shuffle], tf.float32)
+    boxes = box_list.BoxList(corners)
+    boxes.add_field('scores', tf.constant(
+        [exp_scores[i] for i in shuffle], tf.float32))
+    boxes.add_field('weights', tf.constant(
+        [exp_weights[i] for i in shuffle], tf.float32))
+
+    sort_by_score = box_list_ops.sort_by_field(boxes, 'scores')
+    with self.test_session() as sess:
+      corners_out, scores_out, weights_out = sess.run([sort_by_score.get(
+      ), sort_by_score.get_field('scores'), sort_by_score.get_field('weights')])
+      self.assertAllClose(corners_out, exp_corners)
+      self.assertAllClose(scores_out, exp_scores)
+      self.assertAllClose(weights_out, exp_weights)
+
+  def test_sort_by_field_invalid_inputs(self):
+    corners = tf.constant([4 * [0.0], 4 * [0.5], 4 * [1.0], 4 * [2.0], 4 *
+                           [3.0], 4 * [4.0]])
+    misc = tf.constant([[.95, .9], [.5, .3]], tf.float32)
+    weights = tf.constant([.1, .2], tf.float32)
+    boxes = box_list.BoxList(corners)
+    boxes.add_field('misc', misc)
+    boxes.add_field('weights', weights)
+
+    with self.test_session() as sess:
+      with self.assertRaises(ValueError):
+        box_list_ops.sort_by_field(boxes, 'area')
+
+      with self.assertRaises(ValueError):
+        box_list_ops.sort_by_field(boxes, 'misc')
+
+      with self.assertRaisesWithPredicateMatch(errors.InvalidArgumentError,
+                                               'Incorrect field size'):
+        sess.run(box_list_ops.sort_by_field(boxes, 'weights').get())
+
+  def test_visualize_boxes_in_image(self):
+    image = tf.zeros((6, 4, 3))
+    corners = tf.constant([[0, 0, 5, 3],
+                           [0, 0, 3, 2]], tf.float32)
+    boxes = box_list.BoxList(corners)
+    image_and_boxes = box_list_ops.visualize_boxes_in_image(image, boxes)
+    image_and_boxes_bw = tf.to_float(
+        tf.greater(tf.reduce_sum(image_and_boxes, 2), 0.0))
+    exp_result = [[1, 1, 1, 0],
+                  [1, 1, 1, 0],
+                  [1, 1, 1, 0],
+                  [1, 0, 1, 0],
+                  [1, 1, 1, 0],
+                  [0, 0, 0, 0]]
+    with self.test_session() as sess:
+      output = sess.run(image_and_boxes_bw)
+      self.assertAllEqual(output.astype(int), exp_result)
+
+  def test_filter_field_value_equals(self):
+    corners = tf.constant([[0, 0, 1, 1],
+                           [0, 0.1, 1, 1.1],
+                           [0, -0.1, 1, 0.9],
+                           [0, 10, 1, 11],
+                           [0, 10.1, 1, 11.1],
+                           [0, 100, 1, 101]], tf.float32)
+    boxes = box_list.BoxList(corners)
+    boxes.add_field('classes', tf.constant([1, 2, 1, 2, 2, 1]))
+    exp_output1 = [[0, 0, 1, 1], [0, -0.1, 1, 0.9], [0, 100, 1, 101]]
+    exp_output2 = [[0, 0.1, 1, 1.1], [0, 10, 1, 11], [0, 10.1, 1, 11.1]]
+
+    filtered_boxes1 = box_list_ops.filter_field_value_equals(
+        boxes, 'classes', 1)
+    filtered_boxes2 = box_list_ops.filter_field_value_equals(
+        boxes, 'classes', 2)
+    with self.test_session() as sess:
+      filtered_output1, filtered_output2 = sess.run([filtered_boxes1.get(),
+                                                     filtered_boxes2.get()])
+      self.assertAllClose(filtered_output1, exp_output1)
+      self.assertAllClose(filtered_output2, exp_output2)
+
+  def test_filter_greater_than(self):
+    corners = tf.constant([[0, 0, 1, 1],
+                           [0, 0.1, 1, 1.1],
+                           [0, -0.1, 1, 0.9],
+                           [0, 10, 1, 11],
+                           [0, 10.1, 1, 11.1],
+                           [0, 100, 1, 101]], tf.float32)
+    boxes = box_list.BoxList(corners)
+    boxes.add_field('scores', tf.constant([.1, .75, .9, .5, .5, .8]))
+    thresh = .6
+    exp_output = [[0, 0.1, 1, 1.1], [0, -0.1, 1, 0.9], [0, 100, 1, 101]]
+
+    filtered_boxes = box_list_ops.filter_greater_than(boxes, thresh)
+    with self.test_session() as sess:
+      filtered_output = sess.run(filtered_boxes.get())
+      self.assertAllClose(filtered_output, exp_output)
+
+  def test_clip_box_list(self):
+    boxlist = box_list.BoxList(
+        tf.constant([[0.1, 0.1, 0.4, 0.4], [0.1, 0.1, 0.5, 0.5],
+                     [0.6, 0.6, 0.8, 0.8], [0.2, 0.2, 0.3, 0.3]], tf.float32))
+    boxlist.add_field('classes', tf.constant([0, 0, 1, 1]))
+    boxlist.add_field('scores', tf.constant([0.75, 0.65, 0.3, 0.2]))
+    num_boxes = 2
+    clipped_boxlist = box_list_ops.pad_or_clip_box_list(boxlist, num_boxes)
+
+    expected_boxes = [[0.1, 0.1, 0.4, 0.4], [0.1, 0.1, 0.5, 0.5]]
+    expected_classes = [0, 0]
+    expected_scores = [0.75, 0.65]
+    with self.test_session() as sess:
+      boxes_out, classes_out, scores_out = sess.run(
+          [clipped_boxlist.get(), clipped_boxlist.get_field('classes'),
+           clipped_boxlist.get_field('scores')])
+
+      self.assertAllClose(expected_boxes, boxes_out)
+      self.assertAllEqual(expected_classes, classes_out)
+      self.assertAllClose(expected_scores, scores_out)
+
+  def test_pad_box_list(self):
+    boxlist = box_list.BoxList(
+        tf.constant([[0.1, 0.1, 0.4, 0.4], [0.1, 0.1, 0.5, 0.5]], tf.float32))
+    boxlist.add_field('classes', tf.constant([0, 1]))
+    boxlist.add_field('scores', tf.constant([0.75, 0.2]))
+    num_boxes = 4
+    padded_boxlist = box_list_ops.pad_or_clip_box_list(boxlist, num_boxes)
+
+    expected_boxes = [[0.1, 0.1, 0.4, 0.4], [0.1, 0.1, 0.5, 0.5],
+                      [0, 0, 0, 0], [0, 0, 0, 0]]
+    expected_classes = [0, 1, 0, 0]
+    expected_scores = [0.75, 0.2, 0, 0]
+    with self.test_session() as sess:
+      boxes_out, classes_out, scores_out = sess.run(
+          [padded_boxlist.get(), padded_boxlist.get_field('classes'),
+           padded_boxlist.get_field('scores')])
+
+      self.assertAllClose(expected_boxes, boxes_out)
+      self.assertAllEqual(expected_classes, classes_out)
+      self.assertAllClose(expected_scores, scores_out)
+
+
+class ConcatenateTest(tf.test.TestCase):
+
+  def test_invalid_input_box_list_list(self):
+    with self.assertRaises(ValueError):
+      box_list_ops.concatenate(None)
+    with self.assertRaises(ValueError):
+      box_list_ops.concatenate([])
+    with self.assertRaises(ValueError):
+      corners = tf.constant([[0, 0, 0, 0]], tf.float32)
+      boxlist = box_list.BoxList(corners)
+      box_list_ops.concatenate([boxlist, 2])
+
+  def test_concatenate_with_missing_fields(self):
+    corners1 = tf.constant([[0, 0, 0, 0], [1, 2, 3, 4]], tf.float32)
+    scores1 = tf.constant([1.0, 2.1])
+    corners2 = tf.constant([[0, 3, 1, 6], [2, 4, 3, 8]], tf.float32)
+    boxlist1 = box_list.BoxList(corners1)
+    boxlist1.add_field('scores', scores1)
+    boxlist2 = box_list.BoxList(corners2)
+    with self.assertRaises(ValueError):
+      box_list_ops.concatenate([boxlist1, boxlist2])
+
+  def test_concatenate_with_incompatible_field_shapes(self):
+    corners1 = tf.constant([[0, 0, 0, 0], [1, 2, 3, 4]], tf.float32)
+    scores1 = tf.constant([1.0, 2.1])
+    corners2 = tf.constant([[0, 3, 1, 6], [2, 4, 3, 8]], tf.float32)
+    scores2 = tf.constant([[1.0, 1.0], [2.1, 3.2]])
+    boxlist1 = box_list.BoxList(corners1)
+    boxlist1.add_field('scores', scores1)
+    boxlist2 = box_list.BoxList(corners2)
+    boxlist2.add_field('scores', scores2)
+    with self.assertRaises(ValueError):
+      box_list_ops.concatenate([boxlist1, boxlist2])
+
+  def test_concatenate_is_correct(self):
+    corners1 = tf.constant([[0, 0, 0, 0], [1, 2, 3, 4]], tf.float32)
+    scores1 = tf.constant([1.0, 2.1])
+    corners2 = tf.constant([[0, 3, 1, 6], [2, 4, 3, 8], [1, 0, 5, 10]],
+                           tf.float32)
+    scores2 = tf.constant([1.0, 2.1, 5.6])
+
+    exp_corners = [[0, 0, 0, 0],
+                   [1, 2, 3, 4],
+                   [0, 3, 1, 6],
+                   [2, 4, 3, 8],
+                   [1, 0, 5, 10]]
+    exp_scores = [1.0, 2.1, 1.0, 2.1, 5.6]
+
+    boxlist1 = box_list.BoxList(corners1)
+    boxlist1.add_field('scores', scores1)
+    boxlist2 = box_list.BoxList(corners2)
+    boxlist2.add_field('scores', scores2)
+    result = box_list_ops.concatenate([boxlist1, boxlist2])
+    with self.test_session() as sess:
+      corners_output, scores_output = sess.run(
+          [result.get(), result.get_field('scores')])
+      self.assertAllClose(corners_output, exp_corners)
+      self.assertAllClose(scores_output, exp_scores)
+
+
+class NonMaxSuppressionTest(tf.test.TestCase):
+
+  def test_with_invalid_scores_field(self):
+    corners = tf.constant([[0, 0, 1, 1],
+                           [0, 0.1, 1, 1.1],
+                           [0, -0.1, 1, 0.9],
+                           [0, 10, 1, 11],
+                           [0, 10.1, 1, 11.1],
+                           [0, 100, 1, 101]], tf.float32)
+    boxes = box_list.BoxList(corners)
+    boxes.add_field('scores', tf.constant([.9, .75, .6, .95, .5]))
+    iou_thresh = .5
+    max_output_size = 3
+    nms = box_list_ops.non_max_suppression(
+        boxes, iou_thresh, max_output_size)
+    with self.test_session() as sess:
+      with self.assertRaisesWithPredicateMatch(
+          errors.InvalidArgumentError, 'scores has incompatible shape'):
+        sess.run(nms.get())
+
+  def test_select_from_three_clusters(self):
+    corners = tf.constant([[0, 0, 1, 1],
+                           [0, 0.1, 1, 1.1],
+                           [0, -0.1, 1, 0.9],
+                           [0, 10, 1, 11],
+                           [0, 10.1, 1, 11.1],
+                           [0, 100, 1, 101]], tf.float32)
+    boxes = box_list.BoxList(corners)
+    boxes.add_field('scores', tf.constant([.9, .75, .6, .95, .5, .3]))
+    iou_thresh = .5
+    max_output_size = 3
+
+    exp_nms = [[0, 10, 1, 11],
+               [0, 0, 1, 1],
+               [0, 100, 1, 101]]
+    nms = box_list_ops.non_max_suppression(
+        boxes, iou_thresh, max_output_size)
+    with self.test_session() as sess:
+      nms_output = sess.run(nms.get())
+      self.assertAllClose(nms_output, exp_nms)
+
+  def test_select_at_most_two_boxes_from_three_clusters(self):
+    corners = tf.constant([[0, 0, 1, 1],
+                           [0, 0.1, 1, 1.1],
+                           [0, -0.1, 1, 0.9],
+                           [0, 10, 1, 11],
+                           [0, 10.1, 1, 11.1],
+                           [0, 100, 1, 101]], tf.float32)
+    boxes = box_list.BoxList(corners)
+    boxes.add_field('scores', tf.constant([.9, .75, .6, .95, .5, .3]))
+    iou_thresh = .5
+    max_output_size = 2
+
+    exp_nms = [[0, 10, 1, 11],
+               [0, 0, 1, 1]]
+    nms = box_list_ops.non_max_suppression(
+        boxes, iou_thresh, max_output_size)
+    with self.test_session() as sess:
+      nms_output = sess.run(nms.get())
+      self.assertAllClose(nms_output, exp_nms)
+
+  def test_select_at_most_thirty_boxes_from_three_clusters(self):
+    corners = tf.constant([[0, 0, 1, 1],
+                           [0, 0.1, 1, 1.1],
+                           [0, -0.1, 1, 0.9],
+                           [0, 10, 1, 11],
+                           [0, 10.1, 1, 11.1],
+                           [0, 100, 1, 101]], tf.float32)
+    boxes = box_list.BoxList(corners)
+    boxes.add_field('scores', tf.constant([.9, .75, .6, .95, .5, .3]))
+    iou_thresh = .5
+    max_output_size = 30
+
+    exp_nms = [[0, 10, 1, 11],
+               [0, 0, 1, 1],
+               [0, 100, 1, 101]]
+    nms = box_list_ops.non_max_suppression(
+        boxes, iou_thresh, max_output_size)
+    with self.test_session() as sess:
+      nms_output = sess.run(nms.get())
+      self.assertAllClose(nms_output, exp_nms)
+
+  def test_select_single_box(self):
+    corners = tf.constant([[0, 0, 1, 1]], tf.float32)
+    boxes = box_list.BoxList(corners)
+    boxes.add_field('scores', tf.constant([.9]))
+    iou_thresh = .5
+    max_output_size = 3
+
+    exp_nms = [[0, 0, 1, 1]]
+    nms = box_list_ops.non_max_suppression(
+        boxes, iou_thresh, max_output_size)
+    with self.test_session() as sess:
+      nms_output = sess.run(nms.get())
+      self.assertAllClose(nms_output, exp_nms)
+
+  def test_select_from_ten_identical_boxes(self):
+    corners = tf.constant(10 * [[0, 0, 1, 1]], tf.float32)
+    boxes = box_list.BoxList(corners)
+    boxes.add_field('scores', tf.constant(10 * [.9]))
+    iou_thresh = .5
+    max_output_size = 3
+
+    exp_nms = [[0, 0, 1, 1]]
+    nms = box_list_ops.non_max_suppression(
+        boxes, iou_thresh, max_output_size)
+    with self.test_session() as sess:
+      nms_output = sess.run(nms.get())
+      self.assertAllClose(nms_output, exp_nms)
+
+  def test_copy_extra_fields(self):
+    corners = tf.constant([[0, 0, 1, 1],
+                           [0, 0.1, 1, 1.1]], tf.float32)
+    boxes = box_list.BoxList(corners)
+    tensor1 = np.array([[1], [4]])
+    tensor2 = np.array([[1, 1], [2, 2]])
+    boxes.add_field('tensor1', tf.constant(tensor1))
+    boxes.add_field('tensor2', tf.constant(tensor2))
+    new_boxes = box_list.BoxList(tf.constant([[0, 0, 10, 10],
+                                              [1, 3, 5, 5]], tf.float32))
+    new_boxes = box_list_ops._copy_extra_fields(new_boxes, boxes)
+    with self.test_session() as sess:
+      self.assertAllClose(tensor1, sess.run(new_boxes.get_field('tensor1')))
+      self.assertAllClose(tensor2, sess.run(new_boxes.get_field('tensor2')))
+
+
+class CoordinatesConversionTest(tf.test.TestCase):
+
+  def test_to_normalized_coordinates(self):
+    coordinates = tf.constant([[0, 0, 100, 100],
+                               [25, 25, 75, 75]], tf.float32)
+    img = tf.ones((128, 100, 100, 3))
+    boxlist = box_list.BoxList(coordinates)
+    normalized_boxlist = box_list_ops.to_normalized_coordinates(
+        boxlist, tf.shape(img)[1], tf.shape(img)[2])
+    expected_boxes = [[0, 0, 1, 1],
+                      [0.25, 0.25, 0.75, 0.75]]
+
+    with self.test_session() as sess:
+      normalized_boxes = sess.run(normalized_boxlist.get())
+      self.assertAllClose(normalized_boxes, expected_boxes)
+
+  def test_to_normalized_coordinates_already_normalized(self):
+    coordinates = tf.constant([[0, 0, 1, 1],
+                               [0.25, 0.25, 0.75, 0.75]], tf.float32)
+    img = tf.ones((128, 100, 100, 3))
+    boxlist = box_list.BoxList(coordinates)
+    normalized_boxlist = box_list_ops.to_normalized_coordinates(
+        boxlist, tf.shape(img)[1], tf.shape(img)[2])
+
+    with self.test_session() as sess:
+      with self.assertRaisesOpError('assertion failed'):
+        sess.run(normalized_boxlist.get())
+
+  def test_to_absolute_coordinates(self):
+    coordinates = tf.constant([[0, 0, 1, 1],
+                               [0.25, 0.25, 0.75, 0.75]], tf.float32)
+    img = tf.ones((128, 100, 100, 3))
+    boxlist = box_list.BoxList(coordinates)
+    absolute_boxlist = box_list_ops.to_absolute_coordinates(boxlist,
+                                                            tf.shape(img)[1],
+                                                            tf.shape(img)[2])
+    expected_boxes = [[0, 0, 100, 100],
+                      [25, 25, 75, 75]]
+
+    with self.test_session() as sess:
+      absolute_boxes = sess.run(absolute_boxlist.get())
+      self.assertAllClose(absolute_boxes, expected_boxes)
+
+  def test_to_absolute_coordinates_already_abolute(self):
+    coordinates = tf.constant([[0, 0, 100, 100],
+                               [25, 25, 75, 75]], tf.float32)
+    img = tf.ones((128, 100, 100, 3))
+    boxlist = box_list.BoxList(coordinates)
+    absolute_boxlist = box_list_ops.to_absolute_coordinates(boxlist,
+                                                            tf.shape(img)[1],
+                                                            tf.shape(img)[2])
+
+    with self.test_session() as sess:
+      with self.assertRaisesOpError('assertion failed'):
+        sess.run(absolute_boxlist.get())
+
+  def test_convert_to_normalized_and_back(self):
+    coordinates = np.random.uniform(size=(100, 4))
+    coordinates = np.round(np.sort(coordinates) * 200)
+    coordinates[:, 2:4] += 1
+    coordinates[99, :] = [0, 0, 201, 201]
+    img = tf.ones((128, 202, 202, 3))
+
+    boxlist = box_list.BoxList(tf.constant(coordinates, tf.float32))
+    boxlist = box_list_ops.to_normalized_coordinates(boxlist,
+                                                     tf.shape(img)[1],
+                                                     tf.shape(img)[2])
+    boxlist = box_list_ops.to_absolute_coordinates(boxlist,
+                                                   tf.shape(img)[1],
+                                                   tf.shape(img)[2])
+
+    with self.test_session() as sess:
+      out = sess.run(boxlist.get())
+      self.assertAllClose(out, coordinates)
+
+  def test_convert_to_absolute_and_back(self):
+    coordinates = np.random.uniform(size=(100, 4))
+    coordinates = np.sort(coordinates)
+    coordinates[99, :] = [0, 0, 1, 1]
+    img = tf.ones((128, 202, 202, 3))
+
+    boxlist = box_list.BoxList(tf.constant(coordinates, tf.float32))
+    boxlist = box_list_ops.to_absolute_coordinates(boxlist,
+                                                   tf.shape(img)[1],
+                                                   tf.shape(img)[2])
+    boxlist = box_list_ops.to_normalized_coordinates(boxlist,
+                                                     tf.shape(img)[1],
+                                                     tf.shape(img)[2])
+
+    with self.test_session() as sess:
+      out = sess.run(boxlist.get())
+      self.assertAllClose(out, coordinates)
+
+
+class BoxRefinementTest(tf.test.TestCase):
+
+  def test_box_voting(self):
+    candidates = box_list.BoxList(
+        tf.constant([[0.1, 0.1, 0.4, 0.4], [0.6, 0.6, 0.8, 0.8]], tf.float32))
+    candidates.add_field('ExtraField', tf.constant([1, 2]))
+    pool = box_list.BoxList(
+        tf.constant([[0.1, 0.1, 0.4, 0.4], [0.1, 0.1, 0.5, 0.5],
+                     [0.6, 0.6, 0.8, 0.8]], tf.float32))
+    pool.add_field('scores', tf.constant([0.75, 0.25, 0.3]))
+    averaged_boxes = box_list_ops.box_voting(candidates, pool)
+    expected_boxes = [[0.1, 0.1, 0.425, 0.425], [0.6, 0.6, 0.8, 0.8]]
+    expected_scores = [0.5, 0.3]
+    with self.test_session() as sess:
+      boxes_out, scores_out, extra_field_out = sess.run(
+          [averaged_boxes.get(), averaged_boxes.get_field('scores'),
+           averaged_boxes.get_field('ExtraField')])
+
+      self.assertAllClose(expected_boxes, boxes_out)
+      self.assertAllClose(expected_scores, scores_out)
+      self.assertAllEqual(extra_field_out, [1, 2])
+
+  def test_box_voting_fails_with_negative_scores(self):
+    candidates = box_list.BoxList(
+        tf.constant([[0.1, 0.1, 0.4, 0.4]], tf.float32))
+    pool = box_list.BoxList(tf.constant([[0.1, 0.1, 0.4, 0.4]], tf.float32))
+    pool.add_field('scores', tf.constant([-0.2]))
+    averaged_boxes = box_list_ops.box_voting(candidates, pool)
+
+    with self.test_session() as sess:
+      with self.assertRaisesOpError('Scores must be non negative'):
+        sess.run([averaged_boxes.get()])
+
+  def test_box_voting_fails_when_unmatched(self):
+    candidates = box_list.BoxList(
+        tf.constant([[0.1, 0.1, 0.4, 0.4]], tf.float32))
+    pool = box_list.BoxList(tf.constant([[0.6, 0.6, 0.8, 0.8]], tf.float32))
+    pool.add_field('scores', tf.constant([0.2]))
+    averaged_boxes = box_list_ops.box_voting(candidates, pool)
+
+    with self.test_session() as sess:
+      with self.assertRaisesOpError('Each box in selected_boxes must match '
+                                    'with at least one box in pool_boxes.'):
+        sess.run([averaged_boxes.get()])
+
+  def test_refine_boxes(self):
+    pool = box_list.BoxList(
+        tf.constant([[0.1, 0.1, 0.4, 0.4], [0.1, 0.1, 0.5, 0.5],
+                     [0.6, 0.6, 0.8, 0.8]], tf.float32))
+    pool.add_field('ExtraField', tf.constant([1, 2, 3]))
+    pool.add_field('scores', tf.constant([0.75, 0.25, 0.3]))
+    refined_boxes = box_list_ops.refine_boxes(pool, 0.5, 10)
+
+    expected_boxes = [[0.1, 0.1, 0.425, 0.425], [0.6, 0.6, 0.8, 0.8]]
+    expected_scores = [0.5, 0.3]
+    with self.test_session() as sess:
+      boxes_out, scores_out, extra_field_out = sess.run(
+          [refined_boxes.get(), refined_boxes.get_field('scores'),
+           refined_boxes.get_field('ExtraField')])
+
+      self.assertAllClose(expected_boxes, boxes_out)
+      self.assertAllClose(expected_scores, scores_out)
+      self.assertAllEqual(extra_field_out, [1, 3])
+
+  def test_refine_boxes_multi_class(self):
+    pool = box_list.BoxList(
+        tf.constant([[0.1, 0.1, 0.4, 0.4], [0.1, 0.1, 0.5, 0.5],
+                     [0.6, 0.6, 0.8, 0.8], [0.2, 0.2, 0.3, 0.3]], tf.float32))
+    pool.add_field('classes', tf.constant([0, 0, 1, 1]))
+    pool.add_field('scores', tf.constant([0.75, 0.25, 0.3, 0.2]))
+    refined_boxes = box_list_ops.refine_boxes_multi_class(pool, 3, 0.5, 10)
+
+    expected_boxes = [[0.1, 0.1, 0.425, 0.425], [0.6, 0.6, 0.8, 0.8],
+                      [0.2, 0.2, 0.3, 0.3]]
+    expected_scores = [0.5, 0.3, 0.2]
+    with self.test_session() as sess:
+      boxes_out, scores_out, extra_field_out = sess.run(
+          [refined_boxes.get(), refined_boxes.get_field('scores'),
+           refined_boxes.get_field('classes')])
+
+      self.assertAllClose(expected_boxes, boxes_out)
+      self.assertAllClose(expected_scores, scores_out)
+      self.assertAllEqual(extra_field_out, [0, 1, 1])
+
+if __name__ == '__main__':
+  tf.test.main()

object_detection/core/box_list_test.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Tests for object_detection.core.box_list."""
+
+import tensorflow as tf
+
+from object_detection.core import box_list
+
+
+class BoxListTest(tf.test.TestCase):
+  """Tests for BoxList class."""
+
+  def test_num_boxes(self):
+    data = tf.constant([[0, 0, 1, 1], [1, 1, 2, 3], [3, 4, 5, 5]], tf.float32)
+    expected_num_boxes = 3
+
+    boxes = box_list.BoxList(data)
+    with self.test_session() as sess:
+      num_boxes_output = sess.run(boxes.num_boxes())
+      self.assertEquals(num_boxes_output, expected_num_boxes)
+
+  def test_get_correct_center_coordinates_and_sizes(self):
+    boxes = [[10.0, 10.0, 20.0, 15.0], [0.2, 0.1, 0.5, 0.4]]
+    boxes = box_list.BoxList(tf.constant(boxes))
+    centers_sizes = boxes.get_center_coordinates_and_sizes()
+    expected_centers_sizes = [[15, 0.35], [12.5, 0.25], [10, 0.3], [5, 0.3]]
+    with self.test_session() as sess:
+      centers_sizes_out = sess.run(centers_sizes)
+      self.assertAllClose(centers_sizes_out, expected_centers_sizes)
+
+  def test_create_box_list_with_dynamic_shape(self):
+    data = tf.constant([[0, 0, 1, 1], [1, 1, 2, 3], [3, 4, 5, 5]], tf.float32)
+    indices = tf.reshape(tf.where(tf.greater([1, 0, 1], 0)), [-1])
+    data = tf.gather(data, indices)
+    assert data.get_shape().as_list() == [None, 4]
+    expected_num_boxes = 2
+
+    boxes = box_list.BoxList(data)
+    with self.test_session() as sess:
+      num_boxes_output = sess.run(boxes.num_boxes())
+      self.assertEquals(num_boxes_output, expected_num_boxes)
+
+  def test_transpose_coordinates(self):
+    boxes = [[10.0, 10.0, 20.0, 15.0], [0.2, 0.1, 0.5, 0.4]]
+    boxes = box_list.BoxList(tf.constant(boxes))
+    boxes.transpose_coordinates()
+    expected_corners = [[10.0, 10.0, 15.0, 20.0], [0.1, 0.2, 0.4, 0.5]]
+    with self.test_session() as sess:
+      corners_out = sess.run(boxes.get())
+      self.assertAllClose(corners_out, expected_corners)
+
+  def test_box_list_invalid_inputs(self):
+    data0 = tf.constant([[[0, 0, 1, 1], [3, 4, 5, 5]]], tf.float32)
+    data1 = tf.constant([[0, 0, 1], [1, 1, 2], [3, 4, 5]], tf.float32)
+    data2 = tf.constant([[0, 0, 1], [1, 1, 2], [3, 4, 5]], tf.int32)
+
+    with self.assertRaises(ValueError):
+      _ = box_list.BoxList(data0)
+    with self.assertRaises(ValueError):
+      _ = box_list.BoxList(data1)
+    with self.assertRaises(ValueError):
+      _ = box_list.BoxList(data2)
+
+  def test_num_boxes_static(self):
+    box_corners = [[10.0, 10.0, 20.0, 15.0], [0.2, 0.1, 0.5, 0.4]]
+    boxes = box_list.BoxList(tf.constant(box_corners))
+    self.assertEquals(boxes.num_boxes_static(), 2)
+    self.assertEquals(type(boxes.num_boxes_static()), int)
+
+  def test_num_boxes_static_for_uninferrable_shape(self):
+    placeholder = tf.placeholder(tf.float32, shape=[None, 4])
+    boxes = box_list.BoxList(placeholder)
+    self.assertEquals(boxes.num_boxes_static(), None)
+
+  def test_as_tensor_dict(self):
+    boxlist = box_list.BoxList(
+        tf.constant([[0.1, 0.1, 0.4, 0.4], [0.1, 0.1, 0.5, 0.5]], tf.float32))
+    boxlist.add_field('classes', tf.constant([0, 1]))
+    boxlist.add_field('scores', tf.constant([0.75, 0.2]))
+    tensor_dict = boxlist.as_tensor_dict()
+
+    expected_boxes = [[0.1, 0.1, 0.4, 0.4], [0.1, 0.1, 0.5, 0.5]]
+    expected_classes = [0, 1]
+    expected_scores = [0.75, 0.2]
+
+    with self.test_session() as sess:
+      tensor_dict_out = sess.run(tensor_dict)
+      self.assertAllEqual(3, len(tensor_dict_out))
+      self.assertAllClose(expected_boxes, tensor_dict_out['boxes'])
+      self.assertAllEqual(expected_classes, tensor_dict_out['classes'])
+      self.assertAllClose(expected_scores, tensor_dict_out['scores'])
+
+  def test_as_tensor_dict_with_features(self):
+    boxlist = box_list.BoxList(
+        tf.constant([[0.1, 0.1, 0.4, 0.4], [0.1, 0.1, 0.5, 0.5]], tf.float32))
+    boxlist.add_field('classes', tf.constant([0, 1]))
+    boxlist.add_field('scores', tf.constant([0.75, 0.2]))
+    tensor_dict = boxlist.as_tensor_dict(['boxes', 'classes', 'scores'])
+
+    expected_boxes = [[0.1, 0.1, 0.4, 0.4], [0.1, 0.1, 0.5, 0.5]]
+    expected_classes = [0, 1]
+    expected_scores = [0.75, 0.2]
+
+    with self.test_session() as sess:
+      tensor_dict_out = sess.run(tensor_dict)
+      self.assertAllEqual(3, len(tensor_dict_out))
+      self.assertAllClose(expected_boxes, tensor_dict_out['boxes'])
+      self.assertAllEqual(expected_classes, tensor_dict_out['classes'])
+      self.assertAllClose(expected_scores, tensor_dict_out['scores'])
+
+  def test_as_tensor_dict_missing_field(self):
+    boxlist = box_list.BoxList(
+        tf.constant([[0.1, 0.1, 0.4, 0.4], [0.1, 0.1, 0.5, 0.5]], tf.float32))
+    boxlist.add_field('classes', tf.constant([0, 1]))
+    boxlist.add_field('scores', tf.constant([0.75, 0.2]))
+    with self.assertRaises(ValueError):
+      boxlist.as_tensor_dict(['foo', 'bar'])
+
+
+if __name__ == '__main__':
+  tf.test.main()

object_detection/core/box_predictor.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Box predictor for object detectors.
+
+Box predictors are classes that take a high level
+image feature map as input and produce two predictions,
+(1) a tensor encoding box locations, and
+(2) a tensor encoding classes for each box.
+
+These components are passed directly to loss functions
+in our detection models.
+
+These modules are separated from the main model since the same
+few box predictor architectures are shared across many models.
+"""
+from abc import abstractmethod
+import tensorflow as tf
+from object_detection.utils import ops
+from object_detection.utils import static_shape
+
+slim = tf.contrib.slim
+
+BOX_ENCODINGS = 'box_encodings'
+CLASS_PREDICTIONS_WITH_BACKGROUND = 'class_predictions_with_background'
+MASK_PREDICTIONS = 'mask_predictions'
+
+
+class BoxPredictor(object):
+  """BoxPredictor."""
+
+  def __init__(self, is_training, num_classes):
+    """Constructor.
+
+    Args:
+      is_training: Indicates whether the BoxPredictor is in training mode.
+      num_classes: number of classes.  Note that num_classes *does not*
+        include the background category, so if groundtruth labels take values
+        in {0, 1, .., K-1}, num_classes=K (and not K+1, even though the
+        assigned classification targets can range from {0,... K}).
+    """
+    self._is_training = is_training
+    self._num_classes = num_classes
+
+  @property
+  def num_classes(self):
+    return self._num_classes
+
+  def predict(self, image_features, num_predictions_per_location, scope,
+              **params):
+    """Computes encoded object locations and corresponding confidences.
+
+    Takes a high level image feature map as input and produce two predictions,
+    (1) a tensor encoding box locations, and
+    (2) a tensor encoding class scores for each corresponding box.
+    In this interface, we only assume that two tensors are returned as output
+    and do not assume anything about their shapes.
+
+    Args:
+      image_features: A float tensor of shape [batch_size, height, width,
+        channels] containing features for a batch of images.
+      num_predictions_per_location: an integer representing the number of box
+        predictions to be made per spatial location in the feature map.
+      scope: Variable and Op scope name.
+      **params: Additional keyword arguments for specific implementations of
+              BoxPredictor.
+
+    Returns:
+      A dictionary containing at least the following tensors.
+        box_encodings: A float tensor of shape
+          [batch_size, num_anchors, q, code_size] representing the location of
+          the objects, where q is 1 or the number of classes.
+        class_predictions_with_background: A float tensor of shape
+          [batch_size, num_anchors, num_classes + 1] representing the class
+          predictions for the proposals.
+    """
+    with tf.variable_scope(scope):
+      return self._predict(image_features, num_predictions_per_location,
+                           **params)
+
+  # TODO: num_predictions_per_location could be moved to constructor.
+  # This is currently only used by ConvolutionalBoxPredictor.
+  @abstractmethod
+  def _predict(self, image_features, num_predictions_per_location, **params):
+    """Implementations must override this method.
+
+    Args:
+      image_features: A float tensor of shape [batch_size, height, width,
+        channels] containing features for a batch of images.
+      num_predictions_per_location: an integer representing the number of box
+        predictions to be made per spatial location in the feature map.
+      **params: Additional keyword arguments for specific implementations of
+              BoxPredictor.
+
+    Returns:
+      A dictionary containing at least the following tensors.
+        box_encodings: A float tensor of shape
+          [batch_size, num_anchors, q, code_size] representing the location of
+          the objects, where q is 1 or the number of classes.
+        class_predictions_with_background: A float tensor of shape
+          [batch_size, num_anchors, num_classes + 1] representing the class
+          predictions for the proposals.
+    """
+    pass
+
+
+class RfcnBoxPredictor(BoxPredictor):
+  """RFCN Box Predictor.
+
+  Applies a position sensitve ROI pooling on position sensitive feature maps to
+  predict classes and refined locations. See https://arxiv.org/abs/1605.06409
+  for details.
+
+  This is used for the second stage of the RFCN meta architecture. Notice that
+  locations are *not* shared across classes, thus for each anchor, a separate
+  prediction is made for each class.
+  """
+
+  def __init__(self,
+               is_training,
+               num_classes,
+               conv_hyperparams,
+               num_spatial_bins,
+               depth,
+               crop_size,
+               box_code_size):
+    """Constructor.
+
+    Args:
+      is_training: Indicates whether the BoxPredictor is in training mode.
+      num_classes: number of classes.  Note that num_classes *does not*
+        include the background category, so if groundtruth labels take values
+        in {0, 1, .., K-1}, num_classes=K (and not K+1, even though the
+        assigned classification targets can range from {0,... K}).
+      conv_hyperparams: Slim arg_scope with hyperparameters for conolutional
+        layers.
+      num_spatial_bins: A list of two integers `[spatial_bins_y,
+        spatial_bins_x]`.
+      depth: Target depth to reduce the input feature maps to.
+      crop_size: A list of two integers `[crop_height, crop_width]`.
+      box_code_size: Size of encoding for each box.
+    """
+    super(RfcnBoxPredictor, self).__init__(is_training, num_classes)
+    self._conv_hyperparams = conv_hyperparams
+    self._num_spatial_bins = num_spatial_bins
+    self._depth = depth
+    self._crop_size = crop_size
+    self._box_code_size = box_code_size
+
+  @property
+  def num_classes(self):
+    return self._num_classes
+
+  def _predict(self, image_features, num_predictions_per_location,
+               proposal_boxes):
+    """Computes encoded object locations and corresponding confidences.
+
+    Args:
+      image_features: A float tensor of shape [batch_size, height, width,
+        channels] containing features for a batch of images.
+      num_predictions_per_location: an integer representing the number of box
+        predictions to be made per spatial location in the feature map.
+        Currently, this must be set to 1, or an error will be raised.
+      proposal_boxes: A float tensor of shape [batch_size, num_proposals,
+        box_code_size].
+
+    Returns:
+      box_encodings: A float tensor of shape
+        [batch_size, 1, num_classes, code_size] representing the
+        location of the objects.
+      class_predictions_with_background: A float tensor of shape
+        [batch_size, 1, num_classes + 1] representing the class
+        predictions for the proposals.
+    Raises:
+      ValueError: if num_predictions_per_location is not 1.
+    """
+    if num_predictions_per_location != 1:
+      raise ValueError('Currently RfcnBoxPredictor only supports '
+                       'predicting a single box per class per location.')
+
+    batch_size = tf.shape(proposal_boxes)[0]
+    num_boxes = tf.shape(proposal_boxes)[1]
+    def get_box_indices(proposals):
+      proposals_shape = proposals.get_shape().as_list()
+      if any(dim is None for dim in proposals_shape):
+        proposals_shape = tf.shape(proposals)
+      ones_mat = tf.ones(proposals_shape[:2], dtype=tf.int32)
+      multiplier = tf.expand_dims(
+          tf.range(start=0, limit=proposals_shape[0]), 1)
+      return tf.reshape(ones_mat * multiplier, [-1])
+
+    net = image_features
+    with slim.arg_scope(self._conv_hyperparams):
+      net = slim.conv2d(net, self._depth, [1, 1], scope='reduce_depth')
+      # Location predictions.
+      location_feature_map_depth = (self._num_spatial_bins[0] *
+                                    self._num_spatial_bins[1] *
+                                    self.num_classes *
+                                    self._box_code_size)
+      location_feature_map = slim.conv2d(net, location_feature_map_depth,
+                                         [1, 1], activation_fn=None,
+                                         scope='refined_locations')
+      box_encodings = ops.position_sensitive_crop_regions(
+          location_feature_map,
+          boxes=tf.reshape(proposal_boxes, [-1, self._box_code_size]),
+          box_ind=get_box_indices(proposal_boxes),
+          crop_size=self._crop_size,
+          num_spatial_bins=self._num_spatial_bins,
+          global_pool=True)
+      box_encodings = tf.squeeze(box_encodings, squeeze_dims=[1, 2])
+      box_encodings = tf.reshape(box_encodings,
+                                 [batch_size * num_boxes, 1, self.num_classes,
+                                  self._box_code_size])
+
+      # Class predictions.
+      total_classes = self.num_classes + 1  # Account for background class.
+      class_feature_map_depth = (self._num_spatial_bins[0] *
+                                 self._num_spatial_bins[1] *
+                                 total_classes)
+      class_feature_map = slim.conv2d(net, class_feature_map_depth, [1, 1],
+                                      activation_fn=None,
+                                      scope='class_predictions')
+      class_predictions_with_background = ops.position_sensitive_crop_regions(
+          class_feature_map,
+          boxes=tf.reshape(proposal_boxes, [-1, self._box_code_size]),
+          box_ind=get_box_indices(proposal_boxes),
+          crop_size=self._crop_size,
+          num_spatial_bins=self._num_spatial_bins,
+          global_pool=True)
+      class_predictions_with_background = tf.squeeze(
+          class_predictions_with_background, squeeze_dims=[1, 2])
+      class_predictions_with_background = tf.reshape(
+          class_predictions_with_background,
+          [batch_size * num_boxes, 1, total_classes])
+
+    return {BOX_ENCODINGS: box_encodings,
+            CLASS_PREDICTIONS_WITH_BACKGROUND:
+            class_predictions_with_background}
+
+
+class MaskRCNNBoxPredictor(BoxPredictor):
+  """Mask R-CNN Box Predictor.
+
+  See Mask R-CNN: He, K., Gkioxari, G., Dollar, P., & Girshick, R. (2017).
+  Mask R-CNN. arXiv preprint arXiv:1703.06870.
+
+  This is used for the second stage of the Mask R-CNN detector where proposals
+  cropped from an image are arranged along the batch dimension of the input
+  image_features tensor. Notice that locations are *not* shared across classes,
+  thus for each anchor, a separate prediction is made for each class.
+
+  In addition to predicting boxes and classes, optionally this class allows
+  predicting masks and/or keypoints inside detection boxes.
+
+  Currently this box predictor makes per-class predictions; that is, each
+  anchor makes a separate box prediction for each class.
+  """
+
+  def __init__(self,
+               is_training,
+               num_classes,
+               fc_hyperparams,
+               use_dropout,
+               dropout_keep_prob,
+               box_code_size,
+               conv_hyperparams=None,
+               predict_instance_masks=False,
+               mask_prediction_conv_depth=256,
+               predict_keypoints=False):
+    """Constructor.
+
+    Args:
+      is_training: Indicates whether the BoxPredictor is in training mode.
+      num_classes: number of classes.  Note that num_classes *does not*
+        include the background category, so if groundtruth labels take values
+        in {0, 1, .., K-1}, num_classes=K (and not K+1, even though the
+        assigned classification targets can range from {0,... K}).
+      fc_hyperparams: Slim arg_scope with hyperparameters for fully
+        connected ops.
+      use_dropout: Option to use dropout or not.  Note that a single dropout
+        op is applied here prior to both box and class predictions, which stands
+        in contrast to the ConvolutionalBoxPredictor below.
+      dropout_keep_prob: Keep probability for dropout.
+        This is only used if use_dropout is True.
+      box_code_size: Size of encoding for each box.
+      conv_hyperparams: Slim arg_scope with hyperparameters for convolution
+        ops.
+      predict_instance_masks: Whether to predict object masks inside detection
+        boxes.
+      mask_prediction_conv_depth: The depth for the first conv2d_transpose op
+        applied to the image_features in the mask prediciton branch.
+      predict_keypoints: Whether to predict keypoints insde detection boxes.
+
+
+    Raises:
+      ValueError: If predict_instance_masks or predict_keypoints is true.
+    """
+    super(MaskRCNNBoxPredictor, self).__init__(is_training, num_classes)
+    self._fc_hyperparams = fc_hyperparams
+    self._use_dropout = use_dropout
+    self._box_code_size = box_code_size
+    self._dropout_keep_prob = dropout_keep_prob
+    self._conv_hyperparams = conv_hyperparams
+    self._predict_instance_masks = predict_instance_masks
+    self._mask_prediction_conv_depth = mask_prediction_conv_depth
+    self._predict_keypoints = predict_keypoints
+    if self._predict_keypoints:
+      raise ValueError('Keypoint prediction is unimplemented.')
+    if ((self._predict_instance_masks or self._predict_keypoints) and
+        self._conv_hyperparams is None):
+      raise ValueError('`conv_hyperparams` must be provided when predicting '
+                       'masks.')
+
+  @property
+  def num_classes(self):
+    return self._num_classes
+
+  def _predict(self, image_features, num_predictions_per_location):
+    """Computes encoded object locations and corresponding confidences.
+
+    Flattens image_features and applies fully connected ops (with no
+    non-linearity) to predict box encodings and class predictions.  In this
+    setting, anchors are not spatially arranged in any way and are assumed to
+    have been folded into the batch dimension.  Thus we output 1 for the
+    anchors dimension.
+
+    Args:
+      image_features: A float tensor of shape [batch_size, height, width,
+        channels] containing features for a batch of images.
+      num_predictions_per_location: an integer representing the number of box
+        predictions to be made per spatial location in the feature map.
+        Currently, this must be set to 1, or an error will be raised.
+
+    Returns:
+      A dictionary containing the following tensors.
+        box_encodings: A float tensor of shape
+          [batch_size, 1, num_classes, code_size] representing the
+          location of the objects.
+        class_predictions_with_background: A float tensor of shape
+          [batch_size, 1, num_classes + 1] representing the class
+          predictions for the proposals.
+      If predict_masks is True the dictionary also contains:
+        instance_masks: A float tensor of shape
+          [batch_size, 1, num_classes, image_height, image_width]
+      If predict_keypoints is True the dictionary also contains:
+        keypoints: [batch_size, 1, num_keypoints, 2]
+
+    Raises:
+      ValueError: if num_predictions_per_location is not 1.
+    """
+    if num_predictions_per_location != 1:
+      raise ValueError('Currently FullyConnectedBoxPredictor only supports '
+                       'predicting a single box per class per location.')
+    spatial_averaged_image_features = tf.reduce_mean(image_features, [1, 2],
+                                                     keep_dims=True,
+                                                     name='AvgPool')
+    flattened_image_features = slim.flatten(spatial_averaged_image_features)
+    if self._use_dropout:
+      flattened_image_features = slim.dropout(flattened_image_features,
+                                              keep_prob=self._dropout_keep_prob,
+                                              is_training=self._is_training)
+    with slim.arg_scope(self._fc_hyperparams):
+      box_encodings = slim.fully_connected(
+          flattened_image_features,
+          self._num_classes * self._box_code_size,
+          activation_fn=None,
+          scope='BoxEncodingPredictor')
+      class_predictions_with_background = slim.fully_connected(
+          flattened_image_features,
+          self._num_classes + 1,
+          activation_fn=None,
+          scope='ClassPredictor')
+    box_encodings = tf.reshape(
+        box_encodings, [-1, 1, self._num_classes, self._box_code_size])
+    class_predictions_with_background = tf.reshape(
+        class_predictions_with_background, [-1, 1, self._num_classes + 1])
+
+    predictions_dict = {
+        BOX_ENCODINGS: box_encodings,
+        CLASS_PREDICTIONS_WITH_BACKGROUND: class_predictions_with_background
+    }
+
+    if self._predict_instance_masks:
+      with slim.arg_scope(self._conv_hyperparams):
+        upsampled_features = slim.conv2d_transpose(
+            image_features,
+            num_outputs=self._mask_prediction_conv_depth,
+            kernel_size=[2, 2],
+            stride=2)
+        mask_predictions = slim.conv2d(upsampled_features,
+                                       num_outputs=self.num_classes,
+                                       activation_fn=None,
+                                       kernel_size=[1, 1])
+        instance_masks = tf.expand_dims(tf.transpose(mask_predictions,
+                                                     perm=[0, 3, 1, 2]),
+                                        axis=1,
+                                        name='MaskPredictor')
+      predictions_dict[MASK_PREDICTIONS] = instance_masks
+    return predictions_dict
+
+
+class ConvolutionalBoxPredictor(BoxPredictor):
+  """Convolutional Box Predictor.
+
+  Optionally add an intermediate 1x1 convolutional layer after features and
+  predict in parallel branches box_encodings and
+  class_predictions_with_background.
+
+  Currently this box predictor assumes that predictions are "shared" across
+  classes --- that is each anchor makes box predictions which do not depend
+  on class.
+  """
+
+  def __init__(self,
+               is_training,
+               num_classes,
+               conv_hyperparams,
+               min_depth,
+               max_depth,
+               num_layers_before_predictor,
+               use_dropout,
+               dropout_keep_prob,
+               kernel_size,
+               box_code_size,
+               apply_sigmoid_to_scores=False):
+    """Constructor.
+
+    Args:
+      is_training: Indicates whether the BoxPredictor is in training mode.
+      num_classes: number of classes.  Note that num_classes *does not*
+        include the background category, so if groundtruth labels take values
+        in {0, 1, .., K-1}, num_classes=K (and not K+1, even though the
+        assigned classification targets can range from {0,... K}).
+      conv_hyperparams: Slim arg_scope with hyperparameters for convolution ops.
+      min_depth: Minumum feature depth prior to predicting box encodings
+        and class predictions.
+      max_depth: Maximum feature depth prior to predicting box encodings
+        and class predictions. If max_depth is set to 0, no additional
+        feature map will be inserted before location and class predictions.
+      num_layers_before_predictor: Number of the additional conv layers before
+        the predictor.
+      use_dropout: Option to use dropout for class prediction or not.
+      dropout_keep_prob: Keep probability for dropout.
+        This is only used if use_dropout is True.
+      kernel_size: Size of final convolution kernel.  If the
+        spatial resolution of the feature map is smaller than the kernel size,
+        then the kernel size is automatically set to be
+        min(feature_width, feature_height).
+      box_code_size: Size of encoding for each box.
+      apply_sigmoid_to_scores: if True, apply the sigmoid on the output
+        class_predictions.
+
+    Raises:
+      ValueError: if min_depth > max_depth.
+    """
+    super(ConvolutionalBoxPredictor, self).__init__(is_training, num_classes)
+    if min_depth > max_depth:
+      raise ValueError('min_depth should be less than or equal to max_depth')
+    self._conv_hyperparams = conv_hyperparams
+    self._min_depth = min_depth
+    self._max_depth = max_depth
+    self._num_layers_before_predictor = num_layers_before_predictor
+    self._use_dropout = use_dropout
+    self._kernel_size = kernel_size
+    self._box_code_size = box_code_size
+    self._dropout_keep_prob = dropout_keep_prob
+    self._apply_sigmoid_to_scores = apply_sigmoid_to_scores
+
+  def _predict(self, image_features, num_predictions_per_location):
+    """Computes encoded object locations and corresponding confidences.
+
+    Args:
+      image_features: A float tensor of shape [batch_size, height, width,
+        channels] containing features for a batch of images.
+      num_predictions_per_location: an integer representing the number of box
+        predictions to be made per spatial location in the feature map.
+
+    Returns:
+      A dictionary containing the following tensors.
+        box_encodings: A float tensor of shape [batch_size, num_anchors, 1,
+          code_size] representing the location of the objects, where
+          num_anchors = feat_height * feat_width * num_predictions_per_location
+        class_predictions_with_background: A float tensor of shape
+          [batch_size, num_anchors, num_classes + 1] representing the class
+          predictions for the proposals.
+    """
+    features_depth = static_shape.get_depth(image_features.get_shape())
+    depth = max(min(features_depth, self._max_depth), self._min_depth)
+
+    # Add a slot for the background class.
+    num_class_slots = self.num_classes + 1
+    net = image_features
+    with slim.arg_scope(self._conv_hyperparams), \
+         slim.arg_scope([slim.dropout], is_training=self._is_training):
+      # Add additional conv layers before the predictor.
+      if depth > 0 and self._num_layers_before_predictor > 0:
+        for i in range(self._num_layers_before_predictor):
+          net = slim.conv2d(
+              net, depth, [1, 1], scope='Conv2d_%d_1x1_%d' % (i, depth))
+      with slim.arg_scope([slim.conv2d], activation_fn=None,
+                          normalizer_fn=None, normalizer_params=None):
+        box_encodings = slim.conv2d(
+            net, num_predictions_per_location * self._box_code_size,
+            [self._kernel_size, self._kernel_size],
+            scope='BoxEncodingPredictor')
+        if self._use_dropout:
+          net = slim.dropout(net, keep_prob=self._dropout_keep_prob)
+        class_predictions_with_background = slim.conv2d(
+            net, num_predictions_per_location * num_class_slots,
+            [self._kernel_size, self._kernel_size], scope='ClassPredictor')
+        if self._apply_sigmoid_to_scores:
+          class_predictions_with_background = tf.sigmoid(
+              class_predictions_with_background)
+
+    batch_size = static_shape.get_batch_size(image_features.get_shape())
+    if batch_size is None:
+      features_height = static_shape.get_height(image_features.get_shape())
+      features_width = static_shape.get_width(image_features.get_shape())
+      flattened_predictions_size = (features_height * features_width *
+                                    num_predictions_per_location)
+      box_encodings = tf.reshape(
+          box_encodings,
+          [-1, flattened_predictions_size, 1, self._box_code_size])
+      class_predictions_with_background = tf.reshape(
+          class_predictions_with_background,
+          [-1, flattened_predictions_size, num_class_slots])
+    else:
+      box_encodings = tf.reshape(
+          box_encodings, [batch_size, -1, 1, self._box_code_size])
+      class_predictions_with_background = tf.reshape(
+          class_predictions_with_background, [batch_size, -1, num_class_slots])
+    return {BOX_ENCODINGS: box_encodings,
+            CLASS_PREDICTIONS_WITH_BACKGROUND:
+            class_predictions_with_background}

object_detection/core/box_predictor_test.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Tests for object_detection.core.box_predictor."""
+
+import numpy as np
+import tensorflow as tf
+
+from google.protobuf import text_format
+from object_detection.builders import hyperparams_builder
+from object_detection.core import box_predictor
+from object_detection.protos import hyperparams_pb2
+
+
+class MaskRCNNBoxPredictorTest(tf.test.TestCase):
+
+  def _build_arg_scope_with_hyperparams(self,
+                                        op_type=hyperparams_pb2.Hyperparams.FC):
+    hyperparams = hyperparams_pb2.Hyperparams()
+    hyperparams_text_proto = """
+      activation: NONE
+      regularizer {
+        l2_regularizer {
+        }
+      }
+      initializer {
+        truncated_normal_initializer {
+        }
+      }
+    """
+    text_format.Merge(hyperparams_text_proto, hyperparams)
+    hyperparams.op = op_type
+    return hyperparams_builder.build(hyperparams, is_training=True)
+
+  def test_get_boxes_with_five_classes(self):
+    image_features = tf.random_uniform([2, 7, 7, 3], dtype=tf.float32)
+    mask_box_predictor = box_predictor.MaskRCNNBoxPredictor(
+        is_training=False,
+        num_classes=5,
+        fc_hyperparams=self._build_arg_scope_with_hyperparams(),
+        use_dropout=False,
+        dropout_keep_prob=0.5,
+        box_code_size=4,
+    )
+    box_predictions = mask_box_predictor.predict(
+        image_features, num_predictions_per_location=1, scope='BoxPredictor')
+    box_encodings = box_predictions[box_predictor.BOX_ENCODINGS]
+    class_predictions_with_background = box_predictions[
+        box_predictor.CLASS_PREDICTIONS_WITH_BACKGROUND]
+    init_op = tf.global_variables_initializer()
+    with self.test_session() as sess:
+      sess.run(init_op)
+      (box_encodings_shape,
+       class_predictions_with_background_shape) = sess.run(
+           [tf.shape(box_encodings),
+            tf.shape(class_predictions_with_background)])
+      self.assertAllEqual(box_encodings_shape, [2, 1, 5, 4])
+      self.assertAllEqual(class_predictions_with_background_shape, [2, 1, 6])
+
+  def test_value_error_on_predict_instance_masks_with_no_conv_hyperparms(self):
+    with self.assertRaises(ValueError):
+      box_predictor.MaskRCNNBoxPredictor(
+          is_training=False,
+          num_classes=5,
+          fc_hyperparams=self._build_arg_scope_with_hyperparams(),
+          use_dropout=False,
+          dropout_keep_prob=0.5,
+          box_code_size=4,
+          predict_instance_masks=True)
+
+  def test_get_instance_masks(self):
+    image_features = tf.random_uniform([2, 7, 7, 3], dtype=tf.float32)
+    mask_box_predictor = box_predictor.MaskRCNNBoxPredictor(
+        is_training=False,
+        num_classes=5,
+        fc_hyperparams=self._build_arg_scope_with_hyperparams(),
+        use_dropout=False,
+        dropout_keep_prob=0.5,
+        box_code_size=4,
+        conv_hyperparams=self._build_arg_scope_with_hyperparams(
+            op_type=hyperparams_pb2.Hyperparams.CONV),
+        predict_instance_masks=True)
+    box_predictions = mask_box_predictor.predict(
+        image_features, num_predictions_per_location=1, scope='BoxPredictor')
+    mask_predictions = box_predictions[box_predictor.MASK_PREDICTIONS]
+    self.assertListEqual([2, 1, 5, 14, 14],
+                         mask_predictions.get_shape().as_list())
+
+  def test_do_not_return_instance_masks_and_keypoints_without_request(self):
+    image_features = tf.random_uniform([2, 7, 7, 3], dtype=tf.float32)
+    mask_box_predictor = box_predictor.MaskRCNNBoxPredictor(
+        is_training=False,
+        num_classes=5,
+        fc_hyperparams=self._build_arg_scope_with_hyperparams(),
+        use_dropout=False,
+        dropout_keep_prob=0.5,
+        box_code_size=4)
+    box_predictions = mask_box_predictor.predict(
+        image_features, num_predictions_per_location=1, scope='BoxPredictor')
+    self.assertEqual(len(box_predictions), 2)
+    self.assertTrue(box_predictor.BOX_ENCODINGS in box_predictions)
+    self.assertTrue(box_predictor.CLASS_PREDICTIONS_WITH_BACKGROUND
+                    in box_predictions)
+
+  def test_value_error_on_predict_keypoints(self):
+    with self.assertRaises(ValueError):
+      box_predictor.MaskRCNNBoxPredictor(
+          is_training=False,
+          num_classes=5,
+          fc_hyperparams=self._build_arg_scope_with_hyperparams(),
+          use_dropout=False,
+          dropout_keep_prob=0.5,
+          box_code_size=4,
+          predict_keypoints=True)
+
+
+class RfcnBoxPredictorTest(tf.test.TestCase):
+
+  def _build_arg_scope_with_conv_hyperparams(self):
+    conv_hyperparams = hyperparams_pb2.Hyperparams()
+    conv_hyperparams_text_proto = """
+      regularizer {
+        l2_regularizer {
+        }
+      }
+      initializer {
+        truncated_normal_initializer {
+        }
+      }
+    """
+    text_format.Merge(conv_hyperparams_text_proto, conv_hyperparams)
+    return hyperparams_builder.build(conv_hyperparams, is_training=True)
+
+  def test_get_correct_box_encoding_and_class_prediction_shapes(self):
+    image_features = tf.random_uniform([4, 8, 8, 64], dtype=tf.float32)
+    proposal_boxes = tf.random_normal([4, 2, 4], dtype=tf.float32)
+    rfcn_box_predictor = box_predictor.RfcnBoxPredictor(
+        is_training=False,
+        num_classes=2,
+        conv_hyperparams=self._build_arg_scope_with_conv_hyperparams(),
+        num_spatial_bins=[3, 3],
+        depth=4,
+        crop_size=[12, 12],
+        box_code_size=4
+    )
+    box_predictions = rfcn_box_predictor.predict(
+        image_features, num_predictions_per_location=1, scope='BoxPredictor',
+        proposal_boxes=proposal_boxes)
+    box_encodings = box_predictions[box_predictor.BOX_ENCODINGS]
+    class_predictions_with_background = box_predictions[
+        box_predictor.CLASS_PREDICTIONS_WITH_BACKGROUND]
+
+    init_op = tf.global_variables_initializer()
+    with self.test_session() as sess:
+      sess.run(init_op)
+      (box_encodings_shape,
+       class_predictions_shape) = sess.run(
+           [tf.shape(box_encodings),
+            tf.shape(class_predictions_with_background)])
+      self.assertAllEqual(box_encodings_shape, [8, 1, 2, 4])
+      self.assertAllEqual(class_predictions_shape, [8, 1, 3])
+
+
+class ConvolutionalBoxPredictorTest(tf.test.TestCase):
+
+  def _build_arg_scope_with_conv_hyperparams(self):
+    conv_hyperparams = hyperparams_pb2.Hyperparams()
+    conv_hyperparams_text_proto = """
+      activation: RELU_6
+      regularizer {
+        l2_regularizer {
+        }
+      }
+      initializer {
+        truncated_normal_initializer {
+        }
+      }
+    """
+    text_format.Merge(conv_hyperparams_text_proto, conv_hyperparams)
+    return hyperparams_builder.build(conv_hyperparams, is_training=True)
+
+  def test_get_boxes_for_five_aspect_ratios_per_location(self):
+    image_features = tf.random_uniform([4, 8, 8, 64], dtype=tf.float32)
+    conv_box_predictor = box_predictor.ConvolutionalBoxPredictor(
+        is_training=False,
+        num_classes=0,
+        conv_hyperparams=self._build_arg_scope_with_conv_hyperparams(),
+        min_depth=0,
+        max_depth=32,
+        num_layers_before_predictor=1,
+        use_dropout=True,
+        dropout_keep_prob=0.8,
+        kernel_size=1,
+        box_code_size=4
+    )
+    box_predictions = conv_box_predictor.predict(
+        image_features, num_predictions_per_location=5, scope='BoxPredictor')
+    box_encodings = box_predictions[box_predictor.BOX_ENCODINGS]
+    objectness_predictions = box_predictions[
+        box_predictor.CLASS_PREDICTIONS_WITH_BACKGROUND]
+
+    init_op = tf.global_variables_initializer()
+    with self.test_session() as sess:
+      sess.run(init_op)
+      (box_encodings_shape,
+       objectness_predictions_shape) = sess.run(
+           [tf.shape(box_encodings), tf.shape(objectness_predictions)])
+      self.assertAllEqual(box_encodings_shape, [4, 320, 1, 4])
+      self.assertAllEqual(objectness_predictions_shape, [4, 320, 1])
+
+  def test_get_boxes_for_one_aspect_ratio_per_location(self):
+    image_features = tf.random_uniform([4, 8, 8, 64], dtype=tf.float32)
+    conv_box_predictor = box_predictor.ConvolutionalBoxPredictor(
+        is_training=False,
+        num_classes=0,
+        conv_hyperparams=self._build_arg_scope_with_conv_hyperparams(),
+        min_depth=0,
+        max_depth=32,
+        num_layers_before_predictor=1,
+        use_dropout=True,
+        dropout_keep_prob=0.8,
+        kernel_size=1,
+        box_code_size=4
+    )
+    box_predictions = conv_box_predictor.predict(
+        image_features, num_predictions_per_location=1, scope='BoxPredictor')
+    box_encodings = box_predictions[box_predictor.BOX_ENCODINGS]
+    objectness_predictions = box_predictions[
+        box_predictor.CLASS_PREDICTIONS_WITH_BACKGROUND]
+
+    init_op = tf.global_variables_initializer()
+    with self.test_session() as sess:
+      sess.run(init_op)
+      (box_encodings_shape,
+       objectness_predictions_shape) = sess.run(
+           [tf.shape(box_encodings), tf.shape(objectness_predictions)])
+      self.assertAllEqual(box_encodings_shape, [4, 64, 1, 4])
+      self.assertAllEqual(objectness_predictions_shape, [4, 64, 1])
+
+  def test_get_multi_class_predictions_for_five_aspect_ratios_per_location(
+      self):
+    num_classes_without_background = 6
+    image_features = tf.random_uniform([4, 8, 8, 64], dtype=tf.float32)
+    conv_box_predictor = box_predictor.ConvolutionalBoxPredictor(
+        is_training=False,
+        num_classes=num_classes_without_background,
+        conv_hyperparams=self._build_arg_scope_with_conv_hyperparams(),
+        min_depth=0,
+        max_depth=32,
+        num_layers_before_predictor=1,
+        use_dropout=True,
+        dropout_keep_prob=0.8,
+        kernel_size=1,
+        box_code_size=4
+    )
+    box_predictions = conv_box_predictor.predict(
+        image_features,
+        num_predictions_per_location=5,
+        scope='BoxPredictor')
+    box_encodings = box_predictions[box_predictor.BOX_ENCODINGS]
+    class_predictions_with_background = box_predictions[
+        box_predictor.CLASS_PREDICTIONS_WITH_BACKGROUND]
+
+    init_op = tf.global_variables_initializer()
+    with self.test_session() as sess:
+      sess.run(init_op)
+      (box_encodings_shape, class_predictions_with_background_shape
+      ) = sess.run([
+          tf.shape(box_encodings), tf.shape(class_predictions_with_background)])
+      self.assertAllEqual(box_encodings_shape, [4, 320, 1, 4])
+      self.assertAllEqual(class_predictions_with_background_shape,
+                          [4, 320, num_classes_without_background+1])
+
+  def test_get_boxes_for_five_aspect_ratios_per_location_fully_convolutional(
+      self):
+    image_features = tf.placeholder(dtype=tf.float32, shape=[4, None, None, 64])
+    conv_box_predictor = box_predictor.ConvolutionalBoxPredictor(
+        is_training=False,
+        num_classes=0,
+        conv_hyperparams=self._build_arg_scope_with_conv_hyperparams(),
+        min_depth=0,
+        max_depth=32,
+        num_layers_before_predictor=1,
+        use_dropout=True,
+        dropout_keep_prob=0.8,
+        kernel_size=1,
+        box_code_size=4
+    )
+    box_predictions = conv_box_predictor.predict(
+        image_features, num_predictions_per_location=5, scope='BoxPredictor')
+    box_encodings = box_predictions[box_predictor.BOX_ENCODINGS]
+    objectness_predictions = box_predictions[
+        box_predictor.CLASS_PREDICTIONS_WITH_BACKGROUND]
+    init_op = tf.global_variables_initializer()
+
+    resolution = 32
+    expected_num_anchors = resolution*resolution*5
+    with self.test_session() as sess:
+      sess.run(init_op)
+      (box_encodings_shape,
+       objectness_predictions_shape) = sess.run(
+           [tf.shape(box_encodings), tf.shape(objectness_predictions)],
+           feed_dict={image_features:
+                      np.random.rand(4, resolution, resolution, 64)})
+      self.assertAllEqual(box_encodings_shape, [4, expected_num_anchors, 1, 4])
+      self.assertAllEqual(objectness_predictions_shape,
+                          [4, expected_num_anchors, 1])
+
+
+if __name__ == '__main__':
+  tf.test.main()

object_detection/core/data_decoder.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Interface for data decoders.
+
+Data decoders decode the input data and return a dictionary of tensors keyed by
+the entries in core.reader.Fields.
+"""
+from abc import ABCMeta
+from abc import abstractmethod
+
+
+class DataDecoder(object):
+  """Interface for data decoders."""
+  __metaclass__ = ABCMeta
+
+  @abstractmethod
+  def decode(self, data):
+    """Return a single image and associated labels.
+
+    Args:
+      data: a string tensor holding a serialized protocol buffer corresponding
+        to data for a single image.
+
+    Returns:
+      tensor_dict: a dictionary containing tensors. Possible keys are defined in
+          reader.Fields.
+    """
+    pass

object_detection/core/keypoint_ops.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Keypoint operations.
+
+Keypoints are represented as tensors of shape [num_instances, num_keypoints, 2],
+where the last dimension holds rank 2 tensors of the form [y, x] representing
+the coordinates of the keypoint.
+"""
+import numpy as np
+import tensorflow as tf
+
+
+def scale(keypoints, y_scale, x_scale, scope=None):
+  """Scales keypoint coordinates in x and y dimensions.
+
+  Args:
+    keypoints: a tensor of shape [num_instances, num_keypoints, 2]
+    y_scale: (float) scalar tensor
+    x_scale: (float) scalar tensor
+    scope: name scope.
+
+  Returns:
+    new_keypoints: a tensor of shape [num_instances, num_keypoints, 2]
+  """
+  with tf.name_scope(scope, 'Scale'):
+    y_scale = tf.cast(y_scale, tf.float32)
+    x_scale = tf.cast(x_scale, tf.float32)
+    new_keypoints = keypoints * [[[y_scale, x_scale]]]
+    return new_keypoints
+
+
+def clip_to_window(keypoints, window, scope=None):
+  """Clips keypoints to a window.
+
+  This op clips any input keypoints to a window.
+
+  Args:
+    keypoints: a tensor of shape [num_instances, num_keypoints, 2]
+    window: a tensor of shape [4] representing the [y_min, x_min, y_max, x_max]
+      window to which the op should clip the keypoints.
+    scope: name scope.
+
+  Returns:
+    new_keypoints: a tensor of shape [num_instances, num_keypoints, 2]
+  """
+  with tf.name_scope(scope, 'ClipToWindow'):
+    y, x = tf.split(value=keypoints, num_or_size_splits=2, axis=2)
+    win_y_min, win_x_min, win_y_max, win_x_max = tf.unstack(window)
+    y = tf.maximum(tf.minimum(y, win_y_max), win_y_min)
+    x = tf.maximum(tf.minimum(x, win_x_max), win_x_min)
+    new_keypoints = tf.concat([y, x], 2)
+    return new_keypoints
+
+
+def prune_outside_window(keypoints, window, scope=None):
+  """Prunes keypoints that fall outside a given window.
+
+  This function replaces keypoints that fall outside the given window with nan.
+  See also clip_to_window which clips any keypoints that fall outside the given
+  window.
+
+  Args:
+    keypoints: a tensor of shape [num_instances, num_keypoints, 2]
+    window: a tensor of shape [4] representing the [y_min, x_min, y_max, x_max]
+      window outside of which the op should prune the keypoints.
+    scope: name scope.
+
+  Returns:
+    new_keypoints: a tensor of shape [num_instances, num_keypoints, 2]
+  """
+  with tf.name_scope(scope, 'PruneOutsideWindow'):
+    y, x = tf.split(value=keypoints, num_or_size_splits=2, axis=2)
+    win_y_min, win_x_min, win_y_max, win_x_max = tf.unstack(window)
+
+    valid_indices = tf.logical_and(
+        tf.logical_and(y >= win_y_min, y <= win_y_max),
+        tf.logical_and(x >= win_x_min, x <= win_x_max))
+
+    new_y = tf.where(valid_indices, y, np.nan * tf.ones_like(y))
+    new_x = tf.where(valid_indices, x, np.nan * tf.ones_like(x))
+    new_keypoints = tf.concat([new_y, new_x], 2)
+
+    return new_keypoints
+
+
+def change_coordinate_frame(keypoints, window, scope=None):
+  """Changes coordinate frame of the keypoints to be relative to window's frame.
+
+  Given a window of the form [y_min, x_min, y_max, x_max], changes keypoint
+  coordinates from keypoints of shape [num_instances, num_keypoints, 2]
+  to be relative to this window.
+
+  An example use case is data augmentation: where we are given groundtruth
+  keypoints and would like to randomly crop the image to some window. In this
+  case we need to change the coordinate frame of each groundtruth keypoint to be
+  relative to this new window.
+
+  Args:
+    keypoints: a tensor of shape [num_instances, num_keypoints, 2]
+    window: a tensor of shape [4] representing the [y_min, x_min, y_max, x_max]
+      window we should change the coordinate frame to.
+    scope: name scope.
+
+  Returns:
+    new_keypoints: a tensor of shape [num_instances, num_keypoints, 2]
+  """
+  with tf.name_scope(scope, 'ChangeCoordinateFrame'):
+    win_height = window[2] - window[0]
+    win_width = window[3] - window[1]
+    new_keypoints = scale(keypoints - [window[0], window[1]], 1.0 / win_height,
+                          1.0 / win_width)
+    return new_keypoints
+
+
+def to_normalized_coordinates(keypoints, height, width,
+                              check_range=True, scope=None):
+  """Converts absolute keypoint coordinates to normalized coordinates in [0, 1].
+
+  Usually one uses the dynamic shape of the image or conv-layer tensor:
+    keypoints = keypoint_ops.to_normalized_coordinates(keypoints,
+                                                       tf.shape(images)[1],
+                                                       tf.shape(images)[2]),
+
+  This function raises an assertion failed error at graph execution time when
+  the maximum coordinate is smaller than 1.01 (which means that coordinates are
+  already normalized). The value 1.01 is to deal with small rounding errors.
+
+  Args:
+    keypoints: A tensor of shape [num_instances, num_keypoints, 2].
+    height: Maximum value for y coordinate of absolute keypoint coordinates.
+    width: Maximum value for x coordinate of absolute keypoint coordinates.
+    check_range: If True, checks if the coordinates are normalized.
+    scope: name scope.
+
+  Returns:
+    tensor of shape [num_instances, num_keypoints, 2] with normalized
+    coordinates in [0, 1].
+  """
+  with tf.name_scope(scope, 'ToNormalizedCoordinates'):
+    height = tf.cast(height, tf.float32)
+    width = tf.cast(width, tf.float32)
+
+    if check_range:
+      max_val = tf.reduce_max(keypoints)
+      max_assert = tf.Assert(tf.greater(max_val, 1.01),
+                             ['max value is lower than 1.01: ', max_val])
+      with tf.control_dependencies([max_assert]):
+        width = tf.identity(width)
+
+    return scale(keypoints, 1.0 / height, 1.0 / width)
+
+
+def to_absolute_coordinates(keypoints, height, width,
+                            check_range=True, scope=None):
+  """Converts normalized keypoint coordinates to absolute pixel coordinates.
+
+  This function raises an assertion failed error when the maximum keypoint
+  coordinate value is larger than 1.01 (in which case coordinates are already
+  absolute).
+
+  Args:
+    keypoints: A tensor of shape [num_instances, num_keypoints, 2]
+    height: Maximum value for y coordinate of absolute keypoint coordinates.
+    width: Maximum value for x coordinate of absolute keypoint coordinates.
+    check_range: If True, checks if the coordinates are normalized or not.
+    scope: name scope.
+
+  Returns:
+    tensor of shape [num_instances, num_keypoints, 2] with absolute coordinates
+    in terms of the image size.
+
+  """
+  with tf.name_scope(scope, 'ToAbsoluteCoordinates'):
+    height = tf.cast(height, tf.float32)
+    width = tf.cast(width, tf.float32)
+
+    # Ensure range of input keypoints is correct.
+    if check_range:
+      max_val = tf.reduce_max(keypoints)
+      max_assert = tf.Assert(tf.greater_equal(1.01, max_val),
+                             ['maximum keypoint coordinate value is larger '
+                              'than 1.01: ', max_val])
+      with tf.control_dependencies([max_assert]):
+        width = tf.identity(width)
+
+    return scale(keypoints, height, width)
+
+
+def flip_horizontal(keypoints, flip_point, flip_permutation, scope=None):
+  """Flips the keypoints horizontally around the flip_point.
+
+  This operation flips the x coordinate for each keypoint around the flip_point
+  and also permutes the keypoints in a manner specified by flip_permutation.
+
+  Args:
+    keypoints: a tensor of shape [num_instances, num_keypoints, 2]
+    flip_point:  (float) scalar tensor representing the x coordinate to flip the
+      keypoints around.
+    flip_permutation: rank 1 int32 tensor containing the keypoint flip
+      permutation. This specifies the mapping from original keypoint indices
+      to the flipped keypoint indices. This is used primarily for keypoints
+      that are not reflection invariant. E.g. Suppose there are 3 keypoints
+      representing ['head', 'right_eye', 'left_eye'], then a logical choice for
+      flip_permutation might be [0, 2, 1] since we want to swap the 'left_eye'
+      and 'right_eye' after a horizontal flip.
+    scope: name scope.
+
+  Returns:
+    new_keypoints: a tensor of shape [num_instances, num_keypoints, 2]
+  """
+  with tf.name_scope(scope, 'FlipHorizontal'):
+    keypoints = tf.transpose(keypoints, [1, 0, 2])
+    keypoints = tf.gather(keypoints, flip_permutation)
+    v, u = tf.split(value=keypoints, num_or_size_splits=2, axis=2)
+    u = flip_point * 2.0 - u
+    new_keypoints = tf.concat([v, u], 2)
+    new_keypoints = tf.transpose(new_keypoints, [1, 0, 2])
+    return new_keypoints