Merge pull request #2568 from andrefaraujo/master

Adding DELF model

research/delf/delf/python/feature_extractor_test.py

+# Copyright 2017 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Tests for DELF feature extractor."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from delf import feature_extractor
+import numpy as np
+import tensorflow as tf
+
+
+class FeatureExtractorTest(tf.test.TestCase):
+
+  def testNormalizePixelValues(self):
+    image = tf.constant(
+        [[[3, 255, 0], [34, 12, 5]], [[45, 5, 65], [56, 77, 89]]],
+        dtype=tf.uint8)
+    normalized_image = feature_extractor.NormalizePixelValues(
+        image, pixel_value_offset=5.0, pixel_value_scale=2.0)
+    exp_normalized_image = [[[-1.0, 125.0, -2.5], [14.5, 3.5, 0.0]],
+                            [[20.0, 0.0, 30.0], [25.5, 36.0, 42.0]]]
+    with self.test_session() as sess:
+      normalized_image_out = sess.run(normalized_image)
+
+    self.assertAllEqual(normalized_image_out, exp_normalized_image)
+
+  def testCalculateReceptiveBoxes(self):
+    boxes = feature_extractor.CalculateReceptiveBoxes(
+        height=1, width=2, rf=291, stride=32, padding=145)
+    exp_boxes = [[-145., -145., 145., 145.], [-145., -113., 145., 177.]]
+    with self.test_session() as sess:
+      boxes_out = sess.run(boxes)
+
+    self.assertAllEqual(exp_boxes, boxes_out)
+
+  def testCalculateKeypointCenters(self):
+    boxes = [[-10.0, 0.0, 11.0, 21.0], [-2.5, 5.0, 18.5, 26.0],
+             [45.0, -2.5, 66.0, 18.5]]
+    centers = feature_extractor.CalculateKeypointCenters(boxes)
+    with self.test_session() as sess:
+      centers_out = sess.run(centers)
+
+    exp_centers = [[0.5, 10.5], [8.0, 15.5], [55.5, 8.0]]
+
+    self.assertAllEqual(exp_centers, centers_out)
+
+  def testExtractKeypointDescriptor(self):
+    image = tf.constant(
+        [[[0, 255, 255], [128, 64, 196]], [[0, 0, 32], [32, 128, 16]]],
+        dtype=tf.uint8)
+
+    # Arbitrary model function used to test ExtractKeypointDescriptor. The
+    # generated feature_map is a replicated version of the image, concatenated
+    # with zeros to achieve the required dimensionality. The attention is simply
+    # the norm of the input image pixels.
+    def _test_model_fn(image, normalized_image, reuse):
+      del normalized_image, reuse  # Unused variables in the test.
+      image_shape = tf.shape(image)
+      attention = tf.squeeze(tf.norm(image, axis=3))
+      feature_map = tf.concat(
+          [
+              tf.tile(image, [1, 1, 1, 341]),
+              tf.zeros([1, image_shape[1], image_shape[2], 1])
+          ],
+          axis=3)
+      return attention, feature_map
+
+    boxes, feature_scales, features, scores = feature_extractor.ExtractKeypointDescriptor(
+        image,
+        layer_name='resnet_v1_50/block3',
+        image_scales=tf.constant([1.0]),
+        iou=1.0,
+        max_feature_num=10,
+        abs_thres=1.5,
+        model_fn=_test_model_fn)
+
+    exp_boxes = [[-145.0, -145.0, 145.0, 145.0], [-113.0, -145.0, 177.0, 145.0]]
+    exp_feature_scales = [1.0, 1.0]
+    exp_features = np.array(
+        np.concatenate(
+            (np.tile([[-1.0, 127.0 / 128.0, 127.0 / 128.0], [-1.0, -1.0, -0.75]
+                     ], [1, 341]), np.zeros([2, 1])),
+            axis=1))
+    exp_scores = [[1.723042], [1.600781]]
+
+    with self.test_session() as sess:
+      boxes_out, feature_scales_out, features_out, scores_out = sess.run(
+          [boxes, feature_scales, features, scores])
+
+    self.assertAllEqual(exp_boxes, boxes_out)
+    self.assertAllEqual(exp_feature_scales, feature_scales_out)
+    self.assertAllClose(exp_features, features_out)
+    self.assertAllClose(exp_scores, scores_out)
+
+  def testPcaWhitening(self):
+    data = tf.constant([[1.0, 2.0, -2.0], [-5.0, 0.0, 3.0], [-1.0, 2.0, 0.0],
+                        [0.0, 4.0, -1.0]])
+    pca_matrix = tf.constant([[2.0, 0.0, -1.0], [0.0, 1.0, 1.0],
+                              [-1.0, 1.0, 3.0]])
+    pca_mean = tf.constant([1.0, 2.0, 3.0])
+    output_dim = 2
+    use_whitening = True
+    pca_variances = tf.constant([4.0, 1.0])
+
+    output = feature_extractor.ApplyPcaAndWhitening(
+        data, pca_matrix, pca_mean, output_dim, use_whitening, pca_variances)
+
+    exp_output = [[2.5, -5.0], [-6.0, -2.0], [-0.5, -3.0], [1.0, -2.0]]
+
+    with self.test_session() as sess:
+      output_out = sess.run(output)
+
+    self.assertAllEqual(exp_output, output_out)
+
+
+if __name__ == '__main__':
+  tf.test.main()

research/delf/delf/python/feature_io.py

+# Copyright 2017 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Python interface for DelfFeatures proto.
+
+Support read and write of DelfFeatures from/to numpy arrays and file.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from delf import feature_pb2
+from delf import datum_io
+import numpy as np
+import tensorflow as tf
+
+
+def ArraysToDelfFeatures(locations,
+                         scales,
+                         descriptors,
+                         attention,
+                         orientations=None):
+  """Converts DELF features to DelfFeatures proto.
+
+  Args:
+    locations: [N, 2] float array which denotes the selected keypoint
+      locations. N is the number of features.
+    scales: [N] float array with feature scales.
+    descriptors: [N, depth] float array with DELF descriptors.
+    attention: [N] float array with attention scores.
+    orientations: [N] float array with orientations. If None, all orientations
+      are set to zero.
+
+  Returns:
+    delf_features: DelfFeatures object.
+  """
+  num_features = len(attention)
+  assert num_features == locations.shape[0]
+  assert num_features == len(scales)
+  assert num_features == descriptors.shape[0]
+
+  if orientations is None:
+    orientations = np.zeros([num_features], dtype=np.float32)
+  else:
+    assert num_features == len(orientations)
+
+  delf_features = feature_pb2.DelfFeatures()
+  for i in xrange(num_features):
+    delf_feature = delf_features.feature.add()
+    delf_feature.y = locations[i, 0]
+    delf_feature.x = locations[i, 1]
+    delf_feature.scale = scales[i]
+    delf_feature.orientation = orientations[i]
+    delf_feature.strength = attention[i]
+    delf_feature.descriptor.CopyFrom(datum_io.ArrayToDatum(descriptors[i,]))
+
+  return delf_features
+
+
+def DelfFeaturesToArrays(delf_features):
+  """Converts data saved in DelfFeatures to numpy arrays.
+
+  If there are no features, the function returns four empty arrays.
+
+  Args:
+    delf_features: DelfFeatures object.
+
+  Returns:
+    locations: [N, 2] float array which denotes the selected keypoint
+      locations. N is the number of features.
+    scales: [N] float array with feature scales.
+    descriptors: [N, depth] float array with DELF descriptors.
+    attention: [N] float array with attention scores.
+    orientations: [N] float array with orientations.
+  """
+  num_features = len(delf_features.feature)
+  if num_features == 0:
+    return np.array([]), np.array([]), np.array([]), np.array([])
+
+  # Figure out descriptor dimensionality by parsing first one.
+  descriptor_dim = len(
+      datum_io.DatumToArray(delf_features.feature[0].descriptor))
+  locations = np.zeros([num_features, 2])
+  scales = np.zeros([num_features])
+  descriptors = np.zeros([num_features, descriptor_dim])
+  attention = np.zeros([num_features])
+  orientations = np.zeros([num_features])
+
+  for i in xrange(num_features):
+    delf_feature = delf_features.feature[i]
+    locations[i, 0] = delf_feature.y
+    locations[i, 1] = delf_feature.x
+    scales[i] = delf_feature.scale
+    descriptors[i,] = datum_io.DatumToArray(delf_feature.descriptor)
+    attention[i] = delf_feature.strength
+    orientations[i] = delf_feature.orientation
+
+  return locations, scales, descriptors, attention, orientations
+
+
+def SerializeToString(locations,
+                      scales,
+                      descriptors,
+                      attention,
+                      orientations=None):
+  """Converts numpy arrays to serialized DelfFeatures.
+
+  Args:
+    locations: [N, 2] float array which denotes the selected keypoint
+      locations. N is the number of features.
+    scales: [N] float array with feature scales.
+    descriptors: [N, depth] float array with DELF descriptors.
+    attention: [N] float array with attention scores.
+    orientations: [N] float array with orientations. If None, all orientations
+      are set to zero.
+
+  Returns:
+    Serialized DelfFeatures string.
+  """
+  delf_features = ArraysToDelfFeatures(locations, scales, descriptors,
+                                       attention, orientations)
+  return delf_features.SerializeToString()
+
+
+def ParseFromString(string):
+  """Converts serialized DelfFeatures string to numpy arrays.
+
+  Args:
+    string: Serialized DelfFeatures string.
+
+  Returns:
+    locations: [N, 2] float array which denotes the selected keypoint
+      locations. N is the number of features.
+    scales: [N] float array with feature scales.
+    descriptors: [N, depth] float array with DELF descriptors.
+    attention: [N] float array with attention scores.
+    orientations: [N] float array with orientations.
+  """
+  delf_features = feature_pb2.DelfFeatures()
+  delf_features.ParseFromString(string)
+  return DelfFeaturesToArrays(delf_features)
+
+
+def ReadFromFile(file_path):
+  """Helper function to load data from a DelfFeatures format in a file.
+
+  Args:
+    file_path: Path to file containing data.
+
+  Returns:
+    locations: [N, 2] float array which denotes the selected keypoint
+      locations. N is the number of features.
+    scales: [N] float array with feature scales.
+    descriptors: [N, depth] float array with DELF descriptors.
+    attention: [N] float array with attention scores.
+    orientations: [N] float array with orientations.
+  """
+  with tf.gfile.FastGFile(file_path, 'r') as f:
+    return ParseFromString(f.read())
+
+
+def WriteToFile(file_path,
+                locations,
+                scales,
+                descriptors,
+                attention,
+                orientations=None):
+  """Helper function to write data to a file in DelfFeatures format.
+
+  Args:
+    file_path: Path to file that will be written.
+    locations: [N, 2] float array which denotes the selected keypoint
+      locations. N is the number of features.
+    scales: [N] float array with feature scales.
+    descriptors: [N, depth] float array with DELF descriptors.
+    attention: [N] float array with attention scores.
+    orientations: [N] float array with orientations. If None, all orientations
+      are set to zero.
+  """
+  serialized_data = SerializeToString(locations, scales, descriptors, attention,
+                                      orientations)
+  with tf.gfile.FastGFile(file_path, 'w') as f:
+    f.write(serialized_data)

research/delf/delf/python/feature_io_test.py

+# Copyright 2017 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Tests for feature_io, the python interface of DelfFeatures."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from delf import feature_io
+from delf import datum_pb2
+import numpy as np
+import os
+import tensorflow as tf
+
+
+def create_data():
+  """Creates data to be used in tests.
+    Returns:
+      locations: [N, 2] float array which denotes the selected keypoint
+        locations. N is the number of features.
+      scales: [N] float array with feature scales.
+      descriptors: [N, depth] float array with DELF descriptors.
+      attention: [N] float array with attention scores.
+      orientations: [N] float array with orientations.
+    """
+  locations = np.arange(8, dtype=np.float32).reshape(4, 2)
+  scales = np.arange(4, dtype=np.float32)
+  attention = np.arange(4, dtype=np.float32)
+  orientations = np.arange(4, dtype=np.float32)
+  descriptors = np.zeros([4, 1024])
+  descriptors[0,] = np.arange(1024)
+  descriptors[1,] = np.zeros([1024])
+  descriptors[2,] = np.ones([1024])
+  descriptors[3,] = -np.ones([1024])
+
+  return locations, scales, descriptors, attention, orientations
+
+
+class DelfFeaturesIoTest(tf.test.TestCase):
+
+  def testConversionAndBack(self):
+    locations, scales, descriptors, attention, orientations = create_data()
+
+    serialized = feature_io.SerializeToString(locations, scales, descriptors,
+                                              attention, orientations)
+    parsed_data = feature_io.ParseFromString(serialized)
+
+    self.assertAllEqual(locations, parsed_data[0])
+    self.assertAllEqual(scales, parsed_data[1])
+    self.assertAllEqual(descriptors, parsed_data[2])
+    self.assertAllEqual(attention, parsed_data[3])
+    self.assertAllEqual(orientations, parsed_data[4])
+
+  def testConversionAndBackNoOrientations(self):
+    locations, scales, descriptors, attention, _ = create_data()
+
+    serialized = feature_io.SerializeToString(locations, scales, descriptors,
+                                              attention)
+    parsed_data = feature_io.ParseFromString(serialized)
+
+    self.assertAllEqual(locations, parsed_data[0])
+    self.assertAllEqual(scales, parsed_data[1])
+    self.assertAllEqual(descriptors, parsed_data[2])
+    self.assertAllEqual(attention, parsed_data[3])
+    self.assertAllEqual(np.zeros([4]), parsed_data[4])
+
+  def testWriteAndReadToFile(self):
+    locations, scales, descriptors, attention, orientations = create_data()
+
+    tmpdir = tf.test.get_temp_dir()
+    filename = os.path.join(tmpdir, 'test.delf')
+    feature_io.WriteToFile(filename, locations, scales, descriptors, attention,
+                           orientations)
+    data_read = feature_io.ReadFromFile(filename)
+
+    self.assertAllEqual(locations, data_read[0])
+    self.assertAllEqual(scales, data_read[1])
+    self.assertAllEqual(descriptors, data_read[2])
+    self.assertAllEqual(attention, data_read[3])
+    self.assertAllEqual(orientations, data_read[4])
+
+
+if __name__ == '__main__':
+  tf.test.main()

research/delf/setup.py

+# Copyright 2017 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Setup script for delf."""
+
+from setuptools import setup, find_packages
+
+setup(
+    name='delf',
+    version='0.1',
+    include_package_data=True,
+    packages=find_packages(),
+    description='DELF (DEep Local Features)',
+)