Release MobileDet code and model, and require tf_slim installation for OD API. (#8562)

* Merged commit includes the following changes:
311933687  by Sergio Guadarrama:

    Removes spurios use of tf.compat.v2, which results in spurious tf.compat.v1.compat.v2. Adds basic test to nasnet_utils.
    Replaces all remaining import tensorflow as tf with import tensorflow.compat.v1 as tf

--
311766063  by Sergio Guadarrama:

    Removes explicit tf.compat.v1 in all call sites (we already import tf.compat.v1, so this code was  doing tf.compat.v1.compat.v1). The existing code worked in latest version of tensorflow, 2.2, (and 1.15) but not in 1.14 or in 2.0.0a, this CL fixes it.

--
311624958  by Sergio Guadarrama:

    Updates README that doesn't render properly in github documentation

--
310980959  by Sergio Guadarrama:

    Moves research_models/slim off tf.contrib.slim/layers/framework to tf_slim

--
310263156  by Sergio Guadarrama:

    Adds model breakdown for MobilenetV3

--
308640...

research/lstm_object_detection/README.md

-![TensorFlow Requirement: 1.x](https://img.shields.io/badge/TensorFlow%20Requirement-1.x-brightgreen)
-![TensorFlow 2 Not Supported](https://img.shields.io/badge/TensorFlow%202%20Not%20Supported-%E2%9C%95-red.svg)
-
 # Tensorflow Mobile Video Object Detection
 
 Tensorflow mobile video object detection implementation proposed in the
@@ -35,6 +32,7 @@ https://scholar.googleusercontent.com/scholar.bib?q=info:rLqvkztmWYgJ:scholar.go
 * yinxiao@google.com
 * menglong@google.com
 * yongzhe@google.com
+* lzyuan@google.com
 
 
 ## Table of Contents

research/lstm_object_detection/inputs/seq_dataset_builder.py

@@ -23,7 +23,8 @@ Detection configuration framework, they should define their own builder function
 that wraps the build function.
 """
 import tensorflow.compat.v1 as tf
-from tensorflow.contrib import slim as contrib_slim
+import tf_slim as slim
+
 from tensorflow.contrib.training.python.training import sequence_queueing_state_saver as sqss
 from lstm_object_detection.inputs import tf_sequence_example_decoder
 from lstm_object_detection.protos import input_reader_google_pb2
@@ -33,7 +34,7 @@ from object_detection.core import standard_fields as fields
 from object_detection.protos import input_reader_pb2
 from object_detection.utils import ops as util_ops
 
-parallel_reader = contrib_slim.parallel_reader
+parallel_reader = slim.parallel_reader
 # TODO(yinxiao): Make the following variable into configurable proto.
 # Padding size for the labeled objects in each frame. Here we assume each
 # frame has a total number of objects less than _PADDING_SIZE.

research/lstm_object_detection/inputs/tf_sequence_example_decoder.py

@@ -19,11 +19,11 @@ A decoder to decode string tensors containing serialized
 tensorflow.SequenceExample protos.
 """
 import tensorflow.compat.v1 as tf
-from tensorflow.contrib import slim as contrib_slim
+import tf_slim as slim
 from object_detection.core import data_decoder
 from object_detection.core import standard_fields as fields
 
-tfexample_decoder = contrib_slim.tfexample_decoder
+tfexample_decoder = slim.tfexample_decoder
 
 
 class BoundingBoxSequence(tfexample_decoder.ItemHandler):

research/lstm_object_detection/lstm/lstm_cells.py

@@ -15,10 +15,8 @@
 """BottleneckConvLSTMCell implementation."""
 
 import tensorflow.compat.v1 as tf
-
-from tensorflow.contrib import layers as contrib_layers
+import tf_slim as slim
 from tensorflow.contrib import rnn as contrib_rnn
-from tensorflow.contrib import slim
 from tensorflow.contrib.framework.python.ops import variables as contrib_variables
 import lstm_object_detection.lstm.utils as lstm_utils
 
@@ -121,7 +119,7 @@ class BottleneckConvLSTMCell(contrib_rnn.RNNCell):
       if self._pre_bottleneck:
         bottleneck = inputs
       else:
-        bottleneck = contrib_layers.separable_conv2d(
+        bottleneck = slim.separable_conv2d(
             tf.concat([inputs, h], 3),
             self._num_units,
             self._filter_size,
@@ -133,7 +131,7 @@ class BottleneckConvLSTMCell(contrib_rnn.RNNCell):
         if self._viz_gates:
           slim.summaries.add_histogram_summary(bottleneck, 'bottleneck')
 
-      concat = contrib_layers.separable_conv2d(
+      concat = slim.separable_conv2d(
           bottleneck,
           4 * self._num_units,
           self._filter_size,
@@ -243,7 +241,7 @@ class BottleneckConvLSTMCell(contrib_rnn.RNNCell):
       state = tf.reshape(state, [batch_size, height, width, -1])
     with tf.variable_scope('conv_lstm_cell', reuse=tf.AUTO_REUSE):
       scope_name = 'bottleneck_%d' % input_index
-      inputs = contrib_layers.separable_conv2d(
+      inputs = slim.separable_conv2d(
           tf.concat([inputs, state], 3),
           self.output_size[-1],
           self._filter_size,

research/lstm_object_detection/lstm/utils.py

@@ -217,7 +217,8 @@ def quantize_op(inputs,
     # While training, collect EMAs of ranges seen, store in min_var, max_var.
     # TFLite requires that 0.0 is always in the [min; max] range.
     range_min = tf.minimum(tf.reduce_min(inputs), 0.0, 'SafeQuantRangeMin')
-    range_max = tf.maximum(tf.reduce_max(inputs), 0.0, 'SafeQuantRangeMax')
+    # We set the lower_bound of max_range to prevent range collapse.
+    range_max = tf.maximum(tf.reduce_max(inputs), 1e-5, 'SafeQuantRangeMax')
     min_val = moving_averages.assign_moving_average(
         min_var, range_min, ema_decay, name='AssignMinEma')
     max_val = moving_averages.assign_moving_average(

research/lstm_object_detection/meta_architectures/lstm_ssd_meta_arch.py

@@ -25,7 +25,6 @@ for details.
 import abc
 import re
 import tensorflow.compat.v1 as tf
-from tensorflow.contrib import slim as contrib_slim
 
 from object_detection.core import box_list_ops
 from object_detection.core import matcher
@@ -34,8 +33,6 @@ from object_detection.meta_architectures import ssd_meta_arch
 from object_detection.utils import ops
 from object_detection.utils import shape_utils
 
-slim = contrib_slim
-
 
 class LSTMSSDMetaArch(ssd_meta_arch.SSDMetaArch):
   """LSTM Meta-architecture definition."""

research/lstm_object_detection/meta_architectures/lstm_ssd_meta_arch_test.py

@@ -23,7 +23,7 @@ import functools
 
 import numpy as np
 import tensorflow.compat.v1 as tf
-from tensorflow.contrib import slim as contrib_slim
+import tf_slim as slim
 
 from lstm_object_detection.lstm import lstm_cells
 from lstm_object_detection.meta_architectures import lstm_ssd_meta_arch
@@ -39,8 +39,6 @@ from object_detection.utils import test_case
 from object_detection.utils import test_utils
 
 
-slim = contrib_slim
-
 MAX_TOTAL_NUM_BOXES = 5
 NUM_CLASSES = 1
 

research/lstm_object_detection/models/lstm_ssd_interleaved_mobilenet_v2_feature_extractor.py

@@ -16,7 +16,7 @@
 """LSTDInterleavedFeatureExtractor which interleaves multiple MobileNet V2."""
 
 import tensorflow.compat.v1 as tf
-from tensorflow.contrib import slim
+import tf_slim as slim
 
 from tensorflow.python.framework import ops as tf_ops
 from lstm_object_detection.lstm import lstm_cells
@@ -134,9 +134,10 @@ class LSTMSSDInterleavedMobilenetV2FeatureExtractor(
     scope_name = self._base_network_scope + '_2'
     with tf.variable_scope(scope_name, reuse=self._reuse_weights) as base_scope:
       if self._low_res:
-        size_small = preprocessed_inputs.get_shape().as_list()[1] / 2
+        height_small = preprocessed_inputs.get_shape().as_list()[1] // 2
+        width_small = preprocessed_inputs.get_shape().as_list()[2] // 2
         inputs_small = tf.image.resize_images(preprocessed_inputs,
-                                              [size_small, size_small])
+                                              [height_small, width_small])
         # Create end point handle for tflite deployment.
         with tf.name_scope(None):
           inputs_small = tf.identity(
@@ -152,7 +153,8 @@ class LSTMSSDInterleavedMobilenetV2FeatureExtractor(
           scope=base_scope)
       return net, end_points
 
-  def create_lstm_cell(self, batch_size, output_size, state_saver, state_name):
+  def create_lstm_cell(self, batch_size, output_size, state_saver, state_name,
+                       dtype=tf.float32):
     """Create the LSTM cell, and initialize state if necessary.
 
     Args:
@@ -160,6 +162,8 @@ class LSTMSSDInterleavedMobilenetV2FeatureExtractor(
       output_size: output size of the lstm cell, [width, height].
       state_saver: a state saver object with methods `state` and `save_state`.
       state_name: string, the name to use with the state_saver.
+      dtype: dtype to initialize lstm state.
+
     Returns:
       lstm_cell: the lstm cell unit.
       init_state: initial state representations.
@@ -180,7 +184,7 @@ class LSTMSSDInterleavedMobilenetV2FeatureExtractor(
         visualize_gates=False)
 
     if state_saver is None:
-      init_state = lstm_cell.init_state('lstm_state', batch_size, tf.float32)
+      init_state = lstm_cell.init_state('lstm_state', batch_size, dtype)
       step = None
     else:
       step = state_saver.state(state_name + '_step')
@@ -222,7 +226,7 @@ class LSTMSSDInterleavedMobilenetV2FeatureExtractor(
         33, preprocessed_inputs)
     preprocessed_inputs = ops.pad_to_multiple(
         preprocessed_inputs, self._pad_to_multiple)
-    batch_size = preprocessed_inputs.shape[0].value / unroll_length
+    batch_size = preprocessed_inputs.shape[0].value // unroll_length
     batch_axis = 0
     nets = []
 
@@ -250,7 +254,8 @@ class LSTMSSDInterleavedMobilenetV2FeatureExtractor(
       with tf.variable_scope('LSTM', reuse=self._reuse_weights):
         output_size = (large_base_feature_shape[1], large_base_feature_shape[2])
         lstm_cell, init_state, step = self.create_lstm_cell(
-            batch_size, output_size, state_saver, state_name)
+            batch_size, output_size, state_saver, state_name,
+            dtype=preprocessed_inputs.dtype)
 
         nets_seq = [
             tf.split(net, unroll_length, axis=batch_axis) for net in nets
@@ -269,15 +274,16 @@ class LSTMSSDInterleavedMobilenetV2FeatureExtractor(
             scope=None)
         self._states_out = states_out
 
-      batcher_ops = None
+      image_features = {}
       if state_saver is not None:
         self._step = state_saver.state(state_name + '_step')
         batcher_ops = [
             state_saver.save_state(state_name + '_c', states_out[-1][0]),
             state_saver.save_state(state_name + '_h', states_out[-1][1]),
             state_saver.save_state(state_name + '_step', self._step + 1)]
-      image_features = {}
-      with tf_ops.control_dependencies(batcher_ops):
+        with tf_ops.control_dependencies(batcher_ops):
+          image_features['layer_19'] = tf.concat(net_seq, 0)
+      else:
         image_features['layer_19'] = tf.concat(net_seq, 0)
 
       # SSD layers.
@@ -289,4 +295,4 @@ class LSTMSSDInterleavedMobilenetV2FeatureExtractor(
             insert_1x1_conv=True,
             image_features=image_features,
             pool_residual=True)
-    return feature_maps.values()
+    return list(feature_maps.values())

research/lstm_object_detection/models/lstm_ssd_interleaved_mobilenet_v2_feature_extractor_test.py

@@ -15,10 +15,9 @@
 
 """Tests for lstm_ssd_interleaved_mobilenet_v2_feature_extractor."""
 
-import itertools
 import numpy as np
 import tensorflow.compat.v1 as tf
-from tensorflow.contrib import slim
+import tf_slim as slim
 from tensorflow.contrib import training as contrib_training
 
 from lstm_object_detection.models import lstm_ssd_interleaved_mobilenet_v2_feature_extractor
@@ -261,16 +260,16 @@ class LSTMSSDInterleavedMobilenetV2FeatureExtractorTest(
     state_channel = 320
     init_state1 = {
         'lstm_state_c': tf.zeros(
-            [image_height/32, image_width/32, state_channel]),
+            [image_height // 32, image_width // 32, state_channel]),
         'lstm_state_h': tf.zeros(
-            [image_height/32, image_width/32, state_channel]),
+            [image_height // 32, image_width // 32, state_channel]),
         'lstm_state_step': tf.zeros([1])
     }
     init_state2 = {
         'lstm_state_c': tf.random_uniform(
-            [image_height/32, image_width/32, state_channel]),
+            [image_height // 32, image_width // 32, state_channel]),
         'lstm_state_h': tf.random_uniform(
-            [image_height/32, image_width/32, state_channel]),
+            [image_height // 32, image_width // 32, state_channel]),
         'lstm_state_step': tf.zeros([1])
     }
     seq = {'dummy': tf.random_uniform([2, 1, 1, 1])}
@@ -326,7 +325,7 @@ class LSTMSSDInterleavedMobilenetV2FeatureExtractorTest(
     image_tensor = np.random.rand(batch_size, image_height, image_width,
                                   3).astype(np.float32)
     feature_maps = self.execute(graph_fn, [image_tensor])
-    for feature_map, expected_shape in itertools.izip(
+    for feature_map, expected_shape in zip(
         feature_maps, expected_feature_map_shapes):
       self.assertAllEqual(feature_map.shape, expected_shape)
 

research/lstm_object_detection/models/lstm_ssd_mobilenet_v1_feature_extractor.py

@@ -16,7 +16,7 @@
 """LSTMSSDFeatureExtractor for MobilenetV1 features."""
 
 import tensorflow.compat.v1 as tf
-from tensorflow.contrib import slim as contrib_slim
+import tf_slim as slim
 from tensorflow.python.framework import ops as tf_ops
 from lstm_object_detection.lstm import lstm_cells
 from lstm_object_detection.lstm import rnn_decoder
@@ -27,8 +27,6 @@ from object_detection.utils import ops
 from object_detection.utils import shape_utils
 from nets import mobilenet_v1
 
-slim = contrib_slim
-
 
 class LSTMSSDMobileNetV1FeatureExtractor(
     lstm_ssd_meta_arch.LSTMSSDFeatureExtractor):
@@ -85,7 +83,8 @@ class LSTMSSDMobileNetV1FeatureExtractor(
     self._base_network_scope = 'MobilenetV1'
     self._lstm_state_depth = lstm_state_depth
 
-  def create_lstm_cell(self, batch_size, output_size, state_saver, state_name):
+  def create_lstm_cell(self, batch_size, output_size, state_saver, state_name,
+                       dtype=tf.float32):
     """Create the LSTM cell, and initialize state if necessary.
 
     Args:
@@ -93,6 +92,7 @@ class LSTMSSDMobileNetV1FeatureExtractor(
       output_size: output size of the lstm cell, [width, height].
       state_saver: a state saver object with methods `state` and `save_state`.
       state_name: string, the name to use with the state_saver.
+      dtype: dtype to initialize lstm state.
 
     Returns:
       lstm_cell: the lstm cell unit.
@@ -107,7 +107,7 @@ class LSTMSSDMobileNetV1FeatureExtractor(
         visualize_gates=False)
 
     if state_saver is None:
-      init_state = lstm_cell.init_state(state_name, batch_size, tf.float32)
+      init_state = lstm_cell.init_state(state_name, batch_size, dtype)
       step = None
     else:
       step = state_saver.state(state_name + '_step')
@@ -166,11 +166,14 @@ class LSTMSSDMobileNetV1FeatureExtractor(
       with slim.arg_scope(
           [slim.batch_norm], fused=False, is_training=self._is_training):
         # ConvLSTM layers.
-        batch_size = net.shape[0].value / unroll_length
+        batch_size = net.shape[0].value // unroll_length
         with tf.variable_scope('LSTM', reuse=self._reuse_weights) as lstm_scope:
           lstm_cell, init_state, _ = self.create_lstm_cell(
-              batch_size, (net.shape[1].value, net.shape[2].value), state_saver,
-              state_name)
+              batch_size,
+              (net.shape[1].value, net.shape[2].value),
+              state_saver,
+              state_name,
+              dtype=preprocessed_inputs.dtype)
           net_seq = list(tf.split(net, unroll_length))
 
           # Identities added for inputing state tensors externally.
@@ -205,4 +208,4 @@ class LSTMSSDMobileNetV1FeatureExtractor(
               insert_1x1_conv=True,
               image_features=image_features)
 
-    return feature_maps.values()
+    return list(feature_maps.values())

research/lstm_object_detection/models/lstm_ssd_mobilenet_v1_feature_extractor_test.py

@@ -17,14 +17,12 @@
 
 import numpy as np
 import tensorflow.compat.v1 as tf
-from tensorflow.contrib import slim as contrib_slim
+import tf_slim as slim
 from tensorflow.contrib import training as contrib_training
 
 from lstm_object_detection.models import lstm_ssd_mobilenet_v1_feature_extractor as feature_extractor
 from object_detection.models import ssd_feature_extractor_test
 
-slim = contrib_slim
-
 
 class LstmSsdMobilenetV1FeatureExtractorTest(
     ssd_feature_extractor_test.SsdFeatureExtractorTestBase):

research/lstm_object_detection/models/mobilenet_defs.py

@@ -15,14 +15,11 @@
 """Definitions for modified MobileNet models used in LSTD."""
 
 import tensorflow.compat.v1 as tf
-from tensorflow.contrib import slim as contrib_slim
-
+import tf_slim as slim
 from nets import mobilenet_v1
 from nets.mobilenet import conv_blocks as mobilenet_convs
 from nets.mobilenet import mobilenet
 
-slim = contrib_slim
-
 
 def mobilenet_v1_lite_def(depth_multiplier, low_res=False):
   """Conv definitions for a lite MobileNet v1 model.

research/lstm_object_detection/trainer.py

@@ -19,9 +19,15 @@ This file provides a generic training method that can be used to train a
 DetectionModel.
 """
 
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
 import functools
+import six
+from six.moves import range
 import tensorflow.compat.v1 as tf
-from tensorflow.contrib import slim as contrib_slim
+import tf_slim as slim
 
 from object_detection.builders import optimizer_builder
 from object_detection.core import standard_fields as fields
@@ -29,8 +35,6 @@ from object_detection.utils import ops as util_ops
 from object_detection.utils import variables_helper
 from deployment import model_deploy
 
-slim = contrib_slim
-
 
 def create_input_queue(create_tensor_dict_fn):
   """Sets up reader, prefetcher and returns input queue.
@@ -198,7 +202,7 @@ def get_restore_checkpoint_ops(restore_checkpoints, detection_model,
     available_var_map = (
         variables_helper.get_variables_available_in_checkpoint(
             var_map, restore_checkpoint))
-    for var_name, var in available_var_map.iteritems():
+    for var_name, var in six.iteritems(available_var_map):
       if var in vars_restored:
         tf.logging.info('Variable %s contained in multiple checkpoints',
                      var.op.name)
@@ -210,7 +214,7 @@ def get_restore_checkpoint_ops(restore_checkpoints, detection_model,
     available_ema_var_map = {}
     ckpt_reader = tf.train.NewCheckpointReader(restore_checkpoint)
     ckpt_vars_to_shape_map = ckpt_reader.get_variable_to_shape_map()
-    for var_name, var in available_var_map.iteritems():
+    for var_name, var in six.iteritems(available_var_map):
       var_name_ema = var_name + '/ExponentialMovingAverage'
       if var_name_ema in ckpt_vars_to_shape_map:
         available_ema_var_map[var_name_ema] = var
@@ -218,7 +222,7 @@ def get_restore_checkpoint_ops(restore_checkpoints, detection_model,
         available_ema_var_map[var_name] = var
     available_var_map = available_ema_var_map
     init_saver = tf.train.Saver(available_var_map)
-    if available_var_map.keys():
+    if list(available_var_map.keys()):
       restorers.append(init_saver)
     else:
       tf.logging.info('WARNING: Checkpoint %s has no restorable variables',

research/object_detection/README.md

@@ -104,6 +104,25 @@ reporting an issue.
 
 ## Release information
 
+### May 19th, 2020
+We have released
+[MobileDets](https://arxiv.org/abs/2004.14525),
+a set of high-performance models for mobile CPUs, DSPs and EdgeTPUs.
+
+* MobileDets outperform MobileNetV3+SSDLite by 1.7 mAP at comparable mobile CPU
+inference latencies. MobileDets also outperform MobileNetV2+SSDLite by 1.9 mAP
+on mobile CPUs, 3.7 mAP on EdgeTPUs and 3.4 mAP on DSPs while running equally
+fast. MobileDets also offer up to 2x speedup over MnasFPN on EdgeTPUs and DSPs.
+
+For each of the three hardware platforms we have released model definition,
+model checkpoints trained on the COCO14 dataset and converted TFLite models in
+fp32 and/or uint8.
+
+<b>Thanks to contributors</b>: Yunyang Xiong, Hanxiao Liu, Suyog Gupta,
+Berkin Akin, Gabriel Bender, Pieter-Jan Kindermans, Mingxing Tan, Vikas Singh,
+Bo Chen, Quoc Le, Zhichao Lu.
+
+
 ### May 7th, 2020
 We have released a mobile model with the
 [MnasFPN head](https://arxiv.org/abs/1912.01106).
@@ -119,7 +138,7 @@ the COCO14 dataset and a converted TFLite model.
 
 <b>Thanks to contributors</b>: Bo Chen, Golnaz Ghiasi, Hanxiao Liu,
 Tsung-Yi Lin, Dmitry Kalenichenko, Hartwig Adam, Quoc Le, Zhichao Lu,
-Jonathan Huang.
+Jonathan Huang, Hao Xu.
 
 
 

research/object_detection/anchor_generators/flexible_grid_anchor_generator.py

@@ -14,7 +14,7 @@
 # ==============================================================================
 """Generates grid anchors on the fly corresponding to multiple CNN layers."""
 
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 
 from object_detection.anchor_generators import grid_anchor_generator
 from object_detection.core import anchor_generator

research/object_detection/anchor_generators/flexible_grid_anchor_generator_test.py

@@ -15,7 +15,7 @@
 
 """Tests for anchor_generators.flexible_grid_anchor_generator_test.py."""
 import numpy as np
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 
 from object_detection.anchor_generators import flexible_grid_anchor_generator as fg
 from object_detection.utils import test_case

research/object_detection/anchor_generators/grid_anchor_generator.py

@@ -20,7 +20,7 @@ Generates grid anchors on the fly as described in:
 Shaoqing Ren, Kaiming He, Ross Girshick, and Jian Sun.
 """
 
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 
 from object_detection.core import anchor_generator
 from object_detection.core import box_list

research/object_detection/anchor_generators/grid_anchor_generator_test.py

@@ -15,7 +15,7 @@
 
 """Tests for object_detection.grid_anchor_generator."""
 import numpy as np
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 
 from object_detection.anchor_generators import grid_anchor_generator
 from object_detection.utils import test_case

research/object_detection/anchor_generators/multiple_grid_anchor_generator.py

@@ -25,7 +25,7 @@ Cheng-Yang Fu, Alexander C. Berg
 
 import numpy as np
 
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 
 from object_detection.anchor_generators import grid_anchor_generator
 from object_detection.core import anchor_generator

research/object_detection/anchor_generators/multiple_grid_anchor_generator_test.py

@@ -17,7 +17,7 @@
 
 import numpy as np
 
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 
 from object_detection.anchor_generators import multiple_grid_anchor_generator as ag
 from object_detection.utils import test_case