Fix dependency issues (#5815)

* Add a second expected_loss_weights function using an alternative expectation calculation compared to previous. Integrate this op into ssd_meta_arch and losses builder. Affects files that use losses_builder.build to handle the returning of an additional element.

PiperOrigin-RevId: 219190083

* Fix dependency issues.

PiperOrigin-RevId: 222888231

* Update CODEOWNERS 

Add @masonliuw and @yinxiaoli to /research/lstm_object_detection/

CODEOWNERS

@@ -30,7 +30,7 @@
 /research/lfads/ @jazcollins @susillo
 /research/lm_1b/ @oriolvinyals @panyx0718
 /research/lm_commonsense/ @thtrieu
-/research/lstm_object_detection/ @dreamdragon
+/research/lstm_object_detection/ @dreamdragon @masonliuw @yinxiaoli
 /research/marco/ @vincentvanhoucke
 /research/maskgan/ @a-dai
 /research/morph_net/ @gariel-google

research/lstm_object_detection/README

@@ -10,4 +10,7 @@ http://openaccess.thecvf.com/content_cvpr_2018/papers/Liu_Mobile_Video_Object_CV
   year={2018}
 }
 
-If you have any questions regarding this codebase, please contact menglong@google.com
\ No newline at end of file
+If you have any questions regarding this codebase, please contact us:
+masonliuw@gmail.com
+yinxiao@google.com
+menglong@google.com
\ No newline at end of file

research/lstm_object_detection/eval.py

@@ -33,7 +33,7 @@ from lstm_object_detection import evaluator
 from lstm_object_detection import model_builder
 from lstm_object_detection import seq_dataset_builder
 from lstm_object_detection.utils import config_util
-from google3.third_party.tensorflow_models.object_detection.utils import label_map_util
+from object_detection.utils import label_map_util
 
 tf.logging.set_verbosity(tf.logging.INFO)
 flags = tf.app.flags

research/lstm_object_detection/evaluator.py

@@ -23,11 +23,11 @@ DetectionModel.
 import logging
 import tensorflow as tf
 from lstm_object_detection.metrics import coco_evaluation_all_frames
-from google3.third_party.tensorflow_models.object_detection import eval_util
-from google3.third_party.tensorflow_models.object_detection.core import prefetcher
-from google3.third_party.tensorflow_models.object_detection.core import standard_fields as fields
-from google3.third_party.tensorflow_models.object_detection.metrics import coco_evaluation
-from google3.third_party.tensorflow_models.object_detection.utils import object_detection_evaluation
+from object_detection import eval_util
+from object_detection.core import prefetcher
+from object_detection.core import standard_fields as fields
+from object_detection.metrics import coco_evaluation
+from object_detection.utils import object_detection_evaluation
 
 
 # A dictionary of metric names to classes that implement the metric. The classes

research/lstm_object_detection/lstm/lstm_cells.py

@@ -15,11 +15,11 @@
 
 """BottleneckConvLSTMCell implementation."""
 
-import google3
-import tensorflow.google as tf
-import google3.learning.brain.contrib.slim as slim
+import tensorflow as tf
 from tensorflow.contrib.framework.python.ops import variables
 
+slim = tf.contrib.slim
+
 _batch_norm = tf.contrib.layers.batch_norm
 
 
@@ -195,4 +195,5 @@ class BottleneckConvLSTMCell(tf.contrib.rnn.RNNCell):
       ]
     for s, r in zip(state_size, ret_flat):
       r.set_shape([None] + s)
-    return tf.nest.pack_sequence_as(structure=[1, 1], flat_sequence=ret_flat)
+    return tf.contrib.framework.nest.pack_sequence_as(
+        structure=[1, 1], flat_sequence=ret_flat)

research/lstm_object_detection/lstm/lstm_meta_arch.py

@@ -23,11 +23,11 @@ See https://arxiv.org/abs/1711.06368 for details.
 import re
 import tensorflow as tf
 
-from google3.third_party.tensorflow_models.object_detection.core import box_list_ops
-from google3.third_party.tensorflow_models.object_detection.core import standard_fields as fields
-from google3.third_party.tensorflow_models.object_detection.meta_architectures import ssd_meta_arch
-from google3.third_party.tensorflow_models.object_detection.utils import ops
-from google3.third_party.tensorflow_models.object_detection.utils import shape_utils
+from object_detection.core import box_list_ops
+from object_detection.core import standard_fields as fields
+from object_detection.meta_architectures import ssd_meta_arch
+from object_detection.utils import ops
+from object_detection.utils import shape_utils
 
 slim = tf.contrib.slim
 

research/lstm_object_detection/metrics/coco_evaluation_all_frames.py

@@ -17,9 +17,9 @@
 
 import tensorflow as tf
 
-from google3.third_party.tensorflow_models.object_detection.core import standard_fields
-from google3.third_party.tensorflow_models.object_detection.metrics import coco_evaluation
-from google3.third_party.tensorflow_models.object_detection.metrics import coco_tools
+from object_detection.core import standard_fields
+from object_detection.metrics import coco_evaluation
+from object_detection.metrics import coco_tools
 
 
 class CocoEvaluationAllFrames(coco_evaluation.CocoDetectionEvaluator):

research/lstm_object_detection/metrics/coco_evaluation_all_frames_test.py

@@ -18,7 +18,7 @@
 import numpy as np
 import tensorflow as tf
 from lstm_object_detection.metrics import coco_evaluation_all_frames
-from google3.third_party.tensorflow_models.object_detection.core import standard_fields
+from object_detection.core import standard_fields
 
 
 class CocoEvaluationAllFramesTest(tf.test.TestCase):

research/lstm_object_detection/model_builder.py

@@ -16,17 +16,17 @@
 """A function to build a DetectionModel from configuration."""
 from lstm_object_detection.lstm import lstm_meta_arch
 from lstm_object_detection.models.lstm_ssd_mobilenet_v1_feature_extractor import LSTMMobileNetV1FeatureExtractor
-from google3.third_party.tensorflow_models.object_detection.builders import anchor_generator_builder
-from google3.third_party.tensorflow_models.object_detection.builders import box_coder_builder
-from google3.third_party.tensorflow_models.object_detection.builders import box_predictor_builder
-from google3.third_party.tensorflow_models.object_detection.builders import hyperparams_builder
-from google3.third_party.tensorflow_models.object_detection.builders import image_resizer_builder
-from google3.third_party.tensorflow_models.object_detection.builders import losses_builder
-from google3.third_party.tensorflow_models.object_detection.builders import matcher_builder
-from google3.third_party.tensorflow_models.object_detection.builders import model_builder
-from google3.third_party.tensorflow_models.object_detection.builders import post_processing_builder
-from google3.third_party.tensorflow_models.object_detection.builders import region_similarity_calculator_builder as sim_calc
-from google3.third_party.tensorflow_models.object_detection.core import target_assigner
+from object_detection.builders import anchor_generator_builder
+from object_detection.builders import box_coder_builder
+from object_detection.builders import box_predictor_builder
+from object_detection.builders import hyperparams_builder
+from object_detection.builders import image_resizer_builder
+from object_detection.builders import losses_builder
+from object_detection.builders import matcher_builder
+from object_detection.builders import model_builder
+from object_detection.builders import post_processing_builder
+from object_detection.builders import region_similarity_calculator_builder as sim_calc
+from object_detection.core import target_assigner
 
 model_builder.SSD_FEATURE_EXTRACTOR_CLASS_MAP.update({
     'lstm_mobilenet_v1': LSTMMobileNetV1FeatureExtractor,
@@ -125,7 +125,7 @@ def _build_lstm_model(ssd_config, lstm_config, is_training):
   non_max_suppression_fn, score_conversion_fn = post_processing_builder.build(
       ssd_config.post_processing)
   (classification_loss, localization_loss, classification_weight,
-   localization_weight, miner, _) = losses_builder.build(ssd_config.loss)
+   localization_weight, miner, _, _) = losses_builder.build(ssd_config.loss)
 
   normalize_loss_by_num_matches = ssd_config.normalize_loss_by_num_matches
   encode_background_as_zeros = ssd_config.encode_background_as_zeros

research/lstm_object_detection/model_builder_test.py

@@ -20,7 +20,7 @@ from google.protobuf import text_format
 from lstm_object_detection import model_builder
 from lstm_object_detection.lstm import lstm_meta_arch
 from lstm_object_detection.protos import pipeline_pb2 as internal_pipeline_pb2
-from google3.third_party.tensorflow_models.object_detection.protos import pipeline_pb2
+from object_detection.protos import pipeline_pb2
 
 
 class ModelBuilderTest(tf.test.TestCase):

research/lstm_object_detection/models/lstm_ssd_mobilenet_v1_feature_extractor.py

@@ -20,10 +20,10 @@ from tensorflow.python.framework import ops as tf_ops
 from lstm_object_detection.lstm import lstm_cells
 from lstm_object_detection.lstm import lstm_meta_arch
 from lstm_object_detection.lstm import rnn_decoder
-from google3.third_party.tensorflow_models.object_detection.models import feature_map_generators
-from google3.third_party.tensorflow_models.object_detection.utils import context_manager
-from google3.third_party.tensorflow_models.object_detection.utils import ops
-from google3.third_party.tensorflow_models.object_detection.utils import shape_utils
+from object_detection.models import feature_map_generators
+from object_detection.utils import context_manager
+from object_detection.utils import ops
+from object_detection.utils import shape_utils
 from nets import mobilenet_v1
 
 slim = tf.contrib.slim

research/lstm_object_detection/models/lstm_ssd_mobilenet_v1_feature_extractor_test.py

@@ -19,7 +19,7 @@ import numpy as np
 import tensorflow as tf
 
 from lstm_object_detection.models import lstm_ssd_mobilenet_v1_feature_extractor as feature_extactor
-from google3.third_party.tensorflow_models.object_detection.models import ssd_feature_extractor_test
+from object_detection.models import ssd_feature_extractor_test
 
 slim = tf.contrib.slim
 

research/lstm_object_detection/protos/input_reader_google.proto

@@ -2,7 +2,7 @@ syntax = "proto2";
 
 package lstm_object_detection.input_readers;
 
-import "third_party/tensorflow_models/object_detection/protos/input_reader.proto";
+import "object_detection/protos/input_reader.proto";
 
 message GoogleInputReader {
   extend object_detection.protos.ExternalInputReader {

research/lstm_object_detection/protos/pipeline.proto

@@ -2,7 +2,7 @@ syntax = "proto2";
 
 package object_detection.protos;
 
-import "third_party/tensorflow_models/object_detection/protos/pipeline.proto";
+import "object_detection/protos/pipeline.proto";
 
 extend TrainEvalPipelineConfig {
   optional LstmModel lstm_model = 205743444;

research/lstm_object_detection/seq_dataset_builder.py

@@ -24,16 +24,16 @@ that wraps the build function.
 """
 import tensorflow as tf
 import tensorflow.google as google_tf
-from google3.learning.brain.contrib.slim.data import parallel_reader
 from tensorflow.contrib.training.python.training import sequence_queueing_state_saver as sqss
 from lstm_object_detection import tf_sequence_example_decoder
 from lstm_object_detection.protos import input_reader_google_pb2
-from google3.third_party.tensorflow_models.object_detection.core import preprocessor
-from google3.third_party.tensorflow_models.object_detection.core import preprocessor_cache
-from google3.third_party.tensorflow_models.object_detection.core import standard_fields as fields
-from google3.third_party.tensorflow_models.object_detection.protos import input_reader_pb2
-from google3.third_party.tensorflow_models.object_detection.utils import ops as util_ops
+from object_detection.core import preprocessor
+from object_detection.core import preprocessor_cache
+from object_detection.core import standard_fields as fields
+from object_detection.protos import input_reader_pb2
+from object_detection.utils import ops as util_ops
 
+parallel_reader = tf.contrib.slim.parallel_reader
 # TODO(yinxiao): Make the following variable into configurable proto.
 # Padding size for the labeled objects in each frame. Here we assume each
 # frame has a total number of objects less than _PADDING_SIZE.
@@ -142,7 +142,7 @@ def build(input_reader_config,
       min_after_dequeue=input_reader_config.min_after_dequeue)
 
   # TODO(yinxiao): Add loading instance mask option.
-  decoder = tf_sequence_example_decoder.TfSequenceExampleDecoder()
+  decoder = tf_sequence_example_decoder.TFSequenceExampleDecoder()
 
   keys_to_decode = [
       fields.InputDataFields.image, fields.InputDataFields.groundtruth_boxes,

research/lstm_object_detection/seq_dataset_builder_test.py

@@ -25,11 +25,11 @@ from tensorflow.core.example import example_pb2
 from tensorflow.core.example import feature_pb2
 from lstm_object_detection import seq_dataset_builder
 from lstm_object_detection.protos import pipeline_pb2 as internal_pipeline_pb2
-from google3.third_party.tensorflow_models.object_detection.builders import preprocessor_builder
-from google3.third_party.tensorflow_models.object_detection.core import standard_fields as fields
-from google3.third_party.tensorflow_models.object_detection.protos import input_reader_pb2
-from google3.third_party.tensorflow_models.object_detection.protos import pipeline_pb2
-from google3.third_party.tensorflow_models.object_detection.protos import preprocessor_pb2
+from object_detection.builders import preprocessor_builder
+from object_detection.core import standard_fields as fields
+from object_detection.protos import input_reader_pb2
+from object_detection.protos import pipeline_pb2
+from object_detection.protos import preprocessor_pb2
 
 
 class DatasetBuilderTest(parameterized.TestCase):

research/lstm_object_detection/tf_sequence_example_decoder.py

@@ -21,14 +21,74 @@ TODO(yinxiao): When TensorFlow object detection API officially supports
 tensorflow.SequenceExample, merge this decoder.
 """
 import tensorflow as tf
-from google3.learning.brain.contrib.slim.data import tfexample_decoder
-from google3.third_party.tensorflow_models.object_detection.core import data_decoder
-from google3.third_party.tensorflow_models.object_detection.core import standard_fields as fields
+from object_detection.core import data_decoder
+from object_detection.core import standard_fields as fields
 
-slim_example_decoder = tf.contrib.slim.tfexample_decoder
+tfexample_decoder = tf.contrib.slim.tfexample_decoder
 
 
-class TfSequenceExampleDecoder(data_decoder.DataDecoder):
+class BoundingBoxSequence(tfexample_decoder.ItemHandler):
+  """An ItemHandler that concatenates SparseTensors to Bounding Boxes.
+  """
+
+  def __init__(self, keys=None, prefix=None, return_dense=True,
+               default_value=-1.0):
+    """Initialize the bounding box handler.
+
+    Args:
+      keys: A list of four key names representing the ymin, xmin, ymax, xmax
+        in the Example or SequenceExample.
+      prefix: An optional prefix for each of the bounding box keys in the
+        Example or SequenceExample. If provided, `prefix` is prepended to each
+        key in `keys`.
+      return_dense: if True, returns a dense tensor; if False, returns as
+        sparse tensor.
+      default_value: The value used when the `tensor_key` is not found in a
+        particular `TFExample`.
+
+    Raises:
+      ValueError: if keys is not `None` and also not a list of exactly 4 keys
+    """
+    if keys is None:
+      keys = ['ymin', 'xmin', 'ymax', 'xmax']
+    elif len(keys) != 4:
+      raise ValueError('BoundingBoxSequence expects 4 keys but got {}'.format(
+          len(keys)))
+    self._prefix = prefix
+    self._keys = keys
+    self._full_keys = [prefix + k for k in keys]
+    self._return_dense = return_dense
+    self._default_value = default_value
+    super(BoundingBoxSequence, self).__init__(self._full_keys)
+
+  def tensors_to_item(self, keys_to_tensors):
+    """Maps the given dictionary of tensors to a concatenated list of bboxes.
+
+    Args:
+      keys_to_tensors: a mapping of TF-Example keys to parsed tensors.
+
+    Returns:
+      [time, num_boxes, 4] tensor of bounding box coordinates, in order
+          [y_min, x_min, y_max, x_max]. Whether the tensor is a SparseTensor
+          or a dense Tensor is determined by the return_dense parameter. Empty
+          positions in the sparse tensor are filled with -1.0 values.
+    """
+    sides = []
+    for key in self._full_keys:
+      value = keys_to_tensors[key]
+      expanded_dims = tf.concat(
+          [tf.to_int64(tf.shape(value)),
+           tf.constant([1], dtype=tf.int64)], 0)
+      side = tf.sparse_reshape(value, expanded_dims)
+      sides.append(side)
+    bounding_boxes = tf.sparse_concat(2, sides)
+    if self._return_dense:
+      bounding_boxes = tf.sparse_tensor_to_dense(
+          bounding_boxes, default_value=self._default_value)
+    return bounding_boxes
+
+
+class TFSequenceExampleDecoder(data_decoder.DataDecoder):
   """Tensorflow Sequence Example proto decoder."""
 
   def __init__(self):
@@ -62,30 +122,30 @@ class TfSequenceExampleDecoder(data_decoder.DataDecoder):
     }
     self.items_to_handlers = {
         fields.InputDataFields.image:
-            slim_example_decoder.Image(
+            tfexample_decoder.Image(
                 image_key='image/encoded',
                 format_key='image/format',
                 channels=3,
                 repeated=True),
         fields.InputDataFields.source_id: (
-            slim_example_decoder.Tensor('image/source_id')),
+            tfexample_decoder.Tensor('image/source_id')),
         fields.InputDataFields.key: (
-            slim_example_decoder.Tensor('image/key/sha256')),
+            tfexample_decoder.Tensor('image/key/sha256')),
         fields.InputDataFields.filename: (
-            slim_example_decoder.Tensor('image/filename')),
+            tfexample_decoder.Tensor('image/filename')),
         # Object boxes and classes.
         fields.InputDataFields.groundtruth_boxes:
-            tfexample_decoder.BoundingBoxSequence(prefix='bbox/'),
+            BoundingBoxSequence(prefix='bbox/'),
         fields.InputDataFields.groundtruth_classes: (
-            slim_example_decoder.Tensor('bbox/label/index')),
+            tfexample_decoder.Tensor('bbox/label/index')),
         fields.InputDataFields.groundtruth_area:
-            slim_example_decoder.Tensor('area'),
+            tfexample_decoder.Tensor('area'),
         fields.InputDataFields.groundtruth_is_crowd: (
-            slim_example_decoder.Tensor('is_crowd')),
+            tfexample_decoder.Tensor('is_crowd')),
         fields.InputDataFields.groundtruth_difficult: (
-            slim_example_decoder.Tensor('difficult')),
+            tfexample_decoder.Tensor('difficult')),
         fields.InputDataFields.groundtruth_group_of: (
-            slim_example_decoder.Tensor('group_of'))
+            tfexample_decoder.Tensor('group_of'))
     }
 
   def decode(self, tf_seq_example_string_tensor, items=None):

research/lstm_object_detection/tf_sequence_example_decoder_test.py

@@ -13,17 +13,19 @@
 # limitations under the License.
 # ==============================================================================
 
-"""Tests for video_object_detection.tf_sequence_example_decoder."""
+"""Tests for lstm_object_detection.tf_sequence_example_decoder."""
 
 import numpy as np
 import tensorflow as tf
 from tensorflow.core.example import example_pb2
 from tensorflow.core.example import feature_pb2
+from tensorflow.python.framework import dtypes
+from tensorflow.python.ops import parsing_ops
 from lstm_object_detection import tf_sequence_example_decoder
-from google3.third_party.tensorflow_models.object_detection.core import standard_fields as fields
+from object_detection.core import standard_fields as fields
 
 
-class TfSequenceExampleDecoderTest(tf.test.TestCase):
+class TFSequenceExampleDecoderTest(tf.test.TestCase):
   """Tests for sequence example decoder."""
 
   def _EncodeImage(self, image_tensor, encoding_type='jpeg'):
@@ -86,7 +88,7 @@ class TfSequenceExampleDecoderTest(tf.test.TestCase):
                     ]),
             })).SerializeToString()
 
-    example_decoder = tf_sequence_example_decoder.TfSequenceExampleDecoder()
+    example_decoder = tf_sequence_example_decoder.TFSequenceExampleDecoder()
     tensor_dict = example_decoder.decode(tf.convert_to_tensor(sequence_example))
 
     # Test tensor dict image dimension.

research/lstm_object_detection/train.py

@@ -51,7 +51,7 @@ from lstm_object_detection import model_builder
 from lstm_object_detection import seq_dataset_builder
 from lstm_object_detection import trainer
 from lstm_object_detection.utils import config_util
-from google3.third_party.tensorflow_models.object_detection.builders import preprocessor_builder
+from object_detection.builders import preprocessor_builder
 
 flags.DEFINE_string('master', '', 'Name of the TensorFlow master to use.')
 flags.DEFINE_integer('task', 0, 'task id')

research/lstm_object_detection/trainer.py

@@ -23,10 +23,10 @@ import functools
 import tensorflow as tf
 from google3.pyglib import logging
 
-from google3.third_party.tensorflow_models.object_detection.builders import optimizer_builder
-from google3.third_party.tensorflow_models.object_detection.core import standard_fields as fields
-from google3.third_party.tensorflow_models.object_detection.utils import ops as util_ops
-from google3.third_party.tensorflow_models.object_detection.utils import variables_helper
+from object_detection.builders import optimizer_builder
+from object_detection.core import standard_fields as fields
+from object_detection.utils import ops as util_ops
+from object_detection.utils import variables_helper
 from deployment import model_deploy
 
 slim = tf.contrib.slim