Commit b968a6ce authored by Mark Sandler's avatar Mark Sandler Committed by Jonathan Huang
Browse files

Merged commit includes the following changes: (#7800)

280142968  by Zhichao Lu:

    Opensource MobilenetEdgeTPU + ssdlite into third-party object detection APIs on EdgeTPU.

--
280134001  by Zhichao Lu:

    Adds MobilenetEdgeTpu + ssdlite into internal object detection APIs on EdgeTPU.

--
278941778  by Zhichao Lu:

    Add support for fixed input shapes for 'encoded_image_string_tensor' and 'tf_example' inputs.

--
278933274  by Zhichao Lu:

      Adding fool proof check to avoid using 1x1 depthwise conv op.

--
278762192  by Zhichao Lu:

    Ensure correct number of iterations after training resumes.

--
278746440  by Zhichao Lu:

    Internal change.

--
278006953  by Zhichao Lu:

    Internal changes to tf.contrib symbols

--
278006330  by Zhichao Lu:

    Internal changes to tf.contrib symbols

--
277593959  by Zhichao Lu:

      Make the ssd_feature_extractor_test.py PY3 compatible. The "six.zip" will use "itertools.izip" in Python 2 and "zip" in Python 3.

--
277344551  by Zhichao Lu:

    Internal change.

--
277154953  by Zhichao Lu:

    Conditionally use keras based optimizers so that check-pointing works correctly.
    This change also enables summaries on TPU which were previously not enabled
    due to a bug.

--
277087572  by Zhichao Lu:

    Fix resizing boxes when using keep_aspect_ratio_rezier with padding.

--
275898543  by Zhichao Lu:

    Support label_map_proto as input in label_map_util.

--
275347137  by Zhichao Lu:

    Add force_no_resize flag in eval.proto which replaces
    the resize config with identity resizer. This is useful
    when we want to test at the original image resolution.

--

PiperOrigin-RevId: 280142968
parent c3bd5082
......@@ -135,7 +135,6 @@ def eager_train_step(detection_model,
learning_rate,
add_regularization_loss=True,
clip_gradients_value=None,
use_tpu=False,
global_step=None,
num_replicas=1.0):
"""Process a single training batch.
......@@ -192,7 +191,6 @@ def eager_train_step(detection_model,
regularization loss in the losses dictionary.
clip_gradients_value: If this is present, clip the gradients global norm
at this value using `tf.clip_by_global_norm`.
use_tpu: Whether computation should happen on a TPU.
global_step: The current training step. Used for TensorBoard logging
purposes. This step is not updated by this function and must be
incremented separately.
......@@ -223,10 +221,9 @@ def eager_train_step(detection_model,
tf.constant(num_replicas, dtype=tf.float32))
losses_dict['Loss/normalized_total_loss'] = total_loss
if not use_tpu:
for loss_type in losses_dict:
tf.compat.v2.summary.scalar(
loss_type, losses_dict[loss_type], step=global_step)
for loss_type in losses_dict:
tf.compat.v2.summary.scalar(
loss_type, losses_dict[loss_type], step=global_step)
trainable_variables = detection_model.trainable_variables
......@@ -235,10 +232,7 @@ def eager_train_step(detection_model,
if clip_gradients_value:
gradients, _ = tf.clip_by_global_norm(gradients, clip_gradients_value)
optimizer.apply_gradients(zip(gradients, trainable_variables))
if not use_tpu:
tf.compat.v2.summary.scalar('learning_rate', learning_rate,
step=global_step)
tf.compat.v2.summary.scalar('learning_rate', learning_rate, step=global_step)
return total_loss
......@@ -451,11 +445,10 @@ def train_loop(
unpad_groundtruth_tensors)
ckpt = tf.compat.v2.train.Checkpoint(
step=global_step, model=detection_model)
step=global_step, model=detection_model, optimizer=optimizer)
manager = tf.compat.v2.train.CheckpointManager(
ckpt, model_dir, max_to_keep=7)
## Maybe re-enable checkpoint restoration depending on how it works:
# ckpt.restore(manager.latest_checkpoint)
ckpt.restore(manager.latest_checkpoint)
def train_step_fn(features, labels):
return eager_train_step(
......@@ -467,7 +460,6 @@ def train_loop(
learning_rate=learning_rate_fn(),
add_regularization_loss=add_regularization_loss,
clip_gradients_value=clip_gradients_value,
use_tpu=use_tpu,
global_step=global_step,
num_replicas=strategy.num_replicas_in_sync)
......@@ -487,19 +479,22 @@ def train_loop(
return mean_loss
train_input_iter = iter(train_input)
for _ in range(train_steps):
for _ in range(train_steps - global_step.value()):
start_time = time.time()
loss = _dist_train_step(train_input_iter)
global_step.assign_add(1)
end_time = time.time()
if not use_tpu:
tf.compat.v2.summary.scalar(
'steps_per_sec', 1.0 / (end_time - start_time), step=global_step)
# TODO(kaftan): Remove this print after it is no longer helpful for
## debugging.
print('Finished step', global_step, end_time, loss)
if int(global_step.value().numpy()) % checkpoint_every_n == 0:
tf.compat.v2.summary.scalar(
'steps_per_sec', 1.0 / (end_time - start_time),
step=global_step)
if (int(global_step.value()) % 100) == 0:
tf.logging.info(
'Step {} time taken {:.3f}s loss={:.3f}'.format(
global_step.value(), end_time - start_time, loss))
if int(global_step.value()) % checkpoint_every_n == 0:
manager.save()
......@@ -620,14 +615,11 @@ def eager_eval_loop(
return eval_dict, losses_dict, class_agnostic
i = 0
for features, labels in eval_dataset:
for i, (features, labels) in enumerate(eval_dataset):
eval_dict, losses_dict, class_agnostic = compute_eval_dict(features, labels)
end_time = time.time()
# TODO(kaftan): Remove this print after it is no longer helpful for
## debugging.
tf.print('Finished eval dict computation', i, end_time)
i += 1
if i % 100 == 0:
tf.logging.info('Finished eval step %d', i)
if evaluators is None:
if class_agnostic:
......
......@@ -16,6 +16,7 @@
"""Embedded-friendly SSDFeatureExtractor for MobilenetV1 features."""
import tensorflow as tf
from tensorflow.contrib import slim as contrib_slim
from object_detection.meta_architectures import ssd_meta_arch
from object_detection.models import feature_map_generators
......@@ -23,7 +24,7 @@ from object_detection.utils import context_manager
from object_detection.utils import ops
from nets import mobilenet_v1
slim = tf.contrib.slim
slim = contrib_slim
class EmbeddedSSDMobileNetV1FeatureExtractor(ssd_meta_arch.SSDFeatureExtractor):
......
......@@ -23,12 +23,13 @@ Huang et al. (https://arxiv.org/abs/1611.10012)
"""
import tensorflow as tf
from tensorflow.contrib import slim as contrib_slim
from object_detection.meta_architectures import faster_rcnn_meta_arch
from object_detection.utils import variables_helper
from nets import inception_resnet_v2
slim = tf.contrib.slim
slim = contrib_slim
class FasterRCNNInceptionResnetV2FeatureExtractor(
......@@ -211,4 +212,3 @@ class FasterRCNNInceptionResnetV2FeatureExtractor(
second_stage_feature_extractor_scope + '/', '')
variables_to_restore[var_name] = variable
return variables_to_restore
......@@ -19,11 +19,12 @@ See "Rethinking the Inception Architecture for Computer Vision"
https://arxiv.org/abs/1512.00567
"""
import tensorflow as tf
from tensorflow.contrib import slim as contrib_slim
from object_detection.meta_architectures import faster_rcnn_meta_arch
from nets import inception_v2
slim = tf.contrib.slim
slim = contrib_slim
def _batch_norm_arg_scope(list_ops,
......
......@@ -17,12 +17,13 @@
import numpy as np
import tensorflow as tf
from tensorflow.contrib import slim as contrib_slim
from object_detection.meta_architectures import faster_rcnn_meta_arch
from object_detection.utils import shape_utils
from nets import mobilenet_v1
slim = tf.contrib.slim
slim = contrib_slim
def _get_mobilenet_conv_no_last_stride_defs(conv_depth_ratio_in_percentage):
......
......@@ -21,14 +21,16 @@ https://arxiv.org/abs/1707.07012
"""
import tensorflow as tf
from tensorflow.contrib import framework as contrib_framework
from tensorflow.contrib import slim as contrib_slim
from object_detection.meta_architectures import faster_rcnn_meta_arch
from object_detection.utils import variables_helper
from nets.nasnet import nasnet
from nets.nasnet import nasnet_utils
arg_scope = tf.contrib.framework.arg_scope
slim = tf.contrib.slim
arg_scope = contrib_framework.arg_scope
slim = contrib_slim
def nasnet_large_arg_scope_for_detection(is_batch_norm_training=False):
......@@ -322,4 +324,3 @@ class FasterRCNNNASFeatureExtractor(
var_name += '/ExponentialMovingAverage'
variables_to_restore[var_name] = variable
return variables_to_restore
......@@ -19,14 +19,16 @@ Based on PNASNet model: https://arxiv.org/abs/1712.00559
"""
import tensorflow as tf
from tensorflow.contrib import framework as contrib_framework
from tensorflow.contrib import slim as contrib_slim
from object_detection.meta_architectures import faster_rcnn_meta_arch
from object_detection.utils import variables_helper
from nets.nasnet import nasnet_utils
from nets.nasnet import pnasnet
arg_scope = tf.contrib.framework.arg_scope
slim = tf.contrib.slim
arg_scope = contrib_framework.arg_scope
slim = contrib_slim
def pnasnet_large_arg_scope_for_detection(is_batch_norm_training=False):
......
......@@ -25,12 +25,13 @@ the MSRA provided checkpoints
same preprocessing, batch norm scaling, etc.
"""
import tensorflow as tf
from tensorflow.contrib import slim as contrib_slim
from object_detection.meta_architectures import faster_rcnn_meta_arch
from nets import resnet_utils
from nets import resnet_v1
slim = tf.contrib.slim
slim = contrib_slim
class FasterRCNNResnetV1FeatureExtractor(
......
......@@ -26,9 +26,10 @@ of final feature maps.
import collections
import functools
import tensorflow as tf
from tensorflow.contrib import slim as contrib_slim
from object_detection.utils import ops
from object_detection.utils import shape_utils
slim = tf.contrib.slim
slim = contrib_slim
# Activation bound used for TPU v1. Activations will be clipped to
# [-ACTIVATION_BOUND, ACTIVATION_BOUND] when training with
......
......@@ -17,11 +17,12 @@
from abc import abstractmethod
import itertools
import numpy as np
from six.moves import zip
import tensorflow as tf
from google.protobuf import text_format
from tensorflow.contrib import slim as contrib_slim
from object_detection.builders import hyperparams_builder
from object_detection.protos import hyperparams_pb2
from object_detection.utils import test_case
......@@ -53,7 +54,7 @@ class SsdFeatureExtractorTestBase(test_case.TestCase):
return hyperparams_builder.KerasLayerHyperparams(conv_hyperparams)
def conv_hyperparams_fn(self):
with tf.contrib.slim.arg_scope([]) as sc:
with contrib_slim.arg_scope([]) as sc:
return sc
@abstractmethod
......@@ -135,7 +136,7 @@ class SsdFeatureExtractorTestBase(test_case.TestCase):
image_tensor = np.random.rand(batch_size, image_height, image_width,
3).astype(np.float32)
feature_maps = self.execute(graph_fn, [image_tensor])
for feature_map, expected_shape in itertools.izip(
for feature_map, expected_shape in zip(
feature_maps, expected_feature_map_shapes):
self.assertAllEqual(feature_map.shape, expected_shape)
......@@ -168,7 +169,7 @@ class SsdFeatureExtractorTestBase(test_case.TestCase):
np.array(image_height, dtype=np.int32),
np.array(image_width, dtype=np.int32)
])
for feature_map, expected_shape in itertools.izip(
for feature_map, expected_shape in zip(
feature_maps, expected_feature_map_shapes):
self.assertAllEqual(feature_map.shape, expected_shape)
......
......@@ -15,6 +15,7 @@
"""SSDFeatureExtractor for InceptionV2 features."""
import tensorflow as tf
from tensorflow.contrib import slim as contrib_slim
from object_detection.meta_architectures import ssd_meta_arch
from object_detection.models import feature_map_generators
......@@ -22,7 +23,7 @@ from object_detection.utils import ops
from object_detection.utils import shape_utils
from nets import inception_v2
slim = tf.contrib.slim
slim = contrib_slim
class SSDInceptionV2FeatureExtractor(ssd_meta_arch.SSDFeatureExtractor):
......
......@@ -15,6 +15,7 @@
"""SSDFeatureExtractor for InceptionV3 features."""
import tensorflow as tf
from tensorflow.contrib import slim as contrib_slim
from object_detection.meta_architectures import ssd_meta_arch
from object_detection.models import feature_map_generators
......@@ -22,7 +23,7 @@ from object_detection.utils import ops
from object_detection.utils import shape_utils
from nets import inception_v3
slim = tf.contrib.slim
slim = contrib_slim
class SSDInceptionV3FeatureExtractor(ssd_meta_arch.SSDFeatureExtractor):
......
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""SSDFeatureExtractor for MobileNetEdgeTPU features."""
import tensorflow as tf
from object_detection.models import ssd_mobilenet_v3_feature_extractor
from nets.mobilenet import mobilenet_v3
slim = tf.contrib.slim
class SSDMobileNetEdgeTPUFeatureExtractor(
ssd_mobilenet_v3_feature_extractor.SSDMobileNetV3FeatureExtractorBase):
"""MobileNetEdgeTPU feature extractor."""
def __init__(self,
is_training,
depth_multiplier,
min_depth,
pad_to_multiple,
conv_hyperparams_fn,
reuse_weights=None,
use_explicit_padding=False,
use_depthwise=False,
override_base_feature_extractor_hyperparams=False,
scope_name='MobilenetEdgeTPU'):
super(SSDMobileNetEdgeTPUFeatureExtractor, self).__init__(
conv_defs=mobilenet_v3.V3_EDGETPU,
from_layer=['layer_18/expansion_output', 'layer_23'],
is_training=is_training,
depth_multiplier=depth_multiplier,
min_depth=min_depth,
pad_to_multiple=pad_to_multiple,
conv_hyperparams_fn=conv_hyperparams_fn,
reuse_weights=reuse_weights,
use_explicit_padding=use_explicit_padding,
use_depthwise=use_depthwise,
override_base_feature_extractor_hyperparams=override_base_feature_extractor_hyperparams,
scope_name=scope_name
)
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for ssd_mobilenet_edgetpu_feature_extractor."""
import tensorflow as tf
from tensorflow.contrib import slim as contrib_slim
from object_detection.models import ssd_mobilenet_edgetpu_feature_extractor
from object_detection.models import ssd_mobilenet_edgetpu_feature_extractor_testbase
slim = contrib_slim
class SsdMobilenetEdgeTPUFeatureExtractorTest(
ssd_mobilenet_edgetpu_feature_extractor_testbase
._SsdMobilenetEdgeTPUFeatureExtractorTestBase):
def _get_input_sizes(self):
"""Return first two input feature map sizes."""
return [384, 192]
def _create_feature_extractor(self,
depth_multiplier,
pad_to_multiple,
use_explicit_padding=False,
use_keras=False):
"""Constructs a new MobileNetEdgeTPU feature extractor.
Args:
depth_multiplier: float depth multiplier for feature extractor
pad_to_multiple: the nearest multiple to zero pad the input height and
width dimensions to.
use_explicit_padding: use 'VALID' padding for convolutions, but prepad
inputs so that the output dimensions are the same as if 'SAME' padding
were used.
use_keras: if True builds a keras-based feature extractor, if False builds
a slim-based one.
Returns:
an ssd_meta_arch.SSDFeatureExtractor object.
"""
min_depth = 32
return (ssd_mobilenet_edgetpu_feature_extractor
.SSDMobileNetEdgeTPUFeatureExtractor(
False,
depth_multiplier,
min_depth,
pad_to_multiple,
self.conv_hyperparams_fn,
use_explicit_padding=use_explicit_padding))
if __name__ == '__main__':
tf.test.main()
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Base test class for ssd_mobilenet_edgetpu_feature_extractor."""
import abc
import numpy as np
import tensorflow as tf
from object_detection.models import ssd_feature_extractor_test
slim = tf.contrib.slim
class _SsdMobilenetEdgeTPUFeatureExtractorTestBase(
ssd_feature_extractor_test.SsdFeatureExtractorTestBase):
"""Base class for MobilenetEdgeTPU tests."""
@abc.abstractmethod
def _get_input_sizes(self):
"""Return feature map sizes for the two inputs to SSD head."""
pass
def test_extract_features_returns_correct_shapes_128(self):
image_height = 128
image_width = 128
depth_multiplier = 1.0
pad_to_multiple = 1
input_feature_sizes = self._get_input_sizes()
expected_feature_map_shape = [(2, 8, 8, input_feature_sizes[0]),
(2, 4, 4, input_feature_sizes[1]),
(2, 2, 2, 512), (2, 1, 1, 256), (2, 1, 1,
256),
(2, 1, 1, 128)]
self.check_extract_features_returns_correct_shape(
2,
image_height,
image_width,
depth_multiplier,
pad_to_multiple,
expected_feature_map_shape,
use_keras=False)
def test_extract_features_returns_correct_shapes_299(self):
image_height = 299
image_width = 299
depth_multiplier = 1.0
pad_to_multiple = 1
input_feature_sizes = self._get_input_sizes()
expected_feature_map_shape = [(2, 19, 19, input_feature_sizes[0]),
(2, 10, 10, input_feature_sizes[1]),
(2, 5, 5, 512), (2, 3, 3, 256), (2, 2, 2,
256),
(2, 1, 1, 128)]
self.check_extract_features_returns_correct_shape(
2,
image_height,
image_width,
depth_multiplier,
pad_to_multiple,
expected_feature_map_shape,
use_keras=False)
def test_extract_features_returns_correct_shapes_with_pad_to_multiple(self):
image_height = 299
image_width = 299
depth_multiplier = 1.0
pad_to_multiple = 32
input_feature_sizes = self._get_input_sizes()
expected_feature_map_shape = [(2, 20, 20, input_feature_sizes[0]),
(2, 10, 10, input_feature_sizes[1]),
(2, 5, 5, 512), (2, 3, 3, 256), (2, 2, 2,
256),
(2, 1, 1, 128)]
self.check_extract_features_returns_correct_shape(
2, image_height, image_width, depth_multiplier, pad_to_multiple,
expected_feature_map_shape)
def test_preprocess_returns_correct_value_range(self):
image_height = 128
image_width = 128
depth_multiplier = 1
pad_to_multiple = 1
test_image = np.random.rand(4, image_height, image_width, 3)
feature_extractor = self._create_feature_extractor(
depth_multiplier, pad_to_multiple, use_keras=False)
preprocessed_image = feature_extractor.preprocess(test_image)
self.assertTrue(np.all(np.less_equal(np.abs(preprocessed_image), 1.0)))
def test_has_fused_batchnorm(self):
image_height = 128
image_width = 128
depth_multiplier = 1
pad_to_multiple = 1
image_placeholder = tf.placeholder(tf.float32,
[1, image_height, image_width, 3])
feature_extractor = self._create_feature_extractor(
depth_multiplier, pad_to_multiple, use_keras=False)
preprocessed_image = feature_extractor.preprocess(image_placeholder)
_ = feature_extractor.extract_features(preprocessed_image)
self.assertTrue(any('FusedBatchNorm' in op.type
for op in tf.get_default_graph().get_operations()))
......@@ -16,6 +16,7 @@
"""SSDFeatureExtractor for MobilenetV1 features."""
import tensorflow as tf
from tensorflow.contrib import slim as contrib_slim
from object_detection.meta_architectures import ssd_meta_arch
from object_detection.models import feature_map_generators
......@@ -24,7 +25,7 @@ from object_detection.utils import ops
from object_detection.utils import shape_utils
from nets import mobilenet_v1
slim = tf.contrib.slim
slim = contrib_slim
class SSDMobileNetV1FeatureExtractor(ssd_meta_arch.SSDFeatureExtractor):
......
......@@ -21,12 +21,13 @@ from absl.testing import parameterized
import numpy as np
import tensorflow as tf
from tensorflow.contrib import slim as contrib_slim
from object_detection.models import ssd_feature_extractor_test
from object_detection.models import ssd_mobilenet_v1_feature_extractor
from object_detection.models import ssd_mobilenet_v1_keras_feature_extractor
slim = tf.contrib.slim
slim = contrib_slim
@parameterized.parameters(
......
......@@ -18,6 +18,7 @@
import copy
import functools
import tensorflow as tf
from tensorflow.contrib import slim as contrib_slim
from object_detection.meta_architectures import ssd_meta_arch
from object_detection.models import feature_map_generators
......@@ -26,7 +27,7 @@ from object_detection.utils import ops
from object_detection.utils import shape_utils
from nets import mobilenet_v1
slim = tf.contrib.slim
slim = contrib_slim
# A modified config of mobilenet v1 that makes it more detection friendly,
......
......@@ -21,12 +21,13 @@ Keras-based Mobilenet V1 FPN feature extractors in SSD.
from absl.testing import parameterized
import numpy as np
import tensorflow as tf
from tensorflow.contrib import slim as contrib_slim
from object_detection.models import ssd_feature_extractor_test
from object_detection.models import ssd_mobilenet_v1_fpn_feature_extractor
from object_detection.models import ssd_mobilenet_v1_fpn_keras_feature_extractor
slim = tf.contrib.slim
slim = contrib_slim
@parameterized.parameters(
......
......@@ -16,6 +16,7 @@
"""SSDFeatureExtractor for Keras MobilenetV1 features."""
import tensorflow as tf
from tensorflow.contrib import slim as contrib_slim
from object_detection.meta_architectures import ssd_meta_arch
from object_detection.models import feature_map_generators
......@@ -23,7 +24,7 @@ from object_detection.models.keras_models import mobilenet_v1
from object_detection.utils import ops
from object_detection.utils import shape_utils
slim = tf.contrib.slim
slim = contrib_slim
class SSDMobileNetV1KerasFeatureExtractor(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment