Merged commit includes the following changes: (#6407)

233991726  by Sergio Guadarrama:

    Internal change

231925959  by Sergio Guadarrama:

    Internal change

231253502  by Sergio Guadarrama:

    Internal change

229973546  by Sergio Guadarrama:

    Internal change

229870842  by Sergio Guadarrama:

    Internal change

PiperOrigin-RevId: 233991726

research/slim/BUILD

@@ -796,6 +796,12 @@ py_binary(
     srcs = ["export_inference_graph.py"],
     # WARNING: not supported in bazel; will be commented out by copybara.
     # paropts = ["--compress"],
+    deps = [":export_inference_graph_lib"],
+)
+
+py_library(
+    name = "export_inference_graph_lib",
+    srcs = ["export_inference_graph.py"],
     deps = [
         ":dataset_factory",
         ":nets_factory",
@@ -813,7 +819,7 @@ py_test(
         "manual",
     ],
     deps = [
-        ":export_inference_graph",
+        ":export_inference_graph_lib",
         # "//tensorflow",
         # "//tensorflow/python:platform",
     ],

research/slim/nets/inception_v1.py

@@ -28,6 +28,7 @@ trunc_normal = lambda stddev: tf.truncated_normal_initializer(0.0, stddev)
 
 def inception_v1_base(inputs,
                       final_endpoint='Mixed_5c',
+                      include_root_block=True,
                       scope='InceptionV1'):
   """Defines the Inception V1 base architecture.
 
@@ -43,7 +44,11 @@ def inception_v1_base(inputs,
       can be one of ['Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1',
       'Conv2d_2c_3x3', 'MaxPool_3a_3x3', 'Mixed_3b', 'Mixed_3c',
       'MaxPool_4a_3x3', 'Mixed_4b', 'Mixed_4c', 'Mixed_4d', 'Mixed_4e',
-      'Mixed_4f', 'MaxPool_5a_2x2', 'Mixed_5b', 'Mixed_5c']
+      'Mixed_4f', 'MaxPool_5a_2x2', 'Mixed_5b', 'Mixed_5c']. If
+      include_root_block is False, ['Conv2d_1a_7x7', 'MaxPool_2a_3x3',
+      'Conv2d_2b_1x1', 'Conv2d_2c_3x3', 'MaxPool_3a_3x3'] will not be available.
+    include_root_block: If True, include the convolution and max-pooling layers
+      before the inception modules. If False, excludes those layers.
     scope: Optional variable_scope.
 
   Returns:
@@ -59,26 +64,33 @@ def inception_v1_base(inputs,
         weights_initializer=trunc_normal(0.01)):
       with slim.arg_scope([slim.conv2d, slim.max_pool2d],
                           stride=1, padding='SAME'):
+        net = inputs
+        if include_root_block:
           end_point = 'Conv2d_1a_7x7'
           net = slim.conv2d(inputs, 64, [7, 7], stride=2, scope=end_point)
           end_points[end_point] = net
-        if final_endpoint == end_point: return net, end_points
+          if final_endpoint == end_point:
+            return net, end_points
           end_point = 'MaxPool_2a_3x3'
           net = slim.max_pool2d(net, [3, 3], stride=2, scope=end_point)
           end_points[end_point] = net
-        if final_endpoint == end_point: return net, end_points
+          if final_endpoint == end_point:
+            return net, end_points
           end_point = 'Conv2d_2b_1x1'
           net = slim.conv2d(net, 64, [1, 1], scope=end_point)
           end_points[end_point] = net
-        if final_endpoint == end_point: return net, end_points
+          if final_endpoint == end_point:
+            return net, end_points
           end_point = 'Conv2d_2c_3x3'
           net = slim.conv2d(net, 192, [3, 3], scope=end_point)
           end_points[end_point] = net
-        if final_endpoint == end_point: return net, end_points
+          if final_endpoint == end_point:
+            return net, end_points
           end_point = 'MaxPool_3a_3x3'
           net = slim.max_pool2d(net, [3, 3], stride=2, scope=end_point)
           end_points[end_point] = net
-        if final_endpoint == end_point: return net, end_points
+          if final_endpoint == end_point:
+            return net, end_points
 
         end_point = 'Mixed_3b'
         with tf.variable_scope(end_point):

research/slim/nets/inception_v1_test.py

@@ -95,7 +95,8 @@ class InceptionV1Test(tf.test.TestCase):
     inputs = tf.random_uniform((batch_size, height, width, 3))
     _, end_points = inception.inception_v1_base(inputs,
                                                 final_endpoint='Mixed_5c')
-    endpoints_shapes = {'Conv2d_1a_7x7': [5, 112, 112, 64],
+    endpoints_shapes = {
+        'Conv2d_1a_7x7': [5, 112, 112, 64],
         'MaxPool_2a_3x3': [5, 56, 56, 64],
         'Conv2d_2b_1x1': [5, 56, 56, 64],
         'Conv2d_2c_3x3': [5, 56, 56, 192],
@@ -110,7 +111,8 @@ class InceptionV1Test(tf.test.TestCase):
         'Mixed_4f': [5, 14, 14, 832],
         'MaxPool_5a_2x2': [5, 7, 7, 832],
         'Mixed_5b': [5, 7, 7, 832],
-                        'Mixed_5c': [5, 7, 7, 1024]}
+        'Mixed_5c': [5, 7, 7, 1024]
+    }
 
     self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys())
     for endpoint_name in endpoints_shapes:
@@ -139,6 +141,35 @@ class InceptionV1Test(tf.test.TestCase):
     self.assertListEqual(mixed_5c.get_shape().as_list(),
                          [batch_size, 4, 4, 1024])
 
+  def testBuildBaseNetworkWithoutRootBlock(self):
+    batch_size = 5
+    height, width = 28, 28
+    channels = 192
+
+    inputs = tf.random_uniform((batch_size, height, width, channels))
+    _, end_points = inception.inception_v1_base(
+        inputs, include_root_block=False)
+    endpoints_shapes = {
+        'Mixed_3b': [5, 28, 28, 256],
+        'Mixed_3c': [5, 28, 28, 480],
+        'MaxPool_4a_3x3': [5, 14, 14, 480],
+        'Mixed_4b': [5, 14, 14, 512],
+        'Mixed_4c': [5, 14, 14, 512],
+        'Mixed_4d': [5, 14, 14, 512],
+        'Mixed_4e': [5, 14, 14, 528],
+        'Mixed_4f': [5, 14, 14, 832],
+        'MaxPool_5a_2x2': [5, 7, 7, 832],
+        'Mixed_5b': [5, 7, 7, 832],
+        'Mixed_5c': [5, 7, 7, 1024]
+    }
+
+    self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys())
+    for endpoint_name in endpoints_shapes:
+      expected_shape = endpoints_shapes[endpoint_name]
+      self.assertTrue(endpoint_name in end_points)
+      self.assertListEqual(end_points[endpoint_name].get_shape().as_list(),
+                           expected_shape)
+
   def testUnknownImageShape(self):
     tf.reset_default_graph()
     batch_size = 2

research/slim/nets/inception_v2.py

@@ -32,6 +32,7 @@ def inception_v2_base(inputs,
                       depth_multiplier=1.0,
                       use_separable_conv=True,
                       data_format='NHWC',
+                      include_root_block=True,
                       scope=None):
   """Inception v2 (6a2).
 
@@ -45,7 +46,9 @@ def inception_v2_base(inputs,
       can be one of ['Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1',
       'Conv2d_2c_3x3', 'MaxPool_3a_3x3', 'Mixed_3b', 'Mixed_3c', 'Mixed_4a',
       'Mixed_4b', 'Mixed_4c', 'Mixed_4d', 'Mixed_4e', 'Mixed_5a', 'Mixed_5b',
-      'Mixed_5c'].
+      'Mixed_5c']. If include_root_block is False, ['Conv2d_1a_7x7',
+      'MaxPool_2a_3x3', 'Conv2d_2b_1x1', 'Conv2d_2c_3x3', 'MaxPool_3a_3x3'] will
+      not be available.
     min_depth: Minimum depth value (number of channels) for all convolution ops.
       Enforced when depth_multiplier < 1, and not an active constraint when
       depth_multiplier >= 1.
@@ -56,6 +59,8 @@ def inception_v2_base(inputs,
     use_separable_conv: Use a separable convolution for the first layer
       Conv2d_1a_7x7. If this is False, use a normal convolution instead.
     data_format: Data format of the activations ('NHWC' or 'NCHW').
+    include_root_block: If True, include the convolution and max-pooling layers
+      before the inception modules. If False, excludes those layers.
     scope: Optional variable_scope.
 
   Returns:
@@ -93,6 +98,8 @@ def inception_v2_base(inputs,
         padding='SAME',
         data_format=data_format):
 
+      net = inputs
+      if include_root_block:
         # Note that sizes in the comments below assume an input spatial size of
         # 224x224, however, the inputs can be of any size greater 32x32.
 
@@ -109,7 +116,8 @@ def inception_v2_base(inputs,
           # so that the separable convolution is not overparameterized.
           depthwise_multiplier = min(int(depth(64) / 3), 8)
           net = slim.separable_conv2d(
-            inputs, depth(64), [7, 7],
+              inputs,
+              depth(64), [7, 7],
               depth_multiplier=depthwise_multiplier,
               stride=2,
               padding='SAME',
@@ -124,28 +132,37 @@ def inception_v2_base(inputs,
               weights_initializer=trunc_normal(1.0),
               scope=end_point)
         end_points[end_point] = net
-      if end_point == final_endpoint: return net, end_points
+        if end_point == final_endpoint:
+          return net, end_points
         # 112 x 112 x 64
         end_point = 'MaxPool_2a_3x3'
         net = slim.max_pool2d(net, [3, 3], scope=end_point, stride=2)
         end_points[end_point] = net
-      if end_point == final_endpoint: return net, end_points
+        if end_point == final_endpoint:
+          return net, end_points
         # 56 x 56 x 64
         end_point = 'Conv2d_2b_1x1'
-      net = slim.conv2d(net, depth(64), [1, 1], scope=end_point,
+        net = slim.conv2d(
+            net,
+            depth(64), [1, 1],
+            scope=end_point,
             weights_initializer=trunc_normal(0.1))
         end_points[end_point] = net
-      if end_point == final_endpoint: return net, end_points
+        if end_point == final_endpoint:
+          return net, end_points
         # 56 x 56 x 64
         end_point = 'Conv2d_2c_3x3'
         net = slim.conv2d(net, depth(192), [3, 3], scope=end_point)
         end_points[end_point] = net
-      if end_point == final_endpoint: return net, end_points
+        if end_point == final_endpoint:
+          return net, end_points
         # 56 x 56 x 192
         end_point = 'MaxPool_3a_3x3'
         net = slim.max_pool2d(net, [3, 3], scope=end_point, stride=2)
         end_points[end_point] = net
-      if end_point == final_endpoint: return net, end_points
+        if end_point == final_endpoint:
+          return net, end_points
+
       # 28 x 28 x 192
       # Inception module.
       end_point = 'Mixed_3b'

research/slim/nets/inception_v2_test.py

@@ -253,6 +253,33 @@ class InceptionV2Test(tf.test.TestCase):
     self.assertListEqual(pre_pool.get_shape().as_list(),
                          [batch_size, 4, 4, 1024])
 
+  def testBuildBaseNetworkWithoutRootBlock(self):
+    batch_size = 5
+    height, width = 28, 28
+    channels = 192
+
+    inputs = tf.random_uniform((batch_size, height, width, channels))
+    _, end_points = inception.inception_v2_base(
+        inputs, include_root_block=False)
+    endpoints_shapes = {
+        'Mixed_3b': [batch_size, 28, 28, 256],
+        'Mixed_3c': [batch_size, 28, 28, 320],
+        'Mixed_4a': [batch_size, 14, 14, 576],
+        'Mixed_4b': [batch_size, 14, 14, 576],
+        'Mixed_4c': [batch_size, 14, 14, 576],
+        'Mixed_4d': [batch_size, 14, 14, 576],
+        'Mixed_4e': [batch_size, 14, 14, 576],
+        'Mixed_5a': [batch_size, 7, 7, 1024],
+        'Mixed_5b': [batch_size, 7, 7, 1024],
+        'Mixed_5c': [batch_size, 7, 7, 1024]
+    }
+    self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys())
+    for endpoint_name in endpoints_shapes:
+      expected_shape = endpoints_shapes[endpoint_name]
+      self.assertTrue(endpoint_name in end_points)
+      self.assertListEqual(end_points[endpoint_name].get_shape().as_list(),
+                           expected_shape)
+
   def testUnknownImageShape(self):
     tf.reset_default_graph()
     batch_size = 2

research/slim/nets/mobilenet/mobilenet.py

@@ -109,8 +109,8 @@ def depth_multiplier(output_params,
 _Op = collections.namedtuple('Op', ['op', 'params', 'multiplier_func'])
 
 
-def op(opfunc, **params):
-  multiplier = params.pop('multiplier_transorm', depth_multiplier)
+def op(opfunc, multiplier_func=depth_multiplier, **params):
+  multiplier = params.pop('multiplier_transorm', multiplier_func)
   return _Op(opfunc, params=params, multiplier_func=multiplier)
 
 
@@ -226,6 +226,7 @@ def mobilenet_base(  # pylint: disable=invalid-name
   # since it is also set by mobilenet_scope
   # c) set all defaults
   # d) set all extra overrides.
+  # pylint: disable=g-backslash-continuation
   with _scope_all(scope, default_scope='Mobilenet'), \
       safe_arg_scope([slim.batch_norm], is_training=is_training), \
       _set_arg_scope_defaults(conv_defs_defaults), \
@@ -262,8 +263,15 @@ def mobilenet_base(  # pylint: disable=invalid-name
         current_stride *= stride
       # Update params.
       params['stride'] = layer_stride
-      # Only insert rate to params if rate > 1.
+      # Only insert rate to params if rate > 1 and kernel size is not [1, 1].
       if layer_rate > 1:
+        if tuple(params.get('kernel_size', [])) != (1, 1):
+          # We will apply atrous rate in the following cases:
+          # 1) When kernel_size is not in params, the operation then uses
+          #   default kernel size 3x3.
+          # 2) When kernel_size is in params, and if the kernel_size is not
+          #   equal to (1, 1) (there is no need to apply atrous convolution to
+          #   any 1x1 convolution).
           params['rate'] = layer_rate
       # Set padding
       if use_explicit_padding:

research/slim/nets/mobilenet/mobilenet_v2_test.py

@@ -152,6 +152,26 @@ class MobilenetV2Test(tf.test.TestCase):
         output_stride=16)
     self.assertEqual(out.get_shape().as_list()[1:3], [14, 14])
 
+  def testMultiplier(self):
+    op = mobilenet.op
+    new_def = copy.deepcopy(mobilenet_v2.V2_DEF)
+
+    def inverse_multiplier(output_params, multiplier):
+      output_params['num_outputs'] /= multiplier
+
+    new_def['spec'][0] = op(
+        slim.conv2d,
+        kernel_size=(3, 3),
+        multiplier_func=inverse_multiplier,
+        num_outputs=16)
+    _ = mobilenet_v2.mobilenet_base(
+        tf.placeholder(tf.float32, (10, 224, 224, 16)),
+        conv_defs=new_def, depth_multiplier=0.1)
+    s = [op.outputs[0].get_shape().as_list()[-1] for op in find_ops('Conv2D')]
+    # Expect first layer to be 160 (16 / 0.1), and other layers
+    # their max(original size * 0.1, 8)
+    self.assertEqual([160, 8, 48, 8, 48], s[:5])
+
   def testWithOutputStride8AndExplicitPadding(self):
     tf.reset_default_graph()
     out, _ = mobilenet.mobilenet_base(

research/slim/nets/resnet_v1.py

@@ -286,14 +286,21 @@ def resnet_v1_50(inputs,
                  output_stride=None,
                  spatial_squeeze=True,
                  store_non_strided_activations=False,
+                 min_base_depth=8,
+                 depth_multiplier=1,
                  reuse=None,
                  scope='resnet_v1_50'):
   """ResNet-50 model of [1]. See resnet_v1() for arg and return description."""
+  depth_func = lambda d: max(int(d * depth_multiplier), min_base_depth)
   blocks = [
-      resnet_v1_block('block1', base_depth=64, num_units=3, stride=2),
-      resnet_v1_block('block2', base_depth=128, num_units=4, stride=2),
-      resnet_v1_block('block3', base_depth=256, num_units=6, stride=2),
-      resnet_v1_block('block4', base_depth=512, num_units=3, stride=1),
+      resnet_v1_block('block1', base_depth=depth_func(64), num_units=3,
+                      stride=2),
+      resnet_v1_block('block2', base_depth=depth_func(128), num_units=4,
+                      stride=2),
+      resnet_v1_block('block3', base_depth=depth_func(256), num_units=6,
+                      stride=2),
+      resnet_v1_block('block4', base_depth=depth_func(512), num_units=3,
+                      stride=1),
   ]
   return resnet_v1(inputs, blocks, num_classes, is_training,
                    global_pool=global_pool, output_stride=output_stride,
@@ -310,14 +317,21 @@ def resnet_v1_101(inputs,
                   output_stride=None,
                   spatial_squeeze=True,
                   store_non_strided_activations=False,
+                  min_base_depth=8,
+                  depth_multiplier=1,
                   reuse=None,
                   scope='resnet_v1_101'):
   """ResNet-101 model of [1]. See resnet_v1() for arg and return description."""
+  depth_func = lambda d: max(int(d * depth_multiplier), min_base_depth)
   blocks = [
-      resnet_v1_block('block1', base_depth=64, num_units=3, stride=2),
-      resnet_v1_block('block2', base_depth=128, num_units=4, stride=2),
-      resnet_v1_block('block3', base_depth=256, num_units=23, stride=2),
-      resnet_v1_block('block4', base_depth=512, num_units=3, stride=1),
+      resnet_v1_block('block1', base_depth=depth_func(64), num_units=3,
+                      stride=2),
+      resnet_v1_block('block2', base_depth=depth_func(128), num_units=4,
+                      stride=2),
+      resnet_v1_block('block3', base_depth=depth_func(256), num_units=23,
+                      stride=2),
+      resnet_v1_block('block4', base_depth=depth_func(512), num_units=3,
+                      stride=1),
   ]
   return resnet_v1(inputs, blocks, num_classes, is_training,
                    global_pool=global_pool, output_stride=output_stride,
@@ -334,14 +348,21 @@ def resnet_v1_152(inputs,
                   output_stride=None,
                   store_non_strided_activations=False,
                   spatial_squeeze=True,
+                  min_base_depth=8,
+                  depth_multiplier=1,
                   reuse=None,
                   scope='resnet_v1_152'):
   """ResNet-152 model of [1]. See resnet_v1() for arg and return description."""
+  depth_func = lambda d: max(int(d * depth_multiplier), min_base_depth)
   blocks = [
-      resnet_v1_block('block1', base_depth=64, num_units=3, stride=2),
-      resnet_v1_block('block2', base_depth=128, num_units=8, stride=2),
-      resnet_v1_block('block3', base_depth=256, num_units=36, stride=2),
-      resnet_v1_block('block4', base_depth=512, num_units=3, stride=1),
+      resnet_v1_block('block1', base_depth=depth_func(64), num_units=3,
+                      stride=2),
+      resnet_v1_block('block2', base_depth=depth_func(128), num_units=8,
+                      stride=2),
+      resnet_v1_block('block3', base_depth=depth_func(256), num_units=36,
+                      stride=2),
+      resnet_v1_block('block4', base_depth=depth_func(512), num_units=3,
+                      stride=1),
   ]
   return resnet_v1(inputs, blocks, num_classes, is_training,
                    global_pool=global_pool, output_stride=output_stride,
@@ -358,14 +379,21 @@ def resnet_v1_200(inputs,
                   output_stride=None,
                   store_non_strided_activations=False,
                   spatial_squeeze=True,
+                  min_base_depth=8,
+                  depth_multiplier=1,
                   reuse=None,
                   scope='resnet_v1_200'):
   """ResNet-200 model of [2]. See resnet_v1() for arg and return description."""
+  depth_func = lambda d: max(int(d * depth_multiplier), min_base_depth)
   blocks = [
-      resnet_v1_block('block1', base_depth=64, num_units=3, stride=2),
-      resnet_v1_block('block2', base_depth=128, num_units=24, stride=2),
-      resnet_v1_block('block3', base_depth=256, num_units=36, stride=2),
-      resnet_v1_block('block4', base_depth=512, num_units=3, stride=1),
+      resnet_v1_block('block1', base_depth=depth_func(64), num_units=3,
+                      stride=2),
+      resnet_v1_block('block2', base_depth=depth_func(128), num_units=24,
+                      stride=2),
+      resnet_v1_block('block3', base_depth=depth_func(256), num_units=36,
+                      stride=2),
+      resnet_v1_block('block4', base_depth=depth_func(512), num_units=3,
+                      stride=1),
   ]
   return resnet_v1(inputs, blocks, num_classes, is_training,
                    global_pool=global_pool, output_stride=output_stride,

research/slim/nets/resnet_v1_test.py

@@ -550,6 +550,82 @@ class ResnetCompleteNetworkTest(tf.test.TestCase):
       output = sess.run(output, {inputs: images.eval()})
       self.assertEqual(output.shape, (batch, 9, 9, 32))
 
+  def testDepthMultiplier(self):
+    resnets = [
+        resnet_v1.resnet_v1_50, resnet_v1.resnet_v1_101,
+        resnet_v1.resnet_v1_152, resnet_v1.resnet_v1_200
+    ]
+    resnet_names = [
+        'resnet_v1_50', 'resnet_v1_101', 'resnet_v1_152', 'resnet_v1_200'
+    ]
+    for resnet, resnet_name in zip(resnets, resnet_names):
+      depth_multiplier = 0.25
+      global_pool = True
+      num_classes = 10
+      inputs = create_test_input(2, 224, 224, 3)
+      with slim.arg_scope(resnet_utils.resnet_arg_scope()):
+        scope_base = resnet_name + '_base'
+        _, end_points_base = resnet(
+            inputs,
+            num_classes,
+            global_pool=global_pool,
+            min_base_depth=1,
+            scope=scope_base)
+        scope_test = resnet_name + '_test'
+        _, end_points_test = resnet(
+            inputs,
+            num_classes,
+            global_pool=global_pool,
+            min_base_depth=1,
+            depth_multiplier=depth_multiplier,
+            scope=scope_test)
+        for block in ['block1', 'block2', 'block3', 'block4']:
+          block_name_base = scope_base + '/' + block
+          block_name_test = scope_test + '/' + block
+          self.assertTrue(block_name_base in end_points_base)
+          self.assertTrue(block_name_test in end_points_test)
+          self.assertEqual(
+              len(end_points_base[block_name_base].get_shape().as_list()), 4)
+          self.assertEqual(
+              len(end_points_test[block_name_test].get_shape().as_list()), 4)
+          self.assertListEqual(
+              end_points_base[block_name_base].get_shape().as_list()[:3],
+              end_points_test[block_name_test].get_shape().as_list()[:3])
+          self.assertEqual(
+              int(depth_multiplier *
+                  end_points_base[block_name_base].get_shape().as_list()[3]),
+              end_points_test[block_name_test].get_shape().as_list()[3])
+
+  def testMinBaseDepth(self):
+    resnets = [
+        resnet_v1.resnet_v1_50, resnet_v1.resnet_v1_101,
+        resnet_v1.resnet_v1_152, resnet_v1.resnet_v1_200
+    ]
+    resnet_names = [
+        'resnet_v1_50', 'resnet_v1_101', 'resnet_v1_152', 'resnet_v1_200'
+    ]
+    for resnet, resnet_name in zip(resnets, resnet_names):
+      min_base_depth = 5
+      global_pool = True
+      num_classes = 10
+      inputs = create_test_input(2, 224, 224, 3)
+      with slim.arg_scope(resnet_utils.resnet_arg_scope()):
+        _, end_points = resnet(
+            inputs,
+            num_classes,
+            global_pool=global_pool,
+            min_base_depth=min_base_depth,
+            depth_multiplier=0,
+            scope=resnet_name)
+        for block in ['block1', 'block2', 'block3', 'block4']:
+          block_name = resnet_name + '/' + block
+          self.assertTrue(block_name in end_points)
+          self.assertEqual(
+              len(end_points[block_name].get_shape().as_list()), 4)
+          # The output depth is 4 times base_depth.
+          depth_expected = min_base_depth * 4
+          self.assertEqual(
+              end_points[block_name].get_shape().as_list()[3], depth_expected)
 
 if __name__ == '__main__':
   tf.test.main()