imagenet_test.py 12.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import unittest

Karmel Allison's avatar
Karmel Allison committed
22
import tensorflow as tf  # pylint: disable=g-bad-import-order
23

24
from official.resnet import imagenet_main
25
from official.utils.testing import integration
26
27
28

tf.logging.set_verbosity(tf.logging.ERROR)

29
_BATCH_SIZE = 32
30
31
32
33
34
_LABEL_CLASSES = 1001


class BaseTest(tf.test.TestCase):

35
36
37
38
39
  @classmethod
  def setUpClass(cls):  # pylint: disable=invalid-name
    super(BaseTest, cls).setUpClass()
    imagenet_main.define_imagenet_flags()

40
41
42
43
  def tearDown(self):
    super(BaseTest, self).tearDown()
    tf.gfile.DeleteRecursively(self.get_temp_dir())

44
  def _tensor_shapes_helper(self, resnet_size, resnet_version, dtype, with_gpu):
45
46
    """Checks the tensor shapes after each phase of the ResNet model."""
    def reshape(shape):
Karmel Allison's avatar
Karmel Allison committed
47
48
      """Returns the expected dimensions depending on if a GPU is being used."""

49
50
      # If a GPU is used for the test, the shape is returned (already in NCHW
      # form). When GPU is not used, the shape is converted to NHWC.
51
52
53
54
55
56
57
      if with_gpu:
        return shape
      return shape[0], shape[2], shape[3], shape[1]

    graph = tf.Graph()

    with graph.as_default(), self.test_session(
58
        graph=graph, use_gpu=with_gpu, force_gpu=with_gpu):
59
      model = imagenet_main.ImagenetModel(
60
          resnet_size=resnet_size,
61
          data_format='channels_first' if with_gpu else 'channels_last',
62
          resnet_version=resnet_version,
63
64
          dtype=dtype
      )
65
      inputs = tf.random_uniform([1, 224, 224, 3])
66
      output = model(inputs, training=True)
67

68
69
70
71
72
73
74
75
      initial_conv = graph.get_tensor_by_name('resnet_model/initial_conv:0')
      max_pool = graph.get_tensor_by_name('resnet_model/initial_max_pool:0')
      block_layer1 = graph.get_tensor_by_name('resnet_model/block_layer1:0')
      block_layer2 = graph.get_tensor_by_name('resnet_model/block_layer2:0')
      block_layer3 = graph.get_tensor_by_name('resnet_model/block_layer3:0')
      block_layer4 = graph.get_tensor_by_name('resnet_model/block_layer4:0')
      reduce_mean = graph.get_tensor_by_name('resnet_model/final_reduce_mean:0')
      dense = graph.get_tensor_by_name('resnet_model/final_dense:0')
76
77
78
79
80
81
82
83
84
85
86

      self.assertAllEqual(initial_conv.shape, reshape((1, 64, 112, 112)))
      self.assertAllEqual(max_pool.shape, reshape((1, 64, 56, 56)))

      # The number of channels after each block depends on whether we're
      # using the building_block or the bottleneck_block.
      if resnet_size < 50:
        self.assertAllEqual(block_layer1.shape, reshape((1, 64, 56, 56)))
        self.assertAllEqual(block_layer2.shape, reshape((1, 128, 28, 28)))
        self.assertAllEqual(block_layer3.shape, reshape((1, 256, 14, 14)))
        self.assertAllEqual(block_layer4.shape, reshape((1, 512, 7, 7)))
87
        self.assertAllEqual(reduce_mean.shape, reshape((1, 512, 1, 1)))
88
89
90
91
92
      else:
        self.assertAllEqual(block_layer1.shape, reshape((1, 256, 56, 56)))
        self.assertAllEqual(block_layer2.shape, reshape((1, 512, 28, 28)))
        self.assertAllEqual(block_layer3.shape, reshape((1, 1024, 14, 14)))
        self.assertAllEqual(block_layer4.shape, reshape((1, 2048, 7, 7)))
93
        self.assertAllEqual(reduce_mean.shape, reshape((1, 2048, 1, 1)))
94

95
96
      self.assertAllEqual(dense.shape, (1, _LABEL_CLASSES))
      self.assertAllEqual(output.shape, (1, _LABEL_CLASSES))
97

98
99
100
  def tensor_shapes_helper(self, resnet_size, resnet_version, with_gpu=False):
    self._tensor_shapes_helper(resnet_size=resnet_size,
                               resnet_version=resnet_version,
101
                               dtype=tf.float32, with_gpu=with_gpu)
102
103
    self._tensor_shapes_helper(resnet_size=resnet_size,
                               resnet_version=resnet_version,
104
105
                               dtype=tf.float16, with_gpu=with_gpu)

106
  def test_tensor_shapes_resnet_18_v1(self):
107
    self.tensor_shapes_helper(18, resnet_version=1)
108

109
  def test_tensor_shapes_resnet_18_v2(self):
110
    self.tensor_shapes_helper(18, resnet_version=2)
111

112
  def test_tensor_shapes_resnet_34_v1(self):
113
    self.tensor_shapes_helper(34, resnet_version=1)
114

115
  def test_tensor_shapes_resnet_34_v2(self):
116
    self.tensor_shapes_helper(34, resnet_version=2)
117

118
  def test_tensor_shapes_resnet_50_v1(self):
119
    self.tensor_shapes_helper(50, resnet_version=1)
120

121
  def test_tensor_shapes_resnet_50_v2(self):
122
    self.tensor_shapes_helper(50, resnet_version=2)
123
124

  def test_tensor_shapes_resnet_101_v1(self):
125
    self.tensor_shapes_helper(101, resnet_version=1)
126
127

  def test_tensor_shapes_resnet_101_v2(self):
128
    self.tensor_shapes_helper(101, resnet_version=2)
129
130

  def test_tensor_shapes_resnet_152_v1(self):
131
    self.tensor_shapes_helper(152, resnet_version=1)
132
133

  def test_tensor_shapes_resnet_152_v2(self):
134
    self.tensor_shapes_helper(152, resnet_version=2)
135
136

  def test_tensor_shapes_resnet_200_v1(self):
137
    self.tensor_shapes_helper(200, resnet_version=1)
138
139

  def test_tensor_shapes_resnet_200_v2(self):
140
    self.tensor_shapes_helper(200, resnet_version=2)
141
142
143

  @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU')
  def test_tensor_shapes_resnet_18_with_gpu_v1(self):
144
    self.tensor_shapes_helper(18, resnet_version=1, with_gpu=True)
145
146
147

  @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU')
  def test_tensor_shapes_resnet_18_with_gpu_v2(self):
148
    self.tensor_shapes_helper(18, resnet_version=2, with_gpu=True)
149
150
151

  @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU')
  def test_tensor_shapes_resnet_34_with_gpu_v1(self):
152
    self.tensor_shapes_helper(34, resnet_version=1, with_gpu=True)
153
154
155

  @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU')
  def test_tensor_shapes_resnet_34_with_gpu_v2(self):
156
    self.tensor_shapes_helper(34, resnet_version=2, with_gpu=True)
157
158

  @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU')
159
  def test_tensor_shapes_resnet_50_with_gpu_v1(self):
160
    self.tensor_shapes_helper(50, resnet_version=1, with_gpu=True)
161
162

  @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU')
163
  def test_tensor_shapes_resnet_50_with_gpu_v2(self):
164
    self.tensor_shapes_helper(50, resnet_version=2, with_gpu=True)
165
166

  @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU')
167
  def test_tensor_shapes_resnet_101_with_gpu_v1(self):
168
    self.tensor_shapes_helper(101, resnet_version=1, with_gpu=True)
169
170

  @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU')
171
  def test_tensor_shapes_resnet_101_with_gpu_v2(self):
172
    self.tensor_shapes_helper(101, resnet_version=2, with_gpu=True)
173
174

  @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU')
175
  def test_tensor_shapes_resnet_152_with_gpu_v1(self):
176
    self.tensor_shapes_helper(152, resnet_version=1, with_gpu=True)
177
178

  @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU')
179
  def test_tensor_shapes_resnet_152_with_gpu_v2(self):
180
    self.tensor_shapes_helper(152, resnet_version=2, with_gpu=True)
181

182
183
  @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU')
  def test_tensor_shapes_resnet_200_with_gpu_v1(self):
184
    self.tensor_shapes_helper(200, resnet_version=1, with_gpu=True)
185
186
187

  @unittest.skipUnless(tf.test.is_built_with_cuda(), 'requires GPU')
  def test_tensor_shapes_resnet_200_with_gpu_v2(self):
188
    self.tensor_shapes_helper(200, resnet_version=2, with_gpu=True)
189

190
  def resnet_model_fn_helper(self, mode, resnet_version, dtype):
191
    """Tests that the EstimatorSpec is given the appropriate arguments."""
192
193
194
195
196
197
198
199
200
201
202
203
    tf.train.create_global_step()

    input_fn = imagenet_main.get_synth_input_fn()
    dataset = input_fn(True, '', _BATCH_SIZE)
    iterator = dataset.make_one_shot_iterator()
    features, labels = iterator.get_next()
    spec = imagenet_main.imagenet_model_fn(
        features, labels, mode, {
            'dtype': dtype,
            'resnet_size': 50,
            'data_format': 'channels_last',
            'batch_size': _BATCH_SIZE,
204
            'resnet_version': resnet_version,
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
            'loss_scale': 128 if dtype == tf.float16 else 1,
        })

    predictions = spec.predictions
    self.assertAllEqual(predictions['probabilities'].shape,
                        (_BATCH_SIZE, _LABEL_CLASSES))
    self.assertEqual(predictions['probabilities'].dtype, tf.float32)
    self.assertAllEqual(predictions['classes'].shape, (_BATCH_SIZE,))
    self.assertEqual(predictions['classes'].dtype, tf.int64)

    if mode != tf.estimator.ModeKeys.PREDICT:
      loss = spec.loss
      self.assertAllEqual(loss.shape, ())
      self.assertEqual(loss.dtype, tf.float32)

    if mode == tf.estimator.ModeKeys.EVAL:
      eval_metric_ops = spec.eval_metric_ops
      self.assertAllEqual(eval_metric_ops['accuracy'][0].shape, ())
      self.assertAllEqual(eval_metric_ops['accuracy'][1].shape, ())
      self.assertEqual(eval_metric_ops['accuracy'][0].dtype, tf.float32)
      self.assertEqual(eval_metric_ops['accuracy'][1].dtype, tf.float32)
226

227
  def test_resnet_model_fn_train_mode_v1(self):
228
    self.resnet_model_fn_helper(tf.estimator.ModeKeys.TRAIN, resnet_version=1,
229
                                dtype=tf.float32)
230

231
  def test_resnet_model_fn_train_mode_v2(self):
232
    self.resnet_model_fn_helper(tf.estimator.ModeKeys.TRAIN, resnet_version=2,
233
                                dtype=tf.float32)
234
235

  def test_resnet_model_fn_eval_mode_v1(self):
236
    self.resnet_model_fn_helper(tf.estimator.ModeKeys.EVAL, resnet_version=1,
237
                                dtype=tf.float32)
238

239
  def test_resnet_model_fn_eval_mode_v2(self):
240
    self.resnet_model_fn_helper(tf.estimator.ModeKeys.EVAL, resnet_version=2,
241
                                dtype=tf.float32)
Karmel Allison's avatar
Karmel Allison committed
242

243
  def test_resnet_model_fn_predict_mode_v1(self):
244
    self.resnet_model_fn_helper(tf.estimator.ModeKeys.PREDICT, resnet_version=1,
245
                                dtype=tf.float32)
246

247
  def test_resnet_model_fn_predict_mode_v2(self):
248
    self.resnet_model_fn_helper(tf.estimator.ModeKeys.PREDICT, resnet_version=2,
249
                                dtype=tf.float32)
250

251
  def _test_imagenetmodel_shape(self, resnet_version):
Neal Wu's avatar
Neal Wu committed
252
253
254
    batch_size = 135
    num_classes = 246

255
256
    model = imagenet_main.ImagenetModel(
        50, data_format='channels_last', num_classes=num_classes,
257
        resnet_version=resnet_version)
258
259
260
261
262
263
264

    fake_input = tf.random_uniform([batch_size, 224, 224, 3])
    output = model(fake_input, training=True)

    self.assertAllEqual(output.shape, (batch_size, num_classes))

  def test_imagenetmodel_shape_v1(self):
265
    self._test_imagenetmodel_shape(resnet_version=1)
Neal Wu's avatar
Neal Wu committed
266

267
  def test_imagenetmodel_shape_v2(self):
268
    self._test_imagenetmodel_shape(resnet_version=2)
Neal Wu's avatar
Neal Wu committed
269

270
  def test_imagenet_end_to_end_synthetic_v1(self):
271
    integration.run_synthetic(
272
        main=imagenet_main.run_imagenet, tmp_root=self.get_temp_dir(),
273
274
        extra_flags=['-v', '1']
    )
275
276

  def test_imagenet_end_to_end_synthetic_v2(self):
277
    integration.run_synthetic(
278
        main=imagenet_main.run_imagenet, tmp_root=self.get_temp_dir(),
279
280
        extra_flags=['-v', '2']
    )
281
282

  def test_imagenet_end_to_end_synthetic_v1_tiny(self):
283
    integration.run_synthetic(
284
        main=imagenet_main.run_imagenet, tmp_root=self.get_temp_dir(),
285
        extra_flags=['-resnet_version', '1', '-resnet_size', '18']
286
    )
287
288

  def test_imagenet_end_to_end_synthetic_v2_tiny(self):
289
    integration.run_synthetic(
290
        main=imagenet_main.run_imagenet, tmp_root=self.get_temp_dir(),
291
        extra_flags=['-resnet_version', '2', '-resnet_size', '18']
292
    )
293
294

  def test_imagenet_end_to_end_synthetic_v1_huge(self):
295
    integration.run_synthetic(
296
        main=imagenet_main.run_imagenet, tmp_root=self.get_temp_dir(),
297
        extra_flags=['-resnet_version', '1', '-resnet_size', '200']
298
    )
299
300

  def test_imagenet_end_to_end_synthetic_v2_huge(self):
301
    integration.run_synthetic(
302
        main=imagenet_main.run_imagenet, tmp_root=self.get_temp_dir(),
303
        extra_flags=['-resnet_version', '2', '-resnet_size', '200']
304
    )
305

306
307
308
309
310
311
312
  def test_flag_restriction(self):
    with self.assertRaises(SystemExit):
      integration.run_synthetic(
          main=imagenet_main.run_imagenet, tmp_root=self.get_temp_dir(),
          extra_flags=['-resnet_version', '1', '-dtype', 'fp16']
      )

313

314
315
if __name__ == '__main__':
  tf.test.main()