cifar10_test.py 6.56 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tempfile import mkstemp

22
from absl import logging
23
import numpy as np
24
import tensorflow as tf
25

26
from official.r1.resnet import cifar10_main
27
from official.utils.misc import keras_utils
28
from official.utils.testing import integration
29

30
logging.set_verbosity(logging.ERROR)
31

32
_BATCH_SIZE = 128
Neal Wu's avatar
Neal Wu committed
33
34
35
_HEIGHT = 32
_WIDTH = 32
_NUM_CHANNELS = 3
36

37
38

class BaseTest(tf.test.TestCase):
Karmel Allison's avatar
Karmel Allison committed
39
40
  """Tests for the Cifar10 version of Resnet.
  """
41

42
43
  _num_validation_images = None

44
45
46
  @classmethod
  def setUpClass(cls):  # pylint: disable=invalid-name
    super(BaseTest, cls).setUpClass()
47
48
    if keras_utils.is_v2_0:
      tf.compat.v1.disable_eager_execution()
49
    cifar10_main.define_cifar_flags()
50
51
52
53
54

  def setUp(self):
    super(BaseTest, self).setUp()
    self._num_validation_images = cifar10_main.NUM_IMAGES['validation']
    cifar10_main.NUM_IMAGES['validation'] = 4
55

56
57
  def tearDown(self):
    super(BaseTest, self).tearDown()
58
    tf.io.gfile.rmtree(self.get_temp_dir())
59
    cifar10_main.NUM_IMAGES['validation'] = self._num_validation_images
60

61
62
63
  def test_dataset_input_fn(self):
    fake_data = bytearray()
    fake_data.append(7)
Neal Wu's avatar
Neal Wu committed
64
65
    for i in range(_NUM_CHANNELS):
      for _ in range(_HEIGHT * _WIDTH):
66
67
68
        fake_data.append(i)

    _, filename = mkstemp(dir=self.get_temp_dir())
69
70
71
    data_file = open(filename, 'wb')
    data_file.write(fake_data)
    data_file.close()
72

73
    fake_dataset = tf.data.FixedLengthRecordDataset(
Karmel Allison's avatar
Karmel Allison committed
74
        filename, cifar10_main._RECORD_BYTES)  # pylint: disable=protected-access
75
    fake_dataset = fake_dataset.map(
Toby Boyd's avatar
Toby Boyd committed
76
        lambda val: cifar10_main.parse_record(val, False, tf.float32))
77
78
    image, label = tf.compat.v1.data.make_one_shot_iterator(
        fake_dataset).get_next()
79

80
    self.assertAllEqual(label.shape, ())
Neal Wu's avatar
Neal Wu committed
81
    self.assertAllEqual(image.shape, (_HEIGHT, _WIDTH, _NUM_CHANNELS))
82

83
    with self.session() as sess:
84
85
      image, label = sess.run([image, label])

86
      self.assertEqual(label, 7)
87
88
89

      for row in image:
        for pixel in row:
90
          self.assertAllClose(pixel, np.array([-1.225, 0., 1.225]), rtol=1e-3)
91

92
  def cifar10_model_fn_helper(self, mode, resnet_version, dtype):
Toby Boyd's avatar
Toby Boyd committed
93
    input_fn = cifar10_main.get_synth_input_fn(dtype)
94
    dataset = input_fn(True, '', _BATCH_SIZE)
95
    iterator = tf.compat.v1.data.make_initializable_iterator(dataset)
96
97
98
99
100
101
102
    features, labels = iterator.get_next()
    spec = cifar10_main.cifar10_model_fn(
        features, labels, mode, {
            'dtype': dtype,
            'resnet_size': 32,
            'data_format': 'channels_last',
            'batch_size': _BATCH_SIZE,
103
            'resnet_version': resnet_version,
104
            'loss_scale': 128 if dtype == tf.float16 else 1,
Zac Wellmer's avatar
Zac Wellmer committed
105
            'fine_tune': False,
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
        })

    predictions = spec.predictions
    self.assertAllEqual(predictions['probabilities'].shape,
                        (_BATCH_SIZE, 10))
    self.assertEqual(predictions['probabilities'].dtype, tf.float32)
    self.assertAllEqual(predictions['classes'].shape, (_BATCH_SIZE,))
    self.assertEqual(predictions['classes'].dtype, tf.int64)

    if mode != tf.estimator.ModeKeys.PREDICT:
      loss = spec.loss
      self.assertAllEqual(loss.shape, ())
      self.assertEqual(loss.dtype, tf.float32)

    if mode == tf.estimator.ModeKeys.EVAL:
      eval_metric_ops = spec.eval_metric_ops
      self.assertAllEqual(eval_metric_ops['accuracy'][0].shape, ())
      self.assertAllEqual(eval_metric_ops['accuracy'][1].shape, ())
      self.assertEqual(eval_metric_ops['accuracy'][0].dtype, tf.float32)
      self.assertEqual(eval_metric_ops['accuracy'][1].dtype, tf.float32)
126

127
  def test_cifar10_model_fn_train_mode_v1(self):
128
    self.cifar10_model_fn_helper(tf.estimator.ModeKeys.TRAIN, resnet_version=1,
129
                                 dtype=tf.float32)
130

131
  def test_cifar10_model_fn_trainmode__v2(self):
132
    self.cifar10_model_fn_helper(tf.estimator.ModeKeys.TRAIN, resnet_version=2,
133
                                 dtype=tf.float32)
134
135

  def test_cifar10_model_fn_eval_mode_v1(self):
136
    self.cifar10_model_fn_helper(tf.estimator.ModeKeys.EVAL, resnet_version=1,
137
                                 dtype=tf.float32)
138
139

  def test_cifar10_model_fn_eval_mode_v2(self):
140
    self.cifar10_model_fn_helper(tf.estimator.ModeKeys.EVAL, resnet_version=2,
141
                                 dtype=tf.float32)
142
143

  def test_cifar10_model_fn_predict_mode_v1(self):
144
145
    self.cifar10_model_fn_helper(tf.estimator.ModeKeys.PREDICT,
                                 resnet_version=1, dtype=tf.float32)
146
147

  def test_cifar10_model_fn_predict_mode_v2(self):
148
149
    self.cifar10_model_fn_helper(tf.estimator.ModeKeys.PREDICT,
                                 resnet_version=2, dtype=tf.float32)
150

151
  def _test_cifar10model_shape(self, resnet_version):
Neal Wu's avatar
Neal Wu committed
152
153
154
    batch_size = 135
    num_classes = 246

155
    model = cifar10_main.Cifar10Model(32, data_format='channels_last',
156
157
                                      num_classes=num_classes,
                                      resnet_version=resnet_version)
158
    fake_input = tf.random.uniform([batch_size, _HEIGHT, _WIDTH, _NUM_CHANNELS])
159
160
161
162
163
    output = model(fake_input, training=True)

    self.assertAllEqual(output.shape, (batch_size, num_classes))

  def test_cifar10model_shape_v1(self):
164
    self._test_cifar10model_shape(resnet_version=1)
Neal Wu's avatar
Neal Wu committed
165

166
  def test_cifar10model_shape_v2(self):
167
    self._test_cifar10model_shape(resnet_version=2)
Neal Wu's avatar
Neal Wu committed
168

169
  def test_cifar10_end_to_end_synthetic_v1(self):
170
    integration.run_synthetic(
171
        main=cifar10_main.run_cifar, tmp_root=self.get_temp_dir(),
172
173
        extra_flags=['-resnet_version', '1', '-batch_size', '4',
                     '--max_train_steps', '1']
174
    )
175
176

  def test_cifar10_end_to_end_synthetic_v2(self):
177
    integration.run_synthetic(
178
        main=cifar10_main.run_cifar, tmp_root=self.get_temp_dir(),
179
180
        extra_flags=['-resnet_version', '2', '-batch_size', '4',
                     '--max_train_steps', '1']
181
182
    )

183
184
185

if __name__ == '__main__':
  tf.test.main()