Merge remote-tracking branch 'upstream/master'

official/utils/testing/reference_data/resnet/batch-size-32_building_projection_version-2_width-8_channels-4/results.json

+[32, 8, 8, 4, 0.7820245027542114, 0.8173515796661377, 4095.256591796875, 32, 4, 4, 8, -0.3681173324584961, 0.197183296084404, 213.7501678466797]
\ No newline at end of file

official/utils/testing/reference_data/resnet/batch-size-32_building_projection_version-2_width-8_channels-4/tf_version.json

+["1.8.0-dev20180325", "v1.7.0-rc1-750-g6c1737e6c8"]
\ No newline at end of file

official/utils/testing/reference_data/resnet/batch-size-32_building_version-1_width-8_channels-4/results.json

+[32, 8, 8, 4, 0.23128163814544678, 0.22117376327514648, 4100.51806640625, 32, 8, 8, 4, 0.9646798372268677, 0.16614516079425812, 5799.6708984375]
\ No newline at end of file

official/utils/testing/reference_data/resnet/batch-size-32_building_version-1_width-8_channels-4/tf_version.json

+["1.8.0-dev20180325", "v1.7.0-rc1-750-g6c1737e6c8"]
\ No newline at end of file

official/utils/testing/reference_data/resnet/batch-size-32_building_version-2_width-8_channels-4/results.json

+[32, 8, 8, 4, 0.7616699934005737, 0.5485763549804688, 4106.8720703125, 32, 8, 8, 4, 0.42806127667427063, 0.7305102348327637, 5319.9892578125]
\ No newline at end of file

official/utils/testing/reference_data/resnet/batch-size-32_building_version-2_width-8_channels-4/tf_version.json

+["1.8.0-dev20180325", "v1.7.0-rc1-750-g6c1737e6c8"]
\ No newline at end of file

official/utils/testing/reference_data/resnet/batch_norm/results.json

+[32, 16, 16, 3, 0.9722558259963989, 0.18413543701171875, 12374.20703125, 32, 16, 16, 3, 1.6126631498336792, -1.096894383430481, -0.041595458984375]
\ No newline at end of file

official/utils/testing/reference_data/resnet/batch_norm/tf_version.json

+["1.8.0-dev20180325", "v1.7.0-rc1-750-g6c1737e6c8"]
\ No newline at end of file

official/utils/testing/reference_data_test.py

+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""This module tests generic behavior of reference data tests.
+
+This test is not intended to test every layer of interest, and models should
+test the layers that affect them. This test is primarily focused on ensuring
+that reference_data.BaseTest functions as intended. If there is a legitimate
+change such as a change to TensorFlow which changes graph construction, tests
+can be regenerated with the following command:
+
+  $ python3 reference_data_test.py -regen
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import sys
+import unittest
+import warnings
+
+import tensorflow as tf  # pylint: disable=g-bad-import-order
+from official.utils.testing import reference_data
+
+
+class GoldenBaseTest(reference_data.BaseTest):
+  """Class to ensure that reference data testing runs properly."""
+
+  @property
+  def test_name(self):
+    return "reference_data_test"
+
+  def _uniform_random_ops(self, test=False, wrong_name=False, wrong_shape=False,
+                          bad_seed=False, bad_function=False):
+    """Tests number generation and failure modes.
+
+    This test is of a very simple graph: the generation of a 1x1 random tensor.
+    However, it is also used to confirm that the tests are actually checking
+    properly by failing in predefined ways.
+
+    Args:
+      test: Whether or not to run as a test case.
+      wrong_name: Whether to assign the wrong name to the tensor.
+      wrong_shape: Whether to create a tensor with the wrong shape.
+      bad_seed: Whether or not to perturb the random seed.
+      bad_function: Whether to perturb the correctness function.
+    """
+    name = "uniform_random"
+
+    g = tf.Graph()
+    with g.as_default():
+      seed = self.name_to_seed(name)
+      seed = seed + 1 if bad_seed else seed
+      tf.set_random_seed(seed)
+      tensor_name = "wrong_tensor" if wrong_name else "input_tensor"
+      tensor_shape = (1, 2) if wrong_shape else (1, 1)
+      input_tensor = tf.get_variable(
+          tensor_name, dtype=tf.float32,
+          initializer=tf.random_uniform(tensor_shape, maxval=1)
+      )
+
+    def correctness_function(tensor_result):
+      result = float(tensor_result[0, 0])
+      result = result + 0.1 if bad_function else result
+      return [result]
+
+    self._save_or_test_ops(
+        name=name, graph=g, ops_to_eval=[input_tensor], test=test,
+        correctness_function=correctness_function
+    )
+
+  def _dense_ops(self, test=False):
+    name = "dense"
+
+    g = tf.Graph()
+    with g.as_default():
+      tf.set_random_seed(self.name_to_seed(name))
+      input_tensor = tf.get_variable(
+          "input_tensor", dtype=tf.float32,
+          initializer=tf.random_uniform((1, 2), maxval=1)
+      )
+      layer = tf.layers.dense(inputs=input_tensor, units=4)
+      layer = tf.layers.dense(inputs=layer, units=1)
+
+    self._save_or_test_ops(
+        name=name, graph=g, ops_to_eval=[layer], test=test,
+        correctness_function=self.default_correctness_function
+    )
+
+  def test_uniform_random(self):
+    self._uniform_random_ops(test=True)
+
+  def test_tensor_name_error(self):
+    with self.assertRaises(AssertionError):
+      self._uniform_random_ops(test=True, wrong_name=True)
+
+  def test_tensor_shape_error(self):
+    with self.assertRaises(AssertionError):
+      self._uniform_random_ops(test=True, wrong_shape=True)
+
+  @unittest.skipIf(sys.version_info[0] == 2,
+                   "catch_warning doesn't catch tf.logging.warn in py 2.")
+  def test_bad_seed(self):
+    with warnings.catch_warnings(record=True) as warn_catch:
+      self._uniform_random_ops(test=True, bad_seed=True)
+      assert len(warn_catch) == 1, "Test did not warn of minor graph change."
+
+  def test_incorrectness_function(self):
+    with self.assertRaises(AssertionError):
+      self._uniform_random_ops(test=True, bad_function=True)
+
+  def test_dense(self):
+    self._dense_ops(test=True)
+
+  def regenerate(self):
+    self._uniform_random_ops(test=False)
+    self._dense_ops(test=False)
+
+
+if __name__ == "__main__":
+  reference_data.main(argv=sys.argv, test_class=GoldenBaseTest)