Merge remote-tracking branch 'tensorflow/master'

research/gan/image_compression/train.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# pylint:disable=line-too-long
+"""Trains an image compression network with an adversarial loss."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+
+import tensorflow as tf
+
+import data_provider
+import networks
+import summaries
+
+
+tfgan = tf.contrib.gan
+flags = tf.flags
+FLAGS = flags.FLAGS
+
+flags.DEFINE_integer('batch_size', 32, 'The number of images in each batch.')
+
+flags.DEFINE_integer('patch_size', 32, 'The size of the patches to train on.')
+
+flags.DEFINE_integer('bits_per_patch', 1230,
+                     'The number of bits to produce per patch.')
+
+flags.DEFINE_integer('model_depth', 64,
+                     'Number of filters for compression model')
+
+flags.DEFINE_string('master', '', 'Name of the TensorFlow master to use.')
+
+flags.DEFINE_string('train_log_dir', '/tmp/compression/',
+                    'Directory where to write event logs.')
+
+flags.DEFINE_float('generator_lr', 1e-5,
+                   'The compression model learning rate.')
+
+flags.DEFINE_float('discriminator_lr', 1e-6,
+                   'The discriminator learning rate.')
+
+flags.DEFINE_integer('max_number_of_steps', 2000000,
+                     'The maximum number of gradient steps.')
+
+flags.DEFINE_integer(
+    'ps_tasks', 0,
+    'The number of parameter servers. If the value is 0, then the parameters '
+    'are handled locally by the worker.')
+
+flags.DEFINE_integer(
+    'task', 0,
+    'The Task ID. This value is used when training with multiple workers to '
+    'identify each worker.')
+
+flags.DEFINE_float(
+    'weight_factor', 10000.0,
+    'How much to weight the adversarial loss relative to pixel loss.')
+
+flags.DEFINE_string('dataset_dir', None, 'Location of data.')
+
+
+def main(_):
+  if not tf.gfile.Exists(FLAGS.train_log_dir):
+    tf.gfile.MakeDirs(FLAGS.train_log_dir)
+
+  with tf.device(tf.train.replica_device_setter(FLAGS.ps_tasks)):
+    # Put input pipeline on CPU to reserve GPU for training.
+    with tf.name_scope('inputs'), tf.device('/cpu:0'):
+      images = data_provider.provide_data(
+          'train', FLAGS.batch_size, dataset_dir=FLAGS.dataset_dir,
+          patch_size=FLAGS.patch_size)
+
+    # Manually define a GANModel tuple. This is useful when we have custom
+    # code to track variables. Note that we could replace all of this with a
+    # call to `tfgan.gan_model`, but we don't in order to demonstrate some of
+    # TFGAN's flexibility.
+    with tf.variable_scope('generator') as gen_scope:
+      reconstructions, _, prebinary = networks.compression_model(
+          images,
+          num_bits=FLAGS.bits_per_patch,
+          depth=FLAGS.model_depth)
+    gan_model = _get_gan_model(
+        generator_inputs=images,
+        generated_data=reconstructions,
+        real_data=images,
+        generator_scope=gen_scope)
+    summaries.add_reconstruction_summaries(images, reconstructions, prebinary)
+    tfgan.eval.add_gan_model_summaries(gan_model)
+
+    # Define the GANLoss tuple using standard library functions.
+    with tf.name_scope('loss'):
+      gan_loss = tfgan.gan_loss(
+          gan_model,
+          generator_loss_fn=tfgan.losses.least_squares_generator_loss,
+          discriminator_loss_fn=tfgan.losses.least_squares_discriminator_loss,
+          add_summaries=FLAGS.weight_factor > 0)
+
+      # Define the standard pixel loss.
+      l1_pixel_loss = tf.norm(gan_model.real_data - gan_model.generated_data,
+                              ord=1)
+
+      # Modify the loss tuple to include the pixel loss. Add summaries as well.
+      gan_loss = tfgan.losses.combine_adversarial_loss(
+          gan_loss, gan_model, l1_pixel_loss, weight_factor=FLAGS.weight_factor)
+
+    # Get the GANTrain ops using the custom optimizers and optional
+    # discriminator weight clipping.
+    with tf.name_scope('train_ops'):
+      gen_lr, dis_lr = _lr(FLAGS.generator_lr, FLAGS.discriminator_lr)
+      gen_opt, dis_opt = _optimizer(gen_lr, dis_lr)
+      train_ops = tfgan.gan_train_ops(
+          gan_model,
+          gan_loss,
+          generator_optimizer=gen_opt,
+          discriminator_optimizer=dis_opt,
+          summarize_gradients=True,
+          colocate_gradients_with_ops=True,
+          aggregation_method=tf.AggregationMethod.EXPERIMENTAL_ACCUMULATE_N)
+      tf.summary.scalar('generator_lr', gen_lr)
+      tf.summary.scalar('discriminator_lr', dis_lr)
+
+    # Determine the number of generator vs discriminator steps.
+    train_steps = tfgan.GANTrainSteps(
+        generator_train_steps=1,
+        discriminator_train_steps=int(FLAGS.weight_factor > 0))
+
+    # Run the alternating training loop. Skip it if no steps should be taken
+    # (used for graph construction tests).
+    status_message = tf.string_join(
+        ['Starting train step: ',
+         tf.as_string(tf.train.get_or_create_global_step())],
+        name='status_message')
+    if FLAGS.max_number_of_steps == 0: return
+    tfgan.gan_train(
+        train_ops,
+        FLAGS.train_log_dir,
+        tfgan.get_sequential_train_hooks(train_steps),
+        hooks=[tf.train.StopAtStepHook(num_steps=FLAGS.max_number_of_steps),
+               tf.train.LoggingTensorHook([status_message], every_n_iter=10)],
+        master=FLAGS.master,
+        is_chief=FLAGS.task == 0)
+
+
+def _optimizer(gen_lr, dis_lr):
+  # First is generator optimizer, second is discriminator.
+  adam_kwargs = {
+      'epsilon': 1e-8,
+      'beta1': 0.5,
+  }
+  return (tf.train.AdamOptimizer(gen_lr, **adam_kwargs),
+          tf.train.AdamOptimizer(dis_lr, **adam_kwargs))
+
+
+def _lr(gen_lr_base, dis_lr_base):
+  """Return the generator and discriminator learning rates."""
+  gen_lr_kwargs = {
+      'decay_steps': 60000,
+      'decay_rate': 0.9,
+      'staircase': True,
+  }
+  gen_lr = tf.train.exponential_decay(
+      learning_rate=gen_lr_base,
+      global_step=tf.train.get_or_create_global_step(),
+      **gen_lr_kwargs)
+  dis_lr = dis_lr_base
+
+  return gen_lr, dis_lr
+
+
+def _get_gan_model(generator_inputs, generated_data, real_data,
+                   generator_scope):
+  """Manually construct and return a GANModel tuple."""
+  generator_vars = tf.contrib.framework.get_trainable_variables(generator_scope)
+
+  discriminator_fn = networks.discriminator
+  with tf.variable_scope('discriminator') as dis_scope:
+    discriminator_gen_outputs = discriminator_fn(generated_data)
+  with tf.variable_scope(dis_scope, reuse=True):
+    discriminator_real_outputs = discriminator_fn(real_data)
+  discriminator_vars = tf.contrib.framework.get_trainable_variables(
+      dis_scope)
+
+  # Manually construct GANModel tuple.
+  gan_model = tfgan.GANModel(
+      generator_inputs=generator_inputs,
+      generated_data=generated_data,
+      generator_variables=generator_vars,
+      generator_scope=generator_scope,
+      generator_fn=None,  # not necessary
+      real_data=real_data,
+      discriminator_real_outputs=discriminator_real_outputs,
+      discriminator_gen_outputs=discriminator_gen_outputs,
+      discriminator_variables=discriminator_vars,
+      discriminator_scope=dis_scope,
+      discriminator_fn=discriminator_fn)
+
+  return gan_model
+
+
+if __name__ == '__main__':
+  tf.logging.set_verbosity(tf.logging.INFO)
+  tf.app.run()
+

research/gan/image_compression/train_test.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for image_compression.train."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+import numpy as np
+import tensorflow as tf
+import train
+
+FLAGS = tf.flags.FLAGS
+mock = tf.test.mock
+
+
+class TrainTest(tf.test.TestCase):
+
+  def _test_build_graph_helper(self, weight_factor):
+    FLAGS.max_number_of_steps = 0
+    FLAGS.weight_factor = weight_factor
+
+    batch_size = 3
+    patch_size = 16
+
+    FLAGS.batch_size = batch_size
+    FLAGS.patch_size = patch_size
+    mock_imgs = np.zeros([batch_size, patch_size, patch_size, 3],
+                         dtype=np.float32)
+
+    with mock.patch.object(train, 'data_provider') as mock_data_provider:
+      mock_data_provider.provide_data.return_value = mock_imgs
+      train.main(None)
+
+  def test_build_graph_noadversarialloss(self):
+    self._test_build_graph_helper(0.0)
+
+  def test_build_graph_adversarialloss(self):
+    self._test_build_graph_helper(1.0)
+
+
+if __name__ == '__main__':
+  tf.test.main()
+

research/gan/mnist/conditional_eval.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Evaluates a conditional TFGAN trained MNIST model."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+import tensorflow as tf
+
+import data_provider
+import networks
+import util
+
+flags = tf.flags
+tfgan = tf.contrib.gan
+
+
+flags.DEFINE_string('checkpoint_dir', '/tmp/mnist/',
+                    'Directory where the model was written to.')
+
+flags.DEFINE_string('eval_dir', '/tmp/mnist/',
+                    'Directory where the results are saved to.')
+
+flags.DEFINE_integer('num_images_per_class', 10,
+                     'Number of images to generate per class.')
+
+flags.DEFINE_integer('noise_dims', 64,
+                     'Dimensions of the generator noise vector')
+
+flags.DEFINE_string('classifier_filename', None,
+                    'Location of the pretrained classifier. If `None`, use '
+                    'default.')
+
+flags.DEFINE_integer('max_number_of_evaluations', None,
+                     'Number of times to run evaluation. If `None`, run '
+                     'forever.')
+
+FLAGS = flags.FLAGS
+NUM_CLASSES = 10
+
+
+def main(_, run_eval_loop=True):
+  with tf.name_scope('inputs'):
+    noise, one_hot_labels = _get_generator_inputs(
+        FLAGS.num_images_per_class, NUM_CLASSES, FLAGS.noise_dims)
+
+  # Generate images.
+  with tf.variable_scope('Generator'):  # Same scope as in train job.
+    images = networks.conditional_generator((noise, one_hot_labels))
+
+  # Visualize images.
+  reshaped_img = tfgan.eval.image_reshaper(
+      images, num_cols=FLAGS.num_images_per_class)
+  tf.summary.image('generated_images', reshaped_img, max_outputs=1)
+
+  # Calculate evaluation metrics.
+  tf.summary.scalar('MNIST_Classifier_score',
+                    util.mnist_score(images, FLAGS.classifier_filename))
+  tf.summary.scalar('MNIST_Cross_entropy',
+                    util.mnist_cross_entropy(
+                        images, one_hot_labels, FLAGS.classifier_filename))
+
+  # Write images to disk.
+  image_write_ops = tf.write_file(
+      '%s/%s'% (FLAGS.eval_dir, 'conditional_gan.png'),
+      tf.image.encode_png(data_provider.float_image_to_uint8(reshaped_img[0])))
+
+  # For unit testing, use `run_eval_loop=False`.
+  if not run_eval_loop: return
+  tf.contrib.training.evaluate_repeatedly(
+      FLAGS.checkpoint_dir,
+      hooks=[tf.contrib.training.SummaryAtEndHook(FLAGS.eval_dir),
+             tf.contrib.training.StopAfterNEvalsHook(1)],
+      eval_ops=image_write_ops,
+      max_number_of_evaluations=FLAGS.max_number_of_evaluations)
+
+
+def _get_generator_inputs(num_images_per_class, num_classes, noise_dims):
+  # Since we want a grid of numbers for the conditional generator, manually
+  # construct the desired class labels.
+  num_images_generated = num_images_per_class * num_classes
+  noise = tf.random_normal([num_images_generated, noise_dims])
+  labels = [lbl for lbl in range(num_classes) for _
+            in range(num_images_per_class)]
+  one_hot_labels = tf.one_hot(tf.constant(labels), num_classes)
+  return noise, one_hot_labels
+
+
+if __name__ == '__main__':
+  tf.app.run()

research/gan/mnist/conditional_eval_test.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for tfgan.examples.mnist.conditional_eval."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import tensorflow as tf
+import conditional_eval
+
+
+class ConditionalEvalTest(tf.test.TestCase):
+
+  def test_build_graph(self):
+    conditional_eval.main(None, run_eval_loop=False)
+
+
+if __name__ == '__main__':
+  tf.test.main()

research/gan/mnist/data_provider.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Contains code for loading and preprocessing the MNIST data."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+
+import tensorflow as tf
+
+from slim.datasets import dataset_factory as datasets
+
+slim = tf.contrib.slim
+
+
+def provide_data(split_name, batch_size, dataset_dir, num_readers=1,
+                 num_threads=1):
+  """Provides batches of MNIST digits.
+
+  Args:
+    split_name: Either 'train' or 'test'.
+    batch_size: The number of images in each batch.
+    dataset_dir: The directory where the MNIST data can be found.
+    num_readers: Number of dataset readers.
+    num_threads: Number of prefetching threads.
+
+  Returns:
+    images: A `Tensor` of size [batch_size, 28, 28, 1]
+    one_hot_labels: A `Tensor` of size [batch_size, mnist.NUM_CLASSES], where
+      each row has a single element set to one and the rest set to zeros.
+    num_samples: The number of total samples in the dataset.
+
+  Raises:
+    ValueError: If `split_name` is not either 'train' or 'test'.
+  """
+  dataset = datasets.get_dataset('mnist', split_name, dataset_dir=dataset_dir)
+  provider = slim.dataset_data_provider.DatasetDataProvider(
+      dataset,
+      num_readers=num_readers,
+      common_queue_capacity=2 * batch_size,
+      common_queue_min=batch_size,
+      shuffle=(split_name == 'train'))
+  [image, label] = provider.get(['image', 'label'])
+
+  # Preprocess the images.
+  image = (tf.to_float(image) - 128.0) / 128.0
+
+  # Creates a QueueRunner for the pre-fetching operation.
+  images, labels = tf.train.batch(
+      [image, label],
+      batch_size=batch_size,
+      num_threads=num_threads,
+      capacity=5 * batch_size)
+
+  one_hot_labels = tf.one_hot(labels, dataset.num_classes)
+  return images, one_hot_labels, dataset.num_samples
+
+
+def float_image_to_uint8(image):
+  """Convert float image in [-1, 1) to [0, 255] uint8.
+
+  Note that `1` gets mapped to `0`, but `1 - epsilon` gets mapped to 255.
+
+  Args:
+    image: An image tensor. Values should be in [-1, 1).
+
+  Returns:
+    Input image cast to uint8 and with integer values in [0, 255].
+  """
+  image = (image * 128.0) + 128.0
+  return tf.cast(image, tf.uint8)

research/gan/mnist/data_provider_test.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for data_provider."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+
+
+import tensorflow as tf
+
+import data_provider
+
+
+class DataProviderTest(tf.test.TestCase):
+
+  def test_mnist_data_reading(self):
+    dataset_dir = os.path.join(
+        tf.flags.FLAGS.test_srcdir,
+        'google3/third_party/tensorflow_models/gan/mnist/testdata')
+
+    batch_size = 5
+    images, labels, num_samples = data_provider.provide_data(
+        'test', batch_size, dataset_dir)
+    self.assertEqual(num_samples, 10000)
+
+    with self.test_session() as sess:
+      with tf.contrib.slim.queues.QueueRunners(sess):
+        images, labels = sess.run([images, labels])
+        self.assertEqual(images.shape, (batch_size, 28, 28, 1))
+        self.assertEqual(labels.shape, (batch_size, 10))
+
+if __name__ == '__main__':
+  tf.test.main()

research/gan/mnist/eval.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Evaluates a TFGAN trained MNIST model."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+
+import tensorflow as tf
+
+import data_provider
+import networks
+import util
+
+flags = tf.flags
+FLAGS = flags.FLAGS
+tfgan = tf.contrib.gan
+
+
+flags.DEFINE_string('checkpoint_dir', '/tmp/mnist/',
+                    'Directory where the model was written to.')
+
+flags.DEFINE_string('eval_dir', '/tmp/mnist/',
+                    'Directory where the results are saved to.')
+
+flags.DEFINE_string('dataset_dir', None, 'Location of data.')
+
+flags.DEFINE_integer('num_images_generated', 1000,
+                     'Number of images to generate at once.')
+
+flags.DEFINE_boolean('eval_real_images', False,
+                     'If `True`, run Inception network on real images.')
+
+flags.DEFINE_integer('noise_dims', 64,
+                     'Dimensions of the generator noise vector')
+
+flags.DEFINE_string('classifier_filename', None,
+                    'Location of the pretrained classifier. If `None`, use '
+                    'default.')
+
+flags.DEFINE_integer('max_number_of_evaluations', None,
+                     'Number of times to run evaluation. If `None`, run '
+                     'forever.')
+
+
+def main(_, run_eval_loop=True):
+  # Fetch real images.
+  with tf.name_scope('inputs'):
+    real_images, _, _ = data_provider.provide_data(
+        'train', FLAGS.num_images_generated, FLAGS.dataset_dir)
+
+  image_write_ops = None
+  if FLAGS.eval_real_images:
+    tf.summary.scalar('MNIST_Classifier_score',
+                      util.mnist_score(real_images, FLAGS.classifier_filename))
+  else:
+    # In order for variables to load, use the same variable scope as in the
+    # train job.
+    with tf.variable_scope('Generator'):
+      images = networks.unconditional_generator(
+          tf.random_normal([FLAGS.num_images_generated, FLAGS.noise_dims]))
+    tf.summary.scalar('MNIST_Frechet_distance',
+                      util.mnist_frechet_distance(
+                          real_images, images, FLAGS.classifier_filename))
+    tf.summary.scalar('MNIST_Classifier_score',
+                      util.mnist_score(images, FLAGS.classifier_filename))
+    if FLAGS.num_images_generated >= 100:
+      reshaped_images = tfgan.eval.image_reshaper(
+          images[:100, ...], num_cols=10)
+      uint8_images = data_provider.float_image_to_uint8(reshaped_images)
+      image_write_ops = tf.write_file(
+          '%s/%s'% (FLAGS.eval_dir, 'unconditional_gan.png'),
+          tf.image.encode_png(uint8_images[0]))
+
+  # For unit testing, use `run_eval_loop=False`.
+  if not run_eval_loop: return
+  tf.contrib.training.evaluate_repeatedly(
+      FLAGS.checkpoint_dir,
+      hooks=[tf.contrib.training.SummaryAtEndHook(FLAGS.eval_dir),
+             tf.contrib.training.StopAfterNEvalsHook(1)],
+      eval_ops=image_write_ops,
+      max_number_of_evaluations=FLAGS.max_number_of_evaluations)
+
+
+if __name__ == '__main__':
+  tf.app.run()

research/gan/mnist/eval_test.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for tfgan.examples.mnist.eval."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+
+import tensorflow as tf
+import eval  # pylint:disable=redefined-builtin
+
+
+class EvalTest(tf.test.TestCase):
+
+  def _test_build_graph_helper(self, eval_real_images):
+    tf.flags.FLAGS.eval_real_images = eval_real_images
+    eval.main(None, run_eval_loop=False)
+
+  def test_build_graph_realdata(self):
+    self._test_build_graph_helper(True)
+
+  def test_build_graph_generateddata(self):
+    self._test_build_graph_helper(False)
+
+if __name__ == '__main__':
+  tf.test.main()

research/gan/mnist/infogan_eval.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Evaluates an InfoGAN TFGAN trained MNIST model.
+
+The image visualizations, as in https://arxiv.org/abs/1606.03657, show the
+effect of varying a specific latent variable on the image. Each visualization
+focuses on one of the three structured variables. Columns have two of the three
+variables fixed, while the third one is varied. Different rows have different
+random samples from the remaining latents.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+import numpy as np
+
+import tensorflow as tf
+
+import data_provider
+import networks
+import util
+
+flags = tf.flags
+tfgan = tf.contrib.gan
+
+
+flags.DEFINE_string('checkpoint_dir', '/tmp/mnist/',
+                    'Directory where the model was written to.')
+
+flags.DEFINE_string('eval_dir', '/tmp/mnist/',
+                    'Directory where the results are saved to.')
+
+flags.DEFINE_integer('noise_samples', 6,
+                     'Number of samples to draw from the continuous structured '
+                     'noise.')
+
+flags.DEFINE_integer('unstructured_noise_dims', 62,
+                     'The number of dimensions of the unstructured noise.')
+
+flags.DEFINE_integer('continuous_noise_dims', 2,
+                     'The number of dimensions of the continuous noise.')
+
+flags.DEFINE_string('classifier_filename', None,
+                    'Location of the pretrained classifier. If `None`, use '
+                    'default.')
+
+flags.DEFINE_integer('max_number_of_evaluations', None,
+                     'Number of times to run evaluation. If `None`, run '
+                     'forever.')
+
+CAT_SAMPLE_POINTS = np.arange(0, 10)
+CONT_SAMPLE_POINTS = np.linspace(-2.0, 2.0, 10)
+FLAGS = flags.FLAGS
+
+
+def main(_, run_eval_loop=True):
+  with tf.name_scope('inputs'):
+    noise_args = (FLAGS.noise_samples, CAT_SAMPLE_POINTS, CONT_SAMPLE_POINTS,
+                  FLAGS.unstructured_noise_dims, FLAGS.continuous_noise_dims)
+    # Use fixed noise vectors to illustrate the effect of each dimension.
+    display_noise1 = util.get_eval_noise_categorical(*noise_args)
+    display_noise2 = util.get_eval_noise_continuous_dim1(*noise_args)
+    display_noise3 = util.get_eval_noise_continuous_dim2(*noise_args)
+    _validate_noises([display_noise1, display_noise2, display_noise3])
+
+  # Visualize the effect of each structured noise dimension on the generated
+  # image.
+  generator_fn = lambda x: networks.infogan_generator(x, len(CAT_SAMPLE_POINTS))
+  with tf.variable_scope('Generator') as genscope:  # Same scope as in training.
+    categorical_images = generator_fn(display_noise1)
+  reshaped_categorical_img = tfgan.eval.image_reshaper(
+      categorical_images, num_cols=len(CAT_SAMPLE_POINTS))
+  tf.summary.image('categorical', reshaped_categorical_img, max_outputs=1)
+
+  with tf.variable_scope(genscope, reuse=True):
+    continuous1_images = generator_fn(display_noise2)
+  reshaped_continuous1_img = tfgan.eval.image_reshaper(
+      continuous1_images, num_cols=len(CONT_SAMPLE_POINTS))
+  tf.summary.image('continuous1', reshaped_continuous1_img, max_outputs=1)
+
+  with tf.variable_scope(genscope, reuse=True):
+    continuous2_images = generator_fn(display_noise3)
+  reshaped_continuous2_img = tfgan.eval.image_reshaper(
+      continuous2_images, num_cols=len(CONT_SAMPLE_POINTS))
+  tf.summary.image('continuous2', reshaped_continuous2_img, max_outputs=1)
+
+  # Evaluate image quality.
+  all_images = tf.concat(
+      [categorical_images, continuous1_images, continuous2_images], 0)
+  tf.summary.scalar('MNIST_Classifier_score',
+                    util.mnist_score(all_images, FLAGS.classifier_filename))
+
+  # Write images to disk.
+  image_write_ops = []
+  image_write_ops.append(_get_write_image_ops(
+      FLAGS.eval_dir, 'categorical_infogan.png', reshaped_categorical_img[0]))
+  image_write_ops.append(_get_write_image_ops(
+      FLAGS.eval_dir, 'continuous1_infogan.png', reshaped_continuous1_img[0]))
+  image_write_ops.append(_get_write_image_ops(
+      FLAGS.eval_dir, 'continuous2_infogan.png', reshaped_continuous2_img[0]))
+
+  # For unit testing, use `run_eval_loop=False`.
+  if not run_eval_loop: return
+  tf.contrib.training.evaluate_repeatedly(
+      FLAGS.checkpoint_dir,
+      hooks=[tf.contrib.training.SummaryAtEndHook(FLAGS.eval_dir),
+             tf.contrib.training.StopAfterNEvalsHook(1)],
+      eval_ops=image_write_ops,
+      max_number_of_evaluations=FLAGS.max_number_of_evaluations)
+
+
+def _validate_noises(noises):
+  """Sanity check on constructed noise tensors.
+
+  Args:
+    noises: List of 3-tuples of noise vectors.
+  """
+  assert isinstance(noises, (list, tuple))
+  for noise_l in noises:
+    assert len(noise_l) == 3
+    assert isinstance(noise_l[0], np.ndarray)
+    batch_dim = noise_l[0].shape[0]
+    for i, noise in enumerate(noise_l):
+      assert isinstance(noise, np.ndarray)
+      # Check that batch dimensions are all the same.
+      assert noise.shape[0] == batch_dim
+
+      # Check that shapes for corresponding noises are the same.
+      assert noise.shape == noises[0][i].shape
+
+
+def _get_write_image_ops(eval_dir, filename, images):
+  """Create Ops that write images to disk."""
+  return tf.write_file(
+      '%s/%s'% (eval_dir, filename),
+      tf.image.encode_png(data_provider.float_image_to_uint8(images)))
+
+
+if __name__ == '__main__':
+  tf.app.run()

research/gan/mnist/infogan_eval_test.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for tfgan.examples.mnist.infogan_eval."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+import tensorflow as tf
+import infogan_eval
+
+
+class MnistInfoGANEvalTest(tf.test.TestCase):
+
+  def test_build_graph(self):
+    infogan_eval.main(None, run_eval_loop=False)
+
+
+if __name__ == '__main__':
+  tf.test.main()

research/gan/mnist/launch_jobs.sh

+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+#!/bin/bash
+#
+# This script performs the following operations:
+# 1. Downloads the MNIST dataset.
+# 2. Trains an unconditional, conditional, or InfoGAN model on the MNIST
+#    training set.
+# 3. Evaluates the models and writes sample images to disk.
+#
+# These examples are intended to be fast. For better final results, tune
+# hyperparameters or train longer.
+#
+# NOTE: Each training step takes about 0.5 second with a batch size of 32 on
+# CPU. On GPU, it takes ~5 milliseconds.
+#
+# With the default batch size and number of steps, train times are:
+#
+#   unconditional: CPU: 800  steps, ~10 min   GPU: 800  steps, ~1 min
+#   conditional:   CPU: 2000 steps, ~20 min   GPU: 2000 steps, ~2 min
+#   infogan:       CPU: 3000 steps, ~20 min   GPU: 3000 steps, ~6 min
+#
+# Usage:
+# cd models/research/gan/mnist
+# ./launch_jobs.sh ${gan_type} ${git_repo}
+set -e
+
+# Type of GAN to run. Right now, options are `unconditional`, `conditional`, or
+# `infogan`.
+gan_type=$1
+if ! [[ "$gan_type" =~ ^(unconditional|conditional|infogan) ]]; then
+  echo "'gan_type' must be one of: 'unconditional', 'conditional', 'infogan'."
+  exit
+fi
+
+# Location of the git repository.
+git_repo=$2
+if [[ "$git_repo" == "" ]]; then
+  echo "'git_repo' must not be empty."
+  exit
+fi
+
+# Base name for where the checkpoint and logs will be saved to.
+TRAIN_DIR=/tmp/mnist-model
+
+# Base name for where the evaluation images will be saved to.
+EVAL_DIR=/tmp/mnist-model/eval
+
+# Where the dataset is saved to.
+DATASET_DIR=/tmp/mnist-data
+
+# Location of the classifier frozen graph used for evaluation.
+FROZEN_GRAPH="${git_repo}/research/gan/mnist/data/classify_mnist_graph_def.pb"
+
+export PYTHONPATH=$PYTHONPATH:$git_repo:$git_repo/research:$git_repo/research/slim
+
+# A helper function for printing pretty output.
+Banner () {
+  local text=$1
+  local green='\033[0;32m'
+  local nc='\033[0m'  # No color.
+  echo -e "${green}${text}${nc}"
+}
+
+# Download the dataset.
+python "${git_repo}/research/slim/download_and_convert_data.py" \
+  --dataset_name=mnist \
+  --dataset_dir=${DATASET_DIR}
+
+# Run unconditional GAN.
+if [[ "$gan_type" == "unconditional" ]]; then
+  UNCONDITIONAL_TRAIN_DIR="${TRAIN_DIR}/unconditional"
+  UNCONDITIONAL_EVAL_DIR="${EVAL_DIR}/unconditional"
+  NUM_STEPS=3000
+  # Run training.
+  Banner "Starting training unconditional GAN for ${NUM_STEPS} steps..."
+  python "${git_repo}/research/gan/mnist/train.py" \
+    --train_log_dir=${UNCONDITIONAL_TRAIN_DIR} \
+    --dataset_dir=${DATASET_DIR} \
+    --max_number_of_steps=${NUM_STEPS} \
+    --gan_type="unconditional" \
+    --alsologtostderr
+  Banner "Finished training unconditional GAN ${NUM_STEPS} steps."
+
+  # Run evaluation.
+  Banner "Starting evaluation of unconditional GAN..."
+  python "${git_repo}/research/gan/mnist/eval.py" \
+    --checkpoint_dir=${UNCONDITIONAL_TRAIN_DIR} \
+    --eval_dir=${UNCONDITIONAL_EVAL_DIR} \
+    --dataset_dir=${DATASET_DIR} \
+    --eval_real_images=false \
+    --classifier_filename=${FROZEN_GRAPH} \
+    --max_number_of_evaluation=1
+  Banner "Finished unconditional evaluation. See ${UNCONDITIONAL_EVAL_DIR} for output images."
+fi
+
+# Run conditional GAN.
+if [[ "$gan_type" == "conditional" ]]; then
+  CONDITIONAL_TRAIN_DIR="${TRAIN_DIR}/conditional"
+  CONDITIONAL_EVAL_DIR="${EVAL_DIR}/conditional"
+  NUM_STEPS=3000
+  # Run training.
+  Banner "Starting training conditional GAN for ${NUM_STEPS} steps..."
+  python "${git_repo}/research/gan/mnist/train.py" \
+    --train_log_dir=${CONDITIONAL_TRAIN_DIR} \
+    --dataset_dir=${DATASET_DIR} \
+    --max_number_of_steps=${NUM_STEPS} \
+    --gan_type="conditional" \
+    --alsologtostderr
+  Banner "Finished training conditional GAN ${NUM_STEPS} steps."
+
+  # Run evaluation.
+  Banner "Starting evaluation of conditional GAN..."
+  python "${git_repo}/research/gan/mnist/conditional_eval.py" \
+    --checkpoint_dir=${CONDITIONAL_TRAIN_DIR} \
+    --eval_dir=${CONDITIONAL_EVAL_DIR} \
+    --classifier_filename=${FROZEN_GRAPH} \
+    --max_number_of_evaluation=1
+  Banner "Finished conditional evaluation. See ${CONDITIONAL_EVAL_DIR} for output images."
+fi
+
+# Run InfoGAN.
+if [[ "$gan_type" == "infogan" ]]; then
+  INFOGAN_TRAIN_DIR="${TRAIN_DIR}/infogan"
+  INFOGAN_EVAL_DIR="${EVAL_DIR}/infogan"
+  NUM_STEPS=3000
+  # Run training.
+  Banner "Starting training infogan GAN for ${NUM_STEPS} steps..."
+  python "${git_repo}/research/gan/mnist/train.py" \
+    --train_log_dir=${INFOGAN_TRAIN_DIR} \
+    --dataset_dir=${DATASET_DIR} \
+    --max_number_of_steps=${NUM_STEPS} \
+    --gan_type="infogan" \
+    --alsologtostderr
+  Banner "Finished training InfoGAN ${NUM_STEPS} steps."
+
+  # Run evaluation.
+  Banner "Starting evaluation of infogan..."
+  python "${git_repo}/research/gan/mnist/infogan_eval.py" \
+    --checkpoint_dir=${INFOGAN_TRAIN_DIR} \
+    --eval_dir=${INFOGAN_EVAL_DIR} \
+    --classifier_filename=${FROZEN_GRAPH} \
+    --max_number_of_evaluation=1
+  Banner "Finished InfoGAN evaluation. See ${INFOGAN_EVAL_DIR} for output images."
+fi

research/gan/mnist/networks.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Networks for MNIST example using TFGAN."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import tensorflow as tf
+
+ds = tf.contrib.distributions
+layers = tf.contrib.layers
+tfgan = tf.contrib.gan
+
+
+def _generator_helper(
+    noise, is_conditional, one_hot_labels, weight_decay):
+  """Core MNIST generator.
+
+  This function is reused between the different GAN modes (unconditional,
+  conditional, etc).
+
+  Args:
+    noise: A 2D Tensor of shape [batch size, noise dim].
+    is_conditional: Whether to condition on labels.
+    one_hot_labels: Optional labels for conditioning.
+    weight_decay: The value of the l2 weight decay.
+
+  Returns:
+    A generated image in the range [-1, 1].
+  """
+  with tf.contrib.framework.arg_scope(
+      [layers.fully_connected, layers.conv2d_transpose],
+      activation_fn=tf.nn.relu, normalizer_fn=layers.batch_norm,
+      weights_regularizer=layers.l2_regularizer(weight_decay)):
+    net = layers.fully_connected(noise, 1024)
+    if is_conditional:
+      net = tfgan.features.condition_tensor_from_onehot(net, one_hot_labels)
+    net = layers.fully_connected(net, 7 * 7 * 128)
+    net = tf.reshape(net, [-1, 7, 7, 128])
+    net = layers.conv2d_transpose(net, 64, [4, 4], stride=2)
+    net = layers.conv2d_transpose(net, 32, [4, 4], stride=2)
+    # Make sure that generator output is in the same range as `inputs`
+    # ie [-1, 1].
+    net = layers.conv2d(
+        net, 1, [4, 4], normalizer_fn=None, activation_fn=tf.tanh)
+
+    return net
+
+
+def unconditional_generator(noise, weight_decay=2.5e-5):
+  """Generator to produce unconditional MNIST images.
+
+  Args:
+    noise: A single Tensor representing noise.
+    weight_decay: The value of the l2 weight decay.
+
+  Returns:
+    A generated image in the range [-1, 1].
+  """
+  return _generator_helper(noise, False, None, weight_decay)
+
+
+def conditional_generator(inputs, weight_decay=2.5e-5):
+  """Generator to produce MNIST images conditioned on class.
+
+  Args:
+    inputs: A 2-tuple of Tensors (noise, one_hot_labels).
+    weight_decay: The value of the l2 weight decay.
+
+  Returns:
+    A generated image in the range [-1, 1].
+  """
+  noise, one_hot_labels = inputs
+  return _generator_helper(noise, True, one_hot_labels, weight_decay)
+
+
+def infogan_generator(inputs, categorical_dim, weight_decay=2.5e-5):
+  """InfoGAN generator network on MNIST digits.
+
+  Based on a paper https://arxiv.org/abs/1606.03657, their code
+  https://github.com/openai/InfoGAN, and code by pooleb@.
+
+  Args:
+    inputs: A 3-tuple of Tensors (unstructured_noise, categorical structured
+      noise, continuous structured noise). `inputs[0]` and `inputs[2]` must be
+      2D, and `inputs[1]` must be 1D. All must have the same first dimension.
+    categorical_dim: Dimensions of the incompressible categorical noise.
+    weight_decay: The value of the l2 weight decay.
+
+  Returns:
+    A generated image in the range [-1, 1].
+  """
+  unstructured_noise, cat_noise, cont_noise = inputs
+  cat_noise_onehot = tf.one_hot(cat_noise, categorical_dim)
+  all_noise = tf.concat(
+      [unstructured_noise, cat_noise_onehot, cont_noise], axis=1)
+  return _generator_helper(all_noise, False, None, weight_decay)
+
+
+_leaky_relu = lambda x: tf.nn.leaky_relu(x, alpha=0.01)
+
+
+def _discriminator_helper(img, is_conditional, one_hot_labels, weight_decay):
+  """Core MNIST discriminator.
+
+  This function is reused between the different GAN modes (unconditional,
+  conditional, etc).
+
+  Args:
+    img: Real or generated MNIST digits. Should be in the range [-1, 1].
+    is_conditional: Whether to condition on labels.
+    one_hot_labels: Labels to optionally condition the network on.
+    weight_decay: The L2 weight decay.
+
+  Returns:
+    Final fully connected discriminator layer. [batch_size, 1024].
+  """
+  with tf.contrib.framework.arg_scope(
+      [layers.conv2d, layers.fully_connected],
+      activation_fn=_leaky_relu, normalizer_fn=None,
+      weights_regularizer=layers.l2_regularizer(weight_decay),
+      biases_regularizer=layers.l2_regularizer(weight_decay)):
+    net = layers.conv2d(img, 64, [4, 4], stride=2)
+    net = layers.conv2d(net, 128, [4, 4], stride=2)
+    net = layers.flatten(net)
+    if is_conditional:
+      net = tfgan.features.condition_tensor_from_onehot(net, one_hot_labels)
+    net = layers.fully_connected(net, 1024, normalizer_fn=layers.layer_norm)
+
+    return net
+
+
+def unconditional_discriminator(img, unused_conditioning, weight_decay=2.5e-5):
+  """Discriminator network on unconditional MNIST digits.
+
+  Args:
+    img: Real or generated MNIST digits. Should be in the range [-1, 1].
+    unused_conditioning: The TFGAN API can help with conditional GANs, which
+      would require extra `condition` information to both the generator and the
+      discriminator. Since this example is not conditional, we do not use this
+      argument.
+    weight_decay: The L2 weight decay.
+
+  Returns:
+    Logits for the probability that the image is real.
+  """
+  net = _discriminator_helper(img, False, None, weight_decay)
+  return layers.linear(net, 1)
+
+
+def conditional_discriminator(img, conditioning, weight_decay=2.5e-5):
+  """Conditional discriminator network on MNIST digits.
+
+  Args:
+    img: Real or generated MNIST digits. Should be in the range [-1, 1].
+    conditioning: A 2-tuple of Tensors representing (noise, one_hot_labels).
+    weight_decay: The L2 weight decay.
+
+  Returns:
+    Logits for the probability that the image is real.
+  """
+  _, one_hot_labels = conditioning
+  net = _discriminator_helper(img, True, one_hot_labels, weight_decay)
+  return layers.linear(net, 1)
+
+
+def infogan_discriminator(img, unused_conditioning, weight_decay=2.5e-5,
+                          categorical_dim=10, continuous_dim=2):
+  """InfoGAN discriminator network on MNIST digits.
+
+  Based on a paper https://arxiv.org/abs/1606.03657, their code
+  https://github.com/openai/InfoGAN, and code by pooleb@.
+
+  Args:
+    img: Real or generated MNIST digits. Should be in the range [-1, 1].
+    unused_conditioning: The TFGAN API can help with conditional GANs, which
+      would require extra `condition` information to both the generator and the
+      discriminator. Since this example is not conditional, we do not use this
+      argument.
+    weight_decay: The L2 weight decay.
+    categorical_dim: Dimensions of the incompressible categorical noise.
+    continuous_dim: Dimensions of the incompressible continuous noise.
+
+  Returns:
+    Logits for the probability that the image is real, and a list of posterior
+    distributions for each of the noise vectors.
+  """
+  net = _discriminator_helper(img, False, None, weight_decay)
+  logits_real = layers.fully_connected(net, 1, activation_fn=None)
+
+  # Recognition network for latent variables has an additional layer
+  with tf.contrib.framework.arg_scope([layers.batch_norm], is_training=False):
+    encoder = layers.fully_connected(net, 128, normalizer_fn=layers.batch_norm,
+                                     activation_fn=_leaky_relu)
+
+  # Compute logits for each category of categorical latent.
+  logits_cat = layers.fully_connected(
+      encoder, categorical_dim, activation_fn=None)
+  q_cat = ds.Categorical(logits_cat)
+
+  # Compute mean for Gaussian posterior of continuous latents.
+  mu_cont = layers.fully_connected(encoder, continuous_dim, activation_fn=None)
+  sigma_cont = tf.ones_like(mu_cont)
+  q_cont = ds.Normal(loc=mu_cont, scale=sigma_cont)
+
+  return logits_real, [q_cat, q_cont]

research/gan/mnist/train.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Trains a generator on MNIST data."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import functools
+
+
+import tensorflow as tf
+
+import data_provider
+import networks
+import util
+
+flags = tf.flags
+tfgan = tf.contrib.gan
+
+
+flags.DEFINE_integer('batch_size', 32, 'The number of images in each batch.')
+
+flags.DEFINE_string('train_log_dir', '/tmp/mnist/',
+                    'Directory where to write event logs.')
+
+flags.DEFINE_string('dataset_dir', None, 'Location of data.')
+
+flags.DEFINE_integer('max_number_of_steps', 20000,
+                     'The maximum number of gradient steps.')
+
+flags.DEFINE_string(
+    'gan_type', 'unconditional',
+    'Either `unconditional`, `conditional`, or `infogan`.')
+
+flags.DEFINE_integer(
+    'grid_size', 5, 'Grid size for image visualization.')
+
+
+flags.DEFINE_integer(
+    'noise_dims', 64, 'Dimensions of the generator noise vector.')
+
+FLAGS = flags.FLAGS
+
+
+def _learning_rate(gan_type):
+  # First is generator learning rate, second is discriminator learning rate.
+  return {
+      'unconditional': (1e-3, 1e-4),
+      'conditional': (1e-5, 1e-4),
+      'infogan': (0.001, 9e-5),
+  }[gan_type]
+
+
+def main(_):
+  if not tf.gfile.Exists(FLAGS.train_log_dir):
+    tf.gfile.MakeDirs(FLAGS.train_log_dir)
+
+  # Force all input processing onto CPU in order to reserve the GPU for
+  # the forward inference and back-propagation.
+  with tf.name_scope('inputs'):
+    with tf.device('/cpu:0'):
+      images, one_hot_labels, _ = data_provider.provide_data(
+          'train', FLAGS.batch_size, FLAGS.dataset_dir, num_threads=4)
+
+  # Define the GANModel tuple. Optionally, condition the GAN on the label or
+  # use an InfoGAN to learn a latent representation.
+  if FLAGS.gan_type == 'unconditional':
+    gan_model = tfgan.gan_model(
+        generator_fn=networks.unconditional_generator,
+        discriminator_fn=networks.unconditional_discriminator,
+        real_data=images,
+        generator_inputs=tf.random_normal(
+            [FLAGS.batch_size, FLAGS.noise_dims]))
+  elif FLAGS.gan_type == 'conditional':
+    noise = tf.random_normal([FLAGS.batch_size, FLAGS.noise_dims])
+    gan_model = tfgan.gan_model(
+        generator_fn=networks.conditional_generator,
+        discriminator_fn=networks.conditional_discriminator,
+        real_data=images,
+        generator_inputs=(noise, one_hot_labels))
+  elif FLAGS.gan_type == 'infogan':
+    cat_dim, cont_dim = 10, 2
+    generator_fn = functools.partial(
+        networks.infogan_generator, categorical_dim=cat_dim)
+    discriminator_fn = functools.partial(
+        networks.infogan_discriminator, categorical_dim=cat_dim,
+        continuous_dim=cont_dim)
+    unstructured_inputs, structured_inputs = util.get_infogan_noise(
+        FLAGS.batch_size, cat_dim, cont_dim, FLAGS.noise_dims)
+    gan_model = tfgan.infogan_model(
+        generator_fn=generator_fn,
+        discriminator_fn=discriminator_fn,
+        real_data=images,
+        unstructured_generator_inputs=unstructured_inputs,
+        structured_generator_inputs=structured_inputs)
+  tfgan.eval.add_gan_model_image_summaries(gan_model, FLAGS.grid_size)
+
+  # Get the GANLoss tuple. You can pass a custom function, use one of the
+  # already-implemented losses from the losses library, or use the defaults.
+  with tf.name_scope('loss'):
+    mutual_information_penalty_weight = (1.0 if FLAGS.gan_type == 'infogan'
+                                         else 0.0)
+    gan_loss = tfgan.gan_loss(
+        gan_model,
+        gradient_penalty_weight=1.0,
+        mutual_information_penalty_weight=mutual_information_penalty_weight,
+        add_summaries=True)
+    tfgan.eval.add_regularization_loss_summaries(gan_model)
+
+  # Get the GANTrain ops using custom optimizers.
+  with tf.name_scope('train'):
+    gen_lr, dis_lr = _learning_rate(FLAGS.gan_type)
+    train_ops = tfgan.gan_train_ops(
+        gan_model,
+        gan_loss,
+        generator_optimizer=tf.train.AdamOptimizer(gen_lr, 0.5),
+        discriminator_optimizer=tf.train.AdamOptimizer(dis_lr, 0.5),
+        summarize_gradients=True,
+        aggregation_method=tf.AggregationMethod.EXPERIMENTAL_ACCUMULATE_N)
+
+  # Run the alternating training loop. Skip it if no steps should be taken
+  # (used for graph construction tests).
+  status_message = tf.string_join(
+      ['Starting train step: ',
+       tf.as_string(tf.train.get_or_create_global_step())],
+      name='status_message')
+  if FLAGS.max_number_of_steps == 0: return
+  tfgan.gan_train(
+      train_ops,
+      hooks=[tf.train.StopAtStepHook(num_steps=FLAGS.max_number_of_steps),
+             tf.train.LoggingTensorHook([status_message], every_n_iter=10)],
+      logdir=FLAGS.train_log_dir,
+      get_hooks_fn=tfgan.get_joint_train_hooks())
+
+if __name__ == '__main__':
+  tf.logging.set_verbosity(tf.logging.INFO)
+  tf.app.run()

research/gan/mnist/train_test.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for mnist.train."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+import numpy as np
+
+import tensorflow as tf
+import train
+
+FLAGS = tf.flags.FLAGS
+mock = tf.test.mock
+
+
+class TrainTest(tf.test.TestCase):
+
+  @mock.patch.object(train, 'data_provider', autospec=True)
+  def test_run_one_train_step(self, mock_data_provider):
+    FLAGS.max_number_of_steps = 1
+    FLAGS.gan_type = 'unconditional'
+    FLAGS.batch_size = 5
+    FLAGS.grid_size = 1
+    tf.set_random_seed(1234)
+
+    # Mock input pipeline.
+    mock_imgs = np.zeros([FLAGS.batch_size, 28, 28, 1], dtype=np.float32)
+    mock_lbls = np.concatenate(
+        (np.ones([FLAGS.batch_size, 1], dtype=np.int32),
+         np.zeros([FLAGS.batch_size, 9], dtype=np.int32)), axis=1)
+    mock_data_provider.provide_data.return_value = (mock_imgs, mock_lbls, None)
+
+    train.main(None)
+
+  def _test_build_graph_helper(self, gan_type):
+    FLAGS.max_number_of_steps = 0
+    FLAGS.gan_type = gan_type
+
+    # Mock input pipeline.
+    mock_imgs = np.zeros([FLAGS.batch_size, 28, 28, 1], dtype=np.float32)
+    mock_lbls = np.concatenate(
+        (np.ones([FLAGS.batch_size, 1], dtype=np.int32),
+         np.zeros([FLAGS.batch_size, 9], dtype=np.int32)), axis=1)
+    with mock.patch.object(train, 'data_provider') as mock_data_provider:
+      mock_data_provider.provide_data.return_value = (
+          mock_imgs, mock_lbls, None)
+      train.main(None)
+
+  def test_build_graph_unconditional(self):
+    self._test_build_graph_helper('unconditional')
+
+  def test_build_graph_conditional(self):
+    self._test_build_graph_helper('conditional')
+
+  def test_build_graph_infogan(self):
+    self._test_build_graph_helper('infogan')
+
+if __name__ == '__main__':
+  tf.test.main()

research/gan/mnist/util.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""A utility for evaluating MNIST generative models.
+
+These functions use a pretrained MNIST classifier with ~99% eval accuracy to
+measure various aspects of the quality of generated MNIST digits.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+import numpy as np
+import tensorflow as tf
+
+ds = tf.contrib.distributions
+tfgan = tf.contrib.gan
+
+
+__all__ = [
+    'mnist_score',
+    'mnist_frechet_distance',
+    'mnist_cross_entropy',
+    'get_eval_noise_categorical',
+    'get_eval_noise_continuous_dim1',
+    'get_eval_noise_continuous_dim2',
+    'get_infogan_noise',
+]
+
+
+# Prepend `../`, since paths start from `third_party/tensorflow`.
+MODEL_GRAPH_DEF = '../tensorflow_models/gan/mnist/data/classify_mnist_graph_def.pb'
+INPUT_TENSOR = 'inputs:0'
+OUTPUT_TENSOR = 'logits:0'
+
+
+def mnist_score(images, graph_def_filename=None, input_tensor=INPUT_TENSOR,
+                output_tensor=OUTPUT_TENSOR, num_batches=1):
+  """Get MNIST logits of a fully-trained classifier.
+
+  Args:
+    images: A minibatch tensor of MNIST digits. Shape must be
+      [batch, 28, 28, 1].
+    graph_def_filename: Location of a frozen GraphDef binary file on disk. If
+      `None`, uses a default graph.
+    input_tensor: GraphDef's input tensor name.
+    output_tensor: GraphDef's output tensor name.
+    num_batches: Number of batches to split `generated_images` in to in order to
+      efficiently run them through Inception.
+
+  Returns:
+    A logits tensor of [batch, 10].
+  """
+  images.shape.assert_is_compatible_with([None, 28, 28, 1])
+
+  graph_def = _graph_def_from_par_or_disk(graph_def_filename)
+  mnist_classifier_fn = lambda x: tfgan.eval.run_image_classifier(  # pylint: disable=g-long-lambda
+      x, graph_def, input_tensor, output_tensor)
+
+  score = tfgan.eval.classifier_score(
+      images, mnist_classifier_fn, num_batches)
+  score.shape.assert_is_compatible_with([])
+
+  return score
+
+
+def mnist_frechet_distance(real_images, generated_images,
+                           graph_def_filename=None, input_tensor=INPUT_TENSOR,
+                           output_tensor=OUTPUT_TENSOR, num_batches=1):
+  """Frechet distance between real and generated images.
+
+  This technique is described in detail in https://arxiv.org/abs/1706.08500.
+  Please see TFGAN for implementation details
+  (https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/gan/
+  python/eval/python/classifier_metrics_impl.py).
+
+  Args:
+    real_images: Real images to use to compute Frechet Inception distance.
+    generated_images: Generated images to use to compute Frechet Inception
+      distance.
+    graph_def_filename: Location of a frozen GraphDef binary file on disk. If
+      `None`, uses a default graph.
+    input_tensor: GraphDef's input tensor name.
+    output_tensor: GraphDef's output tensor name.
+    num_batches: Number of batches to split images into in order to
+      efficiently run them through the classifier network.
+
+  Returns:
+    The Frechet distance. A floating-point scalar.
+  """
+  real_images.shape.assert_is_compatible_with([None, 28, 28, 1])
+  generated_images.shape.assert_is_compatible_with([None, 28, 28, 1])
+
+  graph_def = _graph_def_from_par_or_disk(graph_def_filename)
+  mnist_classifier_fn = lambda x: tfgan.eval.run_image_classifier(  # pylint: disable=g-long-lambda
+      x, graph_def, input_tensor, output_tensor)
+
+  frechet_distance = tfgan.eval.frechet_classifier_distance(
+      real_images, generated_images, mnist_classifier_fn, num_batches)
+  frechet_distance.shape.assert_is_compatible_with([])
+
+  return frechet_distance
+
+
+def mnist_cross_entropy(images, one_hot_labels, graph_def_filename=None,
+                        input_tensor=INPUT_TENSOR, output_tensor=OUTPUT_TENSOR):
+  """Returns the cross entropy loss of the classifier on images.
+
+  Args:
+    images: A minibatch tensor of MNIST digits. Shape must be
+      [batch, 28, 28, 1].
+    one_hot_labels: The one hot label of the examples. Tensor size is
+      [batch, 10].
+    graph_def_filename: Location of a frozen GraphDef binary file on disk. If
+      `None`, uses a default graph embedded in the par file.
+    input_tensor: GraphDef's input tensor name.
+    output_tensor: GraphDef's output tensor name.
+
+  Returns:
+    A scalar Tensor representing the cross entropy of the image minibatch.
+  """
+  graph_def = _graph_def_from_par_or_disk(graph_def_filename)
+
+  logits = tfgan.eval.run_image_classifier(
+      images, graph_def, input_tensor, output_tensor)
+  return tf.losses.softmax_cross_entropy(
+      one_hot_labels, logits, loss_collection=None)
+
+
+# (joelshor): Refactor the `eval_noise` functions to reuse code.
+def get_eval_noise_categorical(
+    noise_samples, categorical_sample_points, continuous_sample_points,
+    unstructured_noise_dims, continuous_noise_dims):
+  """Create noise showing impact of categorical noise in InfoGAN.
+
+  Categorical noise is constant across columns. Other noise is constant across
+  rows.
+
+  Args:
+    noise_samples: Number of non-categorical noise samples to use.
+    categorical_sample_points: Possible categorical noise points to sample.
+    continuous_sample_points: Possible continuous noise points to sample.
+    unstructured_noise_dims: Dimensions of the unstructured noise.
+    continuous_noise_dims: Dimensions of the continuous noise.
+
+  Returns:
+    Unstructured noise, categorical noise, continuous noise numpy arrays. Each
+    should have shape [noise_samples, ?].
+  """
+  rows, cols = noise_samples, len(categorical_sample_points)
+
+  # Take random draws for non-categorical noise, making sure they are constant
+  # across columns.
+  unstructured_noise = []
+  for _ in xrange(rows):
+    cur_sample = np.random.normal(size=[1, unstructured_noise_dims])
+    unstructured_noise.extend([cur_sample] * cols)
+  unstructured_noise = np.concatenate(unstructured_noise)
+
+  continuous_noise = []
+  for _ in xrange(rows):
+    cur_sample = np.random.choice(
+        continuous_sample_points, size=[1, continuous_noise_dims])
+    continuous_noise.extend([cur_sample] * cols)
+  continuous_noise = np.concatenate(continuous_noise)
+
+  # Increase categorical noise from left to right, making sure they are constant
+  # across rows.
+  categorical_noise = np.tile(categorical_sample_points, rows)
+
+  return unstructured_noise, categorical_noise, continuous_noise
+
+
+def get_eval_noise_continuous_dim1(
+    noise_samples, categorical_sample_points, continuous_sample_points,
+    unstructured_noise_dims, continuous_noise_dims):  # pylint:disable=unused-argument
+  """Create noise showing impact of first dim continuous noise in InfoGAN.
+
+  First dimension of continuous noise is constant across columns. Other noise is
+  constant across rows.
+
+  Args:
+    noise_samples: Number of non-categorical noise samples to use.
+    categorical_sample_points: Possible categorical noise points to sample.
+    continuous_sample_points: Possible continuous noise points to sample.
+    unstructured_noise_dims: Dimensions of the unstructured noise.
+    continuous_noise_dims: Dimensions of the continuous noise.
+
+  Returns:
+    Unstructured noise, categorical noise, continuous noise numpy arrays.
+  """
+  rows, cols = noise_samples, len(continuous_sample_points)
+
+  # Take random draws for non-first-dim-continuous noise, making sure they are
+  # constant across columns.
+  unstructured_noise = []
+  for _ in xrange(rows):
+    cur_sample = np.random.normal(size=[1, unstructured_noise_dims])
+    unstructured_noise.extend([cur_sample] * cols)
+  unstructured_noise = np.concatenate(unstructured_noise)
+
+  categorical_noise = []
+  for _ in xrange(rows):
+    cur_sample = np.random.choice(categorical_sample_points)
+    categorical_noise.extend([cur_sample] * cols)
+  categorical_noise = np.array(categorical_noise)
+
+  cont_noise_dim2 = []
+  for _ in xrange(rows):
+    cur_sample = np.random.choice(continuous_sample_points, size=[1, 1])
+    cont_noise_dim2.extend([cur_sample] * cols)
+  cont_noise_dim2 = np.concatenate(cont_noise_dim2)
+
+  # Increase first dimension of continuous noise from left to right, making sure
+  # they are constant across rows.
+  cont_noise_dim1 = np.expand_dims(np.tile(continuous_sample_points, rows), 1)
+
+  continuous_noise = np.concatenate((cont_noise_dim1, cont_noise_dim2), 1)
+
+  return unstructured_noise, categorical_noise, continuous_noise
+
+
+def get_eval_noise_continuous_dim2(
+    noise_samples, categorical_sample_points, continuous_sample_points,
+    unstructured_noise_dims, continuous_noise_dims):  # pylint:disable=unused-argument
+  """Create noise showing impact of second dim of continuous noise in InfoGAN.
+
+  Second dimension of continuous noise is constant across columns. Other noise
+  is constant across rows.
+
+  Args:
+    noise_samples: Number of non-categorical noise samples to use.
+    categorical_sample_points: Possible categorical noise points to sample.
+    continuous_sample_points: Possible continuous noise points to sample.
+    unstructured_noise_dims: Dimensions of the unstructured noise.
+    continuous_noise_dims: Dimensions of the continuous noise.
+
+  Returns:
+    Unstructured noise, categorical noise, continuous noise numpy arrays.
+  """
+  rows, cols = noise_samples, len(continuous_sample_points)
+
+  # Take random draws for non-first-dim-continuous noise, making sure they are
+  # constant across columns.
+  unstructured_noise = []
+  for _ in xrange(rows):
+    cur_sample = np.random.normal(size=[1, unstructured_noise_dims])
+    unstructured_noise.extend([cur_sample] * cols)
+  unstructured_noise = np.concatenate(unstructured_noise)
+
+  categorical_noise = []
+  for _ in xrange(rows):
+    cur_sample = np.random.choice(categorical_sample_points)
+    categorical_noise.extend([cur_sample] * cols)
+  categorical_noise = np.array(categorical_noise)
+
+  cont_noise_dim1 = []
+  for _ in xrange(rows):
+    cur_sample = np.random.choice(continuous_sample_points, size=[1, 1])
+    cont_noise_dim1.extend([cur_sample] * cols)
+  cont_noise_dim1 = np.concatenate(cont_noise_dim1)
+
+  # Increase first dimension of continuous noise from left to right, making sure
+  # they are constant across rows.
+  cont_noise_dim2 = np.expand_dims(np.tile(continuous_sample_points, rows), 1)
+
+  continuous_noise = np.concatenate((cont_noise_dim1, cont_noise_dim2), 1)
+
+  return unstructured_noise, categorical_noise, continuous_noise
+
+
+def get_infogan_noise(batch_size, categorical_dim, structured_continuous_dim,
+                      total_continuous_noise_dims):
+  """Get unstructured and structured noise for InfoGAN.
+
+  Args:
+    batch_size: The number of noise vectors to generate.
+    categorical_dim: The number of categories in the categorical noise.
+    structured_continuous_dim: The number of dimensions of the uniform
+      continuous noise.
+    total_continuous_noise_dims: The number of continuous noise dimensions. This
+      number includes the structured and unstructured noise.
+
+  Returns:
+    A 2-tuple of structured and unstructured noise. First element is the
+    unstructured noise, and the second is a 2-tuple of
+    (categorical structured noise, continuous structured noise).
+  """
+  # Get unstructurd noise.
+  unstructured_noise = tf.random_normal(
+      [batch_size, total_continuous_noise_dims - structured_continuous_dim])
+
+  # Get categorical noise Tensor.
+  categorical_dist = ds.Categorical(logits=tf.zeros([categorical_dim]))
+  categorical_noise = categorical_dist.sample([batch_size])
+
+  # Get continuous noise Tensor.
+  continuous_dist = ds.Uniform(-tf.ones([structured_continuous_dim]),
+                               tf.ones([structured_continuous_dim]))
+  continuous_noise = continuous_dist.sample([batch_size])
+
+  return [unstructured_noise], [categorical_noise, continuous_noise]
+
+
+def _graph_def_from_par_or_disk(filename):
+  if filename is None:
+    return tfgan.eval.get_graph_def_from_resource(MODEL_GRAPH_DEF)
+  else:
+    return tfgan.eval.get_graph_def_from_disk(filename)

research/gan/mnist/util_test.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for mnist.util."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+import tensorflow as tf
+from google3.third_party.tensorflow_models.gan.mnist  import util
+
+
+# pylint: disable=line-too-long
+# This is a real digit `5` from MNIST.
+REAL_DIGIT = [[-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -0.9765625, -0.859375, -0.859375, -0.859375, -0.015625, 0.0625, 0.3671875, -0.796875, 0.296875, 0.9921875, 0.9296875, -0.0078125, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -0.765625, -0.71875, -0.265625, 0.203125, 0.328125, 0.9765625, 0.9765625, 0.9765625, 0.9765625, 0.9765625, 0.7578125, 0.34375, 0.9765625, 0.890625, 0.5234375, -0.5, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -0.6171875, 0.859375, 0.9765625, 0.9765625, 0.9765625, 0.9765625, 0.9765625, 0.9765625, 0.9765625, 0.9765625, 0.9609375, -0.2734375, -0.359375, -0.359375, -0.5625, -0.6953125, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -0.859375, 0.7109375, 0.9765625, 0.9765625, 0.9765625, 0.9765625, 0.9765625, 0.546875, 0.421875, 0.9296875, 0.8828125, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -0.375, 0.21875, -0.1640625, 0.9765625, 0.9765625, 0.6015625, -0.9140625, -1.0, -0.6640625, 0.203125, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -0.890625, -0.9921875, 0.203125, 0.9765625, -0.296875, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, 0.0859375, 0.9765625, 0.484375, -0.984375, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -0.9140625, 0.484375, 0.9765625, -0.453125, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -0.7265625, 0.8828125, 0.7578125, 0.25, -0.15625, -0.9921875, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -0.3671875, 0.875, 0.9765625, 0.9765625, -0.0703125, -0.8046875, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -0.6484375, 0.453125, 0.9765625, 0.9765625, 0.171875, -0.7890625, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -0.875, -0.2734375, 0.96875, 0.9765625, 0.4609375, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, 0.9453125, 0.9765625, 0.9453125, -0.5, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -0.640625, 0.015625, 0.4296875, 0.9765625, 0.9765625, 0.6171875, -0.984375, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -0.6953125, 0.15625, 0.7890625, 0.9765625, 0.9765625, 0.9765625, 0.953125, 0.421875, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -0.8125, -0.109375, 0.7265625, 0.9765625, 0.9765625, 0.9765625, 0.9765625, 0.5703125, -0.390625, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -0.8203125, -0.484375, 0.6640625, 0.9765625, 0.9765625, 0.9765625, 0.9765625, 0.546875, -0.3671875, -0.984375, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -0.859375, 0.3359375, 0.7109375, 0.9765625, 0.9765625, 0.9765625, 0.9765625, 0.5234375, -0.375, -0.9296875, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -0.5703125, 0.34375, 0.765625, 0.9765625, 0.9765625, 0.9765625, 0.9765625, 0.90625, 0.0390625, -0.9140625, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, 0.0625, 0.9765625, 0.9765625, 0.9765625, 0.65625, 0.0546875, 0.03125, -0.875, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0]]
+ONE_HOT = [[0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0]]
+
+# Uniform noise in [-1, 1].
+FAKE_DIGIT = [[0.778958797454834, 0.8792028427124023, 0.07099628448486328, 0.8518857955932617, -0.3541288375854492, -0.7431280612945557, -0.12607860565185547, 0.17328786849975586, 0.6749839782714844, -0.5402040481567383, 0.9034252166748047, 0.2420203685760498, 0.3455841541290283, 0.1937558650970459, 0.9989571571350098, 0.9039363861083984, -0.955411434173584, 0.6228537559509277, -0.33131909370422363, 0.9653763771057129, 0.864208459854126, -0.05056142807006836, 0.12686634063720703, -0.09225749969482422, 0.49758028984069824, 0.08698725700378418, 0.5533185005187988, 0.20227980613708496], [0.8400616645812988, 0.7409703731536865, -0.6215496063232422, -0.53228759765625, -0.20184636116027832, -0.8568699359893799, -0.8662903308868408, -0.8735041618347168, -0.11022663116455078, -0.8418543338775635, 0.8193502426147461, -0.901512622833252, -0.7680232524871826, 0.6209826469421387, 0.06459426879882812, 0.5341305732727051, -0.4078702926635742, -0.13658642768859863, 0.6602437496185303, 0.848508358001709, -0.23431801795959473, 0.5995683670043945, -0.9807922840118408, 0.2657158374786377, -0.8068397045135498, 0.2438051700592041, -0.2116842269897461, 0.011460304260253906], [0.00040912628173828125, -0.058798789978027344, 0.3239307403564453, 0.5040378570556641, -0.03192305564880371, -0.4816470146179199, -0.14559340476989746, -0.9231269359588623, -0.6602556705474854, -0.2537086009979248, -0.11059761047363281, -0.8174862861633301, 0.6180260181427002, 0.7245023250579834, 0.5007762908935547, -0.1575303077697754, -0.0167086124420166, 0.7173266410827637, 0.1126704216003418, -0.9878268241882324, 0.4538843631744385, -0.4422755241394043, -0.7899672985076904, 0.7349567413330078, -0.4448075294494629, -0.7548923492431641, -0.5739786624908447, 0.30504918098449707], [-0.8488152027130127, 0.43424463272094727, 0.7724254131317139, 0.43314504623413086, 0.7352848052978516, -0.26010799407958984, 0.43951940536499023, -0.7642686367034912, -0.657184362411499, -0.9933960437774658, -0.47258639335632324, 0.10390830039978027, 0.11454653739929199, -0.6156411170959473, -0.23431062698364258, -0.6897118091583252, -0.5721850395202637, -0.3574075698852539, 0.13927006721496582, -0.6530766487121582, 0.32231855392456055, 0.6294634342193604, 0.5507853031158447, -0.4867420196533203, 0.4329197406768799, 0.6168341636657715, -0.8720219135284424, 0.8639121055603027], [0.02407360076904297, -0.11185193061828613, -0.38637852668762207, -0.8244953155517578, -0.648916482925415, 0.44907593727111816, -0.368192195892334, 0.0190126895904541, -0.9450500011444092, 0.41033458709716797, -0.7877917289733887, 0.617938756942749, 0.551692008972168, -0.48288512229919434, 0.019921541213989258, -0.8765170574188232, 0.5651748180389404, 0.850874662399292, -0.5792787075042725, 0.1748213768005371, -0.6905481815338135, -0.521310567855835, 0.062479496002197266, 0.17763280868530273, -0.4628307819366455, 0.8870463371276855, -0.8685822486877441, -0.29169774055480957], [-0.14687561988830566, 0.8801963329315186, 0.11353135108947754, -0.6009430885314941, 0.8818719387054443, -0.8621203899383545, -0.48749589920043945, 0.5224916934967041, 0.7050364017486572, -0.9968757629394531, 0.7235188484191895, 0.662771463394165, 0.588390588760376, -0.9624209403991699, 0.39203453063964844, -0.2210233211517334, 0.9266352653503418, -0.9132544994354248, -0.5175333023071289, 0.7251780033111572, 0.3030557632446289, 0.5743863582611084, 0.14350247383117676, -0.3735086917877197, -0.6299927234649658, 0.5088682174682617, -0.6953752040863037, 0.23583650588989258], [0.25864362716674805, 0.5736973285675049, -0.3975222110748291, -0.6369199752807617, 0.5376672744750977, -0.19951462745666504, 0.6843924522399902, 0.6296660900115967, 0.36865997314453125, -0.7243289947509766, 0.5768749713897705, -0.5493001937866211, 0.31238412857055664, 0.21019506454467773, -0.368206262588501, -0.33148622512817383, -0.3421964645385742, -0.15616083145141602, 0.6617193222045898, 0.4400820732116699, 0.7893157005310059, -0.2935945987701416, 0.6241741180419922, -0.26036930084228516, -0.6958446502685547, 0.27047157287597656, -0.9095940589904785, 0.9525108337402344], [-0.5233585834503174, -0.45003342628479004, 0.15099048614501953, 0.6257956027984619, 0.9017877578735352, -0.18155455589294434, -0.20237135887145996, -0.014468908309936523, 0.01797318458557129, -0.5453977584838867, 0.21428155899047852, -0.9678947925567627, 0.5137600898742676, -0.1094369888305664, -0.13572359085083008, -0.1704423427581787, -0.9122319221496582, 0.8274900913238525, -0.11746454238891602, -0.8701446056365967, -0.9545385837554932, -0.6735866069793701, 0.9445557594299316, -0.3842940330505371, -0.6240942478179932, -0.1673595905303955, -0.3959221839904785, -0.05602693557739258], [0.8833198547363281, 0.14288711547851562, -0.9623878002166748, -0.26968836784362793, 0.9689288139343262, -0.3792128562927246, 0.2520296573638916, 0.1477947235107422, -0.24453139305114746, 0.94329833984375, 0.8014910221099854, 0.5443501472473145, 0.8486857414245605, -0.0795745849609375, -0.30250096321105957, -0.909733772277832, 0.8387842178344727, -0.41989898681640625, -0.8364224433898926, 0.04792976379394531, -0.38036274909973145, 0.12747883796691895, -0.5356688499450684, -0.04269552230834961, -0.2070469856262207, -0.6911153793334961, 0.33954668045043945, 0.25260138511657715], [0.3128373622894287, 0.36142778396606445, -0.2512378692626953, -0.6497259140014648, -0.21787405014038086, -0.9972476959228516, -0.026904821395874023, -0.4214746952056885, -0.3354766368865967, -0.6112637519836426, -0.7594058513641357, 0.09093379974365234, -0.7331845760345459, 0.5222046375274658, -0.8997514247894287, -0.3749384880065918, -0.0775001049041748, -0.26296448707580566, 0.404325008392334, -0.25776195526123047, 0.9136955738067627, 0.2623283863067627, -0.6411356925964355, 0.6646602153778076, -0.12833356857299805, -0.4184732437133789, -0.3663449287414551, -0.5468103885650635], [0.3770618438720703, 0.5572817325592041, -0.3657073974609375, -0.5056321620941162, 0.6555137634277344, 0.9557509422302246, -0.6900768280029297, -0.4980638027191162, 0.05510354042053223, -0.610318660736084, 0.9753992557525635, 0.7569930553436279, 0.4011664390563965, 0.8439173698425293, -0.5921270847320557, -0.2775266170501709, -0.061129093170166016, -0.49707984924316406, 0.5820951461791992, 0.008175849914550781, 0.06372833251953125, 0.3061811923980713, -0.5091361999511719, 0.9751057624816895, 0.4571402072906494, 0.6769094467163086, -0.46695923805236816, 0.44080281257629395], [-0.6510493755340576, -0.032715559005737305, 0.3482983112335205, -0.8135421276092529, 0.1506943702697754, 0.5220685005187988, -0.8834004402160645, -0.908900260925293, 0.3211519718170166, 0.896381139755249, -0.9448244571685791, -0.6193962097167969, -0.009401559829711914, 0.38227057456970215, -0.9219558238983154, -0.029483318328857422, -0.3889012336730957, 0.5242419242858887, 0.7338912487030029, -0.8713808059692383, -0.04948568344116211, -0.797940731048584, -0.9933724403381348, -0.262890100479126, -0.7165846824645996, -0.9763388633728027, -0.4105076789855957, 0.5907857418060303], [-0.6091890335083008, 0.7921168804168701, -0.033307790756225586, -0.9177074432373047, 0.4553513526916504, 0.5754055976867676, 0.6747269630432129, 0.0015664100646972656, -0.36865878105163574, -0.7999486923217773, 0.993431568145752, -0.7310445308685303, -0.49965715408325195, -0.028263330459594727, -0.20190834999084473, -0.7398116588592529, 0.10513901710510254, 0.22136950492858887, 0.42579007148742676, -0.4703383445739746, -0.8729751110076904, 0.28951215744018555, -0.3110074996948242, 0.9935362339019775, -0.29533815383911133, -0.3384673595428467, -0.07292437553405762, 0.8471579551696777], [-0.04648447036743164, -0.6876633167266846, -0.4921104907989502, -0.1925184726715088, -0.5843420028686523, -0.2852492332458496, 0.1251826286315918, -0.5709969997406006, 0.6744673252105713, 0.17812824249267578, 0.675086259841919, 0.1219022274017334, 0.6496286392211914, 0.05892682075500488, 0.33709263801574707, -0.9087743759155273, -0.4785141944885254, 0.2689247131347656, 0.3600578308105469, -0.6822943687438965, -0.0801839828491211, 0.9234893321990967, 0.5037875175476074, -0.8148105144500732, 0.6820621490478516, -0.8345451354980469, 0.9400079250335693, -0.752265453338623], [0.4599113464355469, 0.0292510986328125, -0.7475054264068604, -0.4074263572692871, 0.8800950050354004, 0.41760849952697754, 0.3225588798522949, 0.8359887599945068, -0.8655965328216553, 0.17258906364440918, 0.752063512802124, -0.48968982696533203, -0.9149389266967773, -0.46224403381347656, -0.26379919052124023, -0.9342350959777832, -0.5444085597991943, -0.4576752185821533, -0.12544608116149902, -0.3810274600982666, -0.5277583599090576, 0.3267025947570801, 0.4762594699859619, 0.09010529518127441, -0.2434844970703125, -0.8013439178466797, -0.23540687561035156, 0.8844473361968994], [-0.8922028541564941, -0.7912418842315674, -0.14429497718811035, -0.3925011157989502, 0.1870102882385254, -0.28865933418273926, 0.11481523513793945, -0.9174098968505859, -0.5264348983764648, -0.7296302318572998, 0.44644737243652344, 0.11140275001525879, 0.08105874061584473, 0.3871574401855469, -0.5030152797698975, -0.9560322761535645, -0.07085466384887695, 0.7949709892272949, 0.37600183486938477, -0.8664641380310059, -0.08428549766540527, 0.9169652462005615, 0.9479010105133057, 0.3576698303222656, 0.6677501201629639, -0.34919166564941406, -0.11635351181030273, -0.05966901779174805], [-0.9716870784759521, -0.7901370525360107, 0.9152195453643799, -0.42171406745910645, 0.20472931861877441, -0.9847748279571533, 0.9935972690582275, -0.5975158214569092, 0.9557573795318604, 0.41234254837036133, -0.26670074462890625, 0.869682788848877, 0.4443938732147217, -0.13968205451965332, -0.7745835781097412, -0.5692052841186523, 0.321591854095459, -0.3146944046020508, -0.3921670913696289, -0.36029601097106934, 0.33528995513916016, 0.029220104217529297, -0.8788590431213379, 0.6883881092071533, -0.8260040283203125, -0.43041515350341797, 0.42810845375061035, 0.40050792694091797], [0.6319584846496582, 0.32369279861450195, -0.6308813095092773, 0.07861590385437012, 0.5387494564056396, -0.902024507522583, -0.5346510410308838, 0.10787153244018555, 0.36048197746276855, 0.7801573276519775, -0.39187049865722656, 0.409869909286499, 0.5972449779510498, -0.7578165531158447, 0.30403685569763184, -0.7885205745697021, 0.01341390609741211, -0.023469209671020508, -0.11588907241821289, -0.941727876663208, -0.09618496894836426, 0.8042230606079102, 0.4683551788330078, -0.6497313976287842, 0.7443571090698242, 0.7150907516479492, -0.5759801864624023, -0.6523513793945312], [0.3150167465209961, -0.1853046417236328, 0.1839759349822998, -0.9234504699707031, -0.666614294052124, -0.740748405456543, 0.5700008869171143, 0.4091987609863281, -0.8912513256072998, 0.027419567108154297, 0.07219696044921875, -0.3128829002380371, 0.9465951919555664, 0.8715424537658691, -0.4559173583984375, -0.5862705707550049, -0.6734106540679932, 0.8419013023376465, -0.5523068904876709, -0.5019669532775879, -0.4969966411590576, -0.030994892120361328, 0.7572557926177979, 0.3824281692504883, 0.6366133689880371, -0.13271117210388184, 0.632627010345459, -0.9462625980377197], [-0.3923935890197754, 0.5466675758361816, -0.9815363883972168, -0.12867975234985352, 0.9932572841644287, -0.712486743927002, -0.17107725143432617, -0.41582536697387695, 0.10990118980407715, 0.11867499351501465, -0.7774863243103027, -0.9382553100585938, 0.4290587902069092, -0.1412363052368164, -0.498490571975708, 0.7793624401092529, 0.9015710353851318, -0.5107312202453613, 0.12394595146179199, 0.4988982677459717, -0.5731511116027832, -0.8105146884918213, 0.19758391380310059, 0.4081540107727051, -0.20432400703430176, 0.9924988746643066, -0.16952204704284668, 0.45574188232421875], [-0.023319005966186523, -0.3514130115509033, 0.4652390480041504, 0.7690563201904297, 0.7906622886657715, 0.9922001361846924, -0.6670932769775391, -0.5720524787902832, 0.3491201400756836, -0.18706512451171875, 0.4899415969848633, -0.5637645721435547, 0.8986928462982178, -0.7359066009521484, -0.6342356204986572, 0.6608924865722656, 0.6014213562011719, 0.7906208038330078, 0.19034814834594727, 0.16884255409240723, -0.8803558349609375, 0.4060337543487549, 0.9862797260284424, -0.48758840560913086, 0.3894319534301758, 0.4226539134979248, 0.00042724609375, -0.3623659610748291], [0.7065057754516602, -0.4103825092315674, 0.3343489170074463, -0.9341757297515869, -0.20749878883361816, -0.3589310646057129, -0.9470062255859375, 0.7851138114929199, -0.74678635597229, -0.05102086067199707, 0.16727972030639648, 0.12385678291320801, -0.41132497787475586, 0.5856695175170898, 0.06311273574829102, 0.8238284587860107, 0.2257852554321289, -0.6452250480651855, -0.6373245716094971, -0.1247873306274414, 0.0021851062774658203, -0.5648598670959473, 0.8261771202087402, 0.3713812828063965, -0.7185893058776855, 0.45911359786987305, 0.6722264289855957, -0.31804966926574707], [-0.2952606678009033, -0.12052011489868164, 0.9074821472167969, -0.9850583076477051, 0.6732273101806641, 0.7954013347625732, 0.938248872756958, -0.35915136337280273, -0.8291504383087158, 0.6020793914794922, -0.30096912384033203, 0.6136975288391113, 0.46443629264831543, -0.9057025909423828, 0.43993186950683594, -0.11653375625610352, 0.9514555931091309, 0.7985148429870605, -0.6911783218383789, -0.931948184967041, 0.06829452514648438, 0.711458683013916, 0.17953252792358398, 0.19076848030090332, 0.6339912414550781, -0.38822221755981445, 0.09893226623535156, -0.7663829326629639], [0.2640511989593506, 0.16821074485778809, 0.9845118522644043, 0.13789963722229004, -0.1097862720489502, 0.24889636039733887, 0.12135696411132812, 0.4419589042663574, -0.022361040115356445, 0.3793671131134033, 0.7053709030151367, 0.5814387798309326, 0.24962759017944336, -0.40136122703552246, -0.364804744720459, -0.2518625259399414, 0.25757312774658203, 0.6828348636627197, -0.08237361907958984, -0.07745933532714844, -0.9299273490905762, -0.3066704273223877, 0.22127842903137207, -0.6690163612365723, 0.7477891445159912, 0.5738682746887207, -0.9027633666992188, 0.33333301544189453], [0.44417595863342285, 0.5279359817504883, 0.36589932441711426, -0.049490928649902344, -0.3003063201904297, 0.8447706699371338, 0.8139019012451172, -0.4958505630493164, -0.3640005588531494, -0.7731854915618896, 0.4138674736022949, 0.3080577850341797, -0.5615301132202148, -0.4247474670410156, -0.25992918014526367, -0.7983508110046387, -0.4926283359527588, 0.08595061302185059, 0.9205574989318848, 0.6025278568267822, 0.37128734588623047, -0.1856670379638672, 0.9658422470092773, -0.6652145385742188, -0.2217710018157959, 0.8311929702758789, 0.9117603302001953, 0.4660191535949707], [-0.6164810657501221, -0.9509942531585693, 0.30228447914123535, 0.39792585372924805, -0.45194506645202637, -0.3315877914428711, 0.5393261909484863, 0.030223369598388672, 0.6013507843017578, 0.020318031311035156, 0.17352747917175293, -0.25470709800720215, -0.7904071807861328, 0.5383470058441162, -0.43855953216552734, -0.350492000579834, -0.5038156509399414, 0.0511777400970459, 0.9628784656524658, -0.520085334777832, 0.33083176612854004, 0.3698580265045166, 0.9121549129486084, -0.664802074432373, -0.45629024505615234, -0.44208550453186035, 0.6502475738525391, -0.597346305847168], [-0.5405080318450928, 0.2640984058380127, -0.2119138240814209, 0.04504036903381348, -0.10604047775268555, -0.8093295097351074, 0.20915007591247559, 0.08994936943054199, 0.5347979068756104, -0.550915002822876, 0.3008608818054199, 0.2416989803314209, 0.025292158126831055, -0.2790646553039551, 0.5850257873535156, 0.7020430564880371, -0.9527721405029297, 0.8273317813873291, -0.8028199672698975, 0.3686351776123047, -0.9391682147979736, -0.4261181354522705, 0.1674947738647461, 0.27993154525756836, 0.7567532062530518, 0.9245085716247559, 0.9245762825012207, -0.296356201171875], [-0.7747671604156494, 0.7632861137390137, 0.10236740112304688, 0.14044547080993652, 0.9318621158599854, -0.5622165203094482, -0.6605725288391113, -0.8286299705505371, 0.8717818260192871, -0.22177672386169434, -0.6030778884887695, 0.20917797088623047, 0.31551361083984375, 0.7741527557373047, -0.3320643901824951, -0.9014863967895508, 0.44268250465393066, 0.25649309158325195, -0.5621528625488281, -0.6077632904052734, 0.21485304832458496, -0.658627986907959, -0.9116294384002686, -0.294114351272583, 0.0452420711517334, 0.8542745113372803, 0.7148771286010742, 0.3244490623474121]]
+# pylint: enable=line-too-long
+
+
+def real_digit():
+  return tf.expand_dims(tf.expand_dims(REAL_DIGIT, 0), -1)
+
+
+def fake_digit():
+  return tf.expand_dims(tf.expand_dims(FAKE_DIGIT, 0), -1)
+
+
+def one_hot_real():
+  return tf.constant(ONE_HOT)
+
+
+def one_hot1():
+  return tf.constant([[1.0] + [0.0] * 9])
+
+
+class MnistScoreTest(tf.test.TestCase):
+
+  def test_any_batch_size(self):
+    inputs = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
+    mscore = util.mnist_score(inputs)
+    for batch_size in [4, 16, 30]:
+      with self.test_session() as sess:
+        sess.run(mscore, feed_dict={inputs: np.zeros([batch_size, 28, 28, 1])})
+
+  def test_deterministic(self):
+    m_score = util.mnist_score(real_digit())
+    with self.test_session():
+      m_score1 = m_score.eval()
+      m_score2 = m_score.eval()
+    self.assertEqual(m_score1, m_score2)
+
+    with self.test_session():
+      m_score3 = m_score.eval()
+    self.assertEqual(m_score1, m_score3)
+
+  def test_single_example_correct(self):
+    real_score = util.mnist_score(real_digit())
+    fake_score = util.mnist_score(fake_digit())
+    with self.test_session():
+      self.assertNear(1.0, real_score.eval(), 1e-6)
+      self.assertNear(1.0, fake_score.eval(), 1e-6)
+
+  def test_minibatch_correct(self):
+    mscore = util.mnist_score(
+        tf.concat([real_digit(), real_digit(), fake_digit()], 0))
+    with self.test_session():
+      self.assertNear(1.612828, mscore.eval(), 1e-6)
+
+  def test_batch_splitting_doesnt_change_value(self):
+    for num_batches in [1, 2, 4, 8]:
+      mscore = util.mnist_score(
+          tf.concat([real_digit()] * 4 + [fake_digit()] * 4, 0),
+          num_batches=num_batches)
+      with self.test_session():
+        self.assertNear(1.649209, mscore.eval(), 1e-6)
+
+
+class MnistFrechetDistanceTest(tf.test.TestCase):
+
+  def test_any_batch_size(self):
+    inputs = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
+    fdistance = util.mnist_frechet_distance(inputs, inputs)
+    for batch_size in [4, 16, 30]:
+      with self.test_session() as sess:
+        sess.run(fdistance,
+                 feed_dict={inputs: np.zeros([batch_size, 28, 28, 1])})
+
+  def test_deterministic(self):
+    fdistance = util.mnist_frechet_distance(
+        tf.concat([real_digit()] * 2, 0),
+        tf.concat([fake_digit()] * 2, 0))
+    with self.test_session():
+      fdistance1 = fdistance.eval()
+      fdistance2 = fdistance.eval()
+    self.assertNear(fdistance1, fdistance2, 2e-1)
+
+    with self.test_session():
+      fdistance3 = fdistance.eval()
+    self.assertNear(fdistance1, fdistance3, 2e-1)
+
+  def test_single_example_correct(self):
+    fdistance = util.mnist_frechet_distance(
+        tf.concat([real_digit()] * 2, 0),
+        tf.concat([real_digit()] * 2, 0))
+    with self.test_session():
+      self.assertNear(0.0, fdistance.eval(), 2e-1)
+
+  def test_minibatch_correct(self):
+    fdistance = util.mnist_frechet_distance(
+        tf.concat([real_digit(), real_digit(), fake_digit()], 0),
+        tf.concat([real_digit(), fake_digit(), fake_digit()], 0))
+    with self.test_session():
+      self.assertNear(43.5, fdistance.eval(), 2e-1)
+
+  def test_batch_splitting_doesnt_change_value(self):
+    for num_batches in [1, 2, 4, 8]:
+      fdistance = util.mnist_frechet_distance(
+          tf.concat([real_digit()] * 6 + [fake_digit()] * 2, 0),
+          tf.concat([real_digit()] * 2 + [fake_digit()] * 6, 0),
+          num_batches=num_batches)
+      with self.test_session():
+        self.assertNear(97.8, fdistance.eval(), 2e-1)
+
+
+class MnistCrossEntropyTest(tf.test.TestCase):
+
+  def test_any_batch_size(self):
+    num_classes = 10
+    one_label = np.array([[1] + [0] * (num_classes - 1)])
+    inputs = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
+    one_hot_label = tf.placeholder(tf.int32, shape=[None, num_classes])
+    entropy = util.mnist_cross_entropy(inputs, one_hot_label)
+    for batch_size in [4, 16, 30]:
+      with self.test_session() as sess:
+        sess.run(entropy, feed_dict={
+            inputs: np.zeros([batch_size, 28, 28, 1]),
+            one_hot_label: np.concatenate([one_label] * batch_size)})
+
+  def test_deterministic(self):
+    xent = util.mnist_cross_entropy(real_digit(), one_hot_real())
+    with self.test_session():
+      ent1 = xent.eval()
+      ent2 = xent.eval()
+    self.assertEqual(ent1, ent2)
+
+    with self.test_session():
+      ent3 = xent.eval()
+    self.assertEqual(ent1, ent3)
+
+  def test_single_example_correct(self):
+    # The correct label should have low cross entropy.
+    correct_xent = util.mnist_cross_entropy(real_digit(), one_hot_real())
+    # The incorrect label should have high cross entropy.
+    wrong_xent = util.mnist_cross_entropy(real_digit(), one_hot1())
+    # A random digit should have medium cross entropy for any label.
+    fake_xent1 = util.mnist_cross_entropy(fake_digit(), one_hot_real())
+    fake_xent6 = util.mnist_cross_entropy(fake_digit(), one_hot1())
+    with self.test_session():
+      self.assertNear(0.00996, correct_xent.eval(), 1e-5)
+      self.assertNear(18.63073, wrong_xent.eval(), 1e-5)
+      self.assertNear(2.2, fake_xent1.eval(), 1e-1)
+      self.assertNear(2.2, fake_xent6.eval(), 1e-1)
+
+  def test_minibatch_correct(self):
+    # Reorded minibatches should have the same value.
+    xent1 = util.mnist_cross_entropy(
+        tf.concat([real_digit(), real_digit(), fake_digit()], 0),
+        tf.concat([one_hot_real(), one_hot1(), one_hot1()], 0))
+    xent2 = util.mnist_cross_entropy(
+        tf.concat([real_digit(), fake_digit(), real_digit()], 0),
+        tf.concat([one_hot_real(), one_hot1(), one_hot1()], 0))
+    with self.test_session():
+      self.assertNear(6.972539, xent1.eval(), 1e-5)
+      self.assertNear(xent1.eval(), xent2.eval(), 1e-5)
+
+
+class GetNoiseTest(tf.test.TestCase):
+
+  def test_get_noise_categorical_syntax(self):
+    util.get_eval_noise_categorical(
+        noise_samples=4,
+        categorical_sample_points=np.arange(0, 10),
+        continuous_sample_points=np.linspace(-2.0, 2.0, 10),
+        unstructured_noise_dims=62,
+        continuous_noise_dims=2)
+
+  def test_get_noise_continuous_dim1_syntax(self):
+    util.get_eval_noise_continuous_dim1(
+        noise_samples=4,
+        categorical_sample_points=np.arange(0, 10),
+        continuous_sample_points=np.linspace(-2.0, 2.0, 10),
+        unstructured_noise_dims=62,
+        continuous_noise_dims=2)
+
+  def test_get_noise_continuous_dim2_syntax(self):
+    util.get_eval_noise_continuous_dim2(
+        noise_samples=4,
+        categorical_sample_points=np.arange(0, 10),
+        continuous_sample_points=np.linspace(-2.0, 2.0, 10),
+        unstructured_noise_dims=62,
+        continuous_noise_dims=2)
+
+  def test_get_infogan_syntax(self):
+    util.get_infogan_noise(
+        batch_size=4,
+        categorical_dim=10,
+        structured_continuous_dim=3,
+        total_continuous_noise_dims=62)
+
+
+if __name__ == '__main__':
+  tf.test.main()