Merge branch 'master' into single_cmd_download

CODEOWNERS

 adversarial_crypto/* @dave-andersen
 adversarial_text/* @rsepassi
+adv_imagenet_models/* @AlexeyKurakin
 attention_ocr/* @alexgorban
 audioset/* @plakal @dpwe
 autoencoders/* @snurkabill

adv_imagenet_models/README.md

+# Adversarially trained ImageNet models
+
+Pre-trained ImageNet models from the following papers:
+
+* [Adversarial Machine Learning at Scale](https://arxiv.org/abs/1611.01236)
+* [Ensemble Adversarial Training: Attacks and Defenses](https://arxiv.org/abs/1705.07204)
+
+## Contact
+
+Author: Alexey Kurakin,
+github: [AlexeyKurakin](https://github.com/AlexeyKurakin)
+
+## Pre-requesites and installation
+
+Ensure that you have installed TensorFlow 1.1 or greater
+([instructions](https://www.tensorflow.org/install/)).
+
+You also need copy of ImageNet dataset if you want to run provided example.
+Follow
+[Preparing the dataset](https://github.com/tensorflow/models/tree/master/slim#Data)
+instructions in TF-Slim library to get and preprocess ImageNet data.
+
+## Available models
+
+Following pre-trained models are available:
+
+Network Architecture | Adversarial training | Checkpoint
+---------------------|----------------------|----------------
+Inception v3 | Step L.L. | [adv_inception_v3_2017_08_18.tar.gz](http://download.tensorflow.org/models/adv_inception_v3_2017_08_18.tar.gz)
+Inception v3 | Step L.L. on ensemble of 3 models | [ens3_adv_inception_v3_2017_08_18.tar.gz](http://download.tensorflow.org/models/ens3_adv_inception_v3_2017_08_18.tar.gz)
+Inception v3 | Step L.L. on ensemble of 4 models| [ens4_adv_inception_v3_2017_08_18.tar.gz](http://download.tensorflow.org/models/ens4_adv_inception_v3_2017_08_18.tar.gz)
+Inception ResNet v2 | Step L.L. on ensemble of 3 models | [ens_adv_inception_resnet_v2_2017_08_18.tar.gz](http://download.tensorflow.org/models/ens_adv_inception_resnet_v2_2017_08_18.tar.gz)
+
+All checkpoints are compatible with
+[TF-Slim](https://github.com/tensorflow/models/tree/master/slim)
+implementation of Inception v3 and Inception Resnet v2.
+
+## How to evaluate models on ImageNet test data
+
+Python script `eval_on_adversarial.py` allow you to evaluate provided models
+on white-box adversarial examples generated from ImageNet test set.
+
+Usage is following:
+
+```bash
+# ${MODEL_NAME} - type of network architecture,
+#     either "inception_v3" or "inception_resnet_v2"
+# ${CHECKPOINT_PATH} - path to model checkpoint
+# ${DATASET_DIR} - directory with ImageNet test set
+# ${ADV_METHOD} - which method to use to generate adversarial images,
+#   supported method:
+#     "none" - use clean images from the dataset
+#     "stepll" - one step towards least likely class method (StepLL),
+#         see https://arxiv.org/abs/1611.01236 for details
+#     "stepllnoise" - RAND+StepLL method from https://arxiv.org/abs/1705.07204
+# ${ADV_EPS} - size of adversarial perturbation, ignored when method is none
+python eval_on_adversarial.py \
+  --model_name=${MODEL_NAME} \
+  --checkpoint_path=${CHECKPOINT_PATH} \
+  --dataset_dir=${DATASET_DIR} \
+  --batch_size=50 \
+  --adversarial_method=${ADV_METHOD} \
+  --adversarial_eps=${ADV_EPS}
+```
+
+Below is an example how to evaluate one of the models on RAND+StepLL adversarial
+examples:
+
+```bash
+# Download checkpoint
+CHECKPOINT_DIR=/tmp/checkpoints
+mkdir ${CHECKPOINT_DIR}
+wget http://download.tensorflow.org/models/ens_adv_inception_resnet_v2_2017_08_18.tar.gz
+tar -xvf ens_adv_inception_resnet_v2_2017_08_18.tar.gz
+mv ens_adv_inception_resnet_v2.ckpt* ${CHECKPOINT_DIR}
+rm ens_adv_inception_resnet_v2_2017_08_18.tar.gz
+
+# Run evaluation
+python eval_on_adversarial.py \
+  --model_name=inception_v3 \
+  --checkpoint_path=${CHECKPOINT_DIR}/ens_adv_inception_resnet_v2.ckpt \
+  --dataset_dir=${DATASET_DIR} \
+  --batch_size=50 \
+  --adversarial_method=stepllnoise \
+  --adversarial_eps=16
+```

adv_imagenet_models/eval_on_adversarial.py

+# Copyright 2017 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Script which evaluates model on adversarial examples."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import math
+import imagenet
+import inception_resnet_v2
+
+import tensorflow as tf
+from tensorflow.contrib.slim.nets import inception
+
+
+slim = tf.contrib.slim
+
+tf.app.flags.DEFINE_integer(
+    'batch_size', 50, 'The number of samples in each batch.')
+
+tf.app.flags.DEFINE_integer(
+    'max_num_batches', None,
+    'Max number of batches to evaluate by default use all.')
+
+tf.app.flags.DEFINE_string(
+    'master', '', 'The address of the TensorFlow master to use.')
+
+tf.app.flags.DEFINE_string(
+    'checkpoint_path', '/tmp/tfmodel/',
+    'The directory where the model was written to or an absolute path to a '
+    'checkpoint file.')
+
+tf.app.flags.DEFINE_integer(
+    'num_preprocessing_threads', 4,
+    'The number of threads used to create the batches.')
+
+tf.app.flags.DEFINE_string(
+    'split_name', 'validation', 'The name of the train/test split.')
+
+tf.app.flags.DEFINE_string(
+    'dataset_dir', None, 'The directory where the dataset files are stored.')
+
+tf.app.flags.DEFINE_string(
+    'model_name', 'inception_v3',
+    'Name of the model to use, either "inception_v3" or "inception_resnet_v2"')
+
+tf.app.flags.DEFINE_float(
+    'moving_average_decay', None,
+    'The decay to use for the moving average.'
+    'If left as None, then moving averages are not used.')
+
+tf.app.flags.DEFINE_string(
+    'adversarial_method', 'none',
+    'What kind of adversarial examples to use for evaluation. '
+    'Could be one of: "none", "stepll", "stepllnoise".')
+
+tf.app.flags.DEFINE_float(
+    'adversarial_eps', 0.0,
+    'Size of adversarial perturbation in range [0, 255].')
+
+
+FLAGS = tf.app.flags.FLAGS
+
+
+IMAGE_SIZE = 299
+NUM_CLASSES = 1001
+
+
+def preprocess_for_eval(image, height, width,
+                        central_fraction=0.875, scope=None):
+  """Prepare one image for evaluation.
+
+  If height and width are specified it would output an image with that size by
+  applying resize_bilinear.
+  If central_fraction is specified it would crop the central fraction of the
+  input image.
+
+  Args:
+    image: 3-D Tensor of image. If dtype is tf.float32 then the range should be
+      [0, 1], otherwise it would converted to tf.float32 assuming that the range
+      is [0, MAX], where MAX is largest positive representable number for
+      int(8/16/32) data type (see `tf.image.convert_image_dtype` for details)
+    height: integer
+    width: integer
+    central_fraction: Optional Float, fraction of the image to crop.
+    scope: Optional scope for name_scope.
+  Returns:
+    3-D float Tensor of prepared image.
+  """
+  with tf.name_scope(scope, 'eval_image', [image, height, width]):
+    if image.dtype != tf.float32:
+      image = tf.image.convert_image_dtype(image, dtype=tf.float32)
+    # Crop the central region of the image with an area containing 87.5% of
+    # the original image.
+    if central_fraction:
+      image = tf.image.central_crop(image, central_fraction=central_fraction)
+
+    if height and width:
+      # Resize the image to the specified height and width.
+      image = tf.expand_dims(image, 0)
+      image = tf.image.resize_bilinear(image, [height, width],
+                                       align_corners=False)
+      image = tf.squeeze(image, [0])
+    image = tf.subtract(image, 0.5)
+    image = tf.multiply(image, 2.0)
+    return image
+
+
+def create_model(x, reuse=None):
+  """Create model graph.
+
+  Args:
+    x: input images
+    reuse: reuse parameter which will be passed to underlying variable scopes.
+      Should be None first call and True every subsequent call.
+
+  Returns:
+    (logits, end_points) - tuple of model logits and enpoints
+
+  Raises:
+    ValueError: if model type specified by --model_name flag is invalid.
+  """
+  if FLAGS.model_name == 'inception_v3':
+    with slim.arg_scope(inception.inception_v3_arg_scope()):
+      return inception.inception_v3(
+          x, num_classes=NUM_CLASSES, is_training=False, reuse=reuse)
+  elif FLAGS.model_name == 'inception_resnet_v2':
+    with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
+      return inception_resnet_v2.inception_resnet_v2(
+          x, num_classes=NUM_CLASSES, is_training=False, reuse=reuse)
+  else:
+    raise ValueError('Invalid model name: %s' % (FLAGS.model_name))
+
+
+def step_target_class_adversarial_images(x, eps, one_hot_target_class):
+  """Base code for one step towards target class methods.
+
+  Args:
+    x: source images
+    eps: size of adversarial perturbation
+    one_hot_target_class: one hot encoded target classes for all images
+
+  Returns:
+    tensor with adversarial images
+  """
+  logits, end_points = create_model(x, reuse=True)
+  cross_entropy = tf.losses.softmax_cross_entropy(one_hot_target_class,
+                                                  logits,
+                                                  label_smoothing=0.1,
+                                                  weights=1.0)
+  cross_entropy += tf.losses.softmax_cross_entropy(one_hot_target_class,
+                                                   end_points['AuxLogits'],
+                                                   label_smoothing=0.1,
+                                                   weights=0.4)
+  x_adv = x - eps * tf.sign(tf.gradients(cross_entropy, x)[0])
+  x_adv = tf.clip_by_value(x_adv, -1.0, 1.0)
+  return tf.stop_gradient(x_adv)
+
+
+def stepll_adversarial_images(x, eps):
+  """One step towards least likely class (Step L.L.) adversarial examples.
+
+  This method is an alternative to FGSM which does not use true classes.
+  Method is described in the "Adversarial Machine Learning at Scale" paper,
+  https://arxiv.org/abs/1611.01236
+
+  Args:
+    x: source images
+    eps: size of adversarial perturbation
+
+  Returns:
+    adversarial images
+  """
+  logits, _ = create_model(x, reuse=True)
+  least_likely_class = tf.argmin(logits, 1)
+  one_hot_ll_class = tf.one_hot(least_likely_class, NUM_CLASSES)
+  return step_target_class_adversarial_images(x, eps, one_hot_ll_class)
+
+
+def stepllnoise_adversarial_images(x, eps):
+  """Step L.L. with noise method.
+
+  This is an imporvement of Step L.L. method. This method is better against
+  adversarially trained models which learn to mask gradient.
+  Method is described in the section "New randomized one shot attack" of
+  "Ensemble Adversarial Training: Attacks and Defenses" paper,
+  https://arxiv.org/abs/1705.07204
+
+  Args:
+    x: source images
+    eps: size of adversarial perturbation
+
+  Returns:
+    adversarial images
+  """
+  logits, _ = create_model(x, reuse=True)
+  least_likely_class = tf.argmin(logits, 1)
+  one_hot_ll_class = tf.one_hot(least_likely_class, NUM_CLASSES)
+  x_noise = x + eps / 2 * tf.sign(tf.random_normal(x.shape))
+  return step_target_class_adversarial_images(x_noise, eps / 2,
+                                              one_hot_ll_class)
+
+
+def get_input_images(dataset_images):
+  """Gets input images for the evaluation.
+
+  Args:
+    dataset_images: tensor with dataset images
+
+  Returns:
+    tensor with input images, which is either dataset images or adversarial
+    images.
+
+  Raises:
+    ValueError: if adversarial method specified by --adversarial_method flag
+      is invalid.
+  """
+  # adversarial_eps defines max difference of values of pixels if
+  # pixels are in range [0, 255]. However values of dataset pixels are
+  # in range [-1, 1], so converting epsilon.
+  eps = FLAGS.adversarial_eps / 255 * 2.0
+
+  if FLAGS.adversarial_method == 'stepll':
+    return stepll_adversarial_images(dataset_images, eps)
+  elif FLAGS.adversarial_method == 'stepllnoise':
+    return stepllnoise_adversarial_images(dataset_images, eps)
+  elif FLAGS.adversarial_method == 'none':
+    return dataset_images
+  else:
+    raise ValueError('Invalid adversarial method: %s'
+                     % (FLAGS.adversarial_method))
+
+
+def main(_):
+  if not FLAGS.dataset_dir:
+    raise ValueError('You must supply the dataset directory with --dataset_dir')
+
+  tf.logging.set_verbosity(tf.logging.INFO)
+  with tf.Graph().as_default():
+    tf_global_step = tf.train.get_or_create_global_step()
+
+    ###################
+    # Prepare dataset #
+    ###################
+    dataset = imagenet.get_split(FLAGS.split_name, FLAGS.dataset_dir)
+    provider = slim.dataset_data_provider.DatasetDataProvider(
+        dataset,
+        shuffle=False,
+        common_queue_capacity=2 * FLAGS.batch_size,
+        common_queue_min=FLAGS.batch_size)
+    [dataset_image, label] = provider.get(['image', 'label'])
+    dataset_image = preprocess_for_eval(dataset_image, IMAGE_SIZE, IMAGE_SIZE)
+    dataset_images, labels = tf.train.batch(
+        [dataset_image, label],
+        batch_size=FLAGS.batch_size,
+        num_threads=FLAGS.num_preprocessing_threads,
+        capacity=5 * FLAGS.batch_size)
+
+    ########################################
+    # Define the model and input exampeles #
+    ########################################
+    create_model(tf.placeholder(tf.float32, shape=dataset_images.shape))
+    input_images = get_input_images(dataset_images)
+    logits, _ = create_model(input_images, reuse=True)
+
+    if FLAGS.moving_average_decay > 0:
+      variable_averages = tf.train.ExponentialMovingAverage(
+          FLAGS.moving_average_decay, tf_global_step)
+      variables_to_restore = variable_averages.variables_to_restore(
+          slim.get_model_variables())
+      variables_to_restore[tf_global_step.op.name] = tf_global_step
+    else:
+      variables_to_restore = slim.get_variables_to_restore()
+
+    ######################
+    # Define the metrics #
+    ######################
+    predictions = tf.argmax(logits, 1)
+    labels = tf.squeeze(labels)
+    names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({
+        'Accuracy': slim.metrics.streaming_accuracy(predictions, labels),
+        'Recall_5': slim.metrics.streaming_sparse_recall_at_k(
+            logits, tf.reshape(labels, [-1, 1]), 5),
+    })
+
+    ######################
+    # Run evaluation     #
+    ######################
+    if FLAGS.max_num_batches:
+      num_batches = FLAGS.max_num_batches
+    else:
+      # This ensures that we make a single pass over all of the data.
+      num_batches = math.ceil(dataset.num_samples / float(FLAGS.batch_size))
+
+    if tf.gfile.IsDirectory(FLAGS.checkpoint_path):
+      checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_path)
+    else:
+      checkpoint_path = FLAGS.checkpoint_path
+
+    tf.logging.info('Evaluating %s' % checkpoint_path)
+
+    top1_accuracy, top5_accuracy = slim.evaluation.evaluate_once(
+        master=FLAGS.master,
+        checkpoint_path=checkpoint_path,
+        logdir=None,
+        summary_op=None,
+        num_evals=num_batches,
+        eval_op=list(names_to_updates.values()),
+        final_op=[names_to_values['Accuracy'], names_to_values['Recall_5']],
+        variables_to_restore=variables_to_restore)
+
+    print('Top1 Accuracy: ', top1_accuracy)
+    print('Top5 Accuracy: ', top5_accuracy)
+
+
+if __name__ == '__main__':
+  tf.app.run()

adv_imagenet_models/imagenet.py

+# Copyright 2017 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Provides data for the ImageNet ILSVRC 2012 Dataset plus some bounding boxes.
+
+Some images have one or more bounding boxes associated with the label of the
+image. See details here: http://image-net.org/download-bboxes
+
+WARNING: Don't use for object detection, in this case all the bounding boxes
+of the image belong to just one class.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import tensorflow as tf
+
+slim = tf.contrib.slim
+
+_FILE_PATTERN = '%s-*'
+
+_SPLITS_TO_SIZES = {
+    'train': 1281167,
+    'validation': 50000,
+}
+
+_ITEMS_TO_DESCRIPTIONS = {
+    'image': 'A color image of varying height and width.',
+    'label': 'The label id of the image, integer between 0 and 999',
+    'label_text': 'The text of the label.',
+    'object/bbox': 'A list of bounding boxes.',
+    'object/label': 'A list of labels, one per each object.',
+}
+
+_NUM_CLASSES = 1001
+
+
+def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
+  """Gets a dataset tuple with instructions for reading ImageNet.
+
+  Args:
+    split_name: A train/test split name.
+    dataset_dir: The base directory of the dataset sources.
+    file_pattern: The file pattern to use when matching the dataset sources.
+      It is assumed that the pattern contains a '%s' string so that the split
+      name can be inserted.
+    reader: The TensorFlow reader type.
+
+  Returns:
+    A `Dataset` namedtuple.
+
+  Raises:
+    ValueError: if `split_name` is not a valid train/test split.
+  """
+  if split_name not in _SPLITS_TO_SIZES:
+    raise ValueError('split name %s was not recognized.' % split_name)
+
+  if not file_pattern:
+    file_pattern = _FILE_PATTERN
+  file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
+
+  # Allowing None in the signature so that dataset_factory can use the default.
+  if reader is None:
+    reader = tf.TFRecordReader
+
+  keys_to_features = {
+      'image/encoded': tf.FixedLenFeature(
+          (), tf.string, default_value=''),
+      'image/format': tf.FixedLenFeature(
+          (), tf.string, default_value='jpeg'),
+      'image/class/label': tf.FixedLenFeature(
+          [], dtype=tf.int64, default_value=-1),
+      'image/class/text': tf.FixedLenFeature(
+          [], dtype=tf.string, default_value=''),
+      'image/object/bbox/xmin': tf.VarLenFeature(
+          dtype=tf.float32),
+      'image/object/bbox/ymin': tf.VarLenFeature(
+          dtype=tf.float32),
+      'image/object/bbox/xmax': tf.VarLenFeature(
+          dtype=tf.float32),
+      'image/object/bbox/ymax': tf.VarLenFeature(
+          dtype=tf.float32),
+      'image/object/class/label': tf.VarLenFeature(
+          dtype=tf.int64),
+  }
+
+  items_to_handlers = {
+      'image': slim.tfexample_decoder.Image('image/encoded', 'image/format'),
+      'label': slim.tfexample_decoder.Tensor('image/class/label'),
+      'label_text': slim.tfexample_decoder.Tensor('image/class/text'),
+      'object/bbox': slim.tfexample_decoder.BoundingBox(
+          ['ymin', 'xmin', 'ymax', 'xmax'], 'image/object/bbox/'),
+      'object/label': slim.tfexample_decoder.Tensor('image/object/class/label'),
+  }
+
+  decoder = slim.tfexample_decoder.TFExampleDecoder(
+      keys_to_features, items_to_handlers)
+
+  return slim.dataset.Dataset(
+      data_sources=file_pattern,
+      reader=reader,
+      decoder=decoder,
+      num_samples=_SPLITS_TO_SIZES[split_name],
+      items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
+      num_classes=_NUM_CLASSES)

adv_imagenet_models/inception_resnet_v2.py

+# Copyright 2017 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Contains the definition of the Inception Resnet V2 architecture.
+
+As described in http://arxiv.org/abs/1602.07261.
+
+  Inception-v4, Inception-ResNet and the Impact of Residual Connections
+    on Learning
+  Christian Szegedy, Sergey Ioffe, Vincent Vanhoucke, Alex Alemi
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+import tensorflow as tf
+
+slim = tf.contrib.slim
+
+
+def block35(net, scale=1.0, activation_fn=tf.nn.relu, scope=None, reuse=None):
+  """Builds the 35x35 resnet block."""
+  with tf.variable_scope(scope, 'Block35', [net], reuse=reuse):
+    with tf.variable_scope('Branch_0'):
+      tower_conv = slim.conv2d(net, 32, 1, scope='Conv2d_1x1')
+    with tf.variable_scope('Branch_1'):
+      tower_conv1_0 = slim.conv2d(net, 32, 1, scope='Conv2d_0a_1x1')
+      tower_conv1_1 = slim.conv2d(tower_conv1_0, 32, 3, scope='Conv2d_0b_3x3')
+    with tf.variable_scope('Branch_2'):
+      tower_conv2_0 = slim.conv2d(net, 32, 1, scope='Conv2d_0a_1x1')
+      tower_conv2_1 = slim.conv2d(tower_conv2_0, 48, 3, scope='Conv2d_0b_3x3')
+      tower_conv2_2 = slim.conv2d(tower_conv2_1, 64, 3, scope='Conv2d_0c_3x3')
+    mixed = tf.concat(axis=3, values=[tower_conv, tower_conv1_1, tower_conv2_2])
+    up = slim.conv2d(mixed, net.get_shape()[3], 1, normalizer_fn=None,
+                     activation_fn=None, scope='Conv2d_1x1')
+    net += scale * up
+    if activation_fn:
+      net = activation_fn(net)
+  return net
+
+
+def block17(net, scale=1.0, activation_fn=tf.nn.relu, scope=None, reuse=None):
+  """Builds the 17x17 resnet block."""
+  with tf.variable_scope(scope, 'Block17', [net], reuse=reuse):
+    with tf.variable_scope('Branch_0'):
+      tower_conv = slim.conv2d(net, 192, 1, scope='Conv2d_1x1')
+    with tf.variable_scope('Branch_1'):
+      tower_conv1_0 = slim.conv2d(net, 128, 1, scope='Conv2d_0a_1x1')
+      tower_conv1_1 = slim.conv2d(tower_conv1_0, 160, [1, 7],
+                                  scope='Conv2d_0b_1x7')
+      tower_conv1_2 = slim.conv2d(tower_conv1_1, 192, [7, 1],
+                                  scope='Conv2d_0c_7x1')
+    mixed = tf.concat(axis=3, values=[tower_conv, tower_conv1_2])
+    up = slim.conv2d(mixed, net.get_shape()[3], 1, normalizer_fn=None,
+                     activation_fn=None, scope='Conv2d_1x1')
+    net += scale * up
+    if activation_fn:
+      net = activation_fn(net)
+  return net
+
+
+def block8(net, scale=1.0, activation_fn=tf.nn.relu, scope=None, reuse=None):
+  """Builds the 8x8 resnet block."""
+  with tf.variable_scope(scope, 'Block8', [net], reuse=reuse):
+    with tf.variable_scope('Branch_0'):
+      tower_conv = slim.conv2d(net, 192, 1, scope='Conv2d_1x1')
+    with tf.variable_scope('Branch_1'):
+      tower_conv1_0 = slim.conv2d(net, 192, 1, scope='Conv2d_0a_1x1')
+      tower_conv1_1 = slim.conv2d(tower_conv1_0, 224, [1, 3],
+                                  scope='Conv2d_0b_1x3')
+      tower_conv1_2 = slim.conv2d(tower_conv1_1, 256, [3, 1],
+                                  scope='Conv2d_0c_3x1')
+    mixed = tf.concat(axis=3, values=[tower_conv, tower_conv1_2])
+    up = slim.conv2d(mixed, net.get_shape()[3], 1, normalizer_fn=None,
+                     activation_fn=None, scope='Conv2d_1x1')
+    net += scale * up
+    if activation_fn:
+      net = activation_fn(net)
+  return net
+
+
+def inception_resnet_v2_base(inputs,
+                             final_endpoint='Conv2d_7b_1x1',
+                             output_stride=16,
+                             align_feature_maps=False,
+                             scope=None):
+  """Inception model from  http://arxiv.org/abs/1602.07261.
+
+  Constructs an Inception Resnet v2 network from inputs to the given final
+  endpoint. This method can construct the network up to the final inception
+  block Conv2d_7b_1x1.
+
+  Args:
+    inputs: a tensor of size [batch_size, height, width, channels].
+    final_endpoint: specifies the endpoint to construct the network up to. It
+      can be one of ['Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3',
+      'MaxPool_3a_3x3', 'Conv2d_3b_1x1', 'Conv2d_4a_3x3', 'MaxPool_5a_3x3',
+      'Mixed_5b', 'Mixed_6a', 'PreAuxLogits', 'Mixed_7a', 'Conv2d_7b_1x1']
+    output_stride: A scalar that specifies the requested ratio of input to
+      output spatial resolution. Only supports 8 and 16.
+    align_feature_maps: When true, changes all the VALID paddings in the network
+      to SAME padding so that the feature maps are aligned.
+    scope: Optional variable_scope.
+
+  Returns:
+    tensor_out: output tensor corresponding to the final_endpoint.
+    end_points: a set of activations for external use, for example summaries or
+                losses.
+
+  Raises:
+    ValueError: if final_endpoint is not set to one of the predefined values,
+      or if the output_stride is not 8 or 16, or if the output_stride is 8 and
+      we request an end point after 'PreAuxLogits'.
+  """
+  if output_stride != 8 and output_stride != 16:
+    raise ValueError('output_stride must be 8 or 16.')
+
+  padding = 'SAME' if align_feature_maps else 'VALID'
+
+  end_points = {}
+
+  def add_and_check_final(name, net):
+    end_points[name] = net
+    return name == final_endpoint
+
+  with tf.variable_scope(scope, 'InceptionResnetV2', [inputs]):
+    with slim.arg_scope([slim.conv2d, slim.max_pool2d, slim.avg_pool2d],
+                        stride=1, padding='SAME'):
+      # 149 x 149 x 32
+      net = slim.conv2d(inputs, 32, 3, stride=2, padding=padding,
+                        scope='Conv2d_1a_3x3')
+      if add_and_check_final('Conv2d_1a_3x3', net): return net, end_points
+
+      # 147 x 147 x 32
+      net = slim.conv2d(net, 32, 3, padding=padding,
+                        scope='Conv2d_2a_3x3')
+      if add_and_check_final('Conv2d_2a_3x3', net): return net, end_points
+      # 147 x 147 x 64
+      net = slim.conv2d(net, 64, 3, scope='Conv2d_2b_3x3')
+      if add_and_check_final('Conv2d_2b_3x3', net): return net, end_points
+      # 73 x 73 x 64
+      net = slim.max_pool2d(net, 3, stride=2, padding=padding,
+                            scope='MaxPool_3a_3x3')
+      if add_and_check_final('MaxPool_3a_3x3', net): return net, end_points
+      # 73 x 73 x 80
+      net = slim.conv2d(net, 80, 1, padding=padding,
+                        scope='Conv2d_3b_1x1')
+      if add_and_check_final('Conv2d_3b_1x1', net): return net, end_points
+      # 71 x 71 x 192
+      net = slim.conv2d(net, 192, 3, padding=padding,
+                        scope='Conv2d_4a_3x3')
+      if add_and_check_final('Conv2d_4a_3x3', net): return net, end_points
+      # 35 x 35 x 192
+      net = slim.max_pool2d(net, 3, stride=2, padding=padding,
+                            scope='MaxPool_5a_3x3')
+      if add_and_check_final('MaxPool_5a_3x3', net): return net, end_points
+
+      # 35 x 35 x 320
+      with tf.variable_scope('Mixed_5b'):
+        with tf.variable_scope('Branch_0'):
+          tower_conv = slim.conv2d(net, 96, 1, scope='Conv2d_1x1')
+        with tf.variable_scope('Branch_1'):
+          tower_conv1_0 = slim.conv2d(net, 48, 1, scope='Conv2d_0a_1x1')
+          tower_conv1_1 = slim.conv2d(tower_conv1_0, 64, 5,
+                                      scope='Conv2d_0b_5x5')
+        with tf.variable_scope('Branch_2'):
+          tower_conv2_0 = slim.conv2d(net, 64, 1, scope='Conv2d_0a_1x1')
+          tower_conv2_1 = slim.conv2d(tower_conv2_0, 96, 3,
+                                      scope='Conv2d_0b_3x3')
+          tower_conv2_2 = slim.conv2d(tower_conv2_1, 96, 3,
+                                      scope='Conv2d_0c_3x3')
+        with tf.variable_scope('Branch_3'):
+          tower_pool = slim.avg_pool2d(net, 3, stride=1, padding='SAME',
+                                       scope='AvgPool_0a_3x3')
+          tower_pool_1 = slim.conv2d(tower_pool, 64, 1,
+                                     scope='Conv2d_0b_1x1')
+        net = tf.concat(
+            [tower_conv, tower_conv1_1, tower_conv2_2, tower_pool_1], 3)
+
+      if add_and_check_final('Mixed_5b', net): return net, end_points
+      # TODO(alemi): Register intermediate endpoints
+      net = slim.repeat(net, 10, block35, scale=0.17)
+
+      # 17 x 17 x 1088 if output_stride == 8,
+      # 33 x 33 x 1088 if output_stride == 16
+      use_atrous = output_stride == 8
+
+      with tf.variable_scope('Mixed_6a'):
+        with tf.variable_scope('Branch_0'):
+          tower_conv = slim.conv2d(net, 384, 3, stride=1 if use_atrous else 2,
+                                   padding=padding,
+                                   scope='Conv2d_1a_3x3')
+        with tf.variable_scope('Branch_1'):
+          tower_conv1_0 = slim.conv2d(net, 256, 1, scope='Conv2d_0a_1x1')
+          tower_conv1_1 = slim.conv2d(tower_conv1_0, 256, 3,
+                                      scope='Conv2d_0b_3x3')
+          tower_conv1_2 = slim.conv2d(tower_conv1_1, 384, 3,
+                                      stride=1 if use_atrous else 2,
+                                      padding=padding,
+                                      scope='Conv2d_1a_3x3')
+        with tf.variable_scope('Branch_2'):
+          tower_pool = slim.max_pool2d(net, 3, stride=1 if use_atrous else 2,
+                                       padding=padding,
+                                       scope='MaxPool_1a_3x3')
+        net = tf.concat([tower_conv, tower_conv1_2, tower_pool], 3)
+
+      if add_and_check_final('Mixed_6a', net): return net, end_points
+
+      # TODO(alemi): register intermediate endpoints
+      with slim.arg_scope([slim.conv2d], rate=2 if use_atrous else 1):
+        net = slim.repeat(net, 20, block17, scale=0.10)
+      if add_and_check_final('PreAuxLogits', net): return net, end_points
+
+      if output_stride == 8:
+        # TODO(gpapan): Properly support output_stride for the rest of the net.
+        raise ValueError('output_stride==8 is only supported up to the '
+                         'PreAuxlogits end_point for now.')
+
+      # 8 x 8 x 2080
+      with tf.variable_scope('Mixed_7a'):
+        with tf.variable_scope('Branch_0'):
+          tower_conv = slim.conv2d(net, 256, 1, scope='Conv2d_0a_1x1')
+          tower_conv_1 = slim.conv2d(tower_conv, 384, 3, stride=2,
+                                     padding=padding,
+                                     scope='Conv2d_1a_3x3')
+        with tf.variable_scope('Branch_1'):
+          tower_conv1 = slim.conv2d(net, 256, 1, scope='Conv2d_0a_1x1')
+          tower_conv1_1 = slim.conv2d(tower_conv1, 288, 3, stride=2,
+                                      padding=padding,
+                                      scope='Conv2d_1a_3x3')
+        with tf.variable_scope('Branch_2'):
+          tower_conv2 = slim.conv2d(net, 256, 1, scope='Conv2d_0a_1x1')
+          tower_conv2_1 = slim.conv2d(tower_conv2, 288, 3,
+                                      scope='Conv2d_0b_3x3')
+          tower_conv2_2 = slim.conv2d(tower_conv2_1, 320, 3, stride=2,
+                                      padding=padding,
+                                      scope='Conv2d_1a_3x3')
+        with tf.variable_scope('Branch_3'):
+          tower_pool = slim.max_pool2d(net, 3, stride=2,
+                                       padding=padding,
+                                       scope='MaxPool_1a_3x3')
+        net = tf.concat(
+            [tower_conv_1, tower_conv1_1, tower_conv2_2, tower_pool], 3)
+
+      if add_and_check_final('Mixed_7a', net): return net, end_points
+
+      # TODO(alemi): register intermediate endpoints
+      net = slim.repeat(net, 9, block8, scale=0.20)
+      net = block8(net, activation_fn=None)
+
+      # 8 x 8 x 1536
+      net = slim.conv2d(net, 1536, 1, scope='Conv2d_7b_1x1')
+      if add_and_check_final('Conv2d_7b_1x1', net): return net, end_points
+
+    raise ValueError('final_endpoint (%s) not recognized', final_endpoint)
+
+
+def inception_resnet_v2(inputs, num_classes=1001, is_training=True,
+                        dropout_keep_prob=0.8,
+                        reuse=None,
+                        scope='InceptionResnetV2',
+                        create_aux_logits=True):
+  """Creates the Inception Resnet V2 model.
+
+  Args:
+    inputs: a 4-D tensor of size [batch_size, height, width, 3].
+    num_classes: number of predicted classes.
+    is_training: whether is training or not.
+    dropout_keep_prob: float, the fraction to keep before final layer.
+    reuse: whether or not the network and its variables should be reused. To be
+      able to reuse 'scope' must be given.
+    scope: Optional variable_scope.
+    create_aux_logits: Whether to include the auxilliary logits.
+
+  Returns:
+    logits: the logits outputs of the model.
+    end_points: the set of end_points from the inception model.
+  """
+  end_points = {}
+
+  with tf.variable_scope(scope, 'InceptionResnetV2', [inputs, num_classes],
+                         reuse=reuse) as scope:
+    with slim.arg_scope([slim.batch_norm, slim.dropout],
+                        is_training=is_training):
+
+      net, end_points = inception_resnet_v2_base(inputs, scope=scope)
+
+      if create_aux_logits:
+        with tf.variable_scope('AuxLogits'):
+          aux = end_points['PreAuxLogits']
+          aux = slim.avg_pool2d(aux, 5, stride=3, padding='VALID',
+                                scope='Conv2d_1a_3x3')
+          aux = slim.conv2d(aux, 128, 1, scope='Conv2d_1b_1x1')
+          aux = slim.conv2d(aux, 768, aux.get_shape()[1:3],
+                            padding='VALID', scope='Conv2d_2a_5x5')
+          aux = slim.flatten(aux)
+          aux = slim.fully_connected(aux, num_classes, activation_fn=None,
+                                     scope='Logits')
+          end_points['AuxLogits'] = aux
+
+      with tf.variable_scope('Logits'):
+        net = slim.avg_pool2d(net, net.get_shape()[1:3], padding='VALID',
+                              scope='AvgPool_1a_8x8')
+        net = slim.flatten(net)
+
+        net = slim.dropout(net, dropout_keep_prob, is_training=is_training,
+                           scope='Dropout')
+
+        end_points['PreLogitsFlatten'] = net
+        logits = slim.fully_connected(net, num_classes, activation_fn=None,
+                                      scope='Logits')
+        end_points['Logits'] = logits
+        end_points['Predictions'] = tf.nn.softmax(logits, name='Predictions')
+
+    return logits, end_points
+inception_resnet_v2.default_image_size = 299
+
+
+def inception_resnet_v2_arg_scope(weight_decay=0.00004,
+                                  batch_norm_decay=0.9997,
+                                  batch_norm_epsilon=0.001):
+  """Returns the scope with the default parameters for inception_resnet_v2.
+
+  Args:
+    weight_decay: the weight decay for weights variables.
+    batch_norm_decay: decay for the moving average of batch_norm momentums.
+    batch_norm_epsilon: small float added to variance to avoid dividing by zero.
+
+  Returns:
+    a arg_scope with the parameters needed for inception_resnet_v2.
+  """
+  # Set weight_decay for weights in conv2d and fully_connected layers.
+  with slim.arg_scope([slim.conv2d, slim.fully_connected],
+                      weights_regularizer=slim.l2_regularizer(weight_decay),
+                      biases_regularizer=slim.l2_regularizer(weight_decay)):
+
+    batch_norm_params = {
+        'decay': batch_norm_decay,
+        'epsilon': batch_norm_epsilon,
+    }
+    # Set activation_fn and parameters for batch_norm.
+    with slim.arg_scope([slim.conv2d], activation_fn=tf.nn.relu,
+                        normalizer_fn=slim.batch_norm,
+                        normalizer_params=batch_norm_params) as scope:
+      return scope

ptn/.gitignore

@@ -5,5 +5,4 @@ bazel-out
 bazel-genfiles
 bazel-ptn
 bazel-testlogs
-WORKSPACE
 *.pyc

ptn/eval_ptn.py

@@ -97,7 +97,7 @@ def main(argv=()):
     ## data ##
     ##########
     eval_data = model.get_inputs(
-        FLAGS.data_sst_path,
+        FLAGS.inp_dir,
         FLAGS.dataset_name,
         eval_params.eval_set,
         eval_params.batch_size,

ptn/eval_rotator.py

@@ -91,7 +91,7 @@ def main(argv=()):
     ## data ##
     ##########
     val_data = model.get_inputs(
-        FLAGS.data_sst_path,
+        FLAGS.inp_dir,
         FLAGS.dataset_name,
         'val',
         FLAGS.batch_size,

ptn/model_rotator.py

@@ -190,8 +190,34 @@ def get_train_op_for_scope(loss, optimizer, scopes, params):
 
 
 def get_metrics(inputs, outputs, params):
-  names_to_values, names_to_updates = metrics.rotator_metrics(
-      inputs, outputs, params)
+  """Aggregate the metrics for rotator model.
+  
+  Args:
+    inputs: Input dictionary of the rotator model.
+    outputs: Output dictionary returned by the rotator model.
+    params: Hyperparameters of the rotator model.
+  
+  Returns:
+    names_to_values: metrics->values (dict).
+    names_to_updates: metrics->ops (dict).
+  """
+  names_to_values = dict()
+  names_to_updates = dict()
+  
+  tmp_values, tmp_updates = metrics.add_image_pred_metrics(
+      inputs, outputs, params.num_views, 3*params.image_size**2)
+  names_to_values.update(tmp_values)
+  names_to_updates.update(tmp_updates)
+
+  tmp_values, tmp_updates = metrics.add_mask_pred_metrics(
+      inputs, outputs, params.num_views, params.image_size**2)
+  names_to_values.update(tmp_values)
+  names_to_updates.update(tmp_updates)
+
+  for name, value in names_to_values.iteritems():
+    slim.summaries.add_scalar_summary(
+        value, name, prefix='eval', print_summary=True)
+ 
   return names_to_values, names_to_updates
 
 

qa_kg/util/data_reader.py

@@ -14,7 +14,10 @@
 # ==============================================================================
 
 from collections import namedtuple
-from Queue import Queue
+try:
+  from queue import Queue  # Python 3
+except ImportError:
+  from Queue import Queue  # Python 2
 import re
 import threading
 import numpy as np
@@ -43,7 +46,7 @@ class SampleBuilder:
 
   def read_kb(self):
     kb_raw = []
-    for line in file(self.config.KB_file):
+    for line in open(self.config.KB_file):
       sub, rel, obj = line.strip().split('|')
       kb_raw.append((sub, rel, obj))
     tf.logging.info('# of KB records: %d' % len(kb_raw))
@@ -55,7 +58,7 @@ class SampleBuilder:
       raw = []
       tf.logging.info(
         'Reading data file {}'.format(self.config.data_files[name]))
-      for line in file(self.config.data_files[name]):
+      for line in open(self.config.data_files[name]):
         question, answers = line.strip().split('\t')
         question = question.replace('],', ']')  # ignore ',' in the template
         raw.append((question, answers))

rebar/rebar.py

@@ -26,6 +26,11 @@ import tensorflow.contrib.slim as slim
 from tensorflow.python.ops import init_ops
 import utils as U
 
+try:
+  xrange          # Python 2
+except NameError:
+  xrange = range  # Python 3
+
 FLAGS = tf.flags.FLAGS
 
 Q_COLLECTION = "q_collection"
@@ -293,7 +298,7 @@ class SBN(object):  # REINFORCE
     logQHard = tf.add_n(logQHard)
 
     # REINFORCE
-    learning_signal = tf.stop_gradient(center(reinforce_learning_signal))
+    learning_signal = tf.stop_gradient(U.center(reinforce_learning_signal))
     self.optimizerLoss = -(learning_signal*logQHard +
                            reinforce_model_grad)
     self.lHat = map(tf.reduce_mean, [

rebar/rebar_train.py

@@ -28,6 +28,12 @@ import tensorflow as tf
 import rebar
 import datasets
 import logger as L
+
+try:
+  xrange          # Python 2
+except NameError:
+  xrange = range  # Python 3
+
 gfile = tf.gfile
 
 tf.app.flags.DEFINE_string("working_dir", "/tmp/rebar",

rebar/utils.py

@@ -134,6 +134,3 @@ def logSumExp(t, axis=0, keep_dims = False):
     return tf.expand_dims(res, axis)
   else:
     return res
-
-if __name__ == '__main__':
-  app.run()

skip_thoughts/skip_thoughts/BUILD

-package(default_visibility = [":internal"])
+package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
-package_group(
-    name = "internal",
-    packages = [
-        "//skip_thoughts/...",
-    ],
-)
-
 py_library(
     name = "configuration",
     srcs = ["configuration.py"],

skip_thoughts/skip_thoughts/data/BUILD

-package(default_visibility = ["//skip_thoughts:internal"])
+package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])  # Apache 2.0
 

skip_thoughts/skip_thoughts/ops/BUILD

-package(default_visibility = ["//skip_thoughts:internal"])
+package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])  # Apache 2.0
 

swivel/glove_to_shards.py

@@ -45,6 +45,8 @@ import sys
 
 import tensorflow as tf
 
+from six.moves import xrange
+
 flags = tf.app.flags
 
 flags.DEFINE_string('input', 'coocurrences.bin', 'Vocabulary file')
@@ -131,6 +133,7 @@ def make_shard_files(coocs, nshards, vocab_sz):
 
   return (shard_files, row_sums)
 
+
 def main(_):
   with open(FLAGS.vocab, 'r') as lines:
     orig_vocab_sz = sum(1 for _ in lines)
@@ -193,5 +196,6 @@ def main(_):
 
   print('done!')
 
+
 if __name__ == '__main__':
   tf.app.run()

swivel/nearest.py

@@ -16,6 +16,7 @@
 
 """Simple tool for inspecting nearest neighbors and analogies."""
 
+from __future__ import print_function
 import re
 import sys
 from getopt import GetoptError, getopt
@@ -24,8 +25,8 @@ from vecs import Vecs
 
 try:
   opts, args = getopt(sys.argv[1:], 'v:e:', ['vocab=', 'embeddings='])
-except GetoptError, e:
-  print >> sys.stderr, e
+except GetoptError as e:
+  print(e, file=sys.stderr)
   sys.exit(2)
 
 opt_vocab = 'vocab.txt'
@@ -55,21 +56,21 @@ while True:
   elif len(parts) == 3:
     vs = [vecs.lookup(w) for w in parts]
     if any(v is None for v in vs):
-      print 'not in vocabulary: %s' % (
-          ', '.join(tok for tok, v in zip(parts, vs) if v is None))
+      print('not in vocabulary: %s' % (
+            ', '.join(tok for tok, v in zip(parts, vs) if v is None)))
 
       continue
 
     res = vecs.neighbors(vs[2] - vs[0] + vs[1])
 
   else:
-    print 'use a single word to query neighbors, or three words for analogy'
+    print('use a single word to query neighbors, or three words for analogy')
     continue
 
   if not res:
     continue
 
   for word, sim in res[:20]:
-    print '%0.4f: %s' % (sim, word)
+    print('%0.4f: %s' % (sim, word))
 
-  print
+  print()

swivel/vecs.py

@@ -15,7 +15,9 @@
 import mmap
 import numpy as np
 import os
-import struct
+
+from six import string_types
+
 
 class Vecs(object):
   def __init__(self, vocab_filename, rows_filename, cols_filename=None):
@@ -41,8 +43,8 @@ class Vecs(object):
       rows = np.matrix(
           np.frombuffer(rows_mm, dtype=np.float32).reshape(n, dim))
 
-      # If column vectors were specified, then open them and add them to the row
-      # vectors.
+      # If column vectors were specified, then open them and add them to the
+      # row vectors.
       if cols_filename:
         with open(cols_filename, 'r') as cols_fh:
           cols_mm = mmap.mmap(cols_fh.fileno(), 0, prot=mmap.PROT_READ)
@@ -71,7 +73,7 @@ class Vecs(object):
 
   def neighbors(self, query):
     """Returns the nearest neighbors to the query (a word or vector)."""
-    if isinstance(query, basestring):
+    if isinstance(query, string_types):
       idx = self.word_to_idx.get(query)
       if idx is None:
         return None